From cd79bc24c9b68b67d4508dcc145b88417fa906a9 Mon Sep 17 00:00:00 2001
From: mikkel <7003+mikkel@users.noreply.github.com>
Date: Sun, 18 Oct 2026 16:15:24 -0600
Subject: [PATCH 1/2] test: vendor Python parity fixtures (binary, parser,
 truncation)

Copy binary.json, parser.json and truncation.json byte-for-byte from
255BITS/patchling-py tests/fixtures/parity/ (commit 533749a) and add
test/parity_fixtures.test.js, which pins each file's sha256 and runs
every case against isBinaryContent, parseRelevanceAnswer and truncateUtf8.

truncateUtf8 becomes a named export of src/filter.js only (mirrored into
landing/lib/filter.js, which must stay byte-identical). It is not added to
src/index.js or package.json exports, so the public API is unchanged.
---
 landing/lib/filter.js                |  2 +-
 src/filter.d.ts                      |  8 ++++
 src/filter.js                        |  2 +-
 test/fixtures/parity/binary.json     | 15 +++++++
 test/fixtures/parity/parser.json     | 21 +++++++++
 test/fixtures/parity/truncation.json | 18 ++++++++
 test/parity_fixtures.test.js         | 66 ++++++++++++++++++++++++++++
 7 files changed, 130 insertions(+), 2 deletions(-)
 create mode 100644 test/fixtures/parity/binary.json
 create mode 100644 test/fixtures/parity/parser.json
 create mode 100644 test/fixtures/parity/truncation.json
 create mode 100644 test/parity_fixtures.test.js

diff --git a/landing/lib/filter.js b/landing/lib/filter.js
index a8f38b0..e289f36 100644
--- a/landing/lib/filter.js
+++ b/landing/lib/filter.js
@@ -525,7 +525,7 @@ function truncateText(text, maxBytes, maxTokens) {
  * @param {number} maxBytes
  * @returns {string}
  */
-function truncateUtf8(text, maxBytes) {
+export function truncateUtf8(text, maxBytes) {
   const bytes = new TextEncoder().encode(text);
   if (bytes.length <= maxBytes || maxBytes <= 0) return maxBytes <= 0 ? '' : text;
   let end = maxBytes;
diff --git a/src/filter.d.ts b/src/filter.d.ts
index 99151ca..2cc8ea9 100644
--- a/src/filter.d.ts
+++ b/src/filter.d.ts
@@ -29,6 +29,14 @@ export function parseRelevanceAnswer(raw: unknown): {
  * @returns {number}
  */
 export function countTokens(text: string): number;
+/**
+ * UTF-8 prefix cut on a code-point boundary. Never emits U+FFFD: a character
+ * that does not fit entirely inside `maxBytes` is dropped.
+ * @param {string} text
+ * @param {number} maxBytes
+ * @returns {string}
+ */
+export function truncateUtf8(text: string, maxBytes: number): string;
 /**
  * True when `content` looks like a binary file (NUL, invalid UTF-8, or a
  * dense run of non-text controls). The whole payload is validated. A tail
diff --git a/src/filter.js b/src/filter.js
index a8f38b0..e289f36 100644
--- a/src/filter.js
+++ b/src/filter.js
@@ -525,7 +525,7 @@ function truncateText(text, maxBytes, maxTokens) {
  * @param {number} maxBytes
  * @returns {string}
  */
-function truncateUtf8(text, maxBytes) {
+export function truncateUtf8(text, maxBytes) {
   const bytes = new TextEncoder().encode(text);
   if (bytes.length <= maxBytes || maxBytes <= 0) return maxBytes <= 0 ? '' : text;
   let end = maxBytes;
diff --git a/test/fixtures/parity/binary.json b/test/fixtures/parity/binary.json
new file mode 100644
index 0000000..d5e2d6b
--- /dev/null
+++ b/test/fixtures/parity/binary.json
@@ -0,0 +1,15 @@
+{
+  "description": "Binary detection on the first 8192 bytes. An incomplete trailing UTF-8 sequence of at most 3 bytes is still text. NUL and genuinely invalid bytes are binary.",
+  "cases": [
+    {"id": "nul", "hex": "6100", "expect_binary": true},
+    {"id": "invalid-ff", "hex": "61ff62", "expect_binary": true},
+    {"id": "bad-continuation", "hex": "c341", "expect_binary": true},
+    {"id": "valid-2byte", "hex": "c3a9", "expect_binary": false},
+    {"id": "valid-3byte", "hex": "e282ac", "expect_binary": false},
+    {"id": "valid-4byte", "hex": "f09f9880", "expect_binary": false},
+    {"id": "cut-2byte-at-8192", "fill": "a", "length": 8192, "tail_hex": "c3", "expect_binary": false},
+    {"id": "cut-3byte-at-8192", "fill": "a", "length": 8192, "tail_hex": "e282", "expect_binary": false},
+    {"id": "cut-4byte-at-8192", "fill": "a", "length": 8192, "tail_hex": "f09f98", "expect_binary": false},
+    {"id": "invalid-at-end-of-window", "fill": "a", "length": 8192, "tail_hex": "ff", "expect_binary": true}
+  ]
+}
diff --git a/test/fixtures/parity/parser.json b/test/fixtures/parity/parser.json
new file mode 100644
index 0000000..1eb488d
--- /dev/null
+++ b/test/fixtures/parity/parser.json
@@ -0,0 +1,21 @@
+{
+  "description": "Leading standalone yes/no after whitespace, quotes, markdown bold, backticks, and Answer:/Decision: prefixes. Think tags are stripped first. Anything else is malformed (null).",
+  "cases": [
+    {"id": "plain-yes", "raw": "yes", "decision": "yes"},
+    {"id": "plain-no", "raw": "NO", "decision": "no"},
+    {"id": "punctuation", "raw": "Yes.", "decision": "yes"},
+    {"id": "quoted-yes", "raw": "\"yes\"", "decision": "yes"},
+    {"id": "quoted-no", "raw": "'no'", "decision": "no"},
+    {"id": "bold-yes", "raw": "**yes**", "decision": "yes"},
+    {"id": "backtick-no", "raw": "`no`", "decision": "no"},
+    {"id": "answer-prefix", "raw": "Answer: yes, the parser lives here", "decision": "yes"},
+    {"id": "decision-prefix", "raw": "Decision: no", "decision": "no"},
+    {"id": "negated", "raw": "There is no doubt this file is required", "decision": null},
+    {"id": "explanatory", "raw": "This file is unrelated to the task", "decision": null},
+    {"id": "nope", "raw": "nope", "decision": null},
+    {"id": "leading-yes-later-no", "raw": "yes for the helper, no for the tests", "decision": "yes"},
+    {"id": "think-then-yes", "raw": "<think>no, skip it</think>\nyes", "decision": "yes"},
+    {"id": "empty", "raw": "", "decision": null},
+    {"id": "whitespace", "raw": "  \n  yes — needed", "decision": "yes"}
+  ]
+}
diff --git a/test/fixtures/parity/truncation.json b/test/fixtures/parity/truncation.json
new file mode 100644
index 0000000..b1bcd9f
--- /dev/null
+++ b/test/fixtures/parity/truncation.json
@@ -0,0 +1,18 @@
+{
+  "description": "UTF-8 truncation on a code-point boundary. Never emit U+FFFD. Shared with the JS filter.",
+  "cases": [
+    {"id": "2byte-max-1", "text": "éX", "max_bytes": 1, "expected": ""},
+    {"id": "2byte-max-2", "text": "éX", "max_bytes": 2, "expected": "é"},
+    {"id": "2byte-max-3", "text": "éX", "max_bytes": 3, "expected": "éX"},
+    {"id": "3byte-max-1", "text": "€X", "max_bytes": 1, "expected": ""},
+    {"id": "3byte-max-2", "text": "€X", "max_bytes": 2, "expected": ""},
+    {"id": "3byte-max-3", "text": "€X", "max_bytes": 3, "expected": "€"},
+    {"id": "3byte-max-4", "text": "€X", "max_bytes": 4, "expected": "€X"},
+    {"id": "4byte-max-1", "text": "😀X", "max_bytes": 1, "expected": ""},
+    {"id": "4byte-max-2", "text": "😀X", "max_bytes": 2, "expected": ""},
+    {"id": "4byte-max-3", "text": "😀X", "max_bytes": 3, "expected": ""},
+    {"id": "4byte-max-4", "text": "😀X", "max_bytes": 4, "expected": "😀"},
+    {"id": "4byte-max-5", "text": "😀X", "max_bytes": 5, "expected": "😀X"},
+    {"id": "ascii-max-1", "text": "ab", "max_bytes": 1, "expected": "a"}
+  ]
+}
diff --git a/test/parity_fixtures.test.js b/test/parity_fixtures.test.js
new file mode 100644
index 0000000..85535cc
--- /dev/null
+++ b/test/parity_fixtures.test.js
@@ -0,0 +1,66 @@
+/**
+ * Shared parity fixtures vendored byte-for-byte from 255BITS/patchling-py
+ * (tests/fixtures/parity/, commit 533749a). The Python suite runs the same
+ * cases in tests/test_relevance.py. Each file's sha256 is pinned so a drift
+ * on either side shows up as a failure here.
+ */
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { createHash } from 'node:crypto';
+import { readFileSync } from 'node:fs';
+import { isBinaryContent, parseRelevanceAnswer } from '../src/index.js';
+import { truncateUtf8 } from '../src/filter.js';
+
+const FIXTURE_SHA256 = {
+  'binary.json': 'babcc94fe1dd990f0a69c41819ed9f76a185756ff28fa2d0e14651a190314934',
+  'parser.json': '809a3522c79781da2aa2f10168bafd770aa781dc29c215b1dd885689047d3c7b',
+  'truncation.json': 'c1d9db9b70687c24068f13a0b2a0edcf84ca3c5830e166336ff4da3c35b5c0df',
+};
+
+function loadFixture(name) {
+  const bytes = readFileSync(new URL(`./fixtures/parity/${name}`, import.meta.url));
+  return { bytes, data: JSON.parse(bytes.toString('utf8')) };
+}
+
+const fixtures = Object.fromEntries(
+  Object.keys(FIXTURE_SHA256).map((name) => [name, loadFixture(name)]),
+);
+
+for (const [name, sha256] of Object.entries(FIXTURE_SHA256)) {
+  test(`${name} bytes match the recorded sha256`, () => {
+    assert.equal(createHash('sha256').update(fixtures[name].bytes).digest('hex'), sha256);
+  });
+}
+
+function binaryCaseBytes(c) {
+  if (c.hex !== undefined) return new Uint8Array(Buffer.from(c.hex, 'hex'));
+  const pad = Buffer.alloc(c.length, c.fill, 'utf8');
+  return new Uint8Array(Buffer.concat([pad, Buffer.from(c.tail_hex, 'hex')]));
+}
+
+test('binary.json: every case matches isBinaryContent', () => {
+  const { cases } = fixtures['binary.json'].data;
+  assert.ok(cases.length > 0);
+  for (const c of cases) {
+    assert.equal(isBinaryContent(binaryCaseBytes(c)), c.expect_binary, c.id);
+  }
+});
+
+test('parser.json: every case matches parseRelevanceAnswer', () => {
+  const { cases } = fixtures['parser.json'].data;
+  assert.ok(cases.length > 0);
+  for (const c of cases) {
+    assert.equal(parseRelevanceAnswer(c.raw).decision, c.decision, c.id);
+  }
+});
+
+test('truncation.json: every case matches truncateUtf8 and never emits U+FFFD', () => {
+  const { cases } = fixtures['truncation.json'].data;
+  assert.ok(cases.length > 0);
+  for (const c of cases) {
+    const out = truncateUtf8(c.text, c.max_bytes);
+    assert.equal(out, c.expected, c.id);
+    assert.ok(!out.includes('\uFFFD'), c.id);
+    assert.ok(Buffer.byteLength(out, 'utf8') <= Math.max(c.max_bytes, 0), c.id);
+  }
+});

From c7e621ea302c512d5f9c6c4cb9644d73a51cd486 Mon Sep 17 00:00:00 2001
From: mikkel <7003+mikkel@users.noreply.github.com>
Date: Tue, 20 Oct 2026 16:17:01 -0600
Subject: [PATCH 2/2] test: build windowed binary samples at exactly length
 bytes

The four cut/invalid-at-end cases appended tail_hex on top of a full
8192-byte pad, so the samples were 8193-8195 bytes and the tail sat past
the window the fixture describes. Build them the way patchling-py does,
fill * (length - len(tail)) + tail, and assert each windowed sample is
exactly 8192 bytes. All ten samples are now byte-identical to the Python
test's inputs.
---
 test/parity_fixtures.test.js | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/test/parity_fixtures.test.js b/test/parity_fixtures.test.js
index 85535cc..1799822 100644
--- a/test/parity_fixtures.test.js
+++ b/test/parity_fixtures.test.js
@@ -32,17 +32,28 @@ for (const [name, sha256] of Object.entries(FIXTURE_SHA256)) {
   });
 }
 
+// Same construction as patchling-py test_parity_binary_cut_fixtures:
+// fill * (length - len(tail)) + tail, so `length` is the total size and the
+// tail sits at the very end of that window.
 function binaryCaseBytes(c) {
   if (c.hex !== undefined) return new Uint8Array(Buffer.from(c.hex, 'hex'));
-  const pad = Buffer.alloc(c.length, c.fill, 'utf8');
-  return new Uint8Array(Buffer.concat([pad, Buffer.from(c.tail_hex, 'hex')]));
+  const tail = Buffer.from(c.tail_hex, 'hex');
+  const fill = Buffer.from(c.fill ?? 'a', 'ascii');
+  assert.equal(fill.length, 1, `${c.id}: fill must be one ASCII byte`);
+  assert.ok(tail.length <= c.length, `${c.id}: tail longer than length`);
+  const pad = Buffer.alloc(c.length - tail.length, fill);
+  const bytes = Buffer.concat([pad, tail]);
+  assert.equal(bytes.length, c.length, `${c.id}: sample must be exactly ${c.length} bytes`);
+  return new Uint8Array(bytes);
 }
 
 test('binary.json: every case matches isBinaryContent', () => {
   const { cases } = fixtures['binary.json'].data;
   assert.ok(cases.length > 0);
   for (const c of cases) {
-    assert.equal(isBinaryContent(binaryCaseBytes(c)), c.expect_binary, c.id);
+    const sample = binaryCaseBytes(c);
+    if (c.length !== undefined) assert.equal(sample.length, 8192, `${c.id}: windowed cases are 8192 bytes`);
+    assert.equal(isBinaryContent(sample), c.expect_binary, c.id);
   }
 });