diff --git a/landing/lib/filter.js b/landing/lib/filter.js index a8f38b0..e289f36 100644 --- a/landing/lib/filter.js +++ b/landing/lib/filter.js @@ -525,7 +525,7 @@ function truncateText(text, maxBytes, maxTokens) { * @param {number} maxBytes * @returns {string} */ -function truncateUtf8(text, maxBytes) { +export function truncateUtf8(text, maxBytes) { const bytes = new TextEncoder().encode(text); if (bytes.length <= maxBytes || maxBytes <= 0) return maxBytes <= 0 ? '' : text; let end = maxBytes; diff --git a/src/filter.d.ts b/src/filter.d.ts index 99151ca..2cc8ea9 100644 --- a/src/filter.d.ts +++ b/src/filter.d.ts @@ -29,6 +29,14 @@ export function parseRelevanceAnswer(raw: unknown): { * @returns {number} */ export function countTokens(text: string): number; +/** + * UTF-8 prefix cut on a code-point boundary. Never emits U+FFFD: a character + * that does not fit entirely inside `maxBytes` is dropped. + * @param {string} text + * @param {number} maxBytes + * @returns {string} + */ +export function truncateUtf8(text: string, maxBytes: number): string; /** * True when `content` looks like a binary file (NUL, invalid UTF-8, or a * dense run of non-text controls). The whole payload is validated. A tail diff --git a/src/filter.js b/src/filter.js index a8f38b0..e289f36 100644 --- a/src/filter.js +++ b/src/filter.js @@ -525,7 +525,7 @@ function truncateText(text, maxBytes, maxTokens) { * @param {number} maxBytes * @returns {string} */ -function truncateUtf8(text, maxBytes) { +export function truncateUtf8(text, maxBytes) { const bytes = new TextEncoder().encode(text); if (bytes.length <= maxBytes || maxBytes <= 0) return maxBytes <= 0 ? '' : text; let end = maxBytes; diff --git a/test/fixtures/parity/binary.json b/test/fixtures/parity/binary.json new file mode 100644 index 0000000..d5e2d6b --- /dev/null +++ b/test/fixtures/parity/binary.json @@ -0,0 +1,15 @@ +{ + "description": "Binary detection on the first 8192 bytes. An incomplete trailing UTF-8 sequence of at most 3 bytes is still text. NUL and genuinely invalid bytes are binary.", + "cases": [ + {"id": "nul", "hex": "6100", "expect_binary": true}, + {"id": "invalid-ff", "hex": "61ff62", "expect_binary": true}, + {"id": "bad-continuation", "hex": "c341", "expect_binary": true}, + {"id": "valid-2byte", "hex": "c3a9", "expect_binary": false}, + {"id": "valid-3byte", "hex": "e282ac", "expect_binary": false}, + {"id": "valid-4byte", "hex": "f09f9880", "expect_binary": false}, + {"id": "cut-2byte-at-8192", "fill": "a", "length": 8192, "tail_hex": "c3", "expect_binary": false}, + {"id": "cut-3byte-at-8192", "fill": "a", "length": 8192, "tail_hex": "e282", "expect_binary": false}, + {"id": "cut-4byte-at-8192", "fill": "a", "length": 8192, "tail_hex": "f09f98", "expect_binary": false}, + {"id": "invalid-at-end-of-window", "fill": "a", "length": 8192, "tail_hex": "ff", "expect_binary": true} + ] +} diff --git a/test/fixtures/parity/parser.json b/test/fixtures/parity/parser.json new file mode 100644 index 0000000..1eb488d --- /dev/null +++ b/test/fixtures/parity/parser.json @@ -0,0 +1,21 @@ +{ + "description": "Leading standalone yes/no after whitespace, quotes, markdown bold, backticks, and Answer:/Decision: prefixes. Think tags are stripped first. Anything else is malformed (null).", + "cases": [ + {"id": "plain-yes", "raw": "yes", "decision": "yes"}, + {"id": "plain-no", "raw": "NO", "decision": "no"}, + {"id": "punctuation", "raw": "Yes.", "decision": "yes"}, + {"id": "quoted-yes", "raw": "\"yes\"", "decision": "yes"}, + {"id": "quoted-no", "raw": "'no'", "decision": "no"}, + {"id": "bold-yes", "raw": "**yes**", "decision": "yes"}, + {"id": "backtick-no", "raw": "`no`", "decision": "no"}, + {"id": "answer-prefix", "raw": "Answer: yes, the parser lives here", "decision": "yes"}, + {"id": "decision-prefix", "raw": "Decision: no", "decision": "no"}, + {"id": "negated", "raw": "There is no doubt this file is required", "decision": null}, + {"id": "explanatory", "raw": "This file is unrelated to the task", "decision": null}, + {"id": "nope", "raw": "nope", "decision": null}, + {"id": "leading-yes-later-no", "raw": "yes for the helper, no for the tests", "decision": "yes"}, + {"id": "think-then-yes", "raw": "no, skip it\nyes", "decision": "yes"}, + {"id": "empty", "raw": "", "decision": null}, + {"id": "whitespace", "raw": " \n yes — needed", "decision": "yes"} + ] +} diff --git a/test/fixtures/parity/truncation.json b/test/fixtures/parity/truncation.json new file mode 100644 index 0000000..b1bcd9f --- /dev/null +++ b/test/fixtures/parity/truncation.json @@ -0,0 +1,18 @@ +{ + "description": "UTF-8 truncation on a code-point boundary. Never emit U+FFFD. Shared with the JS filter.", + "cases": [ + {"id": "2byte-max-1", "text": "éX", "max_bytes": 1, "expected": ""}, + {"id": "2byte-max-2", "text": "éX", "max_bytes": 2, "expected": "é"}, + {"id": "2byte-max-3", "text": "éX", "max_bytes": 3, "expected": "éX"}, + {"id": "3byte-max-1", "text": "€X", "max_bytes": 1, "expected": ""}, + {"id": "3byte-max-2", "text": "€X", "max_bytes": 2, "expected": ""}, + {"id": "3byte-max-3", "text": "€X", "max_bytes": 3, "expected": "€"}, + {"id": "3byte-max-4", "text": "€X", "max_bytes": 4, "expected": "€X"}, + {"id": "4byte-max-1", "text": "😀X", "max_bytes": 1, "expected": ""}, + {"id": "4byte-max-2", "text": "😀X", "max_bytes": 2, "expected": ""}, + {"id": "4byte-max-3", "text": "😀X", "max_bytes": 3, "expected": ""}, + {"id": "4byte-max-4", "text": "😀X", "max_bytes": 4, "expected": "😀"}, + {"id": "4byte-max-5", "text": "😀X", "max_bytes": 5, "expected": "😀X"}, + {"id": "ascii-max-1", "text": "ab", "max_bytes": 1, "expected": "a"} + ] +} diff --git a/test/parity_fixtures.test.js b/test/parity_fixtures.test.js new file mode 100644 index 0000000..1799822 --- /dev/null +++ b/test/parity_fixtures.test.js @@ -0,0 +1,77 @@ +/** + * Shared parity fixtures vendored byte-for-byte from 255BITS/patchling-py + * (tests/fixtures/parity/, commit 533749a). The Python suite runs the same + * cases in tests/test_relevance.py. Each file's sha256 is pinned so a drift + * on either side shows up as a failure here. + */ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { readFileSync } from 'node:fs'; +import { isBinaryContent, parseRelevanceAnswer } from '../src/index.js'; +import { truncateUtf8 } from '../src/filter.js'; + +const FIXTURE_SHA256 = { + 'binary.json': 'babcc94fe1dd990f0a69c41819ed9f76a185756ff28fa2d0e14651a190314934', + 'parser.json': '809a3522c79781da2aa2f10168bafd770aa781dc29c215b1dd885689047d3c7b', + 'truncation.json': 'c1d9db9b70687c24068f13a0b2a0edcf84ca3c5830e166336ff4da3c35b5c0df', +}; + +function loadFixture(name) { + const bytes = readFileSync(new URL(`./fixtures/parity/${name}`, import.meta.url)); + return { bytes, data: JSON.parse(bytes.toString('utf8')) }; +} + +const fixtures = Object.fromEntries( + Object.keys(FIXTURE_SHA256).map((name) => [name, loadFixture(name)]), +); + +for (const [name, sha256] of Object.entries(FIXTURE_SHA256)) { + test(`${name} bytes match the recorded sha256`, () => { + assert.equal(createHash('sha256').update(fixtures[name].bytes).digest('hex'), sha256); + }); +} + +// Same construction as patchling-py test_parity_binary_cut_fixtures: +// fill * (length - len(tail)) + tail, so `length` is the total size and the +// tail sits at the very end of that window. +function binaryCaseBytes(c) { + if (c.hex !== undefined) return new Uint8Array(Buffer.from(c.hex, 'hex')); + const tail = Buffer.from(c.tail_hex, 'hex'); + const fill = Buffer.from(c.fill ?? 'a', 'ascii'); + assert.equal(fill.length, 1, `${c.id}: fill must be one ASCII byte`); + assert.ok(tail.length <= c.length, `${c.id}: tail longer than length`); + const pad = Buffer.alloc(c.length - tail.length, fill); + const bytes = Buffer.concat([pad, tail]); + assert.equal(bytes.length, c.length, `${c.id}: sample must be exactly ${c.length} bytes`); + return new Uint8Array(bytes); +} + +test('binary.json: every case matches isBinaryContent', () => { + const { cases } = fixtures['binary.json'].data; + assert.ok(cases.length > 0); + for (const c of cases) { + const sample = binaryCaseBytes(c); + if (c.length !== undefined) assert.equal(sample.length, 8192, `${c.id}: windowed cases are 8192 bytes`); + assert.equal(isBinaryContent(sample), c.expect_binary, c.id); + } +}); + +test('parser.json: every case matches parseRelevanceAnswer', () => { + const { cases } = fixtures['parser.json'].data; + assert.ok(cases.length > 0); + for (const c of cases) { + assert.equal(parseRelevanceAnswer(c.raw).decision, c.decision, c.id); + } +}); + +test('truncation.json: every case matches truncateUtf8 and never emits U+FFFD', () => { + const { cases } = fixtures['truncation.json'].data; + assert.ok(cases.length > 0); + for (const c of cases) { + const out = truncateUtf8(c.text, c.max_bytes); + assert.equal(out, c.expected, c.id); + assert.ok(!out.includes('\uFFFD'), c.id); + assert.ok(Buffer.byteLength(out, 'utf8') <= Math.max(c.max_bytes, 0), c.id); + } +});