diff --git a/README.md b/README.md index f78b538..2c1bce9 100644 --- a/README.md +++ b/README.md @@ -425,3 +425,11 @@ Local preflight with 44 embeddings returned 20/20 exact top candidates from 70 p ### Commercial hypotheses without invented buyer demand (P2 baseline) `npm run market:hypothesis -- github: [/private/market-evidence.json]` builds a **read-only commercial research dossier** from a source-current verified problem. Optional input uses `{ "kind":"market_evidence_input", "observationId":"github:...", "entries":[{"type":"alternative", "url":"https://...", "quote":"...", "observedAt":"2026-10-13"}] }`. Supported types: `alternative`, `manual_workaround`, `buyer_statement`, `price_quote`, `purchase_claim`, `distribution_channel`. Every claim requires a concrete HTTPS source and an excerpt; submitted quotes remain explicitly **unverified**. The report highlights missing buyer interviews, alternative research and independently verified financial evidence. No inferred budget, independently identified buyer, revenue or product-market fit is ever claimed. The CLI does not mutate PostgreSQL or publish any private evidence inputs. + +### Isolated 10k-vector HNSW scalability benchmark + +`npm run diagnostics:knn-load` creates a **temporary PostgreSQL table** containing 10,000 deterministic, deliberately synthetic 384-dimensional vectors (configurable `PIE_KNN_BENCH_VECTORS` 100–20,000). It builds a temporary HNSW index, measures exact vs approximate top-K neighbor recall at multiple anchors, captures actual PostgreSQL query-plan evidence, and reports p50/p95 latency. Its transaction is rolled back after the measurements, including on failure; no production observations, embeddings, model decisions or corroboration edges are changed. + +`PIE_KNN_BENCH_SAMPLES`, `PIE_KNN_BENCH_NEIGHBORS`, and `PIE_KNN_BENCH_EF_SEARCH` control bounded samples, top-K and HNSW effort. The CLI is experimental and its generated vectors are NOT representative of true user-problem distribution or evidence of source-level semantic recall. Production retrieval remains exact until a separate source-grounded evaluation confirms acceptable real-world recall and an index-using query plan at scale. + +**10k synthetic smoke-run (2026-10-13):** PostgreSQL EXPLAIN used `pie_hnsw_load_vector_idx` at 10,000 rows; six synthetic anchors retrieved 10/10 exact neighbors each (`recall@10=1.0` for this sample). HNSW p50/p95 were 0.66/1.47 ms, versus 5.50/17.19 ms for exact search. These times include database round trips, span only six queries and must not be presented as statistically representative production latency. Source-content lineage, filtering and real hard-paraphrase recall are outside this synthetic test; **no production retrieval switch was made**. diff --git a/package.json b/package.json index 0f2ba3c..9fcc170 100644 --- a/package.json +++ b/package.json @@ -34,7 +34,8 @@ "research:review-alternate": "node --env-file-if-exists=.env packages/agents/review-alternate-pairs.mjs", "eval:adjudicate-holdout": "node packages/agents/holdout-adjudication.mjs", "diagnostics:knn": "node packages/agents/knn-audit.mjs", - "market:hypothesis": "node packages/agents/market-hypothesis.mjs" + "market:hypothesis": "node packages/agents/market-hypothesis.mjs", + "diagnostics:knn-load": "node packages/agents/knn-load-benchmark.mjs" }, "dependencies": { "pg": "^8.23.1" diff --git a/packages/agents/knn-load-benchmark.mjs b/packages/agents/knn-load-benchmark.mjs new file mode 100644 index 0000000..3d6eb74 --- /dev/null +++ b/packages/agents/knn-load-benchmark.mjs @@ -0,0 +1,81 @@ +import {performance} from 'node:perf_hooks'; +import {createPool} from '../storage/postgres.mjs'; +import {compareKnnRecall} from './knn-audit.mjs'; + +const VECTOR_DIMENSIONS=384; +export function validateLoadBenchOptions({vectors=10000,neighbors=10,samples=6,efSearch=100}={}){ + if(!Number.isInteger(vectors)||vectors<100||vectors>20000|| + !Number.isInteger(neighbors)||neighbors<1||neighbors>30|| + !Number.isInteger(samples)||samples<1||samples>12|| + !Number.isInteger(efSearch)||efSearch<10||efSearch>300)throw Error('Invalid HNSW load benchmark bounds'); + return {vectors,neighbors,samples,efSearch}; +} +export function summarizeHnswLoad(samples,plan){ + if(!Array.isArray(samples)||!samples.length||samples.some(x=>!Number.isFinite(x.recallAtK)||!Number.isFinite(x.exactMs)||!Number.isFinite(x.annMs)))throw Error('Invalid HNSW measurement'); + const percentile=(vals,p)=>vals.slice().sort((a,b)=>a-b)[Math.min(vals.length-1,Math.ceil(p*vals.length)-1)]; + const ann=samples.map(x=>x.annMs),exact=samples.map(x=>x.exactMs); + return {samples:samples.length,recallMin:Math.min(...samples.map(x=>x.recallAtK)), + recallMean:Number((samples.reduce((s,x)=>s+x.recallAtK,0)/samples.length).toFixed(4)), + annP50Ms:Number(percentile(ann,.5).toFixed(2)),annP95Ms:Number(percentile(ann,.95).toFixed(2)), + exactP50Ms:Number(percentile(exact,.5).toFixed(2)),exactP95Ms:Number(percentile(exact,.95).toFixed(2)), + hnswIndexUsed:/Index Scan using pie_hnsw_load_vector_idx/.test(plan), + planExcerpt:String(plan||'').replace(/\[[^\]]+\]/g,'[vector omitted]').slice(0,1200), + productionQueryChanged:false}; +} +export async function benchmarkHnsw(pool,opts={}){ + const {vectors,neighbors,samples,efSearch}=validateLoadBenchOptions(opts); + const client=await pool.connect(); + try{ + await client.query('BEGIN'); + await client.query('SET LOCAL statement_timeout = 180000'); + await client.query('CREATE TEMP TABLE pie_hnsw_load (id INTEGER PRIMARY KEY, embedding vector(384)) ON COMMIT DROP'); + // Fully synthetic, deterministic vectors; zero write access to production observations. + await client.query(`INSERT INTO pie_hnsw_load(id,embedding) + SELECT i,('[' || array_to_string(ARRAY( + SELECT (sin(i*j*0.000173+j*0.017)+cos(i*0.009+j*j*0.0021))::real + FROM generate_series(1,${VECTOR_DIMENSIONS}) AS j + ),',') || ']')::vector(384) FROM generate_series(1,$1::int) AS i`,[vectors]); + await client.query('CREATE INDEX pie_hnsw_load_vector_idx ON pie_hnsw_load USING hnsw(embedding vector_cosine_ops) WITH(m=12,ef_construction=64)'); + await client.query('ANALYZE pie_hnsw_load'); + await client.query('SET LOCAL hnsw.ef_search = '+efSearch); + const ids=Array.from({length:samples},(_,i)=>1+Math.floor(i*(vectors-1)/Math.max(1,samples-1))); + const measurements=[]; + let plan=''; + for(const id of ids){ + const v=(await client.query('SELECT embedding::text AS vector FROM pie_hnsw_load WHERE id=$1',[id])).rows[0].vector; + // Turning off the ANN index only for the exact reference makes the expected + // neighbor set deterministic. This is a benchmark-local transaction setting. + await client.query('SET LOCAL enable_indexscan=off'); + await client.query('SET LOCAL enable_bitmapscan=off'); + const exactStart=performance.now(); + const exact=(await client.query('SELECT id,embedding <=> $1::vector AS distance FROM pie_hnsw_load WHERE id<>$2 ORDER BY embedding <=> $1::vector LIMIT $3',[v,id,neighbors])).rows; + const exactMs=performance.now()-exactStart; + await client.query('SET LOCAL enable_indexscan=on'); + await client.query('SET LOCAL enable_bitmapscan=on'); + if(!plan){const exp=await client.query('EXPLAIN SELECT id FROM pie_hnsw_load ORDER BY embedding <=> $1::vector LIMIT $2',[v,neighbors]);plan=exp.rows.map(x=>x['QUERY PLAN']).join('\n');} + const annStart=performance.now(); + const annAll=(await client.query('SELECT id,embedding <=> $1::vector AS distance FROM pie_hnsw_load ORDER BY embedding <=> $1::vector LIMIT $2',[v,neighbors+1])).rows; + const ann=annAll.filter(row=>row.id!==id).slice(0,neighbors); + const annMs=performance.now()-annStart; + const key=id=>String(id); + const stats=compareKnnRecall(exact.map(x=>({left_id:key(id),right_id:key(x.id)})),ann.map(x=>({left_id:key(id),right_id:key(x.id)})),{topK:neighbors}); + measurements.push({anchorId:id,recallAtK:stats.recallAtK,exactMs,annMs,missed:stats.missed.length}); + } + const summary=summarizeHnswLoad(measurements,plan); + return {kind:'synthetic_hnsw_load_benchmark',vectors,dimensions:VECTOR_DIMENSIONS,neighbors,efSearch, + ...summary,measurements,independentRealSourceAccuracy:false,productionVectorsUnchanged:true, + caveat:'Synthetic vectors and temporary HNSW index only. Verify recall and query plans on a separate representative real-source dataset before changing production retrieval.'}; + }finally{ + try{await client.query('ROLLBACK');}finally{client.release();} + } +} +if(process.argv[1]?.endsWith('/packages/agents/knn-load-benchmark.mjs')){ + const pool=createPool();try{ + console.log(JSON.stringify(await benchmarkHnsw(pool,{ + vectors:Number(process.env.PIE_KNN_BENCH_VECTORS||10000), + neighbors:Number(process.env.PIE_KNN_BENCH_NEIGHBORS||10), + samples:Number(process.env.PIE_KNN_BENCH_SAMPLES||6), + efSearch:Number(process.env.PIE_KNN_BENCH_EF_SEARCH||100) + }),null,2)); + }finally{await pool.end();} +} diff --git a/tests/knn-load-benchmark.test.mjs b/tests/knn-load-benchmark.test.mjs new file mode 100644 index 0000000..e3823ca --- /dev/null +++ b/tests/knn-load-benchmark.test.mjs @@ -0,0 +1,33 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import {validateLoadBenchOptions,summarizeHnswLoad,benchmarkHnsw} from '../packages/agents/knn-load-benchmark.mjs'; +test('load test has bounded dataset, search, sample and ANN effort',()=>{ + assert.deepEqual(validateLoadBenchOptions(),{vectors:10000,neighbors:10,samples:6,efSearch:100}); + assert.throws(()=>validateLoadBenchOptions({vectors:100000}),/bounds/); + assert.throws(()=>validateLoadBenchOptions({neighbors:0}),/bounds/); + assert.throws(()=>validateLoadBenchOptions({samples:30}),/bounds/); + assert.throws(()=>validateLoadBenchOptions({efSearch:5000}),/bounds/); +}); +test('summary reports recall, latency percentiles and sanitized query plan',()=>{ + const r=summarizeHnswLoad([{recallAtK:.8,exactMs:30,annMs:3},{recallAtK:1,exactMs:20,annMs:2}], + "Index Scan using pie_hnsw_load_vector_idx (embedding <=> '[1,2,3]'::vector)"); + assert.equal(r.recallMin,.8); + assert.equal(r.recallMean,.9); + assert.equal(r.annP95Ms,3); + assert.equal(r.hnswIndexUsed,true); + assert.ok(!r.planExcerpt.includes('[1,2,3]')); + assert.equal(r.productionQueryChanged,false); +}); +test('small integration benchmark drops temporary data and never writes canonical observations',async t=>{ + if(process.env.PIE_INTEGRATION!=='1')return t.skip('PIE_INTEGRATION=1'); + const {createPool}=await import('../packages/storage/postgres.mjs'); + const pool=createPool();try{ + const report=await benchmarkHnsw(pool,{vectors:120,neighbors:5,samples:2,efSearch:40}); + assert.equal(report.vectors,120); + assert.equal(report.measurements.length,2); + assert.ok(report.recallMean>=0&&report.recallMean<=1); + assert.equal(report.productionVectorsUnchanged,true); + const table=await pool.query("SELECT count(*)::int AS n FROM pg_class WHERE relname='pie_hnsw_load'"); + assert.equal(table.rows[0].n,0); + }finally{await pool.end();} +});