From a1e22d6aed253ef656ee32d2015af1b2a8cd68b7 Mon Sep 17 00:00:00 2001 From: drop-agent-bot <341481197+drop-agent-bot@users.noreply.github.com> Date: Mon, 12 Oct 2026 23:57:48 +0200 Subject: [PATCH] feat(research): mine novel cross-discussion leads with alternative local embeddings --- README.md | 14 ++++ package.json | 3 +- packages/agents/mine-alternate-pairs.mjs | 95 ++++++++++++++++++++++++ tests/mine-alternate-pairs.test.mjs | 61 +++++++++++++++ 4 files changed, 172 insertions(+), 1 deletion(-) create mode 100644 packages/agents/mine-alternate-pairs.mjs create mode 100644 tests/mine-alternate-pairs.test.mjs diff --git a/README.md b/README.md index 6f215c2..7b90a21 100644 --- a/README.md +++ b/README.md @@ -383,3 +383,17 @@ PIE_EMBED_COMPARE_MODELS='all-minilm,nomic-embed-text' \ The comparison calls only local Ollama (`127.0.0.1:11434`), supports different vector dimensions, and reports within-model cosine similarities, candidate retrieval thresholds, and pairwise synthetic ranking AUC. Optional JSON reports are private and cannot silently overwrite an existing file. **The eight input pairs are synthetic development cases, not a held-out benchmark**; all numbers must be labeled development-only. Similarity thresholds are model-specific and must never be copied between models without calibration. Local developer smoke test (October 2026): `all-minilm` and `nomic-embed-text` each ranked 11/15 synthetic positive-negative comparisons correctly (development AUC approximately 0.733). Their score ranges overlap; neither clearly wins and no production model change was made. Independently annotated real-source positives and hard negatives are required before choosing a replacement model. + +### Mine novel real-source pairs with an alternative local embedding model (P0) + +After verifying public source reports and saving their source-current extraction decisions, run: + +```bash +PIE_ALTERNATE_EMBED_MODEL=nomic-embed-text \ + PIE_ALTERNATE_MAX_NODES=100 PIE_ALTERNATE_MAX_PAIRS=12 \ + PIE_ALTERNATE_MIN_SIMILARITY=0.4 \ + PIE_ALTERNATE_MINING_REPORT_PATH=/tmp/pie-alternate-pairs.json \ + npm run research:alternate-pairs +``` + +This is a **development candidate-mining** tool, not a corroboration verdict. It loads only latest source-current, model-screened problems, computes alternative embeddings **in memory only** using localhost Ollama, and ranks previously unreviewed pairs from different original conversations. It excludes current-snapshot pairs already semantically decided, caps each source's exposure and reports source URLs for manual inspection. It neither writes vectors to production pgvector nor sends any candidate to a semantic or paid model. The optional JSON report is private, refuses overwrites and never exposes account hashes. Similarity scores are not comparable between embedding models without calibration. Mined pairs must **not** be inserted into a frozen independent holdout without a fresh sampling protocol and leakage review. diff --git a/package.json b/package.json index 2cf6b42..350138a 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,8 @@ "eval:quality-gate": "node packages/agents/holdout-quality.mjs", "eval:annotation-ui": "node packages/agents/annotation-ui.mjs", "eval:embedding-retrieval": "node packages/agents/evaluate-embeddings.mjs", - "eval:compare-embeddings": "node packages/agents/compare-embeddings.mjs" + "eval:compare-embeddings": "node packages/agents/compare-embeddings.mjs", + "research:alternate-pairs": "node packages/agents/mine-alternate-pairs.mjs" }, "dependencies": { "pg": "^8.23.1" diff --git a/packages/agents/mine-alternate-pairs.mjs b/packages/agents/mine-alternate-pairs.mjs new file mode 100644 index 0000000..f0f9547 --- /dev/null +++ b/packages/agents/mine-alternate-pairs.mjs @@ -0,0 +1,95 @@ +import {createPool} from '../storage/postgres.mjs'; +import {writeFile} from 'node:fs/promises'; +import {embedLocalComparisonText} from './compare-embeddings.mjs'; +import {cosineSimilarity} from './evaluate-embeddings.mjs'; + +export const CURRENT_MINING_PROBLEMS_SQL=` +WITH recent AS ( + SELECT d.run_id,d.observation_id,d.statement,d.status,d.source_content_hash, + row_number() OVER(PARTITION BY d.observation_id ORDER BY r.started_at DESC,r.id DESC) AS rn + FROM model_decisions d JOIN model_runs r ON r.id=d.run_id +) +SELECT m.run_id,m.observation_id AS id,m.statement,o.source,o.url, + o.thread_root_id,o.reporter_key +FROM recent m JOIN observations o ON o.id=m.observation_id +WHERE m.rn=1 AND m.status='found' AND m.source_content_hash=o.content_hash +ORDER BY md5(m.observation_id || $2) LIMIT $1`; +export const PREVIOUS_MINING_VERDICTS_SQL=` +SELECT left_run,left_observation,right_run,right_observation,verdict +FROM corroboration_edges +WHERE verdict IN ('same_problem','different_problem','uncertain')`; +const key=(a,b)=>[a,b].sort().join('|'); +function validateCandidate(row){ + return typeof row?.id==='string'&&typeof row.statement==='string'&&row.statement.trim()&& + typeof row.run_id==='string'&&['github','hackernews'].includes(row.source)&&row.url?.startsWith('https://'); +} +/** Rank unlabeled leads, not corroborated pairs or ground-truth labels. */ +export function rankNovelCrossDiscussionPairs(rows,embeddings,priorEdges=[],{limit=12,maxPerObservation=3,minSimilarity=0.25}={}){ + if(!Array.isArray(rows)||!Array.isArray(embeddings)||rows.length!==embeddings.length||!Array.isArray(priorEdges)|| + !Number.isInteger(limit)||limit<1||limit>100||!Number.isInteger(maxPerObservation)||maxPerObservation<1||maxPerObservation>10|| + !Number.isFinite(minSimilarity)||minSimilarity< -1||minSimilarity>1)throw Error('Invalid mining input or budget'); + if(rows.some(row=>!validateCandidate(row)))throw Error('Invalid verified source candidate'); + const excluded=new Set(); + const snapshot=new Map(rows.map(row=>[row.id,row.run_id])); + for(const edge of priorEdges){ + if(!['same_problem','different_problem','uncertain'].includes(edge.verdict))continue; + if(snapshot.get(edge.left_observation)===edge.left_run&&snapshot.get(edge.right_observation)===edge.right_run) + excluded.add(key(edge.left_observation,edge.right_observation)); + } + const candidates=[]; + let sameThreadSkipped=0,alreadyReviewedSkipped=0; + for(let i=0;ib.similarity-a.similarity||key(a.leftId,a.rightId).localeCompare(key(b.leftId,b.rightId))); + const result=[],exposure=new Map(); + for(const row of candidates){ + if(result.length>=limit)break; + if((exposure.get(row.leftId)||0)>=maxPerObservation||(exposure.get(row.rightId)||0)>=maxPerObservation)continue; + exposure.set(row.leftId,(exposure.get(row.leftId)||0)+1); + exposure.set(row.rightId,(exposure.get(row.rightId)||0)+1); + const distinctAccounts=row.left.source===row.right.source&&row.left.reporter_key&&row.right.reporter_key + ?row.left.reporter_key!==row.right.reporter_key:null; + result.push({left:{id:row.leftId,url:row.left.url,source:row.left.source,statement:row.left.statement}, + right:{id:row.rightId,url:row.right.url,source:row.right.source,statement:row.right.statement}, + similarity:Number(row.similarity.toFixed(4)),platformPair:row.platformPair, + distinctSourceAccounts:distinctAccounts,independentlyVerifiedHumans:null, + semanticEquivalence:'unreviewed',commercialDemand:'unknown'}); + } + return {selected:result,eligibleScoredPairs:candidates.length,sameThreadSkipped,alreadyReviewedSkipped}; +} +export async function mineAlternateEmbeddingPairs(pool,{model='nomic-embed-text',maxNodes=100,maxPairs=12, + maxPerObservation=3,minSimilarity=0.25,embed=embedLocalComparisonText}={}){ + if(typeof model!=='string'||!/^[a-z0-9][a-z0-9._/-]*(?::[a-z0-9._-]+)?$/i.test(model)|| + !Number.isInteger(maxNodes)||maxNodes<2||maxNodes>150)throw Error('Invalid local model or node budget'); + const {rows}=await pool.query(CURRENT_MINING_PROBLEMS_SQL,[maxNodes,'alternate-embedding-mining-v1']); + if(rows.length<2)return {kind:'unreviewed_alternate_embedding_candidates',model,sourceObservations:rows.length, + independentHoldout:false,productionEmbeddingsUnchanged:true,pairs:[],reason:'insufficient_current_sources'}; + const {rows:reviewed}=await pool.query(PREVIOUS_MINING_VERDICTS_SQL); + const embeddings=[]; + for(const row of rows)embeddings.push(await embed(row.statement,{model})); + const mining=rankNovelCrossDiscussionPairs(rows,embeddings,reviewed,{limit:maxPairs,maxPerObservation,minSimilarity}); + return {kind:'unreviewed_alternate_embedding_candidates',model,sourceObservations:rows.length, + independentHoldout:false,productionEmbeddingsUnchanged:true,modelJudgmentsPerformed:0, + ...mining,pairs:mining.selected,selected:undefined, + caveat:'Candidate mining only: these are not same-problem labels, independent human confirmations, or market demand. Do not add mined pairs to frozen independent holdouts without a new sampling and leakage protocol.'}; +} +if(process.argv[1]?.endsWith('/packages/agents/mine-alternate-pairs.mjs')){ + const pool=createPool();try{ + const result=await mineAlternateEmbeddingPairs(pool,{ + model:process.env.PIE_ALTERNATE_EMBED_MODEL||'nomic-embed-text', + maxNodes:Number(process.env.PIE_ALTERNATE_MAX_NODES||100), + maxPairs:Number(process.env.PIE_ALTERNATE_MAX_PAIRS||12), + minSimilarity:Number(process.env.PIE_ALTERNATE_MIN_SIMILARITY||0.25) + }); + const path=process.env.PIE_ALTERNATE_MINING_REPORT_PATH; + if(path)await writeFile(path,JSON.stringify(result,null,2)+'\n',{mode:0o600,flag:'wx'}); + console.log(JSON.stringify(result,null,2)); + }finally{await pool.end();} +} diff --git a/tests/mine-alternate-pairs.test.mjs b/tests/mine-alternate-pairs.test.mjs new file mode 100644 index 0000000..bfa7e28 --- /dev/null +++ b/tests/mine-alternate-pairs.test.mjs @@ -0,0 +1,61 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import {rankNovelCrossDiscussionPairs,mineAlternateEmbeddingPairs,CURRENT_MINING_PROBLEMS_SQL,PREVIOUS_MINING_VERDICTS_SQL} from '../packages/agents/mine-alternate-pairs.mjs'; +const row=(id,thread=id,source='github')=>({id,run_id:'run-'+id,statement:'A specific failure for '+id, + source,url:source==='github'?'https://github.com/a/b/issues/'+id:'https://news.ycombinator.com/item?id='+id, + thread_root_id:thread,reporter_key:'account-'+id}); +test('rejects same-story comments, previously reviewed current snapshots, and caps pair exposure',()=>{ + const a=row('a','thread-one','hackernews'),b=row('b','thread-one','hackernews'), + c=row('c','thread-two'),d=row('d','thread-three'); + const prior=[{left_observation:'a',left_run:'run-a',right_observation:'c',right_run:'run-c',verdict:'different_problem'}]; + const result=rankNovelCrossDiscussionPairs([a,b,c,d],[[1,0],[.99,.1],[.95,.15],[.9,.2]],prior,{limit:8,maxPerObservation:1,minSimilarity:-1}); + assert.ok(result.sameThreadSkipped>=1); + assert.equal(result.alreadyReviewedSkipped,1); + assert.equal(result.selected.length,2); + assert.ok(!result.selected.some(x=>x.left.id==='a'&&x.right.id==='b')); + assert.ok(result.selected.every(x=>x.semanticEquivalence==='unreviewed')); + assert.ok(result.selected.every(x=>x.independentlyVerifiedHumans===null)); +}); +test('old verdicts for changed source versions do not block fresh candidates',()=>{ + const a=row('a'),b=row('b'); + const edges=[{left_observation:'a',left_run:'old-run',right_observation:'b',right_run:'run-b',verdict:'different_problem'}]; + const result=rankNovelCrossDiscussionPairs([a,b],[[1,0],[.9,.1]],edges,{minSimilarity:0.1}); + assert.equal(result.selected.length,1); + assert.equal(result.alreadyReviewedSkipped,0); +}); +test('live mining uses source-current SQL and local model without changing pgvector',async()=>{ + const calls=[]; + const a=row('a'),b=row('b','hackernews:thread','hackernews'); + const db={query:async(sql,args)=>{ + calls.push({sql,args}); + if(sql===CURRENT_MINING_PROBLEMS_SQL)return {rows:[a,b]}; + if(sql===PREVIOUS_MINING_VERDICTS_SQL)return {rows:[]}; + throw Error('Unexpected query'); + }}; + const requested=[]; + const result=await mineAlternateEmbeddingPairs(db,{model:'nomic-embed-text',maxNodes:30, + embed:async(text,{model})=>{requested.push({text,model});return text.includes('a')?[1,0]:[.8,.3];}}); + assert.equal(result.pairs.length,1); + assert.equal(result.modelJudgmentsPerformed,0); + assert.equal(result.productionEmbeddingsUnchanged,true); + assert.equal(result.independentHoldout,false); + assert.equal(requested.length,2); + assert.ok(requested.every(x=>x.model==='nomic-embed-text')); + assert.equal(calls[0].sql,CURRENT_MINING_PROBLEMS_SQL); + assert.equal(calls[1].sql,PREVIOUS_MINING_VERDICTS_SQL); + assert.equal(JSON.stringify(result).includes('account-a'),false); +}); +test('mining rejects bad budgets and missing source-grounded inputs',async()=>{ + const a=row('a'),b=row('b'); + assert.throws(()=>rankNovelCrossDiscussionPairs([a,b],[[1,0]],[]),/Invalid mining/); + await assert.rejects(mineAlternateEmbeddingPairs({}, {model:'https://somewhere',maxNodes:30}),/Invalid local/); + const short=await mineAlternateEmbeddingPairs({query:async()=>({rows:[a]})},{maxNodes:10}); + assert.equal(short.pairs.length,0); +}); + +test('source-scoped hashes cannot prove distinct accounts across different platforms',()=>{ + const a=row('a','a','github'),b=row('b','b','hackernews'); + const result=rankNovelCrossDiscussionPairs([a,b],[[1,0],[1,0]],[],{minSimilarity:.2}); + assert.equal(result.selected[0].distinctSourceAccounts,null); + assert.equal(result.selected[0].independentlyVerifiedHumans,null); +});