feat(research): mine novel cross-discussion leads with alternative local embeddings

README.md

@@ -383,3 +383,17 @@ PIE_EMBED_COMPARE_MODELS='all-minilm,nomic-embed-text' \
 The comparison calls only local Ollama (`127.0.0.1:11434`), supports different vector dimensions, and reports within-model cosine similarities, candidate retrieval thresholds, and pairwise synthetic ranking AUC. Optional JSON reports are private and cannot silently overwrite an existing file. **The eight input pairs are synthetic development cases, not a held-out benchmark**; all numbers must be labeled development-only. Similarity thresholds are model-specific and must never be copied between models without calibration.
 
 Local developer smoke test (October 2026): `all-minilm` and `nomic-embed-text` each ranked 11/15 synthetic positive-negative comparisons correctly (development AUC approximately 0.733). Their score ranges overlap; neither clearly wins and no production model change was made. Independently annotated real-source positives and hard negatives are required before choosing a replacement model.
+
+### Mine novel real-source pairs with an alternative local embedding model (P0)
+
+After verifying public source reports and saving their source-current extraction decisions, run:
+
+```bash
+PIE_ALTERNATE_EMBED_MODEL=nomic-embed-text \
+  PIE_ALTERNATE_MAX_NODES=100 PIE_ALTERNATE_MAX_PAIRS=12 \
+  PIE_ALTERNATE_MIN_SIMILARITY=0.4 \
+  PIE_ALTERNATE_MINING_REPORT_PATH=/tmp/pie-alternate-pairs.json \
+  npm run research:alternate-pairs
+```
+
+This is a **development candidate-mining** tool, not a corroboration verdict. It loads only latest source-current, model-screened problems, computes alternative embeddings **in memory only** using localhost Ollama, and ranks previously unreviewed pairs from different original conversations. It excludes current-snapshot pairs already semantically decided, caps each source's exposure and reports source URLs for manual inspection. It neither writes vectors to production pgvector nor sends any candidate to a semantic or paid model. The optional JSON report is private, refuses overwrites and never exposes account hashes. Similarity scores are not comparable between embedding models without calibration. Mined pairs must **not** be inserted into a frozen independent holdout without a fresh sampling protocol and leakage review.

package.json

@@ -29,7 +29,8 @@
     "eval:quality-gate": "node packages/agents/holdout-quality.mjs",
     "eval:annotation-ui": "node packages/agents/annotation-ui.mjs",
     "eval:embedding-retrieval": "node packages/agents/evaluate-embeddings.mjs",
-    "eval:compare-embeddings": "node packages/agents/compare-embeddings.mjs"
+    "eval:compare-embeddings": "node packages/agents/compare-embeddings.mjs",
+    "research:alternate-pairs": "node packages/agents/mine-alternate-pairs.mjs"
   },
   "dependencies": {
     "pg": "^8.23.1"

packages/agents/mine-alternate-pairs.mjs

@@ -0,0 +1,95 @@
+import {createPool} from '../storage/postgres.mjs';
+import {writeFile} from 'node:fs/promises';
+import {embedLocalComparisonText} from './compare-embeddings.mjs';
+import {cosineSimilarity} from './evaluate-embeddings.mjs';
+
+export const CURRENT_MINING_PROBLEMS_SQL=`
+WITH recent AS (
+ SELECT d.run_id,d.observation_id,d.statement,d.status,d.source_content_hash,
+ row_number() OVER(PARTITION BY d.observation_id ORDER BY r.started_at DESC,r.id DESC) AS rn
+ FROM model_decisions d JOIN model_runs r ON r.id=d.run_id
+)
+SELECT m.run_id,m.observation_id AS id,m.statement,o.source,o.url,
+       o.thread_root_id,o.reporter_key
+FROM recent m JOIN observations o ON o.id=m.observation_id
+WHERE m.rn=1 AND m.status='found' AND m.source_content_hash=o.content_hash
+ORDER BY md5(m.observation_id || $2) LIMIT $1`;
+export const PREVIOUS_MINING_VERDICTS_SQL=`
+SELECT left_run,left_observation,right_run,right_observation,verdict
+FROM corroboration_edges
+WHERE verdict IN ('same_problem','different_problem','uncertain')`;
+const key=(a,b)=>[a,b].sort().join('|');
+function validateCandidate(row){
+ return typeof row?.id==='string'&&typeof row.statement==='string'&&row.statement.trim()&&
+  typeof row.run_id==='string'&&['github','hackernews'].includes(row.source)&&row.url?.startsWith('https://');
+}
+/** Rank unlabeled leads, not corroborated pairs or ground-truth labels. */
+export function rankNovelCrossDiscussionPairs(rows,embeddings,priorEdges=[],{limit=12,maxPerObservation=3,minSimilarity=0.25}={}){
+ if(!Array.isArray(rows)||!Array.isArray(embeddings)||rows.length!==embeddings.length||!Array.isArray(priorEdges)||
+  !Number.isInteger(limit)||limit<1||limit>100||!Number.isInteger(maxPerObservation)||maxPerObservation<1||maxPerObservation>10||
+  !Number.isFinite(minSimilarity)||minSimilarity< -1||minSimilarity>1)throw Error('Invalid mining input or budget');
+ if(rows.some(row=>!validateCandidate(row)))throw Error('Invalid verified source candidate');
+ const excluded=new Set();
+ const snapshot=new Map(rows.map(row=>[row.id,row.run_id]));
+ for(const edge of priorEdges){
+  if(!['same_problem','different_problem','uncertain'].includes(edge.verdict))continue;
+  if(snapshot.get(edge.left_observation)===edge.left_run&&snapshot.get(edge.right_observation)===edge.right_run)
+   excluded.add(key(edge.left_observation,edge.right_observation));
+ }
+ const candidates=[];
+ let sameThreadSkipped=0,alreadyReviewedSkipped=0;
+ for(let i=0;i<rows.length;i++)for(let j=i+1;j<rows.length;j++){
+  const a=rows[i],b=rows[j];
+  if(a.id===b.id|| (a.thread_root_id||a.id)===(b.thread_root_id||b.id)){sameThreadSkipped++;continue;}
+  if(excluded.has(key(a.id,b.id))){alreadyReviewedSkipped++;continue;}
+  const similarity=cosineSimilarity(embeddings[i],embeddings[j]);
+  if(similarity<minSimilarity)continue;
+  candidates.push({leftId:a.id,rightId:b.id,similarity,platformPair:[a.source,b.source].sort().join('+'),
+   left:a,right:b});
+ }
+ candidates.sort((a,b)=>b.similarity-a.similarity||key(a.leftId,a.rightId).localeCompare(key(b.leftId,b.rightId)));
+ const result=[],exposure=new Map();
+ for(const row of candidates){
+  if(result.length>=limit)break;
+  if((exposure.get(row.leftId)||0)>=maxPerObservation||(exposure.get(row.rightId)||0)>=maxPerObservation)continue;
+  exposure.set(row.leftId,(exposure.get(row.leftId)||0)+1);
+  exposure.set(row.rightId,(exposure.get(row.rightId)||0)+1);
+  const distinctAccounts=row.left.source===row.right.source&&row.left.reporter_key&&row.right.reporter_key
+   ?row.left.reporter_key!==row.right.reporter_key:null;
+  result.push({left:{id:row.leftId,url:row.left.url,source:row.left.source,statement:row.left.statement},
+   right:{id:row.rightId,url:row.right.url,source:row.right.source,statement:row.right.statement},
+   similarity:Number(row.similarity.toFixed(4)),platformPair:row.platformPair,
+   distinctSourceAccounts:distinctAccounts,independentlyVerifiedHumans:null,
+   semanticEquivalence:'unreviewed',commercialDemand:'unknown'});
+ }
+ return {selected:result,eligibleScoredPairs:candidates.length,sameThreadSkipped,alreadyReviewedSkipped};
+}
+export async function mineAlternateEmbeddingPairs(pool,{model='nomic-embed-text',maxNodes=100,maxPairs=12,
+ maxPerObservation=3,minSimilarity=0.25,embed=embedLocalComparisonText}={}){
+ if(typeof model!=='string'||!/^[a-z0-9][a-z0-9._/-]*(?::[a-z0-9._-]+)?$/i.test(model)||
+  !Number.isInteger(maxNodes)||maxNodes<2||maxNodes>150)throw Error('Invalid local model or node budget');
+ const {rows}=await pool.query(CURRENT_MINING_PROBLEMS_SQL,[maxNodes,'alternate-embedding-mining-v1']);
+ if(rows.length<2)return {kind:'unreviewed_alternate_embedding_candidates',model,sourceObservations:rows.length,
+  independentHoldout:false,productionEmbeddingsUnchanged:true,pairs:[],reason:'insufficient_current_sources'};
+ const {rows:reviewed}=await pool.query(PREVIOUS_MINING_VERDICTS_SQL);
+ const embeddings=[];
+ for(const row of rows)embeddings.push(await embed(row.statement,{model}));
+ const mining=rankNovelCrossDiscussionPairs(rows,embeddings,reviewed,{limit:maxPairs,maxPerObservation,minSimilarity});
+ return {kind:'unreviewed_alternate_embedding_candidates',model,sourceObservations:rows.length,
+  independentHoldout:false,productionEmbeddingsUnchanged:true,modelJudgmentsPerformed:0,
+  ...mining,pairs:mining.selected,selected:undefined,
+  caveat:'Candidate mining only: these are not same-problem labels, independent human confirmations, or market demand. Do not add mined pairs to frozen independent holdouts without a new sampling and leakage protocol.'};
+}
+if(process.argv[1]?.endsWith('/packages/agents/mine-alternate-pairs.mjs')){
+ const pool=createPool();try{
+  const result=await mineAlternateEmbeddingPairs(pool,{
+   model:process.env.PIE_ALTERNATE_EMBED_MODEL||'nomic-embed-text',
+   maxNodes:Number(process.env.PIE_ALTERNATE_MAX_NODES||100),
+   maxPairs:Number(process.env.PIE_ALTERNATE_MAX_PAIRS||12),
+   minSimilarity:Number(process.env.PIE_ALTERNATE_MIN_SIMILARITY||0.25)
+  });
+  const path=process.env.PIE_ALTERNATE_MINING_REPORT_PATH;
+  if(path)await writeFile(path,JSON.stringify(result,null,2)+'\n',{mode:0o600,flag:'wx'});
+  console.log(JSON.stringify(result,null,2));
+ }finally{await pool.end();}
+}

tests/mine-alternate-pairs.test.mjs

@@ -0,0 +1,61 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import {rankNovelCrossDiscussionPairs,mineAlternateEmbeddingPairs,CURRENT_MINING_PROBLEMS_SQL,PREVIOUS_MINING_VERDICTS_SQL} from '../packages/agents/mine-alternate-pairs.mjs';
+const row=(id,thread=id,source='github')=>({id,run_id:'run-'+id,statement:'A specific failure for '+id,
+ source,url:source==='github'?'https://github.com/a/b/issues/'+id:'https://news.ycombinator.com/item?id='+id,
+ thread_root_id:thread,reporter_key:'account-'+id});
+test('rejects same-story comments, previously reviewed current snapshots, and caps pair exposure',()=>{
+ const a=row('a','thread-one','hackernews'),b=row('b','thread-one','hackernews'),
+ c=row('c','thread-two'),d=row('d','thread-three');
+ const prior=[{left_observation:'a',left_run:'run-a',right_observation:'c',right_run:'run-c',verdict:'different_problem'}];
+ const result=rankNovelCrossDiscussionPairs([a,b,c,d],[[1,0],[.99,.1],[.95,.15],[.9,.2]],prior,{limit:8,maxPerObservation:1,minSimilarity:-1});
+ assert.ok(result.sameThreadSkipped>=1);
+ assert.equal(result.alreadyReviewedSkipped,1);
+ assert.equal(result.selected.length,2);
+ assert.ok(!result.selected.some(x=>x.left.id==='a'&&x.right.id==='b'));
+ assert.ok(result.selected.every(x=>x.semanticEquivalence==='unreviewed'));
+ assert.ok(result.selected.every(x=>x.independentlyVerifiedHumans===null));
+});
+test('old verdicts for changed source versions do not block fresh candidates',()=>{
+ const a=row('a'),b=row('b');
+ const edges=[{left_observation:'a',left_run:'old-run',right_observation:'b',right_run:'run-b',verdict:'different_problem'}];
+ const result=rankNovelCrossDiscussionPairs([a,b],[[1,0],[.9,.1]],edges,{minSimilarity:0.1});
+ assert.equal(result.selected.length,1);
+ assert.equal(result.alreadyReviewedSkipped,0);
+});
+test('live mining uses source-current SQL and local model without changing pgvector',async()=>{
+ const calls=[];
+ const a=row('a'),b=row('b','hackernews:thread','hackernews');
+ const db={query:async(sql,args)=>{
+  calls.push({sql,args});
+  if(sql===CURRENT_MINING_PROBLEMS_SQL)return {rows:[a,b]};
+  if(sql===PREVIOUS_MINING_VERDICTS_SQL)return {rows:[]};
+  throw Error('Unexpected query');
+ }};
+ const requested=[];
+ const result=await mineAlternateEmbeddingPairs(db,{model:'nomic-embed-text',maxNodes:30,
+  embed:async(text,{model})=>{requested.push({text,model});return text.includes('a')?[1,0]:[.8,.3];}});
+ assert.equal(result.pairs.length,1);
+ assert.equal(result.modelJudgmentsPerformed,0);
+ assert.equal(result.productionEmbeddingsUnchanged,true);
+ assert.equal(result.independentHoldout,false);
+ assert.equal(requested.length,2);
+ assert.ok(requested.every(x=>x.model==='nomic-embed-text'));
+ assert.equal(calls[0].sql,CURRENT_MINING_PROBLEMS_SQL);
+ assert.equal(calls[1].sql,PREVIOUS_MINING_VERDICTS_SQL);
+ assert.equal(JSON.stringify(result).includes('account-a'),false);
+});
+test('mining rejects bad budgets and missing source-grounded inputs',async()=>{
+ const a=row('a'),b=row('b');
+ assert.throws(()=>rankNovelCrossDiscussionPairs([a,b],[[1,0]],[]),/Invalid mining/);
+ await assert.rejects(mineAlternateEmbeddingPairs({}, {model:'https://somewhere',maxNodes:30}),/Invalid local/);
+ const short=await mineAlternateEmbeddingPairs({query:async()=>({rows:[a]})},{maxNodes:10});
+ assert.equal(short.pairs.length,0);
+});
+
+test('source-scoped hashes cannot prove distinct accounts across different platforms',()=>{
+ const a=row('a','a','github'),b=row('b','b','hackernews');
+ const result=rankNovelCrossDiscussionPairs([a,b],[[1,0],[1,0]],[],{minSimilarity:.2});
+ assert.equal(result.selected[0].distinctSourceAccounts,null);
+ assert.equal(result.selected[0].independentlyVerifiedHumans,null);
+});