chore(db): drop unused ANN indexes on embedding

apps/sim/lib/knowledge/__integration__/scale.integration.ts

@@ -40,6 +40,15 @@ const rows = Number(process.env.KNOWLEDGE_SCALE_DOCUMENTS ?? 250_000)
 const SEED_BATCH_SIZE = 2_000
 const PAGE_SIZE = 500
 const DIMENSIONS = 1536
+/**
+ * `embedding` carries no ANN index in the schema — production serves approximate
+ * retrieval from the `embedding_search` projection, which this fixture does not
+ * populate. The benchmark still measures ANN behaviour over the dense corpus it
+ * seeds directly, so it owns this index rather than borrowing a schema one, and
+ * builds it after the load instead of paying index maintenance on every insert.
+ */
+const BENCHMARK_VECTOR_INDEX = 'embedding_scale_benchmark_hnsw_idx'
+const BENCHMARK_VECTOR_INDEX_DEFINITION = `CREATE INDEX IF NOT EXISTS ${BENCHMARK_VECTOR_INDEX} ON public.embedding USING hnsw (embedding vector_cosine_ops) WITH (m='16', ef_construction='64')`
 const logger = createLogger('KnowledgeScaleIntegration')
 if (reuseReportFile && statSync(reuseReportFile).size > 16 * 1024 * 1024)
   throw new Error('Retained scale report must be at most 16 MiB')
@@ -386,7 +395,6 @@ describe.skipIf(!enabled)('knowledge scale: isolated real PostgreSQL, no provide
   it.skipIf(metadataOnly)(
     'stores a bounded dense corpus and measures ACL/tag-filtered vector and hybrid retrieval',
     async () => {
-      let vectorIndexDefinition: string | undefined
       if (bulkSeed) {
         const other = await db
           .select({ id: embedding.id })
@@ -397,13 +405,6 @@ describe.skipIf(!enabled)('knowledge scale: isolated real PostgreSQL, no provide
           throw new Error(
             'Bulk scale setup requires a database containing only its own fixture chunks'
           )
-        const [index] = await db.execute(
-          sql`SELECT indexdef FROM pg_indexes WHERE schemaname = 'public' AND indexname = 'embedding_vector_hnsw_idx'`
-        )
-        if (typeof index?.indexdef !== 'string')
-          throw new Error('Canonical 1536-dimensional HNSW index is missing')
-        vectorIndexDefinition = index.indexdef
-        await db.execute(sql`DROP INDEX embedding_vector_hnsw_idx`)
       }
       if (!reuseReportFile)
         await measure('seed.vectors', async () => {
@@ -420,20 +421,17 @@ describe.skipIf(!enabled)('knowledge scale: isolated real PostgreSQL, no provide
             }
           }
         })
-      if (vectorIndexDefinition) {
-        const definition = vectorIndexDefinition
-        await measure('seed.hnswBuild', () =>
-          db.transaction(async (tx) => {
-            await tx.execute(sql`SET LOCAL maintenance_work_mem = '2GB'`)
-            await tx.execute(sql`SET LOCAL max_parallel_maintenance_workers = 2`)
-            await tx.execute(sql.raw(definition))
-          })
-        )
-        const [restored] = await db.execute(
-          sql`SELECT indexdef FROM pg_indexes WHERE schemaname = 'public' AND indexname = 'embedding_vector_hnsw_idx'`
-        )
-        expect(restored.indexdef).toBe(vectorIndexDefinition)
-      }
+      await measure('seed.hnswBuild', () =>
+        db.transaction(async (tx) => {
+          await tx.execute(sql`SET LOCAL maintenance_work_mem = '2GB'`)
+          await tx.execute(sql`SET LOCAL max_parallel_maintenance_workers = 2`)
+          await tx.execute(sql.raw(BENCHMARK_VECTOR_INDEX_DEFINITION))
+        })
+      )
+      const [built] = await db.execute(
+        sql`SELECT indexdef FROM pg_indexes WHERE schemaname = 'public' AND indexname = ${BENCHMARK_VECTOR_INDEX}`
+      )
+      expect(built?.indexdef).toEqual(expect.stringContaining('USING hnsw'))
       await db.execute(sql`ANALYZE embedding`)
       await db.execute(sql`ANALYZE document`)
       const [count] = await db.execute(

apps/sim/lib/knowledge/vector-columns.ts

@@ -58,16 +58,17 @@ export function embeddingVectorValues(
 }
 
 /**
- * Cosine distance between a chunk's vector and the query vector, in the exact
- * form the width's HNSW index was built on.
+ * Cosine distance between a chunk's vector and the query vector, used for the
+ * search layer's exact rerank over an already-bounded candidate set.
  *
- * The 3,072 column is compared through a `halfvec` cast because pgvector
- * indexes `vector` only up to 2,000 dimensions, so its index is on that cast
- * expression. Postgres matches an expression index by the expression, so a
- * plain `<=>` against the column here would silently drop to a sequential scan
- * — and the cast belongs here rather than at each call site precisely because
- * getting it wrong is invisible in the results and only shows up as latency.
- * `packages/db/schema.ts` records what the half-precision comparison costs.
+ * `embedding` carries no ANN index — approximate retrieval runs against the
+ * compact `embedding_search` projection — so this expression is never expected
+ * to match one, and its caller wraps it to keep the planner from trying.
+ *
+ * The 3,072 column keeps its `halfvec` cast: the width that made the cast
+ * necessary is unchanged, and `packages/db/schema.ts` records that the
+ * half-precision comparison moved no distance measurably for the one model
+ * that emits this width.
  */
 export function embeddingDistance(
   dimensions: KbEmbeddingDimensions,

packages/db/migrations/0360_drop_unused_embedding_hnsw_indexes.sql

@@ -0,0 +1,27 @@
+-- Drops the ten unused ANN indexes on "embedding". Approximate retrieval moved to the
+-- compact "embedding_search" projection; the only vector ordering left on this table is an
+-- exact rerank wrapped as (distance) + 0, which the planner cannot match to an index.
+-- The last app version that ordered by a bare distance has drained, so nothing reads these.
+COMMIT;--> statement-breakpoint
+SET lock_timeout = 0;--> statement-breakpoint
+-- migration-safe: DROP INDEX CONCURRENTLY IF EXISTS is idempotent on replay and takes no blocking lock.
+DROP INDEX CONCURRENTLY IF EXISTS "embedding_vector_hnsw_idx";--> statement-breakpoint
+-- migration-safe: DROP INDEX CONCURRENTLY IF EXISTS is idempotent on replay and takes no blocking lock.
+DROP INDEX CONCURRENTLY IF EXISTS "embedding_384_vector_hnsw_idx";--> statement-breakpoint
+-- migration-safe: DROP INDEX CONCURRENTLY IF EXISTS is idempotent on replay and takes no blocking lock.
+DROP INDEX CONCURRENTLY IF EXISTS "embedding_768_vector_hnsw_idx";--> statement-breakpoint
+-- migration-safe: DROP INDEX CONCURRENTLY IF EXISTS is idempotent on replay and takes no blocking lock.
+DROP INDEX CONCURRENTLY IF EXISTS "embedding_1024_vector_hnsw_idx";--> statement-breakpoint
+-- migration-safe: DROP INDEX CONCURRENTLY IF EXISTS is idempotent on replay and takes no blocking lock.
+DROP INDEX CONCURRENTLY IF EXISTS "embedding_3072_vector_hnsw_idx";--> statement-breakpoint
+-- migration-safe: DROP INDEX CONCURRENTLY IF EXISTS is idempotent on replay and takes no blocking lock.
+DROP INDEX CONCURRENTLY IF EXISTS "embedding_binary_hnsw_idx";--> statement-breakpoint
+-- migration-safe: DROP INDEX CONCURRENTLY IF EXISTS is idempotent on replay and takes no blocking lock.
+DROP INDEX CONCURRENTLY IF EXISTS "embedding_384_binary_hnsw_idx";--> statement-breakpoint
+-- migration-safe: DROP INDEX CONCURRENTLY IF EXISTS is idempotent on replay and takes no blocking lock.
+DROP INDEX CONCURRENTLY IF EXISTS "embedding_768_binary_hnsw_idx";--> statement-breakpoint
+-- migration-safe: DROP INDEX CONCURRENTLY IF EXISTS is idempotent on replay and takes no blocking lock.
+DROP INDEX CONCURRENTLY IF EXISTS "embedding_1024_binary_hnsw_idx";--> statement-breakpoint
+-- migration-safe: DROP INDEX CONCURRENTLY IF EXISTS is idempotent on replay and takes no blocking lock.
+DROP INDEX CONCURRENTLY IF EXISTS "embedding_3072_binary_hnsw_idx";--> statement-breakpoint
+SET lock_timeout = '5s';