Make schema.sql re-runnable, fix code text search, and fix main's stale lockfile

index/README.md

@@ -1,3 +1,16 @@
 # index
 
 Postgres + pgvector schema and the embed-and-store job (steps 2 and 5).
+
+Local setup (macOS):
+
+```bash
+brew install postgresql@17 pgvector
+brew services run postgresql@17
+index/setup_local.sh
+```
+
+`setup_local.sh` creates the `adhd` role and the `adhd_rag` database that `retrieval/retrieve.py`
+connects to by default, then applies `schema.sql`. Both are safe to re-run. A database created
+from an older `schema.sql` keeps its English-stemmed `text_search` values until rows are rewritten;
+drop and recreate it for a clean start.

index/schema.sql

@@ -1,5 +1,5 @@
 -- Database schema for RAG chunk storage
--- Matches contracts/chunk.schema.json
+-- Matches contracts/chunk.schema.json. Safe to re-run.
 
 -- Enable required extensions
 CREATE EXTENSION IF NOT EXISTS vector;      -- pgvector for embeddings
@@ -32,13 +32,21 @@ CREATE TABLE IF NOT EXISTS chunks (
 
     -- Search indexes (populated by index/ scripts)
     embedding vector(1024),  -- Qwen/Qwen3-Embedding-0.6B, 1024 dimensions
+    embedding_model TEXT,    -- model that produced `embedding`, so vectors from different models never mix
     text_search tsvector,    -- Full-text search
 
     -- Timestamps
-    created_at TIMESTAMP DEFAULT NOW(),
-    updated_at TIMESTAMP DEFAULT NOW()
+    created_at TIMESTAMPTZ DEFAULT NOW(),
+    updated_at TIMESTAMPTZ DEFAULT NOW(),
+
+    CHECK ((embedding IS NULL) = (embedding_model IS NULL))
 );
 
+-- Upgrade tables created by earlier versions of this file.
+ALTER TABLE chunks ADD COLUMN IF NOT EXISTS embedding_model TEXT;
+ALTER TABLE chunks ALTER COLUMN created_at TYPE TIMESTAMPTZ;
+ALTER TABLE chunks ALTER COLUMN updated_at TYPE TIMESTAMPTZ;
+
 -- Indexes for fast retrieval
 -- No ANN index at init. Exact cosine until the table is large, then HNSW.
 -- IVFFlat with lists = 100 assumes on the order of 100,000 rows; do not add it here.
@@ -61,18 +69,19 @@ CREATE INDEX IF NOT EXISTS chunks_file_hash_idx ON chunks(file_hash);
 -- 4. Composite index for incremental updates
 CREATE INDEX IF NOT EXISTS chunks_repo_path_idx ON chunks(repo, path);
 
--- Function to auto-update text_search column
+-- Function to auto-update text_search column.
+-- 'simple' keeps identifiers intact; 'english' stems get_user_settings to get/user/set.
 CREATE OR REPLACE FUNCTION update_text_search()
 RETURNS TRIGGER AS $$
 BEGIN
-    NEW.text_search = to_tsvector('english', NEW.text);
+    NEW.text_search = to_tsvector('simple', NEW.text);
     NEW.updated_at = NOW();
     RETURN NEW;
 END;
 $$ LANGUAGE plpgsql;
 
 -- Trigger to keep text_search in sync
-CREATE TRIGGER chunks_text_search_update
+CREATE OR REPLACE TRIGGER chunks_text_search_update
 BEFORE INSERT OR UPDATE ON chunks
 FOR EACH ROW
 EXECUTE FUNCTION update_text_search();
@@ -92,6 +101,12 @@ FROM chunks
 GROUP BY repo, kind, language
 ORDER BY repo, kind, language;
 
--- Grant permissions (for development)
-GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO adhd;
-GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO adhd;
+-- Grant permissions to the dev role, if it exists (see index/setup_local.sh)
+DO $$
+BEGIN
+    IF EXISTS (SELECT FROM pg_roles WHERE rolname = 'adhd') THEN
+        GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO adhd;
+        GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO adhd;
+    END IF;
+END
+$$;

index/setup_local.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+# Create the local role and database that retrieval/retrieve.py expects, then apply schema.sql.
+# Safe to re-run. Usage: index/setup_local.sh
+set -euo pipefail
+
+PG_BIN=${PG_BIN:-$(dirname "$(command -v psql || echo /opt/homebrew/opt/postgresql@17/bin/psql)")}
+PSQL=("$PG_BIN/psql" -h localhost -v ON_ERROR_STOP=1 -q)
+DB=${DB:-adhd_rag}
+export PGOPTIONS="-c client_min_messages=warning"  # hide "already exists, skipping" notices
+
+"${PSQL[@]}" -d postgres <<'SQL'
+DO $$
+BEGIN
+    IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = 'adhd') THEN
+        CREATE ROLE adhd LOGIN PASSWORD 'dev_password_change_in_prod';
+    END IF;
+END
+$$;
+SQL
+
+if ! "${PSQL[@]}" -d postgres -tAc "SELECT 1 FROM pg_database WHERE datname = '$DB'" | grep -q 1; then
+    "$PG_BIN/createdb" -h localhost -O adhd "$DB"
+fi
+
+"${PSQL[@]}" -d "$DB" -f "$(dirname "$0")/schema.sql"
+echo "Ready: postgresql://adhd:dev_password_change_in_prod@localhost:5432/$DB"

tests/test_schema.py

@@ -0,0 +1,54 @@
+import shutil
+import subprocess
+import uuid
+from pathlib import Path
+
+import pytest
+
+SCHEMA = Path(__file__).parent.parent / "index" / "schema.sql"
+PSQL = shutil.which("psql") or "/opt/homebrew/opt/postgresql@17/bin/psql"
+
+
+def psql(db, *args, check=True):
+    return subprocess.run([PSQL, "-h", "localhost", "-d", db, "-v", "ON_ERROR_STOP=1", "-Atq", *args], capture_output=True, text=True, check=check)
+
+
+def postgres_available():
+    try:
+        return psql("postgres", "-c", "SELECT 1", check=False).returncode == 0
+    except FileNotFoundError:
+        return False
+
+
+pytestmark = pytest.mark.skipif(not postgres_available(), reason="no local Postgres")
+
+
+@pytest.fixture
+def db():
+    name = f"schema_test_{uuid.uuid4().hex[:8]}"
+    psql("postgres", "-c", f"CREATE DATABASE {name}")
+    yield name
+    psql("postgres", "-c", f"DROP DATABASE {name}")
+
+
+ROW = """INSERT INTO chunks (id, repo, path, start_line, end_line, symbol, language, kind, commit_sha, source_url, file_hash, token_count, text)
+VALUES ('t1', 'org/r', 'a.py', 1, 2, NULL, 'python', 'code', repeat('a', 40), 'https://x', repeat('b', 64), 5, 'def get_user_settings(): pass')"""
+
+
+def test_schema_applies_twice(db):
+    psql(db, "-f", str(SCHEMA))
+    psql(db, "-f", str(SCHEMA))
+
+
+def test_text_search_keeps_identifiers(db):
+    psql(db, "-f", str(SCHEMA))
+    psql(db, "-c", ROW)
+    assert "'settings'" in psql(db, "-c", "SELECT text_search FROM chunks").stdout
+
+
+def test_embedding_requires_model(db):
+    psql(db, "-f", str(SCHEMA))
+    psql(db, "-c", ROW)
+    vector = "array_fill(0.1, ARRAY[1024])::vector"
+    assert psql(db, "-c", f"UPDATE chunks SET embedding = {vector}", check=False).returncode != 0
+    psql(db, "-c", f"UPDATE chunks SET embedding = {vector}, embedding_model = 'm'")