feat: route LLM calls through LiteLLM for multi-provider support

backend/app/condense.py

@@ -184,12 +184,11 @@ def _condense_with_claude(section: dict, max_chars: int) -> Optional[dict]:
         return None
 
     try:
-        from anthropic import Anthropic
+        import litellm
 
         allowed_ids = set(_union_memory_ids(elements))
         by_id = _elements_by_id(elements)
 
-        client = Anthropic(api_key=os.environ["ANTHROPIC_API_KEY"])
         model = os.environ.get(
             "CORTEX_CONDENSE_MODEL",
             os.environ.get("CORTEX_EXTRACTION_MODEL", "claude-opus-4-5"),
@@ -203,14 +202,18 @@ def _condense_with_claude(section: dict, max_chars: int) -> Optional[dict]:
         )
         user = _build_user_prompt(section, elements)
 
-        response = client.messages.create(
+        response = litellm.completion(
             model=model,
             max_tokens=300,
-            system=system,
-            messages=[{"role": "user", "content": user}],
+            messages=[
+                {"role": "system", "content": system},
+                {"role": "user", "content": user},
+            ],
+            # Drop provider-unsupported params so one config works across providers.
+            drop_params=True,
         )
 
-        text = response.content[0].text.strip()
+        text = (response.choices[0].message.content or "").strip()
         text = re.sub(r"^```(?:json)?\s*", "", text)
         text = re.sub(r"\s*```$", "", text)
         data = json.loads(text)

backend/app/embeddings.py

@@ -17,6 +17,9 @@
 VECTOR_DIMENSIONS = 384
 VECTOR_MODEL = "cortex-hash-v1"
 DEFAULT_OPENAI_EMBEDDING_MODEL = "text-embedding-3-small"
+# LiteLLM embeddings: a provider prefixed model reaches Voyage, Cohere, Gemini,
+# Bedrock, and more through one interface. Override with CORTEX_EMBEDDING_MODEL.
+DEFAULT_LITELLM_EMBEDDING_MODEL = "openai/text-embedding-3-small"
 
 # Local, CPU-only, no-API-key neural embeddings (MinishLab model2vec, MIT).
 # potion-base-8M emits 256-dim vectors natively (NOT 384) — so it is not
@@ -153,7 +156,7 @@ def model2vec_available() -> bool:
 
 def configured_embedding_provider() -> str:
     raw = os.environ.get("CORTEX_EMBEDDING_PROVIDER", "").strip().lower()
-    if raw in {"hash", "openai", "model2vec"}:
+    if raw in {"hash", "openai", "model2vec", "litellm"}:
         return raw
     # No explicit override: prefer on-device semantic retrieval WHEN the bundled model2vec model is
     # actually present on disk; otherwise degrade to the deterministic hash embedder. CI (no bundled
@@ -166,6 +169,8 @@ def configured_embedding_model() -> str:
     provider = configured_embedding_provider()
     if provider == "openai":
         return os.environ.get("CORTEX_EMBEDDING_MODEL", DEFAULT_OPENAI_EMBEDDING_MODEL).strip() or DEFAULT_OPENAI_EMBEDDING_MODEL
+    if provider == "litellm":
+        return os.environ.get("CORTEX_EMBEDDING_MODEL", DEFAULT_LITELLM_EMBEDDING_MODEL).strip() or DEFAULT_LITELLM_EMBEDDING_MODEL
     if provider == "model2vec":
         return os.environ.get("CORTEX_EMBEDDING_MODEL", DEFAULT_MODEL2VEC_MODEL).strip() or DEFAULT_MODEL2VEC_MODEL
     return VECTOR_MODEL
@@ -208,7 +213,7 @@ def embedding_status(schema_dimensions: int | None = None) -> dict[str, Any]:
         "dimensions": dimensions,
         "schema_dimensions": schema,
         "index_compatible": dimensions == schema,
-        "network_required": provider == "openai",
+        "network_required": provider in ("openai", "litellm"),
         "strict": _strict_embeddings(),
         "degraded": degraded,
         "degraded_reason": "embedding_model_load_failed" if degraded else None,
@@ -230,6 +235,13 @@ def embed_text_result(text: str, dimensions: int = VECTOR_DIMENSIONS) -> Embeddi
             # hash fallback so retrieval never hard-fails offline.
             if _strict_embeddings():
                 raise
+    elif provider == "litellm":
+        try:
+            return _litellm_embedding(text, resolved_dimensions)
+        except Exception:
+            # Same policy as the openai path: fatal in strict mode, otherwise degrade to hash.
+            if _strict_embeddings():
+                raise
     elif provider == "model2vec":
         try:
             result = _model2vec_embedding(text, resolved_dimensions)
@@ -314,6 +326,43 @@ def _openai_embedding(text: str, dimensions: int) -> EmbeddingResult:
     )
 
 
+def _litellm_embedding(text: str, dimensions: int) -> EmbeddingResult:
+    """Embed via the LiteLLM SDK so a provider prefixed model (openai/..., voyage/...,
+    cohere/..., gemini/...) reaches any embedding provider LiteLLM supports.
+
+    Credentials come from that provider's own env var; an optional
+    CORTEX_LITELLM_BASE_URL / CORTEX_LITELLM_API_KEY points at a LiteLLM proxy.
+    Like the OpenAI path, a dimension mismatch raises so the caller can degrade
+    to hash (or fail in strict mode) rather than corrupt the vector index.
+    """
+    import litellm
+
+    model = configured_embedding_model()
+    kwargs: dict[str, Any] = {"model": model, "input": text, "drop_params": True}
+    # text-embedding-3 models honour an explicit output dimension; drop_params
+    # silently ignores it for models that do not support it.
+    if "text-embedding-3" in model:
+        kwargs["dimensions"] = dimensions
+    base_url = os.environ.get("CORTEX_LITELLM_BASE_URL", "").strip()
+    api_key = os.environ.get("CORTEX_LITELLM_API_KEY", "").strip()
+    if base_url:
+        kwargs["api_base"] = base_url
+    if api_key:
+        kwargs["api_key"] = api_key
+
+    response = litellm.embedding(**kwargs)
+    item = response.data[0]
+    vector = item["embedding"] if isinstance(item, dict) else item.embedding
+    if len(vector) != dimensions:
+        raise ValueError(f"LiteLLM embedding dimensions mismatch: expected {dimensions}, got {len(vector)}")
+    return EmbeddingResult(
+        vector=[float(value) for value in vector],
+        model=str(getattr(response, "model", None) or model),
+        provider="litellm",
+        dimensions=dimensions,
+    )
+
+
 def _openai_timeout_seconds() -> float:
     try:
         return max(1.0, float(os.environ.get("CORTEX_EMBEDDING_TIMEOUT_SECONDS", "10")))

backend/app/extractor.py

@@ -301,9 +301,8 @@ def _extract_with_claude_windowed(
 
 
 def _extract_with_claude(raw_text: str, source: str, author_aliases: Iterable[str] | None = None, self_authored: bool = False) -> dict[str, Any]:
-    from anthropic import Anthropic
+    import litellm
 
-    client = Anthropic(api_key=os.environ["ANTHROPIC_API_KEY"])
     aliases = ", ".join(sorted(_identity_alias_tokens(author_aliases))[:12])
     alias_instruction = f"\nUser-authored aliases in speaker-based imports: {aliases}. Only treat preference, style, and negative records as user memory when authored by those aliases or an explicit user/human/me role." if aliases else ""
     prompt = """Extract Cortex memory as strict JSON with keys records, tasks, entities, summary.
@@ -312,13 +311,18 @@ def _extract_with_claude(raw_text: str, source: str, author_aliases: Iterable[st
 entities: list of {id, kind, name, aliases, context}
 Kinds: claim, decision, event, preference, observation, style, negative, procedure. Layers: semantic, episodic, style, decision, preference, negative, procedural. Task kinds: action, question, decision-pending.
 Use stable IDs and keep each memory atomic. Return JSON only.""" + alias_instruction
-    response = client.messages.create(
+    # Routed through LiteLLM: the Anthropic top-level system prompt becomes a
+    # system message, and drop_params lets one config work across providers.
+    response = litellm.completion(
         model=os.environ.get("CORTEX_EXTRACTION_MODEL", "claude-opus-4-5"),
         max_tokens=2500,
-        system=prompt,
-        messages=[{"role": "user", "content": f"Source: {source}\n\n{raw_text[:CLAUDE_EXTRACTION_WINDOW_CHARS]}"}],
+        messages=[
+            {"role": "system", "content": prompt},
+            {"role": "user", "content": f"Source: {source}\n\n{raw_text[:CLAUDE_EXTRACTION_WINDOW_CHARS]}"},
+        ],
+        drop_params=True,
     )
-    text = response.content[0].text.strip()
+    text = (response.choices[0].message.content or "").strip()
     text = re.sub(r"^```(?:json)?\s*", "", text)
     text = re.sub(r"\s*```$", "", text)
     data = json.loads(text)

backend/requirements.txt

@@ -3,6 +3,7 @@ uvicorn>=0.20.0
 pydantic>=2.0.0
 python-dotenv>=1.0.0
 anthropic>=0.40.0
+litellm>=1.89.0,<2.0.0
 sqlite-vec>=0.1.9
 argon2-cffi>=23.1.0
 cryptography>=42.0.0

ingest.py

@@ -19,9 +19,11 @@
 from instrumentation import setup_tracing, get_tracer
 setup_tracing(project_name="cortex")
 
-from anthropic import Anthropic
+import litellm
 
-client = Anthropic(api_key=os.environ.get("ANTHROPIC_API_KEY"))
+# Model routed through LiteLLM (Claude by default). Set CORTEX_MODEL to use any
+# other provider LiteLLM supports; credentials come from that provider's env var.
+DEFAULT_MODEL = os.environ.get("CORTEX_MODEL", "claude-opus-4-5")
 _tracer = get_tracer()
 
 
@@ -89,23 +91,27 @@ def extract_context(raw_text: str, source: str = "unknown") -> dict:
     now = datetime.now().isoformat()
     source_id = make_id("src_", source + now[:16])
 
-    response = client.messages.create(
-        model="claude-opus-4-5",
+    response = litellm.completion(
+        model=DEFAULT_MODEL,
         max_tokens=3000,
-        system=EXTRACTION_SYSTEM_PROMPT,
-        messages=[{
-            "role": "user",
-            "content": (
-                f"Source: {source}\n"
-                f"Source ID: {source_id}\n"
-                f"Captured: {now}\n\n"
-                f"---\n\n{truncated}\n\n---\n\n"
-                f"Extract context as JSON:"
-            )
-        }]
+        messages=[
+            {"role": "system", "content": EXTRACTION_SYSTEM_PROMPT},
+            {
+                "role": "user",
+                "content": (
+                    f"Source: {source}\n"
+                    f"Source ID: {source_id}\n"
+                    f"Captured: {now}\n\n"
+                    f"---\n\n{truncated}\n\n---\n\n"
+                    f"Extract context as JSON:"
+                ),
+            },
+        ],
+        # Drop provider-unsupported params so one config works across providers.
+        drop_params=True,
     )
 
-    raw_output = response.content[0].text.strip()
+    raw_output = (response.choices[0].message.content or "").strip()
     extracted = _parse_json_response(raw_output)
 
     # Ensure required fields

requirements.txt

@@ -1,4 +1,5 @@
 anthropic>=0.40.0
+litellm>=1.89.0,<2.0.0
 mcp>=1.0.0
 redis[hiredis]>=5.0.0
 voyageai>=0.2.0

ui.py

@@ -16,10 +16,14 @@
 from instrumentation import setup_tracing
 setup_tracing(project_name="cortex")
 
-from anthropic import Anthropic
+import litellm
 from redis_store import search_context, get_recent_context
 
-client = Anthropic(api_key=os.environ.get("ANTHROPIC_API_KEY"))
+# Model routed through LiteLLM: keep Claude as the default, but any provider
+# LiteLLM supports works by setting CORTEX_MODEL (e.g. "openai/gpt-4o",
+# "gemini/gemini-2.5-pro", "bedrock/..."). Credentials come from that
+# provider's own env var (ANTHROPIC_API_KEY by default).
+DEFAULT_MODEL = os.environ.get("CORTEX_MODEL", "claude-opus-4-5")
 
 # ── Page config ───────────────────────────────────────────────────────────────
 
@@ -119,24 +123,29 @@ def ask_cortex(question: str, context_chunks: list[dict]) -> str:
         for c in context_chunks
     ])
 
-    response = client.messages.create(
-        model="claude-opus-4-5",
-        max_tokens=1000,
-        system="""You are Cortex — a personal AI that knows everything about the user based on their captured context.
+    system_prompt = """You are Cortex — a personal AI that knows everything about the user based on their captured context.
 
 You have access to the user's second brain: notes, decisions, insights, and memories captured from their AI chats, Slack, iMessage, and other apps.
 
 Answer questions directly and personally, as if you are their most knowledgeable assistant.
 - Reference specific details from the context (dates, sources, exact decisions)
 - Be concise but complete
 - If the context is partial, say so and answer with what you have
-- Never say "based on the provided context" — just answer naturally""",
-        messages=[{
-            "role": "user",
-            "content": f"Context from my second brain:\n\n{context_text}\n\n---\n\nQuestion: {question}"
-        }]
+- Never say "based on the provided context" — just answer naturally"""
+    response = litellm.completion(
+        model=DEFAULT_MODEL,
+        max_tokens=1000,
+        messages=[
+            {"role": "system", "content": system_prompt},
+            {
+                "role": "user",
+                "content": f"Context from my second brain:\n\n{context_text}\n\n---\n\nQuestion: {question}",
+            },
+        ],
+        # Drop provider-unsupported params so one config works across providers.
+        drop_params=True,
     )
-    return response.content[0].text
+    return response.choices[0].message.content or ""
 
 
 # ── Sidebar ───────────────────────────────────────────────────────────────────