From eac0fc8087ee98cb9b01ababdce1630343fbab96 Mon Sep 17 00:00:00 2001 From: Prodman Devokadev Date: Mon, 24 Aug 2026 10:31:13 +0530 Subject: [PATCH 1/3] feat: route LLM calls through LiteLLM for multi-provider support --- backend/app/condense.py | 15 +++++++++------ backend/requirements.txt | 1 + ingest.py | 38 ++++++++++++++++++++++---------------- requirements.txt | 1 + ui.py | 33 +++++++++++++++++++++------------ 5 files changed, 54 insertions(+), 34 deletions(-) diff --git a/backend/app/condense.py b/backend/app/condense.py index d78c5776..108c224e 100644 --- a/backend/app/condense.py +++ b/backend/app/condense.py @@ -184,12 +184,11 @@ def _condense_with_claude(section: dict, max_chars: int) -> Optional[dict]: return None try: - from anthropic import Anthropic + import litellm allowed_ids = set(_union_memory_ids(elements)) by_id = _elements_by_id(elements) - client = Anthropic(api_key=os.environ["ANTHROPIC_API_KEY"]) model = os.environ.get( "CORTEX_CONDENSE_MODEL", os.environ.get("CORTEX_EXTRACTION_MODEL", "claude-opus-4-5"), @@ -203,14 +202,18 @@ def _condense_with_claude(section: dict, max_chars: int) -> Optional[dict]: ) user = _build_user_prompt(section, elements) - response = client.messages.create( + response = litellm.completion( model=model, max_tokens=300, - system=system, - messages=[{"role": "user", "content": user}], + messages=[ + {"role": "system", "content": system}, + {"role": "user", "content": user}, + ], + # Drop provider-unsupported params so one config works across providers. + drop_params=True, ) - text = response.content[0].text.strip() + text = (response.choices[0].message.content or "").strip() text = re.sub(r"^```(?:json)?\s*", "", text) text = re.sub(r"\s*```$", "", text) data = json.loads(text) diff --git a/backend/requirements.txt b/backend/requirements.txt index d96f93db..76a717b7 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -3,6 +3,7 @@ uvicorn>=0.20.0 pydantic>=2.0.0 python-dotenv>=1.0.0 anthropic>=0.40.0 +litellm>=1.89.0,<2.0.0 sqlite-vec>=0.1.9 argon2-cffi>=23.1.0 cryptography>=42.0.0 diff --git a/ingest.py b/ingest.py index 744dbf47..a018c485 100644 --- a/ingest.py +++ b/ingest.py @@ -19,9 +19,11 @@ from instrumentation import setup_tracing, get_tracer setup_tracing(project_name="cortex") -from anthropic import Anthropic +import litellm -client = Anthropic(api_key=os.environ.get("ANTHROPIC_API_KEY")) +# Model routed through LiteLLM (Claude by default). Set CORTEX_MODEL to use any +# other provider LiteLLM supports; credentials come from that provider's env var. +DEFAULT_MODEL = os.environ.get("CORTEX_MODEL", "claude-opus-4-5") _tracer = get_tracer() @@ -89,23 +91,27 @@ def extract_context(raw_text: str, source: str = "unknown") -> dict: now = datetime.now().isoformat() source_id = make_id("src_", source + now[:16]) - response = client.messages.create( - model="claude-opus-4-5", + response = litellm.completion( + model=DEFAULT_MODEL, max_tokens=3000, - system=EXTRACTION_SYSTEM_PROMPT, - messages=[{ - "role": "user", - "content": ( - f"Source: {source}\n" - f"Source ID: {source_id}\n" - f"Captured: {now}\n\n" - f"---\n\n{truncated}\n\n---\n\n" - f"Extract context as JSON:" - ) - }] + messages=[ + {"role": "system", "content": EXTRACTION_SYSTEM_PROMPT}, + { + "role": "user", + "content": ( + f"Source: {source}\n" + f"Source ID: {source_id}\n" + f"Captured: {now}\n\n" + f"---\n\n{truncated}\n\n---\n\n" + f"Extract context as JSON:" + ), + }, + ], + # Drop provider-unsupported params so one config works across providers. + drop_params=True, ) - raw_output = response.content[0].text.strip() + raw_output = (response.choices[0].message.content or "").strip() extracted = _parse_json_response(raw_output) # Ensure required fields diff --git a/requirements.txt b/requirements.txt index add22c92..9bdbe67b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ anthropic>=0.40.0 +litellm>=1.89.0,<2.0.0 mcp>=1.0.0 redis[hiredis]>=5.0.0 voyageai>=0.2.0 diff --git a/ui.py b/ui.py index a8b501ac..bbcc0617 100644 --- a/ui.py +++ b/ui.py @@ -16,10 +16,14 @@ from instrumentation import setup_tracing setup_tracing(project_name="cortex") -from anthropic import Anthropic +import litellm from redis_store import search_context, get_recent_context -client = Anthropic(api_key=os.environ.get("ANTHROPIC_API_KEY")) +# Model routed through LiteLLM: keep Claude as the default, but any provider +# LiteLLM supports works by setting CORTEX_MODEL (e.g. "openai/gpt-4o", +# "gemini/gemini-2.5-pro", "bedrock/..."). Credentials come from that +# provider's own env var (ANTHROPIC_API_KEY by default). +DEFAULT_MODEL = os.environ.get("CORTEX_MODEL", "claude-opus-4-5") # ── Page config ─────────────────────────────────────────────────────────────── @@ -119,10 +123,7 @@ def ask_cortex(question: str, context_chunks: list[dict]) -> str: for c in context_chunks ]) - response = client.messages.create( - model="claude-opus-4-5", - max_tokens=1000, - system="""You are Cortex — a personal AI that knows everything about the user based on their captured context. + system_prompt = """You are Cortex — a personal AI that knows everything about the user based on their captured context. You have access to the user's second brain: notes, decisions, insights, and memories captured from their AI chats, Slack, iMessage, and other apps. @@ -130,13 +131,21 @@ def ask_cortex(question: str, context_chunks: list[dict]) -> str: - Reference specific details from the context (dates, sources, exact decisions) - Be concise but complete - If the context is partial, say so and answer with what you have -- Never say "based on the provided context" — just answer naturally""", - messages=[{ - "role": "user", - "content": f"Context from my second brain:\n\n{context_text}\n\n---\n\nQuestion: {question}" - }] +- Never say "based on the provided context" — just answer naturally""" + response = litellm.completion( + model=DEFAULT_MODEL, + max_tokens=1000, + messages=[ + {"role": "system", "content": system_prompt}, + { + "role": "user", + "content": f"Context from my second brain:\n\n{context_text}\n\n---\n\nQuestion: {question}", + }, + ], + # Drop provider-unsupported params so one config works across providers. + drop_params=True, ) - return response.content[0].text + return response.choices[0].message.content or "" # ── Sidebar ─────────────────────────────────────────────────────────────────── From ec6e2e182613a86986b638be0df20c8c24aa602a Mon Sep 17 00:00:00 2001 From: Prodman Devokadev Date: Mon, 24 Aug 2026 23:34:21 +0530 Subject: [PATCH 2/3] feat: route extractor and embeddings through LiteLLM too --- 2026-01-01 | 0 500 | 2 ++ backend/app/embeddings.py | 53 +++++++++++++++++++++++++++++++++++++-- backend/app/extractor.py | 16 +++++++----- 4 files changed, 63 insertions(+), 8 deletions(-) create mode 100644 2026-01-01 create mode 100644 500 diff --git a/2026-01-01 b/2026-01-01 new file mode 100644 index 00000000..e69de29b diff --git a/500 b/500 new file mode 100644 index 00000000..49259bdc --- /dev/null +++ b/500 @@ -0,0 +1,2 @@ +Invalid search query "\"cli tool python\" stars: language:Python". +"" is not a numeric value - please provide an integer value diff --git a/backend/app/embeddings.py b/backend/app/embeddings.py index 5d1cd135..c69dd854 100644 --- a/backend/app/embeddings.py +++ b/backend/app/embeddings.py @@ -17,6 +17,9 @@ VECTOR_DIMENSIONS = 384 VECTOR_MODEL = "cortex-hash-v1" DEFAULT_OPENAI_EMBEDDING_MODEL = "text-embedding-3-small" +# LiteLLM embeddings: a provider prefixed model reaches Voyage, Cohere, Gemini, +# Bedrock, and more through one interface. Override with CORTEX_EMBEDDING_MODEL. +DEFAULT_LITELLM_EMBEDDING_MODEL = "openai/text-embedding-3-small" # Local, CPU-only, no-API-key neural embeddings (MinishLab model2vec, MIT). # potion-base-8M emits 256-dim vectors natively (NOT 384) — so it is not @@ -153,7 +156,7 @@ def model2vec_available() -> bool: def configured_embedding_provider() -> str: raw = os.environ.get("CORTEX_EMBEDDING_PROVIDER", "").strip().lower() - if raw in {"hash", "openai", "model2vec"}: + if raw in {"hash", "openai", "model2vec", "litellm"}: return raw # No explicit override: prefer on-device semantic retrieval WHEN the bundled model2vec model is # actually present on disk; otherwise degrade to the deterministic hash embedder. CI (no bundled @@ -166,6 +169,8 @@ def configured_embedding_model() -> str: provider = configured_embedding_provider() if provider == "openai": return os.environ.get("CORTEX_EMBEDDING_MODEL", DEFAULT_OPENAI_EMBEDDING_MODEL).strip() or DEFAULT_OPENAI_EMBEDDING_MODEL + if provider == "litellm": + return os.environ.get("CORTEX_EMBEDDING_MODEL", DEFAULT_LITELLM_EMBEDDING_MODEL).strip() or DEFAULT_LITELLM_EMBEDDING_MODEL if provider == "model2vec": return os.environ.get("CORTEX_EMBEDDING_MODEL", DEFAULT_MODEL2VEC_MODEL).strip() or DEFAULT_MODEL2VEC_MODEL return VECTOR_MODEL @@ -208,7 +213,7 @@ def embedding_status(schema_dimensions: int | None = None) -> dict[str, Any]: "dimensions": dimensions, "schema_dimensions": schema, "index_compatible": dimensions == schema, - "network_required": provider == "openai", + "network_required": provider in ("openai", "litellm"), "strict": _strict_embeddings(), "degraded": degraded, "degraded_reason": "embedding_model_load_failed" if degraded else None, @@ -230,6 +235,13 @@ def embed_text_result(text: str, dimensions: int = VECTOR_DIMENSIONS) -> Embeddi # hash fallback so retrieval never hard-fails offline. if _strict_embeddings(): raise + elif provider == "litellm": + try: + return _litellm_embedding(text, resolved_dimensions) + except Exception: + # Same policy as the openai path: fatal in strict mode, otherwise degrade to hash. + if _strict_embeddings(): + raise elif provider == "model2vec": try: result = _model2vec_embedding(text, resolved_dimensions) @@ -314,6 +326,43 @@ def _openai_embedding(text: str, dimensions: int) -> EmbeddingResult: ) +def _litellm_embedding(text: str, dimensions: int) -> EmbeddingResult: + """Embed via the LiteLLM SDK so a provider prefixed model (openai/..., voyage/..., + cohere/..., gemini/...) reaches any embedding provider LiteLLM supports. + + Credentials come from that provider's own env var; an optional + CORTEX_LITELLM_BASE_URL / CORTEX_LITELLM_API_KEY points at a LiteLLM proxy. + Like the OpenAI path, a dimension mismatch raises so the caller can degrade + to hash (or fail in strict mode) rather than corrupt the vector index. + """ + import litellm + + model = configured_embedding_model() + kwargs: dict[str, Any] = {"model": model, "input": text, "drop_params": True} + # text-embedding-3 models honour an explicit output dimension; drop_params + # silently ignores it for models that do not support it. + if "text-embedding-3" in model: + kwargs["dimensions"] = dimensions + base_url = os.environ.get("CORTEX_LITELLM_BASE_URL", "").strip() + api_key = os.environ.get("CORTEX_LITELLM_API_KEY", "").strip() + if base_url: + kwargs["api_base"] = base_url + if api_key: + kwargs["api_key"] = api_key + + response = litellm.embedding(**kwargs) + item = response.data[0] + vector = item["embedding"] if isinstance(item, dict) else item.embedding + if len(vector) != dimensions: + raise ValueError(f"LiteLLM embedding dimensions mismatch: expected {dimensions}, got {len(vector)}") + return EmbeddingResult( + vector=[float(value) for value in vector], + model=str(getattr(response, "model", None) or model), + provider="litellm", + dimensions=dimensions, + ) + + def _openai_timeout_seconds() -> float: try: return max(1.0, float(os.environ.get("CORTEX_EMBEDDING_TIMEOUT_SECONDS", "10"))) diff --git a/backend/app/extractor.py b/backend/app/extractor.py index 69a32107..81937db3 100644 --- a/backend/app/extractor.py +++ b/backend/app/extractor.py @@ -301,9 +301,8 @@ def _extract_with_claude_windowed( def _extract_with_claude(raw_text: str, source: str, author_aliases: Iterable[str] | None = None, self_authored: bool = False) -> dict[str, Any]: - from anthropic import Anthropic + import litellm - client = Anthropic(api_key=os.environ["ANTHROPIC_API_KEY"]) aliases = ", ".join(sorted(_identity_alias_tokens(author_aliases))[:12]) alias_instruction = f"\nUser-authored aliases in speaker-based imports: {aliases}. Only treat preference, style, and negative records as user memory when authored by those aliases or an explicit user/human/me role." if aliases else "" prompt = """Extract Cortex memory as strict JSON with keys records, tasks, entities, summary. @@ -312,13 +311,18 @@ def _extract_with_claude(raw_text: str, source: str, author_aliases: Iterable[st entities: list of {id, kind, name, aliases, context} Kinds: claim, decision, event, preference, observation, style, negative, procedure. Layers: semantic, episodic, style, decision, preference, negative, procedural. Task kinds: action, question, decision-pending. Use stable IDs and keep each memory atomic. Return JSON only.""" + alias_instruction - response = client.messages.create( + # Routed through LiteLLM: the Anthropic top-level system prompt becomes a + # system message, and drop_params lets one config work across providers. + response = litellm.completion( model=os.environ.get("CORTEX_EXTRACTION_MODEL", "claude-opus-4-5"), max_tokens=2500, - system=prompt, - messages=[{"role": "user", "content": f"Source: {source}\n\n{raw_text[:CLAUDE_EXTRACTION_WINDOW_CHARS]}"}], + messages=[ + {"role": "system", "content": prompt}, + {"role": "user", "content": f"Source: {source}\n\n{raw_text[:CLAUDE_EXTRACTION_WINDOW_CHARS]}"}, + ], + drop_params=True, ) - text = response.content[0].text.strip() + text = (response.choices[0].message.content or "").strip() text = re.sub(r"^```(?:json)?\s*", "", text) text = re.sub(r"\s*```$", "", text) data = json.loads(text) From c3bc8c4e7a0f78957934c4599e862737a94d543f Mon Sep 17 00:00:00 2001 From: Prodman Devokadev Date: Mon, 24 Aug 2026 23:34:56 +0530 Subject: [PATCH 3/3] chore: remove stray files accidentally added --- 2026-01-01 | 0 500 | 2 -- 2 files changed, 2 deletions(-) delete mode 100644 2026-01-01 delete mode 100644 500 diff --git a/2026-01-01 b/2026-01-01 deleted file mode 100644 index e69de29b..00000000 diff --git a/500 b/500 deleted file mode 100644 index 49259bdc..00000000 --- a/500 +++ /dev/null @@ -1,2 +0,0 @@ -Invalid search query "\"cli tool python\" stars: language:Python". -"" is not a numeric value - please provide an integer value