diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..66c6bf20 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,27 @@ +.git +.github +.venv +venv +.run +.env +.env.* +!.env.example +backups +backend/data +models +macos/build +outputs +release-artifacts +site +docs +extension +packages +sdk +status +**/__pycache__ +**/*.pyc +*.sqlite +*.sqlite-wal +*.sqlite-shm +.DS_Store +.pytest_cache diff --git a/.env.example b/.env.example index 62520363..c859d50b 100644 --- a/.env.example +++ b/.env.example @@ -1,45 +1,121 @@ -# Cortex local backend -# Copy to .env only when you run the backend outside the packaged macOS app. +# Cortex backend configuration (single-user, local-first profile). +# +# Copy to .env (scripts/setup_local.sh does this and fills in a random token). +# The backend reads ONLY real environment variables; scripts/start_backend.sh, the launchd +# service and docker-compose export this file for you. NEVER commit .env. +# Full reference: docs/CONFIGURATION.md. Multi-tenant server profile: deploy/cortex.env.example. -# Local API auth. Use a long random token. The packaged app generates one. +# ------------------------------------------------------------------------------------------ +# Required +# ------------------------------------------------------------------------------------------ +# Bearer token for every REST call. Use a long random value (the setup script generates one; +# the packaged macOS app generates its own per install). CORTEX_API_KEY=replace-with-a-long-random-token -# Only set this when intentionally running local development scripts with the -# sample dev-local-key token. +# Only set to 1 when intentionally running development scripts with the sample +# "dev-local-key" token. The backend refuses to start with that token otherwise. CORTEX_ALLOW_INSECURE_DEV_TOKEN=0 -# Optional MCP auth for local AI-tool integrations. REST endpoints still require -# CORTEX_API_KEY; /mcp accepts either the app token or this scoped MCP token. -CORTEX_MCP_API_KEY= -CORTEX_MCP_API_KEY_SCOPES=read,write,export,maintenance - -# Optional local sync manifest signing. When set, device-bound -# /v1/sync/changes responses include an HMAC signature for integrity checks. -CORTEX_SYNC_SIGNING_KEY= +# ------------------------------------------------------------------------------------------ +# Storage (user-owned vault + rebuildable SQLite index) +# ------------------------------------------------------------------------------------------ +# To share one memory store with the packaged macOS app, point both at the app's default: +# CORTEX_VAULT_PATH=~/Library/Application Support/Cortex/Cortex.vault +CORTEX_VAULT_PATH=./backend/data/Cortex.vault +CORTEX_DB_PATH=./backend/data/Cortex.vault/index.sqlite -# Local backup retention. Count pruning is enabled by default; age pruning is -# opt-in because users may depend on older local recovery archives. +# Local backup retention (vault/backups/*.zip created by POST /v1/backups or the app). +# Count pruning is on by default; age pruning (days) is opt-in. CORTEX_BACKUP_RETENTION_COUNT=20 CORTEX_BACKUP_RETENTION_DAYS=0 -# Local vault and rebuildable SQLite index. -CORTEX_VAULT_PATH=./backend/data/Cortex.vault -CORTEX_DB_PATH=./backend/data/Cortex.vault/index.sqlite - -# Dev server binding. Conductor workspaces can override CORTEX_PORT. +# ------------------------------------------------------------------------------------------ +# Network +# ------------------------------------------------------------------------------------------ +# Keep 127.0.0.1 unless a TLS reverse proxy sits in front (see docs/SECURITY.md). CORTEX_HOST=127.0.0.1 CORTEX_PORT=8766 CORTEX_PUBLIC_BASE_URL=http://127.0.0.1:8766 CORTEX_CORS_ORIGINS=http://127.0.0.1:8766,http://localhost:8766 +# The browser extension needs CORS from extension origins; set 0 if you do not use it. +CORTEX_ALLOW_EXTENSION_CORS=1 +# uvicorn worker processes (scripts/start_backend.sh --mode fastapi). Keep 1 for a +# single-user SQLite vault; the in-process rate limiter is per worker. +CORTEX_UVICORN_WORKERS=1 +# Server flavour used by scripts/start_backend.sh and deploy/launchd: fastapi | standalone +CORTEX_SERVER_MODE=fastapi -# Optional extraction provider. If unset, Cortex uses deterministic local extraction. -ANTHROPIC_API_KEY= -CORTEX_EXTRACTION_MODEL=claude-opus-4-5 +# ------------------------------------------------------------------------------------------ +# AI-tool (MCP) access +# ------------------------------------------------------------------------------------------ +# Optional separate, scoped token for /mcp so MCP configs never carry the app token. +# REST endpoints still require CORTEX_API_KEY; /mcp accepts either. +CORTEX_MCP_API_KEY= +CORTEX_MCP_API_KEY_SCOPES=read,write,export,maintenance +# Which MCP tool list is advertised by default: core (curated) | full | chatgpt +CORTEX_MCP_TOOL_SURFACE=core +# Streamable-HTTP compatibility for /mcp (SSE + Mcp-Session-Id). 0 forces plain JSON. +CORTEX_MCP_STREAMABLE=1 -# Optional embedding provider. The default stays offline/local with a -# deterministic hash embedding; OpenAI can be enabled for higher recall. +# ------------------------------------------------------------------------------------------ +# Retrieval quality (all local; no network) +# ------------------------------------------------------------------------------------------ +# hash = deterministic offline embeddings (default; works with nothing installed) +# model2vec = real on-device semantic embeddings (run scripts/setup_local.sh --with-model) +# openai = remote OpenAI embeddings (needs OPENAI_API_KEY; data leaves the machine) CORTEX_EMBEDDING_PROVIDER=hash +CORTEX_MODEL2VEC_PATH= +CORTEX_MODEL2VEC_MODEL=minishlab/potion-base-8M +# Used only by the openai provider. Keep 384 for the hash provider's sqlite-vec schema. CORTEX_EMBEDDING_MODEL=text-embedding-3-small CORTEX_EMBEDDING_DIMENSIONS=384 CORTEX_EMBEDDING_STRICT=0 OPENAI_API_KEY= +# Reranker + query planning + temporal decay (same flags the shipped app uses). +CORTEX_RERANK=linear+mmr +CORTEX_QUERY_PLAN=1 +CORTEX_TEMPORAL_DECAY=1 + +# ------------------------------------------------------------------------------------------ +# Extraction (optional LLM condenser) +# ------------------------------------------------------------------------------------------ +# Unset = deterministic local extractor (capture/search work fully offline). +ANTHROPIC_API_KEY= +CORTEX_EXTRACTION_MODEL=claude-opus-4-5 + +# ------------------------------------------------------------------------------------------ +# Behaviour +# ------------------------------------------------------------------------------------------ +# 1 = new captures skip the Review inbox and are retrievable immediately. +CORTEX_AUTO_APPROVE_CAPTURES=0 +# Brand used in user-visible backend strings: Cortex | Doppl +CORTEX_APP_BRAND=Cortex +# Background job worker: inline (default; jobs run inside the API process) | external +# (run scripts/run_memory_worker.py separately; used by the multi-tenant profile). +CORTEX_WORKER_MODE=inline +# Operator metrics at GET /v1/metrics (admin token). 0 = off. +CORTEX_OBSERVABILITY_ENABLED=0 + +# ------------------------------------------------------------------------------------------ +# Fair-use limits (0 = disabled; the multi-tenant profile sets non-zero values) +# ------------------------------------------------------------------------------------------ +CORTEX_RATE_LIMIT_PER_MINUTE=0 +CORTEX_DEFAULT_MEMORY_QUOTA=0 +CORTEX_MAX_CONCURRENT_REQUESTS=8 +CORTEX_RATE_LIMIT_RPS=20 +CORTEX_RATE_LIMIT_BURST=60 + +# ------------------------------------------------------------------------------------------ +# Sync integrity (optional) +# ------------------------------------------------------------------------------------------ +# When set, device-bound /v1/sync/changes responses carry an HMAC signature. +CORTEX_SYNC_SIGNING_KEY= + +# ------------------------------------------------------------------------------------------ +# Multi-tenant / hosted server profile (leave unset for single-user local mode) +# ------------------------------------------------------------------------------------------ +# See deploy/cortex.env.example for the full server profile: CORTEX_SHARD_MODE=bucket, +# CORTEX_SHARD_ROOT, CORTEX_REQUIRE_SCOPED_API_TOKENS=1, CORTEX_AUTH_ENABLED=1, CORTEX_KEK_FILE, +# CORTEX_SMTP_*, CORTEX_OIDC_*_CLIENT_ID/SECRET, CORTEX_BILLING_PROVIDER, CORTEX_TURNSTILE_*. +# Docker proxy profile only: +# CORTEX_DOMAIN=cortex.example.internal diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7fab48f0..e5cc0810 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -156,7 +156,9 @@ jobs: run: python scripts/check_distribution_site.py - name: Validate site update manifest - run: python scripts/validate_update_manifest.py site/downloads/latest.json + # The site feed points at binaries on GitHub Releases (only latest.json + checksums are in git), + # so validate structure + https URLs + checksum-file agreement; release packaging stays strict. + run: python scripts/validate_update_manifest.py --allow-remote-artifacts site/downloads/latest.json obsidian-plugin: name: Obsidian plugin build diff --git a/.gitignore b/.gitignore index 76fe7f3a..96efce5e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,12 +1,39 @@ +# secrets and local configuration .env -.context/ +.env.* +!.env.example +*.pem +*.p12 +*.key + +# python +.venv/ venv/ __pycache__/ *.pyc -.DS_Store -node_modules/ -npm-debug.log* +.pytest_cache/ +.mypy_cache/ +.ruff_cache/ + +# local runtime state, data, backups +.run/ +.context/ backend/data/ +backups/ +models/ +*.sqlite +*.sqlite-wal +*.sqlite-shm +*.sqlite-journal + +# build outputs macos/build/ outputs/ cortex-agent-package.zip +node_modules/ +npm-debug.log* +dist/ +dist-test/ + +# os +.DS_Store diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..fdbda45d --- /dev/null +++ b/Dockerfile @@ -0,0 +1,47 @@ +# Cortex backend (FastAPI engine) for Linux hosts. The macOS app is native and is NOT +# containerised; this image serves the HTTP/MCP API that the app, SDKs, browser extension +# and MCP clients talk to. +# +# docker build -t cortex-backend . +# docker run --rm -p 127.0.0.1:8766:8766 --env-file .env -v cortex-data:/data cortex-backend +# +# Design notes +# - Pinned dependencies from backend/requirements.lock.txt (same set as the local venv). +# - pysqlite3 is compiled against the distro's libsqlite3 so FTS5 + sqlite-vec extension +# loading work regardless of the base image's bundled Python sqlite build. +# - Runs as a non-root user; data lives in /data (mount a volume); loopback-only publish is +# the responsibility of the compose file / host (see docker-compose.yml). +FROM python:3.12-slim-bookworm AS build + +ENV PIP_NO_CACHE_DIR=1 PIP_DISABLE_PIP_VERSION_CHECK=1 PYTHONDONTWRITEBYTECODE=1 +RUN apt-get update \ + && apt-get install -y --no-install-recommends build-essential libsqlite3-dev \ + && rm -rf /var/lib/apt/lists/* +COPY backend/requirements.lock.txt /tmp/requirements.lock.txt +RUN python -m venv /opt/venv \ + && /opt/venv/bin/pip install -r /tmp/requirements.lock.txt + +FROM python:3.12-slim-bookworm +ENV PATH=/opt/venv/bin:$PATH \ + PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + CORTEX_HOST=0.0.0.0 \ + CORTEX_PORT=8766 \ + CORTEX_VAULT_PATH=/data/Cortex.vault \ + CORTEX_DB_PATH=/data/Cortex.vault/index.sqlite +RUN apt-get update \ + && apt-get install -y --no-install-recommends curl libsqlite3-0 sqlite3 \ + && rm -rf /var/lib/apt/lists/* \ + && groupadd --system cortex && useradd --system --gid cortex --home /srv/cortex --shell /usr/sbin/nologin cortex \ + && mkdir -p /srv/cortex /data && chown -R cortex:cortex /srv/cortex /data +COPY --from=build /opt/venv /opt/venv +WORKDIR /srv/cortex +COPY --chown=cortex:cortex backend ./backend +COPY --chown=cortex:cortex scripts ./scripts +COPY --chown=cortex:cortex spec ./spec +USER cortex +VOLUME ["/data"] +EXPOSE 8766 +HEALTHCHECK --interval=30s --timeout=5s --start-period=20s --retries=3 \ + CMD curl -fsS "http://127.0.0.1:${CORTEX_PORT}/health" >/dev/null || exit 1 +CMD ["sh", "-c", "exec uvicorn backend.app.main:app --host \"$CORTEX_HOST\" --port \"$CORTEX_PORT\" --workers \"${CORTEX_UVICORN_WORKERS:-1}\" --proxy-headers --forwarded-allow-ips=\"${CORTEX_FORWARDED_ALLOW_IPS:-*}\""] diff --git a/README.md b/README.md index 8434fc26..be519756 100644 --- a/README.md +++ b/README.md @@ -108,6 +108,22 @@ the smallest cited, model-calibrated context that answers the task. Cortex checks for updates on its own, so once you're on a recent build, new releases arrive automatically. +## Run it in-house (your own machine or server) + +Everything runs on hardware you control with no account or cloud service: + +```bash +scripts/setup_local.sh --with-model # Python 3.12 venv, pinned deps, generated token, offline embeddings +scripts/start_backend.sh # backend on 127.0.0.1:8766 +scripts/health_check.sh # health, readiness, auth +deploy/launchd/install.sh # macOS: always-on service that restarts on crash +``` + +Linux hosts: `docker compose up -d --build`. Details, upgrades, backups and hardening: +[`docs/INSTALLATION.md`](docs/INSTALLATION.md), [`docs/CONFIGURATION.md`](docs/CONFIGURATION.md), +[`docs/OPERATIONS.md`](docs/OPERATIONS.md), [`docs/SECURITY.md`](docs/SECURITY.md), +[`docs/DEPENDENCIES.md`](docs/DEPENDENCIES.md). + ## Build from source ```bash diff --git a/SETUP.md b/SETUP.md index 751cc8ca..d4fd0282 100644 --- a/SETUP.md +++ b/SETUP.md @@ -66,3 +66,9 @@ python3 scripts/adaptation_eval.py codesign --verify --deep --strict --verbose=2 macos/build/Cortex.app python3 scripts/ops_readiness_check.py ``` + +## Run The Backend As An In-House Service + +`scripts/setup_local.sh`, `scripts/start_backend.sh`, `scripts/health_check.sh` and +`deploy/launchd/install.sh` run the backend on this machine without the packaged app. See +`docs/INSTALLATION.md` and `docs/OPERATIONS.md`. diff --git a/backend/app/connectors/agent_sessions.py b/backend/app/connectors/agent_sessions.py index 09d9fb11..8e2ca4aa 100644 --- a/backend/app/connectors/agent_sessions.py +++ b/backend/app/connectors/agent_sessions.py @@ -254,7 +254,7 @@ def _claude_records(root: Path, path: Path, *, session_cap: int) -> list[AgentSe cleaned = _clean_user_text(text) if not cleaned: continue - fingerprint = sha1(cleaned.encode("utf-8")).hexdigest() + fingerprint = sha1(cleaned.encode("utf-8"), usedforsecurity=False).hexdigest() if fingerprint in seen_texts: continue seen_texts.add(fingerprint) @@ -316,7 +316,7 @@ def _codex_records(root: Path, path: Path, *, session_cap: int) -> list[AgentSes cleaned = _clean_user_text(str(payload.get("message") or "")) if not cleaned: continue - fingerprint = sha1(cleaned.encode("utf-8")).hexdigest() + fingerprint = sha1(cleaned.encode("utf-8"), usedforsecurity=False).hexdigest() if fingerprint in seen_texts: continue seen_texts.add(fingerprint) @@ -370,7 +370,7 @@ def _cursor_records(root: Path, path: Path, *, mtime: float, session_cap: int) - cleaned = _clean_user_text(str(prompt.get("text") or "")) if not cleaned: continue - fingerprint = sha1(cleaned.encode("utf-8")).hexdigest() + fingerprint = sha1(cleaned.encode("utf-8"), usedforsecurity=False).hexdigest() if fingerprint in seen_texts: continue seen_texts.add(fingerprint) @@ -465,7 +465,7 @@ def _stable_external_id(agent: str, session_id: str, anchor: str) -> str: raw = f"{agent}:{session_id}:{anchor}" if len(raw) <= 240: return raw - digest = sha1(raw.encode("utf-8")).hexdigest()[:16] + digest = sha1(raw.encode("utf-8"), usedforsecurity=False).hexdigest()[:16] return f"{raw[:220]}#{digest}" diff --git a/backend/app/obsidian_writeback.py b/backend/app/obsidian_writeback.py index 3fc1ff18..5d02d075 100644 --- a/backend/app/obsidian_writeback.py +++ b/backend/app/obsidian_writeback.py @@ -71,7 +71,7 @@ def safe_page_stem(name: str, fallback: str = "page") -> str: def person_page_filename(entity_id: str, name: str) -> str: """``--.md`` — slug for human browsing, hash so the page survives label edits/collisions (same scheme as the native vault's entity MOC pages).""" - short = hashlib.sha1(str(entity_id or "").encode("utf-8")).hexdigest()[:12] + short = hashlib.sha1(str(entity_id or "").encode("utf-8"), usedforsecurity=False).hexdigest()[:12] return f"{safe_page_stem(name, 'person')}--{short}.md" diff --git a/backend/app/storage.py b/backend/app/storage.py index 93d1bb1e..0bed64ee 100644 --- a/backend/app/storage.py +++ b/backend/app/storage.py @@ -14898,7 +14898,7 @@ def _fnv(text: str) -> int: continue seen_edges.add(key) canvas_edges.append({ - "id": hashlib.sha1(("|".join(key)).encode("utf-8")).hexdigest()[:16], + "id": hashlib.sha1(("|".join(key)).encode("utf-8"), usedforsecurity=False).hexdigest()[:16], "fromNode": self.vault.entity_moc_short_id(key[0]), "toNode": self.vault.entity_moc_short_id(key[1]), }) diff --git a/backend/app/vault.py b/backend/app/vault.py index c2ec98c3..ac43020c 100644 --- a/backend/app/vault.py +++ b/backend/app/vault.py @@ -671,7 +671,7 @@ def memory_note_short_id(self, memory_id: str) -> str: summary/content) so it is invariant across edits — that is what makes glob-by-suffix reliable. Sweeps that mutate/delete also confirm the parsed frontmatter id, so even an (astronomically unlikely) collision can at worst rewrite the same logical note.""" - return hashlib.sha1(str(memory_id or "").encode("utf-8")).hexdigest()[:12] + return hashlib.sha1(str(memory_id or "").encode("utf-8"), usedforsecurity=False).hexdigest()[:12] def memory_note_stem(self, record: dict[str, Any]) -> str: """`--` — the SINGLE source of both the on-disk note filename and the @@ -812,7 +812,7 @@ def entity_moc_short_id(self, entity_id: str) -> str: disambiguator and by the stale-page sweep. 16 hex = 64 bits: birthday-collision-safe well past any realistic entity count (an 8-hex/32-bit id collides around tens of thousands of entities and would clobber another entity's page).""" - return hashlib.sha1(str(entity_id or "").encode("utf-8")).hexdigest()[:16] + return hashlib.sha1(str(entity_id or "").encode("utf-8"), usedforsecurity=False).hexdigest()[:16] def entity_moc_stem(self, entity_id: str, label: str | None = None) -> str: """The MOC note filename stem — the SINGLE source of the entity->entity wikilink target and diff --git a/backend/requirements.lock.txt b/backend/requirements.lock.txt new file mode 100644 index 00000000..269a50e4 --- /dev/null +++ b/backend/requirements.lock.txt @@ -0,0 +1,132 @@ +# This file was autogenerated by uv via the following command: +# Regenerate: uv pip compile --universal --python-version 3.12 backend/requirements.txt backend/runtime-requirements.txt -o backend/requirements.lock.txt +annotated-doc==0.0.5 + # via fastapi +annotated-types==0.8.0 + # via pydantic +anthropic==1.15.0 + # via -r backend/requirements.txt +anyio==4.15.1 + # via + # anthropic + # httpx2 + # starlette +argon2-cffi==25.1.0 + # via -r backend/requirements.txt +argon2-cffi-bindings==26.1.0 + # via argon2-cffi +cffi==2.1.1 + # via + # argon2-cffi-bindings + # cryptography +click==8.5.0 + # via + # huggingface-hub + # uvicorn +cloudpickle==3.1.2 + # via joblib +colorama==0.4.6 ; sys_platform == 'win32' + # via tqdm +cryptography==50.0.2 + # via -r backend/requirements.txt +docstring-parser==0.18.0 + # via anthropic +fastapi==0.143.3 + # via -r backend/requirements.txt +filelock==4.2.1 + # via huggingface-hub +fsspec==2026.9.0 + # via huggingface-hub +h11==0.16.0 + # via + # httpcore2 + # uvicorn +hf-xet==1.7.0 ; platform_machine == 'AMD64' or platform_machine == 'ARM64' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64' + # via huggingface-hub +httpcore2==2.13.1 ; sys_platform != 'emscripten' + # via httpx2 +httpx2==2.13.1 + # via + # anthropic + # huggingface-hub +httpx2-jsfetch==1.0 ; sys_platform == 'emscripten' + # via httpx2 +huggingface-hub==2.2.0 + # via + # model2vec + # tokenizers +idna==3.20 + # via + # anyio + # httpx2 +jinja2==3.1.6 + # via model2vec +jiter==0.17.0 + # via anthropic +joblib==1.6.0 + # via model2vec +markupsafe==3.0.4 + # via jinja2 +model2vec==0.10.0 + # via -r backend/runtime-requirements.txt +numpy==2.5.4 + # via model2vec +opentelemetry-api==1.45.1 + # via + # anthropic + # fastapi +packaging==26.3 + # via huggingface-hub +pycparser==3.11 ; implementation_name != 'PyPy' + # via cffi +pydantic==2.14.1 + # via + # -r backend/requirements.txt + # anthropic + # fastapi +pydantic-core==2.50.1 + # via pydantic +pysqlite3==0.6.1 + # via -r backend/runtime-requirements.txt +python-dotenv==1.2.4 + # via -r backend/requirements.txt +pyyaml==6.0.3 + # via huggingface-hub +safetensors==0.8.0 + # via model2vec +sniffio==1.3.1 + # via anthropic +sqlite-vec==0.1.9 + # via + # -r backend/requirements.txt + # -r backend/runtime-requirements.txt +starlette==1.8.0 + # via fastapi +tokenizers==0.23.3 + # via model2vec +tqdm==4.70.1 + # via + # huggingface-hub + # model2vec +truststore==0.10.5 ; sys_platform != 'emscripten' + # via + # httpcore2 + # httpx2 +typing-extensions==4.16.0 + # via + # anthropic + # anyio + # fastapi + # httpx2 + # huggingface-hub + # opentelemetry-api + # pydantic + # pydantic-core + # starlette + # typing-inspection +typing-inspection==0.4.4 + # via + # fastapi + # pydantic +uvicorn==0.54.0 + # via -r backend/requirements.txt diff --git a/backend/tests/test_macos_ui_quality_contract.py b/backend/tests/test_macos_ui_quality_contract.py index 3758fa27..645fdd4f 100644 --- a/backend/tests/test_macos_ui_quality_contract.py +++ b/backend/tests/test_macos_ui_quality_contract.py @@ -64,10 +64,13 @@ def test_native_apple_button_is_full_width_and_entitlement_gated(self) -> None: # no Apple client id, so the button is a real control, never a dead one. source = CORTEX_CLOUD_AUTH.read_text(encoding="utf-8") - self.assertIn("hasAppleSignInEntitlement", source) + # The entitlement probe lives in `AppleSignInSupport.isAvailable`; both the settings + # sign-in surface and the first-run restore step gate on that single signal. + self.assertIn("enum AppleSignInSupport", source) self.assertIn("com.apple.developer.applesignin", source) self.assertIn(".frame(maxWidth: .infinity, minHeight: 44, maxHeight: 44)", source) - self.assertIn("if canUseNativeAppleSignIn {", source) + self.assertIn("if AppleSignInSupport.isAvailable {", source) + self.assertIn("private var canUseNativeAppleSignIn: Bool { AppleSignInSupport.isAvailable }", source) # 4.8 regression guard: SIWA must never again be gated on the web-provider list. self.assertNotIn("canUseNativeAppleSignIn && backendOffersApple", source) # Provider-aware sign-in: a real labeled button per configured provider (never a diff --git a/backend/tests/test_rerank_eval.py b/backend/tests/test_rerank_eval.py index 92ba1296..98d7c85c 100644 --- a/backend/tests/test_rerank_eval.py +++ b/backend/tests/test_rerank_eval.py @@ -11,8 +11,12 @@ def test_no_regression_under_model2vec(self): prev = os.environ.get("CORTEX_EMBEDDING_PROVIDER") os.environ["CORTEX_EMBEDDING_PROVIDER"] = "model2vec" try: - from backend.app.embeddings import embedding_status + from backend.app.embeddings import embedding_status, warmup_embedding_provider + # embedding_status() reports the EFFECTIVE provider, but the hash-fallback latch is + # only set by a live embed. Probe first so an environment without the model2vec + # package (CI installs backend/requirements.txt only) skips instead of failing. + warmup_embedding_provider() if embedding_status().get("provider") != "model2vec": self.skipTest("model2vec embedder unavailable in this environment") from scripts.rerank_eval import run_rerank_eval diff --git a/deploy/launchd/install.sh b/deploy/launchd/install.sh new file mode 100755 index 00000000..800e48f7 --- /dev/null +++ b/deploy/launchd/install.sh @@ -0,0 +1,102 @@ +#!/usr/bin/env bash +# Install the single-user Cortex backend as a macOS launchd LaunchAgent (starts at login, +# restarts on crash, survives logout/login cycles). User-scoped: no sudo, fully reversible +# with deploy/launchd/uninstall.sh. +# +# deploy/launchd/install.sh [--mode fastapi|standalone] [--dry-run] +# +# Prerequisites: scripts/setup_local.sh has been run (./.venv and ./.env exist). +# Logs: .run/launchd.out.log and .run/launchd.err.log in the repo. +# Label: com.cortex.local.backend +# +# For a headless machine, also enable Automatic Login (System Settings > Users & Groups) and +# `sudo pmset -c sleep 0 disksleep 0 autorestart 1`, or promote the plist to /Library/LaunchDaemons. +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/../.." && pwd)" +LABEL="${CORTEX_LAUNCHD_LABEL:-com.cortex.local.backend}" +AGENTS_DIR="$HOME/Library/LaunchAgents" +PLIST="$AGENTS_DIR/$LABEL.plist" +ENV_FILE="${CORTEX_ENV_FILE:-$ROOT/.env}" +RUN_DIR="$ROOT/.run" +MODE="${CORTEX_SERVER_MODE:-fastapi}" +DRY_RUN=0 +while [[ $# -gt 0 ]]; do + case "$1" in + --mode) MODE="$2"; shift ;; + --dry-run) DRY_RUN=1 ;; + -h|--help) sed -n '2,14p' "$0"; exit 0 ;; + *) echo "unknown flag: $1" >&2; exit 2 ;; + esac + shift +done + +[[ -f "$ENV_FILE" ]] || { echo "ERROR: $ENV_FILE missing; run scripts/setup_local.sh first" >&2; exit 1; } +[[ -x "$ROOT/.venv/bin/python" ]] || { echo "ERROR: $ROOT/.venv missing; run scripts/setup_local.sh first" >&2; exit 1; } +mkdir -p "$AGENTS_DIR" "$RUN_DIR" + +# If a foreground/background copy started by scripts/start_backend.sh is running, stop it so the +# service does not fight it for the port. +if [[ -f "$RUN_DIR/backend.pid" ]] && kill -0 "$(cat "$RUN_DIR/backend.pid")" 2>/dev/null; then + echo "==> stopping the manually started backend first" + "$ROOT/scripts/stop_backend.sh" +fi + +cat >"$PLIST" < + + + + Label$LABEL + ProgramArguments + + /bin/bash + $ROOT/scripts/start_backend.sh + --foreground + --mode + $MODE + + EnvironmentVariables + + CORTEX_ENV_FILE$ENV_FILE + PATH/usr/bin:/bin:/usr/sbin:/sbin + + WorkingDirectory$ROOT + RunAtLoad + KeepAlive + ThrottleInterval5 + StandardOutPath$RUN_DIR/launchd.out.log + StandardErrorPath$RUN_DIR/launchd.err.log + ProcessTypeBackground + + +EOF +plutil -lint "$PLIST" >/dev/null +echo "==> wrote $PLIST (mode: $MODE)" + +if [[ "$DRY_RUN" == "1" ]]; then + echo "==> dry run: not loaded. Load with: launchctl load -w \"$PLIST\"" + exit 0 +fi + +launchctl unload "$PLIST" 2>/dev/null || true +launchctl load -w "$PLIST" +echo "==> loaded $LABEL; waiting for health" +for _ in $(seq 1 60); do + if "$ROOT/scripts/health_check.sh" --quiet; then + "$ROOT/scripts/health_check.sh" + cat <&2 +tail -n 30 "$RUN_DIR/launchd.err.log" >&2 || true +exit 1 diff --git a/deploy/launchd/uninstall.sh b/deploy/launchd/uninstall.sh new file mode 100755 index 00000000..877cf986 --- /dev/null +++ b/deploy/launchd/uninstall.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# Remove the launchd LaunchAgent installed by deploy/launchd/install.sh. Data in the vault is +# untouched. +set -euo pipefail +LABEL="${CORTEX_LAUNCHD_LABEL:-com.cortex.local.backend}" +PLIST="$HOME/Library/LaunchAgents/$LABEL.plist" +if [[ -f "$PLIST" ]]; then + launchctl unload "$PLIST" 2>/dev/null || true + rm -f "$PLIST" + echo "removed $PLIST (vault data left in place)" +else + echo "nothing to remove: $PLIST not found" +fi diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..78f83999 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,77 @@ +# Cortex backend on a Linux host you control. +# +# cp .env.example .env # then set CORTEX_API_KEY to a long random token +# docker compose up -d --build # API on 127.0.0.1:8766 (loopback only) +# docker compose --profile proxy up -d # + Caddy with automatic HTTPS on $CORTEX_DOMAIN +# docker compose --profile worker up -d # + external job worker (multi-tenant profile only) +# +# Single-user, local-first (default): just the `api` service; background jobs run inline. +# Multi-tenant server profile: set CORTEX_SHARD_MODE=bucket, CORTEX_WORKER_MODE=external, +# CORTEX_AUTH_ENABLED=1 etc. in .env (see deploy/cortex.env.example) and enable `worker`. +# +# The port is published on 127.0.0.1 only. Expose it to a network exclusively through a TLS +# reverse proxy (the `proxy` profile, or your own) so bearer tokens never travel in clear text. +services: + api: + build: . + image: cortex-backend:local + env_file: .env + environment: + CORTEX_HOST: 0.0.0.0 + CORTEX_VAULT_PATH: /data/Cortex.vault + CORTEX_DB_PATH: /data/Cortex.vault/index.sqlite + CORTEX_SHARD_ROOT: /data/shards + CORTEX_MODEL2VEC_PATH: /models/model2vec + CORTEX_FORWARDED_ALLOW_IPS: "*" + ports: + - "127.0.0.1:${CORTEX_PORT:-8766}:8766" + volumes: + - cortex-data:/data + - ./models:/models:ro # optional: put the model2vec model here (scripts/setup_local.sh --with-model) + restart: unless-stopped + healthcheck: + test: ["CMD", "curl", "-fsS", "http://127.0.0.1:8766/health"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 20s + + worker: + profiles: ["worker"] + image: cortex-backend:local + env_file: .env + environment: + CORTEX_VAULT_PATH: /data/Cortex.vault + CORTEX_DB_PATH: /data/Cortex.vault/index.sqlite + CORTEX_SHARD_ROOT: /data/shards + CORTEX_MODEL2VEC_PATH: /models/model2vec + command: ["python", "scripts/run_memory_worker.py", "--iterations", "0", "--interval-seconds", "${CORTEX_WORKER_INTERVAL:-30}"] + volumes: + - cortex-data:/data + - ./models:/models:ro + depends_on: + api: + condition: service_healthy + restart: unless-stopped + + proxy: + profiles: ["proxy"] + image: caddy:2 + environment: + CORTEX_DOMAIN: ${CORTEX_DOMAIN:?set CORTEX_DOMAIN in .env for the proxy profile} + command: ["caddy", "reverse-proxy", "--from", "${CORTEX_DOMAIN}", "--to", "api:8766"] + ports: + - "80:80" + - "443:443" + volumes: + - caddy-data:/data + - caddy-config:/config + depends_on: + api: + condition: service_healthy + restart: unless-stopped + +volumes: + cortex-data: + caddy-data: + caddy-config: diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 0bb04884..c34c264f 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -316,3 +316,34 @@ The local source-account sync endpoint is enough for beta local app integrations - Local diagnostics and backups - Sanitized support bundle for operational triage - Future: sensitive source blocklist, hosted account deletion, per-source retention rules + +## In-house deployment topology + +Three ways to run the same backend code on infrastructure you control. All share the vault + +SQLite storage model above; none needs an external database, queue or cache. + +``` +(1) Single-user, this Mac (2) Linux host, Docker (3) Multi-tenant server +--------------------------------------- ------------------------------------ ------------------------------- +launchd: com.cortex.local.backend docker compose: api (+worker,+proxy) systemd/launchd: api, worker, backup + scripts/start_backend.sh --foreground uvicorn backend.app.main:app uvicorn --workers 2 (loopback) + uvicorn OR app.standalone_server 127.0.0.1:8766 published Caddy / Cloudflare Tunnel -> TLS + 127.0.0.1:8766 volume: /data (vault + index) sharded SQLite under CORTEX_SHARD_ROOT + ./backend/data/Cortex.vault optional caddy profile (TLS) accounts, scoped tokens, KEK + ./.env (token) ./.env (token) /etc/cortex/cortex.env + +Clients for all three: macOS app, MCP stdio bridge (scripts/cortex_mcp_stdio.py), Python/TS SDK, +browser extension, Obsidian plugin, any HTTP client with a bearer token. +``` + +Entry points: + +| Entry point | Module | When | +|---|---|---| +| FastAPI app | `backend.app.main:app` via uvicorn | servers, Docker, full OpenAPI surface | +| Standalone server | `backend/app/standalone_server.py` (stdlib `ThreadingHTTPServer`, built-in worker thread) | what the packaged macOS app launches; same routes, no uvicorn/FastAPI needed | +| Background worker | `scripts/run_memory_worker.py` | only when `CORTEX_WORKER_MODE=external` (multi-tenant) | +| MCP bridge | `scripts/cortex_mcp_stdio.py` | stdio to `/mcp` proxy for desktop AI tools | + +Operational docs: `docs/INSTALLATION.md`, `docs/CONFIGURATION.md`, `docs/OPERATIONS.md`, +`docs/SECURITY.md`, `docs/DEPENDENCIES.md`. diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md new file mode 100644 index 00000000..25f0c5c6 --- /dev/null +++ b/docs/CONFIGURATION.md @@ -0,0 +1,136 @@ +# Configuration + +The backend reads configuration from **environment variables only** (`backend/app/config.py` +`load_settings()` plus a few module-level reads). It never loads a `.env` file itself; the wrappers +(`scripts/start_backend.sh`, `deploy/launchd/install.sh`, `docker-compose.yml`, `deploy/macmini/*`, +`deploy/systemd/*`) export the file before launching. + +Two files describe the two supported profiles: + +| Profile | Example file | Use | +|---|---|---| +| Single-user, local-first | `.env.example` | one person, one vault, loopback, the mode the macOS app uses | +| Multi-tenant server | `deploy/cortex.env.example` | many users, sharded SQLite, accounts/OAuth, scoped tokens, external worker | + +Values are case-sensitive; booleans accept `1/true/yes/on`. + +## Required + +| Variable | Default | Notes | +|---|---|---| +| `CORTEX_API_KEY` | (none) | Bearer token for every REST call and the admin token in server mode. The backend refuses `dev-local-key` unless `CORTEX_ALLOW_INSECURE_DEV_TOKEN=1`. | + +## Storage + +| Variable | Default | Notes | +|---|---|---| +| `CORTEX_VAULT_PATH` | `backend/data/Cortex.vault` | User-owned records: Markdown + JSON, `events.jsonl`, `backups/`. The macOS app uses `~/Library/Application Support/Cortex/Cortex.vault`. | +| `CORTEX_DB_PATH` | `/index.sqlite` | Rebuildable FTS5 + `sqlite-vec` index. Deleting it is safe; `POST /v1/maintenance/rebuild-index-from-vault` rebuilds. | +| `CORTEX_BACKUP_RETENTION_COUNT` | `20` | In-vault backup zips to keep. | +| `CORTEX_BACKUP_RETENTION_DAYS` | `0` | Age pruning, off by default. | +| `CORTEX_STORE_CACHE_SIZE` | `512` | Per-process store cache (server mode). | + +## Network + +| Variable | Default | Notes | +|---|---|---| +| `CORTEX_HOST` | `127.0.0.1` | Used by the wrappers. Bind elsewhere only behind TLS. | +| `CORTEX_PORT` | `8766` | The macOS app, MCP bridge and SDKs default to this port. | +| `CORTEX_PUBLIC_BASE_URL` | `http://127.0.0.1:8766` | Advertised in OAuth redirects, MCP resources, account pages. | +| `CORTEX_PUBLIC_APP_URL` | = base URL | Browser-facing URL for sign-in pages (server mode). | +| `CORTEX_CORS_ORIGINS` | loopback | Comma-separated allowed origins. | +| `CORTEX_ALLOW_EXTENSION_CORS` | `1` | Allow `chrome-extension://` / `moz-extension://` origins for the browser extension. | +| `CORTEX_UVICORN_WORKERS` | `1` | uvicorn processes (FastAPI mode). Rate limiter is per process. | +| `CORTEX_SERVER_MODE` | `fastapi` | Wrapper-only: `fastapi` (uvicorn) or `standalone` (stdlib server bundled in the app). | +| `CORTEX_MAX_CONCURRENT_REQUESTS` | `8` | Standalone server concurrency gate. | +| `CORTEX_RATE_LIMIT_RPS` / `CORTEX_RATE_LIMIT_BURST` | `20` / `60` | Standalone per-token token bucket. | +| `CORTEX_RATE_LIMIT_PER_MINUTE` | `0` | FastAPI per-user limiter; `0` disables. | + +## Authentication and tokens + +| Variable | Default | Notes | +|---|---|---| +| `CORTEX_MCP_API_KEY` | (empty) | Optional scoped token accepted only by `/mcp`. Keeps the app token out of MCP configs. | +| `CORTEX_MCP_API_KEY_SCOPES` | `read,write,export,maintenance` | Scopes of that token (`destructive` must be added explicitly). | +| `CORTEX_REQUIRE_SCOPED_API_TOKENS` | `0` | Server mode: only minted scoped tokens may call user endpoints; the admin token is control-plane only. | +| `CORTEX_MCP_TOOL_SURFACE` | `core` | Advertised MCP tool list: `core`, `full`, `coding`, `chat`, `chatgpt`. Advertisement never grants authorization. | +| `CORTEX_MCP_STREAMABLE` | `1` | Streamable-HTTP (SSE) compatibility on `/mcp`. | +| `CORTEX_SYNC_SIGNING_KEY` | (empty) | HMAC key for `/v1/sync/changes` manifests. | + +## Retrieval quality (all local) + +| Variable | Default | Notes | +|---|---|---| +| `CORTEX_EMBEDDING_PROVIDER` | `hash` | `hash` (offline, deterministic, 384-d), `model2vec` (offline, semantic, 256-d, needs the model on disk), `openai` (remote). | +| `CORTEX_MODEL2VEC_PATH` | (empty) | Directory produced by `scripts/setup_local.sh --with-model` (`config.json` + weights). | +| `CORTEX_MODEL2VEC_MODEL` | `minishlab/potion-base-8M` | Hugging Face id used when downloading. | +| `CORTEX_EMBEDDING_MODEL` / `CORTEX_EMBEDDING_DIMENSIONS` | `text-embedding-3-small` / `384` | OpenAI provider only. | +| `CORTEX_EMBEDDING_STRICT` | `0` | `1` makes indexing fail instead of falling back to hash when the provider is unavailable. | +| `CORTEX_RERANK` | (app sets `linear+mmr`) | Reranker blend; no-op under the hash provider. | +| `CORTEX_QUERY_PLAN` / `CORTEX_TEMPORAL_DECAY` | (app sets `1`) | Query planning and per-layer temporal decay. | +| `CORTEX_ENTITY_BOOST`, `CORTEX_CONTEXT_HOP`, `CORTEX_CONTEXT_HOP2`, `CORTEX_CONTEXT_HOP_MS`, `CORTEX_CONTEXT_HOP2_MS`, `CORTEX_RERANK_WEIGHTS_PATH`, `CORTEX_RETRIEVAL_FEEDBACK`, `CORTEX_SESSION_PREFETCH` | tuned defaults | Advanced retrieval knobs; see `backend/app/storage.py`. | + +## Extraction + +| Variable | Default | Notes | +|---|---|---| +| `ANTHROPIC_API_KEY` | (empty) | When set, captures are condensed with Claude. Unset = deterministic local extractor; nothing leaves the machine. | +| `CORTEX_EXTRACTION_MODEL` / `CORTEX_CONDENSE_MODEL` | `claude-opus-4-5` | Model ids for extraction / condensation. | +| `CORTEX_EXTRACTION_MODE` | auto | Force `local` or `llm`. | + +## Behaviour + +| Variable | Default | Notes | +|---|---|---| +| `CORTEX_AUTO_APPROVE_CAPTURES` | `0` | `1` skips the Review inbox for new captures. | +| `CORTEX_APP_BRAND` | `Cortex` | User-visible brand in backend strings (`Doppl` for the App Store build). | +| `CORTEX_WORKER_MODE` | `inline` | `external` moves queued jobs to `scripts/run_memory_worker.py`. | +| `CORTEX_WORKER_*` | see script | `--limit`, `--interval-seconds`, `--iterations`, `--worker-id`, `--failed-job-limit` defaults for the worker. | +| `CORTEX_STANDALONE_WORKER_*` | enabled, 15 s | Standalone server's built-in worker thread. | +| `CORTEX_OBSERVABILITY_ENABLED` | `0` | Enables `GET /v1/metrics` (admin). | +| `CORTEX_DELIVERY_ALLOW_INTERNAL` | `0` | Allow `/v1/delivery/send` to private/internal targets (SSRF guard off). Keep `0`. | +| `CORTEX_DEMO_EMBEDDINGS`, `CORTEX_DEMO_PORT`, `CORTEX_SMOKE_PORT` | | Used only by demo/eval scripts. | + +## Multi-tenant server profile only + +| Variable | Notes | +|---|---| +| `CORTEX_SHARD_MODE` (`local`\|`bucket`), `CORTEX_SHARD_ROOT`, `CORTEX_SHARD_COUNT` (16) | Sharded SQLite layout under one root. | +| `CORTEX_HOSTED_RUNTIME_TIER` (`sharded_sqlite`\|`postgres`), `CORTEX_HOSTED_DATABASE_URL` / `DATABASE_URL`, `CORTEX_HOSTED_VECTOR_BACKEND` | Postgres/pgvector tier is a readiness gate (`/ready` reports it), not an implemented storage backend in this tree. | +| `CORTEX_AUTH_ENABLED`, `CORTEX_AUTH_AUTOVERIFY`, `CORTEX_AUTH_EMAIL_MODE` (`log`\|`smtp`), `CORTEX_AUTH_RATE_LIMIT_PER_MINUTE`, `CORTEX_AUTH_ACCESS_TTL_SECONDS`, `CORTEX_AUTH_REFRESH_IDLE_TTL_SECONDS`, `CORTEX_AUTH_REFRESH_ABSOLUTE_TTL_SECONDS`, `CORTEX_ACCOUNTS_DB_PATH` | First-party accounts (argon2id passwords, access/refresh tokens). | +| `CORTEX_SMTP_HOST`, `CORTEX_SMTP_PORT`, `CORTEX_SMTP_USERNAME`, `CORTEX_SMTP_PASSWORD`, `CORTEX_SMTP_FROM_ADDR`, `CORTEX_SMTP_USE_STARTTLS`, `CORTEX_SMTP_USE_SSL` | Verification / reset mail. Mode `smtp` without a host degrades to the log sink. | +| `CORTEX_OIDC_GOOGLE_CLIENT_ID/SECRET`, `CORTEX_OIDC_GITHUB_CLIENT_ID/SECRET`, `CORTEX_OIDC_APPLE_CLIENT_ID/SECRET` | Social sign-in. | +| `CORTEX_KEK` / `CORTEX_KEK_FILE`, `CORTEX_KEK_VERSION`, `CORTEX_REQUIRE_ENCRYPTED_CREDENTIALS` | Key-encryption key for per-user credential encryption (CXE1). Escrow it offline. | +| `CORTEX_DEFAULT_MEMORY_QUOTA`, `CORTEX_PLAN_QUOTAS` (JSON) | Per-user memory quotas. | +| `CORTEX_BILLING_PROVIDER` (`paddle`\|`stripe`), `CORTEX_PADDLE_WEBHOOK_SECRET`, `CORTEX_PADDLE_API_KEY`, `CORTEX_STRIPE_WEBHOOK_SECRET`, `CORTEX_STRIPE_API_KEY` | Dormant unless provider + webhook secret are set. | +| `CORTEX_TURNSTILE_SITEKEY`, `CORTEX_TURNSTILE_SECRET` | Cloudflare Turnstile on public signup. | +| `CORTEX_BROKER__CLIENT_ID/SECRET`, `CORTEX_OAUTH_BROKER_EXCHANGE_URL`, `CORTEX_GITHUB_OAUTH_CLIENT_ID`, `CORTEX_GITHUB_OAUTH_SCOPE`, `CORTEX_MANAGED_OAUTH_*`, `CORTEX_GOOGLE_OAUTH_*` | OAuth token-exchange broker for confidential connectors (Notion, GitHub, Google, Microsoft). | + +## Build-time (macOS packaging only) + +`CORTEX_BUNDLE_PYTHON`, `CORTEX_PYTHON_FRAMEWORK_SOURCE`, `CORTEX_BUNDLE_BACKEND_DEPS`, +`CORTEX_CODESIGN_IDENTITY`, `CORTEX_NESTED_CODESIGN_IDENTITY`, `CORTEX_CODESIGN_TIMESTAMP`, +`CORTEX_ENTITLEMENTS`, `CORTEX_CHILD_ENTITLEMENTS`, `CORTEX_DEVID_ENTITLEMENTS`, +`CORTEX_PROVISIONING_PROFILE`, `CORTEX_DISTRIBUTION_MODE` (`direct`\|`app-store`), +`CORTEX_APP_BUNDLE_NAME`, `CORTEX_DISPLAY_NAME`, `CORTEX_PRODUCT_NAME`, `CORTEX_SKIP_PYC`, +`CORTEX_REQUIRE_MODEL`, `CORTEX_NOTARY_PROFILE`, `CORTEX_RELEASE_STRICT`. See `macos/build.sh` and +`docs/APPLE_RELEASE.md`. + +## Legacy prototype (not used by the app) + +`capture.py`, `ingest.py`, `github_store.py`, `redis_store.py`, `mcp_server.py`, `ui.py` and +`instrumentation.py` at the repo root are the original prototype and read `REDIS_URL`, +`GITHUB_TOKEN`, `VOYAGE_API_KEY`, `ANTHROPIC_API_KEY` and Arize/OpenInference settings from +`requirements.txt`. They are kept for history and CI byte-compilation only; nothing in the backend +or the app imports them. + +## External integrations and where data goes + +| Integration | Trigger | Outbound host | Can be avoided? | +|---|---|---|---| +| Claude extraction | `ANTHROPIC_API_KEY` set | api.anthropic.com | Yes: leave unset (local extractor). | +| OpenAI embeddings | provider `openai` | api.openai.com | Yes: `hash` or `model2vec`. | +| Model download | `setup_local.sh --with-model` | huggingface.co, once | Yes: copy the `models/model2vec` folder from another machine. | +| Source connectors (Obsidian, GitHub, Slack, Notion, Linear, Jira, Readwise, Raindrop, Gmail, Drive, Outlook, Calendar ICS, Zotero) | user connects them | each provider's API | Yes: connect only local notes / file imports. Tokens are stored encrypted in the vault when a KEK is configured. | +| Social sign-in, SMTP, billing, Turnstile | server profile vars | provider hosts | Yes: single-user profile never enables them. | +| Update feed | macOS app checks `latest.json` | the configured feed URL | Yes: point the feed at your own host or disable in the app. | diff --git a/docs/DEPENDENCIES.md b/docs/DEPENDENCIES.md new file mode 100644 index 00000000..d9825370 --- /dev/null +++ b/docs/DEPENDENCIES.md @@ -0,0 +1,74 @@ +# Dependencies, licences, external services and self-hosting limits + +Project licence: MIT (`LICENSE`, copyright Doppl). Everything required to run the backend and build +the macOS app is open source or Apple-provided. + +## Runtime inventory + +| Component | Version / pin | Purpose | Licence | Runs locally? | +|---|---|---|---|---| +| CPython | 3.12 (3.11+) | backend runtime | PSF | yes | +| fastapi | 0.143.3 | HTTP API (`backend/app/main.py`) | MIT | yes | +| starlette | 1.8.0 | ASGI toolkit under FastAPI | BSD-3 | yes | +| uvicorn | 0.54.0 | ASGI server | BSD-3 | yes | +| pydantic / pydantic-core | 2.14.1 / 2.50.1 | request/response models | MIT | yes | +| python-dotenv | 1.2.4 | listed; the backend does not load `.env` itself | BSD-3 | yes | +| anthropic | 1.15.0 | optional Claude extraction client | MIT | client only; service is remote and optional | +| sqlite-vec | 0.1.9 | vector KNN inside SQLite | MIT / Apache-2.0 | yes | +| pysqlite3 | 0.6.1 | SQLite module built against system libsqlite3 (extension loading, FTS5) | zlib-style (pysqlite) | yes, compiled at install | +| model2vec | 0.10.0 | on-device static embeddings | MIT | yes | +| numpy, tokenizers, safetensors, huggingface_hub, joblib, tqdm, fsspec, filelock, hf-xet, cloudpickle, jinja2, markupsafe, docstring_parser, pyyaml, packaging | pinned in lockfile | transitive deps of model2vec | BSD / Apache / MIT / MPL-2.0 (tqdm) | yes; huggingface_hub is used only for the one-time model download | +| argon2-cffi (+bindings), cffi, pycparser | 25.1.0 | password hashing (server profile) | MIT | yes | +| cryptography | 50.0.2 | CXE1 credential encryption, OIDC verification | Apache-2.0 / BSD-3 | yes | +| certifi | 2026.7.22 | CA bundle for outbound TLS | MPL-2.0 | yes | +| anyio, httpx2, httpcore2, h11, sniffio, idna, jiter, typing_extensions, annotated-types, annotated-doc, typing-inspection, opentelemetry-api, truststore | pinned | transitive (anthropic / fastapi) | MIT / BSD / Apache | yes | + +Full pinned list: `backend/requirements.lock.txt` (46 packages, generated with +`uv pip compile --universal`). Dev/test only: pytest, bandit, pip-audit. + +### Build and tooling + +| Tool | Needed for | Available without admin? | +|---|---|---| +| Xcode Command Line Tools (`swiftc`, `codesign`, `plutil`) | `macos/build.sh` (full Xcode not required) | yes (`xcode-select --install`) | +| Python.framework 3.12 at `/Library/Frameworks` | only `CORTEX_BUNDLE_PYTHON=1` / App Store builds | python.org installer | +| Developer ID certificate + notarytool profile | only `macos/package_release.sh` | Apple Developer Program | +| Node 22 + npm | `packages/obsidian-cortex-plugin` (esbuild, typescript), `sdk/typescript` (tsc, node --test), `packages/openclaw-cortex-context` | nodejs.org; not installed on the verification machine, so these packages were not rebuilt (the committed `main.js` is used by the app bundle) | +| Docker Engine + Compose v2 | `Dockerfile`, `docker-compose.yml` | not installed on the verification machine; image untested there | +| sqlite3 CLI | `deploy/backup.sh` (server profile) | preinstalled on macOS; `apt install sqlite3` | + +### Vendored / bundled + +| Item | Location | Licence | +|---|---|---| +| Obsidian API typings | `packages/obsidian-api` | MIT (upstream obsidianmd/obsidian-api) | +| potion-base-8M embedding model | downloaded to `models/model2vec` on demand | MIT (minishlab) | +| Sample notes | `macos/SampleNotes` | project | + +## External services + +| Service | Used by | Required? | Self-hostable alternative | +|---|---|---|---| +| Anthropic API | LLM extraction/condensation when `ANTHROPIC_API_KEY` is set | no | deterministic local extractor (default) | +| OpenAI embeddings | `CORTEX_EMBEDDING_PROVIDER=openai` | no | `hash` (default) or `model2vec` (local semantic) | +| Hugging Face Hub | one-time download of the embedding model | no (optional, once) | copy `models/model2vec` between machines | +| Source connectors: GitHub, Slack, Notion, Linear, Jira, Readwise, Raindrop, Google (Gmail/Drive/Calendar), Microsoft (Outlook), Zotero, Obsidian/local notes | user-initiated syncs | no | local notes, Obsidian vault, file/export imports need no network | +| OAuth brokers, social sign-in (Google/GitHub/Apple), SMTP, Paddle/Stripe, Cloudflare Turnstile | multi-tenant server profile | no | not needed in single-user mode | +| GitHub Releases / `latest.json` feed | macOS app update check | no | host `latest.json` yourself or ignore | +| Cloudflare Tunnel, Caddy/Let's Encrypt | optional ingress for a shared server | no | any reverse proxy with your own certificates | +| Redis, Voyage AI, GitHub (as a store), Arize/Phoenix, Streamlit | legacy prototype files at repo root only | no | not part of the product | + +## Self-hosting boundaries (what is and is not possible) + +- **Fully offline single-user operation: yes.** Backend, MCP, search, review, export, backups and + the macOS app all work with no network after dependencies are installed. Semantic embeddings + need the one-time model download or a copied model folder. +- **Multi-user server on your hardware: yes**, with the sharded-SQLite tier (`deploy/`). The + Postgres/pgvector tier mentioned in the docs is a readiness gate, not an implemented backend in + this tree; do not plan on it. +- **Cloud sync between devices (E2EE design in `docs/ACCOUNTS_ENCRYPTION_DESIGN.md`)**: requires + running the server profile with accounts; there is no third-party dependency, but it is beta. +- **Not self-hostable:** nothing in the core. Third-party source connectors inherently talk to + their vendors; social sign-in and payments inherently need those providers. +- **Licensing constraints:** none beyond MIT for the code; the embedding model is MIT; Obsidian + typings are MIT. Connector use is subject to each provider's API terms. diff --git a/docs/INSTALLATION.md b/docs/INSTALLATION.md new file mode 100644 index 00000000..ce5b2101 --- /dev/null +++ b/docs/INSTALLATION.md @@ -0,0 +1,164 @@ +# Installation (clean machine, in-house) + +This guide gets the Cortex backend running on infrastructure you control, from nothing. It was +verified on macOS 26 (Apple Silicon) with only Xcode Command Line Tools and `uv` installed; the Linux +path uses the same pinned dependency set inside Docker. + +Cortex has two deployable parts: + +| Part | What it is | Where it runs | +|---|---|---| +| Backend engine | Python 3.12 HTTP + MCP server over a user-owned vault (Markdown/JSON) and a rebuildable SQLite index | Your Mac (launchd), any Linux host (Docker/systemd) | +| macOS app | Native SwiftUI client that embeds and launches its own copy of the backend | Your Mac only; optional | + +Everything below works offline after the one-time dependency install. No account, no cloud, no +external database. + +## 1. Prerequisites + +| Requirement | Why | How to get it | +|---|---|---| +| Git | clone the repo | Xcode CLT (`xcode-select --install`) or distro package | +| CPython 3.12 (3.11+ works) | backend runtime | `uv` installs it without admin rights (recommended), or python.org installer | +| C compiler + SQLite headers | `pysqlite3` builds from source so FTS5 and the `sqlite-vec` extension load on every platform | macOS: Xcode CLT. Debian/Ubuntu: `build-essential libsqlite3-dev` | +| `curl` | health checks | preinstalled on macOS; `apt install curl` on Linux | +| `uv` (optional, recommended) | reproducible Python + venv without sudo | `curl -LsSf https://astral.sh/uv/install.sh \| sh` | +| Xcode Command Line Tools | only to build the macOS app (full Xcode is **not** required) | `xcode-select --install` | +| Docker Engine 24+ with Compose v2 | only for the Linux container path | docker.com or distro package | +| Node 22 + npm | only for the optional Obsidian plugin, TypeScript SDK and browser-extension tooling | nodejs.org | + +Disk: the repository is about 1.8 GB because its history contains release artefacts and brand +assets; the backend itself is small. + +## 2. Get the code + +```bash +git clone https://github.com/trace-cortex/cortex-app.git ~/cortex-app +cd ~/cortex-app +``` + +There are no submodules and no Git LFS objects. A ZIP download is sufficient for the backend, but a +clone keeps the branches, tags and history you need for upgrades and rollbacks. + +## 3. Backend on macOS or Linux (bare metal) + +```bash +scripts/setup_local.sh --dev # --dev adds pytest/bandit/pip-audit; --with-model adds on-device embeddings +scripts/start_backend.sh # background process on 127.0.0.1:8766, log in .run/backend.log +scripts/health_check.sh # /health, /ready, authenticated call, bad-token rejection +``` + +`setup_local.sh` is idempotent. It: + +1. finds or installs CPython 3.12 (via `uv` when present), +2. creates `./.venv` from `backend/requirements.lock.txt` (every version pinned), +3. creates `./.env` from `.env.example` with a freshly generated `cxa_` API token (mode 600, never + overwritten on re-run), +4. optionally downloads the `minishlab/potion-base-8M` embedding model (about 30 MB, one-time + internet access) when you pass `--with-model`, after which semantic search is fully offline, +5. verifies that SQLite + `sqlite-vec` load. + +The API token lives only in `.env`. Read it with: + +```bash +grep '^CORTEX_API_KEY=' .env +``` + +### Make it a service (macOS) + +```bash +deploy/launchd/install.sh # user LaunchAgent: starts at login, restarts on crash +launchctl list | grep com.cortex # status +deploy/launchd/uninstall.sh # remove (data stays) +``` + +For a headless Mac, enable Automatic Login and `sudo pmset -c sleep 0 disksleep 0 autorestart 1`, or +copy the generated plist to `/Library/LaunchDaemons` to run at boot without a login session. + +### Make it a service (Linux, no Docker) + +Use the units in `deploy/systemd/` (`cortex-api.service`, `cortex-worker.service`, +`cortex-backup.timer`). They expect the multi-tenant layout under `/srv/cortex` and +`/etc/cortex/cortex.env`; `deploy/bootstrap.sh` renders all of it on Ubuntu 24.04. For a +single-user host, point `ExecStart` at `scripts/start_backend.sh --foreground` and +`EnvironmentFile` at your `.env` instead. + +## 4. Backend in Docker (Linux host) + +```bash +cp .env.example .env +python3 -c 'import secrets; print("cxa_" + secrets.token_hex(32))' # paste into CORTEX_API_KEY +docker compose up -d --build +curl http://127.0.0.1:8766/health +``` + +The port is published on loopback only. Add `--profile proxy` with `CORTEX_DOMAIN` set to put Caddy +with automatic TLS in front, or front it with your own reverse proxy. Data persists in the +`cortex-data` volume. The Docker path could not be executed on the verification machine (Docker is +not installed there); the image definition mirrors the bare-metal install exactly and uses the same +lockfile. + +## 5. macOS app (optional) + +```bash +./macos/build.sh # dev build, ad-hoc signed, ~2 minutes with CLT only +open macos/build/Cortex.app +``` + +The dev build does not bundle a Python framework (`CORTEX_BUNDLE_PYTHON=0` default); it runs the +backend with the `python3.12` found on `PATH`, so run `scripts/setup_local.sh` first and make sure +`~/.local/bin` (uv) or your Python is on `PATH` for GUI apps. The app launches its own backend on +port 8766 against `~/Library/Application Support/Cortex/Cortex.vault` with a token it generates. + +Run **either** the launchd service **or** the app on port 8766. To share one vault between the two, +set `CORTEX_VAULT_PATH` in `.env` to the app's default path above, then paste the service token into +the app under Connections & Privacy > Advanced > backend token. The app reuses a running backend only +when the vault path and token match (`macos/Sources/CortexApp.swift`, backend probe). + +Release packaging (`macos/package_release.sh`) needs a Developer ID certificate, a notarytool +profile and a Python.framework at `/Library/Frameworks/Python.framework/Versions/3.12` to bundle. +None of that is required to run in-house. + +## 6. Connect AI tools (MCP) + +Point any MCP client at the stdio bridge with the backend URL and a token: + +```json +{ + "mcpServers": { + "cortex": { + "command": "/Users/you/cortex-app/.venv/bin/python", + "args": ["/Users/you/cortex-app/scripts/cortex_mcp_stdio.py"], + "env": { "CORTEX_BASE_URL": "http://127.0.0.1:8766", "CORTEX_API_KEY": "" } + } + } +} +``` + +Prefer a scoped MCP token over the app token: set `CORTEX_MCP_API_KEY` in `.env` (any long random +string) and use that in MCP configs; `/mcp` accepts it while REST endpoints keep requiring the +main token. See `docs/MCP_INTEGRATIONS.md` and `docs/EXTERNAL_INTEGRATIONS.md`. + +## 7. Verify the install + +```bash +source .venv/bin/activate +python -m pytest backend/tests -q # ~7 minutes, 2152 tests +python scripts/backend_beta_smoke.py # end-to-end capture/approve/search against a temp vault +python scripts/check_vector_runtime.py # sqlite-vec loadable +``` + +Expected on a clean install: all tests pass, `"vector": {"available": true}`. + +## 8. Upgrade + +```bash +scripts/stop_backend.sh || launchctl unload ~/Library/LaunchAgents/com.cortex.local.backend.plist +git pull +scripts/setup_local.sh # reinstalls pinned deps, keeps .env +scripts/start_backend.sh || launchctl load -w ~/Library/LaunchAgents/com.cortex.local.backend.plist +scripts/health_check.sh +``` + +Schema migrations run automatically on startup (`backend/app/database.py`, `schema_migrations` +table); take a backup first with `scripts/backup_local.sh`. See `docs/OPERATIONS.md`. diff --git a/docs/IN_HOUSE_MIGRATION_REPORT.md b/docs/IN_HOUSE_MIGRATION_REPORT.md new file mode 100644 index 00000000..e2f52c1e --- /dev/null +++ b/docs/IN_HOUSE_MIGRATION_REPORT.md @@ -0,0 +1,119 @@ +# In-house migration report (2026-10-17) + +Scope: take `trace-cortex/cortex-app` and make it run, reproducibly and documented, on +infrastructure the operator controls (first target: this macOS workstation). This file records what +was inspected, what was verified, what changed, and what remains. + +## 1. Repository access and acquisition + +| Item | Result | +|---|---| +| Visibility | public, MIT, default branch `main`, 39 forks, about 1.8 GB of history | +| Control | the operator's GitHub account is an **admin of the `trace-cortex` organisation** and the repository (GitHub still redirects `VamikaSinghal/cortex-app` to it, i.e. it was transferred from that account). No fork was needed; `origin` stays at trace-cortex. | +| Clone | full clone to `~/cortex-app` with all 12 remote branches and 19 tags | +| Submodules / LFS | none | +| Work branch | `chore/in-house-deployment` (nothing pushed) | + +## 2. Codebase audit (summary) + +| Area | Finding | +|---|---| +| Languages | Python 3.12 backend (CI pins 3.12; `str \| None` syntax needs 3.10+, README says 3.11+), Swift 6 SwiftUI macOS app, TypeScript for the Obsidian plugin / TS SDK / OpenClaw adapter, JS browser extension, static HTML site | +| Backend entry points | `backend.app.main:app` (FastAPI + uvicorn) and `backend/app/standalone_server.py` (stdlib HTTP server with built-in job worker; this is what the packaged app launches). Both expose the same routes, `/health`, `/ready`, `/v1/*`, `/mcp` | +| Workers / scheduled jobs | inline job runner by default; `scripts/run_memory_worker.py` for `CORTEX_WORKER_MODE=external`; nightly backup timer in the server kit | +| Persistence | user-owned vault directory (Markdown + JSON + `events.jsonl`) and a rebuildable SQLite index (FTS5 + `sqlite-vec`, WAL). No Postgres, Redis or queue in the product. Schema migrations run on startup. | +| Auth | single bearer token (`CORTEX_API_KEY`), optional scoped tokens (`read/write/export/maintenance/destructive`), optional MCP-only token; multi-tenant profile adds argon2id accounts, OIDC (Google/GitHub/Apple), KEK-encrypted credentials | +| External services | all optional: Anthropic (extraction), OpenAI (embeddings), Hugging Face (one-time model download), 13 source connectors, SMTP, Paddle/Stripe, Turnstile, Cloudflare Tunnel / Caddy for ingress | +| Config surface | 168 `CORTEX_*` variables plus `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, `DATABASE_URL`; catalogued in `docs/CONFIGURATION.md` | +| Tests | 166 test files, pytest canonical (`backend/tests/conftest.py` isolates a temp vault) | +| CI | `.github/workflows/ci.yml`: python checks + 12 eval gates, security scan (bandit, pip-audit), distribution checks, Obsidian plugin build (Node 22), macOS app build (CLT) | +| Legacy prototype | `capture.py`, `ingest.py`, `github_store.py`, `redis_store.py`, `mcp_server.py`, `ui.py`, `instrumentation.py` + root `requirements.txt` (Redis, Voyage, rumps, Streamlit, Arize). Unused by the product. | + +Data flow: sources (notes, exports, connectors, MCP `remember_this`) -> captures (pending) -> +extractor (local deterministic or Claude) -> memories/entities/tasks in the vault -> review inbox +(approve/archive) -> SQLite index (FTS + vectors) -> retrieval (`/v1/search`, `/v1/ask`, +`/v1/context-pack`, MCP tools) -> clients (macOS app, MCP tools, SDKs, extension). + +## 3. Environment reconstruction + +| Step | Result | +|---|---| +| Runtime | this Mac had only Python 3.9, no Homebrew, no Node, no Docker; `uv` was present and installed CPython 3.12.14 without sudo | +| Dependencies | `backend/requirements.txt` + `backend/runtime-requirements.txt` installed cleanly; `pysqlite3` compiled with Xcode CLT; universal lockfile generated (`backend/requirements.lock.txt`, 46 pins) | +| Vector runtime | `scripts/check_vector_runtime.py`: `sqlite-vec v0.1.9 available: true` | +| Backend | both entry points start, pass `/health` and `/ready`, reject bad tokens (401), accept capture -> approve -> search -> ask with a cited answer, serve MCP `initialize` and `tools/list` (110 tools) | +| macOS app | `macos/build.sh` with `CORTEX_BUNDLE_PYTHON=0`, ad-hoc signing: built in about 2 minutes with Command Line Tools only (33 MB bundle), `codesign --verify --deep --strict` passes, build's own backend boot smoke passed | +| Service | launchd LaunchAgent `com.cortex.local.backend` installed; survives `kill -9` (auto-restart in about 15 s, data intact) | +| Backup/restore | `scripts/backup_local.sh` (WAL-safe) and `scripts/restore_local.sh` round-trip verified | +| Restart | stop/start via scripts and via launchd verified; captured data persisted across restarts | +| Docker | `Dockerfile` + `docker-compose.yml` written from the same lockfile; **not executed** (Docker absent on this machine) | +| Node packages | Obsidian plugin, TS SDK, OpenClaw adapter **not rebuilt** (Node absent); committed build output is what the app bundles | + +## 4. Test and gate results + +| Check | Result | +|---|---| +| `python -m compileall backend scripts *.py` | pass | +| `pytest backend/tests` (full, after fixes) | **2152 passed, 0 failed**, 416 subtests, about 6.7 min (before the fix: 2151 passed, 1 failed) | +| `scripts/retrieval_eval.py`, `tool_routing_eval.py`, `adapter_contract_eval.py`, `context_pack_eval.py`, `token_calibration_eval.py`, `agent_task_eval.py --deterministic`, `delivery_eval.py`, `rerank_eval.py` (skips model2vec lift, as in CI), `adaptation_eval.py`, `check_connector_baseline.py`, `check_docs_current.py`, `backend_beta_smoke.py`, `appstore_compliance_lint.py` (46/46) | all pass | +| `bandit -c .bandit.yaml -r backend scripts -ll` | 0 medium/high after fix (was 8 high) | +| `pip-audit --strict` (both requirement files + lockfile) | no known vulnerabilities | +| `scripts/check_distribution_site.py` | pass | +| `scripts/validate_update_manifest.py site/downloads/latest.json` and `ops_readiness_check.py` | pass with the new `--allow-remote-artifacts` mode (site feed binaries live on GitHub Releases; the manifest's sha256 is cross-checked against the committed checksums file). Strict mode, used for packaged release directories, is unchanged. | +| Upstream CI on `main` | failing since 2026-08-05 on the security scan (SHA1 findings, then `pip-audit` on the root `requirements.txt` whose `rumps` dependency cannot build on Linux), the distribution job (DMG not in git) and one `model2vec` test that did not skip without the package. All three are fixed on this branch. | + +## 5. Files changed on `chore/in-house-deployment` + +Added: + +- `scripts/setup_local.sh`, `scripts/start_backend.sh`, `scripts/stop_backend.sh`, + `scripts/health_check.sh`, `scripts/backup_local.sh`, `scripts/restore_local.sh` +- `deploy/launchd/install.sh`, `deploy/launchd/uninstall.sh` +- `Dockerfile`, `docker-compose.yml`, `.dockerignore` +- `backend/requirements.lock.txt` +- `docs/INSTALLATION.md`, `docs/CONFIGURATION.md`, `docs/OPERATIONS.md`, `docs/SECURITY.md`, + `docs/DEPENDENCIES.md`, `docs/IN_HOUSE_MIGRATION_REPORT.md` + +Modified: + +- `.env.example` (full single-user reference with safe placeholders), `.gitignore` (venv, env + files, run state, data, backups, models, SQLite side files, keys) +- `README.md` ("Run it in-house" section), `SETUP.md` (service pointer), `docs/ARCHITECTURE.md` + (deployment topology) +- `backend/app/connectors/agent_sessions.py`, `backend/app/obsidian_writeback.py`, + `backend/app/storage.py`, `backend/app/vault.py`: SHA1 content fingerprints marked + `usedforsecurity=False` (no behaviour change; unblocks the bandit gate) +- `backend/tests/test_macos_ui_quality_contract.py`: assertions updated to the refactored + `AppleSignInSupport` gate in `CortexCloudAuth.swift` (same guarantee, current symbol names) +- `backend/tests/test_rerank_eval.py`: probe the embedder before deciding to skip, so the gate + skips (as CI intends) when `model2vec` is not installed +- `requirements.txt` (legacy prototype): `rumps` limited to macOS so `pip-audit` resolves on Linux +- `scripts/validate_update_manifest.py`, `scripts/ops_readiness_check.py`, `.github/workflows/ci.yml`: + `--allow-remote-artifacts` for the published site feed + +Removed: none. No application architecture, framework or feature was changed. + +## 6. Known issues and blockers + +1. Docker image and Compose stack are untested here (no Docker Engine on this Mac). +2. Node-based packages were not rebuilt; CI's Obsidian plugin job needs Node 22. +3. The macOS dev build runs the backend with whatever `python3.12` is on the GUI `PATH`; a + bundled-Python build needs a Python.framework at `/Library/Frameworks/Python.framework/3.12`. +4. The launchd service and the macOS app both want port 8766; run one, or align vault path and + token (documented in `docs/INSTALLATION.md`). +5. `com.cortex.doppl.ShipIt` (an updater from a previously installed Doppl build) is still + registered with launchd on this machine; it is unrelated to this deployment. +6. Semantic embeddings are off (hash provider) until `scripts/setup_local.sh --with-model` is + run, which downloads about 30 MB from Hugging Face once. + +## 7. Recommended next steps + +- Run `scripts/setup_local.sh --with-model` once to enable on-device semantic search. +- Open a PR from `chore/in-house-deployment` to `main`; it also turns the upstream security CI job + green. +- Schedule `scripts/backup_local.sh` nightly (plist in `docs/OPERATIONS.md`) and copy archives + off-machine. +- Build and run the Docker image on a Linux host to validate the container path. +- Install Node 22 if the Obsidian plugin or TS SDK need rebuilding. +- Decide whether the single-user service should use the app's default vault path so the macOS app + and the service share one memory store. diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md new file mode 100644 index 00000000..51f1a51f --- /dev/null +++ b/docs/OPERATIONS.md @@ -0,0 +1,167 @@ +# Operations runbook (in-house) + +Applies to the single-user, local-first deployment set up by `scripts/setup_local.sh`. The +multi-tenant server profile has its own runbooks in `deploy/README.md`, `deploy/macmini/README.md` +and `docs/OPERATIONAL_READINESS.md`. + +## Layout + +| Path | Contents | +|---|---| +| `.venv/` | Python 3.12 environment from `backend/requirements.lock.txt` | +| `.env` | configuration + API token (mode 600, git-ignored) | +| `backend/data/Cortex.vault/` | the memory store: `captures/`, `memories/`, `entities/`, `events.jsonl`, `settings.json`, `backups/` | +| `backend/data/Cortex.vault/index.sqlite` | rebuildable FTS5 + sqlite-vec index (WAL mode) | +| `.run/backend.pid`, `.run/backend.log` | background process started by `scripts/start_backend.sh` | +| `.run/launchd.out.log`, `.run/launchd.err.log` | service logs when run under launchd | +| `backups/` | archives from `scripts/backup_local.sh` (default destination) | +| `models/model2vec/` | optional on-device embedding model | + +## Start, stop, status + +Manual (foreground-less background process): + +```bash +scripts/start_backend.sh # idempotent; waits for /health +scripts/health_check.sh # 4 checks, exit 0 = healthy +scripts/stop_backend.sh # SIGTERM, then SIGKILL after 15 s +tail -f .run/backend.log +``` + +As a macOS service: + +```bash +deploy/launchd/install.sh # install + start + health check +launchctl list | grep com.cortex.local.backend # " 0 com.cortex.local.backend" = running +launchctl unload ~/Library/LaunchAgents/com.cortex.local.backend.plist # stop +launchctl load -w ~/Library/LaunchAgents/com.cortex.local.backend.plist # start +launchctl kickstart -k gui/$(id -u)/com.cortex.local.backend # restart +deploy/launchd/uninstall.sh # remove the service +``` + +The service has `KeepAlive` set: a crash or `kill -9` is followed by an automatic restart within a +few seconds (verified: recovery after SIGKILL in about 15 s with the default ThrottleInterval, data intact). + +Docker: + +```bash +docker compose up -d --build +docker compose ps +docker compose logs -f api +docker compose restart api +docker compose down # data persists in the cortex-data volume +``` + +## Health contract + +| Endpoint | Auth | Healthy response | +|---|---|---| +| `GET /health` | none | 200, `status: ok`, `health_contract >= 3`, feature list, vault/db paths | +| `GET /ready` | none | 200 with storage diagnostics; 503 `needs_configuration` or a diagnostics payload when the index needs repair | +| `GET /v1/settings` | bearer | 200 proves the token works | +| `GET /v1/diagnostics` | bearer | integrity check: `quick_check`, FTS orphans, relation orphans | +| `GET /v1/reliability/report` | bearer | consolidated reliability report | +| `GET /v1/metrics` | admin | counters/latency when `CORTEX_OBSERVABILITY_ENABLED=1` | + +`scripts/health_check.sh` runs the first three plus a bad-token rejection and is what the launchd +installer and the restart procedure rely on. Wire it into any monitor you already run (cron + +notification, UptimeRobot on the proxy URL, etc.). + +## Backups + +Two complementary mechanisms: + +1. **In-app / API backups** (`POST /v1/backups`, the app's Backup button): writes a full vault zip + under `/backups/` and prunes to `CORTEX_BACKUP_RETENTION_COUNT`. Restore with + `POST /v1/backups/restore-latest` or `scripts/cortex_restore.py` (per-zip, with integrity checks). +2. **File-system backups** (`scripts/backup_local.sh`): WAL-safe snapshot of the index via SQLite's + online backup API plus every vault record and your `.env`, as one `tar.gz`. Runs safely while + the backend is up. Keeps the newest 14 by default. + +Nightly schedule on macOS (user agent, 02:30): + +```bash +cat > ~/Library/LaunchAgents/com.cortex.local.backup.plist < + + + Labelcom.cortex.local.backup + ProgramArguments/bin/bash$HOME/cortex-app/scripts/backup_local.sh + StartCalendarIntervalHour2Minute30 + StandardErrorPath$HOME/cortex-app/.run/backup.err.log + +EOF +launchctl load -w ~/Library/LaunchAgents/com.cortex.local.backup.plist +``` + +Copy archives off the machine (they contain the API token: encrypt or store them where only you +can read them). `rclone`, Time Machine excluding nothing but `.venv`, or any object store works. + +## Restore + +```bash +scripts/stop_backend.sh # or launchctl unload ... +scripts/restore_local.sh backups/cortex-local-.tar.gz --yes +scripts/start_backend.sh # or launchctl load -w ... +scripts/health_check.sh +``` + +The previous vault is kept at `.pre-restore-`; delete it once satisfied. The archived +`.env` is extracted beside the archive for manual comparison, never applied automatically. + +If only the index is damaged, the vault remains authoritative: delete `index.sqlite*` and call +`POST /v1/maintenance/rebuild-index-from-vault` (or restart; the backend rebuilds missing indexes). +`POST /v1/maintenance/repair-storage` performs a backup-first repair, and +`POST /v1/maintenance/rebuild-search` rebuilds FTS/vector tables only. + +## Upgrades + +```bash +scripts/backup_local.sh +scripts/stop_backend.sh || launchctl unload ~/Library/LaunchAgents/com.cortex.local.backend.plist +git fetch && git checkout +scripts/setup_local.sh # reinstalls pinned deps; .env untouched +scripts/start_backend.sh || launchctl load -w ~/Library/LaunchAgents/com.cortex.local.backend.plist +scripts/health_check.sh +``` + +Schema migrations are applied on startup and recorded in `schema_migrations`. Rollback = check out +the previous commit and restore the pre-upgrade backup (migrations are forward-only). + +Dependency updates: edit `backend/requirements.txt` / `backend/runtime-requirements.txt`, then + +```bash +uv pip compile --universal --python-version 3.12 backend/requirements.txt backend/runtime-requirements.txt -o backend/requirements.lock.txt +``` + +and re-run the test suite and `pip-audit -r backend/requirements.lock.txt`. + +## Logs + +| Source | Location | +|---|---| +| background process | `.run/backend.log` (uvicorn access + app logs) | +| launchd service | `.run/launchd.out.log`, `.run/launchd.err.log` | +| audit trail of memory operations | `/events.jsonl`, `GET /v1/audit-log` | +| support bundle (sanitised) | `GET /v1/support/bundle` or `scripts/export_support_bundle.py` | + +Rotate `.run/*.log` with `newsyslog` or logrotate; the backend does not rotate them itself. + +## Troubleshooting + +| Symptom | Cause | Fix | +|---|---|---| +| `RuntimeError: CORTEX_API_KEY uses the insecure sample token` | `.env` still has `dev-local-key` | set a random token (`scripts/setup_local.sh` does) | +| `port 8766 may already be in use` in the macOS app | launchd service (or another backend) owns the port | run one of them, or align vault path + token (see `docs/INSTALLATION.md` section 5) | +| `/ready` returns 503 with `needs_configuration` | server-profile vars half-set (e.g. `CORTEX_HOSTED_RUNTIME_TIER=postgres` without a database URL) | unset them for single-user mode | +| `/ready` 503 with diagnostics | index integrity issue | `POST /v1/maintenance/repair-storage`, then rebuild from vault | +| `vector.available: false` in `check_vector_runtime.py` | `sqlite-vec` could not load (stdlib sqlite without extension support) | ensure `pysqlite3` built: reinstall with a C compiler + SQLite headers present | +| semantic search feels keyword-only | provider is `hash` | `scripts/setup_local.sh --with-model` and restart | +| `SSL: CERTIFICATE_VERIFY_FAILED` on connector sync | Python without a CA bundle | `certifi` is in the lockfile; otherwise set `SSL_CERT_FILE` | +| service restarts in a loop | see `.run/launchd.err.log`; usually a bad `.env` value | fix, then `launchctl kickstart -k gui/$(id -u)/com.cortex.local.backend` | + +## Deleting everything + +`scripts/stop_backend.sh`, `deploy/launchd/uninstall.sh`, then delete the vault directory, +`backups/`, `.run/` and `.env`. `DELETE /v1/user-data` does the in-app equivalent for the memory +store. Nothing else on the machine holds Cortex data. diff --git a/docs/SECURITY.md b/docs/SECURITY.md new file mode 100644 index 00000000..95fac231 --- /dev/null +++ b/docs/SECURITY.md @@ -0,0 +1,78 @@ +# Security: assumptions, findings and hardening for an in-house deployment + +Companion to `docs/SECURITY_REVIEW.md` (the product-level review) and `docs/TRUST_CONTROLS.md` +(per-tool permissions). This document covers what an operator must know to run the backend safely +on their own infrastructure. + +## Threat model for the single-user profile + +- The backend binds to `127.0.0.1:8766`. Anything on the same machine that can reach loopback and + holds the bearer token has full access; nothing off the machine can reach it. +- All memory lives in plain files under the vault plus a SQLite index. Disk encryption (FileVault + on macOS, LUKS on Linux) is the at-rest protection; the backend does not encrypt the vault + itself. Connector credentials are encrypted (CXE1) only when a KEK is configured + (`CORTEX_KEK`/`CORTEX_KEK_FILE`); without one they are stored in `credentials.json` in the vault. +- The API token is the only authentication. Treat `.env` and every backup archive as secrets. + +## Audit results (October 2026, this tree) + +| Check | Result | +|---|---| +| Committed secrets (tokens, keys, certificates) across all tracked files | none found; only test fixtures with obviously fake values (`xoxb-test`, `test-token`) | +| Tracked env/key files | only `.env.example`; `.env`, `*.pem`, `*.key`, `*.p12` are git-ignored | +| `pip-audit --strict` on `backend/requirements.txt`, `backend/runtime-requirements.txt` and the installed lockfile | no known vulnerabilities | +| `bandit -ll` (medium and above) over `backend/` and `scripts/` | 8 high findings (B324, SHA1 used for content fingerprints), fixed by marking them `usedforsecurity=False`; now 0 medium/high. The upstream CI security job had been failing on exactly these since August 2026. | +| Default credentials | the sample `dev-local-key` token is refused at startup unless `CORTEX_ALLOW_INSECURE_DEV_TOKEN=1` | +| Debug/dev endpoints | none exposed without the token; `/capture` HTML page requires `?token=`; `/docs` (OpenAPI) is served by FastAPI mode only and reveals routes, not data | +| Unauthenticated surface | `GET /`, `GET /health`, `GET /ready` (no user data), `/account/*` HTML pages only when accounts are enabled | +| Auth rejection | wrong or missing bearer returns 401 (verified by `scripts/health_check.sh`) | +| Authorization model | scoped tokens (`read`, `write`, `export`, `maintenance`, `destructive`) checked on every call; MCP tool advertisement never grants scope; per-source trust policies and redaction on by default | +| Outbound traffic in the default configuration | none. Anthropic/OpenAI/connector hosts are contacted only when the corresponding key or connector is configured. `certifi` is used for TLS verification; `CORTEX_DISABLE_SSL` exists for packaged-app diagnostics and must stay unset. | +| SSRF | `/v1/delivery/send` refuses private/internal targets and non-HTTPS public targets unless `CORTEX_DELIVERY_ALLOW_INTERNAL=1` | +| SQL injection | all user values are bound parameters; the f-string SQL that bandit skips (B608) interpolates only hard-coded predicate literals (reviewed in `storage.py`) | +| Password hashing (server profile) | argon2id via `argon2-cffi` | + +## Hardening checklist + +1. **Keep loopback.** Do not set `CORTEX_HOST=0.0.0.0` on bare metal. Expose the API to a network + only through a TLS-terminating reverse proxy (Caddy with automatic certificates is provided in + `deploy/Caddyfile` and the Compose `proxy` profile; Cloudflare Tunnel is documented in + `deploy/macmini/README.md`). Bearer tokens over plain HTTP on a LAN are capturable. +2. **One token per client.** Mint scoped MCP tokens (`POST /v1/integrations/mcp-token` or + `CORTEX_MCP_API_KEY`) for AI tools and the browser extension; keep the main token for the app + and operators. Revoke from the app's Connections & Privacy sheet or `GET/DELETE + /v1/integrations/tokens`. +3. **Rotate the main token** by editing `CORTEX_API_KEY` in `.env` and restarting; update the + macOS app's Advanced backend token and any MCP config afterwards. Scoped tokens stored in the + vault are unaffected. +4. **File permissions.** `.env` is created with mode 600; keep the vault directory readable only by + the service user (`chmod -R go-rwx backend/data` or run under a dedicated user/Docker). +5. **Backups are secrets.** `scripts/backup_local.sh` archives include `.env`. Encrypt before + leaving the machine (`age`, `gpg`, or an encrypted bucket). +6. **Firewall.** macOS: the service never listens on a non-loopback interface, so no rule is + required; still enable the application firewall. Linux: `ufw default deny incoming`, allow only + SSH and the proxy ports, as `deploy/bootstrap.sh` does. +7. **Dependencies.** Versions are pinned in `backend/requirements.lock.txt`. Re-run + `pip-audit -r backend/requirements.lock.txt` on each upgrade and after security advisories. +8. **Models and data stay local.** Use the `hash` or `model2vec` embedding provider and leave + `ANTHROPIC_API_KEY`/`OPENAI_API_KEY` unset for a fully offline posture. Source connectors are + opt-in per source. +9. **Server profile extras.** If you enable accounts: generate and escrow the KEK offline, turn off + `CORTEX_AUTH_AUTOVERIFY` before any public exposure, set non-zero rate limits and quotas, + enable Turnstile on signup, and use SMTP with SPF/DKIM for verification mail. +10. **Least privilege for the process.** Run as an unprivileged user (Docker image already does; + `deploy/systemd/cortex-api.service` uses `NoNewPrivileges`, `ProtectSystem=strict`, + `ProtectHome`, `PrivateTmp`). On macOS the LaunchAgent runs as your user; for a shared machine + promote it to a dedicated service account. + +## Residual risks and open items + +- The macOS dev build (`CORTEX_BUNDLE_PYTHON=0`) is ad-hoc signed and runs the backend with the + system `python3.12`; it is for local use only. Distribution requires Developer ID signing and + notarisation (`docs/APPLE_RELEASE.md`). +- `CORTEX_ALLOW_EXTENSION_CORS=1` by default allows browser-extension origins to call the API with + a token; set `0` if you do not use the extension. +- The legacy prototype files at the repository root (`redis_store.py`, `github_store.py`, ...) are + unused by the product and un-audited beyond byte-compilation; do not run them in production. +- bandit's 817 low-severity findings (mostly `assert`, `subprocess` in build/eval scripts and + broad exception handlers) were not individually reviewed. diff --git a/requirements.txt b/requirements.txt index add22c92..8392e6c2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ redis[hiredis]>=5.0.0 voyageai>=0.2.0 requests>=2.31.0 python-dotenv>=1.0.0 -rumps>=0.4.0 +rumps>=0.4.0; sys_platform == "darwin" # macOS menu bar only (pulls pyobjc, which cannot build on Linux) pynput>=1.7.0 pyperclip>=1.8.0 uvicorn>=0.20.0 diff --git a/scripts/backup_local.sh b/scripts/backup_local.sh new file mode 100755 index 00000000..3a975a62 --- /dev/null +++ b/scripts/backup_local.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash +# WAL-safe backup of a single-user local Cortex vault (the mode scripts/start_backend.sh runs). +# +# scripts/backup_local.sh [--dest DIR] [--keep N] +# +# Produces DIR/cortex-local-.tar.gz containing: +# vault/ every user-owned record file (Markdown/JSON) in CORTEX_VAULT_PATH, +# excluding the rebuildable index and the vault's own backups/ folder +# index.sqlite a consistent snapshot taken with SQLite's online backup API +# (copying a live WAL database byte-for-byte is NOT safe) +# env/.env your configuration INCLUDING the API token: the archive is as sensitive +# as the data; store it encrypted/offsite accordingly. +# Keeps the newest N archives (default 14). Safe to run while the backend is up. +# +# For the multi-tenant server profile use deploy/backup.sh instead. +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +ENV_FILE="${CORTEX_ENV_FILE:-$ROOT/.env}" +VENV="${CORTEX_VENV:-$ROOT/.venv}" +DEST="${CORTEX_BACKUP_DIR:-$ROOT/backups}" +KEEP="${CORTEX_BACKUP_KEEP:-14}" +while [[ $# -gt 0 ]]; do + case "$1" in + --dest) DEST="$2"; shift ;; + --keep) KEEP="$2"; shift ;; + -h|--help) sed -n '2,16p' "$0"; exit 0 ;; + *) echo "unknown flag: $1" >&2; exit 2 ;; + esac + shift +done + +if [[ -f "$ENV_FILE" ]]; then + set -a; . "$ENV_FILE"; set +a +fi +PY="$VENV/bin/python"; [[ -x "$PY" ]] || PY="$(command -v python3)" +abs() { "$PY" -c 'import os,sys; print(os.path.abspath(os.path.expanduser(sys.argv[1])))' "$1"; } +VAULT="$(abs "${CORTEX_VAULT_PATH:-$ROOT/backend/data/Cortex.vault}")" +DB="$(abs "${CORTEX_DB_PATH:-$VAULT/index.sqlite}")" +[[ -d "$VAULT" ]] || { echo "ERROR: vault not found at $VAULT" >&2; exit 1; } + +STAMP="$(date -u +%Y%m%d-%H%M%S)" +STAGING="$(mktemp -d)" +trap 'rm -rf "$STAGING"' EXIT +mkdir -p "$DEST" "$STAGING/vault" "$STAGING/env" + +# 1. user-owned records (skip the rebuildable index + nested backup zips) +"$PY" - "$VAULT" "$STAGING/vault" <<'PY' +import os, shutil, sys +src, dst = sys.argv[1:3] +skip_names = {"index.sqlite", "index.sqlite-wal", "index.sqlite-shm", "index.sqlite-journal"} +for dirpath, dirnames, filenames in os.walk(src): + rel = os.path.relpath(dirpath, src) + if rel.split(os.sep)[0] == "backups": + dirnames[:] = [] + continue + out = os.path.join(dst, rel) if rel != "." else dst + os.makedirs(out, exist_ok=True) + for name in filenames: + if name in skip_names: + continue + shutil.copy2(os.path.join(dirpath, name), os.path.join(out, name)) +PY + +# 2. consistent SQLite snapshot via the online backup API +if [[ -f "$DB" ]]; then + "$PY" - "$DB" "$STAGING/index.sqlite" <<'PY' +import sqlite3, sys +src, dst = sys.argv[1:3] +with sqlite3.connect(src) as a, sqlite3.connect(dst) as b: + a.backup(b) +print("index snapshot ok") +PY +fi + +# 3. config (contains the token: treat the archive as secret) +[[ -f "$ENV_FILE" ]] && cp "$ENV_FILE" "$STAGING/env/.env" + +ARCHIVE="$DEST/cortex-local-$STAMP.tar.gz" +tar -czf "$ARCHIVE" -C "$STAGING" . +chmod 600 "$ARCHIVE" +echo "backup written: $ARCHIVE ($(du -h "$ARCHIVE" | cut -f1))" + +# 4. retention +ls -1t "$DEST"/cortex-local-*.tar.gz 2>/dev/null | tail -n +"$((KEEP + 1))" | while read -r old; do rm -f "$old"; done +echo "retained: $(ls -1 "$DEST"/cortex-local-*.tar.gz 2>/dev/null | wc -l | tr -d ' ') archive(s) in $DEST" diff --git a/scripts/health_check.sh b/scripts/health_check.sh new file mode 100755 index 00000000..03818944 --- /dev/null +++ b/scripts/health_check.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +# Health check for a running Cortex backend. +# +# scripts/health_check.sh [--url http://127.0.0.1:8766] [--quiet] +# +# Checks, in order: +# 1. GET /health (unauthenticated; must be 200 and status "ok") +# 2. GET /ready (200 = storage diagnostics ok; 503 = needs attention) +# 3. GET /v1/settings with the token from .env (proves auth works) +# 4. GET /v1/inbox with a bogus token (must be 401; proves auth is enforced) +# Exit 0 only when all four pass. Prints a one-line summary per check. +set -uo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +ENV_FILE="${CORTEX_ENV_FILE:-$ROOT/.env}" +URL="" +QUIET=0 +while [[ $# -gt 0 ]]; do + case "$1" in + --url) URL="$2"; shift ;; + --quiet) QUIET=1 ;; + -h|--help) sed -n '2,12p' "$0"; exit 0 ;; + *) echo "unknown flag: $1" >&2; exit 2 ;; + esac + shift +done + +TOKEN="${CORTEX_API_KEY:-}" +if [[ -f "$ENV_FILE" ]]; then + # Only read the two values we need; do not export the whole file. + [[ -z "$TOKEN" ]] && TOKEN="$(grep -E '^CORTEX_API_KEY=' "$ENV_FILE" | head -1 | cut -d= -f2-)" + PORT_FROM_ENV="$(grep -E '^CORTEX_PORT=' "$ENV_FILE" | head -1 | cut -d= -f2-)" + HOST_FROM_ENV="$(grep -E '^CORTEX_HOST=' "$ENV_FILE" | head -1 | cut -d= -f2-)" +fi +URL="${URL:-http://${HOST_FROM_ENV:-127.0.0.1}:${PORT_FROM_ENV:-8766}}" +URL="${URL%/}" + +say() { [[ "$QUIET" == "1" ]] || echo "$*"; } +fail=0 + +code_and_body() { curl -sS -m 5 -o "$2" -w '%{http_code}' "${@:3}" "$1" 2>/dev/null || echo 000; } + +tmp="$(mktemp)" +trap 'rm -f "$tmp"' EXIT + +c="$(code_and_body "$URL/health" "$tmp")" +if [[ "$c" == "200" ]] && grep -q '"status": *"ok"' "$tmp"; then + say "ok /health 200 (backend_version $(sed -n 's/.*"backend_version": *"\([^"]*\)".*/\1/p' "$tmp" | head -1))" +else + say "FAIL /health -> HTTP $c"; fail=1 +fi + +c="$(code_and_body "$URL/ready" "$tmp")" +if [[ "$c" == "200" ]]; then say "ok /ready 200"; else say "FAIL /ready -> HTTP $c ($(head -c 200 "$tmp"))"; fail=1; fi + +if [[ -n "$TOKEN" ]]; then + c="$(code_and_body "$URL/v1/settings" "$tmp" -H "Authorization: Bearer $TOKEN")" + if [[ "$c" == "200" ]]; then say "ok /v1/settings 200 with configured token"; else say "FAIL /v1/settings with configured token -> HTTP $c"; fail=1; fi +else + say "skip /v1/settings (no CORTEX_API_KEY available)" +fi + +c="$(code_and_body "$URL/v1/inbox" "$tmp" -H "Authorization: Bearer not-a-real-token")" +if [[ "$c" == "401" ]]; then say "ok /v1/inbox rejects a bad token (401)"; else say "FAIL /v1/inbox with bad token -> HTTP $c (expected 401)"; fail=1; fi + +if [[ "$fail" == "0" ]]; then say "HEALTHY $URL"; else say "UNHEALTHY $URL"; fi +exit "$fail" diff --git a/scripts/ops_readiness_check.py b/scripts/ops_readiness_check.py index 00499faf..b0b6ac6d 100644 --- a/scripts/ops_readiness_check.py +++ b/scripts/ops_readiness_check.py @@ -619,7 +619,13 @@ def main() -> None: {"skipped": True, "reason": "local-dmg-only"}, ) else: - site_manifest_result = run_command(root, [sys.executable, "scripts/validate_update_manifest.py", "site/downloads/latest.json"], timeout=60) + # Site feed binaries live on GitHub Releases, not in git: validate the published manifest + # without requiring the DMG/ZIP on disk. The packaged release dir below stays strict. + site_manifest_result = run_command( + root, + [sys.executable, "scripts/validate_update_manifest.py", "--allow-remote-artifacts", "site/downloads/latest.json"], + timeout=60, + ) add_check(checks, "site_update_manifest", site_manifest_result["ok"], "Site update feed validates.", site_manifest_result) strict_package_artifacts = args.include_package or args.require_package_artifacts or release_dir_arg is not None diff --git a/scripts/restore_local.sh b/scripts/restore_local.sh new file mode 100755 index 00000000..8108a3fd --- /dev/null +++ b/scripts/restore_local.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +# Restore a scripts/backup_local.sh archive into the vault from ./.env. +# +# scripts/restore_local.sh [--yes] +# +# Stops nothing by itself: stop the backend first (scripts/stop_backend.sh or launchctl), +# run this, then start it again. The current vault is moved aside to .pre-restore- +# (never deleted) so the operation is reversible. The .env inside the archive is NOT restored +# automatically; it is extracted next to the archive for you to compare. +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +ENV_FILE="${CORTEX_ENV_FILE:-$ROOT/.env}" +VENV="${CORTEX_VENV:-$ROOT/.venv}" +ARCHIVE="${1:-}" +YES="${2:-}" +[[ -f "$ARCHIVE" ]] || { echo "usage: $0 [--yes]" >&2; exit 2; } + +if [[ -f "$ENV_FILE" ]]; then set -a; . "$ENV_FILE"; set +a; fi +PY="$VENV/bin/python"; [[ -x "$PY" ]] || PY="$(command -v python3)" +abs() { "$PY" -c 'import os,sys; print(os.path.abspath(os.path.expanduser(sys.argv[1])))' "$1"; } +VAULT="$(abs "${CORTEX_VAULT_PATH:-$ROOT/backend/data/Cortex.vault}")" +DB="$(abs "${CORTEX_DB_PATH:-$VAULT/index.sqlite}")" + +if [[ -f "$ROOT/.run/backend.pid" ]] && kill -0 "$(cat "$ROOT/.run/backend.pid")" 2>/dev/null; then + echo "ERROR: backend is running (pid $(cat "$ROOT/.run/backend.pid")). Stop it first." >&2 + exit 1 +fi +if curl -fsS -m 2 -o /dev/null "http://${CORTEX_HOST:-127.0.0.1}:${CORTEX_PORT:-8766}/health" 2>/dev/null; then + echo "ERROR: something is still serving on ${CORTEX_HOST:-127.0.0.1}:${CORTEX_PORT:-8766}. Stop it first." >&2 + exit 1 +fi + +STAMP="$(date -u +%Y%m%d-%H%M%S)" +STAGING="$(mktemp -d)" +trap 'rm -rf "$STAGING"' EXIT +tar -xzf "$ARCHIVE" -C "$STAGING" +[[ -d "$STAGING/vault" ]] || { echo "ERROR: archive has no vault/ directory" >&2; exit 1; } + +echo "restore plan:" +echo " archive : $ARCHIVE" +echo " vault : $VAULT (current copy kept at $VAULT.pre-restore-$STAMP)" +echo " index : $DB" +if [[ "$YES" != "--yes" ]]; then + read -r -p "proceed? [y/N] " answer + [[ "$answer" == "y" || "$answer" == "Y" ]] || { echo "aborted"; exit 1; } +fi + +if [[ -d "$VAULT" ]]; then mv "$VAULT" "$VAULT.pre-restore-$STAMP"; fi +mkdir -p "$(dirname "$VAULT")" +cp -R "$STAGING/vault" "$VAULT" +rm -f "$DB" "$DB-wal" "$DB-shm" +if [[ -f "$STAGING/index.sqlite" ]]; then + mkdir -p "$(dirname "$DB")" + cp "$STAGING/index.sqlite" "$DB" + echo "index restored from snapshot" +else + echo "no index snapshot in archive; the backend rebuilds it from the vault on next start" +fi +if [[ -f "$STAGING/env/.env" ]]; then + cp "$STAGING/env/.env" "$ARCHIVE.env.restored" + chmod 600 "$ARCHIVE.env.restored" + echo "archived .env extracted to $ARCHIVE.env.restored (compare with $ENV_FILE manually)" +fi +echo "done. Start the backend again (scripts/start_backend.sh) and run scripts/health_check.sh" diff --git a/scripts/setup_local.sh b/scripts/setup_local.sh new file mode 100755 index 00000000..6dcf8397 --- /dev/null +++ b/scripts/setup_local.sh @@ -0,0 +1,164 @@ +#!/usr/bin/env bash +# One-command, in-house setup for the Cortex backend on macOS or Linux. +# +# scripts/setup_local.sh [--dev] [--with-model] [--python ] +# +# What it does (idempotent, safe to re-run after `git pull`): +# 1. Finds or installs CPython 3.12 (uses `uv` when available, no sudo needed). +# 2. Creates ./.venv and installs backend/requirements.lock.txt (pinned). +# 3. Creates ./.env from .env.example with a freshly generated API token (never overwrites). +# 4. Optionally (--with-model) downloads the on-device embedding model so semantic +# search works offline afterwards. Without it, Cortex uses deterministic hash embeddings. +# 5. Verifies the SQLite + sqlite-vec runtime. +# +# Flags: +# --dev also install pytest, bandit and pip-audit (test + security tooling) +# --with-model download minishlab/potion-base-8M (~30 MB, one-time, needs internet) +# --python P use interpreter P instead of auto-detecting +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +cd "$ROOT" + +PY_VERSION="${CORTEX_PYTHON_VERSION:-3.12}" +VENV="${CORTEX_VENV:-$ROOT/.venv}" +ENV_FILE="${CORTEX_ENV_FILE:-$ROOT/.env}" +MODEL_DIR="${CORTEX_MODEL_DIR:-$ROOT/models/model2vec}" +MODEL_NAME="${CORTEX_MODEL2VEC_MODEL:-minishlab/potion-base-8M}" +DEV=0 +WITH_MODEL=0 +PY_OVERRIDE="" + +while [[ $# -gt 0 ]]; do + case "$1" in + --dev) DEV=1 ;; + --with-model) WITH_MODEL=1 ;; + --python) PY_OVERRIDE="$2"; shift ;; + -h|--help) sed -n '2,20p' "$0"; exit 0 ;; + *) echo "unknown flag: $1" >&2; exit 2 ;; + esac + shift +done + +log() { printf '==> %s\n' "$*"; } + +# --- 1. interpreter -------------------------------------------------------------- +PY="" +if [[ -n "$PY_OVERRIDE" ]]; then + PY="$PY_OVERRIDE" +elif command -v uv >/dev/null 2>&1; then + log "uv found; ensuring CPython $PY_VERSION is installed (no sudo)" + uv python install "$PY_VERSION" >/dev/null 2>&1 || true + PY="$(uv python find "$PY_VERSION")" +else + for candidate in "python$PY_VERSION" python3.13 python3.12 python3.11 python3; do + if command -v "$candidate" >/dev/null 2>&1; then + if "$candidate" -c 'import sys; raise SystemExit(0 if sys.version_info >= (3, 11) else 1)'; then + PY="$(command -v "$candidate")" + break + fi + fi + done +fi +if [[ -z "$PY" ]]; then + cat >&2 <<'EOF' +ERROR: no CPython >= 3.11 found. + Easiest fix (no admin rights): install uv, then re-run this script. + curl -LsSf https://astral.sh/uv/install.sh | sh + Or install Python 3.12 from https://www.python.org/downloads/ and pass --python /path/to/python3.12 +EOF + exit 1 +fi +log "python: $PY ($("$PY" -c 'import platform; print(platform.python_version())'))" + +# --- 2. virtualenv + pinned dependencies -------------------------------------- +if [[ ! -x "$VENV/bin/python" ]]; then + log "creating virtualenv at $VENV" + if command -v uv >/dev/null 2>&1; then + uv venv --python "$PY" "$VENV" >/dev/null + else + "$PY" -m venv "$VENV" + fi +fi +log "installing pinned backend dependencies (backend/requirements.lock.txt)" +if command -v uv >/dev/null 2>&1; then + uv pip install --python "$VENV/bin/python" -q -r backend/requirements.lock.txt + [[ "$DEV" == "1" ]] && uv pip install --python "$VENV/bin/python" -q pytest bandit pip-audit +else + "$VENV/bin/python" -m pip install -q --upgrade pip + "$VENV/bin/python" -m pip install -q -r backend/requirements.lock.txt + [[ "$DEV" == "1" ]] && "$VENV/bin/python" -m pip install -q pytest bandit pip-audit +fi + +# --- 3. .env with a generated token ------------------------------------------------ +if [[ ! -f "$ENV_FILE" ]]; then + log "creating $ENV_FILE from .env.example with a generated API token" + TOKEN="cxa_$("$VENV/bin/python" -c 'import secrets; print(secrets.token_hex(32))')" + "$VENV/bin/python" - "$ROOT/.env.example" "$ENV_FILE" "$TOKEN" <<'PY' +import sys, pathlib +src, dst, token = sys.argv[1:4] +text = pathlib.Path(src).read_text() +text = text.replace("CORTEX_API_KEY=replace-with-a-long-random-token", f"CORTEX_API_KEY={token}") +pathlib.Path(dst).write_text(text) +PY + chmod 600 "$ENV_FILE" +else + log "$ENV_FILE exists; keeping it (secrets are never overwritten)" +fi + +ensure_env() { + # Append KEY=VALUE to the env file only when KEY is absent. Never overwrites. + if ! grep -qE "^$1=" "$ENV_FILE"; then + printf '%s=%s\n' "$1" "$2" >> "$ENV_FILE" + log "added $1 to $(basename "$ENV_FILE")" + fi +} + +# --- 4. optional on-device embedding model --------------------------------------- +if [[ "$WITH_MODEL" == "1" ]]; then + if [[ -f "$MODEL_DIR/config.json" ]]; then + log "embedding model already present at $MODEL_DIR" + else + log "downloading $MODEL_NAME to $MODEL_DIR (one-time, ~30 MB)" + mkdir -p "$(dirname "$MODEL_DIR")" + "$VENV/bin/python" - "$MODEL_NAME" "$MODEL_DIR" <<'PY' +import sys +from model2vec import StaticModel +name, out = sys.argv[1:3] +StaticModel.from_pretrained(name).save_pretrained(out) +print("saved", out) +PY + fi + ensure_env CORTEX_EMBEDDING_PROVIDER model2vec + ensure_env CORTEX_MODEL2VEC_PATH "$MODEL_DIR" + # If the example file set the provider to hash, flip it to model2vec once. + if grep -qE '^CORTEX_EMBEDDING_PROVIDER=hash$' "$ENV_FILE"; then + sed -i.bak 's/^CORTEX_EMBEDDING_PROVIDER=hash$/CORTEX_EMBEDDING_PROVIDER=model2vec/' "$ENV_FILE" && rm -f "$ENV_FILE.bak" + grep -qE '^CORTEX_MODEL2VEC_PATH=' "$ENV_FILE" || printf 'CORTEX_MODEL2VEC_PATH=%s\n' "$MODEL_DIR" >> "$ENV_FILE" + log "switched CORTEX_EMBEDDING_PROVIDER to model2vec" + fi +fi + +# --- 5. runtime verification --------------------------------------------------------- +log "verifying SQLite + sqlite-vec runtime" +if "$VENV/bin/python" scripts/check_vector_runtime.py >/tmp/cortex-vector-check.$$ 2>&1; then + if grep -q '"available": true' /tmp/cortex-vector-check.$$; then + log "sqlite-vec available: vector search enabled" + else + log "WARNING: sqlite-vec not loadable; Cortex will fall back to full-text search only" + fi +else + log "WARNING: vector runtime check failed; see /tmp/cortex-vector-check.$$" +fi +rm -f /tmp/cortex-vector-check.$$ 2>/dev/null || true + +cat <&2; exit 2 ;; + esac + shift +done + +if [[ ! -f "$ENV_FILE" ]]; then + echo "ERROR: $ENV_FILE not found. Run scripts/setup_local.sh first." >&2 + exit 1 +fi +if [[ ! -x "$VENV/bin/python" ]]; then + echo "ERROR: $VENV/bin/python not found. Run scripts/setup_local.sh first." >&2 + exit 1 +fi + +# Load the env file into the process environment (the backend reads only real env vars). +set -a +# shellcheck disable=SC1090 +. "$ENV_FILE" +set +a + +MODE="${MODE:-${CORTEX_SERVER_MODE:-fastapi}}" +HOST="${CORTEX_HOST:-127.0.0.1}" +PORT="${CORTEX_PORT:-8766}" +export CORTEX_PUBLIC_BASE_URL="${CORTEX_PUBLIC_BASE_URL:-http://$HOST:$PORT}" + +if [[ -z "${CORTEX_API_KEY:-}" ]]; then + echo "ERROR: CORTEX_API_KEY is empty in $ENV_FILE." >&2 + exit 1 +fi +if [[ "$CORTEX_API_KEY" == "dev-local-key" && "${CORTEX_ALLOW_INSECURE_DEV_TOKEN:-0}" != "1" ]]; then + echo "ERROR: CORTEX_API_KEY is the insecure sample token. Set a random token (scripts/setup_local.sh does this)." >&2 + exit 1 +fi + +# Resolve the vault/db paths to absolute paths so launchd, cron and the macOS app agree on them. +abs() { "$VENV/bin/python" -c 'import os,sys; print(os.path.abspath(os.path.expanduser(sys.argv[1])))' "$1"; } +export CORTEX_VAULT_PATH="$(abs "${CORTEX_VAULT_PATH:-./backend/data/Cortex.vault}")" +export CORTEX_DB_PATH="$(abs "${CORTEX_DB_PATH:-$CORTEX_VAULT_PATH/index.sqlite}")" +mkdir -p "$CORTEX_VAULT_PATH" +export PYTHONDONTWRITEBYTECODE=1 +export PYTHONUNBUFFERED=1 + +case "$MODE" in + fastapi) + CMD=("$VENV/bin/python" -m uvicorn backend.app.main:app --host "$HOST" --port "$PORT" --workers "${CORTEX_UVICORN_WORKERS:-1}") + [[ "$HOST" != "127.0.0.1" && "$HOST" != "localhost" ]] && CMD+=(--proxy-headers --forwarded-allow-ips="${CORTEX_FORWARDED_ALLOW_IPS:-127.0.0.1}") + WORKDIR="$ROOT" + ;; + standalone) + CMD=("$VENV/bin/python" -m app.standalone_server --host "$HOST" --port "$PORT") + WORKDIR="$ROOT/backend" + ;; + *) + echo "ERROR: unknown mode '$MODE' (fastapi|standalone)" >&2 + exit 2 + ;; +esac + +if [[ "$FOREGROUND" == "1" ]]; then + cd "$WORKDIR" + exec "${CMD[@]}" +fi + +mkdir -p "$RUN_DIR" +if [[ -f "$PIDFILE" ]] && kill -0 "$(cat "$PIDFILE")" 2>/dev/null; then + echo "Cortex backend already running (pid $(cat "$PIDFILE")) on http://$HOST:$PORT" + exit 0 +fi + +( + cd "$WORKDIR" + nohup "${CMD[@]}" >>"$LOGFILE" 2>&1 & + echo $! >"$PIDFILE" +) +PID="$(cat "$PIDFILE")" +echo "starting Cortex backend ($MODE) pid $PID on http://$HOST:$PORT (log: $LOGFILE)" + +for _ in $(seq 1 60); do + if curl -fsS -o /dev/null "http://$HOST:$PORT/health" 2>/dev/null; then + echo "healthy: http://$HOST:$PORT/health" + exit 0 + fi + if ! kill -0 "$PID" 2>/dev/null; then + echo "ERROR: backend exited during startup. Last log lines:" >&2 + tail -n 20 "$LOGFILE" >&2 + rm -f "$PIDFILE" + exit 1 + fi + sleep 1 +done +echo "ERROR: backend did not become healthy within 60s. See $LOGFILE" >&2 +exit 1 diff --git a/scripts/stop_backend.sh b/scripts/stop_backend.sh new file mode 100755 index 00000000..4d187a3a --- /dev/null +++ b/scripts/stop_backend.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# Stop a backend started by scripts/start_backend.sh (background mode). +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +PIDFILE="$ROOT/.run/backend.pid" + +if [[ ! -f "$PIDFILE" ]]; then + echo "no pidfile at $PIDFILE; nothing to stop (launchd-managed? use deploy/launchd/uninstall.sh or launchctl)" + exit 0 +fi + +PID="$(cat "$PIDFILE")" +if ! kill -0 "$PID" 2>/dev/null; then + echo "pid $PID not running; removing stale pidfile" + rm -f "$PIDFILE" + exit 0 +fi + +echo "stopping Cortex backend pid $PID" +kill -TERM "$PID" 2>/dev/null || true +for _ in $(seq 1 15); do + if ! kill -0 "$PID" 2>/dev/null; then + rm -f "$PIDFILE" + echo "stopped" + exit 0 + fi + sleep 1 +done +echo "did not exit after 15s; sending SIGKILL" +kill -KILL "$PID" 2>/dev/null || true +rm -f "$PIDFILE" +echo "stopped (forced)" diff --git a/scripts/validate_update_manifest.py b/scripts/validate_update_manifest.py index 68f06775..d854869a 100755 --- a/scripts/validate_update_manifest.py +++ b/scripts/validate_update_manifest.py @@ -37,7 +37,19 @@ def artifact_path(root: Path, filename: str, url: str) -> Path: return root / filename -def validate(manifest_path: Path) -> dict: +def checksums_from_file(path: Path) -> dict[str, str]: + """Parse a `shasum -a 256` style file into {filename: sha256}.""" + entries: dict[str, str] = {} + if not path.exists(): + return entries + for line in path.read_text(encoding="utf-8").splitlines(): + parts = line.strip().split() + if len(parts) >= 2 and len(parts[0]) == 64: + entries[parts[-1].lstrip("*")] = parts[0].lower() + return entries + + +def validate(manifest_path: Path, *, allow_remote_artifacts: bool = False) -> dict: payload = json.loads(manifest_path.read_text(encoding="utf-8")) missing = sorted(REQUIRED_TOP_LEVEL - set(payload)) if missing: @@ -54,6 +66,8 @@ def validate(manifest_path: Path) -> dict: root = manifest_path.parent seen_kinds: set[str] = set() + remote_artifacts = 0 + published_checksums = checksums_from_file(root / f"{payload['app']}-{payload['version']}-{payload['build']}.checksums.txt") for artifact in artifacts: missing_artifact = sorted(REQUIRED_ARTIFACT - set(artifact)) if missing_artifact: @@ -64,7 +78,22 @@ def validate(manifest_path: Path) -> dict: seen_kinds.add(kind) path = artifact_path(root, str(artifact["filename"]), str(artifact["url"])) if not path.exists(): - raise FileNotFoundError(f"artifact not found: {path}") + if not allow_remote_artifacts: + raise FileNotFoundError(f"artifact not found: {path}") + # The published site feed points at release binaries hosted elsewhere (GitHub + # Releases); the repo keeps only latest.json + the checksums file. Validate what can + # be validated without the binary: an https URL and agreement with the published + # checksums file when it is present. + parsed = urlparse(str(artifact["url"])) + if parsed.scheme != "https": + raise ValueError(f"remote artifact {artifact['filename']} must use an https URL, got {artifact['url']!r}") + expected = published_checksums.get(str(artifact["filename"])) + if expected is not None and expected != str(artifact["sha256"]).lower(): + raise ValueError( + f"sha256 mismatch for {artifact['filename']}: manifest={artifact['sha256']} checksums-file={expected}" + ) + remote_artifacts += 1 + continue size = path.stat().st_size if size != int(artifact["size_bytes"]): raise ValueError(f"size mismatch for {path.name}: manifest={artifact['size_bytes']} actual={size}") @@ -74,15 +103,36 @@ def validate(manifest_path: Path) -> dict: if "dmg" not in seen_kinds: raise ValueError("manifest must include a dmg artifact") + payload["_remote_artifacts"] = remote_artifacts return payload def main() -> None: parser = argparse.ArgumentParser(description="Validate a Cortex update manifest against local release artifacts.") parser.add_argument("manifest", type=Path) + parser.add_argument( + "--allow-remote-artifacts", + action="store_true", + help=( + "Accept artifacts that are not on disk next to the manifest when they point at an https URL " + "(the published site feed). Their sha256 is cross-checked against the release checksums file " + "when present. Release-directory validation must NOT use this flag." + ), + ) args = parser.parse_args() - payload = validate(args.manifest) - print(json.dumps({"status": "ok", "version": payload["version"], "build": payload["build"], "artifacts": len(payload["artifacts"])}, indent=2)) + payload = validate(args.manifest, allow_remote_artifacts=args.allow_remote_artifacts) + print( + json.dumps( + { + "status": "ok", + "version": payload["version"], + "build": payload["build"], + "artifacts": len(payload["artifacts"]), + "remote_artifacts": payload.get("_remote_artifacts", 0), + }, + indent=2, + ) + ) if __name__ == "__main__":