Add in-house deployment kit, docs and lockfile; fix bandit gate and stale UI test

.dockerignore

@@ -0,0 +1,27 @@
+.git
+.github
+.venv
+venv
+.run
+.env
+.env.*
+!.env.example
+backups
+backend/data
+models
+macos/build
+outputs
+release-artifacts
+site
+docs
+extension
+packages
+sdk
+status
+**/__pycache__
+**/*.pyc
+*.sqlite
+*.sqlite-wal
+*.sqlite-shm
+.DS_Store
+.pytest_cache

.env.example

@@ -1,45 +1,121 @@
-# Cortex local backend
-# Copy to .env only when you run the backend outside the packaged macOS app.
+# Cortex backend configuration (single-user, local-first profile).
+#
+# Copy to .env (scripts/setup_local.sh does this and fills in a random token).
+# The backend reads ONLY real environment variables; scripts/start_backend.sh, the launchd
+# service and docker-compose export this file for you. NEVER commit .env.
+# Full reference: docs/CONFIGURATION.md. Multi-tenant server profile: deploy/cortex.env.example.
 
-# Local API auth. Use a long random token. The packaged app generates one.
+# ------------------------------------------------------------------------------------------
+# Required
+# ------------------------------------------------------------------------------------------
+# Bearer token for every REST call. Use a long random value (the setup script generates one;
+# the packaged macOS app generates its own per install).
 CORTEX_API_KEY=replace-with-a-long-random-token
 
-# Only set this when intentionally running local development scripts with the
-# sample dev-local-key token.
+# Only set to 1 when intentionally running development scripts with the sample
+# "dev-local-key" token. The backend refuses to start with that token otherwise.
 CORTEX_ALLOW_INSECURE_DEV_TOKEN=0
 
-# Optional MCP auth for local AI-tool integrations. REST endpoints still require
-# CORTEX_API_KEY; /mcp accepts either the app token or this scoped MCP token.
-CORTEX_MCP_API_KEY=
-CORTEX_MCP_API_KEY_SCOPES=read,write,export,maintenance
-
-# Optional local sync manifest signing. When set, device-bound
-# /v1/sync/changes responses include an HMAC signature for integrity checks.
-CORTEX_SYNC_SIGNING_KEY=
+# ------------------------------------------------------------------------------------------
+# Storage (user-owned vault + rebuildable SQLite index)
+# ------------------------------------------------------------------------------------------
+# To share one memory store with the packaged macOS app, point both at the app's default:
+#   CORTEX_VAULT_PATH=~/Library/Application Support/Cortex/Cortex.vault
+CORTEX_VAULT_PATH=./backend/data/Cortex.vault
+CORTEX_DB_PATH=./backend/data/Cortex.vault/index.sqlite
 
-# Local backup retention. Count pruning is enabled by default; age pruning is
-# opt-in because users may depend on older local recovery archives.
+# Local backup retention (vault/backups/*.zip created by POST /v1/backups or the app).
+# Count pruning is on by default; age pruning (days) is opt-in.
 CORTEX_BACKUP_RETENTION_COUNT=20
 CORTEX_BACKUP_RETENTION_DAYS=0
 
-# Local vault and rebuildable SQLite index.
-CORTEX_VAULT_PATH=./backend/data/Cortex.vault
-CORTEX_DB_PATH=./backend/data/Cortex.vault/index.sqlite
-
-# Dev server binding. Conductor workspaces can override CORTEX_PORT.
+# ------------------------------------------------------------------------------------------
+# Network
+# ------------------------------------------------------------------------------------------
+# Keep 127.0.0.1 unless a TLS reverse proxy sits in front (see docs/SECURITY.md).
 CORTEX_HOST=127.0.0.1
 CORTEX_PORT=8766
 CORTEX_PUBLIC_BASE_URL=http://127.0.0.1:8766
 CORTEX_CORS_ORIGINS=http://127.0.0.1:8766,http://localhost:8766
+# The browser extension needs CORS from extension origins; set 0 if you do not use it.
+CORTEX_ALLOW_EXTENSION_CORS=1
+# uvicorn worker processes (scripts/start_backend.sh --mode fastapi). Keep 1 for a
+# single-user SQLite vault; the in-process rate limiter is per worker.
+CORTEX_UVICORN_WORKERS=1
+# Server flavour used by scripts/start_backend.sh and deploy/launchd: fastapi | standalone
+CORTEX_SERVER_MODE=fastapi
 
-# Optional extraction provider. If unset, Cortex uses deterministic local extraction.
-ANTHROPIC_API_KEY=
-CORTEX_EXTRACTION_MODEL=claude-opus-4-5
+# ------------------------------------------------------------------------------------------
+# AI-tool (MCP) access
+# ------------------------------------------------------------------------------------------
+# Optional separate, scoped token for /mcp so MCP configs never carry the app token.
+# REST endpoints still require CORTEX_API_KEY; /mcp accepts either.
+CORTEX_MCP_API_KEY=
+CORTEX_MCP_API_KEY_SCOPES=read,write,export,maintenance
+# Which MCP tool list is advertised by default: core (curated) | full | chatgpt
+CORTEX_MCP_TOOL_SURFACE=core
+# Streamable-HTTP compatibility for /mcp (SSE + Mcp-Session-Id). 0 forces plain JSON.
+CORTEX_MCP_STREAMABLE=1
 
-# Optional embedding provider. The default stays offline/local with a
-# deterministic hash embedding; OpenAI can be enabled for higher recall.
+# ------------------------------------------------------------------------------------------
+# Retrieval quality (all local; no network)
+# ------------------------------------------------------------------------------------------
+# hash      = deterministic offline embeddings (default; works with nothing installed)
+# model2vec = real on-device semantic embeddings (run scripts/setup_local.sh --with-model)
+# openai    = remote OpenAI embeddings (needs OPENAI_API_KEY; data leaves the machine)
 CORTEX_EMBEDDING_PROVIDER=hash
+CORTEX_MODEL2VEC_PATH=
+CORTEX_MODEL2VEC_MODEL=minishlab/potion-base-8M
+# Used only by the openai provider. Keep 384 for the hash provider's sqlite-vec schema.
 CORTEX_EMBEDDING_MODEL=text-embedding-3-small
 CORTEX_EMBEDDING_DIMENSIONS=384
 CORTEX_EMBEDDING_STRICT=0
 OPENAI_API_KEY=
+# Reranker + query planning + temporal decay (same flags the shipped app uses).
+CORTEX_RERANK=linear+mmr
+CORTEX_QUERY_PLAN=1
+CORTEX_TEMPORAL_DECAY=1
+
+# ------------------------------------------------------------------------------------------
+# Extraction (optional LLM condenser)
+# ------------------------------------------------------------------------------------------
+# Unset = deterministic local extractor (capture/search work fully offline).
+ANTHROPIC_API_KEY=
+CORTEX_EXTRACTION_MODEL=claude-opus-4-5
+
+# ------------------------------------------------------------------------------------------
+# Behaviour
+# ------------------------------------------------------------------------------------------
+# 1 = new captures skip the Review inbox and are retrievable immediately.
+CORTEX_AUTO_APPROVE_CAPTURES=0
+# Brand used in user-visible backend strings: Cortex | Doppl
+CORTEX_APP_BRAND=Cortex
+# Background job worker: inline (default; jobs run inside the API process) | external
+# (run scripts/run_memory_worker.py separately; used by the multi-tenant profile).
+CORTEX_WORKER_MODE=inline
+# Operator metrics at GET /v1/metrics (admin token). 0 = off.
+CORTEX_OBSERVABILITY_ENABLED=0
+
+# ------------------------------------------------------------------------------------------
+# Fair-use limits (0 = disabled; the multi-tenant profile sets non-zero values)
+# ------------------------------------------------------------------------------------------
+CORTEX_RATE_LIMIT_PER_MINUTE=0
+CORTEX_DEFAULT_MEMORY_QUOTA=0
+CORTEX_MAX_CONCURRENT_REQUESTS=8
+CORTEX_RATE_LIMIT_RPS=20
+CORTEX_RATE_LIMIT_BURST=60
+
+# ------------------------------------------------------------------------------------------
+# Sync integrity (optional)
+# ------------------------------------------------------------------------------------------
+# When set, device-bound /v1/sync/changes responses carry an HMAC signature.
+CORTEX_SYNC_SIGNING_KEY=
+
+# ------------------------------------------------------------------------------------------
+# Multi-tenant / hosted server profile (leave unset for single-user local mode)
+# ------------------------------------------------------------------------------------------
+# See deploy/cortex.env.example for the full server profile: CORTEX_SHARD_MODE=bucket,
+# CORTEX_SHARD_ROOT, CORTEX_REQUIRE_SCOPED_API_TOKENS=1, CORTEX_AUTH_ENABLED=1, CORTEX_KEK_FILE,
+# CORTEX_SMTP_*, CORTEX_OIDC_*_CLIENT_ID/SECRET, CORTEX_BILLING_PROVIDER, CORTEX_TURNSTILE_*.
+# Docker proxy profile only:
+# CORTEX_DOMAIN=cortex.example.internal

.github/workflows/ci.yml

@@ -156,7 +156,9 @@ jobs:
         run: python scripts/check_distribution_site.py
 
       - name: Validate site update manifest
-        run: python scripts/validate_update_manifest.py site/downloads/latest.json
+        # The site feed points at binaries on GitHub Releases (only latest.json + checksums are in git),
+        # so validate structure + https URLs + checksum-file agreement; release packaging stays strict.
+        run: python scripts/validate_update_manifest.py --allow-remote-artifacts site/downloads/latest.json
 
   obsidian-plugin:
     name: Obsidian plugin build

.gitignore

@@ -1,12 +1,39 @@
+# secrets and local configuration
 .env
-.context/
+.env.*
+!.env.example
+*.pem
+*.p12
+*.key
+
+# python
+.venv/
 venv/
 __pycache__/
 *.pyc
-.DS_Store
-node_modules/
-npm-debug.log*
+.pytest_cache/
+.mypy_cache/
+.ruff_cache/
+
+# local runtime state, data, backups
+.run/
+.context/
 backend/data/
+backups/
+models/
+*.sqlite
+*.sqlite-wal
+*.sqlite-shm
+*.sqlite-journal
+
+# build outputs
 macos/build/
 outputs/
 cortex-agent-package.zip
+node_modules/
+npm-debug.log*
+dist/
+dist-test/
+
+# os
+.DS_Store

Dockerfile

@@ -0,0 +1,47 @@
+# Cortex backend (FastAPI engine) for Linux hosts. The macOS app is native and is NOT
+# containerised; this image serves the HTTP/MCP API that the app, SDKs, browser extension
+# and MCP clients talk to.
+#
+#   docker build -t cortex-backend .
+#   docker run --rm -p 127.0.0.1:8766:8766 --env-file .env -v cortex-data:/data cortex-backend
+#
+# Design notes
+# - Pinned dependencies from backend/requirements.lock.txt (same set as the local venv).
+# - pysqlite3 is compiled against the distro's libsqlite3 so FTS5 + sqlite-vec extension
+#   loading work regardless of the base image's bundled Python sqlite build.
+# - Runs as a non-root user; data lives in /data (mount a volume); loopback-only publish is
+#   the responsibility of the compose file / host (see docker-compose.yml).
+FROM python:3.12-slim-bookworm AS build
+
+ENV PIP_NO_CACHE_DIR=1 PIP_DISABLE_PIP_VERSION_CHECK=1 PYTHONDONTWRITEBYTECODE=1
+RUN apt-get update \
+ && apt-get install -y --no-install-recommends build-essential libsqlite3-dev \
+ && rm -rf /var/lib/apt/lists/*
+COPY backend/requirements.lock.txt /tmp/requirements.lock.txt
+RUN python -m venv /opt/venv \
+ && /opt/venv/bin/pip install -r /tmp/requirements.lock.txt
+
+FROM python:3.12-slim-bookworm
+ENV PATH=/opt/venv/bin:$PATH \
+    PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1 \
+    CORTEX_HOST=0.0.0.0 \
+    CORTEX_PORT=8766 \
+    CORTEX_VAULT_PATH=/data/Cortex.vault \
+    CORTEX_DB_PATH=/data/Cortex.vault/index.sqlite
+RUN apt-get update \
+ && apt-get install -y --no-install-recommends curl libsqlite3-0 sqlite3 \
+ && rm -rf /var/lib/apt/lists/* \
+ && groupadd --system cortex && useradd --system --gid cortex --home /srv/cortex --shell /usr/sbin/nologin cortex \
+ && mkdir -p /srv/cortex /data && chown -R cortex:cortex /srv/cortex /data
+COPY --from=build /opt/venv /opt/venv
+WORKDIR /srv/cortex
+COPY --chown=cortex:cortex backend ./backend
+COPY --chown=cortex:cortex scripts ./scripts
+COPY --chown=cortex:cortex spec ./spec
+USER cortex
+VOLUME ["/data"]
+EXPOSE 8766
+HEALTHCHECK --interval=30s --timeout=5s --start-period=20s --retries=3 \
+  CMD curl -fsS "http://127.0.0.1:${CORTEX_PORT}/health" >/dev/null || exit 1
+CMD ["sh", "-c", "exec uvicorn backend.app.main:app --host \"$CORTEX_HOST\" --port \"$CORTEX_PORT\" --workers \"${CORTEX_UVICORN_WORKERS:-1}\" --proxy-headers --forwarded-allow-ips=\"${CORTEX_FORWARDED_ALLOW_IPS:-*}\""]

README.md

@@ -108,6 +108,22 @@ the smallest cited, model-calibrated context that answers the task.
 
 Cortex checks for updates on its own, so once you're on a recent build, new releases arrive automatically.
 
+## Run it in-house (your own machine or server)
+
+Everything runs on hardware you control with no account or cloud service:
+
+```bash
+scripts/setup_local.sh --with-model   # Python 3.12 venv, pinned deps, generated token, offline embeddings
+scripts/start_backend.sh              # backend on 127.0.0.1:8766
+scripts/health_check.sh               # health, readiness, auth
+deploy/launchd/install.sh             # macOS: always-on service that restarts on crash
+```
+
+Linux hosts: `docker compose up -d --build`. Details, upgrades, backups and hardening:
+[`docs/INSTALLATION.md`](docs/INSTALLATION.md), [`docs/CONFIGURATION.md`](docs/CONFIGURATION.md),
+[`docs/OPERATIONS.md`](docs/OPERATIONS.md), [`docs/SECURITY.md`](docs/SECURITY.md),
+[`docs/DEPENDENCIES.md`](docs/DEPENDENCIES.md).
+
 ## Build from source
 
 ```bash

SETUP.md

@@ -66,3 +66,9 @@ python3 scripts/adaptation_eval.py
 codesign --verify --deep --strict --verbose=2 macos/build/Cortex.app
 python3 scripts/ops_readiness_check.py
 ```
+
+## Run The Backend As An In-House Service
+
+`scripts/setup_local.sh`, `scripts/start_backend.sh`, `scripts/health_check.sh` and
+`deploy/launchd/install.sh` run the backend on this machine without the packaged app. See
+`docs/INSTALLATION.md` and `docs/OPERATIONS.md`.

backend/app/connectors/agent_sessions.py

@@ -254,7 +254,7 @@ def _claude_records(root: Path, path: Path, *, session_cap: int) -> list[AgentSe
         cleaned = _clean_user_text(text)
         if not cleaned:
             continue
-        fingerprint = sha1(cleaned.encode("utf-8")).hexdigest()
+        fingerprint = sha1(cleaned.encode("utf-8"), usedforsecurity=False).hexdigest()
         if fingerprint in seen_texts:
             continue
         seen_texts.add(fingerprint)
@@ -316,7 +316,7 @@ def _codex_records(root: Path, path: Path, *, session_cap: int) -> list[AgentSes
         cleaned = _clean_user_text(str(payload.get("message") or ""))
         if not cleaned:
             continue
-        fingerprint = sha1(cleaned.encode("utf-8")).hexdigest()
+        fingerprint = sha1(cleaned.encode("utf-8"), usedforsecurity=False).hexdigest()
         if fingerprint in seen_texts:
             continue
         seen_texts.add(fingerprint)
@@ -370,7 +370,7 @@ def _cursor_records(root: Path, path: Path, *, mtime: float, session_cap: int) -
         cleaned = _clean_user_text(str(prompt.get("text") or ""))
         if not cleaned:
             continue
-        fingerprint = sha1(cleaned.encode("utf-8")).hexdigest()
+        fingerprint = sha1(cleaned.encode("utf-8"), usedforsecurity=False).hexdigest()
         if fingerprint in seen_texts:
             continue
         seen_texts.add(fingerprint)
@@ -465,7 +465,7 @@ def _stable_external_id(agent: str, session_id: str, anchor: str) -> str:
     raw = f"{agent}:{session_id}:{anchor}"
     if len(raw) <= 240:
         return raw
-    digest = sha1(raw.encode("utf-8")).hexdigest()[:16]
+    digest = sha1(raw.encode("utf-8"), usedforsecurity=False).hexdigest()[:16]
     return f"{raw[:220]}#{digest}"
 
 

backend/app/obsidian_writeback.py

@@ -71,7 +71,7 @@ def safe_page_stem(name: str, fallback: str = "page") -> str:
 def person_page_filename(entity_id: str, name: str) -> str:
     """``<slug>--<hash12>.md`` — slug for human browsing, hash so the page survives label
     edits/collisions (same scheme as the native vault's entity MOC pages)."""
-    short = hashlib.sha1(str(entity_id or "").encode("utf-8")).hexdigest()[:12]
+    short = hashlib.sha1(str(entity_id or "").encode("utf-8"), usedforsecurity=False).hexdigest()[:12]
     return f"{safe_page_stem(name, 'person')}--{short}.md"
 
 

backend/app/storage.py

@@ -14898,7 +14898,7 @@ def _fnv(text: str) -> int:
                 continue
             seen_edges.add(key)
             canvas_edges.append({
-                "id": hashlib.sha1(("|".join(key)).encode("utf-8")).hexdigest()[:16],
+                "id": hashlib.sha1(("|".join(key)).encode("utf-8"), usedforsecurity=False).hexdigest()[:16],
                 "fromNode": self.vault.entity_moc_short_id(key[0]),
                 "toNode": self.vault.entity_moc_short_id(key[1]),
             })

backend/app/vault.py

@@ -671,7 +671,7 @@ def memory_note_short_id(self, memory_id: str) -> str:
         summary/content) so it is invariant across edits — that is what makes glob-by-suffix
         reliable. Sweeps that mutate/delete also confirm the parsed frontmatter id, so even an
         (astronomically unlikely) collision can at worst rewrite the same logical note."""
-        return hashlib.sha1(str(memory_id or "").encode("utf-8")).hexdigest()[:12]
+        return hashlib.sha1(str(memory_id or "").encode("utf-8"), usedforsecurity=False).hexdigest()[:12]
 
     def memory_note_stem(self, record: dict[str, Any]) -> str:
         """`<summary-slug>--<shortid>` — the SINGLE source of both the on-disk note filename and the
@@ -812,7 +812,7 @@ def entity_moc_short_id(self, entity_id: str) -> str:
         disambiguator and by the stale-page sweep. 16 hex = 64 bits: birthday-collision-safe well
         past any realistic entity count (an 8-hex/32-bit id collides around tens of thousands of
         entities and would clobber another entity's page)."""
-        return hashlib.sha1(str(entity_id or "").encode("utf-8")).hexdigest()[:16]
+        return hashlib.sha1(str(entity_id or "").encode("utf-8"), usedforsecurity=False).hexdigest()[:16]
 
     def entity_moc_stem(self, entity_id: str, label: str | None = None) -> str:
         """The MOC note filename stem — the SINGLE source of the entity->entity wikilink target and