diff --git a/.github/workflows/build-portable-windows.yml b/.github/workflows/build-portable-windows.yml index a21c5b6..36ba45a 100644 --- a/.github/workflows/build-portable-windows.yml +++ b/.github/workflows/build-portable-windows.yml @@ -76,6 +76,7 @@ jobs: if (-not (Test-Path "$bundle/pgsql/bin/pg_ctl.exe")) { throw "Missing portable pg_ctl.exe" } if (-not (Test-Path "$bundle/pgsql/lib/vector.dll")) { throw "Missing compiled pgvector vector.dll" } if (-not (Test-Path "$bundle/pgsql/share/extension/vector.control")) { throw "Missing pgvector extension control file" } + if (-not (Get-ChildItem "$bundle/grammars" -Recurse -Filter "tree_sitter_python.dll" -ErrorAction SilentlyContinue)) { throw "Missing pre-fetched tree-sitter grammars" } if (-not (Test-Path "$bundle/start-contextcortex.bat")) { throw "Missing start-contextcortex.bat" } Write-Host "All portable bundle artifacts verified!" -ForegroundColor Green diff --git a/docs/guide/getting-started.md b/docs/guide/getting-started.md index ccbf5df..f55a021 100644 --- a/docs/guide/getting-started.md +++ b/docs/guide/getting-started.md @@ -78,6 +78,18 @@ On first start, the bundle does these steps: All data stays in `.\data\`. To use a different folder, set `DATA_DIR` in `.env`. The launcher works with Windows PowerShell 5.1 and with PowerShell 7. +The bundle contains the tree-sitter grammar DLLs in `.\grammars\`. The app loads them from this folder and does not download grammars at runtime. To use a different folder, set `TREE_SITTER_LANGUAGE_PACK_CACHE_DIR` in `.env`. + +> [!WARNING] +> **Error "start-contextcortex.ps1 is not digitally signed":** Windows marks files that you extract from a downloaded zip as "from the Internet". If Group Policy sets the execution policy to `RemoteSigned` or stricter, PowerShell does not run the marked launcher script, and `-ExecutionPolicy Bypass` cannot change this. Unblock the zip **before** you extract it (right-click the zip, select **Properties**, then select **Unblock**), or run this command in the bundle folder: +> ```powershell +> Get-ChildItem -Recurse -File | Unblock-File +> ``` +> If Group Policy sets `AllSigned`, you must sign the script or ask your administrator. + +> [!NOTE] +> **Endpoint protection (for example Carbon Black):** If your endpoint protection blocks DLLs, keep the bundle in a folder where it allows executables and DLLs. All the DLLs that the bundle loads are in its own folder (`_internal\`, `pgsql\`, and `grammars\`). + ### Windows Portable Database & Vector Store Sub-Configurations Edit `dist\contextcortex-windows-x64\.env` before running `start-contextcortex.bat` to choose how the standalone bundle stores metadata and vectors: diff --git a/main.py b/main.py index 99775b1..cd5d31c 100644 --- a/main.py +++ b/main.py @@ -33,6 +33,11 @@ def _apply_frozen_bundle_defaults() -> None: # The portable bundle does not include FastEmbed/ONNX, so embeddings must use the API gateway. os.environ.setdefault("EMBEDDING_PROVIDER", "api") os.environ.setdefault("HOST", "127.0.0.1") + # tree-sitter-language-pack loads grammar DLLs from its cache folder (default: + # %LOCALAPPDATA%\tree-sitter-language-pack). Endpoint protection often blocks + # DLLs there, and offline machines cannot download them. The build script + # pre-fetches the grammars into .\grammars next to the exe; use that folder. + os.environ.setdefault("TREE_SITTER_LANGUAGE_PACK_CACHE_DIR", os.path.join(exe_dir, "grammars")) os.makedirs(os.environ["VAULT_PATH"], exist_ok=True) diff --git a/scripts/build_portable_windows.ps1 b/scripts/build_portable_windows.ps1 index 7a6dc8f..0219188 100644 --- a/scripts/build_portable_windows.ps1 +++ b/scripts/build_portable_windows.ps1 @@ -46,6 +46,31 @@ if (Test-Path $BundleDir) { New-Item -ItemType Directory -Path $BundleDir | Out-Null Copy-Item -Path "$FrozenDir\*" -Destination $BundleDir -Recurse -Force +# 3b. Pre-fetch tree-sitter grammar DLLs into .\grammars (the frozen app sets +# TREE_SITTER_LANGUAGE_PACK_CACHE_DIR to this folder). The bundle then needs no +# runtime download and loads no DLLs from %LOCALAPPDATA%. The language list comes +# from EXTENSION_TO_LANGUAGE in app/services/chunking/tree_sitter_loader.py. +Write-Host "Pre-fetching tree-sitter grammars into the bundle..." -ForegroundColor Cyan +$GrammarsDir = Join-Path $BundleDir "grammars" +$env:TREE_SITTER_LANGUAGE_PACK_CACHE_DIR = $GrammarsDir +python -c @" +import tree_sitter_language_pack as t +from app.services.chunking.tree_sitter_loader import EXTENSION_TO_LANGUAGE +langs = sorted({l for l in EXTENSION_TO_LANGUAGE.values() if l != 'text'}) +# prefetch() accepts only download-manifest names (for example 'csharp', not 'c_sharp') +manifest = set(t.manifest_languages()) +fetch = sorted({l if l in manifest else l.replace('_', '') for l in langs}) +missing = [l for l in fetch if l not in manifest] +if missing: + raise SystemExit('Grammars not in the download manifest: %s' % missing) +t.prefetch(fetch) +for l in langs: + t.get_parser(l) +print('Pre-fetched %d grammars into %s' % (len(fetch), t.cache_dir())) +"@ +if ($LASTEXITCODE -ne 0) { throw "tree-sitter grammar pre-fetch failed" } +Remove-Item Env:\TREE_SITTER_LANGUAGE_PACK_CACHE_DIR + # 4. Download & Extract Portable PostgreSQL Windows x64 Binaries $TempDir = Join-Path $env:TEMP "cc_pg_build" if (Test-Path $TempDir) { diff --git a/scripts/portable/.env.example b/scripts/portable/.env.example index 8c86d86..dab62ad 100644 --- a/scripts/portable/.env.example +++ b/scripts/portable/.env.example @@ -32,6 +32,11 @@ EMBEDDING_MODEL=text-embedding-3-small # Optional extra HTTP headers on every gateway request (JSON object, no outer quotes) # LLM_EXTRA_HEADERS={"X-User-Agent":"contextcortex","X-User-Name":"jdoe"} +# Folder for tree-sitter grammar DLLs (default: .\grammars next to contextcortex.exe, +# pre-filled by the build). Keep it inside an allowed folder if endpoint protection +# blocks DLLs in other locations. +# TREE_SITTER_LANGUAGE_PACK_CACHE_DIR=D:\ContextCortex\grammars + # Corporate CA certificates: put the .crt or .pem files in .\certs\ # The launcher adds them to the public root CAs and sets SSL_CERT_FILE, # REQUESTS_CA_BUNDLE, and GIT_SSL_CAINFO. Set SSL_CERT_FILE here only to