Skip to content

Commit aadfd3f

Browse files
perf: prune vendored dirs in Django urls scan; don't import litellm for availability check
- EndpointDetector._scan_django_urls used Path.rglob over the whole tree (venvs, site-packages, worktrees): ~40s on a large workspace. Use os.walk with pruning of vendored/hidden dirs. - LLMHelper.available/debug log eagerly imported litellm (~12s) even when LLM disabled. Use importlib.util.find_spec; import only on completion. Measured README generation on a 36k-file workspace: 53s -> 28s. Co-authored-by: Koru Agent <agent@coru.dev>
1 parent 4442c48 commit aadfd3f

4 files changed

Lines changed: 85 additions & 3 deletions

File tree

‎code2docs/analyzers/endpoint_detector.py‎

Lines changed: 32 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
"""Detect web framework endpoints (Flask, FastAPI, Django) from AST analysis."""
22

3+
import os
34
import re
45
from dataclasses import dataclass, field
56
from pathlib import Path
@@ -23,6 +24,36 @@ class Endpoint:
2324
return_type: Optional[str] = None
2425

2526

27+
# Directories never searched for Django urls.py: virtualenvs, vendored packages,
28+
# VCS/worktree copies and build output. Walking them dominated runtime (~40s on
29+
# a workspace with vendored site-packages). Any dot-directory is skipped too.
30+
_SKIP_DIRS = frozenset(
31+
{
32+
"venv",
33+
"env",
34+
"node_modules",
35+
"site-packages",
36+
"__pycache__",
37+
"dist",
38+
"build",
39+
"target",
40+
"bower_components",
41+
}
42+
)
43+
44+
45+
def _iter_urls_files(root: Path):
46+
"""Yield ``urls.py`` files under *root*, pruning irrelevant directories."""
47+
for dirpath, dirnames, filenames in os.walk(root):
48+
dirnames[:] = [
49+
d
50+
for d in dirnames
51+
if d not in _SKIP_DIRS and not d.startswith(".") and not d.endswith(".egg-info")
52+
]
53+
if "urls.py" in filenames:
54+
yield Path(dirpath) / "urls.py"
55+
56+
2657
class EndpointDetector:
2758
"""Detects web endpoints from decorator patterns in source code."""
2859

@@ -91,7 +122,7 @@ def _scan_django_urls(self, project_path: str) -> List[Endpoint]:
91122
endpoints: List[Endpoint] = []
92123
project = Path(project_path)
93124

94-
for urls_file in project.rglob("urls.py"):
125+
for urls_file in _iter_urls_files(project):
95126
try:
96127
source = urls_file.read_text(encoding="utf-8")
97128
for match in self.DJANGO_URL_PATTERN.finditer(source):

‎code2docs/llm_helper.py‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
and generators fall back to algorithm-based template output.
55
"""
66

7+
import importlib.util
78
import logging
89
from typing import Optional
910

@@ -29,6 +30,18 @@ def _get_litellm():
2930
return _litellm if _litellm is not False else None
3031

3132

33+
def _litellm_installed() -> bool:
34+
"""Cheap availability check: locate litellm without importing it.
35+
36+
Importing litellm costs ~10s, so it must only happen when a completion is
37+
actually requested (see ``_get_litellm`` in ``LLMHelper.complete``).
38+
"""
39+
try:
40+
return importlib.util.find_spec("litellm") is not None
41+
except (ImportError, ValueError):
42+
return False
43+
44+
3245
class LLMHelper:
3346
"""Thin wrapper around litellm for documentation generation.
3447
@@ -45,7 +58,7 @@ def available(self) -> bool:
4558
"""Check if LLM is configured and litellm is installed."""
4659
if self._available is None:
4760
self._available = (
48-
self.config.enabled and bool(self.config.model) and _get_litellm() is not None
61+
self.config.enabled and bool(self.config.model) and _litellm_installed()
4962
)
5063
if self._available:
5164
logger.info("LLM enabled: model=%s", self.config.model)
@@ -54,7 +67,7 @@ def available(self) -> bool:
5467
"LLM disabled (enabled=%s, model=%s, litellm=%s)",
5568
self.config.enabled,
5669
bool(self.config.model),
57-
_get_litellm() is not None,
70+
_litellm_installed(),
5871
)
5972
return self._available
6073

‎tests/test_analyzers.py‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -136,3 +136,17 @@ def test_no_match(self):
136136
)
137137
ep = detector._parse_decorator(func.decorators[0], func)
138138
assert ep is None
139+
140+
141+
def test_endpoint_detector_skips_vendored_and_hidden_dirs(tmp_path):
142+
from code2docs.analyzers.endpoint_detector import EndpointDetector
143+
144+
(tmp_path / "app").mkdir()
145+
(tmp_path / "app" / "urls.py").write_text("urlpatterns = [path('real/', v)]\n")
146+
for skipped in (".subactor/lib/site-packages/x", "venv/lib", "node_modules/y", ".worktrees/t1"):
147+
d = tmp_path / skipped
148+
d.mkdir(parents=True)
149+
(d / "urls.py").write_text("urlpatterns = [path('vendored/', v)]\n")
150+
151+
paths = [e.path for e in EndpointDetector()._scan_django_urls(str(tmp_path))]
152+
assert paths == ["real/"]

‎tests/test_llm_helper.py‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -270,3 +270,27 @@ def test_getting_started_without_llm(self):
270270
content = gen.generate()
271271
assert "Getting Started" in content
272272
assert "Installation" in content
273+
274+
275+
# ── Performance regression tests ─────────────────────────────────────────
276+
277+
278+
class TestLazyLitellmImport:
279+
"""Availability checks must not import litellm (~10s)."""
280+
281+
def test_disabled_helper_never_imports_litellm(self):
282+
from code2docs import llm_helper
283+
from code2docs.config import LLMConfig
284+
285+
with patch("code2docs.llm_helper._get_litellm") as get_litellm:
286+
assert LLMHelper(LLMConfig(enabled=False)).available is False
287+
get_litellm.assert_not_called()
288+
289+
def test_enabled_availability_uses_find_spec_not_import(self):
290+
from code2docs.config import LLMConfig
291+
292+
with patch("code2docs.llm_helper._litellm_installed", return_value=True), patch(
293+
"code2docs.llm_helper._get_litellm"
294+
) as get_litellm:
295+
assert LLMHelper(LLMConfig(enabled=True, model="x")).available is True
296+
get_litellm.assert_not_called()

0 commit comments

Comments
 (0)