From 0a5340f2e47467b59b449f9e23f0d2cada95ba67 Mon Sep 17 00:00:00 2001 From: Tom Softreck Date: Thu, 12 Nov 2026 19:19:26 +0100 Subject: [PATCH] Verify incremental analysis and cache identity (ticket-218) Co-authored-by: Koru Agent --- VERSION | 2 +- code2llm/__init__.py | 2 +- code2llm/api.py | 102 +++++++++- code2llm/cli_exports/orchestrator.py | 11 +- code2llm/core/analyzer.py | 48 ++++- code2llm/core/config.py | 26 ++- code2llm/core/file_cache.py | 16 +- code2llm/core/persistent_cache.py | 68 +++---- pyproject.toml | 2 +- scripts/verify_projection_distribution.py | 34 ++++ setup.py | 2 +- tests/test_cache_invalidation_e2e.py | 34 ++-- tests/test_declarative_collection.py | 23 ++- tests/test_incremental_api.py | 219 ++++++++++++++++++++++ uv.lock | 2 +- 15 files changed, 505 insertions(+), 86 deletions(-) create mode 100644 tests/test_incremental_api.py diff --git a/VERSION b/VERSION index 536bfca..e897069 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.5.185 +0.5.186 diff --git a/code2llm/__init__.py b/code2llm/__init__.py index 3826029..fcee0a3 100644 --- a/code2llm/__init__.py +++ b/code2llm/__init__.py @@ -8,7 +8,7 @@ and entity resolution with multilingual support. """ -__version__ = "0.5.185" +__version__ = "0.5.186" __author__ = "STTS Project" # Core analysis components (lightweight, always needed) diff --git a/code2llm/api.py b/code2llm/api.py index fb05613..53f2fca 100644 --- a/code2llm/api.py +++ b/code2llm/api.py @@ -5,9 +5,16 @@ """ from pathlib import Path +from copy import deepcopy +from dataclasses import dataclass +import hashlib +import json +import os +import sys +import time from typing import Optional -from .core.config import Config, FAST_CONFIG +from .core.config import Config, FAST_CONFIG, analysis_config_digest, analysis_runtime_identity from .core.models import ( AnalysisResult, FunctionInfo, @@ -51,13 +58,102 @@ def analyze_file(file_path: str, config: Optional[Config] = None) -> AnalysisRes Returns: AnalysisResult for the single file. """ - path = Path(file_path).resolve() - return analyze(str(path.parent), config) + from .core.analyzer import ProjectAnalyzer + + return ProjectAnalyzer(config or FAST_CONFIG).analyze_file(file_path) + + +@dataclass(frozen=True) +class IncrementalAnalysis: + """Versioned observation and reused analysis; not an atomic execution snapshot.""" + + analysis: AnalysisResult + identity: dict + incremental: dict + + +def analyze_incremental(project_path: str, *, cache_dir: str, + config: Optional[Config] = None, cache_policy: str = "content", + changed_files=None) -> IncrementalAnalysis: + """Reuse private per-file results, verifying bytes and relinking the full graph. + + ``changed_files`` is an advisory hint, including deleted relative paths. All + selected files are still verified, so an incomplete hint cannot hide edits. + ``content`` reads only this caller-owned private cache; ``off`` never reads + cached pickle. Pickle is internal trusted state, never a host exchange format. + Config/analyzer/parser identities invalidate reuse. Phase timings distinguish + content verification, per-file analysis dispatch and complete graph linking. + The before/after observation detects drift but cannot exclude ABA races; + execute refactors only against a separately immutable snapshot. + """ + from . import __version__ + from .core.analyzer import ProjectAnalyzer + + if cache_policy not in {"content", "off"}: + raise ValueError("cache_policy must be content or off") + root = Path(project_path).resolve(strict=True) + if not root.is_dir(): + raise ValueError("Expected a project directory") + cache = Path(cache_dir).resolve() + if cache.is_relative_to(root): + raise ValueError("Private analysis cache must be outside the source project") + hints = set() + if isinstance(changed_files, (str, bytes, Path)): + raise TypeError("Changed-file hints must be an iterable of paths") + for value in changed_files or (): + path = Path(value) + path = (root / path).resolve() if not path.is_absolute() else path.resolve() + if not path.is_relative_to(root) or path == root: + raise ValueError("Changed-file hint escapes the source project") + hints.add(path.relative_to(root).as_posix()) + cfg = deepcopy(config or FAST_CONFIG) + config_digest = analysis_config_digest(cfg) + if cache_policy == "content": + cache.mkdir(mode=0o700, parents=True, exist_ok=True) + info = cache.stat() + if info.st_uid != os.geteuid() or info.st_mode & 0o077: + raise ValueError("Persistent cache must be a private caller-owned directory") + cfg.performance.cache_dir = str(cache / "file-cache") + cfg.persistent_cache_root = str(cache / "persistent") + else: + cfg.performance.enable_cache = False + cfg.no_cache = True + analyzer = ProjectAnalyzer(cfg, root) + + def inventory(): + selected = {} + for path, _ in sorted(analyzer._collect_files(root)): + source = Path(path) + if not source.resolve(strict=True).is_relative_to(root): + raise ValueError("Selected source escapes the project root") + selected[source.relative_to(root).as_posix()] = hashlib.sha256(source.read_bytes()).hexdigest() + return selected + + started = time.perf_counter() + before = inventory() + inventory_seconds = time.perf_counter() - started + result = analyzer.analyze_project(str(root)) + started = time.perf_counter() + if inventory() != before: + raise RuntimeError("Source changed during analysis; discard the observation") + inventory_seconds += time.perf_counter() - started + identity = dict(schema="code2llm.incremental/v1", analyzer_version=__version__, + parser_runtime=sys.implementation.cache_tag, runtime=analysis_runtime_identity(), + config_sha256=config_digest, + files=before, snapshot_atomic=False) + payload = json.dumps(identity, sort_keys=True, separators=(",", ":")).encode("utf-8") + identity["snapshot_sha256"] = hashlib.sha256(payload).hexdigest() + stats = dict(analyzer.incremental_stats, inventory_seconds=inventory_seconds, + cache_policy=cache_policy, + changed_files_hint=sorted(hints)) + return IncrementalAnalysis(result, identity, stats) __all__ = [ "analyze", "analyze_file", + "analyze_incremental", + "IncrementalAnalysis", "Config", "FAST_CONFIG", "AnalysisResult", diff --git a/code2llm/cli_exports/orchestrator.py b/code2llm/cli_exports/orchestrator.py index e80a341..96ff98d 100644 --- a/code2llm/cli_exports/orchestrator.py +++ b/code2llm/cli_exports/orchestrator.py @@ -109,7 +109,7 @@ def _try_serve_from_cache( args, formats: list, output_dir: Path, source_path: Path ) -> bool: """Return True and restore output from cache if a valid cached export exists.""" - cache = PersistentCache(str(source_path)) + cache = _analysis_export_cache(args, source_path) config_dict = _build_export_config(args, formats) cached_export_dir = cache.get_export_cache_dir(config_dict) if not cached_export_dir: @@ -122,7 +122,7 @@ def _try_serve_from_cache( def _save_to_export_cache(args, formats: list, output_dir: Path, source_path: Path) -> None: """Persist the just-produced output_dir contents into the export cache.""" - cache = PersistentCache(str(source_path)) + cache = _analysis_export_cache(args, source_path) config_dict = _build_export_config(args, formats) export_cache_dir = cache.create_export_cache_dir(config_dict) _copy_to_cache(output_dir, export_cache_dir, verbose=args.verbose) @@ -132,6 +132,12 @@ def _save_to_export_cache(args, formats: list, output_dir: Path, source_path: Pa print(f" Export cached at: {export_cache_dir}") +def _analysis_export_cache(args, source_path): + """Use exactly the manifest namespace that produced the current analysis.""" + context = getattr(args, "_analysis_cache_context", None) + return PersistentCache(str(source_path), **context) if context else PersistentCache(str(source_path)) + + def _run_core_exports( args, result, output_dir: Path, source_path, formats, requested_formats, is_chunked: bool, skip_cache: bool @@ -156,6 +162,7 @@ def _run_exports(args, result, output_dir: Path, source_path: Optional[Path] = N For chunked analysis, exports to subproject subdirectories. Supports export-level caching for repeated runs. """ + args._analysis_cache_context = getattr(result, "stats", {}).get("analysis_cache_context") requested_formats = [f.strip() for f in args.format.split(",")] formats = _expand_all_formats(requested_formats, getattr(args, "png", False)) if getattr(args, "planfile_apply", False) and "planfile" not in formats: diff --git a/code2llm/core/analyzer.py b/code2llm/core/analyzer.py index 6cd1f3d..ddcb6a6 100644 --- a/code2llm/core/analyzer.py +++ b/code2llm/core/analyzer.py @@ -37,6 +37,7 @@ # Results from worker processes are plain dicts; _merge_results rebuilds dataclasses. import hashlib +from dataclasses import fields import logging import multiprocessing as mp import os @@ -61,6 +62,7 @@ ALL_FILENAMES, LANGUAGE_FILENAME_PREFIXES, DEFAULT_PROGRESS_BAR_THRESHOLD, + analysis_config_digest, ) from .models import AnalysisResult, Pattern from .file_cache import FileCache @@ -84,11 +86,13 @@ def __init__( ): """Initialise the analyser with optional config and project path.""" self.config = config or FAST_CONFIG + self._cache_identity = analysis_config_digest(self.config) self.project_path = project_path self.cache = ( FileCache( self.config.performance.cache_dir, self.config.performance.cache_ttl_hours, + analysis_identity=self._cache_identity, ) if self.config.performance.enable_cache else None @@ -109,9 +113,11 @@ def analyze_project(self, project_path: str) -> AnalysisResult: f" - Parallel: {self.config.performance.parallel_enabled}, Workers: {workers}" ) + hash_started = time.perf_counter() pcache, cached_results, files_to_analyze = self._load_from_persistent_cache( files, project_path ) + hash_seconds = time.perf_counter() - hash_started # Watch mode: show what changed if getattr(self.config, "watch", False) and files_to_analyze: @@ -124,11 +130,40 @@ def analyze_project(self, project_path: str) -> AnalysisResult: print(f" ... and {len(files_to_analyze) - 10} more") print() + parse_started = time.perf_counter() fresh_results = self._run_analysis(files_to_analyze) + parse_seconds = time.perf_counter() - parse_started self._store_to_persistent_cache(pcache, files_to_analyze, fresh_results) + link_started = time.perf_counter() merged = self._merge_results(cached_results + fresh_results, str(project_path)) self._post_process(merged, files, cached_results + fresh_results, start_time) + merged.stats["analysis_cache_context"] = { + "analysis_identity": self._cache_identity, + "cache_root": self.config.persistent_cache_root, + } + self.incremental_stats = dict( + analysis_candidates=len(files_to_analyze), cached_files=len(cached_results), + changed_files=sorted(Path(p).relative_to(project_path).as_posix() + for p, _ in files_to_analyze), + deleted_files=getattr(self, "deleted_files", []), + hashing_seconds=hash_seconds, parsing_seconds=parse_seconds, + linking_seconds=time.perf_counter() - link_started, graph_relinked="full", + ) + return merged + + def analyze_file(self, file_path: str) -> AnalysisResult: + """Analyze exactly one source file, without scanning its siblings.""" + start = time.time() + path = Path(file_path).resolve(strict=True) + if not path.is_file(): + raise ValueError("Expected a source file") + root = self._resolve_project_path(str(path.parent)) + module = self._compute_module_name(path.name, path.name, root.name) + files = [(str(path), module)] + results = self._run_analysis(files) + merged = self._merge_results(results, str(root)) + self._post_process(merged, files, results, start) return merged def _resolve_project_path(self, project_path: str) -> Path: @@ -162,7 +197,9 @@ def _load_from_persistent_cache( if getattr(self.config, "no_cache", False): return None, [], files try: - pcache = PersistentCache(str(project_path)) + pcache = PersistentCache(str(project_path), + cache_root=getattr(self.config, "persistent_cache_root", None), + analysis_identity=self._cache_identity) file_paths = [fp for fp, _ in files] changed_paths, cached_paths = pcache.get_changed_files(file_paths) # Drop manifest entries for files that vanished since the last @@ -170,6 +207,7 @@ def _load_from_persistent_cache( # manifest) wouldn't shrink on delete and stale exports could # be re-used. removed = pcache.prune_missing(file_paths) + self.deleted_files = sorted(removed) path_to_module = dict(files) cached_results: List[Dict] = [] for fp in cached_paths: @@ -414,13 +452,7 @@ def _analyze_parallel(self, files: List[Tuple[str, str]]) -> List[Dict]: workers = min(self.config.performance.get_workers(), len(files)) # Convert config to dict for pickle compatibility - config_dict = { - "mode": self.config.mode, - "max_depth_enumeration": self.config.max_depth_enumeration, - "detect_state_machines": self.config.detect_state_machines, - "detect_recursion": self.config.detect_recursion, - "output_dir": self.config.output_dir, - } + config_dict = {item.name: getattr(self.config, item.name) for item in fields(self.config)} # Use forkserver if available (Linux) to prevent deadlocks with native C/Rust extensions # and multi-threaded runtimes when forking. diff --git a/code2llm/core/config.py b/code2llm/core/config.py index 6c49639..3c05c6b 100644 --- a/code2llm/core/config.py +++ b/code2llm/core/config.py @@ -1,12 +1,35 @@ """Configuration and constants for code2llm.""" import os +import hashlib +import json +import sys +from pathlib import Path import psutil -from dataclasses import dataclass, field +from dataclasses import asdict, dataclass, field from typing import List, Set from enum import Enum +def analysis_config_digest(config): + """Bind private payloads to complete configuration and parser/backend bytes.""" + encoded = json.dumps({"config": asdict(config), "runtime": analysis_runtime_identity()}, + sort_keys=True, separators=(",", ":"), + default=lambda value: sorted(value) if isinstance(value, set) else str(value)) + return hashlib.sha256(encoded.encode("utf-8")).hexdigest() + + +def analysis_runtime_identity(): + from code2llm.analysis import native_bridge + + native = native_bridge.code2llm_rust + path = getattr(native, "__file__", None) + return {"parser_runtime": sys.implementation.cache_tag, + "native_version": native_bridge.get_native_version(), + "native_module_sha256": hashlib.sha256(Path(path).read_bytes()).hexdigest() + if path else None} + + def _get_optimal_workers(default: int = 4, max_per_gb: float = 2.0) -> int: """Calculate optimal parallel workers based on CPU and available RAM. @@ -200,6 +223,7 @@ class Config: # Sub-configs for performance performance: PerformanceConfig = field(default_factory=PerformanceConfig) + persistent_cache_root: str | None = None filters: FilterConfig = field(default_factory=FilterConfig) depth: DepthConfig = field(default_factory=DepthConfig) output: OutputConfig = field(default_factory=OutputConfig) diff --git a/code2llm/core/file_cache.py b/code2llm/core/file_cache.py index 6cd3fb9..edef4c1 100644 --- a/code2llm/core/file_cache.py +++ b/code2llm/core/file_cache.py @@ -21,18 +21,24 @@ def make_cache_key(file_path: str, content: str) -> str: class FileCache: """Cache for parsed AST files.""" - def __init__(self, cache_dir: str = ".code2llm_cache", ttl_hours: int = 24): + def __init__(self, cache_dir: str = ".code2llm_cache", ttl_hours: int = 24, + analysis_identity: str = ""): self.cache_dir = Path(cache_dir) self.cache_dir.mkdir(parents=True, exist_ok=True) self.ttl_seconds = ttl_hours * 3600 self._analyzer_version = __version__ + self._analysis_identity = analysis_identity def _get_cache_key_stat(self, file_path: str) -> str: - """Generate cache key from file path, mtime and size (fast, no I/O on content).""" + """Verify content even when a rewrite preserves mtime and size. + + The method name is retained for private-call compatibility. Metadata + alone is never a basis for returning an old analysis payload. + """ identity = hashlib.sha256(os.path.abspath(file_path).encode()).hexdigest()[:16] try: - s = os.stat(file_path) - return f"{self._analyzer_version}_{identity}_{s.st_mtime_ns}_{s.st_size}" + content = hashlib.sha256(Path(file_path).read_bytes()).hexdigest() + return f"{self._analyzer_version}_{self._analysis_identity}_{identity}_content1_{content}" except OSError: return f"{self._analyzer_version}_{identity}_unknown" @@ -76,7 +82,7 @@ def put(self, file_path: str, _content: str, data: Tuple[ast.AST, str]) -> None: pass def get_fast(self, file_path: str) -> Optional[Any]: - """Get cached analysis result using mtime+size key (call before reading file).""" + """Get a cached analysis after verifying current source bytes.""" cache_key = self._get_cache_key_stat(file_path) cache_path = self._get_cache_path(cache_key) diff --git a/code2llm/core/persistent_cache.py b/code2llm/core/persistent_cache.py index cf4e58b..0cc44c9 100644 --- a/code2llm/core/persistent_cache.py +++ b/code2llm/core/persistent_cache.py @@ -16,8 +16,8 @@ └── exports/ # {run_hash}/_complete + copied outputs Invalidation strategy (fastest-first): - L1 mtime + size unchanged → cache hit (no I/O on file content) - L2 content hash unchanged → cache hit (file touched but identical) + Content hash unchanged → cache hit (including touch without edits) + Metadata alone → never proves unchanged source bytes L3 --no-cache / --force → skip cache entirely """ @@ -26,13 +26,13 @@ import logging import os import pickle +import re import shutil import time from pathlib import Path from typing import Any, Dict, List, Optional, Tuple from code2llm import __version__ -from code2llm.analysis.native_bridge import native_check_changed_files logger = logging.getLogger(__name__) @@ -79,12 +79,18 @@ def __init__( cache_root: Optional[str] = None, auto_cleanup: Optional[bool] = None, ttl_days: Optional[float] = None, + analysis_identity: str = "", ): self._project_dir = os.path.realpath(project_dir) self._analyzer_version = __version__ self._root = Path(cache_root or _DEFAULT_ROOT) self._project_hash = hashlib.md5(self._project_dir.encode()).hexdigest()[:12] + self._analysis_identity = analysis_identity self._project_cache = self._root / "projects" / self._project_hash + if analysis_identity: + if not re.fullmatch(r"[0-9a-f]{64}", analysis_identity): + raise ValueError("Analysis identity must be a SHA-256 digest") + self._project_cache /= "analysis-" + analysis_identity self._files_dir = self._project_cache / "files" self._exports_dir = self._project_cache / "exports" self._manifest_path = self._project_cache / "manifest.json" @@ -171,30 +177,11 @@ def put_file_result(self, filepath: str, result: Any) -> None: def get_changed_files(self, filepaths: List[str]) -> Tuple[List[str], List[str]]: """Split *filepaths* into (changed, cached). - Uses L1 (mtime+size) first; falls back to L2 (content hash) when - mtime changed but size matches — handles touch/copy without edit. - Uses native Rust parallel checker when available. + Always verifies content. The native metadata shortcut is deliberately + excluded until its byte/identity contract is separately qualified. """ files_index = self._manifest["files"] - # Fast path: native parallel check across threads - manifest_entries = { - rel: (info["hash"], float(info["mtime"]), int(info["size"])) - for rel, info in files_index.items() - if isinstance(info, dict) and "hash" in info and "mtime" in info and "size" in info - } - native_res = native_check_changed_files( - self._project_dir, filepaths, manifest_entries, self._analyzer_version - ) - if native_res is not None: - changed_res, cached_res, refreshed = native_res - for fp, new_mtime in refreshed: - rel = os.path.relpath(fp, self._project_dir) - if rel in files_index: - files_index[rel]["mtime"] = new_mtime - self._dirty = True - return changed_res, cached_res - changed: List[str] = [] cached: List[str] = [] @@ -206,11 +193,7 @@ def get_changed_files(self, filepaths: List[str]) -> Tuple[List[str], List[str]] continue try: stat = os.stat(fp) - # L1: mtime + size identical → hit - if stat.st_mtime == prev["mtime"] and stat.st_size == prev["size"]: - cached.append(fp) - continue - # L2: content hash check (size may match but mtime drifted) + # Touch/copy and metadata-preserving rewrites both require bytes. h = self.content_hash(fp) if h == prev["hash"]: prev["mtime"] = stat.st_mtime # refresh mtime in manifest @@ -402,6 +385,13 @@ def gc(self, max_age_days: int = 30, max_size_mb: int = _MAX_CACHE_MB) -> int: Returns number of items removed. """ removed = 0 + if not self._analysis_identity: + for namespace in self._project_cache.glob("analysis-*"): + identity = namespace.name.removeprefix("analysis-") + if namespace.is_dir() and not namespace.is_symlink() and re.fullmatch(r"[0-9a-f]{64}", identity): + scoped = PersistentCache(self._project_dir, cache_root=str(self._root), + auto_cleanup=False, analysis_identity=identity) + removed += scoped.gc(max_age_days=max_age_days, max_size_mb=max_size_mb) cutoff = time.time() - (max_age_days * 86400) # Remove old export dirs @@ -502,12 +492,26 @@ def get_all_projects(cache_root: Optional[str] = None) -> List[Dict[str, Any]]: if not root.exists(): return projects for proj_dir in sorted(root.iterdir()): - meta_path = proj_dir / "meta.json" - if meta_path.exists(): + summaries = [] + candidates = [proj_dir / "meta.json"] + candidates.extend(path / "meta.json" for path in sorted(proj_dir.glob("analysis-*")) + if path.is_dir() and not path.is_symlink() + and re.fullmatch(r"analysis-[0-9a-f]{64}", path.name)) + for meta_path in candidates: try: - projects.append(json.loads(meta_path.read_text())) + metadata = json.loads(meta_path.read_text()) + if isinstance(metadata, dict) and isinstance(metadata.get("project"), str): + summaries.append(metadata) except (json.JSONDecodeError, OSError): pass + if summaries: + summary = dict(summaries[0]) + for field in ("files_cached", "cache_size_bytes", "exports"): + summary[field] = sum(row.get(field, 0) for row in summaries) + summary["updated_at"] = max(row.get("updated_at", 0) for row in summaries) + summary["analysis_namespaces"] = [path.parent.name.removeprefix("analysis-") + for path in candidates[1:] if path.exists()] + projects.append(summary) return projects diff --git a/pyproject.toml b/pyproject.toml index 532441a..a2da08f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "code2llm" -version = "0.5.185" +version = "0.5.186" description = "High-performance Python code flow analysis with optimized TOON format - CFG, DFG, call graphs, and intelligent code queries" readme = "README.md" requires-python = ">=3.10" diff --git a/scripts/verify_projection_distribution.py b/scripts/verify_projection_distribution.py index 9b68809..2a598d9 100644 --- a/scripts/verify_projection_distribution.py +++ b/scripts/verify_projection_distribution.py @@ -12,6 +12,7 @@ import hashlib from importlib import metadata import json +import os from pathlib import Path import shutil import subprocess @@ -98,11 +99,44 @@ def check_installed(expected_version): observed = json.loads(first.read_text(encoding="utf-8")) require(observed == expected, "Public builder and exporter differ") check_document(observed, expected_version) + check_incremental_api() return {"status": "PASS", "version": expected_version, "projection_digest": expected["digest"], "records": len(expected["records"]), "source_bytes_verified": False} +def check_incremental_api(): + """Exercise installed public analysis independently of the source test tree.""" + from copy import deepcopy + from code2llm.api import FAST_CONFIG, analyze_file, analyze_incremental + + with tempfile.TemporaryDirectory(prefix="incremental-contract-") as directory: + root = Path(directory) + project = root / "project" + project.mkdir() + source = project / "a.py" + source.write_text("def first():\n return 1\n") + (project / "b.py").write_text("def other():\n return 2\n") + config = deepcopy(FAST_CONFIG) + config.performance.parallel_enabled = False + cache = str(root / "private-cache") + first = analyze_incremental(str(project), cache_dir=cache, config=config) + warm = analyze_incremental(str(project), cache_dir=cache, config=config) + require(warm.identity == first.identity and warm.incremental["analysis_candidates"] == 0, + "Unchanged installed analysis did not reuse its verified cache") + old = source.stat() + source.write_text("def later():\n return 1\n") + os.utime(source, ns=(old.st_atime_ns, old.st_mtime_ns)) + changed = analyze_incremental(str(project), cache_dir=cache, config=config, changed_files=[]) + cold = analyze_incremental(str(project), cache_dir=cache, config=config, cache_policy="off") + require(set(changed.analysis.functions) == set(cold.analysis.functions) == {"a.later", "b.other"}, + "Metadata-preserving rewrite reused stale installed analysis") + require(changed.identity == cold.identity and changed.identity["snapshot_atomic"] is False, + "Installed observation has a false snapshot claim") + require(set(analyze_file(str(source), config).functions) == {"a.later"}, + "Installed analyze_file scanned a sibling") + + def check_artifacts(directory, expected_version): directory = directory.resolve() wheels = sorted(directory.glob("code2llm-*.whl")) diff --git a/setup.py b/setup.py index ce2e190..0609a56 100644 --- a/setup.py +++ b/setup.py @@ -25,7 +25,7 @@ def read_readme(): setup( name="code2llm", - version="0.5.185", + version="0.5.186", description="Python code flow analysis tool - CFG, DFG, and call graph extraction", long_description=read_readme(), long_description_content_type="text/markdown", diff --git a/tests/test_cache_invalidation_e2e.py b/tests/test_cache_invalidation_e2e.py index 35b00f7..3bb6405 100644 --- a/tests/test_cache_invalidation_e2e.py +++ b/tests/test_cache_invalidation_e2e.py @@ -11,7 +11,7 @@ import pytest from code2llm.core.analyzer import ProjectAnalyzer -from code2llm.core.config import Config +from code2llm.core.config import Config, analysis_config_digest from code2llm.core.persistent_cache import PersistentCache @@ -31,31 +31,21 @@ def project(tmp_path, monkeypatch): return tmp_path / "project", fake_home / ".code2llm" +def _analysis_config(project_dir: Path, root: Path): + cfg = Config(output_dir=str(project_dir.parent / "out"), persistent_cache_root=str(root)) + cfg.performance.parallel_enabled = False + cfg.performance.cache_dir = str(root / "file-cache") + return cfg + + def _cache_for(project_dir: Path, root: Path) -> PersistentCache: - return PersistentCache(str(project_dir), cache_root=str(root)) + return PersistentCache(str(project_dir), cache_root=str(root), + analysis_identity=analysis_config_digest(_analysis_config(project_dir, root))) def _run_full_analysis(project_dir: Path, cache_root: Path): - """Run ProjectAnalyzer but forced to use our test cache root.""" - # Patch _DEFAULT_ROOT via a thin PersistentCache subclass injected - # into the analyzer module path. Simpler: monkey-patch the class. - from code2llm.core import analyzer as analyzer_mod - - orig = analyzer_mod.PersistentCache - - class _PC(orig): # type: ignore[misc, valid-type] - def __init__(self, project_dir_): - super().__init__(project_dir_, cache_root=str(cache_root)) - - analyzer_mod.PersistentCache = _PC - try: - cfg = Config(output_dir=str(project_dir.parent / "out")) - # Force sequential to avoid multiprocessing pickling our subclass. - cfg.performance.parallel_enabled = False - a = ProjectAnalyzer(cfg, project_dir) - return a.analyze_project(str(project_dir)) - finally: - analyzer_mod.PersistentCache = orig + """Use the explicit cache configuration instead of replacing its constructor.""" + return ProjectAnalyzer(_analysis_config(project_dir, cache_root), project_dir).analyze_project(str(project_dir)) def test_first_run_populates_manifest(project): diff --git a/tests/test_declarative_collection.py b/tests/test_declarative_collection.py index 9f56b0a..5321861 100644 --- a/tests/test_declarative_collection.py +++ b/tests/test_declarative_collection.py @@ -20,6 +20,7 @@ ALL_EXTENSIONS, ALL_FILENAMES, Config, + analysis_config_digest, ) from code2llm.core.file_analyzer import FileAnalyzer from code2llm.core.persistent_cache import PersistentCache @@ -97,8 +98,9 @@ def test_modifying_declarative_file_invalidates_cache(iac_project, tmp_path): orig = analyzer_mod.PersistentCache class _PC(orig): - def __init__(self, pd): - super().__init__(pd, cache_root=str(cache_root), auto_cleanup=False) + def __init__(self, pd, **kwargs): + kwargs["cache_root"] = str(cache_root) + super().__init__(pd, auto_cleanup=False, **kwargs) analyzer_mod.PersistentCache = _PC try: @@ -108,7 +110,8 @@ def __init__(self, pd): a.analyze_project(str(iac_project)) pc1 = PersistentCache( - str(iac_project), cache_root=str(cache_root), auto_cleanup=False + str(iac_project), cache_root=str(cache_root), auto_cleanup=False, + analysis_identity=analysis_config_digest(cfg) ) assert "pyproject.toml" in pc1._manifest["files"], ( "declarative files must be tracked in the manifest so that " @@ -123,7 +126,8 @@ def __init__(self, pd): a2.analyze_project(str(iac_project)) pc2 = PersistentCache( - str(iac_project), cache_root=str(cache_root), auto_cleanup=False + str(iac_project), cache_root=str(cache_root), auto_cleanup=False, + analysis_identity=analysis_config_digest(cfg) ) h2 = pc2._compute_run_hash({"fmt": "toon"}) assert h1 != h2, ( @@ -143,8 +147,9 @@ def test_dockerfile_edit_invalidates_cache(iac_project, tmp_path): orig = analyzer_mod.PersistentCache class _PC(orig): - def __init__(self, pd): - super().__init__(pd, cache_root=str(cache_root), auto_cleanup=False) + def __init__(self, pd, **kwargs): + kwargs["cache_root"] = str(cache_root) + super().__init__(pd, auto_cleanup=False, **kwargs) analyzer_mod.PersistentCache = _PC try: @@ -152,7 +157,8 @@ def __init__(self, pd): cfg.performance.parallel_enabled = False ProjectAnalyzer(cfg, iac_project).analyze_project(str(iac_project)) pc1 = PersistentCache( - str(iac_project), cache_root=str(cache_root), auto_cleanup=False + str(iac_project), cache_root=str(cache_root), auto_cleanup=False, + analysis_identity=analysis_config_digest(cfg) ) assert "Dockerfile" in pc1._manifest["files"] h1 = pc1._compute_run_hash({"fmt": "toon"}) @@ -161,7 +167,8 @@ def __init__(self, pd): ProjectAnalyzer(cfg, iac_project).analyze_project(str(iac_project)) pc2 = PersistentCache( - str(iac_project), cache_root=str(cache_root), auto_cleanup=False + str(iac_project), cache_root=str(cache_root), auto_cleanup=False, + analysis_identity=analysis_config_digest(cfg) ) h2 = pc2._compute_run_hash({"fmt": "toon"}) assert h1 != h2 diff --git a/tests/test_incremental_api.py b/tests/test_incremental_api.py new file mode 100644 index 0000000..f00a668 --- /dev/null +++ b/tests/test_incremental_api.py @@ -0,0 +1,219 @@ +"""Differential observations against full scans; source bytes are the oracle.""" +from copy import deepcopy +from dataclasses import asdict +from pathlib import Path +import os + +import pytest + +from code2llm.api import FAST_CONFIG, analyze, analyze_file, analyze_incremental +import code2llm.core.persistent_cache as persistent + + +def graph(result): + return {'functions': {name: asdict(item) for name, item in sorted(result.functions.items())}, + 'classes': {name: asdict(item) for name, item in sorted(result.classes.items())}, + 'modules': {name: asdict(item) for name, item in sorted(result.modules.items())}, + 'nodes': {name: asdict(item) for name, item in sorted(result.nodes.items())}, + 'edges': sorted((item.source, item.target) for item in result.edges)} + + +@pytest.fixture +def project(tmp_path, monkeypatch): + root = tmp_path / 'project' + root.mkdir() + (root / 'a.py').write_text('def entry():\n return helper()\n\ndef helper():\n return 1\n') + (root / 'b.py').write_text('def other():\n return 2\n') + cfg = deepcopy(FAST_CONFIG) + cfg.performance.parallel_enabled = False + cfg.performance.cache_dir = str(tmp_path / 'legacy-file-cache') + monkeypatch.setattr(persistent, '_DEFAULT_ROOT', str(tmp_path / 'legacy-persistent')) + return root, cfg, tmp_path / 'private-cache' + + +def cold(root, cfg): + selected = deepcopy(cfg) + selected.performance.enable_cache = False + selected.no_cache = True + return analyze(str(root), selected) + + +def test_single_file_never_returns_sibling_symbols(project): + root, cfg, _ = project + result = analyze_file(str(root / 'a.py'), cfg) + assert set(result.modules) == {'a'} + assert set(result.functions) == {'a.entry', 'a.helper'} + + +def test_legacy_cache_detects_rewrite_preserving_mtime_and_size(project): + root, cfg, _ = project + analyze(str(root), cfg) + path = root / 'a.py' + before = path.stat() + path.write_text(path.read_text().replace('helper', 'newfun')) + os.utime(path, ns=(before.st_atime_ns, before.st_mtime_ns)) + assert graph(analyze(str(root), cfg)) == graph(cold(root, cfg)) + + +def test_warm_api_reuses_file_analysis_but_relinks_graph(project, monkeypatch): + root, cfg, cache = project + first = analyze_incremental(str(root), cache_dir=str(cache), config=cfg) + from code2llm.core.file_analyzer import FileAnalyzer + def unexpected_parse(*args): + raise AssertionError('unchanged file was parsed again') + monkeypatch.setattr(FileAnalyzer, 'analyze_file', unexpected_parse) + second = analyze_incremental(str(root), cache_dir=str(cache), config=cfg, changed_files=[]) + assert second.identity == first.identity + assert second.identity['snapshot_atomic'] is False + assert second.incremental['analysis_candidates'] == 0 + assert second.incremental['cached_files'] == 2 + assert second.incremental['graph_relinked'] == 'full' + assert graph(first.analysis) == graph(second.analysis) + + +@pytest.mark.parametrize('operation', ['edit', 'touch', 'rename', 'delete', 'same-metadata', 'same-bytes', 'config']) +def test_incremental_matches_full_scan_after_change(project, operation): + root, cfg, cache = project + first = analyze_incremental(str(root), cache_dir=str(cache), config=cfg) + a = root / 'a.py' + if operation in {'edit', 'same-metadata'}: + old = a.stat() + a.write_text(a.read_text().replace('helper', 'newfun')) + if operation == 'same-metadata': + os.utime(a, ns=(old.st_atime_ns, old.st_mtime_ns)) + elif operation == 'touch': + old = a.stat() + os.utime(a, ns=(old.st_atime_ns, old.st_mtime_ns + 1000000)) + elif operation == 'rename': + a.rename(root / 'renamed.py') + elif operation == 'delete': + a.unlink() + elif operation == 'same-bytes': + (root / 'copy.py').write_bytes(a.read_bytes()) + elif operation == 'config': + cfg.filters.min_function_lines = 100 + candidate = analyze_incremental(str(root), cache_dir=str(cache), config=cfg, changed_files=['a.py']) + assert graph(candidate.analysis) == graph(cold(root, cfg)) + for name in ('inventory_seconds', 'hashing_seconds', 'parsing_seconds', 'linking_seconds'): + assert candidate.incremental[name] >= 0 + if operation == 'touch': + assert candidate.identity == first.identity + assert candidate.incremental['analysis_candidates'] == 0 + else: + assert candidate.identity['snapshot_sha256'] != first.identity['snapshot_sha256'] + if operation in {'rename', 'delete'}: + assert candidate.incremental['deleted_files'] == ['a.py'] + + +def test_off_policy_never_reads_pickle(project, monkeypatch): + root, cfg, cache = project + import pickle + def refuse(*args, **kwargs): + raise AssertionError('off policy read a cached pickle') + monkeypatch.setattr(pickle, 'load', refuse) + monkeypatch.setattr(pickle, 'loads', refuse) + result = analyze_incremental(str(root), cache_dir=str(cache), config=cfg, cache_policy='off') + assert not cache.exists() + assert graph(result.analysis) == graph(cold(root, cfg)) + + +def test_drift_is_rejected_and_not_advertised_as_atomic(project, monkeypatch): + root, cfg, cache = project + from code2llm.core.analyzer import ProjectAnalyzer + original = ProjectAnalyzer.analyze_project + def mutate(self, path): + result = original(self, path) + (root / 'a.py').write_text('def changed_during_analysis():\n return 1\n') + return result + monkeypatch.setattr(ProjectAnalyzer, 'analyze_project', mutate) + with pytest.raises(RuntimeError, match='Source changed'): + analyze_incremental(str(root), cache_dir=str(cache), config=cfg) + + +@pytest.mark.parametrize('fault', ['inside', 'public', 'hint', 'policy']) +def test_api_refuses_unsafe_cache_and_hint_scope(project, fault): + root, cfg, cache = project + args = dict(cache_dir=str(cache), config=cfg) + if fault == 'inside': + args['cache_dir'] = str(root / 'cache') + elif fault == 'public': + cache.mkdir(mode=0o755) + elif fault == 'hint': + args['changed_files'] = ['../outside.py'] + else: + args['cache_policy'] = 'metadata-only' + with pytest.raises(ValueError): + analyze_incremental(str(root), **args) + + +def test_cycles_are_relinked_after_callee_edit_and_deletion(project): + root, cfg, cache = project + (root / 'a.py').write_text('def first():\n return second()\n') + (root / 'b.py').write_text('def second():\n return first()\n') + initial = analyze_incremental(str(root), cache_dir=str(cache), config=cfg) + assert graph(initial.analysis) == graph(cold(root, cfg)) + (root / 'b.py').write_text('def second():\n return 1\n') + edited = analyze_incremental(str(root), cache_dir=str(cache), config=cfg, changed_files=[]) + assert edited.incremental['changed_files'] == ['b.py'] + assert graph(edited.analysis) == graph(cold(root, cfg)) + (root / 'b.py').unlink() + deleted = analyze_incremental(str(root), cache_dir=str(cache), config=cfg, changed_files=[]) + assert graph(deleted.analysis) == graph(cold(root, cfg)) + + +def test_parallel_workers_receive_full_configuration(project): + root, cfg, cache = project + for index in range(31): + (root / ('module' + str(index) + '.py')).write_text('def tiny():\n return 1\n') + cfg.filters.min_function_lines = 100 + sequential = cold(root, cfg) + cfg.performance.parallel_enabled = True + cfg.performance.parallel_workers = 2 + parallel = cold(root, cfg) + assert not sequential.functions + assert graph(parallel) == graph(sequential) + + +def test_off_and_cached_observations_share_snapshot_identity(project): + root, cfg, cache = project + cached = analyze_incremental(str(root), cache_dir=str(cache), config=cfg) + uncached = analyze_incremental(str(root), cache_dir=str(cache), config=cfg, cache_policy='off') + assert cached.identity == uncached.identity + + +def test_export_cache_follows_the_analysis_namespace(project, tmp_path): + from code2llm.cli_parser import create_parser + from code2llm.cli_exports.orchestrator import _run_exports, _try_serve_from_cache + root, cfg, cache = project + cfg.persistent_cache_root = str(cache) + args = create_parser().parse_args([str(root), '-f', 'projections', '-o', str(tmp_path / 'output')]) + output = tmp_path / 'output' + output.mkdir() + result = analyze(str(root), cfg) + _run_exports(args, result, output, root) + assert _try_serve_from_cache(args, ['projections'], output, root) is True + (root / 'a.py').write_text('def replacement():\n return 3\n') + result = analyze(str(root), cfg) + args._analysis_cache_context = result.stats['analysis_cache_context'] + assert _try_serve_from_cache(args, ['projections'], output, root) is False + + +def test_existing_cache_inventory_and_gc_include_configuration_namespaces(project): + from code2llm.core.persistent_cache import PersistentCache, get_all_projects + root, cfg, cache = project + first = analyze_incremental(str(root), cache_dir=str(cache), config=cfg) + cfg.filters.min_function_lines = 100 + second = analyze_incremental(str(root), cache_dir=str(cache), config=cfg) + assert first.identity['config_sha256'] != second.identity['config_sha256'] + persistent_root = cache / 'persistent' + projects = get_all_projects(str(persistent_root)) + assert len(projects) == 1 + assert len(projects[0]['analysis_namespaces']) == 2 + assert projects[0]['files_cached'] == 4 + parent = PersistentCache(str(root), cache_root=str(persistent_root), auto_cleanup=False) + for namespace in parent._project_cache.glob('analysis-*'): + (namespace / 'files/orphan.pkl').write_bytes(b'not a pickle; never loaded') + assert parent.gc() == 2 + assert not list(parent._project_cache.glob('analysis-*/files/orphan.pkl')) + current = analyze_incremental(str(root), cache_dir=str(cache), config=cfg) + assert current.incremental['cached_files'] == 2 diff --git a/uv.lock b/uv.lock index e0e2cad..8e24df5 100644 --- a/uv.lock +++ b/uv.lock @@ -647,7 +647,7 @@ source = { editable = "packages/code2graph" } [[package]] name = "code2llm" -version = "0.5.185" +version = "0.5.186" source = { editable = "." } dependencies = [ { name = "astroid" },