Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion VERSION
Original file line number Diff line number Diff line change
@@ -1 +1 @@
0.5.185
0.5.186
2 changes: 1 addition & 1 deletion code2llm/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
and entity resolution with multilingual support.
"""

__version__ = "0.5.185"
__version__ = "0.5.186"
__author__ = "STTS Project"

# Core analysis components (lightweight, always needed)
Expand Down
102 changes: 99 additions & 3 deletions code2llm/api.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,16 @@
"""

from pathlib import Path
from copy import deepcopy
from dataclasses import dataclass
import hashlib
import json
import os
import sys
import time
from typing import Optional

from .core.config import Config, FAST_CONFIG
from .core.config import Config, FAST_CONFIG, analysis_config_digest, analysis_runtime_identity
from .core.models import (
AnalysisResult,
FunctionInfo,
Expand Down Expand Up @@ -51,13 +58,102 @@ def analyze_file(file_path: str, config: Optional[Config] = None) -> AnalysisRes
Returns:
AnalysisResult for the single file.
"""
path = Path(file_path).resolve()
return analyze(str(path.parent), config)
from .core.analyzer import ProjectAnalyzer

return ProjectAnalyzer(config or FAST_CONFIG).analyze_file(file_path)


@dataclass(frozen=True)
class IncrementalAnalysis:
"""Versioned observation and reused analysis; not an atomic execution snapshot."""

analysis: AnalysisResult
identity: dict
incremental: dict


def analyze_incremental(project_path: str, *, cache_dir: str,
config: Optional[Config] = None, cache_policy: str = "content",
changed_files=None) -> IncrementalAnalysis:
"""Reuse private per-file results, verifying bytes and relinking the full graph.

``changed_files`` is an advisory hint, including deleted relative paths. All
selected files are still verified, so an incomplete hint cannot hide edits.
``content`` reads only this caller-owned private cache; ``off`` never reads
cached pickle. Pickle is internal trusted state, never a host exchange format.
Config/analyzer/parser identities invalidate reuse. Phase timings distinguish
content verification, per-file analysis dispatch and complete graph linking.
The before/after observation detects drift but cannot exclude ABA races;
execute refactors only against a separately immutable snapshot.
"""
from . import __version__
from .core.analyzer import ProjectAnalyzer

if cache_policy not in {"content", "off"}:
raise ValueError("cache_policy must be content or off")
root = Path(project_path).resolve(strict=True)
if not root.is_dir():
raise ValueError("Expected a project directory")
cache = Path(cache_dir).resolve()
if cache.is_relative_to(root):
raise ValueError("Private analysis cache must be outside the source project")
hints = set()
if isinstance(changed_files, (str, bytes, Path)):
raise TypeError("Changed-file hints must be an iterable of paths")
for value in changed_files or ():
path = Path(value)
path = (root / path).resolve() if not path.is_absolute() else path.resolve()
if not path.is_relative_to(root) or path == root:
raise ValueError("Changed-file hint escapes the source project")
hints.add(path.relative_to(root).as_posix())
cfg = deepcopy(config or FAST_CONFIG)
config_digest = analysis_config_digest(cfg)
if cache_policy == "content":
cache.mkdir(mode=0o700, parents=True, exist_ok=True)
info = cache.stat()
if info.st_uid != os.geteuid() or info.st_mode & 0o077:
raise ValueError("Persistent cache must be a private caller-owned directory")
cfg.performance.cache_dir = str(cache / "file-cache")
cfg.persistent_cache_root = str(cache / "persistent")
else:
cfg.performance.enable_cache = False
cfg.no_cache = True
analyzer = ProjectAnalyzer(cfg, root)

def inventory():
selected = {}
for path, _ in sorted(analyzer._collect_files(root)):
source = Path(path)
if not source.resolve(strict=True).is_relative_to(root):
raise ValueError("Selected source escapes the project root")
selected[source.relative_to(root).as_posix()] = hashlib.sha256(source.read_bytes()).hexdigest()
return selected

started = time.perf_counter()
before = inventory()
inventory_seconds = time.perf_counter() - started
result = analyzer.analyze_project(str(root))
started = time.perf_counter()
if inventory() != before:
raise RuntimeError("Source changed during analysis; discard the observation")
inventory_seconds += time.perf_counter() - started
identity = dict(schema="code2llm.incremental/v1", analyzer_version=__version__,
parser_runtime=sys.implementation.cache_tag, runtime=analysis_runtime_identity(),
config_sha256=config_digest,
files=before, snapshot_atomic=False)
payload = json.dumps(identity, sort_keys=True, separators=(",", ":")).encode("utf-8")
identity["snapshot_sha256"] = hashlib.sha256(payload).hexdigest()
stats = dict(analyzer.incremental_stats, inventory_seconds=inventory_seconds,
cache_policy=cache_policy,
changed_files_hint=sorted(hints))
return IncrementalAnalysis(result, identity, stats)


__all__ = [
"analyze",
"analyze_file",
"analyze_incremental",
"IncrementalAnalysis",
"Config",
"FAST_CONFIG",
"AnalysisResult",
Expand Down
11 changes: 9 additions & 2 deletions code2llm/cli_exports/orchestrator.py
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,7 @@ def _try_serve_from_cache(
args, formats: list, output_dir: Path, source_path: Path
) -> bool:
"""Return True and restore output from cache if a valid cached export exists."""
cache = PersistentCache(str(source_path))
cache = _analysis_export_cache(args, source_path)
config_dict = _build_export_config(args, formats)
cached_export_dir = cache.get_export_cache_dir(config_dict)
if not cached_export_dir:
Expand All @@ -122,7 +122,7 @@ def _try_serve_from_cache(

def _save_to_export_cache(args, formats: list, output_dir: Path, source_path: Path) -> None:
"""Persist the just-produced output_dir contents into the export cache."""
cache = PersistentCache(str(source_path))
cache = _analysis_export_cache(args, source_path)
config_dict = _build_export_config(args, formats)
export_cache_dir = cache.create_export_cache_dir(config_dict)
_copy_to_cache(output_dir, export_cache_dir, verbose=args.verbose)
Expand All @@ -132,6 +132,12 @@ def _save_to_export_cache(args, formats: list, output_dir: Path, source_path: Pa
print(f" Export cached at: {export_cache_dir}")


def _analysis_export_cache(args, source_path):
"""Use exactly the manifest namespace that produced the current analysis."""
context = getattr(args, "_analysis_cache_context", None)
return PersistentCache(str(source_path), **context) if context else PersistentCache(str(source_path))


def _run_core_exports(
args, result, output_dir: Path, source_path, formats, requested_formats,
is_chunked: bool, skip_cache: bool
Expand All @@ -156,6 +162,7 @@ def _run_exports(args, result, output_dir: Path, source_path: Optional[Path] = N
For chunked analysis, exports to subproject subdirectories.
Supports export-level caching for repeated runs.
"""
args._analysis_cache_context = getattr(result, "stats", {}).get("analysis_cache_context")
requested_formats = [f.strip() for f in args.format.split(",")]
formats = _expand_all_formats(requested_formats, getattr(args, "png", False))
if getattr(args, "planfile_apply", False) and "planfile" not in formats:
Expand Down
48 changes: 40 additions & 8 deletions code2llm/core/analyzer.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
# Results from worker processes are plain dicts; _merge_results rebuilds dataclasses.

import hashlib
from dataclasses import fields
import logging
import multiprocessing as mp
import os
Expand All @@ -61,6 +62,7 @@
ALL_FILENAMES,
LANGUAGE_FILENAME_PREFIXES,
DEFAULT_PROGRESS_BAR_THRESHOLD,
analysis_config_digest,
)
from .models import AnalysisResult, Pattern
from .file_cache import FileCache
Expand All @@ -84,11 +86,13 @@ def __init__(
):
"""Initialise the analyser with optional config and project path."""
self.config = config or FAST_CONFIG
self._cache_identity = analysis_config_digest(self.config)
self.project_path = project_path
self.cache = (
FileCache(
self.config.performance.cache_dir,
self.config.performance.cache_ttl_hours,
analysis_identity=self._cache_identity,
)
if self.config.performance.enable_cache
else None
Expand All @@ -109,9 +113,11 @@ def analyze_project(self, project_path: str) -> AnalysisResult:
f" - Parallel: {self.config.performance.parallel_enabled}, Workers: {workers}"
)

hash_started = time.perf_counter()
pcache, cached_results, files_to_analyze = self._load_from_persistent_cache(
files, project_path
)
hash_seconds = time.perf_counter() - hash_started

# Watch mode: show what changed
if getattr(self.config, "watch", False) and files_to_analyze:
Expand All @@ -124,11 +130,40 @@ def analyze_project(self, project_path: str) -> AnalysisResult:
print(f" ... and {len(files_to_analyze) - 10} more")
print()

parse_started = time.perf_counter()
fresh_results = self._run_analysis(files_to_analyze)
parse_seconds = time.perf_counter() - parse_started
self._store_to_persistent_cache(pcache, files_to_analyze, fresh_results)

link_started = time.perf_counter()
merged = self._merge_results(cached_results + fresh_results, str(project_path))
self._post_process(merged, files, cached_results + fresh_results, start_time)
merged.stats["analysis_cache_context"] = {
"analysis_identity": self._cache_identity,
"cache_root": self.config.persistent_cache_root,
}
self.incremental_stats = dict(
analysis_candidates=len(files_to_analyze), cached_files=len(cached_results),
changed_files=sorted(Path(p).relative_to(project_path).as_posix()
for p, _ in files_to_analyze),
deleted_files=getattr(self, "deleted_files", []),
hashing_seconds=hash_seconds, parsing_seconds=parse_seconds,
linking_seconds=time.perf_counter() - link_started, graph_relinked="full",
)
return merged

def analyze_file(self, file_path: str) -> AnalysisResult:
"""Analyze exactly one source file, without scanning its siblings."""
start = time.time()
path = Path(file_path).resolve(strict=True)
if not path.is_file():
raise ValueError("Expected a source file")
root = self._resolve_project_path(str(path.parent))
module = self._compute_module_name(path.name, path.name, root.name)
files = [(str(path), module)]
results = self._run_analysis(files)
merged = self._merge_results(results, str(root))
self._post_process(merged, files, results, start)
return merged

def _resolve_project_path(self, project_path: str) -> Path:
Expand Down Expand Up @@ -162,14 +197,17 @@ def _load_from_persistent_cache(
if getattr(self.config, "no_cache", False):
return None, [], files
try:
pcache = PersistentCache(str(project_path))
pcache = PersistentCache(str(project_path),
cache_root=getattr(self.config, "persistent_cache_root", None),
analysis_identity=self._cache_identity)
file_paths = [fp for fp, _ in files]
changed_paths, cached_paths = pcache.get_changed_files(file_paths)
# Drop manifest entries for files that vanished since the last
# run. Without this the export-level cache key (derived from the
# manifest) wouldn't shrink on delete and stale exports could
# be re-used.
removed = pcache.prune_missing(file_paths)
self.deleted_files = sorted(removed)
path_to_module = dict(files)
cached_results: List[Dict] = []
for fp in cached_paths:
Expand Down Expand Up @@ -414,13 +452,7 @@ def _analyze_parallel(self, files: List[Tuple[str, str]]) -> List[Dict]:
workers = min(self.config.performance.get_workers(), len(files))

# Convert config to dict for pickle compatibility
config_dict = {
"mode": self.config.mode,
"max_depth_enumeration": self.config.max_depth_enumeration,
"detect_state_machines": self.config.detect_state_machines,
"detect_recursion": self.config.detect_recursion,
"output_dir": self.config.output_dir,
}
config_dict = {item.name: getattr(self.config, item.name) for item in fields(self.config)}

# Use forkserver if available (Linux) to prevent deadlocks with native C/Rust extensions
# and multi-threaded runtimes when forking.
Expand Down
26 changes: 25 additions & 1 deletion code2llm/core/config.py
Original file line number Diff line number Diff line change
@@ -1,12 +1,35 @@
"""Configuration and constants for code2llm."""

import os
import hashlib
import json
import sys
from pathlib import Path
import psutil
from dataclasses import dataclass, field
from dataclasses import asdict, dataclass, field
from typing import List, Set
from enum import Enum


def analysis_config_digest(config):
"""Bind private payloads to complete configuration and parser/backend bytes."""
encoded = json.dumps({"config": asdict(config), "runtime": analysis_runtime_identity()},
sort_keys=True, separators=(",", ":"),
default=lambda value: sorted(value) if isinstance(value, set) else str(value))
return hashlib.sha256(encoded.encode("utf-8")).hexdigest()


def analysis_runtime_identity():
from code2llm.analysis import native_bridge

native = native_bridge.code2llm_rust
path = getattr(native, "__file__", None)
return {"parser_runtime": sys.implementation.cache_tag,
"native_version": native_bridge.get_native_version(),
"native_module_sha256": hashlib.sha256(Path(path).read_bytes()).hexdigest()
if path else None}


def _get_optimal_workers(default: int = 4, max_per_gb: float = 2.0) -> int:
"""Calculate optimal parallel workers based on CPU and available RAM.

Expand Down Expand Up @@ -200,6 +223,7 @@ class Config:

# Sub-configs for performance
performance: PerformanceConfig = field(default_factory=PerformanceConfig)
persistent_cache_root: str | None = None
filters: FilterConfig = field(default_factory=FilterConfig)
depth: DepthConfig = field(default_factory=DepthConfig)
output: OutputConfig = field(default_factory=OutputConfig)
Expand Down
16 changes: 11 additions & 5 deletions code2llm/core/file_cache.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,18 +21,24 @@ def make_cache_key(file_path: str, content: str) -> str:
class FileCache:
"""Cache for parsed AST files."""

def __init__(self, cache_dir: str = ".code2llm_cache", ttl_hours: int = 24):
def __init__(self, cache_dir: str = ".code2llm_cache", ttl_hours: int = 24,
analysis_identity: str = ""):
self.cache_dir = Path(cache_dir)
self.cache_dir.mkdir(parents=True, exist_ok=True)
self.ttl_seconds = ttl_hours * 3600
self._analyzer_version = __version__
self._analysis_identity = analysis_identity

def _get_cache_key_stat(self, file_path: str) -> str:
"""Generate cache key from file path, mtime and size (fast, no I/O on content)."""
"""Verify content even when a rewrite preserves mtime and size.

The method name is retained for private-call compatibility. Metadata
alone is never a basis for returning an old analysis payload.
"""
identity = hashlib.sha256(os.path.abspath(file_path).encode()).hexdigest()[:16]
try:
s = os.stat(file_path)
return f"{self._analyzer_version}_{identity}_{s.st_mtime_ns}_{s.st_size}"
content = hashlib.sha256(Path(file_path).read_bytes()).hexdigest()
return f"{self._analyzer_version}_{self._analysis_identity}_{identity}_content1_{content}"
except OSError:
return f"{self._analyzer_version}_{identity}_unknown"

Expand Down Expand Up @@ -76,7 +82,7 @@ def put(self, file_path: str, _content: str, data: Tuple[ast.AST, str]) -> None:
pass

def get_fast(self, file_path: str) -> Optional[Any]:
"""Get cached analysis result using mtime+size key (call before reading file)."""
"""Get a cached analysis after verifying current source bytes."""
cache_key = self._get_cache_key_stat(file_path)
cache_path = self._get_cache_path(cache_key)

Expand Down
Loading
Loading