Skip to content

Commit 0f4e6e8

Browse files
committed
feat: add persistent per-module cache with incremental summaries
1 parent bcacf39 commit 0f4e6e8

6 files changed

Lines changed: 434 additions & 32 deletions

File tree

‎README.md‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,15 @@ attacker-controlled data reaches such a call, the engine correlates the four fac
4343
one critical `exploitable-vulnerability` finding, judged like any other flow. The shipped
4444
advisory database is a small offline sample; a live OSV feed is future work.
4545

46+
`--cache DIR` keeps the results of a directory check on disk, one JSON entry per module.
47+
The entry is keyed by the module's source, the engine and plugin versions, the plugin
48+
code, the security models, the advisories, the dependency graph and the keys of the
49+
modules it imports transitively. On the next run a module whose key is unchanged is
50+
served from the cache: its function summaries seed the project index, its call sites
51+
serve the project plugins and its findings are reported as they were, so editing one file
52+
re-analyses that file and the modules that import it only. Entries are plain data, never
53+
code; an unreadable entry is simply recomputed.
54+
4655
`--plugins` is a directory searched recursively for `plugin.toml` manifests and may be
4756
repeated. The repository ships a first set under `plugins/`: security models for the
4857
standard library, Flask, FastAPI and SQLAlchemy (`models/`), taint detectors for SQL

‎src/coretrace_python/cache.py‎

Lines changed: 278 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,278 @@
1+
"""Persistent per-module cache (architecture §11, §38 Phase 10).
2+
3+
A module's results are stored under a key derived from everything they depend on: its
4+
source text and identity, the engine, schema and plugin API versions, the plugins and
5+
their code, the security models, the advisories, the dependency graph, and the keys of
6+
the project modules it imports transitively. A module whose key is unchanged on a later
7+
run is served from the cache: its summaries seed the project index, its call sites serve
8+
the project plugins and its findings are reported as they were. Entries are JSON, so a
9+
tampered or foreign file can never execute anything; an unreadable entry is a miss.
10+
"""
11+
12+
from __future__ import annotations
13+
14+
import hashlib
15+
import json
16+
import os
17+
from collections.abc import Iterable, Mapping
18+
from dataclasses import dataclass
19+
from pathlib import Path
20+
from types import MappingProxyType
21+
from typing import Any
22+
23+
from coretrace_python.findings import Confidence, Finding, Severity
24+
from coretrace_python.interprocedural import (
25+
CallSite,
26+
ExternalCall,
27+
ExternalSymbol,
28+
FunctionSummary,
29+
KnownFunction,
30+
ModuleGraph,
31+
Target,
32+
UnknownTarget,
33+
)
34+
from coretrace_python.semantic.symbols import SymbolId
35+
from coretrace_python.source import SourceId, SourceSpan
36+
37+
CACHE_FORMAT = 1
38+
39+
40+
@dataclass(frozen=True)
41+
class CachedModule:
42+
"""Everything a later run needs from one module without lowering it again."""
43+
44+
functions: tuple[str, ...]
45+
summaries: Mapping[str, FunctionSummary]
46+
sites: tuple[CallSite, ...]
47+
findings: tuple[Finding, ...]
48+
49+
def __post_init__(self) -> None:
50+
object.__setattr__(self, "summaries", MappingProxyType(dict(self.summaries)))
51+
52+
53+
# --------------------------------------------------------------------------- keys
54+
55+
56+
def fingerprint(*parts: str) -> str:
57+
digest = hashlib.sha256()
58+
for part in parts:
59+
digest.update(part.encode("utf-8"))
60+
digest.update(b"\0")
61+
return digest.hexdigest()
62+
63+
64+
def directory_fingerprint(directory: Path, suffixes: Iterable[str] = (".py", ".toml")) -> str:
65+
"""A digest of the source files under ``directory``, so edited plugin code misses."""
66+
67+
wanted = tuple(suffixes)
68+
parts: list[str] = []
69+
for path in sorted(directory.rglob("*")):
70+
if path.is_file() and path.suffix in wanted:
71+
parts.append(str(path.relative_to(directory)))
72+
parts.append(path.read_text(encoding="utf-8", errors="replace"))
73+
return fingerprint(*parts)
74+
75+
76+
def module_keys(graph: ModuleGraph, own: Mapping[str, str]) -> dict[str, str]:
77+
"""The key of each module: its own key plus those of every module it imports,
78+
transitively, so a change anywhere below a module misses for that module too."""
79+
80+
keys: dict[str, str] = {}
81+
for name in graph.modules:
82+
closure: set[str] = set()
83+
pending = [name]
84+
while pending:
85+
current = pending.pop()
86+
for imported in graph.imports(current):
87+
if imported in own and imported not in closure and imported != name:
88+
closure.add(imported)
89+
pending.append(imported)
90+
keys[name] = fingerprint(own[name], *(own[m] for m in sorted(closure)))
91+
return keys
92+
93+
94+
# --------------------------------------------------------------------------- store
95+
96+
97+
class ProjectCache:
98+
"""A directory of JSON entries, one per module key."""
99+
100+
def __init__(self, directory: Path) -> None:
101+
self.directory = directory
102+
103+
def load(self, key: str) -> CachedModule | None:
104+
path = self.directory / f"{key}.json"
105+
try:
106+
with path.open(encoding="utf-8") as handle:
107+
return decode(json.load(handle))
108+
except (OSError, ValueError, KeyError, TypeError, IndexError, AttributeError):
109+
return None
110+
111+
def store(self, key: str, module: CachedModule) -> None:
112+
self.directory.mkdir(parents=True, exist_ok=True)
113+
path = self.directory / f"{key}.json"
114+
temporary = path.with_name(f"{path.name}.{os.getpid()}.tmp")
115+
temporary.write_text(json.dumps(encode(module)), encoding="utf-8")
116+
temporary.replace(path)
117+
118+
119+
# --------------------------------------------------------------------------- codec
120+
121+
122+
def encode(module: CachedModule) -> dict[str, Any]:
123+
return {
124+
"format": CACHE_FORMAT,
125+
"functions": list(module.functions),
126+
"summaries": {name: _encode_summary(s) for name, s in module.summaries.items()},
127+
"sites": [_encode_site(site) for site in module.sites],
128+
"findings": [_encode_finding(finding) for finding in module.findings],
129+
}
130+
131+
132+
def decode(data: Mapping[str, Any]) -> CachedModule:
133+
if data["format"] != CACHE_FORMAT:
134+
raise ValueError(f"unsupported cache format {data['format']!r}")
135+
return CachedModule(
136+
tuple(_string(name) for name in data["functions"]),
137+
{_string(name): _decode_summary(s) for name, s in data["summaries"].items()},
138+
tuple(_decode_site(site) for site in data["sites"]),
139+
tuple(_decode_finding(finding) for finding in data["findings"]),
140+
)
141+
142+
143+
def _string(value: Any) -> str:
144+
if not isinstance(value, str):
145+
raise TypeError(f"expected a string, got {value!r}")
146+
return value
147+
148+
149+
def _integer(value: Any) -> int:
150+
if not isinstance(value, int) or isinstance(value, bool):
151+
raise TypeError(f"expected an integer, got {value!r}")
152+
return value
153+
154+
155+
def _indices(values: Any) -> frozenset[int]:
156+
return frozenset(_integer(v) for v in values)
157+
158+
159+
def _encode_span(span: SourceSpan) -> list[Any]:
160+
return [str(span.source_id), span.start_line, span.start_column, span.end_line, span.end_column]
161+
162+
163+
def _decode_span(data: Any) -> SourceSpan:
164+
file, line, column, end_line, end_column = data
165+
return SourceSpan(
166+
SourceId(_string(file)),
167+
_integer(line),
168+
_integer(column),
169+
None if end_line is None else _integer(end_line),
170+
None if end_column is None else _integer(end_column),
171+
)
172+
173+
174+
def _encode_finding(finding: Finding) -> dict[str, Any]:
175+
return {
176+
"rule": finding.rule_id,
177+
"message": finding.message,
178+
"severity": finding.severity.value,
179+
"confidence": finding.confidence.value,
180+
"span": _encode_span(finding.span),
181+
"function": finding.function,
182+
"metadata": dict(finding.metadata),
183+
}
184+
185+
186+
def _decode_finding(data: Mapping[str, Any]) -> Finding:
187+
function = data["function"]
188+
return Finding(
189+
_string(data["rule"]),
190+
_string(data["message"]),
191+
Severity(data["severity"]),
192+
Confidence(data["confidence"]),
193+
_decode_span(data["span"]),
194+
None if function is None else _string(function),
195+
{_string(k): _string(v) for k, v in data["metadata"].items()},
196+
)
197+
198+
199+
def _encode_call(call: ExternalCall) -> dict[str, Any]:
200+
return {
201+
"symbol": str(call.symbol),
202+
"arguments": [sorted(deps) for deps in call.argument_dependencies],
203+
"keywords": sorted(call.keyword_dependencies),
204+
"location": _encode_span(call.location),
205+
"call_site": None if call.call_site is None else _encode_span(call.call_site),
206+
}
207+
208+
209+
def _decode_call(data: Mapping[str, Any]) -> ExternalCall:
210+
site = data["call_site"]
211+
return ExternalCall(
212+
SymbolId(_string(data["symbol"])),
213+
tuple(_indices(deps) for deps in data["arguments"]),
214+
_indices(data["keywords"]),
215+
_decode_span(data["location"]),
216+
None if site is None else _decode_span(site),
217+
)
218+
219+
220+
def _encode_summary(summary: FunctionSummary) -> dict[str, Any]:
221+
return {
222+
"name": summary.name,
223+
"parameters": summary.parameters,
224+
"returns": sorted(summary.return_dependencies),
225+
"external_calls": [_encode_call(call) for call in summary.external_calls],
226+
"unsupported": summary.unsupported,
227+
"return_externals": sorted(str(s) for s in summary.return_externals),
228+
}
229+
230+
231+
def _decode_summary(data: Mapping[str, Any]) -> FunctionSummary:
232+
return FunctionSummary(
233+
_string(data["name"]),
234+
_integer(data["parameters"]),
235+
_indices(data["returns"]),
236+
tuple(_decode_call(call) for call in data["external_calls"]),
237+
bool(data["unsupported"]),
238+
frozenset(SymbolId(_string(s)) for s in data["return_externals"]),
239+
)
240+
241+
242+
def _encode_target(target: Target) -> dict[str, Any]:
243+
if isinstance(target, KnownFunction):
244+
return {"kind": "known", "name": target.name}
245+
if isinstance(target, ExternalSymbol):
246+
return {"kind": "external", "symbol": str(target.symbol)}
247+
return {"kind": "unknown"}
248+
249+
250+
def _decode_target(data: Mapping[str, Any]) -> Target:
251+
kind = data["kind"]
252+
if kind == "known":
253+
return KnownFunction(_string(data["name"]))
254+
if kind == "external":
255+
return ExternalSymbol(SymbolId(_string(data["symbol"])))
256+
if kind == "unknown":
257+
return UnknownTarget()
258+
raise ValueError(f"unknown call target kind {kind!r}")
259+
260+
261+
def _encode_site(site: CallSite) -> dict[str, Any]:
262+
return {
263+
"caller": site.caller,
264+
"location": _encode_span(site.location),
265+
"target": _encode_target(site.target),
266+
"arguments": site.arguments,
267+
"keywords": site.keywords,
268+
}
269+
270+
271+
def _decode_site(data: Mapping[str, Any]) -> CallSite:
272+
return CallSite(
273+
_string(data["caller"]),
274+
_decode_span(data["location"]),
275+
_decode_target(data["target"]),
276+
_integer(data["arguments"]),
277+
_integer(data["keywords"]),
278+
)

‎src/coretrace_python/cli.py‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66

77
from coretrace_python import engine
88
from coretrace_python.analysis import AnalysisError
9+
from coretrace_python.cache import ProjectCache
910
from coretrace_python.cfg import CFGError
1011
from coretrace_python.frontend import HIRBuildError, ParseError, build_hir
1112
from coretrace_python.ir.lowering import LoweringError, lower_module
@@ -70,6 +71,14 @@ def build_parser() -> argparse.ArgumentParser:
7071
default=None,
7172
help="with --check, the report format (default: text)",
7273
)
74+
parser.add_argument(
75+
"--cache",
76+
type=Path,
77+
default=None,
78+
metavar="DIR",
79+
help="with --check on a directory, keep per-module results under DIR and reuse "
80+
"them for modules unchanged since the previous run",
81+
)
7382
return parser
7483

7584

@@ -81,6 +90,9 @@ def main(argv: list[str] | None = None) -> int:
8190
if args.format is not None and not args.check:
8291
print("error: --format only applies to --check", file=sys.stderr)
8392
return EXIT_ERROR
93+
if args.cache is not None and not (args.check and args.path.is_dir()):
94+
print("error: --cache only applies to --check on a directory", file=sys.stderr)
95+
return EXIT_ERROR
8496

8597
if args.path.is_dir() and args.emit_ir:
8698
print(f"error: {args.path} is a directory; --emit-ir needs a file", file=sys.stderr)
@@ -92,7 +104,8 @@ def main(argv: list[str] | None = None) -> int:
92104
print(format_module(lower_module(build_hir(source), ssa=args.ssa)))
93105
if args.check:
94106
if args.path.is_dir():
95-
findings = engine.analyze_project(args.path, args.plugins).findings
107+
cache = None if args.cache is None else ProjectCache(args.cache)
108+
findings = engine.analyze_project(args.path, args.plugins, cache=cache).findings
96109
else:
97110
findings = engine.check(SourceManager().load_file(args.path), args.plugins)
98111
print(render(args.format or "text", engine.report(findings)), end="")

0 commit comments

Comments
 (0)