Skip to content

Commit ffb49bf

Browse files
authored
Merge pull request #43 from CoreTrace/feat/parallel
feat: parallel analysis and memory eviction
2 parents e9d546a + 300626c commit ffb49bf

6 files changed

Lines changed: 503 additions & 37 deletions

File tree

‎README.md‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,15 @@ serve the project plugins and its findings are reported as they were, so editing
5252
re-analyses that file and the modules that import it only. Entries are plain data, never
5353
code; an unreadable entry is simply recomputed.
5454

55+
Modules are analysed by strongly connected components of the module graph, imports
56+
first, so a module starts with the final summaries of everything it imports and mutually
57+
importing modules are iterated together. `--jobs N` analyses the components of one wave
58+
in `N` processes; each worker rebuilds the configuration from the project root and the
59+
plugin roots and hands back summaries, call sites and findings as plain data, so the
60+
result is the same whatever `N`. Once a module's results are extracted, its intermediate
61+
representation and derived results are dropped and only its semantic tables stay in
62+
memory.
63+
5564
`--plugins` is a directory searched recursively for `plugin.toml` manifests and may be
5665
repeated. The repository ships a first set under `plugins/`: security models for the
5766
standard library, Flask, FastAPI and SQLAlchemy (`models/`), taint detectors for SQL

‎src/coretrace_python/cache.py‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
FunctionSummary,
2929
KnownFunction,
3030
ModuleGraph,
31+
SummaryIndex,
3132
Target,
3233
UnknownTarget,
3334
)
@@ -140,6 +141,16 @@ def decode(data: Mapping[str, Any]) -> CachedModule:
140141
)
141142

142143

144+
def encode_index(index: SummaryIndex) -> dict[str, Any]:
145+
"""A summary index as plain data, to hand a worker process what it imports."""
146+
147+
return {str(symbol): _encode_summary(index.summaries[symbol]) for symbol in index.symbols}
148+
149+
150+
def decode_index(data: Mapping[str, Any]) -> SummaryIndex:
151+
return SummaryIndex({SymbolId(_string(k)): _decode_summary(v) for k, v in data.items()})
152+
153+
143154
def _string(value: Any) -> str:
144155
if not isinstance(value, str):
145156
raise TypeError(f"expected a string, got {value!r}")

‎src/coretrace_python/cli.py‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,13 @@ def build_parser() -> argparse.ArgumentParser:
7979
help="with --check on a directory, keep per-module results under DIR and reuse "
8080
"them for modules unchanged since the previous run",
8181
)
82+
parser.add_argument(
83+
"--jobs",
84+
type=int,
85+
default=None,
86+
metavar="N",
87+
help="with --check on a directory, analyse independent modules in N processes",
88+
)
8289
return parser
8390

8491

@@ -93,6 +100,12 @@ def main(argv: list[str] | None = None) -> int:
93100
if args.cache is not None and not (args.check and args.path.is_dir()):
94101
print("error: --cache only applies to --check on a directory", file=sys.stderr)
95102
return EXIT_ERROR
103+
if args.jobs is not None and not (args.check and args.path.is_dir()):
104+
print("error: --jobs only applies to --check on a directory", file=sys.stderr)
105+
return EXIT_ERROR
106+
if args.jobs is not None and args.jobs < 1:
107+
print("error: --jobs must be at least 1", file=sys.stderr)
108+
return EXIT_ERROR
96109

97110
if args.path.is_dir() and args.emit_ir:
98111
print(f"error: {args.path} is a directory; --emit-ir needs a file", file=sys.stderr)
@@ -105,7 +118,9 @@ def main(argv: list[str] | None = None) -> int:
105118
if args.check:
106119
if args.path.is_dir():
107120
cache = None if args.cache is None else ProjectCache(args.cache)
108-
findings = engine.analyze_project(args.path, args.plugins, cache=cache).findings
121+
findings = engine.analyze_project(
122+
args.path, args.plugins, cache=cache, jobs=args.jobs or 1
123+
).findings
109124
else:
110125
findings = engine.check(SourceManager().load_file(args.path), args.plugins)
111126
print(render(args.format or "text", engine.report(findings)), end="")

‎src/coretrace_python/engine.py‎

Lines changed: 157 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -6,11 +6,13 @@
66

77
from __future__ import annotations
88

9+
import concurrent.futures
10+
import multiprocessing
911
from collections.abc import Iterable, Mapping, Sequence
1012
from dataclasses import dataclass, field
1113
from pathlib import Path
1214
from types import MappingProxyType
13-
from typing import ClassVar
15+
from typing import Any, ClassVar
1416

1517
from coretrace_python import __version__
1618
from coretrace_python.abstract import ConstantPropagation
@@ -23,7 +25,11 @@
2325
from coretrace_python.cache import (
2426
CachedModule,
2527
ProjectCache,
28+
decode,
29+
decode_index,
2630
directory_fingerprint,
31+
encode,
32+
encode_index,
2733
fingerprint,
2834
module_keys,
2935
)
@@ -128,6 +134,20 @@ def run(cls, ctx: AnalysisContext) -> None:
128134
pass
129135

130136

137+
class ResultsEvicted(TransformationPass):
138+
"""Drops a module's PyIR and every derived result once its summaries, call sites and
139+
findings are extracted (§30); the semantic tables and the engine inputs stay."""
140+
141+
name: ClassVar[str] = "project.results-evicted"
142+
preserves: ClassVar[frozenset[AnyAnalysis]] = frozenset(
143+
{*SEMANTIC_ANALYSES, SecurityModelAnalysis, ProjectSummaries, DependencyAnalysis}
144+
)
145+
146+
@classmethod
147+
def run(cls, ctx: AnalysisContext) -> None:
148+
pass
149+
150+
131151
def _no_keys() -> Mapping[str, str]:
132152
return MappingProxyType({})
133153

@@ -194,12 +214,17 @@ def analyze_project(
194214
plugin_roots: Sequence[Path] = (),
195215
plugins: Sequence[Plugin] = (),
196216
cache: ProjectCache | None = None,
217+
jobs: int = 1,
197218
) -> ProjectAnalysis:
198219
"""Analyse every Python file under ``root`` with a shared summary index (§21) and the
199220
dependency graph of its manifests (§26). ``plugins`` adds plugin instances to the
200221
ones discovered under ``plugin_roots``. With a ``cache``, modules whose key is
201-
unchanged since a previous run are served from it (§11)."""
222+
unchanged since a previous run are served from it (§11). Modules are scheduled by
223+
strongly connected components of the module graph, imports first; with ``jobs``
224+
above one the components of a wave are analysed in that many processes (§29)."""
202225

226+
if jobs < 1:
227+
raise ValueError("jobs must be at least 1")
203228
sources = SourceManager()
204229
findings: list[Finding] = []
205230
dependencies = resolve_dependencies(root, sources)
@@ -244,61 +269,157 @@ def analyze_project(
244269
graph,
245270
{name: fingerprint(configuration, str(files[name].source_id), name, files[name].text) for name in analysable},
246271
)
247-
cached: dict[str, CachedModule] = {}
272+
results: dict[str, CachedModule] = {}
248273
if cache is not None:
249274
for name in analysable:
250275
entry = cache.load(keys[name])
251276
if entry is not None:
252-
cached[name] = entry
253-
fresh = {name: manager for name, manager in analysable.items() if name not in cached}
254-
255-
index = SummaryIndex()
256-
cached_summaries = {
257-
project_symbol(name, function): summary
258-
for name, entry in cached.items()
259-
for function, summary in entry.summaries.items()
260-
}
261-
for manager in fresh.values():
277+
results[name] = entry
278+
reused = tuple(sorted(results))
279+
280+
module_plugins = tuple(p for p in all_plugins if not isinstance(p, ProjectPlugin))
281+
pool = None
282+
if jobs > 1:
283+
pool = concurrent.futures.ProcessPoolExecutor(
284+
max_workers=jobs, mp_context=multiprocessing.get_context("spawn")
285+
)
286+
try:
287+
for wave in graph.schedule():
288+
pending = [c for c in wave if any(m not in results for m in c)]
289+
computed: dict[str, CachedModule] = {}
290+
if pool is None:
291+
for component in pending:
292+
batch = {name: analysable[name] for name in sorted(component)}
293+
computed.update(_analyse_managers(batch, _seed(results, graph, component), module_plugins, affected))
294+
for manager in batch.values():
295+
manager.run(ResultsEvicted)
296+
else:
297+
futures = [
298+
pool.submit(
299+
_analyse_batch,
300+
_Batch(
301+
root,
302+
tuple(plugin_roots),
303+
tuple(plugins),
304+
{name: _path_of(files[name]) for name in sorted(component)},
305+
encode_index(_seed(results, graph, component)),
306+
),
307+
)
308+
for component in pending
309+
]
310+
for future in futures:
311+
computed.update({name: decode(data) for name, data in future.result().items()})
312+
for name, entry in computed.items():
313+
if cache is not None:
314+
cache.store(keys[name], entry)
315+
results.update(computed)
316+
finally:
317+
if pool is not None:
318+
pool.shutdown()
319+
320+
index = _seed(results, graph, frozenset())
321+
call_graphs: dict[str, CallGraph] = {}
322+
for name in sorted(analysable):
323+
entry = results[name]
324+
findings.extend(entry.findings)
325+
sites: dict[str, list[CallSite]] = {function: [] for function in entry.functions}
326+
for site in entry.sites:
327+
sites.setdefault(site.caller, []).append(site)
328+
call_graphs[name] = CallGraph({}, {f: tuple(s) for f, s in sites.items()}, frozenset())
329+
context = ProjectContext(graph, dependencies, advisories, analysable, call_graphs)
330+
for plugin in all_plugins:
331+
if isinstance(plugin, ProjectPlugin):
332+
findings.extend(plugin.analyze_project(context))
333+
return ProjectAnalysis(graph, index, tuple(findings), dependencies, MappingProxyType(keys), reused)
334+
335+
336+
def _seed(results: Mapping[str, CachedModule], graph: ModuleGraph, component: frozenset[str]) -> SummaryIndex:
337+
"""The summaries a component starts from: those of every module it imports,
338+
transitively, all final by the time its wave runs; the whole index for no component."""
339+
340+
if component:
341+
wanted: set[str] = set()
342+
pending = list(component)
343+
while pending:
344+
for imported in graph.imports(pending.pop()):
345+
if imported in results and imported not in wanted:
346+
wanted.add(imported)
347+
pending.append(imported)
348+
else:
349+
wanted = set(results)
350+
return SummaryIndex(
351+
{
352+
project_symbol(name, function): summary
353+
for name in sorted(wanted)
354+
for function, summary in results[name].summaries.items()
355+
}
356+
)
357+
358+
359+
def _analyse_managers(
360+
managers: Mapping[str, AnalysisManager],
361+
seed: SummaryIndex,
362+
plugins: tuple[Plugin, ...],
363+
affected: Mapping[SymbolId, Advisory],
364+
) -> dict[str, CachedModule]:
365+
"""Analyse one component: iterate its summaries to a fixpoint over ``seed`` (§21),
366+
then extract what the rest of the run needs from each module."""
367+
368+
index = seed
369+
for manager in managers.values():
262370
manager.provide(ProjectSummaries, index)
263371
for _ in range(MAX_PROJECT_ITERATIONS):
264372
updated = SummaryIndex(
265373
{
266-
**cached_summaries,
374+
**seed.summaries,
267375
**{
268376
project_symbol(name, function): summary
269-
for name, manager in fresh.items()
377+
for name, manager in managers.items()
270378
for function, summary in _summaries_of(manager).items()
271379
},
272380
}
273381
)
274382
if updated == index:
275383
break
276384
index = updated
277-
for manager in fresh.values():
385+
for manager in managers.values():
278386
manager.run(ProjectSummariesUpdated)
279387
manager.provide(ProjectSummaries, index)
388+
return {name: _analyse_module(manager, plugins, affected) for name, manager in managers.items()}
280389

390+
391+
@dataclass(frozen=True)
392+
class _Batch:
393+
"""One component handed to a worker process: it rebuilds the configuration from the
394+
project root and the plugin roots, so only paths and the imported summaries travel."""
395+
396+
root: Path
397+
plugin_roots: tuple[Path, ...]
398+
plugins: tuple[Plugin, ...]
399+
paths: Mapping[str, Path]
400+
seed: Mapping[str, Any]
401+
402+
403+
def _analyse_batch(batch: _Batch) -> dict[str, dict[str, Any]]:
404+
sources = SourceManager()
405+
managers = {name: _register_all(build_hir(sources.load_file(path))) for name, path in batch.paths.items()}
406+
registry = load_plugins(batch.plugin_roots, next(iter(managers.values())))
407+
all_plugins: tuple[Plugin, ...] = (*(loaded.plugin for loaded in registry), *batch.plugins)
408+
dependencies = resolve_dependencies(batch.root, sources)
409+
advisories = tuple(a for plugin in all_plugins for a in plugin.advisories)
410+
affected = affected_symbols(dependencies, advisories)
411+
models = plugin_models(all_plugins).extended(*advisory_sinks(affected))
412+
for manager in managers.values():
413+
manager.provide(SecurityModelAnalysis, models)
414+
manager.provide(DependencyAnalysis, dependencies)
281415
module_plugins = tuple(p for p in all_plugins if not isinstance(p, ProjectPlugin))
282-
call_graphs: dict[str, CallGraph] = {}
283-
for name in sorted(analysable):
284-
entry = cached.get(name)
285-
if entry is None:
286-
entry = _analyse_module(analysable[name], module_plugins, affected)
287-
if cache is not None:
288-
cache.store(keys[name], entry)
289-
else:
290-
sites: dict[str, list[CallSite]] = {function: [] for function in entry.functions}
291-
for site in entry.sites:
292-
sites.setdefault(site.caller, []).append(site)
293-
call_graphs[name] = CallGraph({}, {f: tuple(s) for f, s in sites.items()}, frozenset())
294-
findings.extend(entry.findings)
295-
context = ProjectContext(graph, dependencies, advisories, analysable, call_graphs)
296-
for plugin in all_plugins:
297-
if isinstance(plugin, ProjectPlugin):
298-
findings.extend(plugin.analyze_project(context))
299-
return ProjectAnalysis(
300-
graph, index, tuple(findings), dependencies, MappingProxyType(keys), tuple(sorted(cached))
301-
)
416+
results = _analyse_managers(managers, decode_index(batch.seed), module_plugins, affected)
417+
return {name: encode(entry) for name, entry in results.items()}
418+
419+
420+
def _path_of(source: SourceFile) -> Path:
421+
assert source.path is not None, "project sources are loaded from files"
422+
return source.path
302423

303424

304425
def _configuration_key(

0 commit comments

Comments
 (0)