|
6 | 6 |
|
7 | 7 | from __future__ import annotations |
8 | 8 |
|
| 9 | +import concurrent.futures |
| 10 | +import multiprocessing |
9 | 11 | from collections.abc import Iterable, Mapping, Sequence |
10 | 12 | from dataclasses import dataclass, field |
11 | 13 | from pathlib import Path |
12 | 14 | from types import MappingProxyType |
13 | | -from typing import ClassVar |
| 15 | +from typing import Any, ClassVar |
14 | 16 |
|
15 | 17 | from coretrace_python import __version__ |
16 | 18 | from coretrace_python.abstract import ConstantPropagation |
|
23 | 25 | from coretrace_python.cache import ( |
24 | 26 | CachedModule, |
25 | 27 | ProjectCache, |
| 28 | + decode, |
| 29 | + decode_index, |
26 | 30 | directory_fingerprint, |
| 31 | + encode, |
| 32 | + encode_index, |
27 | 33 | fingerprint, |
28 | 34 | module_keys, |
29 | 35 | ) |
@@ -128,6 +134,20 @@ def run(cls, ctx: AnalysisContext) -> None: |
128 | 134 | pass |
129 | 135 |
|
130 | 136 |
|
| 137 | +class ResultsEvicted(TransformationPass): |
| 138 | + """Drops a module's PyIR and every derived result once its summaries, call sites and |
| 139 | + findings are extracted (§30); the semantic tables and the engine inputs stay.""" |
| 140 | + |
| 141 | + name: ClassVar[str] = "project.results-evicted" |
| 142 | + preserves: ClassVar[frozenset[AnyAnalysis]] = frozenset( |
| 143 | + {*SEMANTIC_ANALYSES, SecurityModelAnalysis, ProjectSummaries, DependencyAnalysis} |
| 144 | + ) |
| 145 | + |
| 146 | + @classmethod |
| 147 | + def run(cls, ctx: AnalysisContext) -> None: |
| 148 | + pass |
| 149 | + |
| 150 | + |
131 | 151 | def _no_keys() -> Mapping[str, str]: |
132 | 152 | return MappingProxyType({}) |
133 | 153 |
|
@@ -194,12 +214,17 @@ def analyze_project( |
194 | 214 | plugin_roots: Sequence[Path] = (), |
195 | 215 | plugins: Sequence[Plugin] = (), |
196 | 216 | cache: ProjectCache | None = None, |
| 217 | + jobs: int = 1, |
197 | 218 | ) -> ProjectAnalysis: |
198 | 219 | """Analyse every Python file under ``root`` with a shared summary index (§21) and the |
199 | 220 | dependency graph of its manifests (§26). ``plugins`` adds plugin instances to the |
200 | 221 | ones discovered under ``plugin_roots``. With a ``cache``, modules whose key is |
201 | | - unchanged since a previous run are served from it (§11).""" |
| 222 | + unchanged since a previous run are served from it (§11). Modules are scheduled by |
| 223 | + strongly connected components of the module graph, imports first; with ``jobs`` |
| 224 | + above one the components of a wave are analysed in that many processes (§29).""" |
202 | 225 |
|
| 226 | + if jobs < 1: |
| 227 | + raise ValueError("jobs must be at least 1") |
203 | 228 | sources = SourceManager() |
204 | 229 | findings: list[Finding] = [] |
205 | 230 | dependencies = resolve_dependencies(root, sources) |
@@ -244,61 +269,157 @@ def analyze_project( |
244 | 269 | graph, |
245 | 270 | {name: fingerprint(configuration, str(files[name].source_id), name, files[name].text) for name in analysable}, |
246 | 271 | ) |
247 | | - cached: dict[str, CachedModule] = {} |
| 272 | + results: dict[str, CachedModule] = {} |
248 | 273 | if cache is not None: |
249 | 274 | for name in analysable: |
250 | 275 | entry = cache.load(keys[name]) |
251 | 276 | if entry is not None: |
252 | | - cached[name] = entry |
253 | | - fresh = {name: manager for name, manager in analysable.items() if name not in cached} |
254 | | - |
255 | | - index = SummaryIndex() |
256 | | - cached_summaries = { |
257 | | - project_symbol(name, function): summary |
258 | | - for name, entry in cached.items() |
259 | | - for function, summary in entry.summaries.items() |
260 | | - } |
261 | | - for manager in fresh.values(): |
| 277 | + results[name] = entry |
| 278 | + reused = tuple(sorted(results)) |
| 279 | + |
| 280 | + module_plugins = tuple(p for p in all_plugins if not isinstance(p, ProjectPlugin)) |
| 281 | + pool = None |
| 282 | + if jobs > 1: |
| 283 | + pool = concurrent.futures.ProcessPoolExecutor( |
| 284 | + max_workers=jobs, mp_context=multiprocessing.get_context("spawn") |
| 285 | + ) |
| 286 | + try: |
| 287 | + for wave in graph.schedule(): |
| 288 | + pending = [c for c in wave if any(m not in results for m in c)] |
| 289 | + computed: dict[str, CachedModule] = {} |
| 290 | + if pool is None: |
| 291 | + for component in pending: |
| 292 | + batch = {name: analysable[name] for name in sorted(component)} |
| 293 | + computed.update(_analyse_managers(batch, _seed(results, graph, component), module_plugins, affected)) |
| 294 | + for manager in batch.values(): |
| 295 | + manager.run(ResultsEvicted) |
| 296 | + else: |
| 297 | + futures = [ |
| 298 | + pool.submit( |
| 299 | + _analyse_batch, |
| 300 | + _Batch( |
| 301 | + root, |
| 302 | + tuple(plugin_roots), |
| 303 | + tuple(plugins), |
| 304 | + {name: _path_of(files[name]) for name in sorted(component)}, |
| 305 | + encode_index(_seed(results, graph, component)), |
| 306 | + ), |
| 307 | + ) |
| 308 | + for component in pending |
| 309 | + ] |
| 310 | + for future in futures: |
| 311 | + computed.update({name: decode(data) for name, data in future.result().items()}) |
| 312 | + for name, entry in computed.items(): |
| 313 | + if cache is not None: |
| 314 | + cache.store(keys[name], entry) |
| 315 | + results.update(computed) |
| 316 | + finally: |
| 317 | + if pool is not None: |
| 318 | + pool.shutdown() |
| 319 | + |
| 320 | + index = _seed(results, graph, frozenset()) |
| 321 | + call_graphs: dict[str, CallGraph] = {} |
| 322 | + for name in sorted(analysable): |
| 323 | + entry = results[name] |
| 324 | + findings.extend(entry.findings) |
| 325 | + sites: dict[str, list[CallSite]] = {function: [] for function in entry.functions} |
| 326 | + for site in entry.sites: |
| 327 | + sites.setdefault(site.caller, []).append(site) |
| 328 | + call_graphs[name] = CallGraph({}, {f: tuple(s) for f, s in sites.items()}, frozenset()) |
| 329 | + context = ProjectContext(graph, dependencies, advisories, analysable, call_graphs) |
| 330 | + for plugin in all_plugins: |
| 331 | + if isinstance(plugin, ProjectPlugin): |
| 332 | + findings.extend(plugin.analyze_project(context)) |
| 333 | + return ProjectAnalysis(graph, index, tuple(findings), dependencies, MappingProxyType(keys), reused) |
| 334 | + |
| 335 | + |
| 336 | +def _seed(results: Mapping[str, CachedModule], graph: ModuleGraph, component: frozenset[str]) -> SummaryIndex: |
| 337 | + """The summaries a component starts from: those of every module it imports, |
| 338 | + transitively, all final by the time its wave runs; the whole index for no component.""" |
| 339 | + |
| 340 | + if component: |
| 341 | + wanted: set[str] = set() |
| 342 | + pending = list(component) |
| 343 | + while pending: |
| 344 | + for imported in graph.imports(pending.pop()): |
| 345 | + if imported in results and imported not in wanted: |
| 346 | + wanted.add(imported) |
| 347 | + pending.append(imported) |
| 348 | + else: |
| 349 | + wanted = set(results) |
| 350 | + return SummaryIndex( |
| 351 | + { |
| 352 | + project_symbol(name, function): summary |
| 353 | + for name in sorted(wanted) |
| 354 | + for function, summary in results[name].summaries.items() |
| 355 | + } |
| 356 | + ) |
| 357 | + |
| 358 | + |
| 359 | +def _analyse_managers( |
| 360 | + managers: Mapping[str, AnalysisManager], |
| 361 | + seed: SummaryIndex, |
| 362 | + plugins: tuple[Plugin, ...], |
| 363 | + affected: Mapping[SymbolId, Advisory], |
| 364 | +) -> dict[str, CachedModule]: |
| 365 | + """Analyse one component: iterate its summaries to a fixpoint over ``seed`` (§21), |
| 366 | + then extract what the rest of the run needs from each module.""" |
| 367 | + |
| 368 | + index = seed |
| 369 | + for manager in managers.values(): |
262 | 370 | manager.provide(ProjectSummaries, index) |
263 | 371 | for _ in range(MAX_PROJECT_ITERATIONS): |
264 | 372 | updated = SummaryIndex( |
265 | 373 | { |
266 | | - **cached_summaries, |
| 374 | + **seed.summaries, |
267 | 375 | **{ |
268 | 376 | project_symbol(name, function): summary |
269 | | - for name, manager in fresh.items() |
| 377 | + for name, manager in managers.items() |
270 | 378 | for function, summary in _summaries_of(manager).items() |
271 | 379 | }, |
272 | 380 | } |
273 | 381 | ) |
274 | 382 | if updated == index: |
275 | 383 | break |
276 | 384 | index = updated |
277 | | - for manager in fresh.values(): |
| 385 | + for manager in managers.values(): |
278 | 386 | manager.run(ProjectSummariesUpdated) |
279 | 387 | manager.provide(ProjectSummaries, index) |
| 388 | + return {name: _analyse_module(manager, plugins, affected) for name, manager in managers.items()} |
280 | 389 |
|
| 390 | + |
| 391 | +@dataclass(frozen=True) |
| 392 | +class _Batch: |
| 393 | + """One component handed to a worker process: it rebuilds the configuration from the |
| 394 | + project root and the plugin roots, so only paths and the imported summaries travel.""" |
| 395 | + |
| 396 | + root: Path |
| 397 | + plugin_roots: tuple[Path, ...] |
| 398 | + plugins: tuple[Plugin, ...] |
| 399 | + paths: Mapping[str, Path] |
| 400 | + seed: Mapping[str, Any] |
| 401 | + |
| 402 | + |
| 403 | +def _analyse_batch(batch: _Batch) -> dict[str, dict[str, Any]]: |
| 404 | + sources = SourceManager() |
| 405 | + managers = {name: _register_all(build_hir(sources.load_file(path))) for name, path in batch.paths.items()} |
| 406 | + registry = load_plugins(batch.plugin_roots, next(iter(managers.values()))) |
| 407 | + all_plugins: tuple[Plugin, ...] = (*(loaded.plugin for loaded in registry), *batch.plugins) |
| 408 | + dependencies = resolve_dependencies(batch.root, sources) |
| 409 | + advisories = tuple(a for plugin in all_plugins for a in plugin.advisories) |
| 410 | + affected = affected_symbols(dependencies, advisories) |
| 411 | + models = plugin_models(all_plugins).extended(*advisory_sinks(affected)) |
| 412 | + for manager in managers.values(): |
| 413 | + manager.provide(SecurityModelAnalysis, models) |
| 414 | + manager.provide(DependencyAnalysis, dependencies) |
281 | 415 | module_plugins = tuple(p for p in all_plugins if not isinstance(p, ProjectPlugin)) |
282 | | - call_graphs: dict[str, CallGraph] = {} |
283 | | - for name in sorted(analysable): |
284 | | - entry = cached.get(name) |
285 | | - if entry is None: |
286 | | - entry = _analyse_module(analysable[name], module_plugins, affected) |
287 | | - if cache is not None: |
288 | | - cache.store(keys[name], entry) |
289 | | - else: |
290 | | - sites: dict[str, list[CallSite]] = {function: [] for function in entry.functions} |
291 | | - for site in entry.sites: |
292 | | - sites.setdefault(site.caller, []).append(site) |
293 | | - call_graphs[name] = CallGraph({}, {f: tuple(s) for f, s in sites.items()}, frozenset()) |
294 | | - findings.extend(entry.findings) |
295 | | - context = ProjectContext(graph, dependencies, advisories, analysable, call_graphs) |
296 | | - for plugin in all_plugins: |
297 | | - if isinstance(plugin, ProjectPlugin): |
298 | | - findings.extend(plugin.analyze_project(context)) |
299 | | - return ProjectAnalysis( |
300 | | - graph, index, tuple(findings), dependencies, MappingProxyType(keys), tuple(sorted(cached)) |
301 | | - ) |
| 416 | + results = _analyse_managers(managers, decode_index(batch.seed), module_plugins, affected) |
| 417 | + return {name: encode(entry) for name, entry in results.items()} |
| 418 | + |
| 419 | + |
| 420 | +def _path_of(source: SourceFile) -> Path: |
| 421 | + assert source.path is not None, "project sources are loaded from files" |
| 422 | + return source.path |
302 | 423 |
|
303 | 424 |
|
304 | 425 | def _configuration_key( |
|
0 commit comments