Skip to content

Commit 1cb7b7b

Browse files
committed
perf(engine): prepare each worker once and read summaries from a spool
Closes #245
1 parent 074d63f commit 1cb7b7b

4 files changed

Lines changed: 251 additions & 63 deletions

File tree

‎docs/usage.md‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -757,11 +757,24 @@ Measured on one core of a laptop, single process, cold start, with the bundled p
757757

758758
Time grows a little faster than linearly with the size of the project, because the
759759
interprocedural summaries are iterated to a fixpoint over the module graph. A warm
760-
`--cache` brings an unchanged healthchecks to about one second, `--jobs 4` saves a
761-
quarter of the cold time at the cost of one process's memory per job. The
760+
`--cache` brings an unchanged healthchecks to about one second. The
762761
`healthchecks` repository is part of the regression corpus, so the analysis time of a
763762
real 45 000-line project is exercised on every change.
764763

764+
`--jobs` pays when a wave of the module graph holds many components: each process
765+
prepares the project once and reads the summaries of the modules a component imports
766+
from a directory of finished modules, the cache or one of the run's own, at the cost of
767+
one process's memory per job. Measured on a laptop with 8 cores (4 performance, 4
768+
efficiency), macOS, Python 3.14, bundled plugins, cold start (October 2026):
769+
770+
| Project | Python files | `-j 1` | `-j 8` |
771+
|---|---:|---:|---:|
772+
| healthchecks | 653 | 15.3–15.5 s | 7.9–9.2 s |
773+
| mitmproxy | 583 | 51.6 s | 45.6 s |
774+
775+
The modules of a strongly connected import cycle are analysed together by one process,
776+
and a wave waits for its slowest component: a project whose cycles are large gains less.
777+
765778
## Plugins
766779

767780
The package ships 35 plugins, loaded by default. Security models for the standard library

‎src/coretrace_python/cache.py‎

Lines changed: 0 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,6 @@
3535
Mutation,
3636
NonlocalWrite,
3737
PriorCall,
38-
SummaryIndex,
3938
SymbolRead,
4039
Target,
4140
UnknownTarget,
@@ -155,16 +154,6 @@ def decode(data: Mapping[str, Any]) -> CachedModule:
155154
)
156155

157156

158-
def encode_index(index: SummaryIndex) -> dict[str, Any]:
159-
"""A summary index as plain data, to hand a worker process what it imports."""
160-
161-
return {str(symbol): _encode_summary(index.summaries[symbol]) for symbol in index.symbols}
162-
163-
164-
def decode_index(data: Mapping[str, Any]) -> SummaryIndex:
165-
return SummaryIndex({SymbolId(_string(k)): _decode_summary(v) for k, v in data.items()})
166-
167-
168157
def _string(value: Any) -> str:
169158
if not isinstance(value, str):
170159
raise TypeError(f"expected a string, got {value!r}")

‎src/coretrace_python/engine.py‎

Lines changed: 137 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
import hashlib
1111
import multiprocessing
1212
import sys
13+
import tempfile
1314
from collections import Counter
1415
from collections.abc import Callable, Iterable, Mapping, Sequence
1516
from dataclasses import dataclass, field, replace
@@ -30,10 +31,8 @@
3031
CachedModule,
3132
ProjectCache,
3233
decode,
33-
decode_index,
3434
directory_fingerprint,
3535
encode,
36-
encode_index,
3736
fingerprint,
3837
module_keys,
3938
)
@@ -508,9 +507,30 @@ def analyze_project(
508507
module_plugins = tuple(p for p in all_plugins if not isinstance(p, ProjectPlugin))
509508
suffixes = suffix_index(affected, analysable)
510509
pool = None
510+
spooled: tempfile.TemporaryDirectory[str] | None = None
511+
# Where finished modules are kept for the next waves: the cache, else a directory of
512+
# this run's own. Workers read from it what a component imports (#245).
513+
spool = cache
511514
if jobs > 1:
515+
if spool is None:
516+
spooled = tempfile.TemporaryDirectory(prefix="coretrace-")
517+
spool = ProjectCache(Path(spooled.name))
518+
project = _Project(
519+
root,
520+
tuple(plugin_roots),
521+
tuple(plugins),
522+
advisory_paths,
523+
_encode_routes(routes),
524+
tuple(sorted(escaped)),
525+
template_calls,
526+
tuple(sorted(analysable)),
527+
spool.directory,
528+
)
512529
pool = concurrent.futures.ProcessPoolExecutor(
513-
max_workers=process_pool_size(jobs), mp_context=multiprocessing.get_context("spawn")
530+
max_workers=process_pool_size(jobs),
531+
mp_context=multiprocessing.get_context("spawn"),
532+
initializer=_start_worker,
533+
initargs=(project,),
514534
)
515535
try:
516536
for wave in graph.schedule():
@@ -529,29 +549,27 @@ def analyze_project(
529549
pool.submit(
530550
_analyse_batch,
531551
_Batch(
532-
root,
533-
tuple(plugin_roots),
534-
tuple(plugins),
535552
{name: _path_of(files[name]) for name in sorted(component)},
536-
encode_index(_seed(results, graph, component)),
537-
advisory_paths,
538-
_encode_routes(routes),
539-
tuple(sorted(escaped)),
540-
template_calls,
541-
project=tuple(sorted(analysable)),
553+
{
554+
name: (import_name, keys[name])
555+
for name, import_name in _imported_names(graph, component, results).items()
556+
},
542557
),
543558
)
544559
for component in pending
545560
]
546561
for future in futures:
547562
computed.update({name: decode(data) for name, data in future.result().items()})
563+
store = cache if pool is None else spool
548564
for name, entry in computed.items():
549-
if cache is not None:
550-
cache.store(keys[name], entry)
565+
if store is not None:
566+
store.store(keys[name], entry)
551567
results.update(computed)
552568
finally:
553569
if pool is not None:
554570
pool.shutdown()
571+
if spooled is not None:
572+
spooled.cleanup()
555573

556574
index = _index(results, {name: name for name in results})
557575
call_graphs: dict[str, CallGraph] = {}
@@ -720,12 +738,20 @@ def located(name: str) -> str:
720738

721739

722740
def _seed(results: Mapping[str, CachedModule], graph: ModuleGraph, component: frozenset[str]) -> SummaryIndex:
723-
"""The summaries a component starts from: those of every module it imports,
724-
transitively, all final by the time its wave runs, each under the import name the
725-
component's code refers to it by. Of several files with one import name, only the
726-
one a member imports directly, resolved in its own root, goes under that name: the
727-
others are reached through summaries that inline them already, and a name no
728-
member's root resolved is followed into neither file."""
741+
"""The summaries a component starts from (``_imported_names``)."""
742+
743+
return _index(results, _imported_names(graph, component, results))
744+
745+
746+
def _imported_names(
747+
graph: ModuleGraph, component: frozenset[str], results: Mapping[str, CachedModule]
748+
) -> dict[str, str]:
749+
"""The modules whose summaries a component starts from, each with the import name the
750+
component's code refers to it by: every module it imports, transitively, all final
751+
by the time its wave runs. Of several files with one import name, only the one a
752+
member imports directly, resolved in its own root, goes under that name: the others
753+
are reached through summaries that inline them already, and a name no member's root
754+
resolved is followed into neither file."""
729755

730756
wanted: set[str] = set()
731757
pending = list(component)
@@ -743,7 +769,7 @@ def _seed(results: Mapping[str, CachedModule], graph: ModuleGraph, component: fr
743769
chosen = modules if len(modules) == 1 else [name for name in modules if name in direct]
744770
if len(chosen) == 1:
745771
names[chosen[0]] = import_name
746-
return _index(results, names)
772+
return names
747773

748774

749775
def _index(results: Mapping[str, CachedModule], names: Mapping[str, str]) -> SummaryIndex:
@@ -797,57 +823,118 @@ def _analyse_managers(
797823

798824

799825
@dataclass(frozen=True)
800-
class _Batch:
801-
"""One component handed to a worker process, its files under the engine's module
802-
names: it rebuilds the configuration from the project root and the plugin roots, so
803-
only paths and the imported summaries travel."""
826+
class _Project:
827+
"""What every worker process needs of the project, whatever the component it
828+
analyses: it rebuilds the configuration from the project root and the plugin roots
829+
once, when the pool starts it (``_start_worker``), and reads the finished modules
830+
from ``spool``, a ``ProjectCache`` directory the parent writes after each wave."""
804831

805832
root: Path
806833
plugin_roots: tuple[Path, ...]
807834
plugins: tuple[Plugin, ...]
808-
paths: Mapping[str, Path]
809-
seed: Mapping[str, Any]
810-
advisory_paths: tuple[Path, ...] = ()
811-
routes: tuple[tuple[str, str, str, int], ...] = ()
812-
escaped: tuple[str, ...] = ()
813-
template_calls: TemplateCalls = field(default_factory=TemplateCalls)
814-
# Every analysable module of the project, not only the component's: a curated
835+
advisory_paths: tuple[Path, ...]
836+
routes: tuple[tuple[str, str, str, int], ...]
837+
escaped: tuple[str, ...]
838+
template_calls: TemplateCalls
839+
# Every analysable module of the project, not only a component's: a curated
815840
# entry-point suffix matches derived symbols of any project module.
816-
project: tuple[str, ...] = ()
841+
modules: tuple[str, ...]
842+
spool: Path
817843

818844

819-
def _analyse_batch(batch: _Batch) -> dict[str, dict[str, Any]]:
845+
@dataclass(frozen=True)
846+
class _Batch:
847+
"""One component handed to a worker process: its files under the engine's module
848+
names, and the modules its summaries start from, each with its import name and the
849+
key of its entry in the spool. Only these travel, not the summaries themselves."""
850+
851+
paths: Mapping[str, Path]
852+
imported: Mapping[str, tuple[str, str]]
853+
854+
855+
@dataclass
856+
class _Worker:
857+
"""The configuration a worker process prepared once, and the finished modules it has
858+
read from the spool so far, by key."""
859+
860+
managed: Callable[[AnalysisManager], None]
861+
plugins: tuple[Plugin, ...]
862+
affected: Affected
863+
suffixes: SuffixIndex
864+
spool: ProjectCache
865+
read: dict[str, CachedModule] = field(default_factory=dict)
866+
867+
def entries(self, imported: Mapping[str, tuple[str, str]]) -> dict[str, CachedModule]:
868+
found: dict[str, CachedModule] = {}
869+
for name, (_, key) in imported.items():
870+
entry = self.read.get(key)
871+
if entry is None:
872+
entry = self.spool.load(key)
873+
if entry is None:
874+
raise RuntimeError(f"the summaries of {name} are missing from {self.spool.directory}")
875+
self.read[key] = entry
876+
found[name] = entry
877+
return found
878+
879+
880+
# The state of a worker process, set by ``_start_worker`` when the pool starts it.
881+
_WORKER: _Worker | None = None
882+
883+
884+
def _start_worker(project: _Project) -> None:
885+
"""Prepare this worker process for every component it will analyse (#245)."""
886+
887+
global _WORKER
820888
sources = SourceManager()
821-
managers = {name: _register_all(build_hir(sources.load_file(path))) for name, path in batch.paths.items()}
822-
registry = load_plugins(batch.plugin_roots, next(iter(managers.values())))
823-
all_plugins: tuple[Plugin, ...] = (*(loaded.plugin for loaded in registry), *batch.plugins)
824-
dependencies = resolve_dependencies(batch.root, sources)
889+
probe = _register_all(build_hir(sources.add_source("<empty>", "")))
890+
registry = load_plugins(project.plugin_roots, probe)
891+
all_plugins: tuple[Plugin, ...] = (*(loaded.plugin for loaded in registry), *project.plugins)
892+
dependencies = resolve_dependencies(project.root, sources)
825893
file_advisories: list[tuple[Path, tuple[Advisory, ...]]] = []
826-
for path in batch.advisory_paths:
894+
for path in project.advisory_paths:
827895
try:
828896
file_advisories.append((path, load_advisories(path)))
829897
except AdvisoryFileError:
830898
continue
831-
advisories, _ = _merge_advisories(_contributions(registry, batch.plugins, batch.root, file_advisories))
899+
advisories, _ = _merge_advisories(_contributions(registry, project.plugins, project.root, file_advisories))
832900
affected = affected_symbols(dependencies, advisories)
833-
models = plugin_models(all_plugins, batch.root).extended(
901+
models = plugin_models(all_plugins, project.root).extended(
834902
*advisory_sinks(affected), suffixes=advisory_suffix_sinks(affected)
835903
)
836-
routes = _decode_routes(batch.routes)
837-
escaped = frozenset(batch.escaped)
904+
routes = _decode_routes(project.routes)
905+
escaped = frozenset(project.escaped)
838906
members = models.members_by_class()
839-
for manager in managers.values():
907+
signatures = models.signatures()
908+
clearing = models.clearing(escaped)
909+
910+
def managed(manager: AnalysisManager) -> None:
840911
manager.provide(SecurityModelAnalysis, models)
841912
manager.provide(MembersAnalysis, members)
842-
manager.provide(SignaturesAnalysis, models.signatures())
913+
manager.provide(SignaturesAnalysis, signatures)
843914
manager.provide(DependencyAnalysis, dependencies)
844915
manager.provide(RegisteredRoutes, routes)
845916
manager.provide(EscapedTemplates, escaped)
846-
manager.provide(ClearingAnalysis, models.clearing(escaped))
847-
manager.provide(TemplateCallsAnalysis, batch.template_calls)
848-
module_plugins = tuple(p for p in all_plugins if not isinstance(p, ProjectPlugin))
849-
suffixes = suffix_index(affected, batch.project)
850-
results = _analyse_managers(managers, decode_index(batch.seed), module_plugins, affected, suffixes)
917+
manager.provide(ClearingAnalysis, clearing)
918+
manager.provide(TemplateCallsAnalysis, project.template_calls)
919+
920+
_WORKER = _Worker(
921+
managed,
922+
tuple(p for p in all_plugins if not isinstance(p, ProjectPlugin)),
923+
affected,
924+
suffix_index(affected, project.modules),
925+
ProjectCache(project.spool),
926+
)
927+
928+
929+
def _analyse_batch(batch: _Batch) -> dict[str, dict[str, Any]]:
930+
worker = _WORKER
931+
assert worker is not None, "the pool starts every worker with _start_worker"
932+
sources = SourceManager()
933+
managers = {name: _register_all(build_hir(sources.load_file(path))) for name, path in batch.paths.items()}
934+
for manager in managers.values():
935+
worker.managed(manager)
936+
seed = _index(worker.entries(batch.imported), {name: import_name for name, (import_name, _) in batch.imported.items()})
937+
results = _analyse_managers(managers, seed, worker.plugins, worker.affected, worker.suffixes)
851938
return {name: encode(entry) for name, entry in results.items()}
852939

853940

0 commit comments

Comments
 (0)