1010import hashlib
1111import multiprocessing
1212import sys
13+ import tempfile
1314from collections import Counter
1415from collections .abc import Callable , Iterable , Mapping , Sequence
1516from dataclasses import dataclass , field , replace
3031 CachedModule ,
3132 ProjectCache ,
3233 decode ,
33- decode_index ,
3434 directory_fingerprint ,
3535 encode ,
36- encode_index ,
3736 fingerprint ,
3837 module_keys ,
3938)
@@ -508,9 +507,30 @@ def analyze_project(
508507 module_plugins = tuple (p for p in all_plugins if not isinstance (p , ProjectPlugin ))
509508 suffixes = suffix_index (affected , analysable )
510509 pool = None
510+ spooled : tempfile .TemporaryDirectory [str ] | None = None
511+ # Where finished modules are kept for the next waves: the cache, else a directory of
512+ # this run's own. Workers read from it what a component imports (#245).
513+ spool = cache
511514 if jobs > 1 :
515+ if spool is None :
516+ spooled = tempfile .TemporaryDirectory (prefix = "coretrace-" )
517+ spool = ProjectCache (Path (spooled .name ))
518+ project = _Project (
519+ root ,
520+ tuple (plugin_roots ),
521+ tuple (plugins ),
522+ advisory_paths ,
523+ _encode_routes (routes ),
524+ tuple (sorted (escaped )),
525+ template_calls ,
526+ tuple (sorted (analysable )),
527+ spool .directory ,
528+ )
512529 pool = concurrent .futures .ProcessPoolExecutor (
513- max_workers = process_pool_size (jobs ), mp_context = multiprocessing .get_context ("spawn" )
530+ max_workers = process_pool_size (jobs ),
531+ mp_context = multiprocessing .get_context ("spawn" ),
532+ initializer = _start_worker ,
533+ initargs = (project ,),
514534 )
515535 try :
516536 for wave in graph .schedule ():
@@ -529,29 +549,27 @@ def analyze_project(
529549 pool .submit (
530550 _analyse_batch ,
531551 _Batch (
532- root ,
533- tuple (plugin_roots ),
534- tuple (plugins ),
535552 {name : _path_of (files [name ]) for name in sorted (component )},
536- encode_index (_seed (results , graph , component )),
537- advisory_paths ,
538- _encode_routes (routes ),
539- tuple (sorted (escaped )),
540- template_calls ,
541- project = tuple (sorted (analysable )),
553+ {
554+ name : (import_name , keys [name ])
555+ for name , import_name in _imported_names (graph , component , results ).items ()
556+ },
542557 ),
543558 )
544559 for component in pending
545560 ]
546561 for future in futures :
547562 computed .update ({name : decode (data ) for name , data in future .result ().items ()})
563+ store = cache if pool is None else spool
548564 for name , entry in computed .items ():
549- if cache is not None :
550- cache .store (keys [name ], entry )
565+ if store is not None :
566+ store .store (keys [name ], entry )
551567 results .update (computed )
552568 finally :
553569 if pool is not None :
554570 pool .shutdown ()
571+ if spooled is not None :
572+ spooled .cleanup ()
555573
556574 index = _index (results , {name : name for name in results })
557575 call_graphs : dict [str , CallGraph ] = {}
@@ -720,12 +738,20 @@ def located(name: str) -> str:
720738
721739
722740def _seed (results : Mapping [str , CachedModule ], graph : ModuleGraph , component : frozenset [str ]) -> SummaryIndex :
723- """The summaries a component starts from: those of every module it imports,
724- transitively, all final by the time its wave runs, each under the import name the
725- component's code refers to it by. Of several files with one import name, only the
726- one a member imports directly, resolved in its own root, goes under that name: the
727- others are reached through summaries that inline them already, and a name no
728- member's root resolved is followed into neither file."""
741+ """The summaries a component starts from (``_imported_names``)."""
742+
743+ return _index (results , _imported_names (graph , component , results ))
744+
745+
746+ def _imported_names (
747+ graph : ModuleGraph , component : frozenset [str ], results : Mapping [str , CachedModule ]
748+ ) -> dict [str , str ]:
749+ """The modules whose summaries a component starts from, each with the import name the
750+ component's code refers to it by: every module it imports, transitively, all final
751+ by the time its wave runs. Of several files with one import name, only the one a
752+ member imports directly, resolved in its own root, goes under that name: the others
753+ are reached through summaries that inline them already, and a name no member's root
754+ resolved is followed into neither file."""
729755
730756 wanted : set [str ] = set ()
731757 pending = list (component )
@@ -743,7 +769,7 @@ def _seed(results: Mapping[str, CachedModule], graph: ModuleGraph, component: fr
743769 chosen = modules if len (modules ) == 1 else [name for name in modules if name in direct ]
744770 if len (chosen ) == 1 :
745771 names [chosen [0 ]] = import_name
746- return _index ( results , names )
772+ return names
747773
748774
749775def _index (results : Mapping [str , CachedModule ], names : Mapping [str , str ]) -> SummaryIndex :
@@ -797,57 +823,118 @@ def _analyse_managers(
797823
798824
799825@dataclass (frozen = True )
800- class _Batch :
801- """One component handed to a worker process, its files under the engine's module
802- names: it rebuilds the configuration from the project root and the plugin roots, so
803- only paths and the imported summaries travel."""
826+ class _Project :
827+ """What every worker process needs of the project, whatever the component it
828+ analyses: it rebuilds the configuration from the project root and the plugin roots
829+ once, when the pool starts it (``_start_worker``), and reads the finished modules
830+ from ``spool``, a ``ProjectCache`` directory the parent writes after each wave."""
804831
805832 root : Path
806833 plugin_roots : tuple [Path , ...]
807834 plugins : tuple [Plugin , ...]
808- paths : Mapping [str , Path ]
809- seed : Mapping [str , Any ]
810- advisory_paths : tuple [Path , ...] = ()
811- routes : tuple [tuple [str , str , str , int ], ...] = ()
812- escaped : tuple [str , ...] = ()
813- template_calls : TemplateCalls = field (default_factory = TemplateCalls )
814- # Every analysable module of the project, not only the component's: a curated
835+ advisory_paths : tuple [Path , ...]
836+ routes : tuple [tuple [str , str , str , int ], ...]
837+ escaped : tuple [str , ...]
838+ template_calls : TemplateCalls
839+ # Every analysable module of the project, not only a component's: a curated
815840 # entry-point suffix matches derived symbols of any project module.
816- project : tuple [str , ...] = ()
841+ modules : tuple [str , ...]
842+ spool : Path
817843
818844
819- def _analyse_batch (batch : _Batch ) -> dict [str , dict [str , Any ]]:
845+ @dataclass (frozen = True )
846+ class _Batch :
847+ """One component handed to a worker process: its files under the engine's module
848+ names, and the modules its summaries start from, each with its import name and the
849+ key of its entry in the spool. Only these travel, not the summaries themselves."""
850+
851+ paths : Mapping [str , Path ]
852+ imported : Mapping [str , tuple [str , str ]]
853+
854+
855+ @dataclass
856+ class _Worker :
857+ """The configuration a worker process prepared once, and the finished modules it has
858+ read from the spool so far, by key."""
859+
860+ managed : Callable [[AnalysisManager ], None ]
861+ plugins : tuple [Plugin , ...]
862+ affected : Affected
863+ suffixes : SuffixIndex
864+ spool : ProjectCache
865+ read : dict [str , CachedModule ] = field (default_factory = dict )
866+
867+ def entries (self , imported : Mapping [str , tuple [str , str ]]) -> dict [str , CachedModule ]:
868+ found : dict [str , CachedModule ] = {}
869+ for name , (_ , key ) in imported .items ():
870+ entry = self .read .get (key )
871+ if entry is None :
872+ entry = self .spool .load (key )
873+ if entry is None :
874+ raise RuntimeError (f"the summaries of { name } are missing from { self .spool .directory } " )
875+ self .read [key ] = entry
876+ found [name ] = entry
877+ return found
878+
879+
880+ # The state of a worker process, set by ``_start_worker`` when the pool starts it.
881+ _WORKER : _Worker | None = None
882+
883+
884+ def _start_worker (project : _Project ) -> None :
885+ """Prepare this worker process for every component it will analyse (#245)."""
886+
887+ global _WORKER
820888 sources = SourceManager ()
821- managers = { name : _register_all (build_hir (sources .load_file ( path ))) for name , path in batch . paths . items ()}
822- registry = load_plugins (batch .plugin_roots , next ( iter ( managers . values ())) )
823- all_plugins : tuple [Plugin , ...] = (* (loaded .plugin for loaded in registry ), * batch .plugins )
824- dependencies = resolve_dependencies (batch .root , sources )
889+ probe = _register_all (build_hir (sources .add_source ( "<empty>" , "" )))
890+ registry = load_plugins (project .plugin_roots , probe )
891+ all_plugins : tuple [Plugin , ...] = (* (loaded .plugin for loaded in registry ), * project .plugins )
892+ dependencies = resolve_dependencies (project .root , sources )
825893 file_advisories : list [tuple [Path , tuple [Advisory , ...]]] = []
826- for path in batch .advisory_paths :
894+ for path in project .advisory_paths :
827895 try :
828896 file_advisories .append ((path , load_advisories (path )))
829897 except AdvisoryFileError :
830898 continue
831- advisories , _ = _merge_advisories (_contributions (registry , batch .plugins , batch .root , file_advisories ))
899+ advisories , _ = _merge_advisories (_contributions (registry , project .plugins , project .root , file_advisories ))
832900 affected = affected_symbols (dependencies , advisories )
833- models = plugin_models (all_plugins , batch .root ).extended (
901+ models = plugin_models (all_plugins , project .root ).extended (
834902 * advisory_sinks (affected ), suffixes = advisory_suffix_sinks (affected )
835903 )
836- routes = _decode_routes (batch .routes )
837- escaped = frozenset (batch .escaped )
904+ routes = _decode_routes (project .routes )
905+ escaped = frozenset (project .escaped )
838906 members = models .members_by_class ()
839- for manager in managers .values ():
907+ signatures = models .signatures ()
908+ clearing = models .clearing (escaped )
909+
910+ def managed (manager : AnalysisManager ) -> None :
840911 manager .provide (SecurityModelAnalysis , models )
841912 manager .provide (MembersAnalysis , members )
842- manager .provide (SignaturesAnalysis , models . signatures () )
913+ manager .provide (SignaturesAnalysis , signatures )
843914 manager .provide (DependencyAnalysis , dependencies )
844915 manager .provide (RegisteredRoutes , routes )
845916 manager .provide (EscapedTemplates , escaped )
846- manager .provide (ClearingAnalysis , models .clearing (escaped ))
847- manager .provide (TemplateCallsAnalysis , batch .template_calls )
848- module_plugins = tuple (p for p in all_plugins if not isinstance (p , ProjectPlugin ))
849- suffixes = suffix_index (affected , batch .project )
850- results = _analyse_managers (managers , decode_index (batch .seed ), module_plugins , affected , suffixes )
917+ manager .provide (ClearingAnalysis , clearing )
918+ manager .provide (TemplateCallsAnalysis , project .template_calls )
919+
920+ _WORKER = _Worker (
921+ managed ,
922+ tuple (p for p in all_plugins if not isinstance (p , ProjectPlugin )),
923+ affected ,
924+ suffix_index (affected , project .modules ),
925+ ProjectCache (project .spool ),
926+ )
927+
928+
929+ def _analyse_batch (batch : _Batch ) -> dict [str , dict [str , Any ]]:
930+ worker = _WORKER
931+ assert worker is not None , "the pool starts every worker with _start_worker"
932+ sources = SourceManager ()
933+ managers = {name : _register_all (build_hir (sources .load_file (path ))) for name , path in batch .paths .items ()}
934+ for manager in managers .values ():
935+ worker .managed (manager )
936+ seed = _index (worker .entries (batch .imported ), {name : import_name for name , (import_name , _ ) in batch .imported .items ()})
937+ results = _analyse_managers (managers , seed , worker .plugins , worker .affected , worker .suffixes )
851938 return {name : encode (entry ) for name , entry in results .items ()}
852939
853940
0 commit comments