2020import logging
2121import sys
2222from pathlib import Path
23+ from typing import cast
2324
2425from src .extensions .score_source_code_linker .helpers import parse_info_from_known_good
2526
2627logging .basicConfig (level = logging .INFO , format = "%(message)s" )
2728logger = logging .getLogger (__name__ )
2829
2930
31+ def _reference_key (reference : dict [str , object ]) -> str :
32+ """Return a stable, hashable representation of one source-link reference."""
33+ # References are dictionaries and therefore cannot be added to ``seen`` directly.
34+ # Sorting their JSON keys makes semantically identical dictionaries produce the
35+ # same key even when their input key order differs.
36+ return json .dumps (reference , sort_keys = True )
37+
38+
39+ def _merge_sourcelinks_file (
40+ json_file : Path ,
41+ known_good : Path | None ,
42+ merged : list [dict [str , object ]],
43+ seen : set [str ],
44+ ) -> None :
45+ """Add the unique references from one sourcelinks JSON file to ``merged``."""
46+ with open (json_file , encoding = "utf-8" ) as file :
47+ data = cast (list [object ], json .load (file ))
48+ if not data :
49+ return
50+
51+ raw_metadata = data [0 ]
52+ if not isinstance (raw_metadata , dict ) or "repo_name" not in raw_metadata :
53+ logger .warning (
54+ f"Unexpected schema in sourcelinks file '{ json_file } ': "
55+ "expected first element to be a metadata dict "
56+ "with a 'repo_name' key. "
57+ )
58+ return
59+ metadata = cast (dict [str , object ], raw_metadata )
60+ repo_name = metadata ["repo_name" ]
61+ if not isinstance (repo_name , str ):
62+ logger .warning (
63+ f"Unexpected schema in sourcelinks file '{ json_file } ': "
64+ "expected metadata 'repo_name' to be a string. "
65+ )
66+ return
67+
68+ # A known-good file is optional for standalone builds that include
69+ # documentation from external modules. In that case, keep the metadata
70+ # produced by the individual sourcelinks file.
71+ if known_good and repo_name and repo_name != "local_repo" :
72+ hash , repo = parse_info_from_known_good (
73+ known_good_json = known_good , repo_name = repo_name
74+ )
75+ metadata ["hash" ] = hash
76+ metadata ["url" ] = repo
77+
78+ for raw_reference in data [1 :]:
79+ reference = cast (dict [str , object ], raw_reference )
80+ reference .update (metadata )
81+ key = _reference_key (reference )
82+ if key not in seen :
83+ seen .add (key )
84+ merged .append (reference )
85+
86+
3087def main ():
3188 parser = argparse .ArgumentParser (
3289 description = "Merge multiple sourcelinks JSON files into one"
@@ -39,6 +96,7 @@ def main():
3996 )
4097 _ = parser .add_argument (
4198 "--known_good" ,
99+ type = Path ,
42100 help = "Path to a required 'known good' JSON file (provided by Bazel)." ,
43101 )
44102 _ = parser .add_argument (
@@ -49,52 +107,15 @@ def main():
49107 )
50108
51109 args = parser .parse_args ()
52- all_files = [x for x in args .files if "known_good.json" not in str (x )]
53-
54- merged = []
55- seen = set ()
56- for json_file in all_files :
57- with open (json_file ) as f :
58- data = json .load (f )
59- # If the file is empty e.g. '[]' there is nothing to parse, we continue
60- if not data :
61- continue
62- metadata = data [0 ]
63- if not isinstance (metadata , dict ) or "repo_name" not in metadata :
64- logger .warning (
65- f"Unexpected schema in sourcelinks file '{ json_file } ': "
66- "expected first element to be a metadata dict "
67- "with a 'repo_name' key. "
68- )
69- # As we can't deal with bad JSON structure we just skip it
70- continue
71- # A known-good file is optional for standalone builds that include
72- # documentation from external modules. In that case, keep the
73- # metadata produced by the individual sourcelinks file.
74- if (
75- args .known_good
76- and metadata ["repo_name" ]
77- and metadata ["repo_name" ] != "local_repo"
78- ):
79- hash , repo = parse_info_from_known_good (
80- known_good_json = args .known_good , repo_name = metadata ["repo_name" ]
81- )
82- metadata ["hash" ] = hash
83- metadata ["url" ] = repo
84- # In the case that 'metadata[repo_name]' is 'local_module'
85- # hash & url are already existing and empty inside of 'metadata'
86- # Therefore all 3 keys will be written to needlinks in each branch
87-
88- for d in data [1 :]:
89- d .update (metadata )
90- assert isinstance (data , list ), repr (data )
91- for reference in data [1 :]:
92- key = json .dumps (reference , sort_keys = True )
93- if key not in seen :
94- seen .add (key )
95- merged .append (reference )
96- with open (args .output , "w" ) as f :
97- json .dump (merged , f , indent = 2 , ensure_ascii = False )
110+
111+ merged : list [dict [str , object ]] = []
112+ seen : set [str ] = set ()
113+ for json_file in args .files :
114+ if "known_good.json" not in str (json_file ):
115+ _merge_sourcelinks_file (json_file , args .known_good , merged , seen )
116+
117+ with open (args .output , "w" , encoding = "utf-8" ) as file :
118+ json .dump (merged , file , indent = 2 , ensure_ascii = False )
98119
99120 logger .info (f"Merged { len (args .files )} files into { len (merged )} total references" )
100121 return 0
0 commit comments