@@ -114,6 +114,10 @@ def fallback_problems(
114114 f"{ allowed .seeds } ; this run used --seed-start { run .seed_start } --seeds { run .seeds } "
115115 ]
116116 corpora : frozenset [str ] = _corpus_prefixes (run .corpora )
117+ observed_answers : Counter [tuple [str , str , str ]] = _answer_totals (observed )
118+ allowed_answers : Counter [tuple [str , str , str ]] = _answer_totals (
119+ {** allowed .counts , ** allowed .max_counts }
120+ )
117121 problems : list [str ] = []
118122 for key in sorted ({* allowed .counts , * allowed .max_counts , * observed }):
119123 if key [0 ] not in run .engines :
@@ -128,6 +132,8 @@ def fallback_problems(
128132 actual = actual .get (corpus , 0 ),
129133 exact = None if exact is None else exact .get (corpus , 0 ),
130134 bound = None if bound is None else bound .get (corpus , 0 ),
135+ answers_rose = observed_answers [(key [0 ], key [1 ], corpus )]
136+ > allowed_answers [(key [0 ], key [1 ], corpus )],
131137 )
132138 if problem is not None :
133139 problems .append (problem )
@@ -183,7 +189,13 @@ def write_allow_list(
183189
184190
185191def _corpus_problem (
186- * , label : str , answer : bool , actual : int , exact : int | None , bound : int | None
192+ * ,
193+ label : str ,
194+ answer : bool ,
195+ actual : int ,
196+ exact : int | None ,
197+ bound : int | None ,
198+ answers_rose : bool ,
187199) -> str | None :
188200 if answer and exact is None and bound is None :
189201 return f"{ label } : native answered { actual } , not on the allow-list; record it"
@@ -194,6 +206,12 @@ def _corpus_problem(
194206 )
195207 if exact is None and bound is None :
196208 return f"{ label } : { actual } not on the allow-list; port it or list it with a reason"
209+ if actual == 0 and not answers_rose :
210+ return (
211+ f"{ label } : fallback disappeared but native answers did not appear; the stage may be "
212+ "switched off. A port that removes a fallback must report native answers for the "
213+ "work it now does"
214+ )
197215 if actual == 0 :
198216 return f"{ label } : listed but no longer occurs; remove it from the allow-list"
199217 if bound is not None :
@@ -203,6 +221,18 @@ def _corpus_problem(
203221 return None
204222
205223
224+ def _answer_totals (
225+ entries : dict [FallbackKey , dict [str , int ]],
226+ ) -> Counter [tuple [str , str , str ]]:
227+ """Native answer counts summed by engine, stage and corpus."""
228+
229+ totals : Counter [tuple [str , str , str ]] = Counter ()
230+ for (engine , stage , site , _ ), by_corpus in entries .items ():
231+ for corpus , count in by_corpus .items ():
232+ totals [(engine , stage , corpus )] += count * site .endswith (NATIVE_FALLBACK_ANSWER_SUFFIX )
233+ return totals
234+
235+
206236def _label (* , key : FallbackKey , corpus : str ) -> str :
207237 engine , stage , site , kind = key
208238 return f"{ engine } { stage } { site } { kind } ({ corpus } )"
0 commit comments