From 52640ca8fe973d05fbdd70e3f3c71429c3a7c653 Mon Sep 17 00:00:00 2001 From: Robert Jackson Date: Wed, 16 Sep 2026 06:57:46 -0500 Subject: [PATCH] Add no-codebook baseline support and a self-referential prev-label source Two changes to label_radar_data, both needed to make codebook ablations measurable. Allow definition-free categories. The prompt builder emitted the "Each category is defined as follows:" block unconditionally, so passing a category map with empty descriptions produced "No Precipitation: ; Stratiform Precipitation: ;". The block is now skipped when no category has a description, which makes a names-only control arm expressible: the model gets the class names and nothing else, so the codebook's contribution can be attributed instead of assumed. Behaviour is unchanged for every existing caller, since they all pass descriptions. Add previous_label_source, defaulting to "hand" (existing behaviour). use_previous_labels injected the *human* label of the preceding scan. When scans are closely spaced their labels rarely change, so that hint is close to the answer: on the BNF CSAPR2 set (~10 minute spacing), copying the previous hand label with no image at all reaches 0.80 macro-F1 against a two-human Dawid-Skene consensus, beating every model configuration measured there. Passing "llm" instead injects the model's own previous prediction, which keeps ground truth out of the prompt and matches what is available at inference time. The docstring states the trade-offs so the choice is not made by accident. Also imports pandas, which the new null-guard on the previous label needs. Co-Authored-By: Claude Opus 5 (1M context) --- lars/nepho/inference.py | 45 +++++++- tests/test_inference_nocodebook.py | 124 ++++++++++++++++++++ tests/test_inference_prev_label_source.py | 134 ++++++++++++++++++++++ 3 files changed, 298 insertions(+), 5 deletions(-) create mode 100644 tests/test_inference_nocodebook.py create mode 100644 tests/test_inference_prev_label_source.py diff --git a/lars/nepho/inference.py b/lars/nepho/inference.py index 106e96e..53f6638 100644 --- a/lars/nepho/inference.py +++ b/lars/nepho/inference.py @@ -2,6 +2,8 @@ import os import re +import pandas as pd + from ..preprocessing.labels import apply_criteria_to_labels DEFAULT_CATEGORIES = {"No precipitation": "No echoes greater than 10 dBZ present. A circle of echoes near radar site may be present due to ground clutter.", @@ -515,7 +517,8 @@ async def label_radar_data(radar_df, model, categories=None, guidelines=None, mlflow_tracking_uri=None, codebook_path=None, site="Bankhead National Forest", verbose=True, vmin=None, vmax=None, model_output_dir=None, - use_previous_labels=False): + use_previous_labels=False, + previous_label_source="hand"): """ Label radar data using a given model. @@ -559,6 +562,28 @@ async def label_radar_data(radar_df, model, categories=None, guidelines=None, model_output_dir: str: Directory to save model outputs. use_previous_labels: bool or int: If True, the function will use the previous *use_previous_labels* labels as an additional input to the model for labeling. This can be useful if the model is being used to refine or validate existing labels. + previous_label_source (str): Where those previous labels come from, either + ``"hand"`` (default) or ``"llm"``. + + ``"hand"`` injects the *human* label of the preceding scan, read from + the ``label`` column. Note what this implies for evaluation: when scans + are closely spaced their labels rarely change, so the previous hand + label is close to the answer. On the BNF CSAPR2 set (~10 minute + spacing) simply copying it -- with no image at all -- reaches 0.80 + macro-F1 against a two-human Dawid-Skene consensus, which beats every + model configuration measured. Results under this setting therefore say + more about that leaked hint than about reading radar imagery, and are + not comparable with ``use_previous_labels=0`` runs. + + ``"llm"`` instead injects the model's *own* previous prediction, read + from ``llm_label``, which the loop has already filled in for the + preceding row. No ground truth enters the prompt, so the run measures + temporal consistency rather than label leakage, and matches what is + available at inference time. The trade-off is that an early mistake can + propagate forward. + + Requires the frame to be in chronological order with a contiguous + index, since the lookup is positional. Returns ------- @@ -581,9 +606,13 @@ async def label_radar_data(radar_df, model, categories=None, guidelines=None, f" The radar site is the ARM Facility {site} site." \ " Please classify the weather depicted into one of the following categories: " \ f"{', '.join(categories) if categories else ', '.join(categories)}." - prompt += "Each category is defined as follows: " - for category, description in categories.items(): - prompt += f"{category}: {description}; " + # A category map with no descriptions at all is the "no codebook" baseline: + # the model gets the class names and nothing else. Emitting the definitions + # header followed by empty definitions would just be noise, so skip it. + if any(description for description in categories.values()): + prompt += "Each category is defined as follows: " + for category, description in categories.items(): + prompt += f"{category}: {description}; " prompt += f"The reflectivity values range from {vmin} dBZ as indicated by the blue colors to {vmax} dBZ as indicated by the red colors." for key in radar_df.columns: if key.startswith("pct_gates_") and key.endswith("dbz"): @@ -604,9 +633,15 @@ async def label_radar_data(radar_df, model, categories=None, guidelines=None, prompt_with_time = prompt + f"Please provide just the category label for the radar image taken at time {time}." prompt_with_time = prompt_with_time + "Do not provide your reasoning for your selection, just the category." if use_previous_labels: + prev_column = "label" if previous_label_source == "hand" else "llm_label" for i in range(use_previous_labels): if cur_index - i - 1 >= 0: - prev_label = radar_df.loc[cur_index - i - 1, "label"] + prev_label = radar_df.loc[cur_index - i - 1, prev_column] + # In "llm" mode the previous row has already been labelled, + # because this loop walks the frame in index order. The guard + # is for the first row of a frame, where nothing precedes it. + if prev_label is None or prev_label == "" or pd.isna(prev_label): + continue prompt_with_time += f" The label for the previous radar image taken at time {radar_df.loc[cur_index - i - 1, 'time']} is {prev_label}." output_model = await model.chat(prompt_with_time, images=[fi]) diff --git a/tests/test_inference_nocodebook.py b/tests/test_inference_nocodebook.py new file mode 100644 index 0000000..d66d092 --- /dev/null +++ b/tests/test_inference_nocodebook.py @@ -0,0 +1,124 @@ +"""The 'no codebook' baseline arm: class names reach the model, nothing else. + +The sweep's other arms all hand the model a codebook -- per-class definitions plus +annotator guidelines. Without a definition-free control there is no way to separate +what the model knows about storm morphology from what the codebook told it. That +control is expressed by passing a category map whose descriptions are all empty, +so these tests pin down what such a map does to the prompt. +""" +import pytest +import pandas as pd + +from lars.nepho.inference import label_radar_data + +DEFINITIONS_HEADER = "Each category is defined as follows" + +CATEGORIES_WITH_DESCRIPTIONS = {"Clear": "No echoes.", "Rain": "Widespread echoes."} +CATEGORIES_NAMES_ONLY = {"Clear": "", "Rain": ""} + + +class FakeModel: + """Records every prompt it is handed and replies with a fixed label.""" + + def __init__(self, response="Clear"): + self.response = response + self.calls = [] + + async def chat(self, prompt, images=None): + self.calls.append({"prompt": prompt, "images": images}) + return self.response + + +def make_df(n=1): + return pd.DataFrame({ + "file_path": [f"/tmp/img{i}.png" for i in range(n)], + "time": [f"2026-01-0{i + 1}" for i in range(n)], + "label": ["Clear"] * n, + }) + + +async def prompt_for(categories, **kwargs): + model = FakeModel() + await label_radar_data(make_df(1), model, categories=categories, + verbose=False, **kwargs) + return model.calls[0]["prompt"] + + +@pytest.mark.asyncio +async def test_empty_descriptions_drop_the_definitions_block(): + prompt = await prompt_for(CATEGORIES_NAMES_ONLY) + + assert DEFINITIONS_HEADER not in prompt + # The bare "Clear: ; Rain: ;" residue must not survive either. + assert "Clear: ;" not in prompt + assert "Rain: ;" not in prompt + + +@pytest.mark.asyncio +async def test_class_names_still_reach_the_model_without_descriptions(): + prompt = await prompt_for(CATEGORIES_NAMES_ONLY) + + assert "Clear" in prompt + assert "Rain" in prompt + + +@pytest.mark.asyncio +async def test_descriptions_are_emitted_when_present(): + prompt = await prompt_for(CATEGORIES_WITH_DESCRIPTIONS) + + assert DEFINITIONS_HEADER in prompt + assert "Clear: No echoes.;" in prompt + assert "Rain: Widespread echoes.;" in prompt + + +@pytest.mark.asyncio +async def test_a_single_described_class_keeps_the_block(): + """The guard keys off 'any description', not 'all descriptions'.""" + prompt = await prompt_for({"Clear": "", "Rain": "Widespread echoes."}) + + assert DEFINITIONS_HEADER in prompt + assert "Rain: Widespread echoes.;" in prompt + + +@pytest.mark.asyncio +async def test_guidelines_are_independent_of_the_definitions_block(): + """The baseline passes guidelines=None; the grid passes real ones.""" + bare = await prompt_for(CATEGORIES_NAMES_ONLY, guidelines=None) + guided = await prompt_for(CATEGORIES_NAMES_ONLY, guidelines=["Prefer Rain."]) + + assert "annotator guidelines" not in bare + assert "annotator guidelines" in guided + assert "Prefer Rain." in guided + assert DEFINITIONS_HEADER not in guided + + +@pytest.mark.asyncio +async def test_explicit_vmin_vmax_survive_without_a_codebook(): + """The baseline drops codebook_path but must still describe the real colorbar.""" + prompt = await prompt_for(CATEGORIES_NAMES_ONLY, vmin=-20, vmax=80) + + assert "-20 dBZ" in prompt + assert "80 dBZ" in prompt + + +@pytest.mark.asyncio +async def test_labelling_still_works_with_names_only(): + model = FakeModel("Rain") + df = await label_radar_data(make_df(2), model, + categories=CATEGORIES_NAMES_ONLY, verbose=False) + + assert list(df["llm_label"]) == ["Rain", "Rain"] + + +@pytest.mark.asyncio +async def test_parsing_ignores_descriptions(): + """Category matching reads only the keys, so empty values cannot break it.""" + model = FakeModel("Rain") + df = await label_radar_data(make_df(1), model, + categories=CATEGORIES_NAMES_ONLY, verbose=False) + assert df.loc[0, "llm_label"] == "Rain" + + model = FakeModel("nothing here") + df = await label_radar_data(make_df(1), model, + categories=CATEGORIES_NAMES_ONLY, verbose=False) + assert df.loc[0, "llm_label"] == "Unknown" diff --git a/tests/test_inference_prev_label_source.py b/tests/test_inference_prev_label_source.py new file mode 100644 index 0000000..06fe33f --- /dev/null +++ b/tests/test_inference_prev_label_source.py @@ -0,0 +1,134 @@ +"""Where the prev-scan arm gets its 'previous label' from. + +Historically ``use_previous_labels`` injected the *human* label of the +preceding scan. On the BNF CSAPR2 set that is close to leaking the answer: +consecutive scans are ~10 minutes apart and their labels rarely change, so +copying the previous hand label -- with no image at all -- reaches 0.80 +macro-F1 against the two-human consensus, beating every model configuration +measured. ``previous_label_source="llm"`` feeds the model's own previous +prediction instead, which keeps ground truth out of the prompt and matches +what is actually available at inference time. +""" +import pytest +import pandas as pd + +from lars.nepho.inference import label_radar_data + +CATEGORIES = {"Clear": "No echoes.", "Rain": "Widespread echoes."} + + +class ScriptedModel: + """Replies with a fixed sequence, recording each prompt it was given.""" + + def __init__(self, responses): + self.responses = list(responses) + self.prompts = [] + + async def chat(self, prompt, images=None): + self.prompts.append(prompt) + return self.responses[len(self.prompts) - 1] + + +def make_df(hand_labels): + n = len(hand_labels) + return pd.DataFrame({ + "file_path": [f"/tmp/img{i}.png" for i in range(n)], + "time": [f"2026-01-{i + 1:02d}" for i in range(n)], + "label": list(hand_labels), + }) + + +@pytest.mark.asyncio +async def test_hand_source_injects_the_human_label(): + # Hand labels say Rain; the model keeps answering Clear. Under "hand" the + # prompt must quote the human label, not the model's own answer. + model = ScriptedModel(["Clear", "Clear", "Clear"]) + await label_radar_data(make_df(["Rain", "Rain", "Rain"]), model, + categories=CATEGORIES, use_previous_labels=1, + previous_label_source="hand", verbose=False) + + assert "is Rain." in model.prompts[1] + assert "is Clear." not in model.prompts[1] + + +@pytest.mark.asyncio +async def test_llm_source_injects_the_models_own_prediction(): + model = ScriptedModel(["Clear", "Clear", "Clear"]) + await label_radar_data(make_df(["Rain", "Rain", "Rain"]), model, + categories=CATEGORIES, use_previous_labels=1, + previous_label_source="llm", verbose=False) + + # The model said Clear for scene 0, so scene 1 must be told Clear -- and + # must NOT be told the hand label Rain. + assert "is Clear." in model.prompts[1] + assert "is Rain." not in model.prompts[1] + + +@pytest.mark.asyncio +async def test_llm_source_never_leaks_ground_truth(): + """No hand label should appear in any prompt under the llm source.""" + model = ScriptedModel(["Clear"] * 4) + df = make_df(["Rain", "Rain", "Rain", "Rain"]) + await label_radar_data(df, model, categories=CATEGORIES, + use_previous_labels=1, + previous_label_source="llm", verbose=False) + + for prompt in model.prompts: + assert "is Rain." not in prompt + + +@pytest.mark.asyncio +async def test_llm_source_tracks_a_changing_prediction(): + """Each scene is told whatever the model actually said for the one before.""" + model = ScriptedModel(["Clear", "Rain", "Clear"]) + await label_radar_data(make_df(["Clear"] * 3), model, + categories=CATEGORIES, use_previous_labels=1, + previous_label_source="llm", verbose=False) + + assert "is Clear." in model.prompts[1] # scene 0 -> Clear + assert "is Rain." in model.prompts[2] # scene 1 -> Rain + + +@pytest.mark.asyncio +async def test_first_scene_gets_no_previous_label_in_either_mode(): + for source in ("hand", "llm"): + model = ScriptedModel(["Clear", "Clear"]) + await label_radar_data(make_df(["Rain", "Rain"]), model, + categories=CATEGORIES, use_previous_labels=1, + previous_label_source=source, verbose=False) + assert "previous radar image" not in model.prompts[0], source + + +@pytest.mark.asyncio +async def test_default_source_is_hand_for_backward_compatibility(): + model = ScriptedModel(["Clear", "Clear"]) + await label_radar_data(make_df(["Rain", "Rain"]), model, + categories=CATEGORIES, use_previous_labels=1, + verbose=False) + + assert "is Rain." in model.prompts[1] + + +@pytest.mark.asyncio +async def test_llm_source_handles_an_unparseable_previous_reply(): + """A reply that parses to nothing must not inject an empty label.""" + model = ScriptedModel(["not a category at all", "Clear"]) + await label_radar_data(make_df(["Rain", "Rain"]), model, + categories=CATEGORIES, use_previous_labels=1, + previous_label_source="llm", verbose=False) + + # _parse_category falls back to "Unknown", which is a real string, so it is + # passed through rather than producing "is ." -- the malformed case. + assert "is ." not in model.prompts[1] + + +@pytest.mark.asyncio +async def test_multiple_previous_labels_use_the_chosen_source(): + model = ScriptedModel(["Clear", "Rain", "Clear"]) + await label_radar_data(make_df(["Rain"] * 3), model, + categories=CATEGORIES, use_previous_labels=2, + previous_label_source="llm", verbose=False) + + # Scene 2 looks back two steps: scene 1 -> Rain, scene 0 -> Clear. + assert "is Rain." in model.prompts[2] + assert "is Clear." in model.prompts[2]