diff --git a/lars/nepho/inference.py b/lars/nepho/inference.py index 106e96e..53f6638 100644 --- a/lars/nepho/inference.py +++ b/lars/nepho/inference.py @@ -2,6 +2,8 @@ import os import re +import pandas as pd + from ..preprocessing.labels import apply_criteria_to_labels DEFAULT_CATEGORIES = {"No precipitation": "No echoes greater than 10 dBZ present. A circle of echoes near radar site may be present due to ground clutter.", @@ -515,7 +517,8 @@ async def label_radar_data(radar_df, model, categories=None, guidelines=None, mlflow_tracking_uri=None, codebook_path=None, site="Bankhead National Forest", verbose=True, vmin=None, vmax=None, model_output_dir=None, - use_previous_labels=False): + use_previous_labels=False, + previous_label_source="hand"): """ Label radar data using a given model. @@ -559,6 +562,28 @@ async def label_radar_data(radar_df, model, categories=None, guidelines=None, model_output_dir: str: Directory to save model outputs. use_previous_labels: bool or int: If True, the function will use the previous *use_previous_labels* labels as an additional input to the model for labeling. This can be useful if the model is being used to refine or validate existing labels. + previous_label_source (str): Where those previous labels come from, either + ``"hand"`` (default) or ``"llm"``. + + ``"hand"`` injects the *human* label of the preceding scan, read from + the ``label`` column. Note what this implies for evaluation: when scans + are closely spaced their labels rarely change, so the previous hand + label is close to the answer. On the BNF CSAPR2 set (~10 minute + spacing) simply copying it -- with no image at all -- reaches 0.80 + macro-F1 against a two-human Dawid-Skene consensus, which beats every + model configuration measured. Results under this setting therefore say + more about that leaked hint than about reading radar imagery, and are + not comparable with ``use_previous_labels=0`` runs. + + ``"llm"`` instead injects the model's *own* previous prediction, read + from ``llm_label``, which the loop has already filled in for the + preceding row. No ground truth enters the prompt, so the run measures + temporal consistency rather than label leakage, and matches what is + available at inference time. The trade-off is that an early mistake can + propagate forward. + + Requires the frame to be in chronological order with a contiguous + index, since the lookup is positional. Returns ------- @@ -581,9 +606,13 @@ async def label_radar_data(radar_df, model, categories=None, guidelines=None, f" The radar site is the ARM Facility {site} site." \ " Please classify the weather depicted into one of the following categories: " \ f"{', '.join(categories) if categories else ', '.join(categories)}." - prompt += "Each category is defined as follows: " - for category, description in categories.items(): - prompt += f"{category}: {description}; " + # A category map with no descriptions at all is the "no codebook" baseline: + # the model gets the class names and nothing else. Emitting the definitions + # header followed by empty definitions would just be noise, so skip it. + if any(description for description in categories.values()): + prompt += "Each category is defined as follows: " + for category, description in categories.items(): + prompt += f"{category}: {description}; " prompt += f"The reflectivity values range from {vmin} dBZ as indicated by the blue colors to {vmax} dBZ as indicated by the red colors." for key in radar_df.columns: if key.startswith("pct_gates_") and key.endswith("dbz"): @@ -604,9 +633,15 @@ async def label_radar_data(radar_df, model, categories=None, guidelines=None, prompt_with_time = prompt + f"Please provide just the category label for the radar image taken at time {time}." prompt_with_time = prompt_with_time + "Do not provide your reasoning for your selection, just the category." if use_previous_labels: + prev_column = "label" if previous_label_source == "hand" else "llm_label" for i in range(use_previous_labels): if cur_index - i - 1 >= 0: - prev_label = radar_df.loc[cur_index - i - 1, "label"] + prev_label = radar_df.loc[cur_index - i - 1, prev_column] + # In "llm" mode the previous row has already been labelled, + # because this loop walks the frame in index order. The guard + # is for the first row of a frame, where nothing precedes it. + if prev_label is None or prev_label == "" or pd.isna(prev_label): + continue prompt_with_time += f" The label for the previous radar image taken at time {radar_df.loc[cur_index - i - 1, 'time']} is {prev_label}." output_model = await model.chat(prompt_with_time, images=[fi]) diff --git a/tests/test_inference_nocodebook.py b/tests/test_inference_nocodebook.py new file mode 100644 index 0000000..d66d092 --- /dev/null +++ b/tests/test_inference_nocodebook.py @@ -0,0 +1,124 @@ +"""The 'no codebook' baseline arm: class names reach the model, nothing else. + +The sweep's other arms all hand the model a codebook -- per-class definitions plus +annotator guidelines. Without a definition-free control there is no way to separate +what the model knows about storm morphology from what the codebook told it. That +control is expressed by passing a category map whose descriptions are all empty, +so these tests pin down what such a map does to the prompt. +""" +import pytest +import pandas as pd + +from lars.nepho.inference import label_radar_data + +DEFINITIONS_HEADER = "Each category is defined as follows" + +CATEGORIES_WITH_DESCRIPTIONS = {"Clear": "No echoes.", "Rain": "Widespread echoes."} +CATEGORIES_NAMES_ONLY = {"Clear": "", "Rain": ""} + + +class FakeModel: + """Records every prompt it is handed and replies with a fixed label.""" + + def __init__(self, response="Clear"): + self.response = response + self.calls = [] + + async def chat(self, prompt, images=None): + self.calls.append({"prompt": prompt, "images": images}) + return self.response + + +def make_df(n=1): + return pd.DataFrame({ + "file_path": [f"/tmp/img{i}.png" for i in range(n)], + "time": [f"2026-01-0{i + 1}" for i in range(n)], + "label": ["Clear"] * n, + }) + + +async def prompt_for(categories, **kwargs): + model = FakeModel() + await label_radar_data(make_df(1), model, categories=categories, + verbose=False, **kwargs) + return model.calls[0]["prompt"] + + +@pytest.mark.asyncio +async def test_empty_descriptions_drop_the_definitions_block(): + prompt = await prompt_for(CATEGORIES_NAMES_ONLY) + + assert DEFINITIONS_HEADER not in prompt + # The bare "Clear: ; Rain: ;" residue must not survive either. + assert "Clear: ;" not in prompt + assert "Rain: ;" not in prompt + + +@pytest.mark.asyncio +async def test_class_names_still_reach_the_model_without_descriptions(): + prompt = await prompt_for(CATEGORIES_NAMES_ONLY) + + assert "Clear" in prompt + assert "Rain" in prompt + + +@pytest.mark.asyncio +async def test_descriptions_are_emitted_when_present(): + prompt = await prompt_for(CATEGORIES_WITH_DESCRIPTIONS) + + assert DEFINITIONS_HEADER in prompt + assert "Clear: No echoes.;" in prompt + assert "Rain: Widespread echoes.;" in prompt + + +@pytest.mark.asyncio +async def test_a_single_described_class_keeps_the_block(): + """The guard keys off 'any description', not 'all descriptions'.""" + prompt = await prompt_for({"Clear": "", "Rain": "Widespread echoes."}) + + assert DEFINITIONS_HEADER in prompt + assert "Rain: Widespread echoes.;" in prompt + + +@pytest.mark.asyncio +async def test_guidelines_are_independent_of_the_definitions_block(): + """The baseline passes guidelines=None; the grid passes real ones.""" + bare = await prompt_for(CATEGORIES_NAMES_ONLY, guidelines=None) + guided = await prompt_for(CATEGORIES_NAMES_ONLY, guidelines=["Prefer Rain."]) + + assert "annotator guidelines" not in bare + assert "annotator guidelines" in guided + assert "Prefer Rain." in guided + assert DEFINITIONS_HEADER not in guided + + +@pytest.mark.asyncio +async def test_explicit_vmin_vmax_survive_without_a_codebook(): + """The baseline drops codebook_path but must still describe the real colorbar.""" + prompt = await prompt_for(CATEGORIES_NAMES_ONLY, vmin=-20, vmax=80) + + assert "-20 dBZ" in prompt + assert "80 dBZ" in prompt + + +@pytest.mark.asyncio +async def test_labelling_still_works_with_names_only(): + model = FakeModel("Rain") + df = await label_radar_data(make_df(2), model, + categories=CATEGORIES_NAMES_ONLY, verbose=False) + + assert list(df["llm_label"]) == ["Rain", "Rain"] + + +@pytest.mark.asyncio +async def test_parsing_ignores_descriptions(): + """Category matching reads only the keys, so empty values cannot break it.""" + model = FakeModel("Rain") + df = await label_radar_data(make_df(1), model, + categories=CATEGORIES_NAMES_ONLY, verbose=False) + assert df.loc[0, "llm_label"] == "Rain" + + model = FakeModel("nothing here") + df = await label_radar_data(make_df(1), model, + categories=CATEGORIES_NAMES_ONLY, verbose=False) + assert df.loc[0, "llm_label"] == "Unknown" diff --git a/tests/test_inference_prev_label_source.py b/tests/test_inference_prev_label_source.py new file mode 100644 index 0000000..06fe33f --- /dev/null +++ b/tests/test_inference_prev_label_source.py @@ -0,0 +1,134 @@ +"""Where the prev-scan arm gets its 'previous label' from. + +Historically ``use_previous_labels`` injected the *human* label of the +preceding scan. On the BNF CSAPR2 set that is close to leaking the answer: +consecutive scans are ~10 minutes apart and their labels rarely change, so +copying the previous hand label -- with no image at all -- reaches 0.80 +macro-F1 against the two-human consensus, beating every model configuration +measured. ``previous_label_source="llm"`` feeds the model's own previous +prediction instead, which keeps ground truth out of the prompt and matches +what is actually available at inference time. +""" +import pytest +import pandas as pd + +from lars.nepho.inference import label_radar_data + +CATEGORIES = {"Clear": "No echoes.", "Rain": "Widespread echoes."} + + +class ScriptedModel: + """Replies with a fixed sequence, recording each prompt it was given.""" + + def __init__(self, responses): + self.responses = list(responses) + self.prompts = [] + + async def chat(self, prompt, images=None): + self.prompts.append(prompt) + return self.responses[len(self.prompts) - 1] + + +def make_df(hand_labels): + n = len(hand_labels) + return pd.DataFrame({ + "file_path": [f"/tmp/img{i}.png" for i in range(n)], + "time": [f"2026-01-{i + 1:02d}" for i in range(n)], + "label": list(hand_labels), + }) + + +@pytest.mark.asyncio +async def test_hand_source_injects_the_human_label(): + # Hand labels say Rain; the model keeps answering Clear. Under "hand" the + # prompt must quote the human label, not the model's own answer. + model = ScriptedModel(["Clear", "Clear", "Clear"]) + await label_radar_data(make_df(["Rain", "Rain", "Rain"]), model, + categories=CATEGORIES, use_previous_labels=1, + previous_label_source="hand", verbose=False) + + assert "is Rain." in model.prompts[1] + assert "is Clear." not in model.prompts[1] + + +@pytest.mark.asyncio +async def test_llm_source_injects_the_models_own_prediction(): + model = ScriptedModel(["Clear", "Clear", "Clear"]) + await label_radar_data(make_df(["Rain", "Rain", "Rain"]), model, + categories=CATEGORIES, use_previous_labels=1, + previous_label_source="llm", verbose=False) + + # The model said Clear for scene 0, so scene 1 must be told Clear -- and + # must NOT be told the hand label Rain. + assert "is Clear." in model.prompts[1] + assert "is Rain." not in model.prompts[1] + + +@pytest.mark.asyncio +async def test_llm_source_never_leaks_ground_truth(): + """No hand label should appear in any prompt under the llm source.""" + model = ScriptedModel(["Clear"] * 4) + df = make_df(["Rain", "Rain", "Rain", "Rain"]) + await label_radar_data(df, model, categories=CATEGORIES, + use_previous_labels=1, + previous_label_source="llm", verbose=False) + + for prompt in model.prompts: + assert "is Rain." not in prompt + + +@pytest.mark.asyncio +async def test_llm_source_tracks_a_changing_prediction(): + """Each scene is told whatever the model actually said for the one before.""" + model = ScriptedModel(["Clear", "Rain", "Clear"]) + await label_radar_data(make_df(["Clear"] * 3), model, + categories=CATEGORIES, use_previous_labels=1, + previous_label_source="llm", verbose=False) + + assert "is Clear." in model.prompts[1] # scene 0 -> Clear + assert "is Rain." in model.prompts[2] # scene 1 -> Rain + + +@pytest.mark.asyncio +async def test_first_scene_gets_no_previous_label_in_either_mode(): + for source in ("hand", "llm"): + model = ScriptedModel(["Clear", "Clear"]) + await label_radar_data(make_df(["Rain", "Rain"]), model, + categories=CATEGORIES, use_previous_labels=1, + previous_label_source=source, verbose=False) + assert "previous radar image" not in model.prompts[0], source + + +@pytest.mark.asyncio +async def test_default_source_is_hand_for_backward_compatibility(): + model = ScriptedModel(["Clear", "Clear"]) + await label_radar_data(make_df(["Rain", "Rain"]), model, + categories=CATEGORIES, use_previous_labels=1, + verbose=False) + + assert "is Rain." in model.prompts[1] + + +@pytest.mark.asyncio +async def test_llm_source_handles_an_unparseable_previous_reply(): + """A reply that parses to nothing must not inject an empty label.""" + model = ScriptedModel(["not a category at all", "Clear"]) + await label_radar_data(make_df(["Rain", "Rain"]), model, + categories=CATEGORIES, use_previous_labels=1, + previous_label_source="llm", verbose=False) + + # _parse_category falls back to "Unknown", which is a real string, so it is + # passed through rather than producing "is ." -- the malformed case. + assert "is ." not in model.prompts[1] + + +@pytest.mark.asyncio +async def test_multiple_previous_labels_use_the_chosen_source(): + model = ScriptedModel(["Clear", "Rain", "Clear"]) + await label_radar_data(make_df(["Rain"] * 3), model, + categories=CATEGORIES, use_previous_labels=2, + previous_label_source="llm", verbose=False) + + # Scene 2 looks back two steps: scene 1 -> Rain, scene 0 -> Clear. + assert "is Rain." in model.prompts[2] + assert "is Clear." in model.prompts[2]