Add no-codebook baseline support and a self-referential prev-label source

lars/nepho/inference.py

@@ -2,6 +2,8 @@
 import os
 import re
 
+import pandas as pd
+
 from ..preprocessing.labels import apply_criteria_to_labels
 
 DEFAULT_CATEGORIES = {"No precipitation": "No echoes greater than 10 dBZ present. A circle of echoes near radar site may be present due to ground clutter.",
@@ -515,7 +517,8 @@ async def label_radar_data(radar_df, model, categories=None, guidelines=None,
                            mlflow_tracking_uri=None, codebook_path=None,
                            site="Bankhead National Forest",
                            verbose=True, vmin=None, vmax=None, model_output_dir=None,
-                           use_previous_labels=False):
+                           use_previous_labels=False,
+                           previous_label_source="hand"):
     """
     Label radar data using a given model.
 
@@ -559,6 +562,28 @@ async def label_radar_data(radar_df, model, categories=None, guidelines=None,
     model_output_dir: str: Directory to save model outputs.
     use_previous_labels: bool or int: If True, the function will use the previous *use_previous_labels* 
         labels as an additional input to the model for labeling. This can be useful if the model is being used to refine or validate existing labels.
+    previous_label_source (str): Where those previous labels come from, either
+        ``"hand"`` (default) or ``"llm"``.
+
+        ``"hand"`` injects the *human* label of the preceding scan, read from
+        the ``label`` column. Note what this implies for evaluation: when scans
+        are closely spaced their labels rarely change, so the previous hand
+        label is close to the answer. On the BNF CSAPR2 set (~10 minute
+        spacing) simply copying it -- with no image at all -- reaches 0.80
+        macro-F1 against a two-human Dawid-Skene consensus, which beats every
+        model configuration measured. Results under this setting therefore say
+        more about that leaked hint than about reading radar imagery, and are
+        not comparable with ``use_previous_labels=0`` runs.
+
+        ``"llm"`` instead injects the model's *own* previous prediction, read
+        from ``llm_label``, which the loop has already filled in for the
+        preceding row. No ground truth enters the prompt, so the run measures
+        temporal consistency rather than label leakage, and matches what is
+        available at inference time. The trade-off is that an early mistake can
+        propagate forward.
+
+        Requires the frame to be in chronological order with a contiguous
+        index, since the lookup is positional.
 
     Returns
     -------
@@ -581,9 +606,13 @@ async def label_radar_data(radar_df, model, categories=None, guidelines=None,
                 f" The radar site is the ARM Facility {site} site." \
              " Please classify the weather depicted into one of the following categories: " \
              f"{', '.join(categories) if categories else ', '.join(categories)}."
-    prompt += "Each category is defined as follows: "
-    for category, description in categories.items():
-        prompt += f"{category}: {description}; "
+    # A category map with no descriptions at all is the "no codebook" baseline:
+    # the model gets the class names and nothing else. Emitting the definitions
+    # header followed by empty definitions would just be noise, so skip it.
+    if any(description for description in categories.values()):
+        prompt += "Each category is defined as follows: "
+        for category, description in categories.items():
+            prompt += f"{category}: {description}; "
     prompt += f"The reflectivity values range from {vmin} dBZ as indicated by the blue colors to {vmax} dBZ as indicated by the red colors."
     for key in radar_df.columns:
         if key.startswith("pct_gates_") and key.endswith("dbz"):
@@ -604,9 +633,15 @@ async def label_radar_data(radar_df, model, categories=None, guidelines=None,
         prompt_with_time = prompt + f"Please provide just the category label for the radar image taken at time {time}."      
         prompt_with_time = prompt_with_time + "Do not provide your reasoning for your selection, just the category."
         if use_previous_labels:
+            prev_column = "label" if previous_label_source == "hand" else "llm_label"
             for i in range(use_previous_labels):
                 if cur_index - i - 1 >= 0:
-                    prev_label = radar_df.loc[cur_index - i - 1, "label"]
+                    prev_label = radar_df.loc[cur_index - i - 1, prev_column]
+                    # In "llm" mode the previous row has already been labelled,
+                    # because this loop walks the frame in index order. The guard
+                    # is for the first row of a frame, where nothing precedes it.
+                    if prev_label is None or prev_label == "" or pd.isna(prev_label):
+                        continue
                     prompt_with_time += f" The label for the previous radar image taken at time {radar_df.loc[cur_index - i - 1, 'time']} is {prev_label}."
 
         output_model = await model.chat(prompt_with_time, images=[fi])

tests/test_inference_nocodebook.py

@@ -0,0 +1,124 @@
+"""The 'no codebook' baseline arm: class names reach the model, nothing else.
+
+The sweep's other arms all hand the model a codebook -- per-class definitions plus
+annotator guidelines. Without a definition-free control there is no way to separate
+what the model knows about storm morphology from what the codebook told it. That
+control is expressed by passing a category map whose descriptions are all empty,
+so these tests pin down what such a map does to the prompt.
+"""
+import pytest
+import pandas as pd
+
+from lars.nepho.inference import label_radar_data
+
+DEFINITIONS_HEADER = "Each category is defined as follows"
+
+CATEGORIES_WITH_DESCRIPTIONS = {"Clear": "No echoes.", "Rain": "Widespread echoes."}
+CATEGORIES_NAMES_ONLY = {"Clear": "", "Rain": ""}
+
+
+class FakeModel:
+    """Records every prompt it is handed and replies with a fixed label."""
+
+    def __init__(self, response="Clear"):
+        self.response = response
+        self.calls = []
+
+    async def chat(self, prompt, images=None):
+        self.calls.append({"prompt": prompt, "images": images})
+        return self.response
+
+
+def make_df(n=1):
+    return pd.DataFrame({
+        "file_path": [f"/tmp/img{i}.png" for i in range(n)],
+        "time": [f"2026-01-0{i + 1}" for i in range(n)],
+        "label": ["Clear"] * n,
+    })
+
+
+async def prompt_for(categories, **kwargs):
+    model = FakeModel()
+    await label_radar_data(make_df(1), model, categories=categories,
+                           verbose=False, **kwargs)
+    return model.calls[0]["prompt"]
+
+
+@pytest.mark.asyncio
+async def test_empty_descriptions_drop_the_definitions_block():
+    prompt = await prompt_for(CATEGORIES_NAMES_ONLY)
+
+    assert DEFINITIONS_HEADER not in prompt
+    # The bare "Clear: ; Rain: ;" residue must not survive either.
+    assert "Clear: ;" not in prompt
+    assert "Rain: ;" not in prompt
+
+
+@pytest.mark.asyncio
+async def test_class_names_still_reach_the_model_without_descriptions():
+    prompt = await prompt_for(CATEGORIES_NAMES_ONLY)
+
+    assert "Clear" in prompt
+    assert "Rain" in prompt
+
+
+@pytest.mark.asyncio
+async def test_descriptions_are_emitted_when_present():
+    prompt = await prompt_for(CATEGORIES_WITH_DESCRIPTIONS)
+
+    assert DEFINITIONS_HEADER in prompt
+    assert "Clear: No echoes.;" in prompt
+    assert "Rain: Widespread echoes.;" in prompt
+
+
+@pytest.mark.asyncio
+async def test_a_single_described_class_keeps_the_block():
+    """The guard keys off 'any description', not 'all descriptions'."""
+    prompt = await prompt_for({"Clear": "", "Rain": "Widespread echoes."})
+
+    assert DEFINITIONS_HEADER in prompt
+    assert "Rain: Widespread echoes.;" in prompt
+
+
+@pytest.mark.asyncio
+async def test_guidelines_are_independent_of_the_definitions_block():
+    """The baseline passes guidelines=None; the grid passes real ones."""
+    bare = await prompt_for(CATEGORIES_NAMES_ONLY, guidelines=None)
+    guided = await prompt_for(CATEGORIES_NAMES_ONLY, guidelines=["Prefer Rain."])
+
+    assert "annotator guidelines" not in bare
+    assert "annotator guidelines" in guided
+    assert "Prefer Rain." in guided
+    assert DEFINITIONS_HEADER not in guided
+
+
+@pytest.mark.asyncio
+async def test_explicit_vmin_vmax_survive_without_a_codebook():
+    """The baseline drops codebook_path but must still describe the real colorbar."""
+    prompt = await prompt_for(CATEGORIES_NAMES_ONLY, vmin=-20, vmax=80)
+
+    assert "-20 dBZ" in prompt
+    assert "80 dBZ" in prompt
+
+
+@pytest.mark.asyncio
+async def test_labelling_still_works_with_names_only():
+    model = FakeModel("Rain")
+    df = await label_radar_data(make_df(2), model,
+                                categories=CATEGORIES_NAMES_ONLY, verbose=False)
+
+    assert list(df["llm_label"]) == ["Rain", "Rain"]
+
+
+@pytest.mark.asyncio
+async def test_parsing_ignores_descriptions():
+    """Category matching reads only the keys, so empty values cannot break it."""
+    model = FakeModel("Rain")
+    df = await label_radar_data(make_df(1), model,
+                                categories=CATEGORIES_NAMES_ONLY, verbose=False)
+    assert df.loc[0, "llm_label"] == "Rain"
+
+    model = FakeModel("nothing here")
+    df = await label_radar_data(make_df(1), model,
+                                categories=CATEGORIES_NAMES_ONLY, verbose=False)
+    assert df.loc[0, "llm_label"] == "Unknown"

tests/test_inference_prev_label_source.py

@@ -0,0 +1,134 @@
+"""Where the prev-scan arm gets its 'previous label' from.
+
+Historically ``use_previous_labels`` injected the *human* label of the
+preceding scan. On the BNF CSAPR2 set that is close to leaking the answer:
+consecutive scans are ~10 minutes apart and their labels rarely change, so
+copying the previous hand label -- with no image at all -- reaches 0.80
+macro-F1 against the two-human consensus, beating every model configuration
+measured. ``previous_label_source="llm"`` feeds the model's own previous
+prediction instead, which keeps ground truth out of the prompt and matches
+what is actually available at inference time.
+"""
+import pytest
+import pandas as pd
+
+from lars.nepho.inference import label_radar_data
+
+CATEGORIES = {"Clear": "No echoes.", "Rain": "Widespread echoes."}
+
+
+class ScriptedModel:
+    """Replies with a fixed sequence, recording each prompt it was given."""
+
+    def __init__(self, responses):
+        self.responses = list(responses)
+        self.prompts = []
+
+    async def chat(self, prompt, images=None):
+        self.prompts.append(prompt)
+        return self.responses[len(self.prompts) - 1]
+
+
+def make_df(hand_labels):
+    n = len(hand_labels)
+    return pd.DataFrame({
+        "file_path": [f"/tmp/img{i}.png" for i in range(n)],
+        "time": [f"2026-01-{i + 1:02d}" for i in range(n)],
+        "label": list(hand_labels),
+    })
+
+
+@pytest.mark.asyncio
+async def test_hand_source_injects_the_human_label():
+    # Hand labels say Rain; the model keeps answering Clear. Under "hand" the
+    # prompt must quote the human label, not the model's own answer.
+    model = ScriptedModel(["Clear", "Clear", "Clear"])
+    await label_radar_data(make_df(["Rain", "Rain", "Rain"]), model,
+                           categories=CATEGORIES, use_previous_labels=1,
+                           previous_label_source="hand", verbose=False)
+
+    assert "is Rain." in model.prompts[1]
+    assert "is Clear." not in model.prompts[1]
+
+
+@pytest.mark.asyncio
+async def test_llm_source_injects_the_models_own_prediction():
+    model = ScriptedModel(["Clear", "Clear", "Clear"])
+    await label_radar_data(make_df(["Rain", "Rain", "Rain"]), model,
+                           categories=CATEGORIES, use_previous_labels=1,
+                           previous_label_source="llm", verbose=False)
+
+    # The model said Clear for scene 0, so scene 1 must be told Clear -- and
+    # must NOT be told the hand label Rain.
+    assert "is Clear." in model.prompts[1]
+    assert "is Rain." not in model.prompts[1]
+
+
+@pytest.mark.asyncio
+async def test_llm_source_never_leaks_ground_truth():
+    """No hand label should appear in any prompt under the llm source."""
+    model = ScriptedModel(["Clear"] * 4)
+    df = make_df(["Rain", "Rain", "Rain", "Rain"])
+    await label_radar_data(df, model, categories=CATEGORIES,
+                           use_previous_labels=1,
+                           previous_label_source="llm", verbose=False)
+
+    for prompt in model.prompts:
+        assert "is Rain." not in prompt
+
+
+@pytest.mark.asyncio
+async def test_llm_source_tracks_a_changing_prediction():
+    """Each scene is told whatever the model actually said for the one before."""
+    model = ScriptedModel(["Clear", "Rain", "Clear"])
+    await label_radar_data(make_df(["Clear"] * 3), model,
+                           categories=CATEGORIES, use_previous_labels=1,
+                           previous_label_source="llm", verbose=False)
+
+    assert "is Clear." in model.prompts[1]   # scene 0 -> Clear
+    assert "is Rain." in model.prompts[2]    # scene 1 -> Rain
+
+
+@pytest.mark.asyncio
+async def test_first_scene_gets_no_previous_label_in_either_mode():
+    for source in ("hand", "llm"):
+        model = ScriptedModel(["Clear", "Clear"])
+        await label_radar_data(make_df(["Rain", "Rain"]), model,
+                               categories=CATEGORIES, use_previous_labels=1,
+                               previous_label_source=source, verbose=False)
+        assert "previous radar image" not in model.prompts[0], source
+
+
+@pytest.mark.asyncio
+async def test_default_source_is_hand_for_backward_compatibility():
+    model = ScriptedModel(["Clear", "Clear"])
+    await label_radar_data(make_df(["Rain", "Rain"]), model,
+                           categories=CATEGORIES, use_previous_labels=1,
+                           verbose=False)
+
+    assert "is Rain." in model.prompts[1]
+
+
+@pytest.mark.asyncio
+async def test_llm_source_handles_an_unparseable_previous_reply():
+    """A reply that parses to nothing must not inject an empty label."""
+    model = ScriptedModel(["not a category at all", "Clear"])
+    await label_radar_data(make_df(["Rain", "Rain"]), model,
+                           categories=CATEGORIES, use_previous_labels=1,
+                           previous_label_source="llm", verbose=False)
+
+    # _parse_category falls back to "Unknown", which is a real string, so it is
+    # passed through rather than producing "is ." -- the malformed case.
+    assert "is ." not in model.prompts[1]
+
+
+@pytest.mark.asyncio
+async def test_multiple_previous_labels_use_the_chosen_source():
+    model = ScriptedModel(["Clear", "Rain", "Clear"])
+    await label_radar_data(make_df(["Rain"] * 3), model,
+                           categories=CATEGORIES, use_previous_labels=2,
+                           previous_label_source="llm", verbose=False)
+
+    # Scene 2 looks back two steps: scene 1 -> Rain, scene 0 -> Clear.
+    assert "is Rain." in model.prompts[2]
+    assert "is Clear." in model.prompts[2]