Accept optional withheld-area code lists on UK local-area breakdown results

changelog.d/977.added.md

@@ -0,0 +1,3 @@
+UK local-area breakdown results (`LocalAreaBreakdownOutput`, the result of a `breakdown=local_areas` run) can now carry two optional fields, `constituency_withheld_codes` and `local_authority_withheld_codes`, and the contract and the gateway accept a result with or without them. Each will hold, sorted, the ONS codes of that level's areas that the run estimated but disclosure control did not publish: `[]` when none were withheld, and never an area the run did not estimate. No executor fills them yet.
+
+A result without them still validates, and its JSON leaves both keys out rather than writing `null`. This uses field-level `exclude_if`, so the contract now requires pydantic 2.12 or later. The gateway's OpenAPI document lists both fields as optional.

libs/policyengine-simulation-contract/pyproject.toml

@@ -8,7 +8,7 @@ authors = [
 license = {file = "../../LICENSE"}
 requires-python = ">=3.13"
 dependencies = [
-    "pydantic>=2.0",
+    "pydantic>=2.12",
     "packaging>=24",
     # gateway_models embeds the TelemetryEnvelope from the observability lib.
     "policyengine-simulation-observability",

libs/policyengine-simulation-contract/src/policyengine_simulation_contract/macro_output.py

@@ -4,7 +4,7 @@
 
 from typing import Generic, Literal, TypeVar
 
-from pydantic import BaseModel, ConfigDict, RootModel
+from pydantic import BaseModel, ConfigDict, Field, RootModel
 
 from policyengine_simulation_contract.spm import SPMComparisonProvenance
 
@@ -354,6 +354,10 @@ class SingleYearMacroOutput(MacroOutputModel):
     cliff_impact: CliffImpactOutput | None = None
 
 
+def _is_none(value: object) -> bool:
+    return value is None
+
+
 class LocalAreaBreakdownOutput(MacroOutputModel):
     """Completed result of a UK local-area breakdown run.
 
@@ -367,6 +371,15 @@ class LocalAreaBreakdownOutput(MacroOutputModel):
     policyengine.py release that computed it. Each list holds only its own
     level's records. Areas whose income change disclosure control withholds
     are left out of each list, so a list may be empty.
+
+    ``constituency_withheld_codes`` and ``local_authority_withheld_codes``
+    hold, sorted, the ONS codes of that level's areas that the run estimated
+    but disclosure control did not publish, so a client can tell a withheld
+    area from an absent one. An area the run never estimated, such as one with
+    no households in the dataset, is in neither list. An empty list means
+    that no area of the level was withheld. Null means that the result does
+    not report them, as results from older executors do not; the JSON then
+    leaves both keys out rather than writing null.
     """
 
     kind: Literal["local_areas"]
@@ -376,3 +389,9 @@ class LocalAreaBreakdownOutput(MacroOutputModel):
     data_version: str
     constituency_impact: list[ConstituencyImpactRecord]
     local_authority_impact: list[LocalAuthorityImpactRecord]
+    constituency_withheld_codes: list[str] | None = Field(
+        default=None, exclude_if=_is_none
+    )
+    local_authority_withheld_codes: list[str] | None = Field(
+        default=None, exclude_if=_is_none
+    )

libs/policyengine-simulation-contract/tests/test_local_area_breakdown.py

@@ -56,6 +56,14 @@
     "data_version": "microcosm-uk-2024-25-dense-20261019T000432Z-ee5c2dbe",
     "constituency_impact": [CONSTITUENCY],
     "local_authority_impact": [LOCAL_AUTHORITY],
+    "constituency_withheld_codes": ["E14000530", "E14001172"],
+    "local_authority_withheld_codes": [],
+}
+# A result from an executor that does not report the withheld areas.
+BREAKDOWN_WITHOUT_WITHHELD_CODES = {
+    key: value
+    for key, value in BREAKDOWN.items()
+    if not key.endswith("_withheld_codes")
 }
 _ZERO = {"baseline": 0.0, "reform": 0.0}
 _AGE_POVERTY = {"child": _ZERO, "adult": _ZERO, "senior": _ZERO, "all": _ZERO}
@@ -115,6 +123,60 @@ def test_a_breakdown_result_carries_both_area_levels_and_their_provenance():
     assert output.model_dump(mode="json") == BREAKDOWN
 
 
+def test_a_breakdown_result_lists_the_withheld_areas_of_each_level():
+    output = LocalAreaBreakdownOutput.model_validate(BREAKDOWN)
+
+    assert output.constituency_withheld_codes == ["E14000530", "E14001172"]
+    # An empty list reports that disclosure control withheld no area.
+    assert output.local_authority_withheld_codes == []
+
+
+def test_a_breakdown_result_without_withheld_codes_still_validates():
+    output = LocalAreaBreakdownOutput.model_validate(BREAKDOWN_WITHOUT_WITHHELD_CODES)
+
+    # None means the result does not report which areas were withheld.
+    assert output.constituency_withheld_codes is None
+    assert output.local_authority_withheld_codes is None
+    response = JobStatusResponse(
+        status="complete", result=BREAKDOWN_WITHOUT_WITHHELD_CODES
+    )
+    assert isinstance(response.result, LocalAreaBreakdownOutput)
+
+
+def test_a_breakdown_result_without_withheld_codes_dumps_without_them():
+    # A gateway deployed before the fields existed forbids unknown keys, so a
+    # result that does not report them must leave the keys out, not send null.
+    output = LocalAreaBreakdownOutput.model_validate(BREAKDOWN_WITHOUT_WITHHELD_CODES)
+
+    assert output.model_dump(mode="json") == BREAKDOWN_WITHOUT_WITHHELD_CODES
+    assert "withheld_codes" not in output.model_dump_json()
+    response = JobStatusResponse(
+        status="complete", result=BREAKDOWN_WITHOUT_WITHHELD_CODES
+    )
+    assert response.model_dump(mode="json")["result"] == (
+        BREAKDOWN_WITHOUT_WITHHELD_CODES
+    )
+
+
+def test_a_breakdown_result_keeps_an_empty_withheld_list_in_its_dump():
+    # [] reports that no area was withheld, unlike a missing list.
+    output = LocalAreaBreakdownOutput.model_validate(
+        {**BREAKDOWN_WITHOUT_WITHHELD_CODES, "constituency_withheld_codes": []}
+    )
+
+    dumped = output.model_dump(mode="json")
+    assert dumped["constituency_withheld_codes"] == []
+    assert "local_authority_withheld_codes" not in dumped
+
+
+@pytest.mark.parametrize(
+    "field", ["constituency_withheld_codes", "local_authority_withheld_codes"]
+)
+def test_withheld_codes_are_a_list_of_area_codes(field):
+    with pytest.raises(ValidationError):
+        LocalAreaBreakdownOutput.model_validate({**BREAKDOWN, field: "E14000530"})
+
+
 def test_a_job_status_reads_a_breakdown_result_as_a_breakdown():
     response = JobStatusResponse(status="complete", result=BREAKDOWN)
 

projects/policyengine-simulation-entry/tests/test_openapi.py

@@ -122,6 +122,20 @@ def test_local_area_breakdowns_are_public():
     ]["properties"]["result"]["anyOf"]
 
 
+def test_local_area_breakdowns_may_list_the_withheld_areas_of_each_level():
+    breakdown = create_app().openapi()["components"]["schemas"][
+        "LocalAreaBreakdownOutput"
+    ]
+
+    for field in ("constituency_withheld_codes", "local_authority_withheld_codes"):
+        assert breakdown["properties"][field]["anyOf"] == [
+            {"type": "array", "items": {"type": "string"}},
+            {"type": "null"},
+        ]
+        # Optional, so results from executors that do not report them validate.
+        assert field not in breakdown["required"]
+
+
 def test_route_table_is_frozen():
     assert methods(cast("OpenAPIDocument", create_app().openapi())) == EXPECTED_ROUTES
 

projects/policyengine-simulation-gateway/README.md

@@ -104,6 +104,7 @@ instead of a `SingleYearMacroOutput`:
 | `model_version`, `policyengine_version` | The country model and the policyengine.py release that computed it. |
 | `dataset`, `data_version` | The local-area dataset's name and release revision. |
 | `constituency_impact`, `local_authority_impact` | The published areas of each level. Disclosure control leaves withheld areas out, so a list may be empty; each record carries a `disclosure_tier` (`reported` or `approximate`) and 90% interval bounds. |
+| `constituency_withheld_codes`, `local_authority_withheld_codes` | The ONS codes, sorted, of each level's areas that the run estimated but disclosure control did not publish. An area the run never estimated, such as one with no households in the dataset, is in neither list, so an area absent from both the records and this list has no estimate. An empty list means that no area was withheld. Results from older executors don't report them, and a result that doesn't report them leaves both keys out rather than sending null. |
 
 Rules:
 

projects/policyengine-simulation-gateway/tests/golden/openapi.json

@@ -1796,6 +1796,34 @@
             },
             "type": "array",
             "title": "Local Authority Impact"
+          },
+          "constituency_withheld_codes": {
+            "anyOf": [
+              {
+                "items": {
+                  "type": "string"
+                },
+                "type": "array"
+              },
+              {
+                "type": "null"
+              }
+            ],
+            "title": "Constituency Withheld Codes"
+          },
+          "local_authority_withheld_codes": {
+            "anyOf": [
+              {
+                "items": {
+                  "type": "string"
+                },
+                "type": "array"
+              },
+              {
+                "type": "null"
+              }
+            ],
+            "title": "Local Authority Withheld Codes"
           }
         },
         "additionalProperties": false,
@@ -1810,7 +1838,7 @@
           "local_authority_impact"
         ],
         "title": "LocalAreaBreakdownOutput",
-        "description": "Completed result of a UK local-area breakdown run.\n\nA request with ``breakdown: \"local_areas\"`` returns this instead of a\n``SingleYearMacroOutput``. The run covers the requested UK-wide or nation\nregion on the local-area dataset alone, and computes only the\nconstituency and local-authority impacts; every other figure comes from\nthe national run. ``kind`` is always ``local_areas``. ``dataset`` names the\nlocal-area dataset, ``data_version`` its release revision, and\n``model_version`` and ``policyengine_version`` the country model and the\npolicyengine.py release that computed it. Each list holds only its own\nlevel's records. Areas whose income change disclosure control withholds\nare left out of each list, so a list may be empty."
+        "description": "Completed result of a UK local-area breakdown run.\n\nA request with ``breakdown: \"local_areas\"`` returns this instead of a\n``SingleYearMacroOutput``. The run covers the requested UK-wide or nation\nregion on the local-area dataset alone, and computes only the\nconstituency and local-authority impacts; every other figure comes from\nthe national run. ``kind`` is always ``local_areas``. ``dataset`` names the\nlocal-area dataset, ``data_version`` its release revision, and\n``model_version`` and ``policyengine_version`` the country model and the\npolicyengine.py release that computed it. Each list holds only its own\nlevel's records. Areas whose income change disclosure control withholds\nare left out of each list, so a list may be empty.\n\n``constituency_withheld_codes`` and ``local_authority_withheld_codes``\nhold, sorted, the ONS codes of that level's areas that the run estimated\nbut disclosure control did not publish, so a client can tell a withheld\narea from an absent one. An area the run never estimated, such as one with\nno households in the dataset, is in neither list. An empty list means\nthat no area of the level was withheld. Null means that the result does\nnot report them, as results from older executors do not; the JSON then\nleaves both keys out rather than writing null."
       },
       "LocalAuthorityImpactRecord": {
         "properties": {