diff --git a/changelog.d/977.added.md b/changelog.d/977.added.md new file mode 100644 index 000000000..695d7fbd9 --- /dev/null +++ b/changelog.d/977.added.md @@ -0,0 +1,3 @@ +UK local-area breakdown results (`LocalAreaBreakdownOutput`, the result of a `breakdown=local_areas` run) can now carry two optional fields, `constituency_withheld_codes` and `local_authority_withheld_codes`, and the contract and the gateway accept a result with or without them. Each will hold, sorted, the ONS codes of that level's areas that the run estimated but disclosure control did not publish: `[]` when none were withheld, and never an area the run did not estimate. No executor fills them yet. + +A result without them still validates, and its JSON leaves both keys out rather than writing `null`. This uses field-level `exclude_if`, so the contract now requires pydantic 2.12 or later. The gateway's OpenAPI document lists both fields as optional. diff --git a/libs/policyengine-simulation-contract/pyproject.toml b/libs/policyengine-simulation-contract/pyproject.toml index 7214eee3b..1de678f91 100644 --- a/libs/policyengine-simulation-contract/pyproject.toml +++ b/libs/policyengine-simulation-contract/pyproject.toml @@ -8,7 +8,7 @@ authors = [ license = {file = "../../LICENSE"} requires-python = ">=3.13" dependencies = [ - "pydantic>=2.0", + "pydantic>=2.12", "packaging>=24", # gateway_models embeds the TelemetryEnvelope from the observability lib. "policyengine-simulation-observability", diff --git a/libs/policyengine-simulation-contract/src/policyengine_simulation_contract/macro_output.py b/libs/policyengine-simulation-contract/src/policyengine_simulation_contract/macro_output.py index 7ec8bc815..70037d827 100644 --- a/libs/policyengine-simulation-contract/src/policyengine_simulation_contract/macro_output.py +++ b/libs/policyengine-simulation-contract/src/policyengine_simulation_contract/macro_output.py @@ -4,7 +4,7 @@ from typing import Generic, Literal, TypeVar -from pydantic import BaseModel, ConfigDict, RootModel +from pydantic import BaseModel, ConfigDict, Field, RootModel from policyengine_simulation_contract.spm import SPMComparisonProvenance @@ -354,6 +354,10 @@ class SingleYearMacroOutput(MacroOutputModel): cliff_impact: CliffImpactOutput | None = None +def _is_none(value: object) -> bool: + return value is None + + class LocalAreaBreakdownOutput(MacroOutputModel): """Completed result of a UK local-area breakdown run. @@ -367,6 +371,15 @@ class LocalAreaBreakdownOutput(MacroOutputModel): policyengine.py release that computed it. Each list holds only its own level's records. Areas whose income change disclosure control withholds are left out of each list, so a list may be empty. + + ``constituency_withheld_codes`` and ``local_authority_withheld_codes`` + hold, sorted, the ONS codes of that level's areas that the run estimated + but disclosure control did not publish, so a client can tell a withheld + area from an absent one. An area the run never estimated, such as one with + no households in the dataset, is in neither list. An empty list means + that no area of the level was withheld. Null means that the result does + not report them, as results from older executors do not; the JSON then + leaves both keys out rather than writing null. """ kind: Literal["local_areas"] @@ -376,3 +389,9 @@ class LocalAreaBreakdownOutput(MacroOutputModel): data_version: str constituency_impact: list[ConstituencyImpactRecord] local_authority_impact: list[LocalAuthorityImpactRecord] + constituency_withheld_codes: list[str] | None = Field( + default=None, exclude_if=_is_none + ) + local_authority_withheld_codes: list[str] | None = Field( + default=None, exclude_if=_is_none + ) diff --git a/libs/policyengine-simulation-contract/tests/test_local_area_breakdown.py b/libs/policyengine-simulation-contract/tests/test_local_area_breakdown.py index cdd80eaf1..98ecc3ae4 100644 --- a/libs/policyengine-simulation-contract/tests/test_local_area_breakdown.py +++ b/libs/policyengine-simulation-contract/tests/test_local_area_breakdown.py @@ -56,6 +56,14 @@ "data_version": "microcosm-uk-2024-25-dense-20261019T000432Z-ee5c2dbe", "constituency_impact": [CONSTITUENCY], "local_authority_impact": [LOCAL_AUTHORITY], + "constituency_withheld_codes": ["E14000530", "E14001172"], + "local_authority_withheld_codes": [], +} +# A result from an executor that does not report the withheld areas. +BREAKDOWN_WITHOUT_WITHHELD_CODES = { + key: value + for key, value in BREAKDOWN.items() + if not key.endswith("_withheld_codes") } _ZERO = {"baseline": 0.0, "reform": 0.0} _AGE_POVERTY = {"child": _ZERO, "adult": _ZERO, "senior": _ZERO, "all": _ZERO} @@ -115,6 +123,60 @@ def test_a_breakdown_result_carries_both_area_levels_and_their_provenance(): assert output.model_dump(mode="json") == BREAKDOWN +def test_a_breakdown_result_lists_the_withheld_areas_of_each_level(): + output = LocalAreaBreakdownOutput.model_validate(BREAKDOWN) + + assert output.constituency_withheld_codes == ["E14000530", "E14001172"] + # An empty list reports that disclosure control withheld no area. + assert output.local_authority_withheld_codes == [] + + +def test_a_breakdown_result_without_withheld_codes_still_validates(): + output = LocalAreaBreakdownOutput.model_validate(BREAKDOWN_WITHOUT_WITHHELD_CODES) + + # None means the result does not report which areas were withheld. + assert output.constituency_withheld_codes is None + assert output.local_authority_withheld_codes is None + response = JobStatusResponse( + status="complete", result=BREAKDOWN_WITHOUT_WITHHELD_CODES + ) + assert isinstance(response.result, LocalAreaBreakdownOutput) + + +def test_a_breakdown_result_without_withheld_codes_dumps_without_them(): + # A gateway deployed before the fields existed forbids unknown keys, so a + # result that does not report them must leave the keys out, not send null. + output = LocalAreaBreakdownOutput.model_validate(BREAKDOWN_WITHOUT_WITHHELD_CODES) + + assert output.model_dump(mode="json") == BREAKDOWN_WITHOUT_WITHHELD_CODES + assert "withheld_codes" not in output.model_dump_json() + response = JobStatusResponse( + status="complete", result=BREAKDOWN_WITHOUT_WITHHELD_CODES + ) + assert response.model_dump(mode="json")["result"] == ( + BREAKDOWN_WITHOUT_WITHHELD_CODES + ) + + +def test_a_breakdown_result_keeps_an_empty_withheld_list_in_its_dump(): + # [] reports that no area was withheld, unlike a missing list. + output = LocalAreaBreakdownOutput.model_validate( + {**BREAKDOWN_WITHOUT_WITHHELD_CODES, "constituency_withheld_codes": []} + ) + + dumped = output.model_dump(mode="json") + assert dumped["constituency_withheld_codes"] == [] + assert "local_authority_withheld_codes" not in dumped + + +@pytest.mark.parametrize( + "field", ["constituency_withheld_codes", "local_authority_withheld_codes"] +) +def test_withheld_codes_are_a_list_of_area_codes(field): + with pytest.raises(ValidationError): + LocalAreaBreakdownOutput.model_validate({**BREAKDOWN, field: "E14000530"}) + + def test_a_job_status_reads_a_breakdown_result_as_a_breakdown(): response = JobStatusResponse(status="complete", result=BREAKDOWN) diff --git a/libs/policyengine-simulation-contract/uv.lock b/libs/policyengine-simulation-contract/uv.lock index 8915800ea..3fa7e7d15 100644 --- a/libs/policyengine-simulation-contract/uv.lock +++ b/libs/policyengine-simulation-contract/uv.lock @@ -1122,7 +1122,7 @@ requires-dist = [ { name = "modal", marker = "extra == 'test'", specifier = ">=1.4,<2" }, { name = "packaging", specifier = ">=24" }, { name = "policyengine-simulation-observability", editable = "../policyengine-simulation-observability" }, - { name = "pydantic", specifier = ">=2.0" }, + { name = "pydantic", specifier = ">=2.12" }, { name = "pyright", marker = "extra == 'build'", specifier = ">=1.1.401" }, { name = "pytest", marker = "extra == 'test'", specifier = ">=8.3.4" }, { name = "pytest-asyncio", marker = "extra == 'test'", specifier = ">=0.25.3" }, diff --git a/libs/policyengine-stage12-persistence/uv.lock b/libs/policyengine-stage12-persistence/uv.lock index 5e3c5dca8..20d14c19b 100644 --- a/libs/policyengine-stage12-persistence/uv.lock +++ b/libs/policyengine-stage12-persistence/uv.lock @@ -604,7 +604,7 @@ requires-dist = [ { name = "modal", marker = "extra == 'test'", specifier = ">=1.4,<2" }, { name = "packaging", specifier = ">=24" }, { name = "policyengine-simulation-observability", editable = "../policyengine-simulation-observability" }, - { name = "pydantic", specifier = ">=2.0" }, + { name = "pydantic", specifier = ">=2.12" }, { name = "pyright", marker = "extra == 'build'", specifier = ">=1.1.401" }, { name = "pytest", marker = "extra == 'test'", specifier = ">=8.3.4" }, { name = "pytest-asyncio", marker = "extra == 'test'", specifier = ">=0.25.3" }, diff --git a/projects/policyengine-simulation-entry/tests/test_openapi.py b/projects/policyengine-simulation-entry/tests/test_openapi.py index 060b6bb8c..b31f97ada 100644 --- a/projects/policyengine-simulation-entry/tests/test_openapi.py +++ b/projects/policyengine-simulation-entry/tests/test_openapi.py @@ -122,6 +122,20 @@ def test_local_area_breakdowns_are_public(): ]["properties"]["result"]["anyOf"] +def test_local_area_breakdowns_may_list_the_withheld_areas_of_each_level(): + breakdown = create_app().openapi()["components"]["schemas"][ + "LocalAreaBreakdownOutput" + ] + + for field in ("constituency_withheld_codes", "local_authority_withheld_codes"): + assert breakdown["properties"][field]["anyOf"] == [ + {"type": "array", "items": {"type": "string"}}, + {"type": "null"}, + ] + # Optional, so results from executors that do not report them validate. + assert field not in breakdown["required"] + + def test_route_table_is_frozen(): assert methods(cast("OpenAPIDocument", create_app().openapi())) == EXPECTED_ROUTES diff --git a/projects/policyengine-simulation-entry/uv.lock b/projects/policyengine-simulation-entry/uv.lock index c3bfa7d53..e9a587c24 100644 --- a/projects/policyengine-simulation-entry/uv.lock +++ b/projects/policyengine-simulation-entry/uv.lock @@ -881,7 +881,7 @@ requires-dist = [ { name = "modal", marker = "extra == 'test'", specifier = ">=1.4,<2" }, { name = "packaging", specifier = ">=24" }, { name = "policyengine-simulation-observability", editable = "../../libs/policyengine-simulation-observability" }, - { name = "pydantic", specifier = ">=2.0" }, + { name = "pydantic", specifier = ">=2.12" }, { name = "pyright", marker = "extra == 'build'", specifier = ">=1.1.401" }, { name = "pytest", marker = "extra == 'test'", specifier = ">=8.3.4" }, { name = "pytest-asyncio", marker = "extra == 'test'", specifier = ">=0.25.3" }, diff --git a/projects/policyengine-simulation-executor/uv.lock b/projects/policyengine-simulation-executor/uv.lock index 2bd2f7c36..a9b4bb246 100644 --- a/projects/policyengine-simulation-executor/uv.lock +++ b/projects/policyengine-simulation-executor/uv.lock @@ -1883,7 +1883,7 @@ requires-dist = [ { name = "modal", marker = "extra == 'test'", specifier = ">=1.4,<2" }, { name = "packaging", specifier = ">=24" }, { name = "policyengine-simulation-observability", editable = "../../libs/policyengine-simulation-observability" }, - { name = "pydantic", specifier = ">=2.0" }, + { name = "pydantic", specifier = ">=2.12" }, { name = "pyright", marker = "extra == 'build'", specifier = ">=1.1.401" }, { name = "pytest", marker = "extra == 'test'", specifier = ">=8.3.4" }, { name = "pytest-asyncio", marker = "extra == 'test'", specifier = ">=0.25.3" }, diff --git a/projects/policyengine-simulation-gateway/README.md b/projects/policyengine-simulation-gateway/README.md index 033836a95..433d2b651 100644 --- a/projects/policyengine-simulation-gateway/README.md +++ b/projects/policyengine-simulation-gateway/README.md @@ -104,6 +104,7 @@ instead of a `SingleYearMacroOutput`: | `model_version`, `policyengine_version` | The country model and the policyengine.py release that computed it. | | `dataset`, `data_version` | The local-area dataset's name and release revision. | | `constituency_impact`, `local_authority_impact` | The published areas of each level. Disclosure control leaves withheld areas out, so a list may be empty; each record carries a `disclosure_tier` (`reported` or `approximate`) and 90% interval bounds. | +| `constituency_withheld_codes`, `local_authority_withheld_codes` | The ONS codes, sorted, of each level's areas that the run estimated but disclosure control did not publish. An area the run never estimated, such as one with no households in the dataset, is in neither list, so an area absent from both the records and this list has no estimate. An empty list means that no area was withheld. Results from older executors don't report them, and a result that doesn't report them leaves both keys out rather than sending null. | Rules: diff --git a/projects/policyengine-simulation-gateway/tests/golden/openapi.json b/projects/policyengine-simulation-gateway/tests/golden/openapi.json index ae9c9d9c5..a097ff3e5 100644 --- a/projects/policyengine-simulation-gateway/tests/golden/openapi.json +++ b/projects/policyengine-simulation-gateway/tests/golden/openapi.json @@ -1796,6 +1796,34 @@ }, "type": "array", "title": "Local Authority Impact" + }, + "constituency_withheld_codes": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Constituency Withheld Codes" + }, + "local_authority_withheld_codes": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Local Authority Withheld Codes" } }, "additionalProperties": false, @@ -1810,7 +1838,7 @@ "local_authority_impact" ], "title": "LocalAreaBreakdownOutput", - "description": "Completed result of a UK local-area breakdown run.\n\nA request with ``breakdown: \"local_areas\"`` returns this instead of a\n``SingleYearMacroOutput``. The run covers the requested UK-wide or nation\nregion on the local-area dataset alone, and computes only the\nconstituency and local-authority impacts; every other figure comes from\nthe national run. ``kind`` is always ``local_areas``. ``dataset`` names the\nlocal-area dataset, ``data_version`` its release revision, and\n``model_version`` and ``policyengine_version`` the country model and the\npolicyengine.py release that computed it. Each list holds only its own\nlevel's records. Areas whose income change disclosure control withholds\nare left out of each list, so a list may be empty." + "description": "Completed result of a UK local-area breakdown run.\n\nA request with ``breakdown: \"local_areas\"`` returns this instead of a\n``SingleYearMacroOutput``. The run covers the requested UK-wide or nation\nregion on the local-area dataset alone, and computes only the\nconstituency and local-authority impacts; every other figure comes from\nthe national run. ``kind`` is always ``local_areas``. ``dataset`` names the\nlocal-area dataset, ``data_version`` its release revision, and\n``model_version`` and ``policyengine_version`` the country model and the\npolicyengine.py release that computed it. Each list holds only its own\nlevel's records. Areas whose income change disclosure control withholds\nare left out of each list, so a list may be empty.\n\n``constituency_withheld_codes`` and ``local_authority_withheld_codes``\nhold, sorted, the ONS codes of that level's areas that the run estimated\nbut disclosure control did not publish, so a client can tell a withheld\narea from an absent one. An area the run never estimated, such as one with\nno households in the dataset, is in neither list. An empty list means\nthat no area of the level was withheld. Null means that the result does\nnot report them, as results from older executors do not; the JSON then\nleaves both keys out rather than writing null." }, "LocalAuthorityImpactRecord": { "properties": { diff --git a/projects/policyengine-simulation-gateway/uv.lock b/projects/policyengine-simulation-gateway/uv.lock index 082898381..8e709799d 100644 --- a/projects/policyengine-simulation-gateway/uv.lock +++ b/projects/policyengine-simulation-gateway/uv.lock @@ -1304,7 +1304,7 @@ requires-dist = [ { name = "modal", marker = "extra == 'test'", specifier = ">=1.4,<2" }, { name = "packaging", specifier = ">=24" }, { name = "policyengine-simulation-observability", editable = "../../libs/policyengine-simulation-observability" }, - { name = "pydantic", specifier = ">=2.0" }, + { name = "pydantic", specifier = ">=2.12" }, { name = "pyright", marker = "extra == 'build'", specifier = ">=1.1.401" }, { name = "pytest", marker = "extra == 'test'", specifier = ">=8.3.4" }, { name = "pytest-asyncio", marker = "extra == 'test'", specifier = ">=0.25.3" },