From 1d72f0a46f7bd019ab50aabbae0739a97fd24543 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Fri, 6 Nov 2026 11:56:28 +0000 Subject: [PATCH 1/2] Emit NRS's council tax bands by 2022 data zone (chronicle#419) Add the workbook's columns G to N, dwellings in council tax bands A to H, beside nrs_dwellings_total in both record sets of nrs-dwellings-data-zone-2025. They take the total's shape: unit count, aggregation sum, a number cell and the row-5 header, line break included, as a guard. The package round-trips byte for byte through yaml.safe_dump(sort_keys=False, allow_unicode=True, width=88), so the measures were set by loading and dumping it; no row changes. A test checks that the eight bands add up to the total in all 7,392 zones of both sheets. The merged bundle gains 118,272 dwelling facts at existing geography, period and entity keys. Co-Authored-By: Claude Opus 5.5 --- ...s-data-zone-council-tax-bands-419.added.md | 1 + .../source_package.yaml | 208 ++++++++++++++++++ packages/uk_local_2025_sources.md | 2 +- tests/test_chronicle_bundle.py | 20 +- tests/test_chronicle_uk_local_2025.py | 25 ++- 5 files changed, 249 insertions(+), 7 deletions(-) create mode 100644 changelog.d/nrs-data-zone-council-tax-bands-419.added.md diff --git a/changelog.d/nrs-data-zone-council-tax-bands-419.added.md b/changelog.d/nrs-data-zone-council-tax-bands-419.added.md new file mode 100644 index 00000000..4547ae4a --- /dev/null +++ b/changelog.d/nrs-data-zone-council-tax-bands-419.added.md @@ -0,0 +1 @@ +Add the council tax band columns of NRS's dwelling estimates by 2022 data zone, which chronicle#419 asks for (PolicyEngine/microcosm#931): dwellings in bands A to H in each of Scotland's 7,392 data zones, December 2024 and December 2025, as `nrs.dwellings_band_a` to `nrs.dwellings_band_h` beside `nrs.dwellings_total` in `nrs-dwellings-data-zone-2025`. A test checks that the eight bands add up to the zone's total in every zone of both sheets. diff --git a/packages/nrs/dwellings_data_zone_2025/source_package.yaml b/packages/nrs/dwellings_data_zone_2025/source_package.yaml index 070a7f8b..718b13c2 100644 --- a/packages/nrs/dwellings_data_zone_2025/source_package.yaml +++ b/packages/nrs/dwellings_data_zone_2025/source_package.yaml @@ -81358,6 +81358,110 @@ record_sets: expected_cell_type: number expected_column_header_row: 5 expected_column_header: Total number of dwellings + - measure_id: nrs_dwellings_band_a + label: Dwellings, Council Tax band A + ordinal: 1 + column: G + source_column_id: council_tax_band_a + concept: nrs.dwellings_band_a + unit: count + aggregation: sum + expected_cell_type: number + expected_column_header_row: 5 + expected_column_header: 'Council Tax band: + + A' + - measure_id: nrs_dwellings_band_b + label: Dwellings, Council Tax band B + ordinal: 2 + column: H + source_column_id: council_tax_band_b + concept: nrs.dwellings_band_b + unit: count + aggregation: sum + expected_cell_type: number + expected_column_header_row: 5 + expected_column_header: 'Council Tax band: + + B' + - measure_id: nrs_dwellings_band_c + label: Dwellings, Council Tax band C + ordinal: 3 + column: I + source_column_id: council_tax_band_c + concept: nrs.dwellings_band_c + unit: count + aggregation: sum + expected_cell_type: number + expected_column_header_row: 5 + expected_column_header: 'Council Tax band: + + C' + - measure_id: nrs_dwellings_band_d + label: Dwellings, Council Tax band D + ordinal: 4 + column: J + source_column_id: council_tax_band_d + concept: nrs.dwellings_band_d + unit: count + aggregation: sum + expected_cell_type: number + expected_column_header_row: 5 + expected_column_header: 'Council Tax band: + + D' + - measure_id: nrs_dwellings_band_e + label: Dwellings, Council Tax band E + ordinal: 5 + column: K + source_column_id: council_tax_band_e + concept: nrs.dwellings_band_e + unit: count + aggregation: sum + expected_cell_type: number + expected_column_header_row: 5 + expected_column_header: 'Council Tax band: + + E' + - measure_id: nrs_dwellings_band_f + label: Dwellings, Council Tax band F + ordinal: 6 + column: L + source_column_id: council_tax_band_f + concept: nrs.dwellings_band_f + unit: count + aggregation: sum + expected_cell_type: number + expected_column_header_row: 5 + expected_column_header: 'Council Tax band: + + F' + - measure_id: nrs_dwellings_band_g + label: Dwellings, Council Tax band G + ordinal: 7 + column: M + source_column_id: council_tax_band_g + concept: nrs.dwellings_band_g + unit: count + aggregation: sum + expected_cell_type: number + expected_column_header_row: 5 + expected_column_header: 'Council Tax band: + + G' + - measure_id: nrs_dwellings_band_h + label: Dwellings, Council Tax band H + ordinal: 8 + column: N + source_column_id: council_tax_band_h + concept: nrs.dwellings_band_h + unit: count + aggregation: sum + expected_cell_type: number + expected_column_header_row: 5 + expected_column_header: 'Council Tax band: + + H' period_coverage: start_date: '2024-12-31' end_date: '2024-12-31' @@ -162704,6 +162808,110 @@ record_sets: expected_cell_type: number expected_column_header_row: 5 expected_column_header: Total number of dwellings + - measure_id: nrs_dwellings_band_a + label: Dwellings, Council Tax band A + ordinal: 1 + column: G + source_column_id: council_tax_band_a + concept: nrs.dwellings_band_a + unit: count + aggregation: sum + expected_cell_type: number + expected_column_header_row: 5 + expected_column_header: 'Council Tax band: + + A' + - measure_id: nrs_dwellings_band_b + label: Dwellings, Council Tax band B + ordinal: 2 + column: H + source_column_id: council_tax_band_b + concept: nrs.dwellings_band_b + unit: count + aggregation: sum + expected_cell_type: number + expected_column_header_row: 5 + expected_column_header: 'Council Tax band: + + B' + - measure_id: nrs_dwellings_band_c + label: Dwellings, Council Tax band C + ordinal: 3 + column: I + source_column_id: council_tax_band_c + concept: nrs.dwellings_band_c + unit: count + aggregation: sum + expected_cell_type: number + expected_column_header_row: 5 + expected_column_header: 'Council Tax band: + + C' + - measure_id: nrs_dwellings_band_d + label: Dwellings, Council Tax band D + ordinal: 4 + column: J + source_column_id: council_tax_band_d + concept: nrs.dwellings_band_d + unit: count + aggregation: sum + expected_cell_type: number + expected_column_header_row: 5 + expected_column_header: 'Council Tax band: + + D' + - measure_id: nrs_dwellings_band_e + label: Dwellings, Council Tax band E + ordinal: 5 + column: K + source_column_id: council_tax_band_e + concept: nrs.dwellings_band_e + unit: count + aggregation: sum + expected_cell_type: number + expected_column_header_row: 5 + expected_column_header: 'Council Tax band: + + E' + - measure_id: nrs_dwellings_band_f + label: Dwellings, Council Tax band F + ordinal: 6 + column: L + source_column_id: council_tax_band_f + concept: nrs.dwellings_band_f + unit: count + aggregation: sum + expected_cell_type: number + expected_column_header_row: 5 + expected_column_header: 'Council Tax band: + + F' + - measure_id: nrs_dwellings_band_g + label: Dwellings, Council Tax band G + ordinal: 7 + column: M + source_column_id: council_tax_band_g + concept: nrs.dwellings_band_g + unit: count + aggregation: sum + expected_cell_type: number + expected_column_header_row: 5 + expected_column_header: 'Council Tax band: + + G' + - measure_id: nrs_dwellings_band_h + label: Dwellings, Council Tax band H + ordinal: 8 + column: N + source_column_id: council_tax_band_h + concept: nrs.dwellings_band_h + unit: count + aggregation: sum + expected_cell_type: number + expected_column_header_row: 5 + expected_column_header: 'Council Tax band: + + H' period_coverage: start_date: '2025-12-31' end_date: '2025-12-31' diff --git a/packages/uk_local_2025_sources.md b/packages/uk_local_2025_sources.md index ea4599d0..206bc987 100644 --- a/packages/uk_local_2025_sources.md +++ b/packages/uk_local_2025_sources.md @@ -4,7 +4,7 @@ Chronicle preserves the publisher artifacts requested in [issue 313](https://git | Source family | Packages and coverage | Publisher semantics | | --- | --- | --- | -| Scotland households | `nrs-households-dwellings-2025`, `nrs-households-data-zone-2025`, `nrs-dwellings-data-zone-2025` | Council and country private-household estimates: June 2024/2025. Council dwelling stock: September. Data-zone occupied dwellings: September; dwelling stock: December. The small-area occupied-dwelling table does not assert the adjusted private-household estimate. | +| Scotland households | `nrs-households-dwellings-2025`, `nrs-households-data-zone-2025`, `nrs-dwellings-data-zone-2025` | Council and country private-household estimates: June 2024/2025. Council dwelling stock: September. Data-zone occupied dwellings: September; dwelling stock: December, in total and by council tax band A–H ([issue 419](https://github.com/PolicyEngine/chronicle/issues/419)). The small-area occupied-dwelling table does not assert the adjusted private-household estimate. | | Wales households | `welshgov-household-estimates-mid2024`, `welshgov-household-projections-2022-based` | Revised mid-2022/2023/2024 estimates and principal mid-2025 projection. Publisher decimals retained. The estimate selector excludes private-household population and average household size. | | England households | `ons-household-projections-2022-based` | Table 406, all 349 areas, mid-2022 through mid-2047, 2022 boundaries; metropolitan counties remain counties. `source_projection` identifies each publisher projection. | | NI housing stock | `lps-housing-stock-lgd-2026` | Eleven districts and NI country, 1 April 2021–2026. Domestic or mixed-rated dwelling stock. Chronicle retains the published snapshots; consumers calculate roll-forward factors. | diff --git a/tests/test_chronicle_bundle.py b/tests/test_chronicle_bundle.py index 7964f784..124b3357 100644 --- a/tests/test_chronicle_bundle.py +++ b/tests/test_chronicle_bundle.py @@ -562,7 +562,12 @@ def test_build_bundle_writes_merged_consumer_contract(tmp_path): # Survey's chapters 1, 3 and 4 (4,458), HMRC's stamp tax Tables 2e, 3a, # 5a, 7, 9a and 9d (1,135), ONS's land by sector (387) and the council # rows of the Welsh (308) and Scottish (96) council tax tables. - "fact_count": 2062484, + # chronicle#419 adds 118,272 dwelling facts: NRS's council tax bands A to + # H for each of Scotland's 7,392 2022 data zones, December 2024 and + # December 2025. Measured with only nrs-dwellings-data-zone-2025 at + # bundle year 2023: no new geography, period or entity key, duplicate + # key or warning. + "fact_count": 2180756, # The same change adds 82,064 geography keys: 46,363 Scottish 2022 # output areas (UV404), 35,673 LSOA-level keys from the DESNZ LSOA # workbooks (Scotland's data zones already exist), the 26 Northern @@ -686,7 +691,7 @@ def test_build_bundle_writes_merged_consumer_contract(tmp_path): if row_count == 1: assert row["aggregate_fact_key"].startswith("ledger.aggregate_fact.v2:") assert row["semantic_fact_key"].startswith("ledger.semantic_fact.v2:") - assert row_count == 2062484 + assert row_count == 2180756 assert provenance_classes <= { "administrative", "census", @@ -721,7 +726,7 @@ def test_build_bundle_writes_merged_consumer_contract(tmp_path): "jct-obbba-revenue-estimates-2025", "jct-tax-expenditures-2024", ] - assert coverage["fact_count"] == 2062484 + assert coverage["fact_count"] == 2180756 assert coverage["counts"]["by_source"] == { "bea": 446, "bfp_economic_outlook": 5, @@ -759,7 +764,7 @@ def test_build_bundle_writes_merged_consumer_contract(tmp_path): "nbb_national_accounts": 1, "nisra": 29227, "nithc": 8, - "nrs": 359094, + "nrs": 477366, "obr": 4539, "ofgem": 3640, "onem_rva_unemployment": 1, @@ -2532,6 +2537,10 @@ def test_build_bundle_writes_merged_consumer_contract(tmp_path): } for key, count in chronicle_408_period_increments.items(): expected_period_counts[key] = expected_period_counts.get(key, 0) + count + # chronicle#419: NRS's eight council tax bands for each of the 7,392 data + # zones, at December 2024 and December 2025. + for year in (2024, 2025): + expected_period_counts[f"calendar_year:{year}"] += 8 * 7392 assert coverage["counts"]["by_period"] == expected_period_counts # country:NZ: 330 WFF (#209) + 5 national population (#211) + 27 Treasury # (#321) + 3,522 IRD I1 (#318) + 510 IRD I2 (#210) + 103 MSD (#319) @@ -2675,9 +2684,10 @@ def test_build_bundle_writes_merged_consumer_contract(tmp_path): # QRC4 5,757, Table 6 1,208, Wales 308 and Scotland 96), households +1,666 # (the Market Value Survey), returns +1,135 (HMRC's stamp tax), firms +401 # (the land value estimates) and institutional sectors +387 (ONS's land). + # chronicle#419: dwellings +118,272 (NRS's council tax bands by data zone). assert coverage["counts"]["by_entity"] == { "benefit_unit": 33645, - "dwelling": 590264, + "dwelling": 708536, "family": 1700, "firm": 1905, "government": 14071, diff --git a/tests/test_chronicle_uk_local_2025.py b/tests/test_chronicle_uk_local_2025.py index 5d24436e..f5c7816a 100644 --- a/tests/test_chronicle_uk_local_2025.py +++ b/tests/test_chronicle_uk_local_2025.py @@ -2,7 +2,7 @@ import hashlib import json -from collections import defaultdict +from collections import Counter, defaultdict from dataclasses import replace from datetime import datetime from pathlib import Path @@ -107,6 +107,29 @@ def test_household_controls_keep_private_households_projections_and_stock_distin } +def test_scottish_data_zone_council_tax_bands_sum_to_each_zone_total(): + """chronicle#419: columns G to N (bands A to H) add up to column E.""" + facts = load_source_package("nrs-dwellings-data-zone-2025").build_facts(2025) + bands = tuple(f"nrs.dwellings_band_{band}" for band in "abcdefgh") + zones = defaultdict(dict) + for fact in facts: + zones[fact.period.value, fact.geography.id][fact.measure.concept] = fact.value + assert len(facts) == 2 * 7_392 * 9 + assert Counter(period for period, _ in zones) == {2024: 7_392, 2025: 7_392} + for key, values in zones.items(): + assert set(values) == {"nrs.dwellings_total", *bands}, key + assert sum(values[band] for band in bands) == values["nrs.dwellings_total"], key + # Culter - 01 (S01013482, row 6), bands A to H as published. + published = { + 2024: [21, 153, 45, 55, 100, 44, 85, 13], + 2025: [21, 153, 45, 55, 99, 45, 85, 13], + } + for period, counts in published.items(): + assert [zones[period, "S01013482"][band] for band in bands] == counts + assert all(f.source_cell_keys for f in facts) + assert validate_consumer_fact_contract(facts).valid + + def test_rm120_retains_age_sex_residence_and_2023_lad_boundaries(): facts = load_source_package( "ons-census2021-rm120-pcon24-residence2-sex0" From 78d4c2726b81d9e961af23e9d841e9260a069d8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Fri, 6 Nov 2026 12:31:38 +0000 Subject: [PATCH 2/2] Test that each band's header guard rejects a shifted column Vahid's review of 1d72f0a: nothing pinned the 16 new header guards, so a loosened or removed guard would only show once a shifted column broke the band sum. For each sheet and band, the test puts the next band's header in the band's row-5 cell and expects that band's selector to fail on its column header. The workbook is parsed once per session and shared with the band-sum test. Co-Authored-By: Claude Opus 5.5 --- tests/test_chronicle_uk_local_2025.py | 37 ++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/tests/test_chronicle_uk_local_2025.py b/tests/test_chronicle_uk_local_2025.py index f5c7816a..08fc06f4 100644 --- a/tests/test_chronicle_uk_local_2025.py +++ b/tests/test_chronicle_uk_local_2025.py @@ -5,6 +5,7 @@ from collections import Counter, defaultdict from dataclasses import replace from datetime import datetime +from functools import lru_cache from pathlib import Path import pytest @@ -13,6 +14,20 @@ from chronicle.consumer_contract import validate_consumer_fact_contract from chronicle.source_package import SOURCE_PACKAGE_ALIASES, load_source_package +# NRS dwellings by 2022 data zone: council tax bands A to H in columns G to N. +DATA_ZONE_BAND_COLUMNS = dict(zip("ABCDEFGH", "GHIJKLMN", strict=True)) + + +@lru_cache +def _data_zone_dwellings(): + return load_source_package("nrs-dwellings-data-zone-2025") + + +@lru_cache +def _data_zone_dwelling_cells(): + # Parsing the workbook takes over a minute; building facts from its cells, 8s. + return _data_zone_dwellings().build_source_cells(2025) + def test_mid2025_local_authority_population_keeps_all_four_nations(): """Nomis NM_31_1 must retain NI's eleven districts alongside GB.""" @@ -109,7 +124,7 @@ def test_household_controls_keep_private_households_projections_and_stock_distin def test_scottish_data_zone_council_tax_bands_sum_to_each_zone_total(): """chronicle#419: columns G to N (bands A to H) add up to column E.""" - facts = load_source_package("nrs-dwellings-data-zone-2025").build_facts(2025) + facts = _data_zone_dwellings().build_facts(2025, cells=_data_zone_dwelling_cells()) bands = tuple(f"nrs.dwellings_band_{band}" for band in "abcdefgh") zones = defaultdict(dict) for fact in facts: @@ -130,6 +145,26 @@ def test_scottish_data_zone_council_tax_bands_sum_to_each_zone_total(): assert validate_consumer_fact_contract(facts).valid +@pytest.mark.parametrize("sheet", ["2024", "2025"]) +@pytest.mark.parametrize("band", list(DATA_ZONE_BAND_COLUMNS)) +def test_scottish_data_zone_band_header_guards_reject_drift(sheet, band): + """Each band's guard stops the build when its column holds the next band.""" + address = f"{DATA_ZONE_BAND_COLUMNS[band]}5" + next_band = "BCDEFGHA"["ABCDEFGH".index(band)] + changed_cells = [ + replace(cell, raw_value=f"Council Tax band:\n{next_band}") + if (cell.sheet_name, cell.address) == (sheet, address) + else cell + for cell in _data_zone_dwelling_cells() + ] + measure = f"nrs_dwellings_band_{band.lower()}" + with pytest.raises( + ValueError, + match=rf"\.{sheet}\.\w+\.{measure}\.selector' expected column header", + ): + _data_zone_dwellings().build_facts(2025, cells=changed_cells) + + def test_rm120_retains_age_sex_residence_and_2023_lad_boundaries(): facts = load_source_package( "ons-census2021-rm120-pcon24-residence2-sex0"