diff --git a/changelog.d/nrs-data-zone-council-tax-bands-419.added.md b/changelog.d/nrs-data-zone-council-tax-bands-419.added.md new file mode 100644 index 00000000..4547ae4a --- /dev/null +++ b/changelog.d/nrs-data-zone-council-tax-bands-419.added.md @@ -0,0 +1 @@ +Add the council tax band columns of NRS's dwelling estimates by 2022 data zone, which chronicle#419 asks for (PolicyEngine/microcosm#931): dwellings in bands A to H in each of Scotland's 7,392 data zones, December 2024 and December 2025, as `nrs.dwellings_band_a` to `nrs.dwellings_band_h` beside `nrs.dwellings_total` in `nrs-dwellings-data-zone-2025`. A test checks that the eight bands add up to the zone's total in every zone of both sheets. diff --git a/packages/nrs/dwellings_data_zone_2025/source_package.yaml b/packages/nrs/dwellings_data_zone_2025/source_package.yaml index 070a7f8b..718b13c2 100644 --- a/packages/nrs/dwellings_data_zone_2025/source_package.yaml +++ b/packages/nrs/dwellings_data_zone_2025/source_package.yaml @@ -81358,6 +81358,110 @@ record_sets: expected_cell_type: number expected_column_header_row: 5 expected_column_header: Total number of dwellings + - measure_id: nrs_dwellings_band_a + label: Dwellings, Council Tax band A + ordinal: 1 + column: G + source_column_id: council_tax_band_a + concept: nrs.dwellings_band_a + unit: count + aggregation: sum + expected_cell_type: number + expected_column_header_row: 5 + expected_column_header: 'Council Tax band: + + A' + - measure_id: nrs_dwellings_band_b + label: Dwellings, Council Tax band B + ordinal: 2 + column: H + source_column_id: council_tax_band_b + concept: nrs.dwellings_band_b + unit: count + aggregation: sum + expected_cell_type: number + expected_column_header_row: 5 + expected_column_header: 'Council Tax band: + + B' + - measure_id: nrs_dwellings_band_c + label: Dwellings, Council Tax band C + ordinal: 3 + column: I + source_column_id: council_tax_band_c + concept: nrs.dwellings_band_c + unit: count + aggregation: sum + expected_cell_type: number + expected_column_header_row: 5 + expected_column_header: 'Council Tax band: + + C' + - measure_id: nrs_dwellings_band_d + label: Dwellings, Council Tax band D + ordinal: 4 + column: J + source_column_id: council_tax_band_d + concept: nrs.dwellings_band_d + unit: count + aggregation: sum + expected_cell_type: number + expected_column_header_row: 5 + expected_column_header: 'Council Tax band: + + D' + - measure_id: nrs_dwellings_band_e + label: Dwellings, Council Tax band E + ordinal: 5 + column: K + source_column_id: council_tax_band_e + concept: nrs.dwellings_band_e + unit: count + aggregation: sum + expected_cell_type: number + expected_column_header_row: 5 + expected_column_header: 'Council Tax band: + + E' + - measure_id: nrs_dwellings_band_f + label: Dwellings, Council Tax band F + ordinal: 6 + column: L + source_column_id: council_tax_band_f + concept: nrs.dwellings_band_f + unit: count + aggregation: sum + expected_cell_type: number + expected_column_header_row: 5 + expected_column_header: 'Council Tax band: + + F' + - measure_id: nrs_dwellings_band_g + label: Dwellings, Council Tax band G + ordinal: 7 + column: M + source_column_id: council_tax_band_g + concept: nrs.dwellings_band_g + unit: count + aggregation: sum + expected_cell_type: number + expected_column_header_row: 5 + expected_column_header: 'Council Tax band: + + G' + - measure_id: nrs_dwellings_band_h + label: Dwellings, Council Tax band H + ordinal: 8 + column: N + source_column_id: council_tax_band_h + concept: nrs.dwellings_band_h + unit: count + aggregation: sum + expected_cell_type: number + expected_column_header_row: 5 + expected_column_header: 'Council Tax band: + + H' period_coverage: start_date: '2024-12-31' end_date: '2024-12-31' @@ -162704,6 +162808,110 @@ record_sets: expected_cell_type: number expected_column_header_row: 5 expected_column_header: Total number of dwellings + - measure_id: nrs_dwellings_band_a + label: Dwellings, Council Tax band A + ordinal: 1 + column: G + source_column_id: council_tax_band_a + concept: nrs.dwellings_band_a + unit: count + aggregation: sum + expected_cell_type: number + expected_column_header_row: 5 + expected_column_header: 'Council Tax band: + + A' + - measure_id: nrs_dwellings_band_b + label: Dwellings, Council Tax band B + ordinal: 2 + column: H + source_column_id: council_tax_band_b + concept: nrs.dwellings_band_b + unit: count + aggregation: sum + expected_cell_type: number + expected_column_header_row: 5 + expected_column_header: 'Council Tax band: + + B' + - measure_id: nrs_dwellings_band_c + label: Dwellings, Council Tax band C + ordinal: 3 + column: I + source_column_id: council_tax_band_c + concept: nrs.dwellings_band_c + unit: count + aggregation: sum + expected_cell_type: number + expected_column_header_row: 5 + expected_column_header: 'Council Tax band: + + C' + - measure_id: nrs_dwellings_band_d + label: Dwellings, Council Tax band D + ordinal: 4 + column: J + source_column_id: council_tax_band_d + concept: nrs.dwellings_band_d + unit: count + aggregation: sum + expected_cell_type: number + expected_column_header_row: 5 + expected_column_header: 'Council Tax band: + + D' + - measure_id: nrs_dwellings_band_e + label: Dwellings, Council Tax band E + ordinal: 5 + column: K + source_column_id: council_tax_band_e + concept: nrs.dwellings_band_e + unit: count + aggregation: sum + expected_cell_type: number + expected_column_header_row: 5 + expected_column_header: 'Council Tax band: + + E' + - measure_id: nrs_dwellings_band_f + label: Dwellings, Council Tax band F + ordinal: 6 + column: L + source_column_id: council_tax_band_f + concept: nrs.dwellings_band_f + unit: count + aggregation: sum + expected_cell_type: number + expected_column_header_row: 5 + expected_column_header: 'Council Tax band: + + F' + - measure_id: nrs_dwellings_band_g + label: Dwellings, Council Tax band G + ordinal: 7 + column: M + source_column_id: council_tax_band_g + concept: nrs.dwellings_band_g + unit: count + aggregation: sum + expected_cell_type: number + expected_column_header_row: 5 + expected_column_header: 'Council Tax band: + + G' + - measure_id: nrs_dwellings_band_h + label: Dwellings, Council Tax band H + ordinal: 8 + column: N + source_column_id: council_tax_band_h + concept: nrs.dwellings_band_h + unit: count + aggregation: sum + expected_cell_type: number + expected_column_header_row: 5 + expected_column_header: 'Council Tax band: + + H' period_coverage: start_date: '2025-12-31' end_date: '2025-12-31' diff --git a/packages/uk_local_2025_sources.md b/packages/uk_local_2025_sources.md index ea4599d0..206bc987 100644 --- a/packages/uk_local_2025_sources.md +++ b/packages/uk_local_2025_sources.md @@ -4,7 +4,7 @@ Chronicle preserves the publisher artifacts requested in [issue 313](https://git | Source family | Packages and coverage | Publisher semantics | | --- | --- | --- | -| Scotland households | `nrs-households-dwellings-2025`, `nrs-households-data-zone-2025`, `nrs-dwellings-data-zone-2025` | Council and country private-household estimates: June 2024/2025. Council dwelling stock: September. Data-zone occupied dwellings: September; dwelling stock: December. The small-area occupied-dwelling table does not assert the adjusted private-household estimate. | +| Scotland households | `nrs-households-dwellings-2025`, `nrs-households-data-zone-2025`, `nrs-dwellings-data-zone-2025` | Council and country private-household estimates: June 2024/2025. Council dwelling stock: September. Data-zone occupied dwellings: September; dwelling stock: December, in total and by council tax band A–H ([issue 419](https://github.com/PolicyEngine/chronicle/issues/419)). The small-area occupied-dwelling table does not assert the adjusted private-household estimate. | | Wales households | `welshgov-household-estimates-mid2024`, `welshgov-household-projections-2022-based` | Revised mid-2022/2023/2024 estimates and principal mid-2025 projection. Publisher decimals retained. The estimate selector excludes private-household population and average household size. | | England households | `ons-household-projections-2022-based` | Table 406, all 349 areas, mid-2022 through mid-2047, 2022 boundaries; metropolitan counties remain counties. `source_projection` identifies each publisher projection. | | NI housing stock | `lps-housing-stock-lgd-2026` | Eleven districts and NI country, 1 April 2021–2026. Domestic or mixed-rated dwelling stock. Chronicle retains the published snapshots; consumers calculate roll-forward factors. | diff --git a/tests/test_chronicle_bundle.py b/tests/test_chronicle_bundle.py index 7964f784..124b3357 100644 --- a/tests/test_chronicle_bundle.py +++ b/tests/test_chronicle_bundle.py @@ -562,7 +562,12 @@ def test_build_bundle_writes_merged_consumer_contract(tmp_path): # Survey's chapters 1, 3 and 4 (4,458), HMRC's stamp tax Tables 2e, 3a, # 5a, 7, 9a and 9d (1,135), ONS's land by sector (387) and the council # rows of the Welsh (308) and Scottish (96) council tax tables. - "fact_count": 2062484, + # chronicle#419 adds 118,272 dwelling facts: NRS's council tax bands A to + # H for each of Scotland's 7,392 2022 data zones, December 2024 and + # December 2025. Measured with only nrs-dwellings-data-zone-2025 at + # bundle year 2023: no new geography, period or entity key, duplicate + # key or warning. + "fact_count": 2180756, # The same change adds 82,064 geography keys: 46,363 Scottish 2022 # output areas (UV404), 35,673 LSOA-level keys from the DESNZ LSOA # workbooks (Scotland's data zones already exist), the 26 Northern @@ -686,7 +691,7 @@ def test_build_bundle_writes_merged_consumer_contract(tmp_path): if row_count == 1: assert row["aggregate_fact_key"].startswith("ledger.aggregate_fact.v2:") assert row["semantic_fact_key"].startswith("ledger.semantic_fact.v2:") - assert row_count == 2062484 + assert row_count == 2180756 assert provenance_classes <= { "administrative", "census", @@ -721,7 +726,7 @@ def test_build_bundle_writes_merged_consumer_contract(tmp_path): "jct-obbba-revenue-estimates-2025", "jct-tax-expenditures-2024", ] - assert coverage["fact_count"] == 2062484 + assert coverage["fact_count"] == 2180756 assert coverage["counts"]["by_source"] == { "bea": 446, "bfp_economic_outlook": 5, @@ -759,7 +764,7 @@ def test_build_bundle_writes_merged_consumer_contract(tmp_path): "nbb_national_accounts": 1, "nisra": 29227, "nithc": 8, - "nrs": 359094, + "nrs": 477366, "obr": 4539, "ofgem": 3640, "onem_rva_unemployment": 1, @@ -2532,6 +2537,10 @@ def test_build_bundle_writes_merged_consumer_contract(tmp_path): } for key, count in chronicle_408_period_increments.items(): expected_period_counts[key] = expected_period_counts.get(key, 0) + count + # chronicle#419: NRS's eight council tax bands for each of the 7,392 data + # zones, at December 2024 and December 2025. + for year in (2024, 2025): + expected_period_counts[f"calendar_year:{year}"] += 8 * 7392 assert coverage["counts"]["by_period"] == expected_period_counts # country:NZ: 330 WFF (#209) + 5 national population (#211) + 27 Treasury # (#321) + 3,522 IRD I1 (#318) + 510 IRD I2 (#210) + 103 MSD (#319) @@ -2675,9 +2684,10 @@ def test_build_bundle_writes_merged_consumer_contract(tmp_path): # QRC4 5,757, Table 6 1,208, Wales 308 and Scotland 96), households +1,666 # (the Market Value Survey), returns +1,135 (HMRC's stamp tax), firms +401 # (the land value estimates) and institutional sectors +387 (ONS's land). + # chronicle#419: dwellings +118,272 (NRS's council tax bands by data zone). assert coverage["counts"]["by_entity"] == { "benefit_unit": 33645, - "dwelling": 590264, + "dwelling": 708536, "family": 1700, "firm": 1905, "government": 14071, diff --git a/tests/test_chronicle_uk_local_2025.py b/tests/test_chronicle_uk_local_2025.py index 5d24436e..08fc06f4 100644 --- a/tests/test_chronicle_uk_local_2025.py +++ b/tests/test_chronicle_uk_local_2025.py @@ -2,9 +2,10 @@ import hashlib import json -from collections import defaultdict +from collections import Counter, defaultdict from dataclasses import replace from datetime import datetime +from functools import lru_cache from pathlib import Path import pytest @@ -13,6 +14,20 @@ from chronicle.consumer_contract import validate_consumer_fact_contract from chronicle.source_package import SOURCE_PACKAGE_ALIASES, load_source_package +# NRS dwellings by 2022 data zone: council tax bands A to H in columns G to N. +DATA_ZONE_BAND_COLUMNS = dict(zip("ABCDEFGH", "GHIJKLMN", strict=True)) + + +@lru_cache +def _data_zone_dwellings(): + return load_source_package("nrs-dwellings-data-zone-2025") + + +@lru_cache +def _data_zone_dwelling_cells(): + # Parsing the workbook takes over a minute; building facts from its cells, 8s. + return _data_zone_dwellings().build_source_cells(2025) + def test_mid2025_local_authority_population_keeps_all_four_nations(): """Nomis NM_31_1 must retain NI's eleven districts alongside GB.""" @@ -107,6 +122,49 @@ def test_household_controls_keep_private_households_projections_and_stock_distin } +def test_scottish_data_zone_council_tax_bands_sum_to_each_zone_total(): + """chronicle#419: columns G to N (bands A to H) add up to column E.""" + facts = _data_zone_dwellings().build_facts(2025, cells=_data_zone_dwelling_cells()) + bands = tuple(f"nrs.dwellings_band_{band}" for band in "abcdefgh") + zones = defaultdict(dict) + for fact in facts: + zones[fact.period.value, fact.geography.id][fact.measure.concept] = fact.value + assert len(facts) == 2 * 7_392 * 9 + assert Counter(period for period, _ in zones) == {2024: 7_392, 2025: 7_392} + for key, values in zones.items(): + assert set(values) == {"nrs.dwellings_total", *bands}, key + assert sum(values[band] for band in bands) == values["nrs.dwellings_total"], key + # Culter - 01 (S01013482, row 6), bands A to H as published. + published = { + 2024: [21, 153, 45, 55, 100, 44, 85, 13], + 2025: [21, 153, 45, 55, 99, 45, 85, 13], + } + for period, counts in published.items(): + assert [zones[period, "S01013482"][band] for band in bands] == counts + assert all(f.source_cell_keys for f in facts) + assert validate_consumer_fact_contract(facts).valid + + +@pytest.mark.parametrize("sheet", ["2024", "2025"]) +@pytest.mark.parametrize("band", list(DATA_ZONE_BAND_COLUMNS)) +def test_scottish_data_zone_band_header_guards_reject_drift(sheet, band): + """Each band's guard stops the build when its column holds the next band.""" + address = f"{DATA_ZONE_BAND_COLUMNS[band]}5" + next_band = "BCDEFGHA"["ABCDEFGH".index(band)] + changed_cells = [ + replace(cell, raw_value=f"Council Tax band:\n{next_band}") + if (cell.sheet_name, cell.address) == (sheet, address) + else cell + for cell in _data_zone_dwelling_cells() + ] + measure = f"nrs_dwellings_band_{band.lower()}" + with pytest.raises( + ValueError, + match=rf"\.{sheet}\.\w+\.{measure}\.selector' expected column header", + ): + _data_zone_dwellings().build_facts(2025, cells=changed_cells) + + def test_rm120_retains_age_sex_residence_and_2023_lad_boundaries(): facts = load_source_package( "ons-census2021-rm120-pcon24-residence2-sex0"