diff --git a/.gitattributes b/.gitattributes index 6f6eb6ff..5ede0734 100644 --- a/.gitattributes +++ b/.gitattributes @@ -21,5 +21,6 @@ db/data/hmrc/winter_fuel_payment_charge_2025/winter-fuel-payments-charge.html -d # Preserve hash-pinned publisher bytes, including original line endings/spacing. db/data/stats_nz/subnational_population_estimates_2025/subnational-population-estimates-at-30-june-2025-provisional.xlsx binary db/data/stats_nz/regional_council_2025_codes/regional-council-2025-codes.json binary +db/data/stats_nz/territorial_authority_2025_codes/territorial-authority-2025-codes.json binary db/data/stats_nz/national_population_estimates_2025/national-population-estimates-at-30-june-2025-landing-page.html binary db/data/mbie/tenancy_bond_rents_tla_2026/detailed-monthly-tla-tenancy-september.csv binary diff --git a/changelog.d/nz-subnational-population-ta-2025.added.md b/changelog.d/nz-subnational-population-ta-2025.added.md new file mode 100644 index 00000000..03d51ba9 --- /dev/null +++ b/changelog.d/nz-subnational-population-ta-2025.added.md @@ -0,0 +1,6 @@ +Add 67 provisional Stats NZ estimated resident population facts for territorial +authorities at 30 June 2025, each read directly from the pinned workbook's +Table 2. Use publisher TA 2025 codes and names with the existing `local_authority`, +`nz-ta-NNN`, `ta_2025` convention. Preserve local boards, other years, changes, +percentages and the duplicate national row without emitting them; no sums or +reconciliation are added. diff --git a/chronicle/bundle.py b/chronicle/bundle.py index 72f10417..aff99cd0 100644 --- a/chronicle/bundle.py +++ b/chronicle/bundle.py @@ -373,6 +373,7 @@ "stats-nz-census-2023-weekly-rent-rc-99", "stats-nz-household-net-worth-2024", "stats-nz-subnational-population-estimates-2025", + "stats-nz-subnational-population-ta-2025", "treasury-an24-01-fiscal-totals", "treasury-estimates-of-appropriations-2026-27", ) diff --git a/chronicle/source_package.py b/chronicle/source_package.py index 3824b6e6..02d50ae8 100644 --- a/chronicle/source_package.py +++ b/chronicle/source_package.py @@ -242,6 +242,9 @@ "stats-nz-subnational-population-estimates-2025": Path( "stats_nz/subnational_population_estimates_2025" ), + "stats-nz-subnational-population-ta-2025": Path( + "stats_nz/subnational_population_ta_2025" + ), "ons-mye-2025-uk": Path("ons/mye_2025_uk"), "ons-mye-2024-uk-revised-2026": Path("ons/mye_2024_uk_revised_2026"), "slc-student-loan-repayments-england-2026": Path( diff --git a/db/data/stats_nz/territorial_authority_2025_codes/manifest.yaml b/db/data/stats_nz/territorial_authority_2025_codes/manifest.yaml new file mode 100644 index 00000000..42567ce3 --- /dev/null +++ b/db/data/stats_nz/territorial_authority_2025_codes/manifest.yaml @@ -0,0 +1,18 @@ +source_id: stats_nz +package_id: territorial_authority_2025_codes +dataset: stats_nz_territorial_authority_2025_codes +source_page: https://services2.arcgis.com/vKb0s8tBIA3bdocZ/arcgis/rest/services/Territorial_Authority_2025/FeatureServer +table: Territorial authority 2025 code and name attributes (1 January 2025 boundaries) +files: + 2025: + filename: territorial-authority-2025-codes.json + source_url: https://services2.arcgis.com/vKb0s8tBIA3bdocZ/arcgis/rest/services/Territorial_Authority_2025/FeatureServer/0/query?where=1%3D1&outFields=TA2025_V1_00%2CTA2025_V1_00_NAME%2CTA2025_V1_00_NAME_ASCII&returnGeometry=false&orderByFields=TA2025_V1_00&f=pjson + sha256: db033184db1f64bd4c53b0cf070cac3303815f20c9cc428e11389b91033dc225 + size_bytes: 14315 + fetched_at: '2026-10-13T16:42:27+00:00' + storage: + r2: + provider: r2 + bucket: ledger-raw + key: raw/nz/stats_nz/territorial_authority_2025_codes/2025/db033184db1f64bd4c53b0cf070cac3303815f20c9cc428e11389b91033dc225/territorial-authority-2025-codes.json + uri: r2://ledger-raw/raw/nz/stats_nz/territorial_authority_2025_codes/2025/db033184db1f64bd4c53b0cf070cac3303815f20c9cc428e11389b91033dc225/territorial-authority-2025-codes.json diff --git a/db/data/stats_nz/territorial_authority_2025_codes/territorial-authority-2025-codes.json b/db/data/stats_nz/territorial_authority_2025_codes/territorial-authority-2025-codes.json new file mode 100644 index 00000000..388d80d7 --- /dev/null +++ b/db/data/stats_nz/territorial_authority_2025_codes/territorial-authority-2025-codes.json @@ -0,0 +1,521 @@ +{ + "objectIdFieldName" : "OBJECTID", + "uniqueIdField" : + { + "name" : "OBJECTID", + "isSystemMaintained" : true + }, + "globalIdFieldName" : "", + "geometryType" : "esriGeometryPolygon", + "spatialReference" : { + "wkid" : 2193, + "latestWkid" : 2193 + }, + "fields" : [ + { + "name" : "TA2025_V1_00", + "type" : "esriFieldTypeString", + "alias" : "Territorial Authority Code", + "sqlType" : "sqlTypeOther", + "length" : 10, + "domain" : null, + "defaultValue" : null + }, + { + "name" : "TA2025_V1_00_NAME", + "type" : "esriFieldTypeString", + "alias" : "Territorial Authority Name", + "sqlType" : "sqlTypeOther", + "length" : 250, + "domain" : null, + "defaultValue" : null + }, + { + "name" : "TA2025_V1_00_NAME_ASCII", + "type" : "esriFieldTypeString", + "alias" : "TA2025_V1_00_NAME_ASCII", + "sqlType" : "sqlTypeOther", + "length" : 250, + "domain" : null, + "defaultValue" : null + } + ], + "features" : [ + { + "attributes" : { + "TA2025_V1_00" : "001", + "TA2025_V1_00_NAME" : "Far North District", + "TA2025_V1_00_NAME_ASCII" : "Far North District" + } + }, + { + "attributes" : { + "TA2025_V1_00" : "002", + "TA2025_V1_00_NAME" : "Whangarei District", + "TA2025_V1_00_NAME_ASCII" : "Whangarei District" + } + }, + { + "attributes" : { + "TA2025_V1_00" : "003", + "TA2025_V1_00_NAME" : "Kaipara District", + "TA2025_V1_00_NAME_ASCII" : "Kaipara District" + } + }, + { + "attributes" : { + "TA2025_V1_00" : "011", + "TA2025_V1_00_NAME" : "Thames-Coromandel District", + "TA2025_V1_00_NAME_ASCII" : "Thames-Coromandel District" + } + }, + { + "attributes" : { + "TA2025_V1_00" : "012", + "TA2025_V1_00_NAME" : "Hauraki District", + "TA2025_V1_00_NAME_ASCII" : "Hauraki District" + } + }, + { + "attributes" : { + "TA2025_V1_00" : "013", + "TA2025_V1_00_NAME" : "Waikato District", + "TA2025_V1_00_NAME_ASCII" : "Waikato District" + } + }, + { + "attributes" : { + "TA2025_V1_00" : "015", + "TA2025_V1_00_NAME" : "Matamata-Piako District", + "TA2025_V1_00_NAME_ASCII" : "Matamata-Piako District" + } + }, + { + "attributes" : { + "TA2025_V1_00" : "016", + "TA2025_V1_00_NAME" : "Hamilton City", + "TA2025_V1_00_NAME_ASCII" : "Hamilton City" + } + }, + { + "attributes" : { + "TA2025_V1_00" : "017", + "TA2025_V1_00_NAME" : "Waipa District", + "TA2025_V1_00_NAME_ASCII" : "Waipa District" + } + }, + { + "attributes" : { + "TA2025_V1_00" : "018", + "TA2025_V1_00_NAME" : "Ōtorohanga District", + "TA2025_V1_00_NAME_ASCII" : "Otorohanga District" + } + }, + { + "attributes" : { + "TA2025_V1_00" : "019", + "TA2025_V1_00_NAME" : "South Waikato District", + "TA2025_V1_00_NAME_ASCII" : "South Waikato District" + } + }, + { + "attributes" : { + "TA2025_V1_00" : "020", + "TA2025_V1_00_NAME" : "Waitomo District", + "TA2025_V1_00_NAME_ASCII" : "Waitomo District" + } + }, + { + "attributes" : { + "TA2025_V1_00" : "021", + "TA2025_V1_00_NAME" : "Taupo District", + "TA2025_V1_00_NAME_ASCII" : "Taupo District" + } + }, + { + "attributes" : { + "TA2025_V1_00" : "022", + "TA2025_V1_00_NAME" : "Western Bay of Plenty District", + "TA2025_V1_00_NAME_ASCII" : "Western Bay of Plenty District" + } + }, + { + "attributes" : { + "TA2025_V1_00" : "023", + "TA2025_V1_00_NAME" : "Tauranga City", + "TA2025_V1_00_NAME_ASCII" : "Tauranga City" + } + }, + { + "attributes" : { + "TA2025_V1_00" : "024", + "TA2025_V1_00_NAME" : "Rotorua District", + "TA2025_V1_00_NAME_ASCII" : "Rotorua District" + } + }, + { + "attributes" : { + "TA2025_V1_00" : "025", + "TA2025_V1_00_NAME" : "Whakatane District", + "TA2025_V1_00_NAME_ASCII" : "Whakatane District" + } + }, + { + "attributes" : { + "TA2025_V1_00" : "026", + "TA2025_V1_00_NAME" : "Kawerau District", + "TA2025_V1_00_NAME_ASCII" : "Kawerau District" + } + }, + { + "attributes" : { + "TA2025_V1_00" : "027", + "TA2025_V1_00_NAME" : "Ōpōtiki District", + "TA2025_V1_00_NAME_ASCII" : "Opotiki District" + } + }, + { + "attributes" : { + "TA2025_V1_00" : "028", + "TA2025_V1_00_NAME" : "Gisborne District", + "TA2025_V1_00_NAME_ASCII" : "Gisborne District" + } + }, + { + "attributes" : { + "TA2025_V1_00" : "029", + "TA2025_V1_00_NAME" : "Wairoa District", + "TA2025_V1_00_NAME_ASCII" : "Wairoa District" + } + }, + { + "attributes" : { + "TA2025_V1_00" : "030", + "TA2025_V1_00_NAME" : "Hastings District", + "TA2025_V1_00_NAME_ASCII" : "Hastings District" + } + }, + { + "attributes" : { + "TA2025_V1_00" : "031", + "TA2025_V1_00_NAME" : "Napier City", + "TA2025_V1_00_NAME_ASCII" : "Napier City" + } + }, + { + "attributes" : { + "TA2025_V1_00" : "032", + "TA2025_V1_00_NAME" : "Central Hawke's Bay District", + "TA2025_V1_00_NAME_ASCII" : "Central Hawke's Bay District" + } + }, + { + "attributes" : { + "TA2025_V1_00" : "033", + "TA2025_V1_00_NAME" : "New Plymouth District", + "TA2025_V1_00_NAME_ASCII" : "New Plymouth District" + } + }, + { + "attributes" : { + "TA2025_V1_00" : "034", + "TA2025_V1_00_NAME" : "Stratford District", + "TA2025_V1_00_NAME_ASCII" : "Stratford District" + } + }, + { + "attributes" : { + "TA2025_V1_00" : "035", + "TA2025_V1_00_NAME" : "South Taranaki District", + "TA2025_V1_00_NAME_ASCII" : "South Taranaki District" + } + }, + { + "attributes" : { + "TA2025_V1_00" : "036", + "TA2025_V1_00_NAME" : "Ruapehu District", + "TA2025_V1_00_NAME_ASCII" : "Ruapehu District" + } + }, + { + "attributes" : { + "TA2025_V1_00" : "037", + "TA2025_V1_00_NAME" : "Whanganui District", + "TA2025_V1_00_NAME_ASCII" : "Whanganui District" + } + }, + { + "attributes" : { + "TA2025_V1_00" : "038", + "TA2025_V1_00_NAME" : "Rangitikei District", + "TA2025_V1_00_NAME_ASCII" : "Rangitikei District" + } + }, + { + "attributes" : { + "TA2025_V1_00" : "039", + "TA2025_V1_00_NAME" : "Manawatu District", + "TA2025_V1_00_NAME_ASCII" : "Manawatu District" + } + }, + { + "attributes" : { + "TA2025_V1_00" : "040", + "TA2025_V1_00_NAME" : "Palmerston North City", + "TA2025_V1_00_NAME_ASCII" : "Palmerston North City" + } + }, + { + "attributes" : { + "TA2025_V1_00" : "041", + "TA2025_V1_00_NAME" : "Tararua District", + "TA2025_V1_00_NAME_ASCII" : "Tararua District" + } + }, + { + "attributes" : { + "TA2025_V1_00" : "042", + "TA2025_V1_00_NAME" : "Horowhenua District", + "TA2025_V1_00_NAME_ASCII" : "Horowhenua District" + } + }, + { + "attributes" : { + "TA2025_V1_00" : "043", + "TA2025_V1_00_NAME" : "Kapiti Coast District", + "TA2025_V1_00_NAME_ASCII" : "Kapiti Coast District" + } + }, + { + "attributes" : { + "TA2025_V1_00" : "044", + "TA2025_V1_00_NAME" : "Porirua City", + "TA2025_V1_00_NAME_ASCII" : "Porirua City" + } + }, + { + "attributes" : { + "TA2025_V1_00" : "045", + "TA2025_V1_00_NAME" : "Upper Hutt City", + "TA2025_V1_00_NAME_ASCII" : "Upper Hutt City" + } + }, + { + "attributes" : { + "TA2025_V1_00" : "046", + "TA2025_V1_00_NAME" : "Lower Hutt City", + "TA2025_V1_00_NAME_ASCII" : "Lower Hutt City" + } + }, + { + "attributes" : { + "TA2025_V1_00" : "047", + "TA2025_V1_00_NAME" : "Wellington City", + "TA2025_V1_00_NAME_ASCII" : "Wellington City" + } + }, + { + "attributes" : { + "TA2025_V1_00" : "048", + "TA2025_V1_00_NAME" : "Masterton District", + "TA2025_V1_00_NAME_ASCII" : "Masterton District" + } + }, + { + "attributes" : { + "TA2025_V1_00" : "049", + "TA2025_V1_00_NAME" : "Carterton District", + "TA2025_V1_00_NAME_ASCII" : "Carterton District" + } + }, + { + "attributes" : { + "TA2025_V1_00" : "050", + "TA2025_V1_00_NAME" : "South Wairarapa District", + "TA2025_V1_00_NAME_ASCII" : "South Wairarapa District" + } + }, + { + "attributes" : { + "TA2025_V1_00" : "051", + "TA2025_V1_00_NAME" : "Tasman District", + "TA2025_V1_00_NAME_ASCII" : "Tasman District" + } + }, + { + "attributes" : { + "TA2025_V1_00" : "052", + "TA2025_V1_00_NAME" : "Nelson City", + "TA2025_V1_00_NAME_ASCII" : "Nelson City" + } + }, + { + "attributes" : { + "TA2025_V1_00" : "053", + "TA2025_V1_00_NAME" : "Marlborough District", + "TA2025_V1_00_NAME_ASCII" : "Marlborough District" + } + }, + { + "attributes" : { + "TA2025_V1_00" : "054", + "TA2025_V1_00_NAME" : "Kaikoura District", + "TA2025_V1_00_NAME_ASCII" : "Kaikoura District" + } + }, + { + "attributes" : { + "TA2025_V1_00" : "055", + "TA2025_V1_00_NAME" : "Buller District", + "TA2025_V1_00_NAME_ASCII" : "Buller District" + } + }, + { + "attributes" : { + "TA2025_V1_00" : "056", + "TA2025_V1_00_NAME" : "Grey District", + "TA2025_V1_00_NAME_ASCII" : "Grey District" + } + }, + { + "attributes" : { + "TA2025_V1_00" : "057", + "TA2025_V1_00_NAME" : "Westland District", + "TA2025_V1_00_NAME_ASCII" : "Westland District" + } + }, + { + "attributes" : { + "TA2025_V1_00" : "058", + "TA2025_V1_00_NAME" : "Hurunui District", + "TA2025_V1_00_NAME_ASCII" : "Hurunui District" + } + }, + { + "attributes" : { + "TA2025_V1_00" : "059", + "TA2025_V1_00_NAME" : "Waimakariri District", + "TA2025_V1_00_NAME_ASCII" : "Waimakariri District" + } + }, + { + "attributes" : { + "TA2025_V1_00" : "060", + "TA2025_V1_00_NAME" : "Christchurch City", + "TA2025_V1_00_NAME_ASCII" : "Christchurch City" + } + }, + { + "attributes" : { + "TA2025_V1_00" : "062", + "TA2025_V1_00_NAME" : "Selwyn District", + "TA2025_V1_00_NAME_ASCII" : "Selwyn District" + } + }, + { + "attributes" : { + "TA2025_V1_00" : "063", + "TA2025_V1_00_NAME" : "Ashburton District", + "TA2025_V1_00_NAME_ASCII" : "Ashburton District" + } + }, + { + "attributes" : { + "TA2025_V1_00" : "064", + "TA2025_V1_00_NAME" : "Timaru District", + "TA2025_V1_00_NAME_ASCII" : "Timaru District" + } + }, + { + "attributes" : { + "TA2025_V1_00" : "065", + "TA2025_V1_00_NAME" : "Mackenzie District", + "TA2025_V1_00_NAME_ASCII" : "Mackenzie District" + } + }, + { + "attributes" : { + "TA2025_V1_00" : "066", + "TA2025_V1_00_NAME" : "Waimate District", + "TA2025_V1_00_NAME_ASCII" : "Waimate District" + } + }, + { + "attributes" : { + "TA2025_V1_00" : "067", + "TA2025_V1_00_NAME" : "Chatham Islands Territory", + "TA2025_V1_00_NAME_ASCII" : "Chatham Islands Territory" + } + }, + { + "attributes" : { + "TA2025_V1_00" : "068", + "TA2025_V1_00_NAME" : "Waitaki District", + "TA2025_V1_00_NAME_ASCII" : "Waitaki District" + } + }, + { + "attributes" : { + "TA2025_V1_00" : "069", + "TA2025_V1_00_NAME" : "Central Otago District", + "TA2025_V1_00_NAME_ASCII" : "Central Otago District" + } + }, + { + "attributes" : { + "TA2025_V1_00" : "070", + "TA2025_V1_00_NAME" : "Queenstown-Lakes District", + "TA2025_V1_00_NAME_ASCII" : "Queenstown-Lakes District" + } + }, + { + "attributes" : { + "TA2025_V1_00" : "071", + "TA2025_V1_00_NAME" : "Dunedin City", + "TA2025_V1_00_NAME_ASCII" : "Dunedin City" + } + }, + { + "attributes" : { + "TA2025_V1_00" : "072", + "TA2025_V1_00_NAME" : "Clutha District", + "TA2025_V1_00_NAME_ASCII" : "Clutha District" + } + }, + { + "attributes" : { + "TA2025_V1_00" : "073", + "TA2025_V1_00_NAME" : "Southland District", + "TA2025_V1_00_NAME_ASCII" : "Southland District" + } + }, + { + "attributes" : { + "TA2025_V1_00" : "074", + "TA2025_V1_00_NAME" : "Gore District", + "TA2025_V1_00_NAME_ASCII" : "Gore District" + } + }, + { + "attributes" : { + "TA2025_V1_00" : "075", + "TA2025_V1_00_NAME" : "Invercargill City", + "TA2025_V1_00_NAME_ASCII" : "Invercargill City" + } + }, + { + "attributes" : { + "TA2025_V1_00" : "076", + "TA2025_V1_00_NAME" : "Auckland", + "TA2025_V1_00_NAME_ASCII" : "Auckland" + } + }, + { + "attributes" : { + "TA2025_V1_00" : "999", + "TA2025_V1_00_NAME" : "Area Outside Territorial Authority", + "TA2025_V1_00_NAME_ASCII" : "Area Outside Territorial Authority" + } + } + ] +} \ No newline at end of file diff --git a/docs/pe-nz-population-source-notes.md b/docs/pe-nz-population-source-notes.md index 40c534b6..679c6c5b 100644 --- a/docs/pe-nz-population-source-notes.md +++ b/docs/pe-nz-population-source-notes.md @@ -2,7 +2,8 @@ This records the bounded source-ingestion work for demographic packages 1 and 2 of [issue #176](https://github.com/PolicyEngine/chronicle/issues/176), checked on -30 August 2026 UTC. It does not mark the demographic target surface complete. +30 August 2026 UTC, with G8-d2 territorial-authority evidence added on +13 October 2026 UTC. It does not mark the demographic target surface complete. ## Pinned publisher evidence @@ -10,6 +11,7 @@ of [issue #176](https://github.com/PolicyEngine/chronicle/issues/176), checked o |---|---|---|---:| | Subnational population workbook | At 30 June 2025, published 29 October 2025 | `001e8a896cfb50f5ed17836dc815b235e3bcca55ee91c9869a2afaeb054b50a6` | 97,990 | | Regional council code/name attributes | Boundaries at 1 January 2025 | `e5d53fa6abf742121d123bff970c77658a16e8c9b405b83d1a978cc22920d660` | 3,381 | +| Territorial authority code/name/ASCII-name attributes | Boundaries at 1 January 2025; 68 features, fetched 13 October 2026 | `db033184db1f64bd4c53b0cf070cac3303815f20c9cc428e11389b91033dc225` | 14,315 | | National population release landing page only | At 30 June 2025, published 19 August 2025 | `b2d2751800857895b1498d87df6f03ac3511ec8c7a9b17cc3150c8ab56533053` | 61,286 | All artifacts came directly from Stats NZ's website or Stats NZ's official @@ -18,17 +20,54 @@ retrieval timestamps, sizes, and immutable country-scoped R2 keys: - [Subnational workbook manifest](../db/data/stats_nz/subnational_population_estimates_2025/manifest.yaml). - [REGC 2025 classification manifest](../db/data/stats_nz/regional_council_2025_codes/manifest.yaml). +- [TA 2025 classification manifest](../db/data/stats_nz/territorial_authority_2025_codes/manifest.yaml). - [National landing-page evidence manifest](../db/data/stats_nz/national_population_estimates_2025/manifest.yaml). The objects use `r2://ledger-raw/raw/nz/stats_nz//2025//`. -After upload, all three remote objects were downloaded through the authorized +After upload, the original three remote objects were downloaded through the authorized Chronicle/Wrangler path and their byte counts and SHA-256 hashes matched the local publisher artifacts. +The NZ hub published the TA 2025 classification to its declared R2 key on +13 October 2026 and verified it by streaming the remote object back: its +SHA-256 matched the manifest. The workbook and its existing manifest are +unchanged. The two population references share a 30 June 2025 reference date, not a publication date. The national landing page is evidence of the missing download surface, **not** a single-year-age data artifact or a national source package. -The classification is geography evidence only and emits no population facts. +Both classifications are geography evidence only and emit no population facts. + +## G8-d2: territorial-authority population + +The [territorial-authority package](../packages/stats_nz/subnational_population_ta_2025/README.md) +reads the same immutable workbook's Table 2 and admits exactly 67 cells from +column E, headed `2025 P`. These are published, rounded all-age estimated +resident populations at 30 June 2025, recorded with equal coverage dates +`2025-06-30`, `provenance_class: census`, and `assertion: observation`. +The period label and concept evidence retain their provisional status. + +Every workbook row is matched by name to the publisher's +`TA2025_V1_00_NAME_ASCII` field after removing workbook macrons and case-folding; +row order alone does not assign a code. Each three-digit code appears once as +`nz-ta-NNN`, with `geography_level: local_authority` and +`geography_vintage: ta_2025`, matching the MBIE tenancy-bond rent convention. +`geography_name` preserves `TA2025_V1_00_NAME`; the value label preserves the +workbook's row text, including its macrons. Classification code `999`, Area +Outside Territorial Authority, has no workbook row and emits no fact. + +Guards retain every selected row header, `2025 P`, Table 2!A3:A5, the boundary +footnote at A100, and `P provisional` at A105. The worksheet is preserved over +its complete used range, including its 21 Auckland local-board rows, 2023 and +2024 P columns, change and percent columns, and New Zealand row. Those cells +emit no facts in this package. Local boards need a geography level absent from +the current Chronicle contract; G8-d2b is the hub's separate follow-up. + +Table 2!E96 publishes 5,324,700 for New Zealand. The existing package already +reads that national figure from Table 1!D25, so it is not emitted again. +Footnote 3 includes inlets, islands, ships and oil rigs outside territorial +authority areas; published rounding also qualifies comparisons with the +territorial-authority figures. No total, Auckland local-board sum, residual, +share, aging, imputation, or reconciliation is computed or emitted. ## S7: partial workbook package diff --git a/docs/pe-nz-source-checklist.md b/docs/pe-nz-source-checklist.md index bf62d171..4b9f13fe 100644 --- a/docs/pe-nz-source-checklist.md +++ b/docs/pe-nz-source-checklist.md @@ -54,6 +54,7 @@ tests, and record a verified `raw/nz/...` R2 URI. | Package from #176 | Artifact pinned | Package valid | `raw/nz` verified | Notes | |---|---:|---:|---:|---| | `stats_nz/subnational_population_estimates_2025` | [x] | [ ] | [x] | **Partial:** 85 validated workbook counts (16 regions + published NZ total; all ages/four broad age bands). Five-year-age × sex coverage is missing; its ADE structure request returned HTTP 401. Package-completion box stays unchecked. [Evidence and access notes](pe-nz-population-source-notes.md). | +| `stats_nz/subnational_population_ta_2025` | [x] | [x] | [x] | G8-d2: 67 provisional all-age estimated resident populations at 30 June 2025, one Table 2!E cell per territorial authority; `local_authority`, `nz-ta-NNN`, `ta_2025`. Same unchanged workbook; codes and names come from the pinned 68-feature Stats NZ TA 2025 classification. The hub published the classification to its `raw/nz/stats_nz/...` key and verified it by SHA-256 on 2026-10-13. Local boards, previous years, change/percent columns and the duplicate NZ row are preserved but not emitted; no sums or reconciliation. [Scope](../packages/stats_nz/subnational_population_ta_2025/README.md) and [evidence](pe-nz-population-source-notes.md). | | `stats_nz/national_population_estimates_2025` | [ ] | [ ] | [ ] | **Blocked:** no original-vintage single-year-age artifact acquired. The 19 August 2025 release points to Infoshare, not an XLSX. Landing-page evidence only is hash-pinned in `raw/nz`; it emits zero facts. [Evidence and access notes](pe-nz-population-source-notes.md). | | `stats_nz/census_2023_households_by_region` | [x] | [ ] | [x] | **Partial:** B5: 216 published 2023 Census counts, 12 per place (NZ, 16 regional councils and separate Area Outside Region): households, three tenure categories and eight usual-resident household sizes. Requested household composition and social-renting split are absent from this product. #176 named the S2 ArcGIS hub regional-council layer as the source; the published ArcGIS household and family layers are SA1/SA2 only, so regional household composition needs the Aotearoa Data Explorer (`CEN23_FHH_038`, API key or manual export) and stays open. Package-completion box stays unchecked until then. National root plus per-place members share the 18 unchanged receipt-pinned CSVs with the family package; the hub published the 18 objects to their `raw/nz/stats_nz/...` keys and verified each by SHA-256 on 2026-10-10. [Scope and evidence](../packages/stats_nz/census_2023_households_by_region/README.md). | | `stats_nz/census_2023_family_type` | [x] | [x] | [x] | B5: 72 published 2023 Census counts, four per place for the same 18 places: families and the three exact family-type labels. Two requested package IDs retained as family roots with per-place members; shared artifacts committed once. Other years, percentages and Māori sections preserved but not emitted; the hub published the 18 objects to their `raw/nz/stats_nz/...` keys and verified each by SHA-256 on 2026-10-10. [Scope](../packages/stats_nz/census_2023_family_type/README.md) and [evidence](../packages/stats_nz/census_2023_households_by_region/evidence.md). | diff --git a/packages/stats_nz/subnational_population_estimates_2025/README.md b/packages/stats_nz/subnational_population_estimates_2025/README.md index bc0cec1c..825c54b6 100644 --- a/packages/stats_nz/subnational_population_estimates_2025/README.md +++ b/packages/stats_nz/subnational_population_estimates_2025/README.md @@ -16,9 +16,11 @@ and its unchanged XLSX provide these cells: Table 3's columns are 0–14, 15–39, 40–64, and 65+. All selected counts include all sexes; no sex-specific counts appear in these worksheets. Both source worksheets are parsed over their complete used ranges (2,265 cells), including -unselected years and explanatory footnotes. The package does not ingest -territorial authority/local board tables, island subtotals, percentages, median -ages, or population-change components. +unselected years and explanatory footnotes. Territorial-authority all-age +counts from Table 2 are ingested separately by +[subnational_population_ta_2025](../subnational_population_ta_2025/README.md). +Local-board tables, island subtotals, percentages, median ages, and +population-change components are not ingested by this package. ## Reference period and population definition @@ -66,7 +68,7 @@ warns that the two packages word that value differently. ## Artifacts and remaining access gaps See [NZ population source notes](../../../docs/pe-nz-population-source-notes.md) -for the three pinned checksums, raw storage, the detailed S7 API access failure, +for the pinned checksums, raw storage, the detailed S7 API access failure, and S8's missing single-year-age artifact. The wave-1 checklist deliberately keeps the package-completion checkbox unchecked. diff --git a/packages/stats_nz/subnational_population_ta_2025/README.md b/packages/stats_nz/subnational_population_ta_2025/README.md new file mode 100644 index 00000000..f8ac5874 --- /dev/null +++ b/packages/stats_nz/subnational_population_ta_2025/README.md @@ -0,0 +1,90 @@ +# Stats NZ territorial-authority population, 30 June 2025 + +This package emits **67 publisher-backed estimated resident population (ERP) +counts**, one for each territorial authority in Table 2 of the unchanged +[29 October 2025 subnational population release workbook](https://www.stats.govt.nz/information-releases/subnational-population-estimates-at-30-june-2025/). +Each fact reads one column-E source cell, headed `2025 P`, with scale 1. +No population is computed from other cells. + +| Source selection | Geography | Facts | +|---|---|---:| +| Table 2!E8:E11 and E33:E95, where column A names the territorial authority | TA 2025 | 67 | + +The complete Table 2 used range is preserved as source cells, including +unselected values and footnotes. The workbook and its existing manifest are +shared with [the regional population package](../subnational_population_estimates_2025/README.md) +and are unchanged. + +## Reference period and source semantics + +- Reference date: **30 June 2025**, represented as `calendar_year: 2025` with + equal start/end coverage dates `2025-06-30`. These are point-in-time counts, + rather than annual averages. +- Publication vintage: **29 October 2025**. All admitted figures are + **provisional**, retained in the source period label and concept evidence. + The pinned source remains this vintage if the publisher later revises it. +- Population: all ages and all sexes, estimated resident population using the + 2023 census base, including adjustments for census under/overcount, + residents temporarily overseas on census night, and subsequent births, + deaths and net migration. No age-band or sex-specific fact is emitted. +- Provenance: `provenance_class: census` and `assertion: observation`, as in + the regional package. A provisional historical ERP is still a publisher + observation; these fields do not assert that its value is final. +- Concept: `stats_nz.estimated_resident_population`, with `source_label` + relation and Stats NZ source evidence. No canonical equivalence or + downstream population calibration is claimed. +- Geography boundaries: **1 January 2025**, recorded as + `geography_level: local_authority`, `geography_id: nz-ta-NNN` with a + three-digit TA code, and `geography_vintage: ta_2025`. This is the geography + convention used by the MBIE tenancy-bond rent facts. + +The separately pinned [Stats NZ TA 2025 classification](../../../db/data/stats_nz/territorial_authority_2025_codes/manifest.yaml) +contains 68 code/name/ASCII-name entries. Workbook headers match its +`TA2025_V1_00_NAME_ASCII` values after removing macrons and case-folding; +row order never assigns a code. Every admitted code is used once. +Code `999`, Area Outside Territorial Authority, has no workbook row and is +excluded. Each `geography_name` is the publisher's `TA2025_V1_00_NAME`, while +the value `label` is the exact workbook row text. For example, `nz-ta-002` +has geography name `Whangarei District` and value label `Whangārei district`. +The `geography` dimension label is Chronicle's existing `Geography` wording. +The classification names `nz-ta-018` and `nz-ta-027` “Ōtorohanga District” +and “Ōpōtiki District”; MBIE's package uses “Otorohanga District” and +“Opotiki District”. The merged bundle therefore adds two geography-name +warnings while preserving each publisher's spelling. + +Each selected row has an exact column-A row-header guard and a `2025 P` +column-header guard. Guards also cover A3 `Territorial authority and Auckland +local board areas`, A4 `At 30 June 2023–2025`, A5 `Territorial authority and +Auckland local board area(2)`, A100 `2. Boundaries at 1 January 2025.`, and +A105 `P provisional`. Macron differences are not normalized in these guards. + +## Preserved but not emitted + +- The **21 Auckland local-board rows** at 12–32, named in column B with blank + column A. Local boards require a geography level the current Chronicle + contract does not provide; the NZ hub owns a separate G8-d2b follow-up. +- The **2023 and 2024 P columns**, and the **change and percent columns**. +- The **New Zealand(3) row** at 96. Table 2!E96 publishes 5,324,700; the + existing regional package already emits that number from Table 1!D25. + +Footnote 3 explains that the national population includes inlets, islands, +ships and oil rigs outside territorial authority areas. Values remain as the +publisher rounded them. Neither an authority sum nor an Auckland local-board +sum is emitted; published authority counts are not reconciled with the national +row. No residual, share, aging, interpolation, imputation, target activation, +or computed population enters Chronicle. + +## Artifacts and bounded validation + +[NZ population source notes](../../../docs/pe-nz-population-source-notes.md) +record the workbook and classification checksums and sizes. The workbook is +already published and verified. The NZ hub published the TA classification to +its declared `raw/nz/stats_nz/...` key on 13 October 2026 and verified the +remote object by SHA-256 against the manifest. + +The package-specific tests build an independent expected map from the workbook +and classification, compare every admitted cell, check ten exact sample cells, +and exclude local boards, code `999`, and the national row. The existing +regional package retains its 85 facts. Source-package and consumer-contract +validation plus a one-source build are required; independent source-fidelity +and boundary judges must review the candidate before landing. diff --git a/packages/stats_nz/subnational_population_ta_2025/source_package.yaml b/packages/stats_nz/subnational_population_ta_2025/source_package.yaml new file mode 100644 index 00000000..2f0d1695 --- /dev/null +++ b/packages/stats_nz/subnational_population_ta_2025/source_package.yaml @@ -0,0 +1,763 @@ +# 67 published Table 2!E cells only; never local boards or the national row. +# Row-to-code mapping uses the pinned TA2025 NAME_ASCII field after macron +# removal and case-folding of workbook row text, never workbook row order. +# Geography names preserve TA2025 NAME; labels/guards preserve workbook text. +# Shared guard list is reused by every row. Source artifact/manifest unchanged. +schema_version: ledger.source_package.v1 +package_id: stats-nz-subnational-population-ta-2025 +label: Stats NZ 30 June 2025 provisional estimated resident population by territorial authority +dimension_labels: + geography: Geography +artifact: + source_name: stats_nz + source_table: 'Subnational population estimates: At 30 June 2025 (provisional), Table 2' + resource_package: db + resource_directory: data/stats_nz/subnational_population_estimates_2025 + manifest: manifest.yaml + vintage: stats_nz_subnational_population_2025_10_29 + extracted_at: '2026-10-13' + extraction_method: xlsx used-range cell parse restricted to Table 2 + parser: xlsx_used_range + sheets: + - Table 2 + artifact_year: 2025 +record_sets: + - record_set_id: stats_nz.cy2025.subnational_population_ta.all_ages + record_set_spec_id: stats_nz.subnational_population_ta.all_ages.v1 + source_record_id_prefix: stats_nz.cy2025.subnational_population_ta.all_ages + provenance_class: census + assertion: observation + sheet_name: Table 2 + period_type: calendar_year + period: 2025 + period_coverage: + start_date: '2025-06-30' + end_date: '2025-06-30' + basis: calendar + source_period_label: At 30 June 2025 (provisional) + notes: Point-in-time estimated resident population, not an annual average; published + 29 October 2025 using the 2023 census base. + geography_id: nz-ta-001 + geography_level: local_authority + geography_name: Far North District + geography_vintage: ta_2025 + entity: person + entity_role: resident_population + domain: resident_population + groupby_dimension: geography + measures: + - measure_id: population + label: Estimated resident population + ordinal: 0 + column: E + source_column_id: estimated_resident_population + concept: stats_nz.estimated_resident_population + source_concept: stats_nz.estimated_resident_population + concept_relation: source_label + concept_authority: stats_nz + concept_evidence_url: https://www.stats.govt.nz/information-releases/subnational-population-estimates-at-30-june-2025/ + concept_evidence_notes: 'Table 2 publishes provisional estimated resident population + counts at 30 June 2025, using 1 January 2025 boundaries and the 2023 census base. + ERP includes adjustments for net census under/overcount, residents temporarily overseas + on census night, and subsequent births, deaths, and net migration. These counts + include all ages and all sexes. Each territorial-authority value is one published + rounded cell, with no local-board sums or reconciliation to the separately published + New Zealand total. The pinned Stats NZ TA 2025 classification supplies codes and + geography names; the workbook supplies value labels. Source-only facts: no uprating, + interpolation, target activation, or calibration.' + unit: count + aggregation: sum + expected_cell_type: number + expected_column_header_row: 6 + expected_column_header: 2025 P + rows: + - value_id: nz-ta-001 + label: Far North district + ordinal: 0 + row_number: 8 + expected_row_header_column: A + expected_row_header: Far North district + geography_id: nz-ta-001 + geography_name: Far North District + table_record_kind: total + guard_cells: &id001 + - column: A + row: 3 + expected_value: Territorial authority and Auckland local board areas + label: source table scope + - column: A + row: 4 + expected_value: At 30 June 2023–2025 + label: point-in-time release period + - column: A + row: 5 + expected_value: Territorial authority and Auckland local board area(2) + label: source geography heading + - column: A + row: 100 + expected_value: 2. Boundaries at 1 January 2025. + label: territorial authority boundary vintage + - column: A + row: 105 + expected_value: P provisional + label: publication status + - value_id: nz-ta-002 + label: Whangārei district + ordinal: 1 + row_number: 9 + expected_row_header_column: A + expected_row_header: Whangārei district + geography_id: nz-ta-002 + geography_name: Whangarei District + table_record_kind: total + guard_cells: *id001 + - value_id: nz-ta-003 + label: Kaipara district + ordinal: 2 + row_number: 10 + expected_row_header_column: A + expected_row_header: Kaipara district + geography_id: nz-ta-003 + geography_name: Kaipara District + table_record_kind: total + guard_cells: *id001 + - value_id: nz-ta-076 + label: Auckland + ordinal: 3 + row_number: 11 + expected_row_header_column: A + expected_row_header: Auckland + geography_id: nz-ta-076 + geography_name: Auckland + table_record_kind: total + guard_cells: *id001 + - value_id: nz-ta-011 + label: Thames-Coromandel district + ordinal: 4 + row_number: 33 + expected_row_header_column: A + expected_row_header: Thames-Coromandel district + geography_id: nz-ta-011 + geography_name: Thames-Coromandel District + table_record_kind: total + guard_cells: *id001 + - value_id: nz-ta-012 + label: Hauraki district + ordinal: 5 + row_number: 34 + expected_row_header_column: A + expected_row_header: Hauraki district + geography_id: nz-ta-012 + geography_name: Hauraki District + table_record_kind: total + guard_cells: *id001 + - value_id: nz-ta-013 + label: Waikato district + ordinal: 6 + row_number: 35 + expected_row_header_column: A + expected_row_header: Waikato district + geography_id: nz-ta-013 + geography_name: Waikato District + table_record_kind: total + guard_cells: *id001 + - value_id: nz-ta-015 + label: Matamata-Piako district + ordinal: 7 + row_number: 36 + expected_row_header_column: A + expected_row_header: Matamata-Piako district + geography_id: nz-ta-015 + geography_name: Matamata-Piako District + table_record_kind: total + guard_cells: *id001 + - value_id: nz-ta-016 + label: Hamilton city + ordinal: 8 + row_number: 37 + expected_row_header_column: A + expected_row_header: Hamilton city + geography_id: nz-ta-016 + geography_name: Hamilton City + table_record_kind: total + guard_cells: *id001 + - value_id: nz-ta-017 + label: Waipa district + ordinal: 9 + row_number: 38 + expected_row_header_column: A + expected_row_header: Waipa district + geography_id: nz-ta-017 + geography_name: Waipa District + table_record_kind: total + guard_cells: *id001 + - value_id: nz-ta-018 + label: Ōtorohanga district + ordinal: 10 + row_number: 39 + expected_row_header_column: A + expected_row_header: Ōtorohanga district + geography_id: nz-ta-018 + geography_name: Ōtorohanga District + table_record_kind: total + guard_cells: *id001 + - value_id: nz-ta-019 + label: South Waikato district + ordinal: 11 + row_number: 40 + expected_row_header_column: A + expected_row_header: South Waikato district + geography_id: nz-ta-019 + geography_name: South Waikato District + table_record_kind: total + guard_cells: *id001 + - value_id: nz-ta-020 + label: Waitomo district + ordinal: 12 + row_number: 41 + expected_row_header_column: A + expected_row_header: Waitomo district + geography_id: nz-ta-020 + geography_name: Waitomo District + table_record_kind: total + guard_cells: *id001 + - value_id: nz-ta-021 + label: Taupō district + ordinal: 13 + row_number: 42 + expected_row_header_column: A + expected_row_header: Taupō district + geography_id: nz-ta-021 + geography_name: Taupo District + table_record_kind: total + guard_cells: *id001 + - value_id: nz-ta-022 + label: Western Bay of Plenty district + ordinal: 14 + row_number: 43 + expected_row_header_column: A + expected_row_header: Western Bay of Plenty district + geography_id: nz-ta-022 + geography_name: Western Bay of Plenty District + table_record_kind: total + guard_cells: *id001 + - value_id: nz-ta-023 + label: Tauranga city + ordinal: 15 + row_number: 44 + expected_row_header_column: A + expected_row_header: Tauranga city + geography_id: nz-ta-023 + geography_name: Tauranga City + table_record_kind: total + guard_cells: *id001 + - value_id: nz-ta-024 + label: Rotorua district + ordinal: 16 + row_number: 45 + expected_row_header_column: A + expected_row_header: Rotorua district + geography_id: nz-ta-024 + geography_name: Rotorua District + table_record_kind: total + guard_cells: *id001 + - value_id: nz-ta-025 + label: Whakatāne district + ordinal: 17 + row_number: 46 + expected_row_header_column: A + expected_row_header: Whakatāne district + geography_id: nz-ta-025 + geography_name: Whakatane District + table_record_kind: total + guard_cells: *id001 + - value_id: nz-ta-026 + label: Kawerau district + ordinal: 18 + row_number: 47 + expected_row_header_column: A + expected_row_header: Kawerau district + geography_id: nz-ta-026 + geography_name: Kawerau District + table_record_kind: total + guard_cells: *id001 + - value_id: nz-ta-027 + label: Ōpōtiki district + ordinal: 19 + row_number: 48 + expected_row_header_column: A + expected_row_header: Ōpōtiki district + geography_id: nz-ta-027 + geography_name: Ōpōtiki District + table_record_kind: total + guard_cells: *id001 + - value_id: nz-ta-028 + label: Gisborne district + ordinal: 20 + row_number: 49 + expected_row_header_column: A + expected_row_header: Gisborne district + geography_id: nz-ta-028 + geography_name: Gisborne District + table_record_kind: total + guard_cells: *id001 + - value_id: nz-ta-029 + label: Wairoa district + ordinal: 21 + row_number: 50 + expected_row_header_column: A + expected_row_header: Wairoa district + geography_id: nz-ta-029 + geography_name: Wairoa District + table_record_kind: total + guard_cells: *id001 + - value_id: nz-ta-030 + label: Hastings district + ordinal: 22 + row_number: 51 + expected_row_header_column: A + expected_row_header: Hastings district + geography_id: nz-ta-030 + geography_name: Hastings District + table_record_kind: total + guard_cells: *id001 + - value_id: nz-ta-031 + label: Napier city + ordinal: 23 + row_number: 52 + expected_row_header_column: A + expected_row_header: Napier city + geography_id: nz-ta-031 + geography_name: Napier City + table_record_kind: total + guard_cells: *id001 + - value_id: nz-ta-032 + label: Central Hawke's Bay district + ordinal: 24 + row_number: 53 + expected_row_header_column: A + expected_row_header: Central Hawke's Bay district + geography_id: nz-ta-032 + geography_name: Central Hawke's Bay District + table_record_kind: total + guard_cells: *id001 + - value_id: nz-ta-033 + label: New Plymouth district + ordinal: 25 + row_number: 54 + expected_row_header_column: A + expected_row_header: New Plymouth district + geography_id: nz-ta-033 + geography_name: New Plymouth District + table_record_kind: total + guard_cells: *id001 + - value_id: nz-ta-034 + label: Stratford district + ordinal: 26 + row_number: 55 + expected_row_header_column: A + expected_row_header: Stratford district + geography_id: nz-ta-034 + geography_name: Stratford District + table_record_kind: total + guard_cells: *id001 + - value_id: nz-ta-035 + label: South Taranaki district + ordinal: 27 + row_number: 56 + expected_row_header_column: A + expected_row_header: South Taranaki district + geography_id: nz-ta-035 + geography_name: South Taranaki District + table_record_kind: total + guard_cells: *id001 + - value_id: nz-ta-036 + label: Ruapehu district + ordinal: 28 + row_number: 57 + expected_row_header_column: A + expected_row_header: Ruapehu district + geography_id: nz-ta-036 + geography_name: Ruapehu District + table_record_kind: total + guard_cells: *id001 + - value_id: nz-ta-037 + label: Whanganui district + ordinal: 29 + row_number: 58 + expected_row_header_column: A + expected_row_header: Whanganui district + geography_id: nz-ta-037 + geography_name: Whanganui District + table_record_kind: total + guard_cells: *id001 + - value_id: nz-ta-038 + label: Rangitikei district + ordinal: 30 + row_number: 59 + expected_row_header_column: A + expected_row_header: Rangitikei district + geography_id: nz-ta-038 + geography_name: Rangitikei District + table_record_kind: total + guard_cells: *id001 + - value_id: nz-ta-039 + label: Manawatū district + ordinal: 31 + row_number: 60 + expected_row_header_column: A + expected_row_header: Manawatū district + geography_id: nz-ta-039 + geography_name: Manawatu District + table_record_kind: total + guard_cells: *id001 + - value_id: nz-ta-040 + label: Palmerston North city + ordinal: 32 + row_number: 61 + expected_row_header_column: A + expected_row_header: Palmerston North city + geography_id: nz-ta-040 + geography_name: Palmerston North City + table_record_kind: total + guard_cells: *id001 + - value_id: nz-ta-041 + label: Tararua district + ordinal: 33 + row_number: 62 + expected_row_header_column: A + expected_row_header: Tararua district + geography_id: nz-ta-041 + geography_name: Tararua District + table_record_kind: total + guard_cells: *id001 + - value_id: nz-ta-042 + label: Horowhenua district + ordinal: 34 + row_number: 63 + expected_row_header_column: A + expected_row_header: Horowhenua district + geography_id: nz-ta-042 + geography_name: Horowhenua District + table_record_kind: total + guard_cells: *id001 + - value_id: nz-ta-043 + label: Kapiti Coast district + ordinal: 35 + row_number: 64 + expected_row_header_column: A + expected_row_header: Kapiti Coast district + geography_id: nz-ta-043 + geography_name: Kapiti Coast District + table_record_kind: total + guard_cells: *id001 + - value_id: nz-ta-044 + label: Porirua city + ordinal: 36 + row_number: 65 + expected_row_header_column: A + expected_row_header: Porirua city + geography_id: nz-ta-044 + geography_name: Porirua City + table_record_kind: total + guard_cells: *id001 + - value_id: nz-ta-045 + label: Upper Hutt city + ordinal: 37 + row_number: 66 + expected_row_header_column: A + expected_row_header: Upper Hutt city + geography_id: nz-ta-045 + geography_name: Upper Hutt City + table_record_kind: total + guard_cells: *id001 + - value_id: nz-ta-046 + label: Lower Hutt city + ordinal: 38 + row_number: 67 + expected_row_header_column: A + expected_row_header: Lower Hutt city + geography_id: nz-ta-046 + geography_name: Lower Hutt City + table_record_kind: total + guard_cells: *id001 + - value_id: nz-ta-047 + label: Wellington city + ordinal: 39 + row_number: 68 + expected_row_header_column: A + expected_row_header: Wellington city + geography_id: nz-ta-047 + geography_name: Wellington City + table_record_kind: total + guard_cells: *id001 + - value_id: nz-ta-048 + label: Masterton district + ordinal: 40 + row_number: 69 + expected_row_header_column: A + expected_row_header: Masterton district + geography_id: nz-ta-048 + geography_name: Masterton District + table_record_kind: total + guard_cells: *id001 + - value_id: nz-ta-049 + label: Carterton district + ordinal: 41 + row_number: 70 + expected_row_header_column: A + expected_row_header: Carterton district + geography_id: nz-ta-049 + geography_name: Carterton District + table_record_kind: total + guard_cells: *id001 + - value_id: nz-ta-050 + label: South Wairarapa district + ordinal: 42 + row_number: 71 + expected_row_header_column: A + expected_row_header: South Wairarapa district + geography_id: nz-ta-050 + geography_name: South Wairarapa District + table_record_kind: total + guard_cells: *id001 + - value_id: nz-ta-051 + label: Tasman district + ordinal: 43 + row_number: 72 + expected_row_header_column: A + expected_row_header: Tasman district + geography_id: nz-ta-051 + geography_name: Tasman District + table_record_kind: total + guard_cells: *id001 + - value_id: nz-ta-052 + label: Nelson city + ordinal: 44 + row_number: 73 + expected_row_header_column: A + expected_row_header: Nelson city + geography_id: nz-ta-052 + geography_name: Nelson City + table_record_kind: total + guard_cells: *id001 + - value_id: nz-ta-053 + label: Marlborough district + ordinal: 45 + row_number: 74 + expected_row_header_column: A + expected_row_header: Marlborough district + geography_id: nz-ta-053 + geography_name: Marlborough District + table_record_kind: total + guard_cells: *id001 + - value_id: nz-ta-054 + label: Kaikoura district + ordinal: 46 + row_number: 75 + expected_row_header_column: A + expected_row_header: Kaikoura district + geography_id: nz-ta-054 + geography_name: Kaikoura District + table_record_kind: total + guard_cells: *id001 + - value_id: nz-ta-055 + label: Buller district + ordinal: 47 + row_number: 76 + expected_row_header_column: A + expected_row_header: Buller district + geography_id: nz-ta-055 + geography_name: Buller District + table_record_kind: total + guard_cells: *id001 + - value_id: nz-ta-056 + label: Grey district + ordinal: 48 + row_number: 77 + expected_row_header_column: A + expected_row_header: Grey district + geography_id: nz-ta-056 + geography_name: Grey District + table_record_kind: total + guard_cells: *id001 + - value_id: nz-ta-057 + label: Westland district + ordinal: 49 + row_number: 78 + expected_row_header_column: A + expected_row_header: Westland district + geography_id: nz-ta-057 + geography_name: Westland District + table_record_kind: total + guard_cells: *id001 + - value_id: nz-ta-058 + label: Hurunui district + ordinal: 50 + row_number: 79 + expected_row_header_column: A + expected_row_header: Hurunui district + geography_id: nz-ta-058 + geography_name: Hurunui District + table_record_kind: total + guard_cells: *id001 + - value_id: nz-ta-059 + label: Waimakariri district + ordinal: 51 + row_number: 80 + expected_row_header_column: A + expected_row_header: Waimakariri district + geography_id: nz-ta-059 + geography_name: Waimakariri District + table_record_kind: total + guard_cells: *id001 + - value_id: nz-ta-060 + label: Christchurch city + ordinal: 52 + row_number: 81 + expected_row_header_column: A + expected_row_header: Christchurch city + geography_id: nz-ta-060 + geography_name: Christchurch City + table_record_kind: total + guard_cells: *id001 + - value_id: nz-ta-062 + label: Selwyn district + ordinal: 53 + row_number: 82 + expected_row_header_column: A + expected_row_header: Selwyn district + geography_id: nz-ta-062 + geography_name: Selwyn District + table_record_kind: total + guard_cells: *id001 + - value_id: nz-ta-063 + label: Ashburton district + ordinal: 54 + row_number: 83 + expected_row_header_column: A + expected_row_header: Ashburton district + geography_id: nz-ta-063 + geography_name: Ashburton District + table_record_kind: total + guard_cells: *id001 + - value_id: nz-ta-064 + label: Timaru district + ordinal: 55 + row_number: 84 + expected_row_header_column: A + expected_row_header: Timaru district + geography_id: nz-ta-064 + geography_name: Timaru District + table_record_kind: total + guard_cells: *id001 + - value_id: nz-ta-065 + label: Mackenzie district + ordinal: 56 + row_number: 85 + expected_row_header_column: A + expected_row_header: Mackenzie district + geography_id: nz-ta-065 + geography_name: Mackenzie District + table_record_kind: total + guard_cells: *id001 + - value_id: nz-ta-066 + label: Waimate district + ordinal: 57 + row_number: 86 + expected_row_header_column: A + expected_row_header: Waimate district + geography_id: nz-ta-066 + geography_name: Waimate District + table_record_kind: total + guard_cells: *id001 + - value_id: nz-ta-067 + label: Chatham Islands territory + ordinal: 58 + row_number: 87 + expected_row_header_column: A + expected_row_header: Chatham Islands territory + geography_id: nz-ta-067 + geography_name: Chatham Islands Territory + table_record_kind: total + guard_cells: *id001 + - value_id: nz-ta-068 + label: Waitaki district + ordinal: 59 + row_number: 88 + expected_row_header_column: A + expected_row_header: Waitaki district + geography_id: nz-ta-068 + geography_name: Waitaki District + table_record_kind: total + guard_cells: *id001 + - value_id: nz-ta-069 + label: Central Otago district + ordinal: 60 + row_number: 89 + expected_row_header_column: A + expected_row_header: Central Otago district + geography_id: nz-ta-069 + geography_name: Central Otago District + table_record_kind: total + guard_cells: *id001 + - value_id: nz-ta-070 + label: Queenstown-Lakes district + ordinal: 61 + row_number: 90 + expected_row_header_column: A + expected_row_header: Queenstown-Lakes district + geography_id: nz-ta-070 + geography_name: Queenstown-Lakes District + table_record_kind: total + guard_cells: *id001 + - value_id: nz-ta-071 + label: Dunedin city + ordinal: 62 + row_number: 91 + expected_row_header_column: A + expected_row_header: Dunedin city + geography_id: nz-ta-071 + geography_name: Dunedin City + table_record_kind: total + guard_cells: *id001 + - value_id: nz-ta-072 + label: Clutha district + ordinal: 63 + row_number: 92 + expected_row_header_column: A + expected_row_header: Clutha district + geography_id: nz-ta-072 + geography_name: Clutha District + table_record_kind: total + guard_cells: *id001 + - value_id: nz-ta-073 + label: Southland district + ordinal: 64 + row_number: 93 + expected_row_header_column: A + expected_row_header: Southland district + geography_id: nz-ta-073 + geography_name: Southland District + table_record_kind: total + guard_cells: *id001 + - value_id: nz-ta-074 + label: Gore district + ordinal: 65 + row_number: 94 + expected_row_header_column: A + expected_row_header: Gore district + geography_id: nz-ta-074 + geography_name: Gore District + table_record_kind: total + guard_cells: *id001 + - value_id: nz-ta-075 + label: Invercargill city + ordinal: 66 + row_number: 95 + expected_row_header_column: A + expected_row_header: Invercargill city + geography_id: nz-ta-075 + geography_name: Invercargill City + table_record_kind: total + guard_cells: *id001 diff --git a/tests/test_chronicle_bundle.py b/tests/test_chronicle_bundle.py index 0e413217..d1403cdd 100644 --- a/tests/test_chronicle_bundle.py +++ b/tests/test_chronicle_bundle.py @@ -383,6 +383,12 @@ def test_build_bundle_writes_merged_consumer_contract(tmp_path): # year 2023: no new geography, period or entity keys, duplicate keys or # warnings. Its SALT and PTC column fixes, its line 8a relabel and the # Historic Table 2 QBI column fix change no count. + # G8-d2: one-source build at year 2023 measured 67 CY2025 person facts, + # one package/table and 67 TA geography keys; only nz-ta-067 is new to + # the default bundle. Publisher, period and entity keys already exist. + # Standalone warnings/duplicates are zero. Comparing the declared names + # with MBIE through the bundle warning function adds two geography-name + # warnings (nz-ta-018 and nz-ta-027); both publisher spellings are kept. assert summary["counts"] == { "aggregate_duplicate_key_count": 0, "entity_count": 12, @@ -391,8 +397,8 @@ def test_build_bundle_writes_merged_consumer_contract(tmp_path): # Inner/Outer London also add two region-level geography keys. # chronicle#343 adds 63 OBR surcharge-costing projections and 22,940 ONS # constituency sales and median prices, at existing geographies. - "fact_count": 1030137, - "geography_count": 20548, + "fact_count": 1030204, + "geography_count": 20549, "period_count": 511, # 467 before chronicle#292 moved the congressional-district and # state_2022 rows from their ty2023 restamp to TY2022. There the CD @@ -414,7 +420,7 @@ def test_build_bundle_writes_merged_consumer_contract(tmp_path): "semantic_duplicate_key_count": 2324, "skipped_source_count": 10, "source_count": 56, - "source_package_count": 379, + "source_package_count": 380, # 1 semantic-duplicate warning, plus the publisher wording Chronicle # keeps as published: values two packages word differently, groupby # rows that drift inside one package (chronicle#265, #266), and the @@ -433,7 +439,7 @@ def test_build_bundle_writes_merged_consumer_contract(tmp_path): # predecessors, so there is no sex=all wording warning. # chronicle#343 adds one: ONS's 'Semi-detached houses' against DESNZ's # 'Semi detached' for property_type=semi_detached, each as published. - "warning_count": 118, + "warning_count": 120, } row_count = 0 provenance_classes = set() @@ -451,14 +457,14 @@ def test_build_bundle_writes_merged_consumer_contract(tmp_path): if row_count == 1: assert row["aggregate_fact_key"].startswith("ledger.aggregate_fact.v2:") assert row["semantic_fact_key"].startswith("ledger.semantic_fact.v2:") - assert row_count == 1030137 + assert row_count == 1030204 assert provenance_classes <= { "administrative", "census", "model_output", "survey_aggregate", } - assert source_packages["source_package_count"] == 379 + assert source_packages["source_package_count"] == 380 assert source_packages["skipped_source_count"] == 10 assert sorted(item["source"] for item in source_packages["skipped_sources"]) == [ "census-acs-s0101-congressional-district-age-2024", @@ -472,7 +478,7 @@ def test_build_bundle_writes_merged_consumer_contract(tmp_path): "jct-obbba-revenue-estimates-2025", "jct-tax-expenditures-2024", ] - assert coverage["fact_count"] == 1030137 + assert coverage["fact_count"] == 1030204 assert coverage["counts"]["by_source"] == { "bea": 446, "bfp_economic_outlook": 5, @@ -526,13 +532,13 @@ def test_build_bundle_writes_merged_consumer_contract(tmp_path): "statbel_fiscal_income_distribution": 14600, "statbel_population_structure": 36, "treasury": 139, - "stats_nz": 1948, + "stats_nz": 2015, "usda_snap": 852, "voa": 3001, "welshgov": 9567, } table_counts = coverage["counts"]["by_source_table"] - assert len(table_counts) == 321 + assert len(table_counts) == 322 assert ( table_counts["bea:NIPA Table 7.8. Supplements to Wages and Salaries by Type"] == 1 @@ -584,6 +590,13 @@ def test_build_bundle_writes_merged_consumer_contract(tmp_path): ] == 85 ) + assert ( + table_counts[ + "stats_nz:Subnational population estimates: At 30 June 2025 " + "(provisional), Table 2" + ] + == 67 + ) assert table_counts["ird:Taxable income distribution of individuals 2025"] == 3522 assert table_counts["ird:Wage and salary distributions for individuals 2026"] == 510 assert ( @@ -1661,6 +1674,8 @@ def test_build_bundle_writes_merged_consumer_contract(tmp_path): expected_period_counts[f"fiscal_year:{fiscal_year}"] += count # chronicle#211: Stats NZ's population estimates at 30 June 2025. expected_period_counts["calendar_year:2025"] += 85 + # G8-d2 reads Table 2 at the same point-in-time publisher period. + expected_period_counts["calendar_year:2025"] += 67 # chronicle#322: 16 areas x 10 vehicle/fuel columns annually, and the # ONS extension (166 columns minus seven existing series, plus three # annual tourism columns). @@ -1805,9 +1820,10 @@ def test_build_bundle_writes_merged_consumer_contract(tmp_path): coverage["counts"]["by_geography"][f"region:{regional_council:02d}"] == 30 ) assert coverage["counts"]["by_geography"]["region:99"] == 25 - assert coverage["counts"]["by_geography"]["local_authority:nz-ta-054"] == 56 - assert coverage["counts"]["by_geography"]["local_authority:nz-ta-057"] == 77 - assert coverage["counts"]["by_geography"]["local_authority:nz-ta-066"] == 70 + assert coverage["counts"]["by_geography"]["local_authority:nz-ta-054"] == 57 + assert coverage["counts"]["by_geography"]["local_authority:nz-ta-057"] == 78 + assert coverage["counts"]["by_geography"]["local_authority:nz-ta-066"] == 71 + assert coverage["counts"]["by_geography"]["local_authority:nz-ta-067"] == 1 assert ( coverage["counts"]["by_geography"]["statistical_scope:nz-mbie-location-na"] == 84 @@ -1830,7 +1846,7 @@ def test_build_bundle_writes_merged_consumer_contract(tmp_path): assert coverage["counts"]["by_geography"]["country:K03000001"] == 21401 assert coverage["counts"]["by_geography"]["country:K04000001"] == 1148 assert coverage["counts"]["by_geography"]["statistical_scope:ofgem:london"] == 216 - assert len(coverage["counts"]["by_geography"]) == 20548 + assert len(coverage["counts"]["by_geography"]) == 20549 for region in ( "auckland", "bay-of-plenty", @@ -1859,7 +1875,7 @@ def test_build_bundle_writes_merged_consumer_contract(tmp_path): "household": 155015, "institutional_sector": 4468, "pension_plan": 2, - "person": 496036, + "person": 496103, "return": 14600, "social_protection_scheme": 36, "tax_unit": 45268, @@ -1867,7 +1883,7 @@ def test_build_bundle_writes_merged_consumer_contract(tmp_path): assert not coverage["duplicates"]["aggregate_fact_keys"] assert len(coverage["duplicates"]["semantic_fact_keys"]) == 2324 assert Counter(warning["code"] for warning in summary["warnings"]) == { - "conflicting_geography_name_across_packages": 78, + "conflicting_geography_name_across_packages": 80, "conflicting_groupby_value_label": 23, "conflicting_value_label_across_packages": 16, "duplicate_semantic_fact_key": 1, @@ -1898,8 +1914,14 @@ def test_build_bundle_writes_merged_consumer_contract(tmp_path): ] assert Counter(warning["key"].split(":")[0] for warning in geography_names) == { "county": 77, + "local_authority": 2, "statistical_scope": 1, } + assert sorted( + warning["key"] + for warning in geography_names + if warning["key"].startswith("local_authority:") + ) == ["local_authority:nz-ta-018", "local_authority:nz-ta-027"] assert sorted( warning["key"] for warning in summary["warnings"] @@ -1934,6 +1956,7 @@ def test_build_bundle_writes_merged_consumer_contract(tmp_path): "0", "E", "S", + "n", } for source in ( "dfe-funded-early-education-childcare-2026", diff --git a/tests/test_chronicle_nz_subnational_population_ta.py b/tests/test_chronicle_nz_subnational_population_ta.py new file mode 100644 index 00000000..d6d46697 --- /dev/null +++ b/tests/test_chronicle_nz_subnational_population_ta.py @@ -0,0 +1,368 @@ +"""Independent publisher-cell gates for the 30 June 2025 TA population facts.""" + +from __future__ import annotations + +from dataclasses import replace +from functools import lru_cache +import hashlib +import json +from pathlib import Path +import unicodedata + +import openpyxl +import pytest +import yaml + +from chronicle.consumer_contract import ( + consumer_fact_rows, + validate_consumer_fact_contract, +) +from chronicle.core import validate_facts +from chronicle.source_package import ( + SOURCE_PACKAGE_ALIASES, + load_source_package, + validate_source_package, +) +from chronicle.sources.cells import build_source_cell_key, validate_source_cells +from chronicle.sources.specs import ( + build_cells_by_sheet_address, + resolve_source_record, +) + + +ALIAS = "stats-nz-subnational-population-ta-2025" +REPO_ROOT = Path(__file__).resolve().parents[1] +PACKAGE_PATH = REPO_ROOT / "packages/stats_nz/subnational_population_ta_2025" +DATA_ROOT = REPO_ROOT / "db/data/stats_nz" +WORKBOOK_PATH = ( + DATA_ROOT + / "subnational_population_estimates_2025" + / "subnational-population-estimates-at-30-june-2025-provisional.xlsx" +) +CLASSIFICATION_PATH = ( + DATA_ROOT + / "territorial_authority_2025_codes" + / "territorial-authority-2025-codes.json" +) +WORKBOOK_SHA256 = "001e8a896cfb50f5ed17836dc815b235e3bcca55ee91c9869a2afaeb054b50a6" +CLASSIFICATION_SHA256 = ( + "db033184db1f64bd4c53b0cf070cac3303815f20c9cc428e11389b91033dc225" +) +SHEET = "Table 2" +SHARED_GUARDS = { + "A3": "Territorial authority and Auckland local board areas", + "A4": "At 30 June 2023–2025", + "A5": "Territorial authority and Auckland local board area(2)", + "E6": "2025 P", + "A100": "2. Boundaries at 1 January 2025.", + "A105": "P provisional", +} +SAMPLES = ( + (8, "Far North district", "001", 73_700), + (9, "Whangārei district", "002", 100_600), + (11, "Auckland", "076", 1_816_000), + (37, "Hamilton city", "016", 192_100), + (42, "Taupō district", "021", 42_700), + (68, "Wellington city", "047", 210_800), + (81, "Christchurch city", "060", 419_200), + (87, "Chatham Islands territory", "067", 620), + (90, "Queenstown-Lakes district", "070", 53_800), + (95, "Invercargill city", "075", 58_000), +) + + +def _ascii_name(value): + return "".join( + character + for character in unicodedata.normalize("NFD", value) + if not unicodedata.combining(character) + ).casefold() + + +@lru_cache +def _publisher_evidence(): + """Read expected codes, labels and values without consulting package YAML.""" + classification = json.loads(CLASSIFICATION_PATH.read_bytes()) + assert not classification.get("exceededTransferLimit", False) + attributes = [feature["attributes"] for feature in classification["features"]] + assert len(attributes) == 68 + by_code = {item["TA2025_V1_00"]: item for item in attributes} + by_name = {item["TA2025_V1_00_NAME_ASCII"].casefold(): item for item in attributes} + assert len(by_code) == len(by_name) == 68 + assert "999" in by_code + + workbook = openpyxl.load_workbook(WORKBOOK_PATH, data_only=True) + try: + sheet = workbook[SHEET] + values = {cell.coordinate: cell.value for row in sheet for cell in row} + expected = {} + local_boards = {} + for row_number in range(8, 96): + label = sheet[f"A{row_number}"].value + if label is None: + local_boards[row_number] = sheet[f"B{row_number}"].value + continue + # Match the publisher's name, not its position in the classification. + item = by_name[_ascii_name(label)] + geography_id = f"nz-ta-{item['TA2025_V1_00']}" + assert geography_id not in expected + expected[geography_id] = { + "row": row_number, + "label": label, + "name": item["TA2025_V1_00_NAME"], + "value": sheet[f"E{row_number}"].value, + } + assert len(expected) == 67 + assert set(expected) == {f"nz-ta-{code}" for code in by_code if code != "999"} + assert set(local_boards) == set(range(12, 33)) + assert all("local board area" in label for label in local_boards.values()) + return expected, local_boards, values + finally: + workbook.close() + + +# Coordinates come from publisher evidence, so deleting YAML guards cannot +# silently delete their regression cases. +EXPECTED_GUARDS = { + **SHARED_GUARDS, + **{f"A{item['row']}": item["label"] for item in _publisher_evidence()[0].values()}, +} + + +@pytest.fixture(scope="module") +def population(): + package = load_source_package(PACKAGE_PATH) + cells = package.build_source_cells(2025) + facts = package.build_facts(2025, cells=cells) + return package, cells, facts + + +def _facts_by_id(facts): + result = {fact.geography.id: fact for fact in facts} + assert len(result) == len(facts) == 67 + return result + + +def _guard_index(specs): + guards = {} + for spec in specs: + selector = spec.selector + assert selector.sheet_name == SHEET + assert not selector.range_label_guards + entries = ( + (selector.expected_row_header_address, selector.expected_row_header), + (selector.expected_column_header_address, selector.expected_column_header), + *((guard.address, guard.expected_value) for guard in selector.guard_cells), + ) + for address, expected in entries: + if address is None: + continue + if address in guards: + assert guards[address][1] == expected + else: + guards[address] = (spec, expected) + return guards + + +@pytest.mark.parametrize( + ("path", "expected_sha", "expected_size"), + ( + (WORKBOOK_PATH, WORKBOOK_SHA256, 97_990), + (CLASSIFICATION_PATH, CLASSIFICATION_SHA256, 14_315), + ), + ids=("workbook", "classification"), +) +def test_publisher_bytes_and_manifests_remain_pinned(path, expected_sha, expected_size): + content = path.read_bytes() + manifest = yaml.safe_load((path.parent / "manifest.yaml").read_text()) + entry = manifest["files"][2025] + assert hashlib.sha256(content).hexdigest() == entry["sha256"] == expected_sha + assert len(content) == entry["size_bytes"] == expected_size + assert entry["filename"] == path.name + assert entry["storage"]["r2"]["uri"].startswith( + f"r2://ledger-raw/raw/nz/stats_nz/{path.parent.name}/2025/{expected_sha}/" + ) + + +def test_package_validates_and_preserves_the_complete_table_2(population): + package, cells, facts = population + assert SOURCE_PACKAGE_ALIASES[ALIAS] == Path( + "stats_nz/subnational_population_ta_2025" + ) + assert load_source_package(ALIAS).package_path == package.package_path + report = validate_source_package(PACKAGE_PATH, year=2025) + assert report.valid + assert not report.warnings + assert report.counts["source_record_count"] == len(facts) == 67 + assert package.artifact.sheets == (SHEET,) + assert len(cells) == 106 * 21 == 2_226 + assert {cell.sheet_name for cell in cells} == {SHEET} + assert validate_source_cells(cells).valid + assert validate_facts(facts).valid + + +def test_every_ta_fact_matches_independent_publisher_mapping(population): + package, cells, facts = population + expected, _boards, _values = _publisher_evidence() + actual = _facts_by_id(facts) + assert actual.keys() == expected.keys() + cells_by_key = {build_source_cell_key(cell): cell for cell in cells} + specs_by_record = { + spec.source_record_id: spec for spec in package.build_source_record_specs(2025) + } + selected_keys = set() + for geography_id, item in expected.items(): + fact = actual[geography_id] + spec = specs_by_record[fact.source_record_id] + assert fact.value == item["value"] + assert type(fact.value) is int + assert fact.geography.level == "local_authority" + assert fact.geography.name == item["name"] + assert fact.geography.vintage == "ta_2025" + assert fact.layout.groupby_value_label == item["label"] + assert fact.layout.groupby_value_id == geography_id + assert fact.dimension_labels["geography"] == "Geography" + assert fact.dimension_value_labels["geography"][geography_id] == item["label"] + assert fact.layout.groupby_dimension_label == "Geography" + assert spec.selector.sheet_name == SHEET + assert spec.selector.address == f"E{item['row']}" + assert spec.selector.expected_row_header_address == f"A{item['row']}" + assert spec.selector.expected_row_header == item["label"] + assert spec.selector.expected_column_header_address == "E6" + assert spec.selector.expected_column_header == "2025 P" + assert spec.selector.end_address is None + assert spec.selector.expected_cell_type == "number" + assert spec.value_scale == 1 + assert spec.divisor_selector is None + assert spec.round_to is None + value_cell, *context = [cells_by_key[key] for key in fact.source_cell_keys] + assert (value_cell.sheet_name, value_cell.address) == (SHEET, f"E{item['row']}") + assert value_cell.raw_value == fact.value + assert all(type(cell.raw_value) is str for cell in context) + selected_keys.add(fact.source_cell_keys[0]) + assert fact.source.source_sha256 == WORKBOOK_SHA256 + assert fact.measure.unit == "count" + assert fact.aggregation.method == "sum" + assert len(selected_keys) == 67 + + +@pytest.mark.parametrize( + ("row_number", "label", "code", "value"), + SAMPLES, + ids=[f"E{row}-{code}" for row, _label, code, _value in SAMPLES], +) +def test_ten_published_sample_cells(population, row_number, label, code, value): + _package, cells, facts = population + expected, _boards, values = _publisher_evidence() + geography_id = f"nz-ta-{code}" + fact = _facts_by_id(facts)[geography_id] + cells_by_key = {build_source_cell_key(cell): cell for cell in cells} + assert values[f"A{row_number}"] == label + assert values[f"E{row_number}"] == expected[geography_id]["value"] == value + assert fact.value == value + assert fact.layout.groupby_value_label == label + assert cells_by_key[fact.source_cell_keys[0]].address == f"E{row_number}" + + +def test_excludes_local_boards_outside_area_and_national_row(population): + package, cells, facts = population + expected, boards, values = _publisher_evidence() + specs = package.build_source_record_specs(2025) + selected = {spec.selector.address for spec in specs} + assert selected == {f"E{item['row']}" for item in expected.values()} + assert not selected & {f"E{row}" for row in boards} + assert "E96" not in selected + assert values["A96"] == "New Zealand(3)" + assert values["E96"] == 5_324_700 + assert {fact.geography.id for fact in facts}.isdisjoint({"NZ", "999", "nz-ta-999"}) + assert not {fact.layout.groupby_value_label for fact in facts} & set( + boards.values() + ) + # All excluded rows, years and change columns remain in parsed source cells. + addresses = {cell.address for cell in cells} + assert {f"E{row}" for row in boards} | {"E96", "C8", "D8", "F8", "G8"} <= addresses + + +def test_auckland_reads_published_total_cell_without_local_board_arithmetic(population): + _package, cells, facts = population + cells_by_key = {build_source_cell_key(cell): cell for cell in cells} + auckland = _facts_by_id(facts)["nz-ta-076"] + assert auckland.value == 1_816_000 + assert cells_by_key[auckland.source_cell_keys[0]].address == "E11" + assert not {cells_by_key[key].address for key in auckland.source_cell_keys} & { + f"E{row}" for row in range(12, 33) + } + + +def test_period_population_and_provisional_status_are_source_backed(population): + _package, cells, facts = population + cell_index = {build_source_cell_key(cell): cell for cell in cells} + rows = consumer_fact_rows(facts) + assert validate_consumer_fact_contract(facts).valid + assert len({row["aggregate_fact_key"] for row in rows}) == 67 + assert len({row["semantic_fact_key"] for row in rows}) == 67 + for fact, row in zip(facts, rows, strict=True): + assert fact.period.type == "calendar_year" + assert fact.period.value == 2025 + coverage = fact.period_coverage + assert coverage.start_date == coverage.end_date == "2025-06-30" + assert coverage.basis == "calendar" + assert coverage.source_period_label == "At 30 June 2025 (provisional)" + assert fact.assertion == "observation" + assert fact.provenance_class == "census" + assert fact.entity.name == "person" + assert fact.entity.role == "resident_population" + assert not fact.filters + assert not fact.constraints + assert fact.measure.concept == "stats_nz.estimated_resident_population" + assert fact.source.vintage == "stats_nz_subnational_population_2025_10_29" + assert "Table 2" in fact.source.source_table + assert ( + row["geography"]["name"] + == _publisher_evidence()[0][fact.geography.id]["name"] + ) + assert row["concept_alignment"]["relation"] == "source_label" + notes = row["concept_alignment"]["evidence_notes"] + assert "provisional" in notes.lower() + assert "1 January 2025" in notes + assert "2023 census" in notes + lineage_addresses = {cell_index[key].address for key in fact.source_cell_keys} + assert SHARED_GUARDS.keys() <= lineage_addresses + + +@pytest.mark.parametrize("build_year", (2023, 2025, 2026)) +def test_requested_build_year_cannot_change_publisher_reference_period( + population, build_year +): + package, cells, facts = population + assert package.artifact.artifact_year == 2025 + assert package.build_facts(build_year, cells=cells) == facts + + +def test_every_required_guard_is_declared_and_literal(population): + package, _cells, _facts = population + guards = _guard_index(package.build_source_record_specs(2025)) + assert len(EXPECTED_GUARDS) == 73 + assert EXPECTED_GUARDS.keys() <= guards.keys() + for address, expected in EXPECTED_GUARDS.items(): + assert guards[address][1] == expected + assert _publisher_evidence()[2][address] == expected + + +@pytest.mark.parametrize("address", sorted(EXPECTED_GUARDS)) +def test_each_guard_rejects_mutated_source_cell(population, address): + package, cells, _facts = population + guards = _guard_index(package.build_source_record_specs(2025)) + assert address in guards + spec, expected = guards[address] + cell_index = build_cells_by_sheet_address(cells) + assert cell_index[(SHEET, address)].raw_value == expected + resolve_source_record(cells, spec, cells_by_sheet_address=cell_index) + changed_index = { + **cell_index, + (SHEET, address): replace( + cell_index[(SHEET, address)], raw_value="unexpected publisher layout" + ), + } + with pytest.raises(ValueError, match="expected"): + resolve_source_record(cells, spec, cells_by_sheet_address=changed_index) diff --git a/tests/test_source_package_alias_drift.py b/tests/test_source_package_alias_drift.py index 1c9620aa..0e9cf838 100644 --- a/tests/test_source_package_alias_drift.py +++ b/tests/test_source_package_alias_drift.py @@ -163,7 +163,7 @@ def test_nz_bundle_sources_match_nz_prefixed_aliases(): ) assert NZ_BUNDLE_SOURCE_PREFIXES == ("ird", "mbie", "msd", "stats_nz", "treasury") - assert len(NZ_BUNDLE_SOURCES) == 66 + assert len(NZ_BUNDLE_SOURCES) == 67 assert NZ_BUNDLE_SOURCES == expected == nz_bundle_sources_from_aliases() assert_nz_bundle_sources_match_aliases() @@ -193,7 +193,7 @@ def test_nz_bundle_sources_cover_every_nz_package_on_disk(): for manifest in (root / prefix).rglob("source_package.yaml") } - assert len(on_disk) == 66 + assert len(on_disk) == 67 assert on_disk == {SOURCE_PACKAGE_ALIASES[alias] for alias in NZ_BUNDLE_SOURCES}