From df43c735ba1f1e23923972e445497f6ec7346d75 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 12 Oct 2026 13:10:37 -0400 Subject: [PATCH 1/5] Add HT2 state Schedule 1 line 1 and Schedule A lines 7, 8a, 14 facts soi-historic-table-2-state-broad-2022 now reads eight more columns of 22in55cmcsv.csv for every state, each pinned to the TY2022 documentation guide's definition: - N00700/A00700 (Schedule 1 line 1) as taxable_state_local_income_tax_refunds_* - N18300/A18300 (Schedule A line 7) as taxes_paid_deduction_* - N19300/A19300 (Schedule A line 8a) as Table 2.1's mortgage_interest_paid_* - N19700/A19700 (Schedule A line 14) as Table 2.1's charitable_* The congressional-district package's charitable_* move to Table 2.1's concept and label for the same line 14 columns, so state, district and national rows join on concept as well as measure id. Co-Authored-By: Claude Opus 5.5 --- .../source_package.yaml | 8 +- .../source_package.yaml | 92 +++++++++ tests/test_chronicle_soi_doc_guide_columns.py | 95 +++++++++ ...test_chronicle_soi_ht2_state_schedule_a.py | 180 ++++++++++++++++++ tests/test_chronicle_source_package.py | 23 ++- 5 files changed, 391 insertions(+), 7 deletions(-) create mode 100644 tests/test_chronicle_soi_ht2_state_schedule_a.py diff --git a/packages/irs_soi/congressional_district_2022/source_package.yaml b/packages/irs_soi/congressional_district_2022/source_package.yaml index 540610b7..a024286d 100644 --- a/packages/irs_soi/congressional_district_2022/source_package.yaml +++ b/packages/irs_soi/congressional_district_2022/source_package.yaml @@ -14412,24 +14412,24 @@ record_sets: value_scale: 1000 expected_cell_type: number - measure_id: charitable_returns - label: Returns with charitable contributions deduction + label: Returns with contributions deduction ordinal: 38 column: CR source_column_id: N19700 expected_column_header_row: 1 expected_column_header: N19700 - concept: irs_soi.returns_with_charitable_contributions_deduction + concept: irs_soi.returns_with_contributions_deduction unit: count aggregation: sum expected_cell_type: number - measure_id: charitable_amount - label: Charitable contributions deduction + label: Contributions deduction amount ordinal: 39 column: CS source_column_id: A19700 expected_column_header_row: 1 expected_column_header: A19700 - concept: irs_soi.charitable_contributions_deduction + concept: irs_soi.contributions_deduction unit: usd aggregation: sum value_scale: 1000 diff --git a/packages/irs_soi/historic_table_2_state_broad_2022/source_package.yaml b/packages/irs_soi/historic_table_2_state_broad_2022/source_package.yaml index c0f1cfdb..c7e76cd3 100644 --- a/packages/irs_soi/historic_table_2_state_broad_2022/source_package.yaml +++ b/packages/irs_soi/historic_table_2_state_broad_2022/source_package.yaml @@ -781,6 +781,98 @@ record_sets: expected_column_header_row: 1 expected_column_header: A11070 value_scale: 1000 + - measure_id: taxable_state_local_income_tax_refunds_returns + label: Returns with taxable refunds of state and local income taxes + ordinal: 53 + column: AG + source_column_id: N00700 + concept: irs_soi.returns_with_taxable_state_local_income_tax_refunds + unit: count + aggregation: sum + expected_cell_type: number + expected_column_header_row: 1 + expected_column_header: N00700 + - measure_id: taxable_state_local_income_tax_refunds_amount + label: Taxable refunds of state and local income taxes + ordinal: 54 + column: AH + source_column_id: A00700 + concept: irs_soi.taxable_state_local_income_tax_refunds + unit: usd + aggregation: sum + expected_cell_type: number + expected_column_header_row: 1 + expected_column_header: A00700 + value_scale: 1000 + - measure_id: taxes_paid_deduction_returns + label: Returns with taxes paid deduction + ordinal: 55 + column: CD + source_column_id: N18300 + concept: irs_soi.returns_with_taxes_paid_deduction + unit: count + aggregation: sum + expected_cell_type: number + expected_column_header_row: 1 + expected_column_header: N18300 + - measure_id: taxes_paid_deduction_amount + label: Taxes paid deduction amount + ordinal: 56 + column: CE + source_column_id: A18300 + concept: irs_soi.taxes_paid_deduction + unit: usd + aggregation: sum + expected_cell_type: number + expected_column_header_row: 1 + expected_column_header: A18300 + value_scale: 1000 + - measure_id: mortgage_interest_paid_returns + label: Returns with home mortgage interest paid to financial institutions + ordinal: 57 + column: CH + source_column_id: N19300 + concept: irs_soi.returns_with_home_mortgage_interest_paid_to_financial_institutions + unit: count + aggregation: sum + expected_cell_type: number + expected_column_header_row: 1 + expected_column_header: N19300 + - measure_id: mortgage_interest_paid_amount + label: Home mortgage interest paid to financial institutions amount + ordinal: 58 + column: CI + source_column_id: A19300 + concept: irs_soi.home_mortgage_interest_paid_to_financial_institutions + unit: usd + aggregation: sum + expected_cell_type: number + expected_column_header_row: 1 + expected_column_header: A19300 + value_scale: 1000 + - measure_id: charitable_returns + label: Returns with contributions deduction + ordinal: 59 + column: CP + source_column_id: N19700 + concept: irs_soi.returns_with_contributions_deduction + unit: count + aggregation: sum + expected_cell_type: number + expected_column_header_row: 1 + expected_column_header: N19700 + - measure_id: charitable_amount + label: Contributions deduction amount + ordinal: 60 + column: CQ + source_column_id: A19700 + concept: irs_soi.contributions_deduction + unit: usd + aggregation: sum + expected_cell_type: number + expected_column_header_row: 1 + expected_column_header: A19700 + value_scale: 1000 - record_set_id: irs_soi.ty2022.historic_table_2.state_broad.ak provenance_class: administrative record_set_spec_id: irs_soi.historic_table_2.state_broad_totals.v1 diff --git a/tests/test_chronicle_soi_doc_guide_columns.py b/tests/test_chronicle_soi_doc_guide_columns.py index b4188daf..214a55f4 100644 --- a/tests/test_chronicle_soi_doc_guide_columns.py +++ b/tests/test_chronicle_soi_doc_guide_columns.py @@ -97,6 +97,16 @@ "Number of returns with qualified dividends", ), "qualified_dividends_amount": ("1040:3a", "Qualified dividends amount"), + # Schedule 1 line 1 reports only the taxable part of these refunds; the + # guides' wording drops "taxable", so the ids and labels carry it. + "taxable_state_local_income_tax_refunds_returns": ( + "Schedule 1:1", + "Number of returns with state and local income tax refunds", + ), + "taxable_state_local_income_tax_refunds_amount": ( + "Schedule 1:1", + "State and local income tax refunds amount", + ), "schedule_c_income_returns": ( "Schedule 1:3", "Number of returns with business or professional net income (less loss)", @@ -187,6 +197,11 @@ "Schedule A:5e", "Limited state and local taxes", ), + "taxes_paid_deduction_returns": ( + "Schedule A:7", + "Number of returns with taxes paid", + ), + "taxes_paid_deduction_amount": ("Schedule A:7", "Taxes paid amount"), "mortgage_interest_paid_returns": ( "Schedule A:8a", "Number of returns with mortgage interest paid", @@ -755,6 +770,86 @@ def test_congressional_district_interest_parts_share_table_2_1_ids(): assert district[measure_id].label == table_2_1[measure_id]["label"] +# Schedule A lines that Historic Table 2's state rows, the congressional-district +# rows and Table 2.1's national rows all publish, by the IRS variables the +# geographic files read for them. +SHARED_SCHEDULE_A_LINES = { + "mortgage_interest_paid": ("N19300", "A19300"), # line 8a + "charitable": ("N19700", "A19700"), # line 14 +} + + +def test_schedule_a_lines_shared_across_grains_carry_table_2_1_ids(): + """A consumer can join state, district and national rows on concept. + + Every package that reads one of these IRS variables declares Table 2.1's + measure id, concept and label for the same Schedule A line. The + congressional-district file used ``irs_soi.charitable_contributions_ + deduction`` for line 14 while Table 2.1 used ``irs_soi.contributions_ + deduction``; docs/concept-migrations.md records the move to Table 2.1's. + """ + table_2_1 = { + measure["measure_id"]: measure + for record_set in yaml.safe_load( + (IRS_SOI_PACKAGES / "table_2_1" / "source_package.yaml").read_text() + )["record_sets"] + for measure in record_set["measures"] + } + readers: dict[str, set[str]] = defaultdict(set) + for part, variables in SHARED_SCHEDULE_A_LINES.items(): + for variable, suffix in zip(variables, ("returns", "amount"), strict=True): + measure_id = f"{part}_{suffix}" + for read in _column_reads(): + if read.header != variable: + continue + where = f"{read.package_dir}:{read.measure_id} ({variable})" + assert read.measure_id == measure_id, where + assert read.concept == table_2_1[measure_id]["concept"], where + assert read.label == table_2_1[measure_id]["label"], where + readers[variable].add(read.package_dir) + + for variables in SHARED_SCHEDULE_A_LINES.values(): + for variable in variables: + assert { + "congressional_district_2022", + "historic_table_2_state_broad_2022", + } <= readers[variable], variable + + +# Columns no geographic package selected until the state packages read them. +# One IRS variable carries one measure id, concept and label in every package, +# so a later vintage or a district selection cannot fork the identity. +STATE_ONLY_COLUMNS = { + "N00700": ( + "taxable_state_local_income_tax_refunds_returns", + "irs_soi.returns_with_taxable_state_local_income_tax_refunds", + ), + "A00700": ( + "taxable_state_local_income_tax_refunds_amount", + "irs_soi.taxable_state_local_income_tax_refunds", + ), + "N18300": ( + "taxes_paid_deduction_returns", + "irs_soi.returns_with_taxes_paid_deduction", + ), + "A18300": ("taxes_paid_deduction_amount", "irs_soi.taxes_paid_deduction"), +} + + +def test_state_only_columns_keep_one_identity(): + seen: dict[str, set[str]] = defaultdict(set) + for read in _column_reads(): + if read.header not in STATE_ONLY_COLUMNS: + continue + where = f"{read.package_dir}:{read.measure_id} ({read.header})" + assert (read.measure_id, read.concept) == STATE_ONLY_COLUMNS[read.header], where + seen[read.header].add(read.package_dir) + + assert {header: sorted(dirs) for header, dirs in seen.items()} == { + header: ["historic_table_2_state_broad_2022"] for header in STATE_ONLY_COLUMNS + } + + @cache def _facts(package_id: str, year: int) -> dict[str, Any]: package = load_source_package(package_id) diff --git a/tests/test_chronicle_soi_ht2_state_schedule_a.py b/tests/test_chronicle_soi_ht2_state_schedule_a.py new file mode 100644 index 00000000..8d3eeeb7 --- /dev/null +++ b/tests/test_chronicle_soi_ht2_state_schedule_a.py @@ -0,0 +1,180 @@ +"""Historic Table 2 state rows for Schedule 1 line 1 and Schedule A lines 7, 8a, 14. + +``soi-historic-table-2-state-broad-2022`` reads eight more columns of +``22in55cmcsv.csv``. The TY2022 documentation guide (``22incmdocguide.doc``, +extract in ``tests/data/irs_soi_doc_guides``) defines them: + +- ``N00700``/``A00700``: "state and local income tax refunds", Schedule 1 + line 1, which the 2022 form words "Taxable refunds, credits, or offsets of + state and local income taxes"; +- ``N18300``/``A18300``: "taxes paid", Schedule A line 7 ("Add lines 5e and 6"); +- ``N19300``/``A19300``: "mortgage interest paid", Schedule A line 8a; +- ``N19700``/``A19700``: "Total charitable contributions", Schedule A line 14. + +``test_chronicle_soi_doc_guide_columns`` pins each measure id to its guide +definition. This module checks the facts themselves against the published +cells, against the file's own US row, and against Publication 1304 Table 2.1. +""" + +from __future__ import annotations + +import csv +from functools import cache +from pathlib import Path +from typing import Any + +import pytest + +from chronicle.source_package import load_source_package + +REPO_ROOT = Path(__file__).resolve().parents[1] +HT2_DIR = REPO_ROOT / "db" / "data" / "irs_soi" / "historic_table_2" +PACKAGE_ID = "soi-historic-table-2-state-broad-2022" +RECORD_PREFIX = "irs_soi.ty2022.historic_table_2.state_broad." + +# measure_id -> IRS variable. Amount columns are published in thousands. +NEW_MEASURES = { + "taxable_state_local_income_tax_refunds_returns": "N00700", + "taxable_state_local_income_tax_refunds_amount": "A00700", + "taxes_paid_deduction_returns": "N18300", + "taxes_paid_deduction_amount": "A18300", + "mortgage_interest_paid_returns": "N19300", + "mortgage_interest_paid_amount": "A19300", + "charitable_returns": "N19700", + "charitable_amount": "A19700", +} +# Historic Table 2 publishes these rows besides the 50 states and DC. +NON_STATE_ROWS = frozenset({"US", "OA", "PR"}) + + +@cache +def _csv_all_returns_rows(filename: str) -> dict[str, dict[str, str]]: + """Each geography's AGI_STUB 0 row, parsed with the csv module alone.""" + with (HT2_DIR / filename).open(newline="", encoding="utf-8-sig") as file: + rows = list(csv.DictReader(file)) + return {row["STATE"]: row for row in rows if row["AGI_STUB"] == "0"} + + +def _cell(row: dict[str, str], variable: str) -> int: + return int(row[variable].replace(",", "")) + + +def _scale(variable: str) -> int: + return 1000 if variable.startswith("A") else 1 + + +@cache +def _facts() -> dict[str, Any]: + package = load_source_package(PACKAGE_ID) + return {fact.source_record_id: fact for fact in package.build_facts(2023)} + + +def _state_fact(state: str, measure_id: str) -> Any: + return _facts()[f"{RECORD_PREFIX}{state.lower()}.all.{measure_id}"] + + +def test_every_new_fact_is_its_published_cell(): + """Exhaustive over all 51 states and eight columns: 408 facts.""" + rows = _csv_all_returns_rows("22in55cmcsv.csv") + states = sorted(set(rows) - NON_STATE_ROWS) + + assert len(states) == 51 + for state in states: + for measure_id, variable in NEW_MEASURES.items(): + fact = _state_fact(state, measure_id) + where = f"{state}:{measure_id}" + assert fact.value == _cell(rows[state], variable) * _scale(variable), where + assert fact.layout.source_column_id == variable, where + assert fact.period.value == 2022, where + assert fact.measure.unit == ( + "usd" if variable.startswith("A") else "count" + ), where + + +def test_states_other_areas_and_puerto_rico_add_to_the_us_row(): + """Accounting identity of the published file, checked through the facts. + + Amounts add exactly. Return counts are each rounded to the nearest ten, so + 53 rounded cells can miss the US cell by at most 5 x 53. + """ + rows = _csv_all_returns_rows("22in55cmcsv.csv") + states = sorted(set(rows) - NON_STATE_ROWS) + + for measure_id, variable in NEW_MEASURES.items(): + scale = _scale(variable) + total = sum(_state_fact(state, measure_id).value for state in states) + total += (_cell(rows["OA"], variable) + _cell(rows["PR"], variable)) * scale + us = _cell(rows["US"], variable) * scale + if variable.startswith("A"): + assert total == us, measure_id + else: + assert abs(total - us) <= 5 * 53, measure_id + + +def test_state_sums_track_table_2_1_lines_8a_and_14(): + """Differential check against the national publication's same lines. + + Table 2.1 (Publication 1304) tabulates itemized returns from the SOI + sample; Historic Table 2 tabulates the state files. The 51 state rows come + to 0.989-1.000 of Table 2.1's TY2022 line 8a and line 14 totals. A column + one place over would not: the line 8b and 13 neighbours are 2% and 0.3% of + these totals. + """ + table_2_1 = load_source_package("soi-table-2-1") + national = { + fact.source_record_id.rsplit(".", 1)[1]: fact.value + for fact in table_2_1.build_facts(2022) + if fact.source_record_id.startswith( + "irs_soi.ty2022.table_2_1.itemized_all_returns.all." + ) + } + rows = _csv_all_returns_rows("22in55cmcsv.csv") + states = sorted(set(rows) - NON_STATE_ROWS) + ratios = { + measure_id: sum(_state_fact(state, measure_id).value for state in states) + / national[measure_id] + for measure_id in ( + "mortgage_interest_paid_returns", + "mortgage_interest_paid_amount", + "charitable_returns", + "charitable_amount", + ) + } + + assert ratios == pytest.approx( + { + "mortgage_interest_paid_returns": 1.0003, + "mortgage_interest_paid_amount": 0.9959, + "charitable_returns": 0.9918, + "charitable_amount": 0.9887, + }, + abs=0.0001, + ) + + +def test_ty2023_file_carries_the_same_variables_in_the_same_columns(): + """A TY2023 twin of this package can read these columns unchanged. + + ``23in55cmcsv.csv`` inserts two columns at DP, after every column read + here, and ``23incmdocguide.doc`` defines the eight variables in the same + words as the TY2022 guide. + """ + package = load_source_package(PACKAGE_ID) + columns = { + measure["measure_id"]: measure["column"] + for record_set in package.record_sets + for measure in record_set.payload["measures"] + if measure["measure_id"] in NEW_MEASURES + } + with (HT2_DIR / "23in55cmcsv.csv").open(newline="", encoding="utf-8-sig") as file: + header = next(csv.reader(file)) + + def index(letters: str) -> int: + value = 0 + for letter in letters: + value = value * 26 + ord(letter) - ord("A") + 1 + return value - 1 + + assert { + measure_id: header[index(column)] for measure_id, column in columns.items() + } == NEW_MEASURES diff --git a/tests/test_chronicle_source_package.py b/tests/test_chronicle_source_package.py index 4c353a49..7f45e0c2 100644 --- a/tests/test_chronicle_source_package.py +++ b/tests/test_chronicle_source_package.py @@ -1633,8 +1633,8 @@ def test_national_soi_source_package_aliases_validate_fixture_counts(): "soi-historic-table-2-state-broad-2022": { "record_set_count": 51, "row_count": 51, - "measure_count": 2703, - "source_record_count": 2703, + "measure_count": 3111, + "source_record_count": 3111, "source_region_count": 51, }, "soi-historic-table-2-state-eitc-2022": { @@ -3917,7 +3917,7 @@ def test_soi_historic_table_2_state_broad_package_builds_2022_state_facts(): assert validate_source_cells(cells).valid assert validate_facts(facts).valid assert len(cells) == 8_476 - assert len(facts) == 2_703 + assert len(facts) == 3_111 ca_returns = values_by_record[ "irs_soi.ty2022.historic_table_2.state_broad.ca.all.return_count" @@ -3960,6 +3960,23 @@ def test_soi_historic_table_2_state_broad_package_builds_2022_state_facts(): assert ca_returns.geography.id == "0400000US06" assert ca_partnership.layout.source_column_id == "A26270" + # Schedule 1 line 1 and Schedule A lines 7, 8a and 14: the CA cells of + # 22in55cmcsv.csv, amounts in thousands of dollars. + ca_prefix = "irs_soi.ty2022.historic_table_2.state_broad.ca.all." + for measure_id, column, value in ( + ("taxable_state_local_income_tax_refunds_returns", "N00700", 638_320), + ("taxable_state_local_income_tax_refunds_amount", "A00700", 1_061_330_000), + ("taxes_paid_deduction_returns", "N18300", 2_836_990), + ("taxes_paid_deduction_amount", "A18300", 25_884_637_000), + ("mortgage_interest_paid_returns", "N19300", 2_327_780), + ("mortgage_interest_paid_amount", "A19300", 35_409_552_000), + ("charitable_returns", "N19700", 2_287_850), + ("charitable_amount", "A19700", 34_813_548_000), + ): + fact = values_by_record[ca_prefix + measure_id] + assert fact.value == value, measure_id + assert fact.layout.source_column_id == column, measure_id + def test_soi_historic_table_2_state_agi_package_builds_taxable_interest_facts( tmp_path, From 39dd12f5142f77bd0033030df4215ae78fced60b Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 12 Oct 2026 13:16:44 -0400 Subject: [PATCH 2/5] Pin bundle counts and record the line 14 concept move The 20 IRS SOI packages at bundle year 2023, built against the parent commit, gain 408 TY2022 tax-unit facts, eight per state, and 204 semantic duplicate keys: the district file's state-total rows publish Schedule A lines 8a and 14 under the same concepts as the new state rows. No new geography, period or entity keys and no new warnings. docs/concept-migrations.md records the district charitable_* move to Table 2.1's contributions concept and label. Co-Authored-By: Claude Opus 5.5 --- .../irs-soi-ht2-state-schedule-a.added.md | 9 +++++ docs/concept-migrations.md | 35 +++++++++++++++++++ tests/test_chronicle_bundle.py | 27 +++++++++----- 3 files changed, 63 insertions(+), 8 deletions(-) create mode 100644 changelog.d/irs-soi-ht2-state-schedule-a.added.md diff --git a/changelog.d/irs-soi-ht2-state-schedule-a.added.md b/changelog.d/irs-soi-ht2-state-schedule-a.added.md new file mode 100644 index 00000000..919e88cc --- /dev/null +++ b/changelog.d/irs-soi-ht2-state-schedule-a.added.md @@ -0,0 +1,9 @@ +Add four IRS SOI Historic Table 2 column pairs to every state row of +`soi-historic-table-2-state-broad-2022`, each pinned to the TY2022 +documentation guide: taxable refunds of state and local income taxes +(`N00700`/`A00700`, Schedule 1 line 1), the taxes paid deduction +(`N18300`/`A18300`, Schedule A line 7), home mortgage interest paid to +financial institutions (`N19300`/`A19300`, line 8a) and the contributions +deduction (`N19700`/`A19700`, line 14). The last two use Table 2.1's ids, +concepts and labels, and the congressional-district package's `charitable_*` +move to Table 2.1's concept and label for the same line 14 columns. diff --git a/docs/concept-migrations.md b/docs/concept-migrations.md index b87fadfc..73b67a6a 100644 --- a/docs/concept-migrations.md +++ b/docs/concept-migrations.md @@ -149,3 +149,38 @@ The line 10 ids may not be read from a geographic file. Two differential checks compare published values: the district US row against Historic Table 2's, and the line 8a column and four-part sum against Table 2.1. A property check holds every limited SALT row to the $10,000 line 5e cap, within publisher rounding. + +## 2026-10-12: IRS SOI Schedule A line 14 takes Table 2.1's contributions concept + +| Package | Columns | Measure id | Old concept / label | New concept / label | +|---|---|---|---|---| +| `soi-congressional-district-2022` | `N19700` | `charitable_returns` | `irs_soi.returns_with_charitable_contributions_deduction` / "Returns with charitable contributions deduction" | `irs_soi.returns_with_contributions_deduction` / "Returns with contributions deduction" | +| same | `A19700` | `charitable_amount` | `irs_soi.charitable_contributions_deduction` / "Charitable contributions deduction" | `irs_soi.contributions_deduction` / "Contributions deduction amount" | + +`measure_id`, `source_record_id`, values and lineage stay. + +**Why.** The congressional-district and Historic Table 2 guides define +`N19700`/`A19700` as "Total charitable contributions", Schedule A line 14 (the +2022 form's "Add lines 11 through 13"). Table 2.1's `charitable_*` read its +"Contributions deduction: Total" columns, the same line, under +`irs_soi.contributions_deduction`. Two ids named one line. For TY2022 the +district file's US `A19700` is $206.1B and Historic Table 2's is $221.0B, +0.927 and 0.994 of Table 2.1's $222.4B. + +**Added alongside.** `soi-historic-table-2-state-broad-2022` now reads the same +line 14 columns, and line 8a's `N19300`/`A19300`, for every state under Table +2.1's ids, concepts and labels. It also reads two columns no package selected +before, under new ids: `N00700`/`A00700` (Schedule 1 line 1, taxable refunds of +state and local income taxes) as `taxable_state_local_income_tax_refunds_*`, and +`N18300`/`A18300` (Schedule A line 7, taxes paid) as `taxes_paid_deduction_*`. + +**Consumer effect.** Consumers that select by `measure_id` see no change from +the rename. The district file's state-total rows and the new Historic Table 2 +state rows publish the same lines 8a and 14 for the same states, so they share +semantic fact keys, as the other columns of the two files already do. + +**Pinned by** `tests/test_chronicle_soi_doc_guide_columns.py`: +`test_schedule_a_lines_shared_across_grains_carry_table_2_1_ids` holds every +package that reads `N19300`, `A19300`, `N19700` or `A19700` to Table 2.1's +measure id, concept and label, and `test_state_only_columns_keep_one_identity` +gives `N00700`, `A00700`, `N18300` and `A18300` one identity in every package. diff --git a/tests/test_chronicle_bundle.py b/tests/test_chronicle_bundle.py index 5171ea84..0e8d9700 100644 --- a/tests/test_chronicle_bundle.py +++ b/tests/test_chronicle_bundle.py @@ -383,13 +383,20 @@ def test_build_bundle_writes_merged_consumer_contract(tmp_path): # year 2023: no new geography, period or entity keys, duplicate keys or # warnings. Its SALT and PTC column fixes, its line 8a relabel and the # Historic Table 2 QBI column fix change no count. + # Historic Table 2's state package adds Schedule 1 line 1 and Schedule A + # lines 7, 8a and 14: 408 TY2022 tax-unit facts (eight measures at 51 + # states), eight at each state geography. The district file's state-total + # rows publish lines 8a and 14 under the same concepts, so 51 states x four + # measures add 204 semantic duplicate keys. Measured with the 20 IRS SOI + # packages at bundle year 2023 against the parent commit: no new geography, + # period or entity keys and no new warnings. assert summary["counts"] == { "aggregate_duplicate_key_count": 0, "entity_count": 12, "error_count": 0, # chronicle#322 adds 4,947 ONS observations and 3,200 DESNZ ones. # Inner/Outer London also add two region-level geography keys. - "fact_count": 1007134, + "fact_count": 1007542, "geography_count": 20548, "period_count": 511, # 467 before chronicle#292 moved the congressional-district and @@ -409,7 +416,8 @@ def test_build_bundle_writes_merged_consumer_contract(tmp_path): # export: 5,244 NRS + 5,406 UC + 12 DfI overlap groups are retired. # Independent census cuts and the older-only combined Metro/Glider # journey series remain publisher facts with their original provenance. - "semantic_duplicate_key_count": 2324, + # Historic Table 2's state lines 8a and 14 add 204 (see above). + "semantic_duplicate_key_count": 2528, "skipped_source_count": 10, "source_count": 56, "source_package_count": 377, @@ -447,7 +455,7 @@ def test_build_bundle_writes_merged_consumer_contract(tmp_path): if row_count == 1: assert row["aggregate_fact_key"].startswith("ledger.aggregate_fact.v2:") assert row["semantic_fact_key"].startswith("ledger.semantic_fact.v2:") - assert row_count == 1007134 + assert row_count == 1007542 assert provenance_classes <= { "administrative", "census", @@ -468,7 +476,7 @@ def test_build_bundle_writes_merged_consumer_contract(tmp_path): "jct-obbba-revenue-estimates-2025", "jct-tax-expenditures-2024", ] - assert coverage["fact_count"] == 1007134 + assert coverage["fact_count"] == 1007542 assert coverage["counts"]["by_source"] == { "bea": 446, "bfp_economic_outlook": 5, @@ -494,7 +502,7 @@ def test_build_bundle_writes_merged_consumer_contract(tmp_path): "hmrc": 31515, "ici": 12, "ird": 4362, - "irs_soi": 43963, + "irs_soi": 44371, "isc": 4, "jrc_euromod_be": 90, "kff": 52, @@ -1771,6 +1779,9 @@ def test_build_bundle_writes_merged_consumer_contract(tmp_path): # The congressional-district file's Schedule A line 8b, 8c and 9 columns: # six measures at its 480 rows. expected_period_counts["tax_year:2022"] += 2880 + # Historic Table 2's state Schedule 1 line 1 and Schedule A lines 7, 8a + # and 14: eight measures at 51 states. + expected_period_counts["tax_year:2022"] += 408 # BEA NIPA Table 7.8 line 17 (B4923C): the bundle year's own column. expected_period_counts["calendar_year:2023"] += 1 assert coverage["counts"]["by_period"] == expected_period_counts @@ -1800,7 +1811,7 @@ def test_build_bundle_writes_merged_consumer_contract(tmp_path): assert coverage["counts"]["by_geography"]["nuts1:BE3"] == 3662 assert coverage["counts"]["by_geography"]["commune:11002"] == 1 assert coverage["counts"]["by_geography"]["country:0100000US"] == 2116 - assert coverage["counts"]["by_geography"]["state:0400000US06"] == 255 + assert coverage["counts"]["by_geography"]["state:0400000US06"] == 263 assert ( coverage["counts"]["by_geography"]["congressional_district:5001700US0601"] == 62 ) @@ -1841,10 +1852,10 @@ def test_build_bundle_writes_merged_consumer_contract(tmp_path): "person": 496036, "return": 14600, "social_protection_scheme": 36, - "tax_unit": 45268, + "tax_unit": 45676, } assert not coverage["duplicates"]["aggregate_fact_keys"] - assert len(coverage["duplicates"]["semantic_fact_keys"]) == 2324 + assert len(coverage["duplicates"]["semantic_fact_keys"]) == 2528 assert Counter(warning["code"] for warning in summary["warnings"]) == { "conflicting_geography_name_across_packages": 78, "conflicting_groupby_value_label": 23, From e6c39c616a63ff1f7f55b5cdbe9f12661f0203e1 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 12 Oct 2026 14:21:19 -0400 Subject: [PATCH 3/5] Check labels for state-only columns and accept matching later readers Review nits on e3bd56e9: the identity test promised measure id, concept and label but checked two; it also rejected a correctly mapped TY2023 twin. The bundle comment now says what the measurement compared. Co-Authored-By: Claude Opus 5.5 --- tests/test_chronicle_bundle.py | 4 +-- tests/test_chronicle_soi_doc_guide_columns.py | 26 +++++++++++++------ 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/tests/test_chronicle_bundle.py b/tests/test_chronicle_bundle.py index 0e8d9700..2399eaa5 100644 --- a/tests/test_chronicle_bundle.py +++ b/tests/test_chronicle_bundle.py @@ -388,8 +388,8 @@ def test_build_bundle_writes_merged_consumer_contract(tmp_path): # states), eight at each state geography. The district file's state-total # rows publish lines 8a and 14 under the same concepts, so 51 states x four # measures add 204 semantic duplicate keys. Measured with the 20 IRS SOI - # packages at bundle year 2023 against the parent commit: no new geography, - # period or entity keys and no new warnings. + # packages at bundle year 2023, with and without these measures: no new + # geography, period or entity keys and no new warnings. assert summary["counts"] == { "aggregate_duplicate_key_count": 0, "entity_count": 12, diff --git a/tests/test_chronicle_soi_doc_guide_columns.py b/tests/test_chronicle_soi_doc_guide_columns.py index 214a55f4..4ed8c1f0 100644 --- a/tests/test_chronicle_soi_doc_guide_columns.py +++ b/tests/test_chronicle_soi_doc_guide_columns.py @@ -816,23 +816,30 @@ def test_schedule_a_lines_shared_across_grains_carry_table_2_1_ids(): } <= readers[variable], variable -# Columns no geographic package selected until the state packages read them. -# One IRS variable carries one measure id, concept and label in every package, -# so a later vintage or a district selection cannot fork the identity. +# Columns that only Historic Table 2's state packages read. One IRS variable +# carries one measure id, concept and label in every package that reads it, so +# a later vintage or a district selection cannot fork the identity. STATE_ONLY_COLUMNS = { "N00700": ( "taxable_state_local_income_tax_refunds_returns", "irs_soi.returns_with_taxable_state_local_income_tax_refunds", + "Returns with taxable refunds of state and local income taxes", ), "A00700": ( "taxable_state_local_income_tax_refunds_amount", "irs_soi.taxable_state_local_income_tax_refunds", + "Taxable refunds of state and local income taxes", ), "N18300": ( "taxes_paid_deduction_returns", "irs_soi.returns_with_taxes_paid_deduction", + "Returns with taxes paid deduction", + ), + "A18300": ( + "taxes_paid_deduction_amount", + "irs_soi.taxes_paid_deduction", + "Taxes paid deduction amount", ), - "A18300": ("taxes_paid_deduction_amount", "irs_soi.taxes_paid_deduction"), } @@ -842,12 +849,15 @@ def test_state_only_columns_keep_one_identity(): if read.header not in STATE_ONLY_COLUMNS: continue where = f"{read.package_dir}:{read.measure_id} ({read.header})" - assert (read.measure_id, read.concept) == STATE_ONLY_COLUMNS[read.header], where + assert (read.measure_id, read.concept, read.label) == STATE_ONLY_COLUMNS[ + read.header + ], where seen[read.header].add(read.package_dir) - assert {header: sorted(dirs) for header, dirs in seen.items()} == { - header: ["historic_table_2_state_broad_2022"] for header in STATE_ONLY_COLUMNS - } + # Further readers (a TY2023 twin, a district selection) are welcome once + # they pass the identity check above. + for header in STATE_ONLY_COLUMNS: + assert "historic_table_2_state_broad_2022" in seen[header], header @cache From 7d096e57660ebdcab2a83a43dab959636e00faf2 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 12 Oct 2026 21:16:04 -0400 Subject: [PATCH 4/5] Point the legacy-ETL gap rows at the new HT2 state measures docs/us-legacy-etl-cleanup.md (chronicle#348) now names the Historic Table 2 state rows for four retired series: - salt_* names taxes_paid_deduction_* (Schedule A line 7) beside the line 5e and 5b measures; the legacy rows named no IRS line; - mortgage interest and charitable rows list Historic Table 2's state rows next to the district file's state totals, separate IRS tabulations of the same lines; - state_local_refunds_* is covered at state grain; HT2's US row and Table 1.4 still go unselected nationally. The QBI row's chronicle#345 note is now past tense. Co-Authored-By: Claude Opus 5.5 --- docs/us-legacy-etl-cleanup.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/us-legacy-etl-cleanup.md b/docs/us-legacy-etl-cleanup.md index a7dea079..b3ba1bec 100644 --- a/docs/us-legacy-etl-cleanup.md +++ b/docs/us-legacy-etl-cleanup.md @@ -121,14 +121,14 @@ counted 32,067,810. The retired tests pinned the invented levels: for example | `db/etl_soi_credits.py` `ctc_claims`, `ctc_amount`, `actc_claims`, `actc_amount` (state) | `soi-historic-table-2-state-broad-2022` `ctc_*` (`N07225`/`A07225`), `actc_*` (`N11070`/`A11070`) | None for 2022. | | `db/etl_soi_credits.py` EITC by number of children (national) | `soi-table-2-5-eitc-agi-children-2022`, `soi-table-2-5-eitc-agi-children-2023`; by state: `soi-historic-table-2-state-eitc-2022` `eitc_no_children_*` through `eitc_three_or_more_children_*` | None. | | `db/etl_soi_credits.py` CTC and ACTC by number of children (national) | — | Gap: no packaged IRS table publishes CTC or ACTC by number of qualifying children. | -| `db/etl_soi_deductions.py` `salt_claims`, `salt_amount` (state) | `soi-historic-table-2-state-broad-2022` `limited_state_local_taxes_*`, `real_estate_taxes_*` | The legacy "SALT" named no IRS line; HT2's total-taxes-paid column (`N18300`/`A18300`) is not packaged at state grain. | -| `db/etl_soi_deductions.py` `mortgage_interest_claims`, `mortgage_interest_amount` (state) | state: `soi-congressional-district-2022` `interest_paid_deduction_*` (`N19300`/`A19300`, state rows for all 51 geographies; chronicle#345 relabels the column as Schedule A line 8a home mortgage interest); national: `soi-table-2-1` `mortgage_interest_paid_*` | None for 2022. | -| `db/etl_soi_deductions.py` `charitable_claims`, `charitable_amount` (state) | state: `soi-congressional-district-2022` `charitable_*` (`N19700`/`A19700`, state rows for all 51 geographies); national: `soi-table-2-1` `charitable_*` | None for 2022. | +| `db/etl_soi_deductions.py` `salt_claims`, `salt_amount` (state) | `soi-historic-table-2-state-broad-2022` `taxes_paid_deduction_*` (`N18300`/`A18300`, Schedule A line 7), `limited_state_local_taxes_*` (line 5e), `real_estate_taxes_*` (line 5b) | The legacy "SALT" named no IRS line, so no one measure is a like-for-like replacement. None for 2022. | +| `db/etl_soi_deductions.py` `mortgage_interest_claims`, `mortgage_interest_amount` (state) | state: `soi-historic-table-2-state-broad-2022` `mortgage_interest_paid_*` (`N19300`/`A19300`, Schedule A line 8a, Historic Table 2's state rows) and `soi-congressional-district-2022` `mortgage_interest_paid_*` (the district file's state-total rows for the same column); national: `soi-table-2-1` `mortgage_interest_paid_*` | None for 2022. The two state series are separate IRS tabulations of one line and differ by state. | +| `db/etl_soi_deductions.py` `charitable_claims`, `charitable_amount` (state) | state: `soi-historic-table-2-state-broad-2022` `charitable_*` (`N19700`/`A19700`, Schedule A line 14, Historic Table 2's state rows) and `soi-congressional-district-2022` `charitable_*` (the district file's state-total rows for the same column); national: `soi-table-2-1` `charitable_*` | None for 2022. The two state series are separate IRS tabulations of one line and differ by state (WY `A19700`: $572M in the district file, $1,204M in Historic Table 2). | | `db/etl_soi_deductions.py` `medical_claims`, `medical_amount` (state) | `soi-historic-table-2-state-broad-2022` `medical_dental_expense_*` | None for 2022. | -| `db/etl_soi_deductions.py` `qbi_claims`, `qbi_amount` (state) | `soi-congressional-district-2022` `qualified_business_income_deduction_*` (`N04475`/`A04475`, state rows for all 51 geographies); `soi-historic-table-2-state-broad-2022` `qbi_claims`, `qbi_amount` | None for 2022. The broad package's `qbi_*` read `N03270`/`A03270` on main; chronicle#345 repins them to `N04475`/`A04475`. | +| `db/etl_soi_deductions.py` `qbi_claims`, `qbi_amount` (state) | `soi-congressional-district-2022` `qualified_business_income_deduction_*` (`N04475`/`A04475`, state rows for all 51 geographies); `soi-historic-table-2-state-broad-2022` `qbi_claims`, `qbi_amount` | None for 2022. chronicle#345 repinned the broad package's `qbi_*` from `N03270`/`A03270` to `N04475`/`A04475`. | | `db/etl_soi_income_sources.py` taxable and tax-exempt interest, ordinary and qualified dividends, Schedule C, partnership and S corporation, rental and royalty (national and state) | national: `soi-historic-table-2` (US row); state: `soi-historic-table-2-state-broad-2022` `taxable_interest_*`, `tax_exempt_interest_*`, `ordinary_dividends_*`, `qualified_dividends_*`, `schedule_c_income_*`, `partnership_scorp_income_*`, `rental_royalty_income_*` | None for 2022. | | `db/etl_soi_income_sources.py` `short_term_capital_gains_*`, `long_term_capital_gains_*` | `soi-historic-table-2`, `soi-historic-table-2-state-broad-2022` and `soi-table-1-4` publish net capital gain (`net_capital_gains_*`) only | Gap: no packaged table splits gains into short and long term. | -| `db/etl_soi_income_sources.py` `state_local_refunds_*` | — | Gap: HT2 publishes `N00700`/`A00700`, but no package selects it. | +| `db/etl_soi_income_sources.py` `state_local_refunds_*` | state: `soi-historic-table-2-state-broad-2022` `taxable_state_local_income_tax_refunds_*` (`N00700`/`A00700`, Schedule 1 line 1) | None at state grain for 2022. Gap nationally: HT2's US row and Table 1.4's "State income tax refunds" columns publish it, but no package selects them. | | `db/etl_soi_income_sources.py` `alimony_received_*` | national: `soi-table-1-4` `alimony_received_*` | Gap at state grain: no package carries state alimony. | ## Retired: hand-typed values from other publishers From a838ee45a9ca14fa36997a9e46bad2b46dbe510b Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 12 Oct 2026 21:56:14 -0400 Subject: [PATCH 5/5] Name the line 7 and refund identity test for what it checks Confirmation-review nits at 7d096e57: STATE_ONLY_COLUMNS read as "only the state package may read these" while the test accepts matching district readers, so it is now SINGLE_IDENTITY_COLUMNS. The cleanup doc's national refunds gap also names the district file's US row, which publishes N00700/A00700 unselected. Co-Authored-By: Claude Opus 5.5 --- docs/concept-migrations.md | 2 +- docs/us-legacy-etl-cleanup.md | 2 +- tests/test_chronicle_soi_doc_guide_columns.py | 17 +++++++++-------- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/docs/concept-migrations.md b/docs/concept-migrations.md index 73b67a6a..a6a8bd96 100644 --- a/docs/concept-migrations.md +++ b/docs/concept-migrations.md @@ -182,5 +182,5 @@ semantic fact keys, as the other columns of the two files already do. **Pinned by** `tests/test_chronicle_soi_doc_guide_columns.py`: `test_schedule_a_lines_shared_across_grains_carry_table_2_1_ids` holds every package that reads `N19300`, `A19300`, `N19700` or `A19700` to Table 2.1's -measure id, concept and label, and `test_state_only_columns_keep_one_identity` +measure id, concept and label, and `test_line_7_and_refund_columns_keep_one_identity` gives `N00700`, `A00700`, `N18300` and `A18300` one identity in every package. diff --git a/docs/us-legacy-etl-cleanup.md b/docs/us-legacy-etl-cleanup.md index b3ba1bec..ab158ce6 100644 --- a/docs/us-legacy-etl-cleanup.md +++ b/docs/us-legacy-etl-cleanup.md @@ -128,7 +128,7 @@ counted 32,067,810. The retired tests pinned the invented levels: for example | `db/etl_soi_deductions.py` `qbi_claims`, `qbi_amount` (state) | `soi-congressional-district-2022` `qualified_business_income_deduction_*` (`N04475`/`A04475`, state rows for all 51 geographies); `soi-historic-table-2-state-broad-2022` `qbi_claims`, `qbi_amount` | None for 2022. chronicle#345 repinned the broad package's `qbi_*` from `N03270`/`A03270` to `N04475`/`A04475`. | | `db/etl_soi_income_sources.py` taxable and tax-exempt interest, ordinary and qualified dividends, Schedule C, partnership and S corporation, rental and royalty (national and state) | national: `soi-historic-table-2` (US row); state: `soi-historic-table-2-state-broad-2022` `taxable_interest_*`, `tax_exempt_interest_*`, `ordinary_dividends_*`, `qualified_dividends_*`, `schedule_c_income_*`, `partnership_scorp_income_*`, `rental_royalty_income_*` | None for 2022. | | `db/etl_soi_income_sources.py` `short_term_capital_gains_*`, `long_term_capital_gains_*` | `soi-historic-table-2`, `soi-historic-table-2-state-broad-2022` and `soi-table-1-4` publish net capital gain (`net_capital_gains_*`) only | Gap: no packaged table splits gains into short and long term. | -| `db/etl_soi_income_sources.py` `state_local_refunds_*` | state: `soi-historic-table-2-state-broad-2022` `taxable_state_local_income_tax_refunds_*` (`N00700`/`A00700`, Schedule 1 line 1) | None at state grain for 2022. Gap nationally: HT2's US row and Table 1.4's "State income tax refunds" columns publish it, but no package selects them. | +| `db/etl_soi_income_sources.py` `state_local_refunds_*` | state: `soi-historic-table-2-state-broad-2022` `taxable_state_local_income_tax_refunds_*` (`N00700`/`A00700`, Schedule 1 line 1) | None at state grain for 2022. Gap nationally: Historic Table 2's and the district file's US rows and Table 1.4's "State income tax refunds" columns publish it, but no package selects them. | | `db/etl_soi_income_sources.py` `alimony_received_*` | national: `soi-table-1-4` `alimony_received_*` | Gap at state grain: no package carries state alimony. | ## Retired: hand-typed values from other publishers diff --git a/tests/test_chronicle_soi_doc_guide_columns.py b/tests/test_chronicle_soi_doc_guide_columns.py index 4ed8c1f0..80431969 100644 --- a/tests/test_chronicle_soi_doc_guide_columns.py +++ b/tests/test_chronicle_soi_doc_guide_columns.py @@ -816,10 +816,11 @@ def test_schedule_a_lines_shared_across_grains_carry_table_2_1_ids(): } <= readers[variable], variable -# Columns that only Historic Table 2's state packages read. One IRS variable -# carries one measure id, concept and label in every package that reads it, so -# a later vintage or a district selection cannot fork the identity. -STATE_ONLY_COLUMNS = { +# Schedule 1 line 1 and Schedule A line 7, first read by Historic Table 2's +# state package. One IRS variable carries one measure id, concept and label in +# every package that reads it, so a later vintage or a district selection cannot +# fork the identity. +SINGLE_IDENTITY_COLUMNS = { "N00700": ( "taxable_state_local_income_tax_refunds_returns", "irs_soi.returns_with_taxable_state_local_income_tax_refunds", @@ -843,20 +844,20 @@ def test_schedule_a_lines_shared_across_grains_carry_table_2_1_ids(): } -def test_state_only_columns_keep_one_identity(): +def test_line_7_and_refund_columns_keep_one_identity(): seen: dict[str, set[str]] = defaultdict(set) for read in _column_reads(): - if read.header not in STATE_ONLY_COLUMNS: + if read.header not in SINGLE_IDENTITY_COLUMNS: continue where = f"{read.package_dir}:{read.measure_id} ({read.header})" - assert (read.measure_id, read.concept, read.label) == STATE_ONLY_COLUMNS[ + assert (read.measure_id, read.concept, read.label) == SINGLE_IDENTITY_COLUMNS[ read.header ], where seen[read.header].add(read.package_dir) # Further readers (a TY2023 twin, a district selection) are welcome once # they pass the identity check above. - for header in STATE_ONLY_COLUMNS: + for header in SINGLE_IDENTITY_COLUMNS: assert "historic_table_2_state_broad_2022" in seen[header], header