From a6a1deb1abe9e850b68cbc3e6ef540fa959db054 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 12 Oct 2026 11:06:59 -0400 Subject: [PATCH 1/4] Add spec-declared benefit-unit reporting classifiers Implement weighted income cells, strict assigned receipt groups, and typed baseline mappings carried into scenario branches by stable unit ID. Declare NZ A1/A2 parameters, shared MC11 predicate and expected structural zeros; keep unresolved gross payment and Veterans Pension inputs fail-closed. Co-Authored-By: GPT-6.1 Sol Co-Authored-By: Claude Opus 5.5 --- .../1183-reporting-classifiers.added.md | 5 + docs/transport-entitlement.md | 57 ++ .../microcosm/build/nz/as_rate_bridge.json | 15 +- .../microcosm/build/nz/country_package.json | 5 + .../src/microcosm/build/nz/gates.json | 2 +- .../build/nz/reporting_classifiers.json | 743 ++++++++++++++++++ .../build/transport/artifact_types.py | 8 + .../microcosm/build/transport/entitlement.py | 3 + .../src/microcosm/build/transport/registry.py | 2 + .../build/transport/reporting_kernels.py | 327 ++++++++ .../shared/test_nz_spec_package.py | 277 +++++++ .../test_transport_reporting_kernels.py | 641 +++++++++++++++ .../src/microcosm/frame/reporting.py | 429 ++++++++++ .../engine_free/shared/test_reporting.py | 538 +++++++++++++ 14 files changed, 3050 insertions(+), 2 deletions(-) create mode 100644 changelog.d/1183-reporting-classifiers.added.md create mode 100644 packages/microcosm-build/src/microcosm/build/nz/reporting_classifiers.json create mode 100644 packages/microcosm-build/src/microcosm/build/transport/reporting_kernels.py create mode 100644 packages/microcosm-build/tests/engine_free/shared/test_transport_reporting_kernels.py create mode 100644 packages/microcosm-frame/src/microcosm/frame/reporting.py create mode 100644 packages/microcosm-frame/tests/engine_free/shared/test_reporting.py diff --git a/changelog.d/1183-reporting-classifiers.added.md b/changelog.d/1183-reporting-classifiers.added.md new file mode 100644 index 0000000000..28b4591a63 --- /dev/null +++ b/changelog.d/1183-reporting-classifiers.added.md @@ -0,0 +1,5 @@ +Add spec-declared benefit-unit income deciles and receipt groups, including +weighted stable-ID ties, assigned before-tax benefit income, mixed-unit counts, +and typed baseline cells carried across scenario branches. Declare the NZ A1/A2 +parameters and shared MC11 superannuitant predicate; refuse unavailable gross +main-benefit and Veterans Pension inputs. Production graph wiring remains G8-w. diff --git a/docs/transport-entitlement.md b/docs/transport-entitlement.md index bee2aaef38..782566ac4e 100644 --- a/docs/transport-entitlement.md +++ b/docs/transport-entitlement.md @@ -96,6 +96,63 @@ does not accept an `aging_index` parameter. If pre-calibration quantile maps are aged too, those variants must branch from CREATE. V3 declares its receipt override before its independent problem and calibration. +Reporting classifiers are available as three declarative kernels: +`reporting.income_decile@1`, `reporting.carry_income_decile@1`, and +`reporting.receipt_group@1`. Their `classifier` parameter is canonical JSON, +with a resource `classifier_sha256` and a `scenario` label. Income kernels +also declare `entities`, person first, including the reporting entity and +the stored weight entity. They reconstruct the Frame and resolve the unit's +inherited weights; the declared weight kind is checked. Every kernel owns +exactly one all-row classifier column: `income_decile` as `int64`, or +`receipt_group` as `string`. + +The baseline income kernel runs only for `boundary_scenario`. Declared market +and benefit components must be before tax. Benefit components multiply the +engine payment by assigned boolean receipt and their annualisation factor; +missing flags, net components and explicitly excluded outcome columns refuse. +Member amounts sum in stable person-ID order. Income ties sort by the declared +unique tie key. The provisional `weighted_midpoint` convention assigns a unit +at cumulative weight before it plus half its own weight; an exact boundary +enters the higher decile. Zero-weight units have deterministic labels and no +weighted mass. A single positive-weight unit enters decile 6 with ten bins; +an all-zero population refuses. Each receipt reports all decile weighted shares. +Bin positions within the floating-point roundoff bound of an integer boundary +are snapped to that boundary, preserving this convention when weights scale. + +The baseline node declares `baseline_cells` with nominal type +`microcosm.reporting.income-cells`, version 1. Variant nodes consume it through +an ordinary typed artifact edge naming the baseline producer, for example: + +```json +{"name": "baseline_cells", "producer": "baseline.income", "artifact": "baseline_cells"} +``` + +Typed artifact edges already support different populations. The carry kernel +checks the baseline scenario, classifier digest, entity, ID column, bin count, +and exact unit-ID set, then joins the saved labels by ID. It reads no income +amounts and reports weighted shares under the variant's weights. This preserves +baseline cells when calibration or assigned receipt changes. Wiring these +kernels into the production graph is the separate G8-w package. + +Receipt predicates name assigned person flag columns and an any-member +reduction. All flags must be nonmissing booleans. The declared priority selects +the first matching group; the default handles unmatched units. Scenario receipts +include `mixed_units`, predicate counts and group counts. A mixed unit matches +more than one predicate, regardless of which member receives each programme. + +`nz/reporting_classifiers.json` records addendum A1/A2 parameters and provenance. +Its superannuitant predicate is also referenced by MC11 in `as_rate_bridge.json`. +The cell-support gate entry explicitly lists the S0–S4 superannuitant structural +zeros in its notes, and the classifier resource records them as data. The gate +continues to identify actual structural zeros from the measured amounts. +The income declaration deliberately refuses unresolved before-tax main-benefit +and Veterans Pension payment columns. The current bindings request net weekly +main-benefit payments and a separate gross Jobseeker rate; the bindings do not +declare a verified before-tax paid amount. The assigned Veterans Pension flag is also +unsupported in the current closure and must be supplied before classification. +Family IDs used for ties are deterministic head-person IDs for the same pinned +donor and unit rule; source-household keys shared by clones do not replace them. + Engine components and masks can declare an NZS/VP alternative; an explicit exclusion input and counted excluded-unit reference can declare the fallback. The toy tests exercise these declaration and grouping shapes with invented diff --git a/packages/microcosm-build/src/microcosm/build/nz/as_rate_bridge.json b/packages/microcosm-build/src/microcosm/build/nz/as_rate_bridge.json index b64820a453..19ed1cd33b 100644 --- a/packages/microcosm-build/src/microcosm/build/nz/as_rate_bridge.json +++ b/packages/microcosm-build/src/microcosm/build/nz/as_rate_bridge.json @@ -418,7 +418,20 @@ "superannuitant_units": { "central": "exclude_and_count", "scenario_s5": "reg17_superannuitant_base_rate", - "status": "The harness has no NZ Superannuation or Veteran's Pension base-rate branch. G7 implements one from the reg 17 text if it can; otherwise the MC11 fallback stands: superannuitant units are excluded from the central run and counted. Scenario S5 carries the alternative. Awaiting D1 (MC11)." + "status": "The harness has no NZ Superannuation or Veteran's Pension base-rate branch. G7 implements one from the reg 17 text if it can; otherwise the MC11 fallback stands: superannuitant units are excluded from the central run and counted. Scenario S5 carries the alternative. Awaiting D1 (MC11).", + "predicate": { + "resource": "reporting_classifiers", + "path": [ + "receipt_group", + "predicates", + "superannuitant" + ] + }, + "predicate_citation": { + "item": "A2", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "predicate_note": "MC11 exclude-and-count and reporting must select this one assigned-member receipt predicate. A bridge implementation must resolve the reference rather than infer superannuitants from entitlement or age." }, "beneficiary_note": "The harness computes the base rate from zero-income Jobseeker Support for every scenario, beneficiary or not; accommodation_supplement_non_beneficiary itself comes from the receipt layer through the group encode, not from this bridge.", "differential_expectations": { diff --git a/packages/microcosm-build/src/microcosm/build/nz/country_package.json b/packages/microcosm-build/src/microcosm/build/nz/country_package.json index eb45411384..bd8a3008ba 100644 --- a/packages/microcosm-build/src/microcosm/build/nz/country_package.json +++ b/packages/microcosm-build/src/microcosm/build/nz/country_package.json @@ -92,6 +92,11 @@ "kind": "legacy_json", "schema_id": "legacy_json" }, + { + "path": "reporting_classifiers.json", + "kind": "legacy_json", + "schema_id": "legacy_json" + }, { "path": "scenarios.json", "kind": "legacy_json", diff --git a/packages/microcosm-build/src/microcosm/build/nz/gates.json b/packages/microcosm-build/src/microcosm/build/nz/gates.json index 423410d0dd..2ba1c2f21a 100644 --- a/packages/microcosm-build/src/microcosm/build/nz/gates.json +++ b/packages/microcosm-build/src/microcosm/build/nz/gates.json @@ -232,7 +232,7 @@ "target_maximum_single_donor_dollar_share": 0.05 }, "evidence_absent_blocks": true, - "notes": "MC15: missing, malformed or scenario-incomplete source-household diagnostics block. Cells below ESS 100, above contribution CV 2 or donor dollar share 10% are suppressed, never reported; properly suppressed cells do not block. Structural-zero cells are listed separately. Subgroup rows report entitlement, without invented subgroup payment comparators." + "notes": "MC15: missing, malformed or scenario-incomplete source-household diagnostics block. Cells below ESS 100, above contribution CV 2 or donor dollar share 10% are suppressed, never reported; properly suppressed cells do not block. Structural-zero cells are listed separately. Subgroup rows report entitlement, without invented subgroup payment comparators. G8 addendum A2 explicitly expects structural-zero cells: S0/beneficiary_superannuitant_other/superannuitant, S1/beneficiary_superannuitant_other/superannuitant, S2/beneficiary_superannuitant_other/superannuitant, S3/beneficiary_superannuitant_other/superannuitant, S4/beneficiary_superannuitant_other/superannuitant. The machine-readable list is reporting_classifiers.json structural_zero_cells; actual structural-zero diagnostics are derived from reported amounts." }, { "id": "weight_effective_sample_size_target_a", diff --git a/packages/microcosm-build/src/microcosm/build/nz/reporting_classifiers.json b/packages/microcosm-build/src/microcosm/build/nz/reporting_classifiers.json new file mode 100644 index 0000000000..6d918aff72 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/nz/reporting_classifiers.json @@ -0,0 +1,743 @@ +{ + "version": 1, + "country": "nz", + "description": "Benefit-unit reporting classifiers implementing the G8 method addendum A1 and A2. Executable declarations are selected individually; provenance and activation gaps remain explicit spec data.", + "status": "A1 provisional for confirmation at D2; activation blocked by missing before-tax paid main-benefit amounts and Veterans Pension receipt/amount inputs", + "income_decile": { + "entity": "family", + "person_entity": "person", + "unit_id_column": "family_id", + "membership_column": "person_family_id", + "weight_entity": "household", + "expected_weight_kind": "calibrated", + "tie_key": "family_id", + "boundary_scenario": "S0", + "n_bins": 10, + "rank_method": "weighted_midpoint", + "components": [ + { + "entity": "person", + "column": "employment_income", + "kind": "market", + "tax_basis": "before_tax", + "annualisation_factor": 1 + }, + { + "entity": "person", + "column": "nonfarm_self_employment_income", + "kind": "market", + "tax_basis": "before_tax", + "annualisation_factor": 1 + }, + { + "entity": "person", + "column": "farm_self_employment_income", + "kind": "market", + "tax_basis": "before_tax", + "annualisation_factor": 1 + }, + { + "entity": "person", + "column": "interest_income", + "kind": "market", + "tax_basis": "before_tax", + "annualisation_factor": 1 + }, + { + "entity": "person", + "column": "dividend_income", + "kind": "market", + "tax_basis": "before_tax", + "annualisation_factor": 1 + }, + { + "entity": "person", + "column": "rental_income", + "kind": "market", + "tax_basis": "before_tax", + "annualisation_factor": 1 + }, + { + "entity": "person", + "column": "private_pension_income", + "kind": "market", + "tax_basis": "before_tax", + "annualisation_factor": 1 + }, + { + "entity": "person", + "column": null, + "kind": "benefit", + "tax_basis": "before_tax", + "annualisation_factor": 52, + "receipt_column": "receives_jobseeker_support" + }, + { + "entity": "person", + "column": null, + "kind": "benefit", + "tax_basis": "before_tax", + "annualisation_factor": 52, + "receipt_column": "receives_sole_parent_support" + }, + { + "entity": "person", + "column": null, + "kind": "benefit", + "tax_basis": "before_tax", + "annualisation_factor": 52, + "receipt_column": "receives_supported_living_payment" + }, + { + "entity": "person", + "column": "nz_super_weekly_amount_before_tax", + "kind": "benefit", + "tax_basis": "before_tax", + "annualisation_factor": 52, + "receipt_column": "receives_nz_super" + }, + { + "entity": "person", + "column": null, + "kind": "benefit", + "tax_basis": "before_tax", + "annualisation_factor": 52, + "receipt_column": "receives_veterans_pension" + } + ], + "excluded_columns": [ + { + "entity": "family", + "column": "accommodation_supplement" + }, + { + "entity": "family", + "column": "accommodation_supplement_rounded_weekly_payment" + }, + { + "entity": "family", + "column": "working_for_families" + }, + { + "entity": "family", + "column": "family_tax_credit_prescribed_amount" + }, + { + "entity": "person", + "column": "jobseeker_support_net_weekly_payment" + }, + { + "entity": "person", + "column": "sole_parent_support_net_weekly_payment" + }, + { + "entity": "person", + "column": "supported_living_payment_net_weekly_payment" + } + ] + }, + "receipt_group": { + "entity": "family", + "person_entity": "person", + "unit_id_column": "family_id", + "membership_column": "person_family_id", + "predicates": { + "superannuitant": { + "entity": "person", + "columns": [ + "receives_nz_super", + "receives_veterans_pension" + ], + "reduction": "any" + }, + "beneficiary": { + "entity": "person", + "columns": [ + "receives_jobseeker_support", + "receives_sole_parent_support", + "receives_supported_living_payment" + ], + "reduction": "any" + } + }, + "priority": [ + "superannuitant", + "beneficiary" + ], + "default": "other" + }, + "baseline_cell_contract": { + "boundary_scenario": "S0", + "scenario_ids": [ + "S0", + "S1", + "S2", + "S3", + "S4", + "S5", + "V1", + "V2", + "V3" + ], + "join_column": "family_id", + "variant_rule": "carry_baseline_cell_by_unit_id", + "diagnostics": "report_each_scenario_weighted_decile_shares", + "citation": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + } + }, + "rank_method_note": "Provisional within-unit weighted rank convention: sort by income then stable unit id, assign floor(n_bins * (preceding_weight + unit_weight / 2) / total_weight) + 1, clamped to 1..n_bins. Tied units are ordered and may split across bins; no row is split. This midpoint convention adds a precise boundary convention to A1 and awaits hub confirmation.", + "tie_key_evidence": { + "stability": "Same pinned donor and unit rule preserve person ids across rebuilds; each family id is its head person id. Units sharing source household lineage remain distinct units.", + "implementation": [ + "packages/microcosm-frame/src/microcosm/frame/unit_construction.py:388-409", + "packages/microcosm-build/src/microcosm/build/transport/population_kernels.py:209-228", + "packages/microcosm-frame/src/microcosm/frame/transport.py:843-856" + ], + "citation": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + } + }, + "blocked_inputs": [ + { + "receipt_column": "receives_jobseeker_support", + "required": "before-tax weekly paid engine amount after the income test, multiplied by assigned receipt; null column refuses activation", + "unavailable_bound_outputs": [ + "jobseeker_support_net_weekly_payment", + "jobseeker_support_gross_weekly_rate", + "jobseeker_support_weekly_income_test_reduction" + ], + "citation": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + } + }, + { + "receipt_column": "receives_sole_parent_support", + "required": "before-tax weekly paid engine amount after the income test, multiplied by assigned receipt; null column refuses activation", + "unavailable_bound_outputs": [ + "sole_parent_support_net_weekly_payment" + ], + "citation": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + } + }, + { + "receipt_column": "receives_supported_living_payment", + "required": "before-tax weekly paid engine amount after the income test, multiplied by assigned receipt; null column refuses activation", + "unavailable_bound_outputs": [ + "supported_living_payment_net_weekly_payment" + ], + "citation": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + } + }, + { + "receipt_column": "receives_veterans_pension", + "required": "before-tax weekly paid engine amount after the income test, multiplied by assigned receipt; null column refuses activation", + "unavailable_bound_outputs": [], + "citation": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + } + } + ], + "unsupported_receipt_inputs": [ + { + "column": "receives_veterans_pension", + "evidence": "axiom_input_closure.json receipt_flags omits this flag; main_benefit_receiving_veterans_pension defaults false because Veterans Pension is not modelled in v0", + "required_behavior": "Missing flag is refused; a missing flag never implies false.", + "citation": { + "item": "A2", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + } + } + ], + "disclosures": [ + { + "note": "NZ Super is overstated for single people who share a home because the shares-residence input defaults false; this moves their deciles upward.", + "evidence": "axiom_input_closure.json:1045-1052", + "citation": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + } + }, + { + "note": "Market-income components follow the existing family gross-income concept mapping. Rental income is after business expenses. The rental and private-pension concept descriptions do not explicitly say before tax; their before-tax transport interpretation must be verified before activation.", + "evidence": "packages/microcosm-frame/src/microcosm/frame/adapters/axiom_concept_mappings/nz.json:670-688; packages/microcosm-frame/src/microcosm/frame/concepts.py:990-1000,1040-1051", + "citation": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + } + } + ], + "mixed_unit_audit": { + "measure": "units_satisfying_multiple_receipt_predicates", + "per_scenario": true, + "citation": { + "item": "A2", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + } + }, + "structural_zero_cells": [ + { + "scenario": "S0", + "cell": "beneficiary_superannuitant_other/superannuitant" + }, + { + "scenario": "S1", + "cell": "beneficiary_superannuitant_other/superannuitant" + }, + { + "scenario": "S2", + "cell": "beneficiary_superannuitant_other/superannuitant" + }, + { + "scenario": "S3", + "cell": "beneficiary_superannuitant_other/superannuitant" + }, + { + "scenario": "S4", + "cell": "beneficiary_superannuitant_other/superannuitant" + } + ], + "structural_zero_citation": { + "item": "A2", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "parameter_provenance": { + "/income_decile/entity": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/person_entity": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/unit_id_column": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/membership_column": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/weight_entity": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/expected_weight_kind": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/tie_key": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/boundary_scenario": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/n_bins": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/rank_method": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/components/0/entity": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/components/0/column": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/components/0/kind": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/components/0/tax_basis": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/components/0/annualisation_factor": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/components/1/entity": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/components/1/column": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/components/1/kind": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/components/1/tax_basis": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/components/1/annualisation_factor": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/components/2/entity": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/components/2/column": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/components/2/kind": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/components/2/tax_basis": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/components/2/annualisation_factor": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/components/3/entity": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/components/3/column": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/components/3/kind": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/components/3/tax_basis": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/components/3/annualisation_factor": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/components/4/entity": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/components/4/column": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/components/4/kind": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/components/4/tax_basis": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/components/4/annualisation_factor": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/components/5/entity": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/components/5/column": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/components/5/kind": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/components/5/tax_basis": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/components/5/annualisation_factor": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/components/6/entity": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/components/6/column": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/components/6/kind": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/components/6/tax_basis": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/components/6/annualisation_factor": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/components/7/entity": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/components/7/column": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/components/7/kind": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/components/7/tax_basis": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/components/7/annualisation_factor": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/components/7/receipt_column": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/components/8/entity": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/components/8/column": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/components/8/kind": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/components/8/tax_basis": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/components/8/annualisation_factor": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/components/8/receipt_column": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/components/9/entity": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/components/9/column": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/components/9/kind": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/components/9/tax_basis": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/components/9/annualisation_factor": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/components/9/receipt_column": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/components/10/entity": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/components/10/column": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/components/10/kind": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/components/10/tax_basis": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/components/10/annualisation_factor": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/components/10/receipt_column": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/components/11/entity": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/components/11/column": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/components/11/kind": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/components/11/tax_basis": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/components/11/annualisation_factor": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/components/11/receipt_column": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/excluded_columns/0/entity": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/excluded_columns/0/column": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/excluded_columns/1/entity": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/excluded_columns/1/column": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/excluded_columns/2/entity": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/excluded_columns/2/column": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/excluded_columns/3/entity": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/excluded_columns/3/column": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/excluded_columns/4/entity": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/excluded_columns/4/column": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/excluded_columns/5/entity": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/excluded_columns/5/column": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/excluded_columns/6/entity": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/income_decile/excluded_columns/6/column": { + "item": "A1", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/receipt_group/entity": { + "item": "A2", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/receipt_group/person_entity": { + "item": "A2", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/receipt_group/unit_id_column": { + "item": "A2", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/receipt_group/membership_column": { + "item": "A2", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/receipt_group/predicates/superannuitant/entity": { + "item": "A2", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/receipt_group/predicates/superannuitant/columns/0": { + "item": "A2", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/receipt_group/predicates/superannuitant/columns/1": { + "item": "A2", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/receipt_group/predicates/superannuitant/reduction": { + "item": "A2", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/receipt_group/predicates/beneficiary/entity": { + "item": "A2", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/receipt_group/predicates/beneficiary/columns/0": { + "item": "A2", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/receipt_group/predicates/beneficiary/columns/1": { + "item": "A2", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/receipt_group/predicates/beneficiary/columns/2": { + "item": "A2", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/receipt_group/predicates/beneficiary/reduction": { + "item": "A2", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/receipt_group/priority/0": { + "item": "A2", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/receipt_group/priority/1": { + "item": "A2", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + }, + "/receipt_group/default": { + "item": "A2", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + } + }, + "as_non_beneficiary_audit": { + "nz_super": "The existing accommodation_supplement_non_beneficiary state binding negates any_true over assigned member receipt flags, including NZ Super. A unit with any assigned NZ Super receipt is therefore not a non-beneficiary.", + "veterans_pension": "The same binding omits the unsupported Veterans Pension flag, so it must be extended when that receipt input is activated. A missing Veterans Pension flag continues to block reporting rather than classify its recipients as other.", + "evidence": "axiom_input_closure.json:1183-1204", + "citation": { + "item": "A2", + "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + } + } +} diff --git a/packages/microcosm-build/src/microcosm/build/transport/artifact_types.py b/packages/microcosm-build/src/microcosm/build/transport/artifact_types.py index 46e13c8d90..7dc2c8def1 100644 --- a/packages/microcosm-build/src/microcosm/build/transport/artifact_types.py +++ b/packages/microcosm-build/src/microcosm/build/transport/artifact_types.py @@ -21,6 +21,7 @@ "KERNEL_OUTPUTS", "COMPARISON_TYPE", "CELL_DIAGNOSTICS_TYPE", + "INCOME_CELLS_TYPE", "DIAGNOSTICS_TYPE", "EXPORT_DESCRIPTOR_TYPE", "EXPORT_READBACK_TYPE", @@ -35,6 +36,8 @@ COMPARISON_TYPE = ArtifactType("microcosm.targets.comparison", 1) #: Source-household support and donor dollar influence in each scenario cell. CELL_DIAGNOSTICS_TYPE = ArtifactType("microcosm.takeup.cell-diagnostics", 1) +#: Baseline reporting cells, joined across population branches by unit id. +INCOME_CELLS_TYPE = ArtifactType("microcosm.reporting.income-cells", 1) #: One calibration's ``microcosm-diagnostics`` model, validated, as canonical #: JSON; the type version is the diagnostics schema version. DIAGNOSTICS_TYPE = ArtifactType( @@ -54,6 +57,11 @@ #: builds its ``ArtifactOutput`` tuples from this table cannot drift. KERNEL_OUTPUTS: Mapping[str, Mapping[str, ArtifactType]] = MappingProxyType( { + "reporting.income_decile@1": MappingProxyType( + {"baseline_cells": INCOME_CELLS_TYPE} + ), + "reporting.carry_income_decile@1": MappingProxyType({}), + "reporting.receipt_group@1": MappingProxyType({}), "targets.compile@1": MappingProxyType({"surface": TARGET_SURFACE_TYPE}), "targets.problem@1": MappingProxyType({"problem": PROBLEM_TYPE}), "takeup.compare@1": MappingProxyType({"comparison": COMPARISON_TYPE}), diff --git a/packages/microcosm-build/src/microcosm/build/transport/entitlement.py b/packages/microcosm-build/src/microcosm/build/transport/entitlement.py index d06fcec41d..728b6317c2 100644 --- a/packages/microcosm-build/src/microcosm/build/transport/entitlement.py +++ b/packages/microcosm-build/src/microcosm/build/transport/entitlement.py @@ -65,6 +65,9 @@ "takeup.cell_diagnostics@1", "takeup.compare@1", "transport.scenario_override@1", + "reporting.income_decile@1", + "reporting.carry_income_decile@1", + "reporting.receipt_group@1", } ) _FIELDS = frozenset( diff --git a/packages/microcosm-build/src/microcosm/build/transport/registry.py b/packages/microcosm-build/src/microcosm/build/transport/registry.py index 5a65a1a806..57a9d74497 100644 --- a/packages/microcosm-build/src/microcosm/build/transport/registry.py +++ b/packages/microcosm-build/src/microcosm/build/transport/registry.py @@ -47,6 +47,7 @@ from .gate_kernels import register_gate_kernel from .geography_kernels import register_geography_kernels from .population_kernels import register_population_kernels +from .reporting_kernels import register_reporting_kernels from .takeup_kernels import register_takeup_kernels from .target_kernels import register_target_kernels from .terminal_kernels import register_terminal_kernels @@ -380,6 +381,7 @@ def build_transport_registry( kernels.register(SimulateCounterfactualKernel(engine_mapping, dependencies)) kernels.register(SimulateSolveZeroKernel(engine_mapping, dependencies)) register_takeup_kernels(kernels) + register_reporting_kernels(kernels) return TransportRegistry( kernels, codecs, engine_mapping, MappingProxyType(refs), schema, nesting ) diff --git a/packages/microcosm-build/src/microcosm/build/transport/reporting_kernels.py b/packages/microcosm-build/src/microcosm/build/transport/reporting_kernels.py new file mode 100644 index 0000000000..46d1431b9e --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/transport/reporting_kernels.py @@ -0,0 +1,327 @@ +"""Spec-declared unit reporting classifiers and baseline cell transport. + +The baseline classifier emits a typed unit-id mapping. Other populations read +that mapping through an explicit artifact edge, retaining the baseline cells +while reporting their own weighted shares. No income is recomputed on that +path. Receipt classifications reduce assigned member flags using the declared +priority and record mixed-unit counts in each scenario's node receipt. +""" + +from __future__ import annotations + +import json +import sys + +import pandas as pd + +from microcosm.frame import reporting +from microcosm.graph import ( + ROWS_ALL, + Capabilities, + Determinism, + KernelBase, + KernelContext, + KernelRegistry, + KernelResult, + Numeric, + SeedSource, + source_hash, +) +from microcosm.graph.canonical import canonical_json +from microcosm.graph.population import dtype_for_token + +from . import graph_inputs +from .artifact_types import INCOME_CELLS_TYPE +from .graph_inputs import ( + canonical_document_param, + context_frame, + entities_param, + require_outputs, + require_params, + sha256_param, + string_param, +) + +__all__ = [ + "REPORTING_INCOME_DECILE", + "REPORTING_CARRY_INCOME_DECILE", + "REPORTING_RECEIPT_GROUP", + "IncomeDecileKernel", + "CarryIncomeDecileKernel", + "ReceiptGroupKernel", + "decode_income_cells", + "register_reporting_kernels", +] + + +def decode_income_cells(payload: bytes) -> dict: + """Validate a complete baseline mapping before admitting it as an input.""" + try: + document = json.loads(payload) + except (ValueError, TypeError, UnicodeDecodeError) as error: + raise ValueError("Invalid baseline income cells JSON.") from error + required = { + "schema_version", + "kind", + "baseline_scenario", + "entity", + "unit_id_column", + "n_bins", + "classifier_sha256", + "unit_ids", + "deciles", + } + if ( + not isinstance(document, dict) + or set(document) != required + or type(document["schema_version"]) is not int + or document["schema_version"] != 1 + or document["kind"] != "reporting_income_cells" + ): + raise ValueError("Invalid baseline income cells schema.") + for name in ("baseline_scenario", "entity", "unit_id_column"): + if not isinstance(document[name], str) or not document[name]: + raise ValueError(f"Baseline income cells need {name}.") + digest = document["classifier_sha256"] + if ( + not isinstance(digest, str) + or len(digest) != 64 + or not set(digest) <= set("0123456789abcdef") + ): + raise ValueError("Invalid baseline classifier digest.") + if type(document["n_bins"]) is not int or document["n_bins"] < 1: + raise ValueError("Invalid baseline n_bins.") + ids, labels = document["unit_ids"], document["deciles"] + if ( + not isinstance(ids, list) + or not ids + or any(type(value) is not int for value in ids) + or ids != sorted(set(ids)) + or not isinstance(labels, list) + or len(labels) != len(ids) + or any( + type(value) is not int or not 1 <= value <= document["n_bins"] + for value in labels + ) + ): + raise ValueError( + "Baseline cells need unique sorted unit ids and valid deciles." + ) + return document + + +def _classifier(context, ref, *, income): + sha256_param(context, ref, "classifier_sha256") + spec = canonical_document_param(context, ref, "classifier") + validator = ( + reporting.validate_income_classifier + if income + else reporting.validate_receipt_classifier + ) + return validator(spec) + + +def _contract(context, ref, entity, column, dtype, *, carry=False): + if context.node.sources or context.sources: + raise ValueError(f"{ref} declares no sources.") + if any( + item.rows != ROWS_ALL for item in (*context.node.inputs, *context.node.outputs) + ): + raise ValueError(f"{ref} requires all-row slices.") + outputs = context.node.outputs + if len(outputs) != 1 or ( + outputs[0].entity, + outputs[0].column, + outputs[0].dtype, + ) != (entity, column, dtype): + raise ValueError(f"{ref} must own exactly {entity}.{column} as {dtype}.") + if carry: + if ( + len(context.node.artifact_inputs) != 1 + or context.node.artifact_inputs[0].name != "baseline_cells" + or context.node.artifact_inputs[0].type != INCOME_CELLS_TYPE + or set(context.artifacts) != {"baseline_cells"} + or context.artifacts["baseline_cells"].type != INCOME_CELLS_TYPE + ): + raise ValueError(f"{ref} needs exactly the typed baseline_cells artifact.") + elif context.node.artifact_inputs or context.artifacts: + raise ValueError(f"{ref} declares no artifact inputs.") + require_outputs(context, ref) + + +def _weighted_frame(context, ref, spec): + entities = entities_param(context, ref, "entities") + if entities[0] != spec["person_entity"] or spec["entity"] not in entities: + raise ValueError(f"{ref} entity schema disagrees with its classifier.") + frame = context_frame( + context, ref, entities=entities, weight_entity=spec["weight_entity"] + ) + weights = frame.resolve_weights(spec["entity"]) + if weights.kind.value != spec["expected_weight_kind"]: + raise ValueError( + f"{ref} requires {spec['expected_weight_kind']!r} unit weights." + ) + return frame, weights.values + + +def _column(table, spec, name, values): + return { + (spec["entity"], name): pd.Series( + values, index=pd.Index(table[spec["unit_id_column"]].to_numpy(copy=True)) + ) + } + + +class _ReportingKernel(KernelBase): + capabilities = Capabilities( + Determinism.DETERMINISTIC, + numeric=Numeric.PLATFORM_BITWISE, + seed_source=SeedSource.NONE, + dependencies=("numpy", "pandas"), + ) + + def implementation_hash(self) -> str: + return source_hash( + sys.modules[__name__], + reporting, + graph_inputs, + dependencies=self.capabilities.dependencies, + ) + + +_INCOME_PARAMS = frozenset({"classifier", "classifier_sha256", "scenario", "entities"}) + + +class IncomeDecileKernel(_ReportingKernel): + """Compute cells once on the declared baseline, exporting their unit ids.""" + + ref = "reporting.income_decile@1" + + def run(self, context: KernelContext) -> KernelResult: + require_params(context, self.ref, required=_INCOME_PARAMS) + spec = _classifier(context, self.ref, income=True) + scenario = string_param(context, self.ref, "scenario") + if scenario != spec["boundary_scenario"]: + raise ValueError( + f"{self.ref} only computes the declared baseline scenario." + ) + _contract(context, self.ref, spec["entity"], "income_decile", "int64") + frame, weights = _weighted_frame(context, self.ref, spec) + table = frame.table(spec["entity"]) + if spec["tie_key"] not in table: + raise ValueError(f"Missing stable tie key {spec['tie_key']!r}.") + income = reporting.annual_unit_income( + {entity: frame.table(entity) for entity in frame.schema.entities}, spec + ) + labels, diagnostics = reporting.weighted_deciles( + income, + weights, + table[spec["tie_key"]].to_numpy(), + spec["n_bins"], + rank_method=spec["rank_method"], + ) + cells = table.loc[:, [spec["unit_id_column"]]].copy() + cells["income_decile"] = labels + cells = cells.sort_values(spec["unit_id_column"]) + payload = canonical_json( + { + "schema_version": 1, + "kind": "reporting_income_cells", + "baseline_scenario": scenario, + "entity": spec["entity"], + "unit_id_column": spec["unit_id_column"], + "n_bins": spec["n_bins"], + "classifier_sha256": context.params["classifier_sha256"], + "unit_ids": cells[spec["unit_id_column"]].tolist(), + "deciles": cells["income_decile"].tolist(), + } + ) + decode_income_cells(payload) + return KernelResult( + columns=_column(table, spec, "income_decile", labels), + artifacts={"baseline_cells": payload}, + receipt={"scenario": scenario, **diagnostics}, + ) + + +class CarryIncomeDecileKernel(_ReportingKernel): + """Join a baseline artifact by unit id; only shares use the current weights.""" + + ref = "reporting.carry_income_decile@1" + + def run(self, context: KernelContext) -> KernelResult: + require_params(context, self.ref, required=_INCOME_PARAMS) + spec = _classifier(context, self.ref, income=True) + scenario = string_param(context, self.ref, "scenario") + _contract( + context, self.ref, spec["entity"], "income_decile", "int64", carry=True + ) + baseline = decode_income_cells(context.artifacts["baseline_cells"].payload) + expected = { + "baseline_scenario": spec["boundary_scenario"], + "entity": spec["entity"], + "unit_id_column": spec["unit_id_column"], + "n_bins": spec["n_bins"], + "classifier_sha256": context.params["classifier_sha256"], + } + if any(baseline[key] != value for key, value in expected.items()): + raise ValueError("Baseline cells disagree with the declared classifier.") + frame, weights = _weighted_frame(context, self.ref, spec) + table = frame.table(spec["entity"]) + labels = reporting.carry_baseline_deciles( + table[spec["unit_id_column"]].to_numpy(), + baseline["unit_ids"], + baseline["deciles"], + spec["n_bins"], + ) + return KernelResult( + columns=_column(table, spec, "income_decile", labels), + receipt={ + "scenario": scenario, + "baseline_scenario": baseline["baseline_scenario"], + "n_units": len(table), + "weighted_shares": reporting.decile_weighted_shares( + labels, weights, spec["n_bins"] + ), + }, + ) + + +class ReceiptGroupKernel(_ReportingKernel): + """Assigned receipt groups, using the declared shared unit predicates.""" + + ref = "reporting.receipt_group@1" + + def run(self, context: KernelContext) -> KernelResult: + require_params( + context, + self.ref, + required=frozenset({"classifier", "classifier_sha256", "scenario"}), + ) + spec = _classifier(context, self.ref, income=False) + _contract(context, self.ref, spec["entity"], "receipt_group", "string") + groups, diagnostics = reporting.classify_receipt_groups(context.tables, spec) + groups = groups.astype(dtype_for_token("string")) + return KernelResult( + columns={(spec["entity"], "receipt_group"): groups}, + receipt={ + "scenario": string_param(context, self.ref, "scenario"), + **diagnostics, + }, + ) + + +REPORTING_INCOME_DECILE = IncomeDecileKernel() +REPORTING_CARRY_INCOME_DECILE = CarryIncomeDecileKernel() +REPORTING_RECEIPT_GROUP = ReceiptGroupKernel() + + +def register_reporting_kernels(registry: KernelRegistry) -> KernelRegistry: + """Explicit, idempotent registration; implementation hashes bind no spec.""" + for kernel in ( + REPORTING_INCOME_DECILE, + REPORTING_CARRY_INCOME_DECILE, + REPORTING_RECEIPT_GROUP, + ): + registry.register(kernel) + return registry diff --git a/packages/microcosm-build/tests/engine_free/shared/test_nz_spec_package.py b/packages/microcosm-build/tests/engine_free/shared/test_nz_spec_package.py index d448fa05fa..15fdb4077f 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_nz_spec_package.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_nz_spec_package.py @@ -121,6 +121,18 @@ def _input_surface() -> dict[str, dict[str, Any]]: return {module["path"]: module for module in surface["modules"]} +def _reporting_parameter_paths(value: object, path: str) -> Iterator[str]: + """Each executable parameter leaf needs its own method citation.""" + if isinstance(value, Mapping): + for key, child in value.items(): + yield from _reporting_parameter_paths(child, f"{path}/{key}") + elif isinstance(value, list): + for index, child in enumerate(value): + yield from _reporting_parameter_paths(child, f"{path}/{index}") + else: + yield path + + # --------------------------------------------------------------------------- # Validators (exercised by Hypothesis below, applied to the package here) # --------------------------------------------------------------------------- @@ -1653,3 +1665,268 @@ def test_donor_pin_and_us_support_provenance_are_explicit(self) -> None: assert artifact["size_bytes"] == 462915783 assert {"donor_country_code", "support_stratum"} <= set(donor.outputs) assert "US donor support records" in donor.survey + + +class TestReportingClassifiers: + def test_reporting_resource_is_packaged_and_loader_visible(self) -> None: + from microcosm.build.transport.compose import load_transport_spec + + resources = load_transport_spec("nz")["resources"] + assert resources["reporting_classifiers"] == _load("reporting_classifiers.json") + + def test_every_executable_parameter_cites_its_addendum_item(self) -> None: + document = _load("reporting_classifiers.json") + paths = { + path: item + for classifier, item in (("income_decile", "A1"), ("receipt_group", "A2")) + for path in _reporting_parameter_paths( + document[classifier], f"/{classifier}" + ) + } + assert set(document["parameter_provenance"]) == set(paths) + for path, item in paths.items(): + citation = document["parameter_provenance"][path] + assert citation["item"] == item + assert citation["source"].endswith("/g8-method-addendum.md") + + def test_income_names_assigned_receipt_and_gross_components_only(self) -> None: + document = _load("reporting_classifiers.json") + classifier = document["income_decile"] + assert classifier["entity"] == "family" + assert classifier["membership_column"] == "person_family_id" + assert classifier["weight_entity"] == "household" + assert classifier["expected_weight_kind"] == "calibrated" + assert classifier["tie_key"] == classifier["unit_id_column"] == "family_id" + assert classifier["n_bins"] == 10 + assert classifier["rank_method"] == "weighted_midpoint" + market = [row for row in classifier["components"] if row["kind"] == "market"] + assert {row["column"] for row in market} == { + "employment_income", + "nonfarm_self_employment_income", + "farm_self_employment_income", + "interest_income", + "dividend_income", + "rental_income", + "private_pension_income", + } + assert all(row["annualisation_factor"] == 1 for row in market) + benefits = [row for row in classifier["components"] if row["kind"] == "benefit"] + assert all(row["annualisation_factor"] == 52 for row in benefits) + assert all(row["tax_basis"] == "before_tax" for row in classifier["components"]) + super_income = next( + row for row in benefits if row["receipt_column"] == "receives_nz_super" + ) + assert super_income["column"] == "nz_super_weekly_amount_before_tax" + assert super_income["column"] in _bindings()["nz_super"]["variables"] + assert all("entitled" not in row["receipt_column"] for row in benefits) + + def test_unavailable_before_tax_paid_amounts_refuse_activation(self) -> None: + from microcosm.frame.reporting import validate_income_classifier + + document = _load("reporting_classifiers.json") + classifier = document["income_decile"] + missing = { + row["receipt_column"] + for row in classifier["components"] + if row["column"] is None + } + assert missing == { + "receives_jobseeker_support", + "receives_sole_parent_support", + "receives_supported_living_payment", + "receives_veterans_pension", + } + assert missing == {row["receipt_column"] for row in document["blocked_inputs"]} + with pytest.raises(ValueError, match="(?:column|before.tax)"): + validate_income_classifier(classifier) + paid = _bindings()["main_benefits_rates"]["variables"] + assert { + "jobseeker_support_net_weekly_payment", + "sole_parent_support_net_weekly_payment", + "supported_living_payment_net_weekly_payment", + } <= set(paid) + assert not any("before_tax" in name for name in paid) + + @pytest.mark.parametrize( + ("entity", "column"), + [ + ("person", "jobseeker_support_net_weekly_payment"), + ("person", "sole_parent_support_net_weekly_payment"), + ("person", "supported_living_payment_net_weekly_payment"), + ("family", "accommodation_supplement"), + ("family", "accommodation_supplement_rounded_weekly_payment"), + ("family", "working_for_families"), + ], + ) + def test_spec_refuses_net_benefit_and_outcome_income_columns( + self, entity: str, column: str + ) -> None: + from microcosm.frame.reporting import validate_income_classifier + + classifier = _load("reporting_classifiers.json")["income_decile"] + # Resolve only the unavailable amount declarations for this contract + # test; these fixture names make no claim about available engine outputs. + for index, component in enumerate(classifier["components"]): + if component["column"] is None: + component["column"] = f"fixture_before_tax_amount_{index}" + classifier["components"].append( + { + "entity": entity, + "column": column, + "kind": "market", + "tax_basis": "before_tax", + "annualisation_factor": 1, + } + ) + with pytest.raises(ValueError, match="Excluded income component"): + validate_income_classifier(classifier) + + def test_mc11_and_receipt_group_select_one_superannuitant_predicate(self) -> None: + from microcosm.build.transport.compose import load_transport_spec + from microcosm.frame.reporting import validate_receipt_classifier + + document = _load("reporting_classifiers.json") + classifier = validate_receipt_classifier(document["receipt_group"]) + reference = _load("as_rate_bridge.json")["superannuitant_units"]["predicate"] + selected = load_transport_spec("nz")["resources"][reference["resource"]] + for key in reference["path"]: + selected = selected[key] + assert selected == classifier["predicates"]["superannuitant"] + assert selected == { + "entity": "person", + "columns": ["receives_nz_super", "receives_veterans_pension"], + "reduction": "any", + } + assert classifier["priority"] == ["superannuitant", "beneficiary"] + assert classifier["default"] == "other" + assert classifier["predicates"]["beneficiary"]["columns"] == [ + "receives_jobseeker_support", + "receives_sole_parent_support", + "receives_supported_living_payment", + ] + + def test_unsupported_veterans_receipt_is_explicit_and_required(self) -> None: + document = _load("reporting_classifiers.json") + declared = _load("axiom_input_closure.json")["receipt_flags"]["columns"] + (unsupported,) = document["unsupported_receipt_inputs"] + assert unsupported["column"] == "receives_veterans_pension" + assert f"person.{unsupported['column']}" not in declared + assert ( + unsupported["column"] + in document["receipt_group"]["predicates"]["superannuitant"]["columns"] + ) + assert "missing flag never implies false" in unsupported["required_behavior"] + assert document["mixed_unit_audit"]["per_scenario"] is True + + def test_as_non_beneficiary_binding_agrees_for_supported_receipt_flags( + self, + ) -> None: + document = _load("reporting_classifiers.json") + classifier = document["receipt_group"] + expected = { + f"person.{column}" + for predicate in classifier["predicates"].values() + for column in predicate["columns"] + } + entries = _load("axiom_input_closure.json")["entries"] + (entry,) = [ + row + for row in entries + if row["input"] == "accommodation_supplement_non_beneficiary" + ] + binding = entry["state_binding"] + assert binding["test"] == "any_true" + assert binding["group_rule"] == "any_member" + assert binding["negate"] is True + assert "person.receives_nz_super" in binding["columns"] + assert expected - set(binding["columns"]) == { + "person.receives_veterans_pension" + } + assert ( + "must be extended" + in document["as_non_beneficiary_audit"]["veterans_pension"] + ) + + def test_baseline_cells_are_declared_for_every_scenario_and_variant(self) -> None: + document = _load("reporting_classifiers.json") + contract = document["baseline_cell_contract"] + assert contract["scenario_ids"] == [ + row["id"] for row in _load("scenarios.json")["scenarios"] + ] + assert ( + contract["boundary_scenario"] + == document["income_decile"]["boundary_scenario"] + == "S0" + ) + assert contract["join_column"] == "family_id" + assert contract["variant_rule"] == "carry_baseline_cell_by_unit_id" + assert contract["diagnostics"] == "report_each_scenario_weighted_decile_shares" + + def test_pinned_donor_unit_ids_survive_rebuilds_and_row_permutations(self) -> None: + import pandas as pd + + from microcosm.frame.unit_construction import ( + BenefitUnitRule, + DependentChildRule, + build_benefit_units, + ) + + person = pd.DataFrame( + { + "person_id": [102, 101, 202, 201], + "person_household_id": [10, 10, 20, 20], + "age": [30, 30, 30, 30], + "partner_person_id": pd.array([101, 102, None, None], dtype="Int64"), + "parent_1_person_id": pd.array([None] * 4, dtype="Int64"), + "parent_2_person_id": pd.array([None] * 4, dtype="Int64"), + } + ) + household = pd.DataFrame( + { + "household_id": [10, 20], + "reference_person_id": [102, 202], + "source_household_id": ["same_source", "same_source"], + } + ) + rule = BenefitUnitRule( + entity="family", + dependent_child=DependentChildRule(max_age=17, financial_independence=None), + unparented_child="reference_person_unit", + split_parents="first_parent_unit", + provenance={"purpose": "Adult-only stable-id reporting fixture"}, + ) + units, membership = build_benefit_units(person, household, rule) + repeated, repeated_membership = build_benefit_units(person, household, rule) + permuted, permuted_membership = build_benefit_units( + person.iloc[[3, 1, 0, 2]], household.iloc[::-1], rule + ) + pd.testing.assert_frame_equal(units, repeated) + pd.testing.assert_frame_equal(units, permuted) + pd.testing.assert_series_equal(membership, repeated_membership) + pd.testing.assert_series_equal(membership, permuted_membership.sort_index()) + assert units["family_id"].tolist() == [102, 201, 202] + assert units["family_source_household_id"].tolist() == ["same_source"] * 3 + + def test_s0_through_s4_structural_zero_cells_are_listed_on_support_entry( + self, + ) -> None: + document = _load("reporting_classifiers.json") + zeros = document["structural_zero_cells"] + assert zeros == [ + { + "scenario": f"S{index}", + "cell": "beneficiary_superannuitant_other/superannuitant", + } + for index in range(5) + ] + support = next( + row + for row in _load("gates.json")["gates"] + if row["id"] == "reported_cell_support" + ) + for row in zeros: + assert f"{row['scenario']}/{row['cell']}" in support["notes"] + assert ( + "actual structural-zero diagnostics are derived from reported amounts" + in (support["notes"]) + ) diff --git a/packages/microcosm-build/tests/engine_free/shared/test_transport_reporting_kernels.py b/packages/microcosm-build/tests/engine_free/shared/test_transport_reporting_kernels.py new file mode 100644 index 0000000000..9d87c6c0ee --- /dev/null +++ b/packages/microcosm-build/tests/engine_free/shared/test_transport_reporting_kernels.py @@ -0,0 +1,641 @@ +"""Executed reporting classifiers over invented benefit-unit populations.""" + +from __future__ import annotations + +import copy +import json +from dataclasses import replace +from hashlib import sha256 + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.transport.artifact_types import INCOME_CELLS_TYPE +from microcosm.build.transport.compose import compose_transport_graph +from microcosm.build.transport.reporting_kernels import ( + REPORTING_CARRY_INCOME_DECILE, + REPORTING_INCOME_DECILE, + REPORTING_RECEIPT_GROUP, + decode_income_cells, + register_reporting_kernels, +) +from microcosm.build.transport.takeup_kernels import reporting_cell_declarations +from microcosm.frame import EntitySchema, Frame, WeightKind, Weights +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + ArtifactType, + ArtifactValue, + Capabilities, + ContentStore, + Determinism, + Graph, + KernelBase, + KernelContext, + KernelRegistry, + KernelResult, + Node, + NumericScope, + Owned, + Slice, + SourceRef, + StructuralDelta, + WeightTransition, + compile_graph, + run_graph, +) +from microcosm.graph.canonical import canonical_json +from test_support.microcosm_build.transport_entitlement import make_entitlement_fixture +from test_support.paths import paths_for + +_ENTITIES = ("person", "household", "family") +_PATHS = paths_for("microcosm-build") +_CELL_TYPES = ("single", "couple", "sole_parent", "couple_with_children") + + +def _income_spec(): + return { + "entity": "family", + "person_entity": "person", + "unit_id_column": "family_id", + "membership_column": "person_family_id", + "weight_entity": "household", + "expected_weight_kind": "calibrated", + "tie_key": "family_id", + "boundary_scenario": "S0", + "n_bins": 10, + "rank_method": "weighted_midpoint", + "components": [ + { + "entity": "person", + "column": "market", + "kind": "market", + "tax_basis": "before_tax", + "annualisation_factor": 1, + }, + { + "entity": "person", + "column": "main_amount", + "kind": "benefit", + "tax_basis": "before_tax", + "annualisation_factor": 52, + "receipt_column": "receives_main", + }, + { + "entity": "person", + "column": "pension_amount", + "kind": "benefit", + "tax_basis": "before_tax", + "annualisation_factor": 52, + "receipt_column": "receives_pension", + }, + ], + "excluded_columns": [{"entity": "family", "column": "housing_support"}], + } + + +def _receipt_spec(): + return { + "entity": "family", + "person_entity": "person", + "unit_id_column": "family_id", + "membership_column": "person_family_id", + "predicates": { + "superannuitant": { + "entity": "person", + "columns": ["receives_pension", "receives_veteran"], + "reduction": "any", + }, + "beneficiary": { + "entity": "person", + "columns": ["receives_main"], + "reduction": "any", + }, + }, + "priority": ["superannuitant", "beneficiary"], + "default": "other", + } + + +def _frame(*, weight_kind=WeightKind.CALIBRATED): + units = np.arange(40, dtype=np.int64) + person_units = np.repeat(units, 2) + first = np.tile([True, False], len(units)) + second = ~first + person = pd.DataFrame( + { + "person_id": np.arange(80, dtype=np.int64), + "person_household_id": person_units, + "person_family_id": person_units, + "market": np.where(first, person_units * 10000, 0).astype(np.float64), + "main_amount": np.full(80, 10, dtype=np.float64), + "pension_amount": np.full(80, 20, dtype=np.float64), + "receives_main": first & np.isin(person_units % 4, [0, 3]), + "receives_pension": second & np.isin(person_units % 4, [1, 3]), + "receives_veteran": np.zeros(80, dtype=bool), + } + ) + household = pd.DataFrame( + { + "household_id": units, + # Clones sharing a source identity remain distinct unit observations. + "source_household_id": np.repeat(np.arange(20, dtype=np.int64), 2), + } + ) + family = pd.DataFrame( + { + "family_id": units, + "family_household_id": units, + "family_type": pd.Series(np.tile(_CELL_TYPES, 10), dtype="string"), + "housing_support": np.full(40, 1000000, dtype=np.float64), + } + ) + return Frame( + {"person": person, "household": household, "family": family}, + EntitySchema(group_entities=("household", "family")), + {"household": Weights(np.ones(40), weight_kind)}, + pd.Series(["all"] * 80, name="stratum"), + ) + + +def _node(kernel, *, population="S0", scenario="S0", spec=None): + receipt = kernel is REPORTING_RECEIPT_GROUP + spec = (_receipt_spec() if receipt else _income_spec()) if spec is None else spec + carry = kernel is REPORTING_CARRY_INCOME_DECILE + params = { + "classifier": canonical_json(spec).decode(), + "classifier_sha256": sha256(canonical_json(spec)).hexdigest(), + "scenario": scenario, + } + if not receipt: + params["entities"] = _ENTITIES + person_columns = ( + ("receives_main", "receives_pension", "receives_veteran") + if receipt + else ( + "market", + "main_amount", + "pension_amount", + "receives_main", + "receives_pension", + ) + ) + return Node( + f"{population}.{kernel.ref.split('.')[1].split('@')[0]}", + kernel.ref, + population=population, + inputs=( + Slice("person", person_columns), + Slice("household", ("source_household_id",)), + Slice("family", ("family_household_id", "family_type")), + ), + outputs=( + Owned( + "family", + "receipt_group" if receipt else "income_decile", + "string" if receipt else "int64", + ), + ), + params=params, + artifact_outputs=(ArtifactOutput("baseline_cells", INCOME_CELLS_TYPE),) + if kernel is REPORTING_INCOME_DECILE + else (), + artifact_inputs=( + ArtifactInput( + "baseline_cells", + "S0.income_decile", + "baseline_cells", + INCOME_CELLS_TYPE, + ), + ) + if carry + else (), + ) + + +def _artifact(payload, *, kind=INCOME_CELLS_TYPE): + return ArtifactValue(payload, kind, "a" * 64, "b" * 64, NumericScope()) + + +def _context(node, *, frame=None, artifacts=None): + frame = _frame() if frame is None else frame + return KernelContext( + node=node, + tables={entity: frame.table(entity).copy() for entity in _ENTITIES}, + weights={"household": frame.resolve_weights("household")}, + strata=frame.strata.copy(), + params=node.params, + rng=np.random.default_rng(0), + sources={}, + artifacts={} if artifacts is None else artifacts, + ) + + +def _baseline_payload(): + return REPORTING_INCOME_DECILE.run( + _context(_node(REPORTING_INCOME_DECILE)) + ).artifacts["baseline_cells"] + + +def test_baseline_uses_annual_assigned_income_calibrated_unit_weights_and_reports_shares(): + result = REPORTING_INCOME_DECILE.run(_context(_node(REPORTING_INCOME_DECILE))) + expected = np.repeat(np.arange(1, 11, dtype=np.int64), 4) + np.testing.assert_array_equal(result.columns["family", "income_decile"], expected) + assert result.receipt["weighted_shares"] == pytest.approx([0.1] * 10) + assert result.receipt["scenario"] == "S0" + cells = decode_income_cells(result.artifacts["baseline_cells"]) + assert cells["unit_ids"] == list(range(40)) + assert cells["deciles"] == expected.tolist() + + +def test_receipt_any_member_priority_and_mixed_count(): + result = REPORTING_RECEIPT_GROUP.run(_context(_node(REPORTING_RECEIPT_GROUP))) + assert ( + result.columns["family", "receipt_group"].tolist() + == ["beneficiary", "superannuitant", "other", "superannuitant"] * 10 + ) + assert str(result.columns["family", "receipt_group"].dtype) == "string" + assert result.receipt["mixed_units"] == 10 + assert result.receipt["group_counts"] == { + "superannuitant": 20, + "beneficiary": 10, + "other": 10, + } + + +def test_carry_retains_baseline_cells_after_weights_income_and_receipt_change(): + node = _node(REPORTING_CARRY_INCOME_DECILE, scenario="V3") + context = _context( + node, artifacts={"baseline_cells": _artifact(_baseline_payload())} + ) + context.weights["household"] = Weights(np.arange(1, 41), WeightKind.CALIBRATED) + context.tables["person"]["market"] *= -1 + context.tables["person"]["receives_main"] = False + result = REPORTING_CARRY_INCOME_DECILE.run(context) + np.testing.assert_array_equal( + result.columns["family", "income_decile"], + np.repeat(np.arange(1, 11, dtype=np.int64), 4), + ) + assert result.receipt["weighted_shares"] == pytest.approx( + [sum(range(4 * i + 1, 4 * i + 5)) / 820 for i in range(10)] + ) + + +def test_veteran_receipt_on_another_member_uses_superannuitant_group(): + context = _context(_node(REPORTING_RECEIPT_GROUP)) + context.tables["person"].loc[5, "receives_veteran"] = True + result = REPORTING_RECEIPT_GROUP.run(context) + assert result.columns["family", "receipt_group"].loc[2] == "superannuitant" + assert result.receipt["group_counts"]["superannuitant"] == 21 + assert result.receipt["mixed_units"] == 10 + + +def test_income_benefit_amount_uses_assigned_receipt_and_declared_annualisation(): + spec = _income_spec() + spec["n_bins"] = 40 + node = _node(REPORTING_INCOME_DECILE, spec=spec) + context = _context(node) + context.tables["person"].loc[0, "market"] = 5000 + context.tables["person"].loc[0, "main_amount"] = 300 + assigned = REPORTING_INCOME_DECILE.run(context).columns["family", "income_decile"] + assert assigned.loc[0] == 3 + context.tables["person"].loc[0, "receives_main"] = False + unassigned = REPORTING_INCOME_DECILE.run(context).columns["family", "income_decile"] + assert unassigned.loc[0] == 1 + + +@pytest.mark.parametrize("bad", [None, 1, "yes"]) +def test_receipt_invalid_flags_refused_even_when_higher_priority_matches(bad): + context = _context(_node(REPORTING_RECEIPT_GROUP)) + context.tables["person"]["receives_main"] = context.tables["person"][ + "receives_main" + ].astype(object) + context.tables["person"].loc[2, "receives_main"] = bad + with pytest.raises(ValueError, match="boolean flags"): + REPORTING_RECEIPT_GROUP.run(context) + + +def test_receipt_missing_flag_is_refused(): + context = _context(_node(REPORTING_RECEIPT_GROUP)) + del context.tables["person"]["receives_veteran"] + with pytest.raises(ValueError, match="receives_veteran.*missing"): + REPORTING_RECEIPT_GROUP.run(context) + + +@pytest.mark.parametrize( + "kernel", + [REPORTING_INCOME_DECILE, REPORTING_CARRY_INCOME_DECILE, REPORTING_RECEIPT_GROUP], +) +def test_reporting_refuses_wrong_owned_output_dtype(kernel): + node = _node(kernel) + node = replace(node, outputs=(replace(node.outputs[0], dtype="float64"),)) + artifacts = ( + {"baseline_cells": _artifact(_baseline_payload())} + if kernel is REPORTING_CARRY_INCOME_DECILE + else None + ) + with pytest.raises(ValueError, match="must own exactly"): + kernel.run(_context(node, artifacts=artifacts)) + + +@pytest.mark.parametrize( + "kernel", [REPORTING_INCOME_DECILE, REPORTING_CARRY_INCOME_DECILE] +) +@pytest.mark.parametrize("kind", [None, WeightKind.DESIGN]) +def test_income_refuses_missing_or_noncalibrated_source_weights(kernel, kind): + node = _node(kernel) + context = _context( + node, + artifacts={"baseline_cells": _artifact(_baseline_payload())} + if kernel is REPORTING_CARRY_INCOME_DECILE + else None, + ) + context.weights.clear() + if kind is not None: + context.weights["household"] = Weights(np.ones(40), kind) + with pytest.raises(ValueError, match="effective weights|calibrated"): + kernel.run(context) + + +def test_recomputing_deciles_on_variant_is_refused(): + with pytest.raises(ValueError, match="only computes.*baseline"): + REPORTING_INCOME_DECILE.run( + _context(_node(REPORTING_INCOME_DECILE, scenario="V1")) + ) + + +@pytest.mark.parametrize( + "field,value", + [ + ("baseline_scenario", "S1"), + ("entity", "household"), + ("unit_id_column", "household_id"), + ("n_bins", 11), + ("classifier_sha256", "c" * 64), + ], +) +def test_carry_refuses_artifact_classifier_mismatch(field, value): + document = decode_income_cells(_baseline_payload()) + document[field] = value + with pytest.raises(ValueError, match="disagree.*classifier"): + REPORTING_CARRY_INCOME_DECILE.run( + _context( + _node(REPORTING_CARRY_INCOME_DECILE, scenario="V1"), + artifacts={"baseline_cells": _artifact(canonical_json(document))}, + ) + ) + + +@pytest.mark.parametrize( + "change", + [ + "missing_id", + "different_id", + "duplicate_id", + "out_of_range", + "float_label", + "bad_digest", + ], +) +def test_carry_refuses_invalid_cells_and_changed_population(change): + document = decode_income_cells(_baseline_payload()) + if change == "missing_id": + document["unit_ids"].pop() + document["deciles"].pop() + elif change == "different_id": + document["unit_ids"][-1] = 400 + elif change == "duplicate_id": + document["unit_ids"][-1] = 38 + elif change == "out_of_range": + document["deciles"][0] = 0 + elif change == "float_label": + document["deciles"][0] = 1.0 + else: + document["classifier_sha256"] = "invalid" + with pytest.raises(ValueError, match="same unit ids|unique sorted|digest"): + REPORTING_CARRY_INCOME_DECILE.run( + _context( + _node(REPORTING_CARRY_INCOME_DECILE, scenario="V1"), + artifacts={"baseline_cells": _artifact(canonical_json(document))}, + ) + ) + + +def test_carry_refuses_wrong_typed_artifact(): + node = _node(REPORTING_CARRY_INCOME_DECILE, scenario="V1") + with pytest.raises(ValueError, match="type"): + REPORTING_CARRY_INCOME_DECILE.run( + _context( + node, + artifacts={ + "baseline_cells": _artifact( + _baseline_payload(), kind=ArtifactType("other", 1) + ) + }, + ) + ) + + +class _Create(KernelBase): + ref = "test.reporting_create@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, structural=StructuralDelta.CREATE + ) + + def run(self, context): + declaration = json.loads(context.sources["fixture"].read_text()) + if declaration != {"n_units": 40}: + raise ValueError("Unsupported reporting fixture population.") + return KernelResult(frame=_frame(weight_kind=WeightKind.DESIGN)) + + +class _Reweight(KernelBase): + ref = "test.reporting_reweight@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, structural=StructuralDelta.REWEIGHT + ) + + def run(self, context): + values = context.weights["household"].values.copy() + if context.params["variant"]: + values *= np.arange(1, len(values) + 1) + return KernelResult(weights=Weights(values, WeightKind.CALIBRATED)) + + +class _Redraw(KernelBase): + ref = "test.reporting_redraw@1" + capabilities = Capabilities(Determinism.DETERMINISTIC) + + def run(self, context): + person = context.tables["person"] + ids = pd.Index(person["person_id"], name="person_id") + return KernelResult( + columns={ + ("person", "market"): pd.Series( + -person["market"].to_numpy(), index=ids + ), + ("person", "receives_main"): pd.Series( + np.zeros(len(person), dtype=bool), index=ids + ), + } + ) + + +def test_native_cross_branch_artifact_preserves_baseline_cells_for_every_scenario( + tmp_path, +): + declaration = json.loads( + (_PATHS.package / "src/microcosm/build/nz/scenarios.json").read_text() + ) + frame = _frame(weight_kind=WeightKind.DESIGN) + structural = { + ("person", "person_id"), + ("person", "person_household_id"), + ("person", "person_family_id"), + ("household", "household_id"), + ("family", "family_id"), + } + create = Node( + "open", + _Create.ref, + sources=("fixture",), + structural=StructuralDelta.CREATE, + outputs=tuple( + Owned(entity, column, str(frame.table(entity)[column].dtype)) + for entity in _ENTITIES + for column in frame.table(entity) + if (entity, column) not in structural + ), + ) + nodes = [create] + for row in declaration["scenarios"]: + name = row["id"] + variant = row["tier"] == "calibration" + nodes.append( + Node( + name, + _Reweight.ref, + structural=StructuralDelta.REWEIGHT, + base="open", + inputs=(Slice("household", ("source_household_id",)),), + params={"variant": variant}, + weights=WeightTransition("household", "calibrated", mass="free"), + mass="free", + ) + ) + if variant: + nodes.append( + Node( + f"{name}.redraw", + _Redraw.ref, + population=name, + inputs=(Slice("person", ("market", "receives_main")),), + outputs=( + Owned("person", "market", "float64", rewrite=True), + Owned("person", "receives_main", "bool", rewrite=True), + ), + ) + ) + nodes.extend( + [ + _node( + REPORTING_INCOME_DECILE + if name == "S0" + else REPORTING_CARRY_INCOME_DECILE, + population=name, + scenario=name, + ), + _node(REPORTING_RECEIPT_GROUP, population=name, scenario=name), + ] + ) + registry = register_reporting_kernels(KernelRegistry()) + for kernel in (_Create(), _Reweight(), _Redraw()): + registry.register(kernel) + graph = Graph( + country="toy", + sources=(SourceRef("fixture", "raw-bytes-v1"),), + nodes=tuple(nodes), + ) + fixture_source = tmp_path / "population.json" + fixture_source.write_bytes(canonical_json({"n_units": 40})) + store = ContentStore(tmp_path / "store") + manifest = run_graph( + compile_graph(graph), + sources={"fixture": fixture_source}, + store=store, + kernels=registry, + ) + baseline = manifest.populations["S0"].table("family").set_index("family_id") + cells = reporting_cell_declarations( + declaration["gap_conventions"]["reporting_cells"] + ) + assert len(cells) == 43 + assert ( + len(set(zip(baseline.income_decile, baseline.family_type, strict=True))) == 40 + ) + assert set(baseline.receipt_group) == {"beneficiary", "superannuitant", "other"} + assert set(manifest.populations) == { + "open", + *[row["id"] for row in declaration["scenarios"]], + } + for row in declaration["scenarios"]: + name = row["id"] + current = manifest.populations[name].table("family").set_index("family_id") + pd.testing.assert_series_equal(current.income_decile, baseline.income_decile) + receipt = manifest.nodes[f"{name}.receipt_group"].receipt + assert receipt["scenario"] == name + assert receipt["mixed_units"] == (0 if row["tier"] == "calibration" else 10) + if row["tier"] == "calibration": + weights = manifest.populations[name].resolve_weights("family").values + np.testing.assert_array_equal(weights, np.arange(1, 41)) + shares = manifest.nodes[f"{name}.carry_income_decile"].receipt[ + "weighted_shares" + ] + assert shares == pytest.approx( + [sum(range(4 * i + 1, 4 * i + 5)) / 820 for i in range(10)] + ) + assert shares != pytest.approx([0.1] * 10) + assert set(current.receipt_group) == {"superannuitant", "other"} + + +def test_reporting_registration_and_declarative_entitlement_composer(tmp_path): + registry = register_reporting_kernels(KernelRegistry()) + register_reporting_kernels(registry) + for kernel in ( + REPORTING_INCOME_DECILE, + REPORTING_CARRY_INCOME_DECILE, + REPORTING_RECEIPT_GROUP, + ): + assert registry.get(kernel.ref) is kernel + assert kernel.implementation_hash() == kernel.implementation_hash() + fixture = make_entitlement_fixture(tmp_path / "fixture", variants=False) + for kernel in ( + REPORTING_INCOME_DECILE, + REPORTING_CARRY_INCOME_DECILE, + REPORTING_RECEIPT_GROUP, + ): + assert fixture.registry.kernels.get(kernel.ref) is kernel + spec = copy.deepcopy(fixture.spec) + classifier = _receipt_spec() + classifier["predicates"]["superannuitant"]["columns"] = ["receives_pension"] + spec["resources"]["entitlement_graph"]["nodes"].append( + { + "id": "toy.reporting_receipt", + "kernel": REPORTING_RECEIPT_GROUP.ref, + "population": "nz.scn.S0", + "inputs": [ + {"entity": "person", "columns": ["receives_main", "receives_pension"]} + ], + "outputs": [ + {"entity": "family", "column": "receipt_group", "dtype": "string"} + ], + "params": { + "classifier": classifier, + "classifier_sha256": sha256(canonical_json(classifier)).hexdigest(), + "scenario": "S0", + }, + } + ) + graph = compose_transport_graph(spec, fixture.config) + assert graph.node("toy.reporting_receipt").kernel == REPORTING_RECEIPT_GROUP.ref diff --git a/packages/microcosm-frame/src/microcosm/frame/reporting.py b/packages/microcosm-frame/src/microcosm/frame/reporting.py new file mode 100644 index 0000000000..88d5b0a2a3 --- /dev/null +++ b/packages/microcosm-frame/src/microcosm/frame/reporting.py @@ -0,0 +1,429 @@ +"""Reporting strata from declared income, unit weights and assigned receipt. + +Policy choices belong to the classifier declaration. These helpers do not +infer programs, tax treatment or annualisation from column names. A variant +uses :func:`carry_baseline_deciles` to preserve the baseline's unit membership, +then reports its own weighted cell shares. +""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from math import fsum +from numbers import Integral, Real + +import numpy as np +import pandas as pd + +from microcosm.frame.schema import EntitySchema +from microcosm.frame.weights import WeightKind + +__all__ = [ + "annual_unit_income", + "carry_baseline_deciles", + "classify_receipt_groups", + "decile_weighted_shares", + "validate_income_classifier", + "validate_receipt_classifier", + "weighted_deciles", +] + + +def _text(value, name): + if not isinstance(value, str) or not value.strip(): + raise ValueError(f"{name} must be nonempty text.") + return value + + +def _mapping(value, fields, name): + if not isinstance(value, Mapping) or set(value) != fields: + raise ValueError(f"{name} requires exactly {sorted(fields)}.") + return dict(value) + + +def _sequence(value, name): + if isinstance(value, str | bytes) or not isinstance(value, Sequence): + raise ValueError(f"{name} must be a sequence.") + return list(value) + + +def _bin_count(value): + if isinstance(value, bool) or not isinstance(value, Integral) or value < 1: + raise ValueError("n_bins must be a positive integer.") + return int(value) + + +def _common(spec): + for name in ("entity", "person_entity", "unit_id_column", "membership_column"): + _text(spec[name], name) + if spec["entity"] == spec["person_entity"]: + raise ValueError("The reporting entity must differ from the person entity.") + + +def validate_income_classifier(spec: Mapping) -> dict: + """Validate the complete executable annual-gross-income declaration.""" + + result = _mapping( + spec, + { + "entity", + "person_entity", + "unit_id_column", + "membership_column", + "weight_entity", + "expected_weight_kind", + "tie_key", + "boundary_scenario", + "n_bins", + "rank_method", + "components", + "excluded_columns", + }, + "Income classifier", + ) + _common(result) + for name in ("weight_entity", "tie_key", "boundary_scenario"): + _text(result[name], name) + if result["expected_weight_kind"] not in {kind.value for kind in WeightKind}: + raise ValueError("expected_weight_kind must declare a known weight kind.") + result["n_bins"] = _bin_count(result["n_bins"]) + if result["rank_method"] != "weighted_midpoint": + raise ValueError("rank_method must be weighted_midpoint.") + excluded = [] + for row in _sequence(result["excluded_columns"], "excluded_columns"): + row = _mapping(row, {"entity", "column"}, "Excluded income column") + excluded.append( + (_text(row["entity"], "excluded entity"), _text(row["column"], "column")) + ) + if len(set(excluded)) != len(excluded): + raise ValueError("Excluded income columns must be distinct.") + components = _sequence(result["components"], "components") + if not components: + raise ValueError("Income components must be nonempty.") + seen = set() + validated = [] + for component in components: + if not isinstance(component, Mapping): + raise ValueError("Each income component must be an object.") + fields = {"entity", "column", "kind", "tax_basis", "annualisation_factor"} + if component.get("kind") == "benefit": + fields.add("receipt_column") + component = _mapping(component, fields, "Income component") + entity = _text(component["entity"], "component entity") + column = _text(component["column"], "component column") + if entity not in {result["entity"], result["person_entity"]}: + raise ValueError( + "Income components must belong to persons or reporting units." + ) + if component["kind"] not in {"market", "benefit"}: + raise ValueError("Income component kind must be market or benefit.") + if component["tax_basis"] != "before_tax": + raise ValueError( + "Every income component must be before_tax; net income is refused." + ) + factor = component["annualisation_factor"] + if ( + isinstance(factor, bool) + or not isinstance(factor, Real) + or not np.isfinite(factor) + or factor <= 0 + ): + raise ValueError("annualisation_factor must be finite and positive.") + key = (entity, column) + if key in excluded: + raise ValueError(f"Excluded income component {entity}.{column} is refused.") + if key in seen: + raise ValueError("Income components must be distinct.") + seen.add(key) + if component["kind"] == "benefit": + _text(component["receipt_column"], "receipt_column") + validated.append(component) + result["components"] = validated + return result + + +def validate_receipt_classifier(spec: Mapping) -> dict: + """Validate the shared member predicates and their reporting priority.""" + + result = _mapping( + spec, + { + "entity", + "person_entity", + "unit_id_column", + "membership_column", + "predicates", + "priority", + "default", + }, + "Receipt classifier", + ) + _common(result) + _text(result["default"], "default") + predicates = result["predicates"] + if not isinstance(predicates, Mapping) or not predicates: + raise ValueError("Receipt predicates must be a nonempty object.") + validated = {} + for name, predicate in predicates.items(): + _text(name, "predicate name") + predicate = _mapping( + predicate, {"entity", "columns", "reduction"}, "Receipt predicate" + ) + if predicate["entity"] != result["person_entity"]: + raise ValueError("Receipt predicates must read the declared person entity.") + if predicate["reduction"] != "any": + raise ValueError("Receipt predicates must use any-member reduction.") + columns = _sequence(predicate["columns"], "predicate columns") + if not columns or any(not isinstance(c, str) or not c.strip() for c in columns): + raise ValueError("Receipt predicate columns must be nonempty text.") + if len(set(columns)) != len(columns): + raise ValueError("Receipt predicate columns must be distinct.") + validated[name] = predicate + priority = _sequence(result["priority"], "priority") + if ( + not all(isinstance(name, str) for name in priority) + or len(set(priority)) != len(priority) + or set(priority) != set(validated) + or result["default"] in validated + ): + raise ValueError( + "Priority must name every predicate once and exclude the default." + ) + result["predicates"] = validated + return result + + +def _integer_ids(values, name, *, unique=True): + values = np.asarray(values, dtype=object) + if values.ndim != 1 or any( + isinstance(value, bool | np.bool_) or not isinstance(value, Integral) + for value in values + ): + raise ValueError(f"{name} must contain nonmissing integer ids.") + if unique and len(set(values)) != len(values): + raise ValueError(f"{name} must contain unique ids.") + return values + + +def _tie_ids(values, size): + values = np.asarray(values, dtype=object) + if values.ndim != 1 or len(values) != size: + raise ValueError("Tie ids must have one id per unit.") + integers = all( + isinstance(v, Integral) and not isinstance(v, bool | np.bool_) for v in values + ) + strings = all(isinstance(v, str) and v for v in values) + if not (integers or strings) or len(set(values)) != size: + raise ValueError("Tie ids must be unique integers or unique nonempty strings.") + return values + + +def _numeric(values, name): + source = np.asarray(values) + if source.ndim != 1 or source.dtype.kind not in "iuf": + raise ValueError(f"{name} must be a finite numeric vector.") + result = source.astype(np.float64) + if not np.isfinite(result).all(): + raise ValueError(f"{name} must be a finite numeric vector.") + return result + + +def _weights(values, size): + weights = _numeric(values, "Weights") + if len(weights) != size or (weights < 0).any() or not (weights > 0).any(): + raise ValueError( + "Weights must align to units, be nonnegative and have positive mass." + ) + return weights + + +def _column(tables, entity, column): + if entity not in tables or column not in tables[entity]: + raise ValueError(f"Required classifier column {entity}.{column} is missing.") + return tables[entity][column] + + +def _relations(tables, spec): + units = _integer_ids( + _column(tables, spec["entity"], spec["unit_id_column"]), "Unit ids" + ) + schema = EntitySchema( + person_entity=spec["person_entity"], group_entities=(spec["entity"],) + ) + person_ids = _integer_ids( + _column(tables, spec["person_entity"], schema.person_id_column), "Person ids" + ) + membership = _integer_ids( + _column(tables, spec["person_entity"], spec["membership_column"]), + "Unit memberships", + unique=False, + ) + rows = pd.Index(units).get_indexer(membership) + if (rows < 0).any(): + raise ValueError("Unit memberships contain orphan ids.") + if len(units) == 0 or (np.bincount(rows, minlength=len(units)) == 0).any(): + raise ValueError("Every reporting unit must have at least one member.") + order = np.lexsort((person_ids, rows)) + starts = np.flatnonzero(np.r_[True, np.diff(rows[order]) != 0]) + return units, rows, order, starts + + +def _assigned_flags(values, name): + flags = values.to_numpy(dtype=object) + if any(not isinstance(value, bool | np.bool_) for value in flags): + raise ValueError( + f"Assigned receipt {name} must contain boolean flags without missing values." + ) + return flags.astype(bool) + + +def annual_unit_income(tables: Mapping[str, pd.DataFrame], spec: Mapping) -> np.ndarray: + """Annual before-tax assigned-receipt income in reporting-unit row order. + + Person components sum in stable person-id order within each unit. Benefit + amounts are nonnegative engine payments multiplied by assigned receipt; + market income can include losses. Neither entitlement nor the reporting + outcome is implicitly included. + """ + + spec = validate_income_classifier(spec) + units, _, order, starts = _relations(tables, spec) + contributions = [] + try: + with np.errstate(over="raise", invalid="raise"): + for component in spec["components"]: + entity, column = component["entity"], component["column"] + values = _numeric(_column(tables, entity, column), f"{entity}.{column}") + if component["kind"] == "benefit": + if (values < 0).any(): + raise ValueError("Benefit payments must be nonnegative.") + receipt_column = component["receipt_column"] + flags = _assigned_flags( + _column(tables, entity, receipt_column), + f"{entity}.{receipt_column}", + ) + values = values * flags + values = values * component["annualisation_factor"] + if entity == spec["person_entity"]: + values = np.add.reduceat(values[order], starts) + contributions.append(values) + result = np.sum(contributions, axis=0) + except FloatingPointError as error: + raise ValueError("Annual unit income must remain finite.") from error + if len(result) != len(units) or not np.isfinite(result).all(): + raise ValueError("Annual unit income must remain finite and align to unit ids.") + return result + + +def decile_weighted_shares(labels, weights, n_bins) -> list[float]: + """Weighted shares for fixed unit cells under the current scenario weights.""" + + n_bins = _bin_count(n_bins) + labels = _integer_ids(labels, "Decile labels", unique=False) + if any(label < 1 or label > n_bins for label in labels): + raise ValueError("Decile labels must lie within the declared bins.") + weights = _weights(weights, len(labels)) + scaled = weights / weights.max() + total = fsum(scaled) + return [fsum(scaled[labels == label]) / total for label in range(1, n_bins + 1)] + + +def weighted_deciles(incomes, weights, tie_ids, n_bins, *, rank_method): + """Assign individual units by their sorted cumulative-weight midpoints. + + Sort by income then the declared unique tie id. A unit's rank is the + weight preceding it plus half its own weight, divided by total weight. + Bins are left-closed: a midpoint exactly on a boundary enters the higher + bin. Zero-weight units use the cumulative mass at their position, clipped + to the final bin at rank one. An all-zero population is refused. + """ + + n_bins = _bin_count(n_bins) + if rank_method != "weighted_midpoint": + raise ValueError("rank_method must be weighted_midpoint.") + incomes = _numeric(incomes, "Incomes") + weights = _weights(weights, len(incomes)) + tie_ids = _tie_ids(tie_ids, len(incomes)) + order = np.lexsort((tie_ids, incomes)) + scaled = weights[order].astype(np.longdouble) / np.max(weights) + total = np.sum(scaled, dtype=np.longdouble) + midpoint = (np.cumsum(scaled, dtype=np.longdouble) - scaled / 2) / total + positions = midpoint * n_bins + boundaries = np.rint(positions) + # Rescaling float64 input weights can move a theoretical boundary by + # rounding error. Preserve the left-closed convention at that precision. + roundoff = np.finfo(np.float64).eps * n_bins * 4 + positions = np.where( + np.abs(positions - boundaries) <= roundoff, boundaries, positions + ) + sorted_labels = np.minimum(np.floor(positions).astype(np.int64) + 1, n_bins) + labels = np.empty(len(incomes), dtype=np.int64) + labels[order] = sorted_labels + return labels, { + "rank_method": rank_method, + "n_units": len(labels), + "zero_weight_units": int(np.count_nonzero(weights == 0)), + "weighted_shares": decile_weighted_shares(labels, weights, n_bins), + } + + +def carry_baseline_deciles( + unit_ids, baseline_ids, baseline_deciles, n_bins +) -> np.ndarray: + """Align fixed baseline cells by id, refusing population changes or invalid cells.""" + + n_bins = _bin_count(n_bins) + unit_ids = _integer_ids(unit_ids, "Variant unit ids") + baseline_ids = _integer_ids(baseline_ids, "Baseline unit ids") + deciles = _integer_ids(baseline_deciles, "Baseline deciles", unique=False) + if len(baseline_ids) != len(deciles) or any( + value < 1 or value > n_bins for value in deciles + ): + raise ValueError( + "Baseline deciles must align to ids and lie in the declared bins." + ) + if not len(unit_ids) or set(unit_ids) != set(baseline_ids): + raise ValueError("Variant and baseline must contain exactly the same unit ids.") + rows = pd.Index(baseline_ids).get_indexer(unit_ids) + return deciles[rows].astype(np.int64) + + +def classify_receipt_groups(tables: Mapping[str, pd.DataFrame], spec: Mapping): + """Classify units by declared any-member assigned receipt and priority. + + Mixed units satisfy more than one reporting predicate. Every input flag + is validated even if a higher-priority predicate already selects a unit. + """ + + spec = validate_receipt_classifier(spec) + units, _, order, starts = _relations(tables, spec) + predicates = {} + for name, predicate in spec["predicates"].items(): + flags = [ + _assigned_flags( + _column(tables, predicate["entity"], column), + f"{predicate['entity']}.{column}", + ) + for column in predicate["columns"] + ] + person_match = np.logical_or.reduce(flags) + predicates[name] = np.logical_or.reduceat(person_match[order], starts) + groups = np.full(len(units), spec["default"], dtype=object) + for name in reversed(spec["priority"]): + groups[predicates[name]] = name + result = pd.Series( + groups, index=pd.Index(units, name=spec["unit_id_column"]), name="receipt_group" + ) + return result, { + "n_units": len(units), + "mixed_units": int( + np.count_nonzero(np.sum(list(predicates.values()), axis=0) > 1) + ), + "group_counts": { + name: int(np.count_nonzero(groups == name)) + for name in [*spec["priority"], spec["default"]] + }, + "predicate_counts": { + name: int(np.count_nonzero(mask)) for name, mask in predicates.items() + }, + } diff --git a/packages/microcosm-frame/tests/engine_free/shared/test_reporting.py b/packages/microcosm-frame/tests/engine_free/shared/test_reporting.py new file mode 100644 index 0000000000..f873142bae --- /dev/null +++ b/packages/microcosm-frame/tests/engine_free/shared/test_reporting.py @@ -0,0 +1,538 @@ +"""Unit reporting classifiers using declared inputs and fixed baseline cells.""" + +from copy import deepcopy + +import numpy as np +import pandas as pd +import pytest +from hypothesis import example, given, settings +from hypothesis import strategies as st + +from microcosm.frame.reporting import ( + annual_unit_income, + carry_baseline_deciles, + classify_receipt_groups, + decile_weighted_shares, + validate_income_classifier, + validate_receipt_classifier, + weighted_deciles, +) + + +def _income_spec(): + return { + "entity": "family", + "person_entity": "person", + "unit_id_column": "family_id", + "membership_column": "person_family_id", + "weight_entity": "household", + "expected_weight_kind": "calibrated", + "tie_key": "family_id", + "boundary_scenario": "baseline", + "n_bins": 10, + "rank_method": "weighted_midpoint", + "components": [ + { + "entity": "person", + "column": "market_income", + "kind": "market", + "tax_basis": "before_tax", + "annualisation_factor": 1, + }, + { + "entity": "family", + "column": "capital_income", + "kind": "market", + "tax_basis": "before_tax", + "annualisation_factor": 1, + }, + { + "entity": "person", + "column": "main_payment", + "kind": "benefit", + "tax_basis": "before_tax", + "annualisation_factor": 52, + "receipt_column": "receives_main", + }, + { + "entity": "person", + "column": "pension_payment", + "kind": "benefit", + "tax_basis": "before_tax", + "annualisation_factor": 52, + "receipt_column": "receives_pension", + }, + ], + "excluded_columns": [ + {"entity": "family", "column": "housing_assistance"}, + {"entity": "family", "column": "tax_credit"}, + {"entity": "person", "column": "main_net_payment"}, + ], + } + + +def _receipt_spec(): + return { + "entity": "family", + "person_entity": "person", + "unit_id_column": "family_id", + "membership_column": "person_family_id", + "predicates": { + "superannuitant": { + "entity": "person", + "columns": ["receives_pension", "receives_veteran"], + "reduction": "any", + }, + "beneficiary": { + "entity": "person", + "columns": ["receives_main"], + "reduction": "any", + }, + }, + "priority": ["superannuitant", "beneficiary"], + "default": "other", + } + + +def _tables(): + return { + "family": pd.DataFrame( + { + "family_id": [100, 200, 300, 400], + "family_source_household_id": [77, 77, 88, 99], + "capital_income": [-25, 0, 10, 0], + "housing_assistance": [999, 999, 999, 999], + "tax_credit": [999, 999, 999, 999], + } + ), + "person": pd.DataFrame( + { + "person_id": [7, 8, 9, 10, 11], + "person_family_id": [100, 100, 200, 300, 400], + "market_income": [500, 250, -20, 0, 1000], + "main_payment": [100, 200, 50, 40, 10], + "main_net_payment": [80, 160, 40, 32, 8], + "pension_payment": [100, 200, 50, 300, 10], + "receives_main": [False, True, True, False, False], + "receives_pension": [True, False, False, True, False], + "receives_veteran": [False, False, False, False, False], + } + ), + } + + +def _deciles(incomes, weights, ids=None, bins=10): + if ids is None: + ids = np.arange(len(incomes)) + return weighted_deciles( + incomes, weights, ids, bins, rank_method="weighted_midpoint" + ) + + +def test_declared_income_uses_gross_assigned_amounts_and_annual_factors(): + tables = _tables() + before = deepcopy(tables) + np.testing.assert_array_equal( + annual_unit_income(tables, _income_spec()), [16_325, 2580, 15_610, 1000] + ) + for entity in tables: + pd.testing.assert_frame_equal(tables[entity], before[entity]) + + +def test_annual_income_person_permutation_has_stable_summation(): + tables = _tables() + spec = _income_spec() + spec["components"] = spec["components"][:1] + tables["person"] = pd.DataFrame( + { + "person_id": [10, 20, 30, 40, 50, 60], + "person_family_id": [100, 100, 100, 200, 300, 400], + "market_income": [1e16, -1e16, 1, 2, 3, 4], + } + ) + expected = annual_unit_income(tables, spec) + tables["person"] = tables["person"].iloc[[2, 0, 5, 4, 3, 1]] + np.testing.assert_array_equal(annual_unit_income(tables, spec), expected) + + +def test_declared_unit_benefit_amounts_are_supported(): + tables = _tables() + spec = _income_spec() + component = dict(spec["components"][2], entity="family") + spec["components"] = [component] + tables["family"]["main_payment"] = [100, 200, 300, 400] + tables["family"]["receives_main"] = [True, False, True, False] + np.testing.assert_array_equal( + annual_unit_income(tables, spec), [5200, 0, 15_600, 0] + ) + + +def test_exact_weighted_midpoint_boundaries_and_uneven_weights(): + labels, diagnostics = _deciles([0, 1, 2, 3, 4], [1, 1, 2, 2, 4]) + np.testing.assert_array_equal(labels, [1, 2, 4, 6, 9]) + np.testing.assert_allclose( + diagnostics["weighted_shares"], [0.1, 0.1, 0, 0.2, 0, 0.2, 0, 0, 0.4, 0] + ) + assert diagnostics["zero_weight_units"] == 0 + np.testing.assert_array_equal(_deciles([0, 1], [2, 8])[0], [2, 7]) + + +def test_boundary_ties_split_by_stable_unit_id(): + labels, diagnostics = _deciles( + [100, 100, 100, 100], [1, 1, 1, 1], [40, 10, 30, 20], 2 + ) + np.testing.assert_array_equal(labels, [2, 1, 2, 1]) + assert diagnostics["weighted_shares"] == [0.5, 0.5] + + +def test_zero_weights_use_their_sorted_boundary_position(): + labels, diagnostics = _deciles([-1, 0, 1, 2, 3], [0, 1, 0, 1, 0]) + np.testing.assert_array_equal(labels, [1, 3, 6, 8, 10]) + assert diagnostics["zero_weight_units"] == 3 + assert diagnostics["weighted_shares"] == [0, 0, 0.5, 0, 0, 0, 0, 0.5, 0, 0] + + +def test_single_unit_has_midpoint_cell_and_reports_full_share(): + labels, diagnostics = _deciles([0], [17], [10]) + np.testing.assert_array_equal(labels, [6]) + assert diagnostics["weighted_shares"][5] == 1 + + +def test_all_equal_income_units_fill_every_decile_by_id(): + ids = np.arange(10)[::-1] + labels, diagnostics = _deciles(np.zeros(10), np.ones(10), ids) + np.testing.assert_array_equal(labels, np.arange(1, 11)[::-1]) + np.testing.assert_allclose(diagnostics["weighted_shares"], np.full(10, 0.1)) + + +def test_cloned_source_households_are_separate_reporting_units(): + tables = _tables() + units = tables["family"] + assert units["family_source_household_id"].tolist()[:2] == [77, 77] + labels, _ = _deciles([0, 0, 10, 20], [2, 2, 2, 2], units["family_id"], 4) + np.testing.assert_array_equal(labels, [1, 2, 3, 4]) + + +@st.composite +def _weighted_samples(draw): + rows = draw( + st.lists( + st.tuples(st.integers(-100, 100), st.integers(0, 100)), + min_size=1, + max_size=30, + ).filter(lambda rows: any(weight > 0 for _, weight in rows)) + ) + incomes, weights = np.array(rows).T + ids = np.arange(len(rows)) * 17 - 35 + permutation = np.array(draw(st.permutations(range(len(rows))))) + return incomes, weights, ids, permutation + + +@settings(max_examples=100, deadline=None, database=None) +@given(_weighted_samples()) +def test_weighted_deciles_permutation_invariance(sample): + incomes, weights, ids, permutation = sample + labels, diagnostics = _deciles(incomes, weights, ids) + permuted, reordered_diagnostics = _deciles( + incomes[permutation], weights[permutation], ids[permutation] + ) + np.testing.assert_array_equal(permuted, labels[permutation]) + assert reordered_diagnostics == diagnostics + + +@settings(max_examples=100, deadline=None, database=None) +@example( + sample=(np.arange(3), np.array([3, 2, 5]), np.arange(3), np.arange(3)), scale=0.3 +) +@example( + sample=(np.arange(3), np.array([1, 3, 6]), np.arange(3), np.arange(3)), scale=0.1 +) +@example( + sample=(np.arange(3), np.array([1, 3, 6]), np.arange(3), np.arange(3)), scale=1.1 +) +@given( + _weighted_samples(), + st.floats(min_value=0.125, max_value=1000, allow_nan=False, allow_infinity=False), +) +def test_weighted_deciles_weight_scale_invariance(sample, scale): + incomes, weights, ids, _ = sample + labels, diagnostics = _deciles(incomes, weights, ids) + scaled, scaled_diagnostics = _deciles(incomes, weights * scale, ids) + np.testing.assert_array_equal(scaled, labels) + np.testing.assert_allclose( + scaled_diagnostics["weighted_shares"], + diagnostics["weighted_shares"], + rtol=1e-14, + atol=1e-14, + ) + + +def test_variant_reweight_carries_baseline_cells_by_unit_id(): + ids = np.array([10, 20, 30, 40, 50]) + baseline, _ = _deciles([0, 1, 2, 3, 4], [1, 1, 2, 2, 4], ids) + variant_ids = ids[::-1] + variant_weights = np.array([1, 1, 1, 1, 96]) + variant = carry_baseline_deciles(variant_ids, ids, baseline, 10) + np.testing.assert_array_equal(variant, baseline[::-1]) + recomputed, _ = _deciles([900, 800, 700, 600, 500], variant_weights, variant_ids) + assert not np.array_equal(recomputed, variant) + shares = decile_weighted_shares(variant, variant_weights, 10) + assert shares[0] == pytest.approx(0.96) + assert shares[8] == pytest.approx(0.01) + + +@pytest.mark.parametrize( + "unit_ids,baseline_ids,labels", + [ + ([1, 2], [1, 3], [1, 2]), + ([1, 2], [1, 2, 3], [1, 2, 3]), + ([1, 1], [1, 2], [1, 2]), + ([1, 2], [1, 1], [1, 2]), + ([1, 2], [1, 2], [1]), + ([1, 2], [1, 2], [0, 1]), + ([1, 2], [1, 2], [1, 11]), + ([1.0, 2], [1, 2], [1, 2]), + ([1, 2], [1, 2], [True, 2]), + ([], [], []), + ], +) +def test_baseline_carry_refuses_population_or_label_changes( + unit_ids, baseline_ids, labels +): + with pytest.raises(ValueError): + carry_baseline_deciles(unit_ids, baseline_ids, labels, 10) + + +def test_receipt_priority_member_reduction_and_mixed_unit_count(): + groups, diagnostics = classify_receipt_groups(_tables(), _receipt_spec()) + assert groups.index.tolist() == [100, 200, 300, 400] + assert groups.tolist() == [ + "superannuitant", + "beneficiary", + "superannuitant", + "other", + ] + assert diagnostics["mixed_units"] == 1 + assert diagnostics["group_counts"] == { + "superannuitant": 2, + "beneficiary": 1, + "other": 1, + } + assert diagnostics["predicate_counts"] == {"superannuitant": 2, "beneficiary": 2} + + +def test_veterans_receipt_satisfies_the_same_superannuitant_predicate(): + tables = _tables() + tables["person"]["receives_pension"] = False + tables["person"].loc[4, "receives_veteran"] = True + groups, diagnostics = classify_receipt_groups(tables, _receipt_spec()) + assert groups.tolist() == ["beneficiary", "beneficiary", "other", "superannuitant"] + assert diagnostics["mixed_units"] == 0 + + +def test_receipt_priority_is_spec_data(): + spec = _receipt_spec() + spec["priority"] = ["beneficiary", "superannuitant"] + groups, diagnostics = classify_receipt_groups(_tables(), spec) + assert groups.loc[100] == "beneficiary" + assert diagnostics["mixed_units"] == 1 + + +@pytest.mark.parametrize("value", [None, pd.NA, np.nan, 0, 1, "False", [], 0.5]) +def test_receipt_invalid_flags_refused_even_in_preempted_group(value): + tables = _tables() + tables["person"]["receives_main"] = tables["person"]["receives_main"].astype(object) + tables["person"].at[0, "receives_main"] = value + with pytest.raises(ValueError, match="boolean flags"): + classify_receipt_groups(tables, _receipt_spec()) + with pytest.raises(ValueError, match="boolean flags"): + annual_unit_income(tables, _income_spec()) + + +def test_missing_receipt_flag_is_refused(): + tables = _tables() + del tables["person"]["receives_main"] + with pytest.raises(ValueError, match="receives_main.*missing"): + classify_receipt_groups(tables, _receipt_spec()) + with pytest.raises(ValueError, match="receives_main.*missing"): + annual_unit_income(tables, _income_spec()) + + +@pytest.mark.parametrize( + "column", ["main_net_payment", "housing_assistance", "tax_credit"] +) +def test_excluded_net_or_outcome_income_components_refused(column): + spec = _income_spec() + entity = "person" if column == "main_net_payment" else "family" + spec["components"].append( + { + "entity": entity, + "column": column, + "kind": "market", + "tax_basis": "before_tax", + "annualisation_factor": 1, + } + ) + with pytest.raises(ValueError, match="Excluded income component"): + annual_unit_income(_tables(), spec) + + +def test_net_tax_basis_refused_in_a_gross_income_mix(): + spec = _income_spec() + spec["components"][2]["tax_basis"] = "after_tax" + with pytest.raises(ValueError, match="before_tax.*net income"): + annual_unit_income(_tables(), spec) + + +@pytest.mark.parametrize("value", [np.nan, np.inf, -np.inf, "100", True]) +def test_nonfinite_or_nonnumeric_income_refused(value): + tables = _tables() + tables["person"]["market_income"] = tables["person"]["market_income"].astype(object) + tables["person"].at[0, "market_income"] = value + with pytest.raises(ValueError, match="finite numeric"): + annual_unit_income(tables, _income_spec()) + + +def test_negative_benefit_payments_are_refused(): + tables = _tables() + tables["person"].loc[0, "main_payment"] = -1 + with pytest.raises(ValueError, match="nonnegative"): + annual_unit_income(tables, _income_spec()) + + +@pytest.mark.parametrize( + "consumer,spec", + [(annual_unit_income, _income_spec), (classify_receipt_groups, _receipt_spec)], +) +@pytest.mark.parametrize( + "mutation", + [ + "duplicate_unit", + "duplicate_person", + "orphan", + "missing_membership", + "empty_unit", + "float_id", + ], +) +def test_invalid_unit_identity_or_memberships_refused(consumer, spec, mutation): + tables = _tables() + if mutation == "duplicate_unit": + tables["family"].loc[1, "family_id"] = 100 + elif mutation == "duplicate_person": + tables["person"].loc[1, "person_id"] = 7 + elif mutation == "orphan": + tables["person"].loc[0, "person_family_id"] = 999 + elif mutation == "missing_membership": + tables["person"].loc[0, "person_family_id"] = np.nan + elif mutation == "empty_unit": + tables["person"] = tables["person"].iloc[:-1] + else: + tables["family"]["family_id"] = tables["family"]["family_id"].astype(float) + with pytest.raises(ValueError): + consumer(tables, spec()) + + +@pytest.mark.parametrize( + "incomes,weights,ids,bins,method", + [ + ([0], [0], [1], 10, "weighted_midpoint"), + ([0, 1], [1, -1], [1, 2], 10, "weighted_midpoint"), + ([0, 1], [1], [1, 2], 10, "weighted_midpoint"), + ([0], [np.inf], [1], 10, "weighted_midpoint"), + ([np.nan], [1], [1], 10, "weighted_midpoint"), + ([0, 1], [1, 1], [1, 1], 10, "weighted_midpoint"), + ([0, 1], [1, 1], [1, "two"], 10, "weighted_midpoint"), + ([0], [1], [None], 10, "weighted_midpoint"), + ([0], [1], [1], 0, "weighted_midpoint"), + ([0], [1], [1], True, "weighted_midpoint"), + ([0], [1], [1], 10, "unweighted"), + ], +) +def test_invalid_decile_inputs_refused(incomes, weights, ids, bins, method): + with pytest.raises(ValueError): + weighted_deciles(incomes, weights, ids, bins, rank_method=method) + + +@pytest.mark.parametrize("factor", [0, -1, np.nan, np.inf, True, "52"]) +def test_invalid_annualisation_factor_refused(factor): + spec = _income_spec() + spec["components"][2]["annualisation_factor"] = factor + with pytest.raises(ValueError, match="annualisation_factor"): + validate_income_classifier(spec) + + +@pytest.mark.parametrize( + "change", + [ + "unknown", + "benefit_missing_receipt", + "market_receipt", + "duplicate", + "invalid_kind", + "empty", + "weight_kind", + "rank_method", + ], +) +def test_malformed_income_spec_refused(change): + spec = _income_spec() + if change == "unknown": + spec["unknown"] = True + elif change == "benefit_missing_receipt": + del spec["components"][2]["receipt_column"] + elif change == "market_receipt": + spec["components"][0]["receipt_column"] = "receives_main" + elif change == "duplicate": + spec["components"].append(spec["components"][0]) + elif change == "invalid_kind": + spec["components"][0]["kind"] = "entitlement" + elif change == "empty": + spec["components"] = [] + elif change == "weight_kind": + spec["expected_weight_kind"] = "unknown" + else: + spec["rank_method"] = "unweighted" + with pytest.raises(ValueError): + validate_income_classifier(spec) + + +@pytest.mark.parametrize( + "change", + [ + "unknown", + "no_predicates", + "duplicate_priority", + "missing_priority", + "bad_reduction", + "bad_entity", + "empty_columns", + "duplicate_columns", + "default_is_predicate", + ], +) +def test_malformed_receipt_spec_refused(change): + spec = _receipt_spec() + if change == "unknown": + spec["unknown"] = True + elif change == "no_predicates": + spec["predicates"] = {} + elif change == "duplicate_priority": + spec["priority"] = ["superannuitant", "superannuitant"] + elif change == "missing_priority": + spec["priority"] = ["superannuitant"] + elif change == "bad_reduction": + spec["predicates"]["beneficiary"]["reduction"] = "all" + elif change == "bad_entity": + spec["predicates"]["beneficiary"]["entity"] = "family" + elif change == "empty_columns": + spec["predicates"]["beneficiary"]["columns"] = [] + elif change == "duplicate_columns": + spec["predicates"]["beneficiary"]["columns"] = [ + "receives_main", + "receives_main", + ] + else: + spec["default"] = "superannuitant" + with pytest.raises(ValueError): + validate_receipt_classifier(spec) From bced1c25384e7ead1f0e5980a7bed8f3d9317a9e Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 12 Oct 2026 11:06:59 -0400 Subject: [PATCH 2/4] Regenerate the NZ golden country spec for the reporting classifiers The as_rate_bridge, gates and country_package resource hashes move, the new reporting_classifiers resource is added, and the fingerprint changes; each hash equals sha256 of the committed file. Co-Authored-By: Claude Opus 5.5 --- .../microcosm-build/tests/golden/nz_country_spec.json | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/microcosm-build/tests/golden/nz_country_spec.json b/packages/microcosm-build/tests/golden/nz_country_spec.json index 0a9c751b51..1eeb647c29 100644 --- a/packages/microcosm-build/tests/golden/nz_country_spec.json +++ b/packages/microcosm-build/tests/golden/nz_country_spec.json @@ -1,6 +1,6 @@ { "country": "nz", - "fingerprint": "7c7064be95a241a4757246ff9de728ad75c2501c162ae2232214c39414cba61d", + "fingerprint": "a1a8179e64c1953cc5a7ebcc1bf63d700801b70394dfa79dddc74fea5405d32c", "gate_ids": [ "calibration_per_family_fit", "national_and_region_admin_aggregates", @@ -38,18 +38,19 @@ }, "resource_hashes": { "as_area_crosswalk.json": "cfb76c99898204598b0609e5462b2fba9d081872a13ac5dd973cb406bb6f7469", - "as_rate_bridge.json": "5eea897159508fb71820f22cd8ccf4993075a750462bd97312d9e89ab01cba15", + "as_rate_bridge.json": "2dad25ba1ed46f37b2ca4fb75f0d99d6e8d36a46b40233ca51fce762e3655d0d", "axiom_input_closure.json": "c3c164f437b05d598af7e604d2cb46365a8323a8043da49b08c5e0cdf32e2ec1", "axiom_rules_bindings.json": "72905af77132ff6a3c01ada4b6210be4c369d9c6dc49dc8c1b673e4bac0b5fb1", "benefit_unit_rule.json": "b4ed95161177668964ffcc39556c14600e2e08bda350be9ea5f52e83cc3afd41", - "country_package.json": "6c4c1d19a0666958a4dc1079e5363084a347e817fc0c9cca59eada45bdc18873", + "country_package.json": "d1059312ee47721961978394aa3a754d55fbc8e6c2802ddc755b4a1dcfee920d", "currency_bridge.json": "aa6a4b286a6a1a015088f32ac013148553422c3d45eeef7d7a15af08388857ca", "export_contract.json": "2c7414fefe395c4f5822643dcd3bb3544ebb2040259aa007b828a99bd01ae210", - "gates.json": "18a78dde49b6b979bf46039fa1d779bf30362b853514739e66bd669c2aaebeba", + "gates.json": "13e5ea49330571f30ac65cfb286908448df685a6d81e39c1d7d38824678eb7cf", "geography_spine.json": "353821844d85d2622c328b4986dd9e3a8075c13498b9f56d349c4578edf7495a", "holdout_references.json": "52b5a2360502156da1392a68350e4e7f61b5fd420fa74428819b65886a9f8655", "precal_references.json": "43b13849e3d74ebca7d7a9d46817463e3ad1333c3aedd7a3eac39e870510fba0", "release_contract.json": "6cd5e3a4a2bf30e4025e04e964c393b24040937d13db7e6a07b3966a55f3b114", + "reporting_classifiers.json": "1de71e406021c5f22dcdf40a14bb5509fb21b5ebcf261c8397329007299e2e2f", "scenarios.json": "30f96f3063773d77e3c70119c8f5d99d3528eca87428903da00ca071ab34cfaf", "source_stages.json": "cc82c68878ac0fc07ecbd796c1a66de6f34786ef8785125046425c37f26aff31", "spec/bundle.yaml": "25ef74ecbc00c3f2ecca2d30ddd5e46a582657863c217da33ee9a396569d2cf2", @@ -79,6 +80,7 @@ "holdout_references.json", "precal_references.json", "release_contract.json", + "reporting_classifiers.json", "scenarios.json", "source_stages.json", "target_references.json" From 70f1706cebe1f0fefa1d197c7390121a3cceec6b Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 13 Oct 2026 03:26:12 -0400 Subject: [PATCH 3/4] Pin the reg 17/18 source and drop the last local citation path The A2 method note now cites the pinned axiom-corpus release, path and sha256 of the SSR 2018 PCO XML it was read from. The MC11 predicate citation in as_rate_bridge.json points to the in-repo method note instead of a local absolute path, and the tie-key evidence names build_benefit_units rather than stale line numbers. NZ golden regenerated for the two resource hashes. Co-Authored-By: Claude Opus 5.5 --- docs/nz-reporting-classifiers-method.md | 6 +++++- .../src/microcosm/build/nz/as_rate_bridge.json | 2 +- .../src/microcosm/build/nz/reporting_classifiers.json | 2 +- packages/microcosm-build/tests/golden/nz_country_spec.json | 6 +++--- 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/docs/nz-reporting-classifiers-method.md b/docs/nz-reporting-classifiers-method.md index 3343883881..39cb0a93e4 100644 --- a/docs/nz-reporting-classifiers-method.md +++ b/docs/nz-reporting-classifiers-method.md @@ -69,7 +69,11 @@ who share a home and moving those units upward in the income ranking. ## A2 Receipt group The legal basis is the Social Security Regulations 2018 (LI 2018/202), read for -this method as at 5 April 2026. [Regulation 17, PCO +this method as at 5 April 2026. The text read is the PCO XML pinned in +`TheAxiomFoundation/axiom-corpus` release `nz-rulespec-2026-07-25`, at +`data/corpus/sources/nz/regulation/2026-06-16-rulespec-nz-pco/regulation/public/2018/0202/wholeof.xml` +(sha256 `5f1250b903ce1b618eebd6370d6db8c51cb377b9fdb5cf4d171e259cd0e85d8f`). The +legislation.govt.nz links below open the latest version. [Regulation 17, PCO LMS96264](https://www.legislation.govt.nz/regulation/public/2018/0202/latest/LMS96264.html) defines a beneficiary by payment of a main benefit, New Zealand Superannuation, or a veteran's pension. Everyone else is a non-beneficiary for that regulation. diff --git a/packages/microcosm-build/src/microcosm/build/nz/as_rate_bridge.json b/packages/microcosm-build/src/microcosm/build/nz/as_rate_bridge.json index 19ed1cd33b..567a4e119b 100644 --- a/packages/microcosm-build/src/microcosm/build/nz/as_rate_bridge.json +++ b/packages/microcosm-build/src/microcosm/build/nz/as_rate_bridge.json @@ -429,7 +429,7 @@ }, "predicate_citation": { "item": "A2", - "source": "/Users/maxghenis/reviews/nz-hub/notes/1011/g8-method-addendum.md" + "source": "docs/nz-reporting-classifiers-method.md#a2-receipt-group" }, "predicate_note": "MC11 exclude-and-count and reporting must select this one assigned-member receipt predicate. A bridge implementation must resolve the reference rather than infer superannuitants from entitlement or age." }, diff --git a/packages/microcosm-build/src/microcosm/build/nz/reporting_classifiers.json b/packages/microcosm-build/src/microcosm/build/nz/reporting_classifiers.json index d1c3931b9e..9ca1423e8c 100644 --- a/packages/microcosm-build/src/microcosm/build/nz/reporting_classifiers.json +++ b/packages/microcosm-build/src/microcosm/build/nz/reporting_classifiers.json @@ -191,7 +191,7 @@ "tie_key_evidence": { "stability": "Same pinned donor and unit rule preserve person ids across rebuilds; each family id is its head person id. Units sharing source household lineage remain distinct units.", "implementation": [ - "packages/microcosm-frame/src/microcosm/frame/unit_construction.py:388-409", + "packages/microcosm-frame/src/microcosm/frame/unit_construction.py build_benefit_units (each family id is its head person's id)", "packages/microcosm-build/src/microcosm/build/transport/population_kernels.py:209-228", "packages/microcosm-frame/src/microcosm/frame/transport.py:843-856" ], diff --git a/packages/microcosm-build/tests/golden/nz_country_spec.json b/packages/microcosm-build/tests/golden/nz_country_spec.json index f218f96ac5..c1a498971a 100644 --- a/packages/microcosm-build/tests/golden/nz_country_spec.json +++ b/packages/microcosm-build/tests/golden/nz_country_spec.json @@ -1,6 +1,6 @@ { "country": "nz", - "fingerprint": "9dbec94ad8adc301129436ce33add8c285d7a339bd2be3660ba54bf5d4b317d1", + "fingerprint": "a41318afb8914164bf7abc6d11757a4fff32f3c529f68207d850ce1c3762a9c5", "gate_ids": [ "calibration_per_family_fit", "national_and_region_admin_aggregates", @@ -38,7 +38,7 @@ }, "resource_hashes": { "as_area_crosswalk.json": "2c296c9204efc37e8e4798d5ca23945e5b025830277a02ebad642725762f4783", - "as_rate_bridge.json": "2dad25ba1ed46f37b2ca4fb75f0d99d6e8d36a46b40233ca51fce762e3655d0d", + "as_rate_bridge.json": "c6e21af7b090283431973a5610cc3adf5d783a4a01fb0051d5dc6c8347eeabb0", "axiom_input_closure.json": "c3c164f437b05d598af7e604d2cb46365a8323a8043da49b08c5e0cdf32e2ec1", "axiom_rules_bindings.json": "72905af77132ff6a3c01ada4b6210be4c369d9c6dc49dc8c1b673e4bac0b5fb1", "benefit_unit_rule.json": "5c1cf16daaae482ea3ec53495f4c2e4ae452d928c6240a5284dbbf30cd2dfe37", @@ -50,7 +50,7 @@ "holdout_references.json": "52b5a2360502156da1392a68350e4e7f61b5fd420fa74428819b65886a9f8655", "precal_references.json": "43b13849e3d74ebca7d7a9d46817463e3ad1333c3aedd7a3eac39e870510fba0", "release_contract.json": "6cd5e3a4a2bf30e4025e04e964c393b24040937d13db7e6a07b3966a55f3b114", - "reporting_classifiers.json": "1df7c97a98cbe2b38089d4f53596487202bb7a4b672fe289172bb793122f4d68", + "reporting_classifiers.json": "b5af9e4737af4717d86aedfe651df8b3c18c3875763726e7a6a0b9b1168ed133", "scenarios.json": "30f96f3063773d77e3c70119c8f5d99d3528eca87428903da00ca071ab34cfaf", "source_stages.json": "cc82c68878ac0fc07ecbd796c1a66de6f34786ef8785125046425c37f26aff31", "spec/bundle.yaml": "25ef74ecbc00c3f2ecca2d30ddd5e46a582657863c217da33ee9a396569d2cf2", From 8eb7d88fecd44b4332567a52080a66d1b626963c Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 13 Oct 2026 04:41:29 -0400 Subject: [PATCH 4/4] List the NZ reporting_classifiers resource in the generation-zero view test The expected legacy_json set in test_nz_generation_zero_views_come_from_ the_country_spec_seam predates the new nz/reporting_classifiers.json resource (the same gap CI found for #1249's chronicle_feed). Co-Authored-By: Claude Opus 5.5 --- .../tests/engine_free/shared/test_spec_engine_country_bundles.py | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_country_bundles.py b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_country_bundles.py index f39358e700..dce12e27eb 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_country_bundles.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_country_bundles.py @@ -232,6 +232,7 @@ def test_nz_generation_zero_views_come_from_the_country_spec_seam() -> None: "holdout_references.json", "precal_references.json", "release_contract.json", + "reporting_classifiers.json", "scenarios.json", "source_stages.json", "target_references.json",