From 9a42184dc5d0ab5f6749c314503841b9062795fd Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 14 Oct 2026 01:38:21 -0400 Subject: [PATCH 1/5] Reset negative HBAI household income to zero before housing costs HBAI: "Negative incomes BHC are reset to zero, but negative AHC incomes calculated from the adjusted BHC incomes are possible." (Background Information and Methodology report FYE 2025.) - hbai_household_net_income is now max(0, the new hbai_household_net_income_before_reset_to_zero), which holds the old formula. AHC, equivalised income, deciles, poverty flags and gaps follow from the reset figure. - household_income_decile no longer gives -1 to negative incomes: the reset figure is never negative, and HBAI ranks those households in the bottom decile. - Labour supply responses measure the figure before the reset, the household's actual budget constraint. Fixes #2401 Co-Authored-By: Claude Opus 5.5 --- .../hbai-negative-income-reset.fixed.md | 1 + policyengine_uk/dynamics/labour_supply.py | 16 +- policyengine_uk/dynamics/progression.py | 10 +- .../income/hbai_negative_income.yaml | 162 ++++++++++++++++++ .../tests/test_household_income_decile.py | 21 ++- .../household/income/deep_poverty_gap.py | 6 +- .../income/hbai_household_net_income.py | 23 ++- .../income/hbai_household_net_income_ahc.py | 9 + ...usehold_net_income_before_reset_to_zero.py | 37 ++++ .../income/household_income_decile.py | 13 +- .../income/in_relative_poverty_ahc.py | 6 +- .../income/in_relative_poverty_bhc.py | 6 +- .../variables/household/income/poverty_gap.py | 6 +- .../household/income/poverty_gap_ahc.py | 5 + .../household/income/poverty_gap_bhc.py | 4 + 15 files changed, 285 insertions(+), 40 deletions(-) create mode 100644 changelog.d/hbai-negative-income-reset.fixed.md create mode 100644 policyengine_uk/tests/policy/baseline/household/income/hbai_negative_income.yaml create mode 100644 policyengine_uk/variables/household/income/hbai_household_net_income_before_reset_to_zero.py diff --git a/changelog.d/hbai-negative-income-reset.fixed.md b/changelog.d/hbai-negative-income-reset.fixed.md new file mode 100644 index 0000000000..b6860605b8 --- /dev/null +++ b/changelog.d/hbai-negative-income-reset.fixed.md @@ -0,0 +1 @@ +- HBAI household net income before housing costs now resets a negative figure to zero, as HBAI does; income after housing costs, deciles, poverty flags and poverty gaps follow from the reset figure. The figure before the reset is the new `hbai_household_net_income_before_reset_to_zero`, which the labour supply responses now use. diff --git a/policyengine_uk/dynamics/labour_supply.py b/policyengine_uk/dynamics/labour_supply.py index 9949f8b1a6..38a65086e6 100644 --- a/policyengine_uk/dynamics/labour_supply.py +++ b/policyengine_uk/dynamics/labour_supply.py @@ -96,7 +96,7 @@ class LabourSupplyResponseData(BaseModel): def apply_labour_supply_responses( sim: Simulation, - target_variable: str = "hbai_household_net_income", + target_variable: str = "hbai_household_net_income_before_reset_to_zero", input_variable: str = "employment_income", year: int = 2025, count_adults: int = 2, @@ -110,7 +110,11 @@ def apply_labour_supply_responses( Args: sim: PolicyEngine simulation object (should have baseline attribute) - target_variable: Variable that drives labour supply decisions + target_variable: Variable that drives labour supply decisions. The + default is HBAI net income before HBAI resets a negative income + to zero: the household's actual budget constraint. The reset is + a statistical convention; measured on the reset figure, a pay + rise for a household below zero would look fully withdrawn. input_variable: Variable representing labour supply (typically employment_income) year: Year for calculation count_adults: Number of adults to calculate responses for @@ -194,7 +198,7 @@ def apply_labour_supply_responses( def apply_progression_responses( sim: Simulation, - target_variable: str = "hbai_household_net_income", + target_variable: str = "hbai_household_net_income_before_reset_to_zero", input_variable: str = "employment_income", year: int = 2025, count_adults: int = 2, @@ -208,7 +212,11 @@ def apply_progression_responses( Args: sim: PolicyEngine simulation object (should have baseline attribute) - target_variable: Variable that drives labour supply decisions + target_variable: Variable that drives labour supply decisions. The + default is HBAI net income before HBAI resets a negative income + to zero: the household's actual budget constraint. The reset is + a statistical convention; measured on the reset figure, a pay + rise for a household below zero would look fully withdrawn. input_variable: Variable representing labour supply (typically employment_income) year: Year for calculation count_adults: Number of adults to calculate responses for diff --git a/policyengine_uk/dynamics/progression.py b/policyengine_uk/dynamics/progression.py index 58cf7cfc1e..28416e018e 100644 --- a/policyengine_uk/dynamics/progression.py +++ b/policyengine_uk/dynamics/progression.py @@ -16,7 +16,7 @@ def calculate_derivative( sim: Simulation, - target_variable: str = "hbai_household_net_income", + target_variable: str = "hbai_household_net_income_before_reset_to_zero", input_variable: str = "employment_income", year: int = 2025, count_adults: int = 2, @@ -30,7 +30,9 @@ def calculate_derivative( Args: sim: PolicyEngine simulation object - target_variable: Variable to measure changes in (typically household_net_income) + target_variable: Variable to measure changes in (by default HBAI net + income before a negative income is reset to zero, so that the + rate reflects the household's actual budget constraint) input_variable: Variable to change (typically employment_income) year: Year for calculation count_adults: Number of adults to apply changes to @@ -89,7 +91,7 @@ def calculate_derivative( def calculate_relative_income_change( sim: Simulation, - target_variable: str = "hbai_household_net_income", + target_variable: str = "hbai_household_net_income_before_reset_to_zero", year: int = 2025, ) -> pd.DataFrame: """Calculate relative change in income between baseline and scenario. @@ -135,7 +137,7 @@ def calculate_relative_income_change( def calculate_derivative_change( sim: Simulation, - target_variable: str = "hbai_household_net_income", + target_variable: str = "hbai_household_net_income_before_reset_to_zero", input_variable: str = "employment_income", year: int = 2025, count_adults: int = 2, diff --git a/policyengine_uk/tests/policy/baseline/household/income/hbai_negative_income.yaml b/policyengine_uk/tests/policy/baseline/household/income/hbai_negative_income.yaml new file mode 100644 index 0000000000..2e1bd94f09 --- /dev/null +++ b/policyengine_uk/tests/policy/baseline/household/income/hbai_negative_income.yaml @@ -0,0 +1,162 @@ +# HBAI resets a negative household income before housing costs to zero and +# derives income after housing costs from the reset figure, which can be +# negative: "Negative incomes BHC are reset to zero, but negative AHC incomes +# calculated from the adjusted BHC incomes are possible. Where incomes have +# been adjusted to zero BHC, income AHC is derived from the adjusted BHC +# income." (HBAI Background Information and Methodology report FYE 2025, +# "Negative incomes"). +# +# The cases use 2024 (FYE 2025), where the absolute poverty lines are HBAI's +# published FYE 2025 lines: for a single adult 15,040.08 a year before housing +# costs and 11,276.52 after (see poverty/hbai_family_type_lines.yaml). +# Universal Credit is not claimed, so the household's only income is what the +# case gives it. + +- name: Deductions above income leave a negative figure that is reset to zero + period: 2024 + absolute_error_margin: 0.1 + input: + people: + adult: + age: 40 + maintenance_expenses: 5_000 + benunits: + benunit: + members: [adult] + would_claim_uc: false + households: + household: + members: [adult] + council_tax: 1_500 + rent: 6_000 + tenure_type: RENT_PRIVATELY + output: + hbai_household_net_income_before_reset_to_zero: -6_500 + hbai_household_net_income: 0 + real_hbai_household_net_income: 0 + equiv_hbai_household_net_income: 0 + # Income after housing costs is the reset figure less rent: 0 - 6,000. + hbai_household_net_income_ahc: -6_000 + # Divided by the 0.58 AHC equivalence factor for a single adult. + equiv_hbai_household_net_income_ahc: -10_344.83 + # The before-housing-costs gap is the whole line, never more. + poverty_gap_bhc: 15_040.08 + poverty_gap: 15_040.08 + deep_poverty_gap: 7_520.04 + # The after-housing-costs gap is the line plus the negative income. + poverty_gap_ahc: 17_276.52 + in_poverty_bhc: true + in_poverty_ahc: true + in_deep_poverty_bhc: true + in_deep_poverty_ahc: true + in_poverty: true + in_deep_poverty: true + +- name: An owner-occupier's housing costs come off the reset income + period: 2024 + absolute_error_margin: 0.1 + input: + people: + adult: + age: 40 + maintenance_expenses: 3_000 + benunits: + benunit: + members: [adult] + would_claim_uc: false + households: + household: + members: [adult] + council_tax: 1_000 + tenure_type: OWNED_WITH_MORTGAGE + mortgage_interest_repayment: 2_000 + water_and_sewerage_charges: 400 + structural_insurance_payments: 100 + output: + hbai_household_net_income_before_reset_to_zero: -4_000 + hbai_household_net_income: 0 + hbai_household_net_income_ahc: -2_500 + +- name: A negative figure given directly is reset; housing costs still come off + period: 2024 + absolute_error_margin: 0.1 + input: + people: + adult: + age: 40 + benunits: + benunit: + members: [adult] + households: + household: + members: [adult] + hbai_household_net_income_before_reset_to_zero: -25_000 + rent: 4_800 + tenure_type: RENT_PRIVATELY + output: + hbai_household_net_income: 0 + hbai_household_net_income_ahc: -4_800 + poverty_gap_bhc: 15_040.08 + poverty_gap_ahc: 16_076.52 + +- name: A figure of exactly zero stays zero + period: 2024 + absolute_error_margin: 0.1 + input: + people: + adult: + age: 40 + benunits: + benunit: + members: [adult] + households: + household: + members: [adult] + hbai_household_net_income_before_reset_to_zero: 0 + output: + hbai_household_net_income: 0 + hbai_household_net_income_ahc: 0 + +- name: A positive income is unchanged by the reset + period: 2024 + absolute_error_margin: 0.1 + input: + people: + adult: + age: 40 + benunits: + benunit: + members: [adult] + households: + household: + members: [adult] + hbai_household_net_income_before_reset_to_zero: 12_345 + rent: 5_000 + tenure_type: RENT_PRIVATELY + output: + hbai_household_net_income: 12_345 + hbai_household_net_income_ahc: 7_345 + # 12,345 is below the 15,040.08 line: a gap of 2,695.08. + poverty_gap_bhc: 2_695.08 + +- name: Earnings above deductions give the same income with or without the reset + period: 2024 + absolute_error_margin: 0.1 + input: + people: + adult: + age: 40 + employment_income: 20_000 + benunits: + benunit: + members: [adult] + would_claim_uc: false + households: + household: + members: [adult] + council_tax: 1_500 + output: + # 20,000 less Income Tax of 20% x (20,000 - 12,570) = 1,486, Class 1 NICs + # of 8% x (20,000 - 12,570) = 594.40 and Council Tax of 1,500. + hbai_household_net_income_before_reset_to_zero: 16_419.60 + hbai_household_net_income: 16_419.60 diff --git a/policyengine_uk/tests/test_household_income_decile.py b/policyengine_uk/tests/test_household_income_decile.py index 4baa0f085e..49b4bc0804 100644 --- a/policyengine_uk/tests/test_household_income_decile.py +++ b/policyengine_uk/tests/test_household_income_decile.py @@ -32,20 +32,27 @@ def test_household_income_decile_caps_at_ten(): assert result.max() == 10 -def test_household_income_decile_keeps_negative_incomes_outside_deciles(): +def test_household_income_decile_ranks_incomes_reset_to_zero_in_the_bottom_decile(): + # HBAI resets a negative income to zero before ranking, so + # equiv_hbai_household_net_income is never negative and a household whose + # income was reset sits in the bottom decile, not outside the deciles. Two + # of twenty equally weighted households have their income reset: a tenth + # of the population, so both fill the bottom decile. household = FakeHousehold( { - "equiv_hbai_household_net_income": np.array([-5, 0, 10], dtype=float), - "household_count_people": np.ones(3), - "household_weight": np.ones(3), + "equiv_hbai_household_net_income": np.concatenate( + [[0.0, 0.0], np.arange(1, 19, dtype=float) * 1_000] + ), + "household_count_people": np.ones(20), + "household_weight": np.ones(20), } ) result = household_income_decile.formula(household, 2025, None) - assert result[0] == -1 - assert np.all(result[1:] >= 1) - assert np.all(result[1:] <= 10) + assert result[0] == result[1] == 1 + assert np.all(result >= 1) + assert np.all(result <= 10) def test_pre_budget_change_income_decile_domain_is_negative_or_one_to_ten(): diff --git a/policyengine_uk/variables/household/income/deep_poverty_gap.py b/policyengine_uk/variables/household/income/deep_poverty_gap.py index 610bf7793a..15bcf4804c 100644 --- a/policyengine_uk/variables/household/income/deep_poverty_gap.py +++ b/policyengine_uk/variables/household/income/deep_poverty_gap.py @@ -3,7 +3,11 @@ class deep_poverty_gap(Variable): label = "deep poverty gap" - documentation = "The financial gap between net household income and the deep poverty line (before housing costs)." + documentation = ( + "The financial gap between net household income and the deep poverty " + "line (before housing costs). Income is HBAI income with a negative " + "figure reset to zero, so the gap is at most the deep poverty line." + ) entity = Household definition_period = YEAR value_type = float diff --git a/policyengine_uk/variables/household/income/hbai_household_net_income.py b/policyengine_uk/variables/household/income/hbai_household_net_income.py index 0dc95db9e3..f95899a98a 100644 --- a/policyengine_uk/variables/household/income/hbai_household_net_income.py +++ b/policyengine_uk/variables/household/income/hbai_household_net_income.py @@ -73,22 +73,21 @@ class hbai_household_net_income(Variable): value_type = float entity = Household label = "Household net income (HBAI definition)" - documentation = "Disposable income for the household, following the definition used for official poverty statistics" + documentation = ( + "Disposable income for the household before housing costs, following " + "the definition used for official poverty statistics. As in HBAI, a " + 'negative income is reset to zero ("Negative incomes BHC are reset to ' + 'zero"), so this is never negative; the figure before the reset is ' + "hbai_household_net_income_before_reset_to_zero. Income after housing " + "costs is derived from this reset figure and can be negative." + ) unit = GBP definition_period = YEAR + reference = "https://www.gov.uk/government/statistics/households-below-average-income-for-financial-years-ending-1995-to-2025/households-below-average-income-background-information-and-methodology-report-fye-2025#negative-incomes" def formula(household, period, parameters): - abolish_council_tax = parameters.gov.contrib.abolish_council_tax(period) - if abolish_council_tax: - adds = [ - a for a in HBAI_HOUSEHOLD_NET_INCOME_ADDS if a != "council_tax_benefit" - ] - subtracts = [ - s for s in HBAI_HOUSEHOLD_NET_INCOME_SUBTRACTS if s != "council_tax" - ] - return add(household, period, adds) - add(household, period, subtracts) - return add(household, period, HBAI_HOUSEHOLD_NET_INCOME_ADDS) - add( - household, period, HBAI_HOUSEHOLD_NET_INCOME_SUBTRACTS + return max_( + 0, household("hbai_household_net_income_before_reset_to_zero", period) ) diff --git a/policyengine_uk/variables/household/income/hbai_household_net_income_ahc.py b/policyengine_uk/variables/household/income/hbai_household_net_income_ahc.py index f1116ac76a..cd28ea29b5 100644 --- a/policyengine_uk/variables/household/income/hbai_household_net_income_ahc.py +++ b/policyengine_uk/variables/household/income/hbai_household_net_income_ahc.py @@ -7,9 +7,18 @@ class hbai_household_net_income_ahc(Variable): value_type = float entity = Household label = "Household net income, after housing costs" + documentation = ( + "HBAI household net income after housing costs. As in HBAI, it is " + "derived from the before-housing-costs income after a negative figure " + 'has been reset to zero, and it can itself be negative: "negative AHC ' + 'incomes calculated from the adjusted BHC incomes are possible".' + ) definition_period = YEAR unit = GBP + reference = "https://www.gov.uk/government/statistics/households-below-average-income-for-financial-years-ending-1995-to-2025/households-below-average-income-background-information-and-methodology-report-fye-2025#negative-incomes" + # The reset before-housing-costs income, not + # hbai_household_net_income_before_reset_to_zero. adds = ["hbai_household_net_income"] subtracts = [ "rent", diff --git a/policyengine_uk/variables/household/income/hbai_household_net_income_before_reset_to_zero.py b/policyengine_uk/variables/household/income/hbai_household_net_income_before_reset_to_zero.py new file mode 100644 index 0000000000..7d988cd7e2 --- /dev/null +++ b/policyengine_uk/variables/household/income/hbai_household_net_income_before_reset_to_zero.py @@ -0,0 +1,37 @@ +from policyengine_uk.model_api import * +from policyengine_uk.variables.household.income.hbai_household_net_income import ( + HBAI_HOUSEHOLD_NET_INCOME_ADDS, + HBAI_HOUSEHOLD_NET_INCOME_SUBTRACTS, +) + + +class hbai_household_net_income_before_reset_to_zero(Variable): + value_type = float + entity = Household + label = "Household net income (HBAI definition) before a negative income is reset to zero" + documentation = ( + "HBAI household net income before housing costs, before HBAI resets a " + "negative figure to zero. It can be negative when deductions such as " + "Council Tax, Income Tax, pension contributions and maintenance paid " + "exceed income. hbai_household_net_income is this figure reset to zero " + "when negative. Use this variable for a household's actual budget " + "constraint (marginal rates, labour supply responses) and to diagnose " + "negative incomes." + ) + unit = GBP + definition_period = YEAR + reference = "https://www.gov.uk/government/statistics/households-below-average-income-for-financial-years-ending-1995-to-2025/households-below-average-income-background-information-and-methodology-report-fye-2025#negative-incomes" + + def formula(household, period, parameters): + abolish_council_tax = parameters.gov.contrib.abolish_council_tax(period) + if abolish_council_tax: + adds = [ + a for a in HBAI_HOUSEHOLD_NET_INCOME_ADDS if a != "council_tax_benefit" + ] + subtracts = [ + s for s in HBAI_HOUSEHOLD_NET_INCOME_SUBTRACTS if s != "council_tax" + ] + return add(household, period, adds) - add(household, period, subtracts) + return add(household, period, HBAI_HOUSEHOLD_NET_INCOME_ADDS) - add( + household, period, HBAI_HOUSEHOLD_NET_INCOME_SUBTRACTS + ) diff --git a/policyengine_uk/variables/household/income/household_income_decile.py b/policyengine_uk/variables/household/income/household_income_decile.py index e68d4c11e3..124650f60a 100644 --- a/policyengine_uk/variables/household/income/household_income_decile.py +++ b/policyengine_uk/variables/household/income/household_income_decile.py @@ -5,7 +5,12 @@ class household_income_decile(Variable): label = "household income decile" - documentation = "Decile of household income (person-weighted)" + documentation = ( + "Decile of equivalised HBAI household net income before housing costs " + "(person-weighted). As in HBAI, a negative income is reset to zero " + "before ranking, so a household whose income was negative before the " + "reset falls in the bottom decile." + ) entity = Household definition_period = YEAR value_type = int @@ -15,8 +20,4 @@ def formula(household, period, parameters): count_people = household("household_count_people", period) household_weight = household("household_weight", period) weighted_income = MicroSeries(income, weights=household_weight * count_people) - decile = weighted_income.decile_rank().values - # Set negatives to -1. - # This avoids the bottom decile summing to a negative number, - # which would flip the % change in the interface. - return where(income < 0, -1, decile) + return weighted_income.decile_rank().values diff --git a/policyengine_uk/variables/household/income/in_relative_poverty_ahc.py b/policyengine_uk/variables/household/income/in_relative_poverty_ahc.py index c2998ad1ed..1c5ed6ed0b 100644 --- a/policyengine_uk/variables/household/income/in_relative_poverty_ahc.py +++ b/policyengine_uk/variables/household/income/in_relative_poverty_ahc.py @@ -7,8 +7,10 @@ class in_relative_poverty_ahc(Variable): "Whether the household's equivalised HBAI net income, after housing " "costs, is below 60% of the contemporary median. The median is taken " "over individuals, as in DWP's Households Below Average Income series, " - "so each household counts once per member. Negative incomes stay in " - "the median and count as below the line. The absolute-line twin is " + "so each household counts once per member. Income after housing " + "costs is derived from the before-housing-costs income reset to zero " + "and can be negative, as in HBAI; negative incomes stay in the median " + "and count as below the line. The absolute-line twin is " "in_poverty_ahc." ) entity = Household diff --git a/policyengine_uk/variables/household/income/in_relative_poverty_bhc.py b/policyengine_uk/variables/household/income/in_relative_poverty_bhc.py index e54006ae7d..09381dc2af 100644 --- a/policyengine_uk/variables/household/income/in_relative_poverty_bhc.py +++ b/policyengine_uk/variables/household/income/in_relative_poverty_bhc.py @@ -7,9 +7,9 @@ class in_relative_poverty_bhc(Variable): "Whether the household's equivalised HBAI net income, before housing " "costs, is below 60% of the contemporary median. The median is taken " "over individuals, as in DWP's Households Below Average Income series, " - "so each household counts once per member. Negative incomes stay in " - "the median and count as below the line. The absolute-line twin is " - "in_poverty_bhc." + "so each household counts once per member. As in HBAI, a negative " + "income is reset to zero; it stays in the median and counts as below " + "the line. The absolute-line twin is in_poverty_bhc." ) entity = Household definition_period = YEAR diff --git a/policyengine_uk/variables/household/income/poverty_gap.py b/policyengine_uk/variables/household/income/poverty_gap.py index b08a2169e3..7b30a7e6ad 100644 --- a/policyengine_uk/variables/household/income/poverty_gap.py +++ b/policyengine_uk/variables/household/income/poverty_gap.py @@ -3,7 +3,11 @@ class poverty_gap(Variable): label = "poverty gap" - documentation = "The financial gap between net household income and the poverty line (before housing costs)." + documentation = ( + "The financial gap between net household income and the poverty line " + "(before housing costs). Income is HBAI income with a negative figure " + "reset to zero, so the gap is at most the poverty line." + ) entity = Household definition_period = YEAR value_type = float diff --git a/policyengine_uk/variables/household/income/poverty_gap_ahc.py b/policyengine_uk/variables/household/income/poverty_gap_ahc.py index bd21d62a00..1c0f5a195e 100644 --- a/policyengine_uk/variables/household/income/poverty_gap_ahc.py +++ b/policyengine_uk/variables/household/income/poverty_gap_ahc.py @@ -5,6 +5,11 @@ class poverty_gap_ahc(Variable): value_type = float entity = Household label = "Positive financial gap between net household income and the poverty line, after housing costs" + documentation = ( + "Income is HBAI income after housing costs, derived as in HBAI from " + "the before-housing-costs income reset to zero. It can be negative, " + "so the gap can exceed the poverty line by up to the housing costs." + ) definition_period = YEAR def formula(household, period, parameters): diff --git a/policyengine_uk/variables/household/income/poverty_gap_bhc.py b/policyengine_uk/variables/household/income/poverty_gap_bhc.py index 6bc516cf24..26952125c7 100644 --- a/policyengine_uk/variables/household/income/poverty_gap_bhc.py +++ b/policyengine_uk/variables/household/income/poverty_gap_bhc.py @@ -5,6 +5,10 @@ class poverty_gap_bhc(Variable): value_type = float entity = Household label = "Positive financial gap between net household income and the poverty line" + documentation = ( + "Income is HBAI income before housing costs with a negative figure " + "reset to zero, as in HBAI, so the gap is at most the poverty line." + ) definition_period = YEAR def formula(household, period, parameters): From b488187a22c4f8bd817d3955ce94c3d1ce23aed8 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 14 Oct 2026 01:48:46 -0400 Subject: [PATCH 2/5] Property-test the HBAI reset and document it - test_hbai_negative_income_properties: the reset, a differential against the components summed outside the model, AHC from the reset figure, unchanged poverty headcounts, gaps measured from the reset, the means identity, deciles 1-10 rising with income, and the labour supply default. Nine planted mutants are each caught. - The PAWHP property checks its accounting identity on HBAI income before the reset, which the reset would break for a negative income. - docs: a "Negative incomes" section on the HBAI page. Co-Authored-By: Claude Opus 5.5 --- docs/book/validation/hbai.md | 9 +- .../test_hbai_negative_income_properties.py | 413 ++++++++++++++++++ .../test_pawhp_household_income_properties.py | 11 +- 3 files changed, 429 insertions(+), 4 deletions(-) create mode 100644 policyengine_uk/tests/test_hbai_negative_income_properties.py diff --git a/docs/book/validation/hbai.md b/docs/book/validation/hbai.md index 4d57f8f8b2..02cb934dbc 100644 --- a/docs/book/validation/hbai.md +++ b/docs/book/validation/hbai.md @@ -6,7 +6,8 @@ PolicyEngine UK includes variables that match to the Households Below Average In These are: -- `hbai_household_net_income`: Household net income, before housing costs +- `hbai_household_net_income`: Household net income, before housing costs, with a negative figure reset to zero +- `hbai_household_net_income_before_reset_to_zero`: The same income before a negative figure is reset to zero - `hbai_household_net_income_ahc`: Household net income, after housing costs - `equiv_hbai_household_net_income`: Equivalised household net income, before housing costs (BHC) - `equiv_hbai_household_net_income_ahc`: Equivalised household net income, after housing costs (AHC) @@ -44,6 +45,12 @@ The table below shows how the HBAI income components map to PolicyEngine variabl | Structural insurance premiums (for owner occupiers) | `structural_insurance_payments` | | Ground rent and service charges |`housing_service_charges`| +## Negative incomes + +HBAI resets a negative income before housing costs to zero: "Negative incomes BHC are reset to zero, but negative AHC incomes calculated from the adjusted BHC incomes are possible. Where incomes have been adjusted to zero BHC, income AHC is derived from the adjusted BHC income." ([HBAI Background Information and Methodology report FYE 2025](https://www.gov.uk/government/statistics/households-below-average-income-for-financial-years-ending-1995-to-2025/households-below-average-income-background-information-and-methodology-report-fye-2025#negative-incomes)). + +PolicyEngine does the same. `hbai_household_net_income` is `hbai_household_net_income_before_reset_to_zero` reset to zero when negative, and `hbai_household_net_income_ahc` subtracts housing costs from the reset figure, so it can be negative. The equivalised incomes, `household_income_decile`, the poverty flags and the poverty gaps all use the reset figure: a household whose income was negative is in the bottom decile and counts as in poverty, and its before-housing-costs poverty gap is the whole poverty line. The labour supply responses use the figure before the reset, which is the household's actual budget constraint. + ## Nowcasting/forecasting diff --git a/policyengine_uk/tests/test_hbai_negative_income_properties.py b/policyengine_uk/tests/test_hbai_negative_income_properties.py new file mode 100644 index 0000000000..028fde0646 --- /dev/null +++ b/policyengine_uk/tests/test_hbai_negative_income_properties.py @@ -0,0 +1,413 @@ +"""HBAI resets a negative household income to zero before housing costs. + +"Negative incomes BHC are reset to zero, but negative AHC incomes calculated +from the adjusted BHC incomes are possible. Where incomes have been adjusted +to zero BHC, income AHC is derived from the adjusted BHC income." (HBAI +Background Information and Methodology report FYE 2025, "Negative incomes".) + +Properties, for any population of households (b is +hbai_household_net_income_before_reset_to_zero, HC the HBAI housing costs: +rent, water and sewerage charges, mortgage interest and structural insurance): + +1. Reset: hbai_household_net_income = max(0, b), so it and its equivalised + and real versions are never negative. +2. Differential: b equals the HBAI components summed outside the model + (HBAI_HOUSEHOLD_NET_INCOME_ADDS less HBAI_HOUSEHOLD_NET_INCOME_SUBTRACTS). +3. After housing costs: hbai_household_net_income_ahc = max(0, b) - HC, which + is negative whenever HC exceeds the reset income. +4. Headcounts: every poverty flag (absolute, deep, relative; BHC and AHC) + equals the flag computed from the income before the reset. Zero is below + any absolute line, and the reset moves no one across the median as long as + fewer than half of people have an income of zero or less before the reset. +5. Gaps: each gap is measured from the reset income, so the BHC gaps never + exceed their line, and every gap is at most the gap before the reset, with + equality when b >= 0. +6. Means: the weighted total of the reset income exceeds that of b by exactly + the weighted sum of the negative parts of b. +7. Deciles: every household has a decile from 1 to 10 (none is left out), and + deciles never fall as equivalised income rises. +8. Labour supply responses measure b, the actual budget constraint: a pay rise + for a household below zero raises b, while the reset income stays at zero. + +Comparisons allow float32 rounding: the model stores values as float32. +""" + +import inspect + +import numpy as np +from hypothesis import HealthCheck, example, given, settings +from hypothesis import strategies as st +from microdf import MicroSeries + +from policyengine_uk import Simulation +from policyengine_uk.dynamics import labour_supply, progression +from policyengine_uk.variables.household.income.hbai_household_net_income import ( + HBAI_HOUSEHOLD_NET_INCOME_ADDS, + HBAI_HOUSEHOLD_NET_INCOME_SUBTRACTS, +) + +BEFORE_RESET = "hbai_household_net_income_before_reset_to_zero" +HOUSING_COSTS = [ + "rent", + "water_and_sewerage_charges", + "mortgage_interest_repayment", + "structural_insurance_payments", +] +YEARS = [2024, 2025, 2026] +PROPERTY_SETTINGS = settings( + max_examples=12, + deadline=None, + derandomize=True, + suppress_health_check=[HealthCheck.too_slow, HealthCheck.data_too_large], +) + + +def money(*values): + return st.sampled_from(values) + + +@st.composite +def households(draw): + renter = draw(st.booleans()) + return { + "adults": [ + { + "age": draw(st.sampled_from([25, 40, 55, 70])), + "employment_income": draw(money(0, 0, 6_000, 20_000, 60_000)), + # Maintenance paid and pension contributions are deducted in + # HBAI and can exceed income, which makes b negative. + "maintenance_expenses": draw(money(0, 0, 4_000, 15_000, 40_000)), + "personal_pension_contributions": draw(money(0, 0, 5_000)), + } + for _ in range(draw(st.integers(1, 2))) + ], + "children": draw(st.integers(0, 2)), + "would_claim_uc": draw(st.booleans()), + "household": { + "council_tax": draw(money(0, 1_200, 2_500)), + "tenure_type": "RENT_PRIVATELY" if renter else "OWNED_WITH_MORTGAGE", + "rent": draw(money(4_000, 9_000, 15_000)) if renter else 0, + "mortgage_interest_repayment": 0 if renter else draw(money(0, 3_000)), + "water_and_sewerage_charges": draw(money(0, 450)), + "structural_insurance_payments": 0 if renter else draw(money(0, 250)), + }, + } + + +@st.composite +def populations(draw): + return { + "year": draw(st.sampled_from(YEARS)), + "households": draw(st.lists(households(), min_size=2, max_size=8)), + } + + +def adult(age, earnings, maintenance=0, pension=0): + return { + "age": age, + "employment_income": earnings, + "maintenance_expenses": maintenance, + "personal_pension_contributions": pension, + } + + +def home(council_tax, rent=0, mortgage=0, water=0, insurance=0): + return { + "council_tax": council_tax, + "tenure_type": "RENT_PRIVATELY" if rent else "OWNED_WITH_MORTGAGE", + "rent": rent, + "mortgage_interest_repayment": mortgage, + "water_and_sewerage_charges": water, + "structural_insurance_payments": insurance, + } + + +# A population the properties always check, whatever Hypothesis draws: two +# households whose deductions exceed their income (one renting, one buying), +# and four with positive incomes, so most people are above zero and the +# relative-poverty properties apply. +PINNED = { + "year": 2024, + "households": [ + { + "adults": [adult(40, 6_000, maintenance=15_000)], + "children": 0, + "would_claim_uc": False, + "household": home(1_200, rent=9_000, water=450), + }, + { + "adults": [adult(55, 0, maintenance=4_000, pension=5_000)], + "children": 1, + "would_claim_uc": False, + "household": home(2_500, mortgage=3_000, insurance=250), + }, + { + "adults": [adult(40, 20_000), adult(40, 60_000)], + "children": 2, + "would_claim_uc": True, + "household": home(2_500, mortgage=3_000), + }, + { + "adults": [adult(25, 20_000)], + "children": 0, + "would_claim_uc": True, + "household": home(1_200, rent=9_000), + }, + { + "adults": [adult(70, 0)], + "children": 0, + "would_claim_uc": True, + "household": home(1_200, rent=4_000), + }, + { + "adults": [adult(55, 60_000)], + "children": 0, + "would_claim_uc": True, + "household": home(2_500), + }, + ], +} + + +def situation(population): + year = population["year"] + people, benunits, groups = {}, {}, {} + for h, household in enumerate(population["households"]): + members = [] + for a, person in enumerate(household["adults"]): + name = f"h{h}_a{a}" + people[name] = {key: {year: value} for key, value in person.items()} + members.append(name) + for c in range(household["children"]): + name = f"h{h}_c{c}" + people[name] = {"age": {year: 8}} + members.append(name) + benunits[f"b{h}"] = { + "members": members, + "would_claim_uc": {year: household["would_claim_uc"]}, + } + groups[f"h{h}"] = { + "members": members, + **{key: {year: value} for key, value in household["household"].items()}, + } + return {"people": people, "benunits": benunits, "households": groups} + + +def household_values(sim, variable, year): + values = np.asarray(sim.calculate(variable, year), dtype=float) + entity = sim.tax_benefit_system.get_variable(variable).entity.key + if entity == "household": + return values + return np.asarray(sim.map_result(values, entity, "household"), dtype=float) + + +def run(population): + year = population["year"] + sim = Simulation(situation=situation(population)) + names = [ + BEFORE_RESET, + "hbai_household_net_income", + "real_hbai_household_net_income", + "inflation_adjustment", + "hbai_household_net_income_ahc", + "equiv_hbai_household_net_income", + "equiv_hbai_household_net_income_ahc", + "household_equivalisation_bhc", + "household_equivalisation_ahc", + "household_weight", + "household_count_people", + "poverty_line_bhc", + "poverty_line_ahc", + "poverty_line", + "deep_poverty_line", + "poverty_threshold_bhc", + "poverty_threshold_ahc", + "poverty_gap_bhc", + "poverty_gap_ahc", + "poverty_gap", + "deep_poverty_gap", + "in_poverty_bhc", + "in_poverty_ahc", + "in_deep_poverty_bhc", + "in_deep_poverty_ahc", + "in_poverty", + "in_deep_poverty", + "in_relative_poverty_bhc", + "in_relative_poverty_ahc", + "household_income_decile", + ] + values = {name: household_values(sim, name, year) for name in names} + values["housing_costs"] = sum( + household_values(sim, name, year) for name in HOUSING_COSTS + ) + values["components"] = sum( + household_values(sim, name, year) for name in HBAI_HOUSEHOLD_NET_INCOME_ADDS + ) - sum( + household_values(sim, name, year) + for name in HBAI_HOUSEHOLD_NET_INCOME_SUBTRACTS + ) + return values, sim + + +def person_median(income, v): + weights = v["household_weight"] * v["household_count_people"] + return float(MicroSeries(income, weights=weights).median()) + + +@PROPERTY_SETTINGS +@example(PINNED) +@given(populations()) +def test_reset_income_and_its_measures(population): + v, _ = run(population) + b = v[BEFORE_RESET] + reset = np.maximum(0, b) + + # 1. The reset, and the measures built on it. + assert np.allclose(v["hbai_household_net_income"], reset, atol=0.01) + assert np.all(v["hbai_household_net_income"] >= 0) + assert np.all(v["equiv_hbai_household_net_income"] >= 0) + assert np.allclose( + v["equiv_hbai_household_net_income"], + reset / v["household_equivalisation_bhc"], + atol=0.01, + ) + assert np.allclose( + v["real_hbai_household_net_income"], + reset * v["inflation_adjustment"], + atol=0.5, + ) + + # 2. Differential: the components summed outside the model. + assert np.allclose(b, v["components"], atol=1) + + # 3. After housing costs, from the reset income. + ahc = reset - v["housing_costs"] + assert np.allclose(v["hbai_household_net_income_ahc"], ahc, atol=0.5) + assert np.allclose( + v["equiv_hbai_household_net_income_ahc"], + ahc / v["household_equivalisation_ahc"], + atol=0.5, + ) + assert np.all(v["hbai_household_net_income_ahc"] >= -v["housing_costs"] - 0.5) + + # 6. Means: the reset adds exactly the negative parts of b. + w = v["household_weight"] + assert np.isclose( + np.sum(w * v["hbai_household_net_income"]) - np.sum(w * b), + np.sum(w * np.maximum(0, -b)), + atol=1, + ) + + +@PROPERTY_SETTINGS +@example(PINNED) +@given(populations()) +def test_poverty_headcounts_are_unchanged_and_gaps_measured_from_the_reset( + population, +): + v, _ = run(population) + b = v[BEFORE_RESET] + reset = np.maximum(0, b) + hc = v["housing_costs"] + equiv_bhc = b / v["household_equivalisation_bhc"] + equiv_ahc = (b - hc) / v["household_equivalisation_ahc"] + + # 4. Absolute and deep poverty flags: the same before and after the reset. + assert np.array_equal(v["in_poverty_bhc"], equiv_bhc < v["poverty_threshold_bhc"]) + assert np.array_equal(v["in_poverty_ahc"], equiv_ahc < v["poverty_threshold_ahc"]) + assert np.array_equal( + v["in_deep_poverty_bhc"], equiv_bhc < v["poverty_threshold_bhc"] / 2 + ) + assert np.array_equal( + v["in_deep_poverty_ahc"], equiv_ahc < v["poverty_threshold_ahc"] / 2 + ) + assert np.array_equal(v["in_poverty"], b < v["poverty_line"]) + assert np.array_equal(v["in_deep_poverty"], b < v["deep_poverty_line"]) + + # 4. Relative poverty, while fewer than half of people are at or below + # zero before the reset (otherwise the median itself is reset). + median_bhc = person_median(equiv_bhc, v) + if median_bhc > 0: + assert np.array_equal( + v["in_relative_poverty_bhc"], equiv_bhc < 0.6 * median_bhc + ) + median_ahc = person_median(equiv_ahc, v) + if median_ahc > 0: + assert np.array_equal( + v["in_relative_poverty_ahc"], equiv_ahc < 0.6 * median_ahc + ) + + # 5. Gaps from the reset income: never more than the gap before the reset, + # equal to it when b >= 0, and the BHC gaps never more than their line. + for gap, line, income, before in [ + ("poverty_gap_bhc", "poverty_line_bhc", reset, b), + ("poverty_gap", "poverty_line", reset, b), + ("deep_poverty_gap", "deep_poverty_line", reset, b), + ("poverty_gap_ahc", "poverty_line_ahc", reset - hc, b - hc), + ]: + expected = np.maximum(0, v[line] - income) + assert np.allclose(v[gap], expected, atol=0.5), gap + unreset_gap = np.maximum(0, v[line] - before) + assert np.all(v[gap] <= unreset_gap + 0.5), gap + assert np.allclose(v[gap][b >= 0], unreset_gap[b >= 0], atol=0.5), gap + for gap, line in [ + ("poverty_gap_bhc", "poverty_line_bhc"), + ("poverty_gap", "poverty_line"), + ("deep_poverty_gap", "deep_poverty_line"), + ]: + assert np.all(v[gap] <= v[line] + 0.5), gap + + +@PROPERTY_SETTINGS +@example(PINNED) +@given(populations()) +def test_every_household_has_a_decile_that_rises_with_income(population): + v, _ = run(population) + decile = v["household_income_decile"] + income = v["equiv_hbai_household_net_income"] + + # 7. No household is left out of the deciles, and a lower income never + # has a higher decile. + assert np.all((decile >= 1) & (decile <= 10)) + lower = income[:, None] < income[None, :] + assert not np.any(lower & (decile[:, None] > decile[None, :])) + + +def test_labour_supply_responses_default_to_the_income_before_the_reset(): + for function in [ + labour_supply.apply_labour_supply_responses, + labour_supply.apply_progression_responses, + progression.calculate_derivative, + progression.calculate_relative_income_change, + progression.calculate_derivative_change, + ]: + default = inspect.signature(function).parameters["target_variable"].default + assert default == BEFORE_RESET, function.__name__ + + +def test_a_pay_rise_below_zero_raises_the_income_before_the_reset(): + # 8. A worker whose maintenance payments exceed their pay: b is negative, + # so the reset income is zero both before and after a pay rise. Measured + # on b, the rise is kept net of tax and NICs; measured on the reset income + # it would look fully withdrawn. + year = 2026 + population = { + "year": year, + "households": [ + { + "adults": [adult(40, 20_000, maintenance=40_000)], + "children": 0, + "would_claim_uc": False, + "household": home(1_200, rent=9_000), + } + ], + } + sim = Simulation(situation=situation(population)) + assert sim.calculate(BEFORE_RESET, year)[0] < 0 + kept = progression.calculate_derivative(sim, year=year, count_adults=1) + reset_kept = progression.calculate_derivative( + sim, target_variable="hbai_household_net_income", year=year, count_adults=1 + ) + # Basic-rate Income Tax and main-rate Class 1 NICs: 20% + 8%. + assert np.isclose(kept[0], 0.72, atol=0.01) + assert reset_kept[0] == 0 diff --git a/policyengine_uk/tests/test_pawhp_household_income_properties.py b/policyengine_uk/tests/test_pawhp_household_income_properties.py index 5711eb2014..4ff4255422 100644 --- a/policyengine_uk/tests/test_pawhp_household_income_properties.py +++ b/policyengine_uk/tests/test_pawhp_household_income_properties.py @@ -8,8 +8,8 @@ the 2023 to 2027 qualifying weeks: switching PAWHP off (gov.social_security_scotland.pawhp.active) lowers household_benefits, household_gross_income and hbai_benefits by exactly the household's pawhp, -and household_net_income and hbai_household_net_income by its pawhp less -the change in the household's income tax. It changes neither the Winter +and household_net_income and HBAI income (before the reset of a negative +figure to zero) by its pawhp less the change in the household's income tax. It changes neither the Winter Fuel Payment nor the Cost-of-Living Payments (the pensioner payment is keyed on the Winter Fuel Payment). Before the fix the income measures did not move at all. @@ -33,7 +33,12 @@ # Either side of pensionable age (66 to 67 over these years) and of 80. AGES = [40, 60, 67, 70, 79, 80, 85] GROSS_MEASURES = ["household_benefits", "household_gross_income", "hbai_benefits"] -NET_MEASURES = ["household_net_income", "hbai_household_net_income"] +# HBAI income before the reset to zero: the accounting identity holds for the +# sum of components, which the reset would break for a negative income. +NET_MEASURES = [ + "household_net_income", + "hbai_household_net_income_before_reset_to_zero", +] UNCHANGED = ["winter_fuel_allowance", "cost_of_living_support_payment"] PAWHP_OFF = {"gov.social_security_scotland.pawhp.active": False} PROPERTY_SETTINGS = settings( From 2d7c3d1ea9cf012b413992e936a13c22294e6126 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 14 Oct 2026 07:22:47 -0400 Subject: [PATCH 3/5] Address review: decile ties, a reset median, Council Tax abolition, #2085 - #2085's channel identity (test_trading_loss_properties) now measures HBAI income before the reset, the sum the identity holds for. - household_income_decile documents the tie rule: households at zero share the rank at the top of their group, so they are in decile 1 while they hold at most a tenth of people. Unit tests pin both sides of that line; the property test checks they share the lowest decile, and decile 1 below a tenth (new pinned population). - When the median before the reset is at or below zero, the reset makes it zero and no one is in relative poverty before housing costs: checked. - YAML: the reset under gov.contrib.abolish_council_tax. - Changelog: headcounts, gaps, bottom-decile moves and unchanged labour supply responses. Co-Authored-By: Claude Opus 5.5 --- .../hbai-negative-income-reset.fixed.md | 2 +- .../income/hbai_negative_income.yaml | 25 ++++++++++ .../test_hbai_negative_income_properties.py | 47 ++++++++++++++++++- .../tests/test_household_income_decile.py | 44 +++++++++++------ .../tests/test_trading_loss_properties.py | 12 +++-- ...usehold_net_income_before_reset_to_zero.py | 7 +-- .../income/household_income_decile.py | 6 ++- 7 files changed, 117 insertions(+), 26 deletions(-) diff --git a/changelog.d/hbai-negative-income-reset.fixed.md b/changelog.d/hbai-negative-income-reset.fixed.md index b6860605b8..345ba4dd00 100644 --- a/changelog.d/hbai-negative-income-reset.fixed.md +++ b/changelog.d/hbai-negative-income-reset.fixed.md @@ -1 +1 @@ -- HBAI household net income before housing costs now resets a negative figure to zero, as HBAI does; income after housing costs, deciles, poverty flags and poverty gaps follow from the reset figure. The figure before the reset is the new `hbai_household_net_income_before_reset_to_zero`, which the labour supply responses now use. +- HBAI household net income before housing costs now resets a negative figure to zero, as HBAI does; income after housing costs, deciles, poverty flags and poverty gaps follow from the reset figure. Poverty headcounts do not change; poverty gaps fall and mean incomes rise by the sum of the negative incomes. Households whose income is reset now count in the bottom income decile instead of being left out with decile -1, so bottom-decile figures move. The figure before the reset is the new `hbai_household_net_income_before_reset_to_zero`, which the labour supply responses use, so they are unchanged. diff --git a/policyengine_uk/tests/policy/baseline/household/income/hbai_negative_income.yaml b/policyengine_uk/tests/policy/baseline/household/income/hbai_negative_income.yaml index 2e1bd94f09..48e8bbde3b 100644 --- a/policyengine_uk/tests/policy/baseline/household/income/hbai_negative_income.yaml +++ b/policyengine_uk/tests/policy/baseline/household/income/hbai_negative_income.yaml @@ -160,3 +160,28 @@ # of 8% x (20,000 - 12,570) = 594.40 and Council Tax of 1,500. hbai_household_net_income_before_reset_to_zero: 16_419.60 hbai_household_net_income: 16_419.60 + +- name: With Council Tax abolished, the reset applies to income without it + period: 2024 + absolute_error_margin: 0.1 + input: + gov.contrib.abolish_council_tax: true + people: + adult: + age: 40 + maintenance_expenses: 3_000 + benunits: + benunit: + members: [adult] + would_claim_uc: false + households: + household: + members: [adult] + council_tax: 1_500 + rent: 2_000 + tenure_type: RENT_PRIVATELY + output: + # Maintenance paid only: the reform drops Council Tax from the deductions. + hbai_household_net_income_before_reset_to_zero: -3_000 + hbai_household_net_income: 0 + hbai_household_net_income_ahc: -2_000 diff --git a/policyengine_uk/tests/test_hbai_negative_income_properties.py b/policyengine_uk/tests/test_hbai_negative_income_properties.py index 028fde0646..6b4fd24fe9 100644 --- a/policyengine_uk/tests/test_hbai_negative_income_properties.py +++ b/policyengine_uk/tests/test_hbai_negative_income_properties.py @@ -19,13 +19,17 @@ equals the flag computed from the income before the reset. Zero is below any absolute line, and the reset moves no one across the median as long as fewer than half of people have an income of zero or less before the reset. + Otherwise the BHC median is itself reset to zero, and no one is in relative + poverty before housing costs. 5. Gaps: each gap is measured from the reset income, so the BHC gaps never exceed their line, and every gap is at most the gap before the reset, with equality when b >= 0. 6. Means: the weighted total of the reset income exceeds that of b by exactly the weighted sum of the negative parts of b. 7. Deciles: every household has a decile from 1 to 10 (none is left out), and - deciles never fall as equivalised income rises. + deciles never fall as equivalised income rises. Households at zero share + one decile, the lowest; it is decile 1 when they hold less than a tenth of + people (equal incomes take the rank at the top of their group). 8. Labour supply responses measure b, the actual budget constraint: a pay rise for a household below zero raises b, while the reset income stays at zero. @@ -169,6 +173,31 @@ def home(council_tax, rent=0, mortgage=0, water=0, insurance=0): } +# One household whose maintenance payments exceed its income among ten single +# earners: the household at zero holds 1 of 11 people, under a tenth, so it is +# in decile 1. +PINNED_FEW_AT_ZERO = { + "year": 2025, + "households": [ + { + "adults": [adult(40, 0, maintenance=15_000)], + "children": 0, + "would_claim_uc": False, + "household": home(1_200, rent=6_000), + } + ] + + [ + { + "adults": [adult(30, earnings)], + "children": 0, + "would_claim_uc": True, + "household": home(1_200, rent=6_000), + } + for earnings in range(10_000, 60_000, 5_000) + ], +} + + def situation(population): year = population["year"] people, benunits, groups = {}, {}, {} @@ -256,6 +285,7 @@ def person_median(income, v): @PROPERTY_SETTINGS @example(PINNED) +@example(PINNED_FEW_AT_ZERO) @given(populations()) def test_reset_income_and_its_measures(population): v, _ = run(population) @@ -301,6 +331,7 @@ def test_reset_income_and_its_measures(population): @PROPERTY_SETTINGS @example(PINNED) +@example(PINNED_FEW_AT_ZERO) @given(populations()) def test_poverty_headcounts_are_unchanged_and_gaps_measured_from_the_reset( population, @@ -331,6 +362,10 @@ def test_poverty_headcounts_are_unchanged_and_gaps_measured_from_the_reset( assert np.array_equal( v["in_relative_poverty_bhc"], equiv_bhc < 0.6 * median_bhc ) + else: + # The median is reset to zero with everyone else, so the line is zero + # and no reset income is below it. + assert not np.any(v["in_relative_poverty_bhc"]) median_ahc = person_median(equiv_ahc, v) if median_ahc > 0: assert np.array_equal( @@ -360,6 +395,7 @@ def test_poverty_headcounts_are_unchanged_and_gaps_measured_from_the_reset( @PROPERTY_SETTINGS @example(PINNED) +@example(PINNED_FEW_AT_ZERO) @given(populations()) def test_every_household_has_a_decile_that_rises_with_income(population): v, _ = run(population) @@ -372,6 +408,15 @@ def test_every_household_has_a_decile_that_rises_with_income(population): lower = income[:, None] < income[None, :] assert not np.any(lower & (decile[:, None] > decile[None, :])) + # Households at zero (every reset household among them) share the lowest + # decile, which is decile 1 when they hold less than a tenth of people. + zero = income == 0 + if zero.any(): + assert np.all(decile[zero] == decile.min()) + people = v["household_weight"] * v["household_count_people"] + if people[zero].sum() / people.sum() < 0.1 - 1e-6: + assert np.all(decile[zero] == 1) + def test_labour_supply_responses_default_to_the_income_before_the_reset(): for function in [ diff --git a/policyengine_uk/tests/test_household_income_decile.py b/policyengine_uk/tests/test_household_income_decile.py index 49b4bc0804..34cecd830d 100644 --- a/policyengine_uk/tests/test_household_income_decile.py +++ b/policyengine_uk/tests/test_household_income_decile.py @@ -32,27 +32,41 @@ def test_household_income_decile_caps_at_ten(): assert result.max() == 10 -def test_household_income_decile_ranks_incomes_reset_to_zero_in_the_bottom_decile(): - # HBAI resets a negative income to zero before ranking, so - # equiv_hbai_household_net_income is never negative and a household whose - # income was reset sits in the bottom decile, not outside the deciles. Two - # of twenty equally weighted households have their income reset: a tenth - # of the population, so both fill the bottom decile. - household = FakeHousehold( +def zero_and_positive_incomes(zeros, total=20): + return FakeHousehold( { "equiv_hbai_household_net_income": np.concatenate( - [[0.0, 0.0], np.arange(1, 19, dtype=float) * 1_000] - ), - "household_count_people": np.ones(20), - "household_weight": np.ones(20), + [np.zeros(zeros), np.arange(1, total - zeros + 1, dtype=float)] + ) + * 1_000, + "household_count_people": np.ones(total), + "household_weight": np.ones(total), } ) - result = household_income_decile.formula(household, 2025, None) - assert result[0] == result[1] == 1 - assert np.all(result >= 1) - assert np.all(result <= 10) +def test_household_income_decile_ranks_incomes_reset_to_zero_in_the_bottom_decile(): + # HBAI resets a negative income to zero before ranking, so + # equiv_hbai_household_net_income is never negative and a household whose + # income was reset sits in the bottom decile, not outside the deciles. + # Equal incomes share the rank at the top of their group: one or two of + # twenty equally weighted households at zero (at most a tenth of people) + # are all in decile 1. + for zeros in [1, 2]: + result = household_income_decile.formula( + zero_and_positive_incomes(zeros), 2025, None + ) + assert np.all(result[:zeros] == 1) + assert np.all((result >= 1) & (result <= 10)) + + +def test_household_income_decile_tied_zeros_above_a_tenth_share_a_higher_decile(): + # Documented limit: when more than a tenth of people are tied at zero they + # share the decile of the top of their group (three of twenty: decile 2), + # and decile 1 is empty. On the enhanced FRS under 1% of people are at zero. + result = household_income_decile.formula(zero_and_positive_incomes(3), 2025, None) + assert np.all(result[:3] == 2) + assert 1 not in result def test_pre_budget_change_income_decile_domain_is_negative_or_one_to_ten(): diff --git a/policyengine_uk/tests/test_trading_loss_properties.py b/policyengine_uk/tests/test_trading_loss_properties.py index f2360db709..c74afc311c 100644 --- a/policyengine_uk/tests/test_trading_loss_properties.py +++ b/policyengine_uk/tests/test_trading_loss_properties.py @@ -34,8 +34,9 @@ child's) come off their applicable income, never below zero, and the award never falls. 6. Household income: market income falls by exactly the household's losses, - and HBAI net income moves only through the loss itself, taxes and - means-tested or passported benefits; every other component is unchanged. + and HBAI net income (before HBAI resets a negative figure to zero) moves + only through the loss itself, taxes and means-tested or passported + benefits; every other component is unchanged. 7. Class 4 (SSCBA 1992 Sch 2 para 3): Class 4 profits are self-employment profits after capital allowances and the trading allowance, less the separate Class 4 loss balance and the year's loss, never below zero. @@ -727,7 +728,10 @@ def test_household_income_moves_only_through_its_channels(units): components = sorted( set(HBAI_HOUSEHOLD_NET_INCOME_ADDS + HBAI_HOUSEHOLD_NET_INCOME_SUBTRACTS) ) - variables = components + ["household_market_income", "hbai_household_net_income"] + # HBAI income before the reset to zero: the channel identity holds for the + # sum of components, which the reset breaks once income crosses zero. + hbai = "hbai_household_net_income_before_reset_to_zero" + variables = components + ["household_market_income", hbai] a_sim = Simulation(situation=family_situation(units)) b_sim = Simulation(situation=family_situation(without)) @@ -753,7 +757,7 @@ def household_values(sim, variable): if v in HBAI_CHANNELS ) assert np.allclose( - a["hbai_household_net_income"] - b["hbai_household_net_income"], + a[hbai] - b[hbai], channels, atol=1, ) diff --git a/policyengine_uk/variables/household/income/hbai_household_net_income_before_reset_to_zero.py b/policyengine_uk/variables/household/income/hbai_household_net_income_before_reset_to_zero.py index 7d988cd7e2..7f3802ed21 100644 --- a/policyengine_uk/variables/household/income/hbai_household_net_income_before_reset_to_zero.py +++ b/policyengine_uk/variables/household/income/hbai_household_net_income_before_reset_to_zero.py @@ -11,9 +11,10 @@ class hbai_household_net_income_before_reset_to_zero(Variable): label = "Household net income (HBAI definition) before a negative income is reset to zero" documentation = ( "HBAI household net income before housing costs, before HBAI resets a " - "negative figure to zero. It can be negative when deductions such as " - "Council Tax, Income Tax, pension contributions and maintenance paid " - "exceed income. hbai_household_net_income is this figure reset to zero " + "negative figure to zero. It can be negative: self-employment losses " + "count as negative income, and deductions such as Council Tax, Income " + "Tax, pension contributions and maintenance paid can exceed income. " + "hbai_household_net_income is this figure reset to zero " "when negative. Use this variable for a household's actual budget " "constraint (marginal rates, labour supply responses) and to diagnose " "negative incomes." diff --git a/policyengine_uk/variables/household/income/household_income_decile.py b/policyengine_uk/variables/household/income/household_income_decile.py index 124650f60a..9db722f354 100644 --- a/policyengine_uk/variables/household/income/household_income_decile.py +++ b/policyengine_uk/variables/household/income/household_income_decile.py @@ -8,8 +8,10 @@ class household_income_decile(Variable): documentation = ( "Decile of equivalised HBAI household net income before housing costs " "(person-weighted). As in HBAI, a negative income is reset to zero " - "before ranking, so a household whose income was negative before the " - "reset falls in the bottom decile." + "before ranking. Households with equal incomes share one rank, the " + "rank at the top of their group, so households at zero fall in the " + "bottom decile as long as they hold at most a tenth of people (on the " + "enhanced FRS they hold under 1%)." ) entity = Household definition_period = YEAR From 59d57bdafdb5feda95baa14092cc19bab5f992dc Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 14 Oct 2026 08:21:29 -0400 Subject: [PATCH 4/5] State when the reset leaves headcounts and the bottom decile unchanged Round 2 review wording fixes: - docs and the relative poverty docstrings: a reset income is below the relative line while the median person's income is positive. - docs and changelog: households at zero are in the bottom decile while they hold at most a tenth of people (0.26% to 0.29% on the enhanced FRS, 2025-26 to 2030-31). - changelog: income totals, not means, rise by the sum of the negative incomes. Co-Authored-By: Claude Opus 5.5 --- changelog.d/hbai-negative-income-reset.fixed.md | 2 +- docs/book/validation/hbai.md | 2 +- .../variables/household/income/in_relative_poverty_ahc.py | 4 ++-- .../variables/household/income/in_relative_poverty_bhc.py | 4 +++- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/changelog.d/hbai-negative-income-reset.fixed.md b/changelog.d/hbai-negative-income-reset.fixed.md index 345ba4dd00..ffb8891556 100644 --- a/changelog.d/hbai-negative-income-reset.fixed.md +++ b/changelog.d/hbai-negative-income-reset.fixed.md @@ -1 +1 @@ -- HBAI household net income before housing costs now resets a negative figure to zero, as HBAI does; income after housing costs, deciles, poverty flags and poverty gaps follow from the reset figure. Poverty headcounts do not change; poverty gaps fall and mean incomes rise by the sum of the negative incomes. Households whose income is reset now count in the bottom income decile instead of being left out with decile -1, so bottom-decile figures move. The figure before the reset is the new `hbai_household_net_income_before_reset_to_zero`, which the labour supply responses use, so they are unchanged. +- HBAI household net income before housing costs now resets a negative figure to zero, as HBAI does; income after housing costs, deciles, poverty flags and poverty gaps follow from the reset figure. Poverty headcounts do not change while the median person's income is positive (as on the enhanced FRS); poverty gaps fall and income totals rise by the sum of the negative incomes. Households whose income is reset now count in the bottom income decile, while households at zero hold at most a tenth of people, instead of being left out with decile -1, so bottom-decile figures move. The figure before the reset is the new `hbai_household_net_income_before_reset_to_zero`, which the labour supply responses use, so they are unchanged. diff --git a/docs/book/validation/hbai.md b/docs/book/validation/hbai.md index 02cb934dbc..7349f3e2bd 100644 --- a/docs/book/validation/hbai.md +++ b/docs/book/validation/hbai.md @@ -49,7 +49,7 @@ The table below shows how the HBAI income components map to PolicyEngine variabl HBAI resets a negative income before housing costs to zero: "Negative incomes BHC are reset to zero, but negative AHC incomes calculated from the adjusted BHC incomes are possible. Where incomes have been adjusted to zero BHC, income AHC is derived from the adjusted BHC income." ([HBAI Background Information and Methodology report FYE 2025](https://www.gov.uk/government/statistics/households-below-average-income-for-financial-years-ending-1995-to-2025/households-below-average-income-background-information-and-methodology-report-fye-2025#negative-incomes)). -PolicyEngine does the same. `hbai_household_net_income` is `hbai_household_net_income_before_reset_to_zero` reset to zero when negative, and `hbai_household_net_income_ahc` subtracts housing costs from the reset figure, so it can be negative. The equivalised incomes, `household_income_decile`, the poverty flags and the poverty gaps all use the reset figure: a household whose income was negative is in the bottom decile and counts as in poverty, and its before-housing-costs poverty gap is the whole poverty line. The labour supply responses use the figure before the reset, which is the household's actual budget constraint. +PolicyEngine does the same. `hbai_household_net_income` is `hbai_household_net_income_before_reset_to_zero` reset to zero when negative, and `hbai_household_net_income_ahc` subtracts housing costs from the reset figure, so it can be negative. The equivalised incomes, `household_income_decile`, the poverty flags and the poverty gaps all use the reset figure. A household whose income was negative is below every absolute poverty line, and its before-housing-costs poverty gap is the whole line. It is below the relative line while the median person's income is positive; if more than half of people had income at or below zero, the reset would make the median, and so the relative line, zero. Households with equal incomes share the rank at the top of their group, so households at zero are in the bottom decile while they hold at most a tenth of people. On the enhanced FRS under 1% of people have a negative income before the reset, so both conditions hold. The labour supply responses use the figure before the reset, which is the household's actual budget constraint. ## Nowcasting/forecasting diff --git a/policyengine_uk/variables/household/income/in_relative_poverty_ahc.py b/policyengine_uk/variables/household/income/in_relative_poverty_ahc.py index 1c5ed6ed0b..4417118798 100644 --- a/policyengine_uk/variables/household/income/in_relative_poverty_ahc.py +++ b/policyengine_uk/variables/household/income/in_relative_poverty_ahc.py @@ -10,8 +10,8 @@ class in_relative_poverty_ahc(Variable): "so each household counts once per member. Income after housing " "costs is derived from the before-housing-costs income reset to zero " "and can be negative, as in HBAI; negative incomes stay in the median " - "and count as below the line. The absolute-line twin is " - "in_poverty_ahc." + "and count as below the line while the median is positive. The " + "absolute-line twin is in_poverty_ahc." ) entity = Household definition_period = YEAR diff --git a/policyengine_uk/variables/household/income/in_relative_poverty_bhc.py b/policyengine_uk/variables/household/income/in_relative_poverty_bhc.py index 09381dc2af..2a02a45afe 100644 --- a/policyengine_uk/variables/household/income/in_relative_poverty_bhc.py +++ b/policyengine_uk/variables/household/income/in_relative_poverty_bhc.py @@ -9,7 +9,9 @@ class in_relative_poverty_bhc(Variable): "over individuals, as in DWP's Households Below Average Income series, " "so each household counts once per member. As in HBAI, a negative " "income is reset to zero; it stays in the median and counts as below " - "the line. The absolute-line twin is in_poverty_bhc." + "the line while the median is positive (if more than half of people " + "were at or below zero, the median and the line would be zero). The " + "absolute-line twin is in_poverty_bhc." ) entity = Household definition_period = YEAR From 06f47f7885ac64b3b63107432f382f1debface19 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 15 Oct 2026 00:43:04 -0400 Subject: [PATCH 5/5] Exercise service charges in HBAI reset and computed BHC tests --- .../hbai-negative-income-reset.fixed.md | 2 +- .../income/hbai_negative_income.yaml | 20 +++--- .../test_hbai_housing_costs_properties.py | 62 ++++++++++++++++++- .../test_hbai_negative_income_properties.py | 8 ++- 4 files changed, 78 insertions(+), 14 deletions(-) diff --git a/changelog.d/hbai-negative-income-reset.fixed.md b/changelog.d/hbai-negative-income-reset.fixed.md index ffb8891556..6be5627122 100644 --- a/changelog.d/hbai-negative-income-reset.fixed.md +++ b/changelog.d/hbai-negative-income-reset.fixed.md @@ -1 +1 @@ -- HBAI household net income before housing costs now resets a negative figure to zero, as HBAI does; income after housing costs, deciles, poverty flags and poverty gaps follow from the reset figure. Poverty headcounts do not change while the median person's income is positive (as on the enhanced FRS); poverty gaps fall and income totals rise by the sum of the negative incomes. Households whose income is reset now count in the bottom income decile, while households at zero hold at most a tenth of people, instead of being left out with decile -1, so bottom-decile figures move. The figure before the reset is the new `hbai_household_net_income_before_reset_to_zero`, which the labour supply responses use, so they are unchanged. +- HBAI household net income before housing costs now resets a negative figure to zero, as HBAI does; income after housing costs, deciles, poverty flags and poverty gaps follow from the reset figure. Poverty headcounts do not change while both person-weighted BHC and AHC median incomes are positive (as on the enhanced FRS); poverty gaps fall and income totals rise by the sum of the negative incomes. Households whose income is reset now count in the bottom income decile instead of being left out with decile -1, as long as households at zero hold at most a tenth of people, so bottom-decile figures move. The figure before the reset is the new `hbai_household_net_income_before_reset_to_zero`, which the labour supply responses use, so they are unchanged. diff --git a/policyengine_uk/tests/policy/baseline/household/income/hbai_negative_income.yaml b/policyengine_uk/tests/policy/baseline/household/income/hbai_negative_income.yaml index 48e8bbde3b..fe5d3c9046 100644 --- a/policyengine_uk/tests/policy/baseline/household/income/hbai_negative_income.yaml +++ b/policyengine_uk/tests/policy/baseline/household/income/hbai_negative_income.yaml @@ -29,22 +29,23 @@ members: [adult] council_tax: 1_500 rent: 6_000 + housing_service_charges: 300 tenure_type: RENT_PRIVATELY output: hbai_household_net_income_before_reset_to_zero: -6_500 hbai_household_net_income: 0 real_hbai_household_net_income: 0 equiv_hbai_household_net_income: 0 - # Income after housing costs is the reset figure less rent: 0 - 6,000. - hbai_household_net_income_ahc: -6_000 + # Income after housing costs is reset BHC less rent and service charges. + hbai_household_net_income_ahc: -6_300 # Divided by the 0.58 AHC equivalence factor for a single adult. - equiv_hbai_household_net_income_ahc: -10_344.83 + equiv_hbai_household_net_income_ahc: -10_862.07 # The before-housing-costs gap is the whole line, never more. poverty_gap_bhc: 15_040.08 poverty_gap: 15_040.08 deep_poverty_gap: 7_520.04 # The after-housing-costs gap is the line plus the negative income. - poverty_gap_ahc: 17_276.52 + poverty_gap_ahc: 17_576.52 in_poverty_bhc: true in_poverty_ahc: true in_deep_poverty_bhc: true @@ -72,10 +73,11 @@ mortgage_interest_repayment: 2_000 water_and_sewerage_charges: 400 structural_insurance_payments: 100 + housing_service_charges: 600 output: hbai_household_net_income_before_reset_to_zero: -4_000 hbai_household_net_income: 0 - hbai_household_net_income_ahc: -2_500 + hbai_household_net_income_ahc: -3_100 - name: A negative figure given directly is reset; housing costs still come off period: 2024 @@ -92,12 +94,13 @@ members: [adult] hbai_household_net_income_before_reset_to_zero: -25_000 rent: 4_800 + housing_service_charges: 200 tenure_type: RENT_PRIVATELY output: hbai_household_net_income: 0 - hbai_household_net_income_ahc: -4_800 + hbai_household_net_income_ahc: -5_000 poverty_gap_bhc: 15_040.08 - poverty_gap_ahc: 16_076.52 + poverty_gap_ahc: 16_276.52 - name: A figure of exactly zero stays zero period: 2024 @@ -132,10 +135,11 @@ members: [adult] hbai_household_net_income_before_reset_to_zero: 12_345 rent: 5_000 + housing_service_charges: 450 tenure_type: RENT_PRIVATELY output: hbai_household_net_income: 12_345 - hbai_household_net_income_ahc: 7_345 + hbai_household_net_income_ahc: 6_895 # 12,345 is below the 15,040.08 line: a gap of 2,695.08. poverty_gap_bhc: 2_695.08 diff --git a/policyengine_uk/tests/test_hbai_housing_costs_properties.py b/policyengine_uk/tests/test_hbai_housing_costs_properties.py index b204e3e4fd..f3ddeeb2b0 100644 --- a/policyengine_uk/tests/test_hbai_housing_costs_properties.py +++ b/policyengine_uk/tests/test_hbai_housing_costs_properties.py @@ -1,8 +1,10 @@ """HBAI AHC deducts every housing cost once, including service charges. -BHC is supplied directly to isolate the housing-cost contract from the -calculation and zero floor of income before housing costs. Each vector -population contains a household with zero BHC and positive service charges. +The generated properties supply BHC directly to isolate the housing-cost +contract from the calculation and zero floor of income before housing costs. +Each vector population contains a household with zero BHC and positive service +charges. A separate differential case computes BHC to ensure service charges +are deducted only after housing costs. """ import numpy as np @@ -154,3 +156,57 @@ def test_extra_service_charges_reduce_ahc_by_the_same_amount( np.testing.assert_allclose( before_ahc - after_ahc, extra_input, rtol=1e-6, atol=0.03 ) + + +@pytest.mark.parametrize("year", [2024, 2025, 2026]) +def test_service_charges_do_not_change_computed_bhc(year): + # Positive computed BHC exposes a misplaced BHC deduction that the zero + # floor could hide. Neither BHC variable is supplied as an input. + simulation = Simulation( + situation={ + "people": { + "person": { + "age": {year: 40}, + "employment_income": {year: 60_000}, + } + }, + "benunits": { + "benunit": { + "members": ["person"], + "would_claim_uc": {year: False}, + } + }, + "households": { + "household": { + "members": ["person"], + "tenure_type": {year: "OWNED_OUTRIGHT"}, + **{ + name: {year: 1_200 if name == "housing_service_charges" else 0} + for name in HOUSING_COSTS + }, + } + }, + } + ) + bhc_variables = ( + "hbai_household_net_income_before_reset_to_zero", + "hbai_household_net_income", + ) + before_bhc = { + name: np.asarray(simulation.calculate(name, year)).copy() + for name in bhc_variables + } + before_ahc = np.asarray( + simulation.calculate("hbai_household_net_income_ahc", year) + ).copy() + assert all(np.all(income > 2_400) for income in before_bhc.values()) + + simulation.set_input("housing_service_charges", year, [2_400]) + simulation.reset_calculations() + + for name in bhc_variables: + np.testing.assert_array_equal( + simulation.calculate(name, year), before_bhc[name] + ) + after_ahc = np.asarray(simulation.calculate("hbai_household_net_income_ahc", year)) + np.testing.assert_allclose(before_ahc - after_ahc, 1_200, rtol=0, atol=0.01) diff --git a/policyengine_uk/tests/test_hbai_negative_income_properties.py b/policyengine_uk/tests/test_hbai_negative_income_properties.py index 6b4fd24fe9..29336fe3c7 100644 --- a/policyengine_uk/tests/test_hbai_negative_income_properties.py +++ b/policyengine_uk/tests/test_hbai_negative_income_properties.py @@ -7,7 +7,8 @@ Properties, for any population of households (b is hbai_household_net_income_before_reset_to_zero, HC the HBAI housing costs: -rent, water and sewerage charges, mortgage interest and structural insurance): +rent, water and sewerage charges, mortgage interest, structural insurance, +and ground rent and service charges): 1. Reset: hbai_household_net_income = max(0, b), so it and its equivalised and real versions are never negative. @@ -56,6 +57,7 @@ "water_and_sewerage_charges", "mortgage_interest_repayment", "structural_insurance_payments", + "housing_service_charges", ] YEARS = [2024, 2025, 2026] PROPERTY_SETTINGS = settings( @@ -89,6 +91,7 @@ def households(draw): "would_claim_uc": draw(st.booleans()), "household": { "council_tax": draw(money(0, 1_200, 2_500)), + "housing_service_charges": draw(money(300, 600, 1_200)), "tenure_type": "RENT_PRIVATELY" if renter else "OWNED_WITH_MORTGAGE", "rent": draw(money(4_000, 9_000, 15_000)) if renter else 0, "mortgage_interest_repayment": 0 if renter else draw(money(0, 3_000)), @@ -115,7 +118,7 @@ def adult(age, earnings, maintenance=0, pension=0): } -def home(council_tax, rent=0, mortgage=0, water=0, insurance=0): +def home(council_tax, rent=0, mortgage=0, water=0, insurance=0, charges=600): return { "council_tax": council_tax, "tenure_type": "RENT_PRIVATELY" if rent else "OWNED_WITH_MORTGAGE", @@ -123,6 +126,7 @@ def home(council_tax, rent=0, mortgage=0, water=0, insurance=0): "mortgage_interest_repayment": mortgage, "water_and_sewerage_charges": water, "structural_insurance_payments": insurance, + "housing_service_charges": charges, }