diff --git a/changelog.d/nz-spec-line-citations.fixed.md b/changelog.d/nz-spec-line-citations.fixed.md new file mode 100644 index 0000000000..3dfc17f318 --- /dev/null +++ b/changelog.d/nz-spec-line-citations.fixed.md @@ -0,0 +1,3 @@ +Replace drifting New Zealand spec citations with Python symbols, JSON Pointers, +and a documentation heading. Guard every NZ JSON/YAML resource against local +file line citations and require commit pins for foreign file line citations. diff --git a/packages/microcosm-build/src/microcosm/build/nz/benefit_unit_rule.json b/packages/microcosm-build/src/microcosm/build/nz/benefit_unit_rule.json index 9941ed4917..bf3be694eb 100644 --- a/packages/microcosm-build/src/microcosm/build/nz/benefit_unit_rule.json +++ b/packages/microcosm-build/src/microcosm/build/nz/benefit_unit_rule.json @@ -2,7 +2,7 @@ "version": 1, "country": "nz", "method_card_row": "MC7", - "description": "The New Zealand benefit unit (engine Family entity): an adult, their co-resident partner, and their dependent children; every other adult forms their own unit. An adult for unit construction is any person excluded from the declared dependent-child rule, including a financially independent young person or an independent teen parent. The unit id family.family_id and membership person.person_family_id are structural graph columns (microcosm.graph.executor._structural_columns, executor.py:638-645); a non-structural node cannot own either (executor.py:1549-1554). family.family_household_id is the ordinary family column linking each unit to its household. The shared transport CREATE kernel (G4/G5a; transport/population_kernels.py:208-232) builds units from donor concept pointers under this spec. Family weights are inherited from the household through the Frame.", + "description": "The New Zealand benefit unit (engine Family entity): an adult, their co-resident partner, and their dependent children; every other adult forms their own unit. An adult for unit construction is any person excluded from the declared dependent-child rule, including a financially independent young person or an independent teen parent. The unit id family.family_id and membership person.person_family_id are structural graph columns (microcosm.graph.executor._structural_columns); a non-structural node cannot own either (microcosm.graph.executor._validate_population_declaration). family.family_household_id is the ordinary family column linking each unit to its household. The shared transport CREATE kernel (G4/G5a; microcosm.build.transport.population_kernels.TransportCreateKernel) builds units from donor concept pointers under this spec. Family weights are inherited from the household through the Frame.", "status": "settled by the NZ hub on 2026-10-12 under G8 method addendum B8 (MC7)", "entity": "family", "engine_entity": "Family", @@ -19,7 +19,7 @@ "parent_2_person_id" ], "household_reference_person": "reference_person_id", - "citation": "packages/microcosm-frame/src/microcosm/frame/concepts.py:889-928" + "citation": "packages/microcosm-frame/src/microcosm/frame/concepts.py, the \"# --- Relationships\" block: _pointer(\"partner_person_id\") through _pointer(\"reference_person_id\")" }, "source_household_lineage": { "concept": "fact:household.source_household_id", @@ -37,7 +37,7 @@ }, { "member": "dependent_child", - "rule": "A dependent child joins the unit of its co-resident parent parent_1_person_id; if that parent is itself dependent, follow the parent chain to that parent's unit. A lone co-resident parent is always parent 1 (concepts.py:900-907), so parent_2_person_id never chooses the unit." + "rule": "A dependent child joins the unit of its co-resident parent parent_1_person_id; if that parent is itself dependent, follow the parent chain to that parent's unit. A lone co-resident parent is always parent 1 (concepts.py _pointer(\"parent_1_person_id\")), so parent_2_person_id never chooses the unit." }, { "member": "other_adult", @@ -55,7 +55,7 @@ "hours_concept": "fact:person.usual_weekly_hours", "donor_variable": "weekly_hours_worked_before_lsr", "basis": "Social Security Act 2018, Schedule 2, financially independent (DLM6784504, limb a) and full employment (DLM6784514, limbs a and b), as at 1 April 2026; G8 method addendum B8. The test applies only while below age_limit_years.", - "concept_evidence": "concepts.py:1069-1083 declares usual weekly hours across all jobs and self-employment in worked weeks in the donor reference year, with transport CARRY. adapters/policyengine_us_concepts.py:352-362 maps weekly_hours_worked_before_lsr through Identity to this concept; transport.py:843-849 decodes the donor through that mapping. docs/concept-coverage/policyengine-us.md:31 records the same binding.", + "concept_evidence": "concepts.py concept(\"fact:person.usual_weekly_hours\") declares usual weekly hours across all jobs and self-employment in worked weeks in the donor reference year, with transport CARRY. microcosm.frame.adapters.policyengine_us_concepts.POLICYENGINE_US_CONCEPT_MAPPING maps weekly_hours_worked_before_lsr through Identity to this concept; microcosm.frame.transport.read_populace_us_donor decodes the donor through that mapping. docs/concept-coverage/policyengine-us.md#inputs-each-concept-feeds records the same binding.", "approximation": "The donor usual rate in worked weeks is used for the statutory average weekly employment test; employment contracts and the averaging interval are not observed." }, "parent_independence_min_age": 16, diff --git a/packages/microcosm-build/src/microcosm/build/nz/reporting_classifiers.json b/packages/microcosm-build/src/microcosm/build/nz/reporting_classifiers.json index 9ca1423e8c..eb5e7411c7 100644 --- a/packages/microcosm-build/src/microcosm/build/nz/reporting_classifiers.json +++ b/packages/microcosm-build/src/microcosm/build/nz/reporting_classifiers.json @@ -192,8 +192,8 @@ "stability": "Same pinned donor and unit rule preserve person ids across rebuilds; each family id is its head person id. Units sharing source household lineage remain distinct units.", "implementation": [ "packages/microcosm-frame/src/microcosm/frame/unit_construction.py build_benefit_units (each family id is its head person's id)", - "packages/microcosm-build/src/microcosm/build/transport/population_kernels.py:209-228", - "packages/microcosm-frame/src/microcosm/frame/transport.py:843-856" + "microcosm.build.transport.population_kernels.TransportCreateKernel", + "microcosm.frame.transport.read_populace_us_donor" ], "citation": { "item": "A1", @@ -260,7 +260,7 @@ "disclosures": [ { "note": "NZ Super is overstated for single people who share a home because the shares-residence input defaults false; this moves their deciles upward.", - "evidence": "axiom_input_closure.json:1045-1052", + "evidence": "axiom_input_closure.json#/entries[input=nz_super_shares_residence_with_person_at_least_adult_age]", "citation": { "item": "A1", "source": "docs/nz-reporting-classifiers-method.md#a1-income-decile-provisional" @@ -268,7 +268,7 @@ }, { "note": "Market-income components follow the existing family gross-income concept mapping. Rental income is after business expenses. The rental and private-pension concept descriptions do not explicitly say before tax; their before-tax transport interpretation must be verified before activation.", - "evidence": "packages/microcosm-frame/src/microcosm/frame/adapters/axiom_concept_mappings/nz.json:670-688; packages/microcosm-frame/src/microcosm/frame/concepts.py:990-1000,1040-1051", + "evidence": "packages/microcosm-frame/src/microcosm/frame/adapters/axiom_concept_mappings/nz.json#/bindings[canonical_input=nz:statutes~1social_security~1accommodation_supplement~1core#input.accommodation_supplement_relevant_weekly_income]; packages/microcosm-frame/src/microcosm/frame/concepts.py _amount(\"rental_income\") and _amount(\"private_pension_income\")", "citation": { "item": "A1", "source": "docs/nz-reporting-classifiers-method.md#a1-income-decile-provisional" @@ -734,7 +734,7 @@ "as_non_beneficiary_audit": { "nz_super": "The existing accommodation_supplement_non_beneficiary state binding negates any_true over assigned member receipt flags, including NZ Super. A unit with any assigned NZ Super receipt is therefore not a non-beneficiary.", "veterans_pension": "The same binding omits the unsupported Veterans Pension flag, so it must be extended when that receipt input is activated. A missing Veterans Pension flag continues to block reporting rather than classify its recipients as other.", - "evidence": "axiom_input_closure.json:1183-1204", + "evidence": "axiom_input_closure.json#/entries[input=accommodation_supplement_non_beneficiary]", "citation": { "item": "A2", "source": "docs/nz-reporting-classifiers-method.md#a2-receipt-group" diff --git a/packages/microcosm-build/src/microcosm/build/nz/scenarios.json b/packages/microcosm-build/src/microcosm/build/nz/scenarios.json index 6fba457c8f..8a819e9923 100644 --- a/packages/microcosm-build/src/microcosm/build/nz/scenarios.json +++ b/packages/microcosm-build/src/microcosm/build/nz/scenarios.json @@ -150,7 +150,7 @@ "annualisation": { "weeks_per_year": 52, "method_card_row": "MC12", - "note": "the conformance harness uses 365/7 weeks (ops main run.py:168); the model states and uses 52" + "note": "the conformance harness uses 365/7 weeks (WEEKS_IN_MODEL_YEAR in TheAxiomFoundation/ops nz-lane/emtr_reproduction/run.py at ba6e7d6e); the model states and uses 52" }, "groupings": { "method_card_row": "MC3", diff --git a/packages/microcosm-build/tests/engine_free/shared/test_nz_spec_package.py b/packages/microcosm-build/tests/engine_free/shared/test_nz_spec_package.py index bab2dc5346..11110db7df 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_nz_spec_package.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_nz_spec_package.py @@ -19,12 +19,120 @@ (``modal_area``); - every scenario names exactly one method-card row and a tier from {entitlement, calibration}, and changes exactly one knob from the centre - (``scenario_errors``). + (``scenario_errors``); +- no resource cites a line of a repository file, and foreign-file line + citations require a same-resource commit pin (``unpinned_line_citations``); +- JSON citations refuse index steps into arrays consisting of mappings. + Instead, ``file.json#/entries[input=target]`` selects exactly one record. + A selector extends an RFC 6901 token naming an array; ordinary pointer + steps may follow it, e.g. ``/entries[input=target]/state_binding/columns/0``. + Comma-separated terms are a conjunction, e.g. ``[input=target,module=nz]``. + Fields are identifiers and values are nonempty strings; brackets, commas, + equals signs and whitespace are reserved. Parse the selector before + decoding RFC 6901 ``~0`` (tilde) and ``~1`` (slash) escapes. No match or + multiple matches is an error. Indexes into scalar arrays remain valid. Differential checks compare two records of one fact: the loader's target references with a direct parse, the rules bindings' module digests with the committed Axiom input surface, and the export contract's formula-owned list -with the bound variables. +with the bound variables, and source citations with the code and JSON nodes +they identify. + +The line guard catches drift-prone citations that an author (human or model) +would plausibly write in NZ spec prose. It is not a defence against deliberate +obfuscation, such as spelling out line numbers or splitting them across keys. + +Recognized line citations (for any extension, or known checkout filenames +such as LICENSE and .gitignore): + +- The filename may be plain, backtick-wrapped, single/double/curly-quoted, + an inline Markdown link's destination or text (including backticks), an + image link, or reference-link text. A link attributes its locator to the + destination (without query or fragment) and every filename in its label; + all must be pinned foreign files for an exemption. +- Forward: file:12, file:L12, file：L12 (full-width colon), file#L12, + file @ L12, file L12, file, L12, file (L12), file [L12], file line 12, + file lines L12-L20, file l.12, file ll.12-20, and file at/on line 12. + Locators may themselves be quoted or backtick-wrapped. +- Reverse: line 12 in file, lines 12-20 of file, ll.12-20 in file, + and L12-L20 of/in file. Words ignore case; the L prefix is uppercase. +- Numbers may form comma lists or ranges with hyphen, en dash, em dash, + or "to", with optional spaces and an L on either endpoint. +- HTTP(S) URLs may be bare or Markdown destinations, inside parentheses, + quotes, angle brackets or HTML hrefs, or in Markdown reference definitions. + GitHub blob/raw/tree/blame//path and raw.githubusercontent.com URLs + accept #L12, #L12-L20, #L12-20 and #L12C3-L20C9 fragments (also lowercase + l/c). Query strings before fragments and percent-encoded URL paths work. + Trailing punctuation and symbols, including Unicode dashes and ellipses, + cannot hide a complete line fragment. Ununderstood fragments stay scannable. + PolicyEngine/microcosm URLs at any ref, including commit permalinks, name + local files. Foreign URLs require a same-resource {path, commit} pin for + the exact repository-relative path; basename matching is prose-only. + Paths normalize leading ./; a trailing slash names a directory, not a file. + +In-scope examples: + file:12 + file:L12 + file：L12 + file#L12 + file @ L12 + file L12 + file, L12 + file (L12) + file [L12] + file line 12 + file lines L12-L20 + file l.12 + file ll.12-20 + file at line 12 + file on line 12 + line 12 in file + lines 12-20 of file + ll.12-20 in file + L12-L20 of file + file:12,14-20 + file lines 12–20 + file lines 12—20 + file lines 12 to 20 + `file` lines 12-20 + 'file' lines 12-20 + "file" lines 12-20 + ‘file’ lines 12-20 + “file” lines 12-20 + file: `L12` + file: 'line 12' + [file](file) lines 12-20 + [`file`](file) L12 + [file][source] lines 12-20 + [the source](file) lines 12-20 + lines 12-20 of [the source](file) + [upstream.rs](file) L12 + [the source](file#L12) + [the source](file:L12) + [file:L12](upstream.rs) + [the source](file?raw=1) L12 + ![x](file) L5 + LICENSE:12 + .gitignore:12 + [the source](https://github.com/PolicyEngine/microcosm/blob/main/file#L12) + https://github.com/PolicyEngine/microcosm/raw/main/file#L12-L20 + https://raw.githubusercontent.com/PolicyEngine/microcosm/main/file#L12C3-L20C9 + + https://github.com/PolicyEngine/microcosm/tree/main/file#L5-10: see the field. + +Out of scope: bare prose numbers ("line 12" without a file), unknown +extensionless names ("unknown_source:12"), spelled-out numbers ("file line +twelve"), locators split across keys or values ({"file": "line", "n": 12}), +HTML/entity-encoded markup ("file:12"), encoded locator syntax (#%4C5 or +#L%35), repeatedly encoded URL paths, Unicode lookalike locator letters, +zero-width characters inserted into locators, +escaped/nested Markdown labels, link titles, and arbitrary Markdown rendering +or reference-link destination lookup. The guard scans parsed resource strings +for explicit file/line locators. +In plain prose, lowercase l12 is ambiguous with a Python identifier; use +uppercase L12 or an explicit line/l./ll. label. Symbols, heading anchors, +JSON Pointers and keyed selectors remain opaque, including locator-like tokens. +Versioned symbols such as takeup.assign@1 also remain valid. """ from __future__ import annotations @@ -32,11 +140,19 @@ import copy import dataclasses import hashlib +import importlib +import inspect import json import math import re +import subprocess +import unicodedata from collections.abc import Callable, Iterator, Mapping, Sequence +from functools import cache +from pathlib import PurePosixPath +from types import SimpleNamespace from typing import Any +from urllib.parse import unquote, urlsplit import pytest from hypothesis import assume, given, settings @@ -49,6 +165,8 @@ reference_fact_selectors, ) from microcosm.build.spec_engine import load_yaml12 +from microcosm.frame import concepts as frame_concepts +from microcosm.frame.input_closure import INPUT_CLOSURE_FORMAT from test_support.paths import paths_for _BUILD = paths_for("microcosm-build") @@ -97,10 +215,27 @@ def _nested_keys(value: object) -> Iterator[str]: yield from _nested_keys(child) +def _nested_strings(value: object) -> Iterator[str]: + """Every string in a payload, mapping keys included, at any depth.""" + + if isinstance(value, Mapping): + for key, child in value.items(): + yield str(key) + yield from _nested_strings(child) + elif isinstance(value, list): + for child in value: + yield from _nested_strings(child) + elif isinstance(value, str): + yield value + + def _package_payloads() -> dict[str, object]: payloads: dict[str, object] = {} for path in sorted(NZ_ROOT.rglob("*")): - if not path.is_file(): + if not path.is_file() or not ( + path.suffix == ".json" + or (path.parent == NZ_ROOT / "spec" and path.suffix == ".yaml") + ): continue name = path.relative_to(NZ_ROOT).as_posix() text = path.read_text(encoding="utf-8") @@ -459,6 +594,476 @@ def scenario_errors(document: Mapping[str, Any]) -> list[str]: return errors +#: A file citation, with any extension, optionally qualified by directories. +_SOURCE_FILE = ( + r"(?P[A-Za-z0-9_./-]*[A-Za-z0-9_-]\.[A-Za-z0-9][A-Za-z0-9_-]*)" + r"(?![A-Za-z0-9_.-])" +) + + +def _line_citation_patterns(source: str) -> tuple[re.Pattern[str], ...]: + """Share the documented locator grammar with extensionless filenames.""" + + quotes = r"[`'\"‘’“”]*" + # Keep wrappers attached to their filename or complete Markdown link. + wrapped = ( + rf"\[?{quotes}{source}{quotes}" + r"(?:\](?:\((?:[^()\n]|\([^()\n]*\))*\)|\[[^\]\n]*\])?)?" + ) + number = r"L?\s*\d+(?!\w)" + span = rf"{number}(?:\s*(?:[-–—]|(?i:to))\s*{number})?" + numbers = rf"{span}(?:\s*,\s*{span})*" + label = r"(?i:lines?\b\s+|ll?\.\s*)" + forward = ( + wrapped + + rf"(?:\s*[:：]\s*{quotes}(?:{label})?|\s*@\s*{quotes}(?:{label}|(?=L))|" + rf"\s*#(?=L)|\s*,?\s*[\[(]?\s*" + rf"(?:(?i:at|on)\s+)?{quotes}(?:{label}|(?=L))){quotes}{numbers}" + ) + reverse = rf"\b(?:{label}|(?=L)){quotes}{numbers}{quotes}\s+(?i:of|in)\s+" + wrapped + # Lowercase URL fragments must also survive raw fallback scanning. Keep + # bare l12 and file:l12 ambiguous with symbols, as in the prose grammar. + lower_fragment = wrapped + r"\s*#l\d+(?:[Cc]\d+)?(?!\w)" + return (re.compile(forward), re.compile(reverse), re.compile(lower_fragment)) + + +#: Keep every locator spelling from #1141, now for any file extension. +_LINE_CITATIONS = _line_citation_patterns(_SOURCE_FILE) +_MARKDOWN_LINK = re.compile( + r"(?P!?\[(?P