From 273aade201bf4e3c853d2fd3d56cb7ed6cadbe77 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 24 Sep 2026 16:15:41 -0400 Subject: [PATCH 01/10] Restore WEIND/WEMIND before pooling and carry industry + worked_last_year The #720 restoration now appends the ASEC longest-job industry recodes WEIND (detailed, 0-23) and WEMIND (major, 0-15) to every vintage from its pinned Census person member. No H5 vintage carries them, so they sit outside the 2026-08-23 offline-fix receipt and are recorded as such. After restoring, each vintage is checked against two identities that hold in all three members: WEMIND is the major group of WEIND (the pinned WEIND_TO_WEMIND function), and WEIND carries a worker code (1-22) exactly where WKSWORK > 0. The release-time org_wages carry reads them next to POCCU2 and emits detailed_industry_recode, major_industry_recode and worked_last_year (WKSWORK > 0), refusing missing or out-of-range codes and checking the halves of the identities that hold on every spine. Gate bands follow the measured shares. Part of #719. Co-Authored-By: Claude Opus 5.5 --- .../us_runtime/asec_census_person_columns.py | 125 +++++++++++++++++- .../microcosm/build/us_runtime/org_wages.py | 92 ++++++++++++- .../test_us_asec_census_person_columns.py | 97 +++++++++++++- 3 files changed, 310 insertions(+), 4 deletions(-) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_census_person_columns.py b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_census_person_columns.py index b0c5650405..c142b46ffa 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_census_person_columns.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_census_person_columns.py @@ -57,10 +57,24 @@ ``A_FTPT`` same; ``eligibility_inputs`` ``derive_us_eligibility_inputs_from_manifest`` (``is_full_time_college_student``) +``WEIND`` ``org_wages.derive_us_org_occupation_inputs`` -> + ``detailed_industry_recode`` (#719) +``WEMIND`` same -> ``major_industry_recode`` (#719) ====================== ====================================================== The other thirteen columns the 2026-08-23 offline fix appended are not restored; :data:`ASEC_CENSUS_PERSON_COLUMNS_NOT_RESTORED` records why. + +``WEIND`` and ``WEMIND`` (industry of the longest job last year, detailed and +major groups) were never in that offline fix: no H5 vintage carries them, not +even 2024, so every vintage gets them from its member. +:data:`ASEC_CENSUS_PERSON_COLUMNS_BEYOND_OFFLINE_FIX` records that. Once they +are restored, two identities that hold in all three pinned members are +checked on the vintage (:func:`_check_work_experience_universe`): ``WEMIND`` +is the function :data:`WEIND_TO_WEMIND` of ``WEIND``, and ``WEIND`` carries a +worker code (1--22) exactly where ``WKSWORK > 0``. The second identity is +ASEC-only: an ACS person may carry an industry from a job held one to five +years ago, so the release-time consumer can check only its forward half. """ from __future__ import annotations @@ -81,9 +95,11 @@ __all__ = [ "ASEC_CENSUS_PERSON_COLUMNS", + "ASEC_CENSUS_PERSON_COLUMNS_BEYOND_OFFLINE_FIX", "ASEC_CENSUS_PERSON_COLUMNS_NOT_RESTORED", "ASEC_CENSUS_PERSON_COLUMN_NAMES", "ASEC_CENSUS_PERSON_IDENTITY_COLUMNS", + "WEIND_TO_WEMIND", "AsecCensusPersonColumn", "AsecCensusPersonColumnsError", "restore_asec_census_person_columns", @@ -107,6 +123,48 @@ class AsecCensusPersonColumn: _NOW_YES_NO = frozenset({1, 2}) _ENROLLMENT = frozenset({0, 1, 2}) +#: ``WEIND``: 0 = not in universe (under 15), 1--21 = civilian industry of the +#: longest job last year, 22 = Armed Forces, 23 = did not work last year (age +#: 15+). ``WEMIND`` aggregates it into 15 major groups (0 = not in universe). +_DETAILED_INDUSTRY_CODES = frozenset(range(24)) +_MAJOR_INDUSTRY_CODES = frozenset(range(16)) +_WORK_EXPERIENCE_READER = ( + "microcosm.build.us_runtime.org_wages.derive_us_org_occupation_inputs" +) + +#: ``WEMIND`` as a function of ``WEIND``, identical in all three pinned person +#: members (pppub23/24/25, 2026-09-24: every ``WEIND`` value co-occurs with +#: exactly one ``WEMIND`` value) and nested as the codebook's major and +#: detailed industry groups are. +WEIND_TO_WEMIND: MappingProxyType[int, int] = MappingProxyType( + { + 0: 0, + 1: 1, + 2: 2, + 3: 3, + 4: 4, + 5: 4, + 6: 5, + 7: 5, + 8: 6, + 9: 6, + 10: 7, + 11: 8, + 12: 8, + 13: 9, + 14: 9, + 15: 10, + 16: 10, + 17: 11, + 18: 11, + 19: 12, + 20: 12, + 21: 13, + 22: 14, + 23: 15, + } +) +_WORKER_INDUSTRY_CODES = frozenset(range(1, 23)) _HEALTH_COVERAGE_READER = ( "microcosm.build.us_runtime.cps_carried._fill_health_coverage_inputs" ) @@ -204,6 +262,16 @@ class AsecCensusPersonColumn: "(is_full_time_college_student <- A_HSCOL == 2 & A_FTPT == 1)", ), ), + AsecCensusPersonColumn( + "WEIND", + _DETAILED_INDUSTRY_CODES, + (f"{_WORK_EXPERIENCE_READER} -> detailed_industry_recode",), + ), + AsecCensusPersonColumn( + "WEMIND", + _MAJOR_INDUSTRY_CODES, + (f"{_WORK_EXPERIENCE_READER} -> major_industry_recode",), + ), ) ASEC_CENSUS_PERSON_COLUMN_NAMES: tuple[str, ...] = tuple( column.name for column in ASEC_CENSUS_PERSON_COLUMNS @@ -247,6 +315,22 @@ class AsecCensusPersonColumn: } ) +#: Reviewed columns that the 2026-08-23 offline fix never appended, and why +#: they are restored anyway. +ASEC_CENSUS_PERSON_COLUMNS_BEYOND_OFFLINE_FIX: MappingProxyType[str, str] = ( + MappingProxyType( + { + column: ( + "Industry of the longest job last year (#719): the county-file " + "person schema carries industry with occupation (POCCU2) on " + "every record. No H5 vintage carries it, not even 2024, so " + "every vintage restores it from its pinned member." + ) + for column in ("WEIND", "WEMIND") + } + ) +) + #: Census identity carried by both the H5 and the person member. After the #: PERIDNUM join each must agree row for row, so a join that pairs the wrong #: people cannot pass on a coincidence of keys. @@ -368,6 +452,40 @@ def _value_summary(spec: AsecCensusPersonColumn, values: pd.Series) -> dict[str, return {str(code): int(counts.get(code, 0)) for code in sorted(spec.domain)} +def _check_work_experience_universe( + table: pd.DataFrame, h5_label: str +) -> dict[str, object]: + """Check the ASEC work-experience identities on one restored vintage.""" + + detailed = table["WEIND"].to_numpy(dtype=np.int64) + major = table["WEMIND"].to_numpy(dtype=np.int64) + expected_major = np.array([WEIND_TO_WEMIND[code] for code in detailed]) + mismatch = major != expected_major + if mismatch.any(): + _refuse( + f"{h5_label} WEMIND disagrees with the major group of WEIND on " + f"{int(mismatch.sum())} rows; the join paired the wrong people or " + "the member changed its industry recodes." + ) + record: dict[str, object] = {"weind_to_wemind_rows_verified": len(table)} + if "WKSWORK" not in table.columns: + record["worker_code_iff_weeks_worked"] = "not checked: H5 lacks WKSWORK" + return record + weeks = pd.to_numeric(table["WKSWORK"], errors="coerce") + if weeks.isna().any(): + _refuse(f"{h5_label} WKSWORK has missing or non-numeric values.") + worker = np.isin(detailed, list(_WORKER_INDUSTRY_CODES)) + disagree = worker != weeks.gt(0).to_numpy() + if disagree.any(): + _refuse( + f"{h5_label} WEIND carries a worker code (1--22) where WKSWORK is " + f"not positive, or the reverse, on {int(disagree.sum())} rows." + ) + record["worker_code_iff_weeks_worked"] = "verified" + record["worker_rows"] = int(worker.sum()) + return record + + def restore_asec_census_person_columns( person: pd.DataFrame, *, @@ -391,7 +509,10 @@ def restore_asec_census_person_columns( person has no member row, or a member person is absent from the H5; when ``PH_SEQ``/``P_SEQ``/``A_LINENO``/``A_AGE`` disagree after the join; and when a reviewed column the H5 already carries is not integer or differs - from the member on any row. A column the H5 carries is never overwritten. + from the member on any row; and when ``WEMIND`` is not the major group of + ``WEIND``, or (where the H5 carries ``WKSWORK``) ``WEIND`` holds a worker + code on a row without positive weeks worked or the reverse. A column the + H5 carries is never overwritten. New columns are appended as ``int64`` after the H5's own, in reviewed order; every H5 column keeps its position, dtype and values (checked). @@ -482,6 +603,7 @@ def restore_asec_census_person_columns( ) if not unchanged: _refuse(f"Restoring Census columns changed an existing {h5_label} column.") + work_experience = _check_work_experience_universe(result, h5_label) return result, { "operation": "exact_source_join", @@ -503,4 +625,5 @@ def restore_asec_census_person_columns( "columns_added": list(added), "columns_verified_equal": verified, "member_values": values, + "work_experience_universe": work_experience, } diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/org_wages.py b/packages/microcosm-build/src/microcosm/build/us_runtime/org_wages.py index d309067756..a90b53fdad 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/org_wages.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/org_wages.py @@ -53,6 +53,7 @@ from microcosm.build.gates import GateResult from microcosm.build.source_manifest import SourceStageSpec, load_source_manifest +from microcosm.build.us_runtime.asec_census_person_columns import WEIND_TO_WEMIND from microcosm.frame import Frame from microcosm.frame.units import US_SCHEMA @@ -119,6 +120,9 @@ "cps_race", "is_hispanic", "detailed_occupation_recode", + "detailed_industry_recode", + "major_industry_recode", + "worked_last_year", "has_never_worked", "is_military", "is_computer_scientist", @@ -137,12 +141,18 @@ "PRDTRACE", "PRDTHSP", "POCCU2", + "WEIND", + "WEMIND", + "WKSWORK", "employment_income_before_lsr", "weekly_hours_worked_before_lsr", "hours_worked_last_week", "weeks_worked", ) +#: WEIND worker codes: civilian industry groups 1--21 and Armed Forces 22. +_WORKER_INDUSTRY_CODES = np.arange(1, 23) + FLSA_OVERTIME_OCCUPATION_CODES: dict[str, int] = { "has_never_worked": 53, "is_military": 52, @@ -307,6 +317,12 @@ "cps_race": (0.95, 1.0), "is_hispanic": (0.05, 0.30), "detailed_occupation_recode": (0.65, 0.95), + # Measured on the pinned inputs: 0.822-0.824 of weighted ASEC persons (all + # of age 15+) and 0.80 of ACS persons (age 16+) carry a nonzero industry + # recode; 0.520-0.523 (ASEC) and 0.52 (ACS) worked last year. + "detailed_industry_recode": (0.70, 0.92), + "major_industry_recode": (0.70, 0.92), + "worked_last_year": (0.40, 0.62), "has_never_worked": (0.15, 0.35), "is_military": (0.0005, 0.01), "is_computer_scientist": (0.005, 0.05), @@ -532,16 +548,60 @@ def load_org_2024_donor( return result +def _complete_codes(person: pd.DataFrame, column: str, upper: int) -> np.ndarray: + values = pd.to_numeric(person[column], errors="coerce").to_numpy(dtype=np.float64) + invalid = ~np.isfinite(values) | (values != np.floor(values)) + invalid |= (values < 0) | (values > upper) + if invalid.any(): + raise ValueError( + f"ORG work-experience carry needs {column} as complete integer codes " + f"in [0, {upper}]; {int(invalid.sum())} row(s) are missing or outside." + ) + return values.astype(np.int64) + + def derive_us_org_occupation_inputs(person: pd.DataFrame) -> pd.DataFrame: - """Carry CPS race/ethnicity and derive the exact POCCU2 FLSA flags.""" + """Carry CPS race/ethnicity, occupation, industry and worked-last-year. + + Occupation derives the exact POCCU2 FLSA flags. The work-experience + carries (#719) are the longest-job industry recodes ``WEIND`` and + ``WEMIND`` and ``worked_last_year = WKSWORK > 0``; on ACS rows the release + predictor join supplies all three from native ``INDP``/``WKWN`` before this + runs. Both spines satisfy the checked identities: ``WEMIND`` is the major + group of ``WEIND``, and positive weeks worked carry a worker industry code. + The converse holds on ASEC rows only (an ACS industry can come from a job + held one to five years ago) and is checked before pooling, where the rows + are known to be ASEC (``asec_census_person_columns``). + """ missing = [ - column for column in ("PRDTRACE", "PRDTHSP", "POCCU2") if column not in person + column + for column in ("PRDTRACE", "PRDTHSP", "POCCU2", "WEIND", "WEMIND", "WKSWORK") + if column not in person ] if missing: raise ValueError( f"ORG occupation derivation needs person column(s): {missing}." ) + detailed_industry = _complete_codes(person, "WEIND", 23) + major_industry = _complete_codes(person, "WEMIND", 15) + weeks_worked = _complete_codes(person, "WKSWORK", 52) + expected_major = np.array([WEIND_TO_WEMIND[code] for code in detailed_industry]) + mismatched_major = int((major_industry != expected_major).sum()) + if mismatched_major: + raise ValueError( + f"ORG work-experience carry: WEMIND is not the major group of WEIND " + f"on {mismatched_major} row(s)." + ) + worked = weeks_worked > 0 + worked_without_industry = int( + (worked & ~np.isin(detailed_industry, _WORKER_INDUSTRY_CODES)).sum() + ) + if worked_without_industry: + raise ValueError( + "ORG work-experience carry: positive WKSWORK without a worker WEIND " + f"code (1--22) on {worked_without_industry} row(s)." + ) occupation = pd.to_numeric(person["POCCU2"], errors="coerce").fillna(0).astype(int) result = pd.DataFrame(index=person.index) result["cps_race"] = ( @@ -551,6 +611,9 @@ def derive_us_org_occupation_inputs(person: pd.DataFrame) -> pd.DataFrame: pd.to_numeric(person["PRDTHSP"], errors="coerce").fillna(0).ne(0) ) result["detailed_occupation_recode"] = occupation.astype(np.int16) + result["detailed_industry_recode"] = detailed_industry.astype(np.int16) + result["major_industry_recode"] = major_industry.astype(np.int16) + result["worked_last_year"] = worked for variable, code in FLSA_OVERTIME_OCCUPATION_CODES.items(): result[variable] = occupation.eq(code) result["is_executive_administrative_professional"] = occupation.isin( @@ -993,6 +1056,27 @@ def share(column: str) -> float: "positive_without_weeks": int(((premium > 0) & (weeks <= 0)).sum()), "positive_always_exempt": int(((premium > 0) & (never | military)).sum()), } + detailed_industry = pd.to_numeric( + person["detailed_industry_recode"], errors="coerce" + ).to_numpy(dtype=np.float64) + major_industry = pd.to_numeric( + person["major_industry_recode"], errors="coerce" + ).to_numpy(dtype=np.float64) + worked = person["worked_last_year"].fillna(False).astype(bool).to_numpy() + known_detailed = np.isin(detailed_industry, list(WEIND_TO_WEMIND)) + expected_major = np.full(detailed_industry.shape, np.nan) + expected_major[known_detailed] = [ + WEIND_TO_WEMIND[int(code)] for code in detailed_industry[known_detailed] + ] + work_experience_violations = { + "unknown_industry_code": int((~known_detailed).sum()), + "major_group_mismatch": int( + (known_detailed & (major_industry != expected_major)).sum() + ), + "worked_without_worker_industry": int( + (worked & ~np.isin(detailed_industry, _WORKER_INDUSTRY_CODES)).sum() + ), + } return { "nonzero_shares": {column: share(column) for column in _SHARE_BANDS}, "share_bands": {column: list(band) for column, band in _SHARE_BANDS.items()}, @@ -1002,6 +1086,7 @@ def share(column: str) -> float: }, "fsla_overtime_premium_weighted_total": float(np.nansum(premium * weights)), "constraint_violations": violations, + "work_experience_violations": work_experience_violations, } @@ -1032,6 +1117,9 @@ def us_org_wages_signal_gate(frame: Frame) -> GateResult: for name, count in summary["constraint_violations"].items(): if count: failures.append(f"fsla_overtime_premium: {count} {name} violation(s).") + for name, count in summary["work_experience_violations"].items(): + if count: + failures.append(f"industry/worked_last_year: {count} {name} row(s).") return GateResult( name="org_wages_signal", passed=not failures, diff --git a/packages/microcosm-build/tests/test_us_asec_census_person_columns.py b/packages/microcosm-build/tests/test_us_asec_census_person_columns.py index 1087649aa5..c1b1dae580 100644 --- a/packages/microcosm-build/tests/test_us_asec_census_person_columns.py +++ b/packages/microcosm-build/tests/test_us_asec_census_person_columns.py @@ -17,8 +17,10 @@ from microcosm.build.us_runtime.asec_census_person_columns import ( ASEC_CENSUS_PERSON_COLUMN_NAMES, ASEC_CENSUS_PERSON_COLUMNS, + ASEC_CENSUS_PERSON_COLUMNS_BEYOND_OFFLINE_FIX, ASEC_CENSUS_PERSON_COLUMNS_NOT_RESTORED, ASEC_CENSUS_PERSON_IDENTITY_COLUMNS, + WEIND_TO_WEMIND, AsecCensusPersonColumnsError, restore_asec_census_person_columns, ) @@ -70,6 +72,12 @@ #: H5 row order differs from the member's, so a positional copy would fail. _H5_ORDER = [3, 0, 7, 5, 1, 6, 2, 4] +#: Member-order industry codes: six workers (one in the Armed Forces) and two +#: people 15+ who did not work last year. Every member person is 16+. +_WEIND = [7, 16, 23, 22, 23, 9, 1, 21] +#: Weeks worked last year consistent with ``_WEIND`` (positive iff 1--22). +_WKSWORK = [52, 40, 0, 52, 0, 12, 26, 3] + def _digest(path: Path) -> str: return hashlib.sha256(path.read_bytes()).hexdigest() @@ -95,6 +103,9 @@ def _member() -> pd.DataFrame: else: codes = sorted(spec.domain) member[spec.name] = [codes[(row * 3) % len(codes)] for row in range(8)] + # The industry pair is rigid: WEMIND is the major group of WEIND. + member["WEIND"] = _WEIND + member["WEMIND"] = [WEIND_TO_WEMIND[code] for code in _WEIND] return member @@ -364,12 +375,96 @@ def test_default_pins_are_the_shared_census_person_pins(sources): def test_the_review_places_every_offline_fix_column_exactly_once(): restored = set(ASEC_CENSUS_PERSON_COLUMN_NAMES) not_restored = set(ASEC_CENSUS_PERSON_COLUMNS_NOT_RESTORED) + beyond = set(ASEC_CENSUS_PERSON_COLUMNS_BEYOND_OFFLINE_FIX) assert len(restored) == len(ASEC_CENSUS_PERSON_COLUMN_NAMES) assert restored.isdisjoint(not_restored) - assert restored | not_restored == _RECEIPT_720_ADDED + assert beyond == {"WEIND", "WEMIND"} + assert beyond <= restored + assert beyond.isdisjoint(_RECEIPT_720_ADDED) + assert (restored - beyond) | not_restored == _RECEIPT_720_ADDED assert not restored & set(ASEC_CENSUS_PERSON_IDENTITY_COLUMNS) +def test_weind_to_wemind_is_the_codebook_nesting(): + """Pinned from all three members; every major group nests detailed ones.""" + + assert dict(WEIND_TO_WEMIND) == { + 0: 0, + 1: 1, + 2: 2, + 3: 3, + 4: 4, + 5: 4, + 6: 5, + 7: 5, + 8: 6, + 9: 6, + 10: 7, + 11: 8, + 12: 8, + 13: 9, + 14: 9, + 15: 10, + 16: 10, + 17: 11, + 18: 11, + 19: 12, + 20: 12, + 21: 13, + 22: 14, + 23: 15, + } + specs = {spec.name: spec for spec in ASEC_CENSUS_PERSON_COLUMNS} + assert set(WEIND_TO_WEMIND) == specs["WEIND"].domain + assert set(WEIND_TO_WEMIND.values()) == specs["WEMIND"].domain + majors = list(WEIND_TO_WEMIND.values()) + assert majors == sorted(majors) + + +def test_restores_industry_and_verifies_the_worker_universe(sources): + member, csv, _, pin = sources + person = _h5_person(member) + person["WKSWORK"] = np.asarray(_WKSWORK, dtype=np.int64)[_H5_ORDER] + result, record = _restore(person, csv, pin) + joined = member.set_index("PERIDNUM").loc[person["PERIDNUM"].astype(str)] + assert result["WEIND"].tolist() == joined["WEIND"].tolist() + assert result["WEMIND"].tolist() == joined["WEMIND"].tolist() + assert record["work_experience_universe"] == { + "weind_to_wemind_rows_verified": 8, + "worker_code_iff_weeks_worked": "verified", + "worker_rows": 6, + } + + +def test_records_that_the_worker_universe_needs_wkswork(sources): + member, csv, _, pin = sources + _, record = _restore(_h5_person(member), csv, pin) + assert record["work_experience_universe"] == { + "weind_to_wemind_rows_verified": 8, + "worker_code_iff_weeks_worked": "not checked: H5 lacks WKSWORK", + } + + +@pytest.mark.parametrize("weeks", [0, 30]) +def test_refuses_a_worker_code_that_disagrees_with_weeks_worked(sources, weeks): + member, csv, _, pin = sources + person = _h5_person(member) + wkswork = np.asarray(_WKSWORK, dtype=np.int64)[_H5_ORDER] + # H5 row 1 is member row 0 (a worker); H5 row 6 is member row 2 (no work). + wkswork[1 if weeks == 0 else 6] = weeks + person["WKSWORK"] = wkswork + with pytest.raises(AsecCensusPersonColumnsError, match="worker code"): + _restore(person, csv, pin) + + +def test_refuses_a_major_group_that_is_not_the_detailed_groups_parent(tmp_path): + member = _member() + member.loc[0, "WEMIND"] = 6 # WEIND 7 nests in major group 5 + csv, _, pin = _write(tmp_path, member) + with pytest.raises(AsecCensusPersonColumnsError, match="major group of WEIND"): + _restore(_h5_person(member), csv, pin) + + def _cited_objects(reader: str): for dotted in re.findall(r"\b(?:microcosm|microunit)(?:\.\w+)+", reader): parts = dotted.split(".") From 69644ce38977d1b8b716ce86904ff63aeb790265 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 24 Sep 2026 16:24:15 -0400 Subject: [PATCH 02/10] Map ACS INDP and WKWN onto the CPS work-experience columns at release The release predictor join now also materializes WEIND, WEMIND and WKSWORK for ACS-spine people, so the org_wages carry gives every released person record an industry and a worked-last-year indicator, the same way ACS OCCP already reaches detailed_occupation_recode through POCCU2. ACS_INDP_TO_WEIND (267 codes) reads every civilian code's group from the pinned 2025 ASEC member, where INDUSTRY determines WEIND exactly and the 2022 Census code list matches ACS 2024 INDP. The eight ACS-only codes are reviewed: the seven Armed Forces branch codes collapse to CPS 9890's group 22, and 9920 is the did-not-work group 23. Blank INDP mirrors the OCCP treatment: 0 below age 16 (age 15 keeps the out-of-universe sentinel), 23 from age 16 where only ESR 6 may be blank. WEMIND follows WEIND by the pinned nesting; WKSWORK is WKWN (weeks in the past 12 months) or 0. Positive weeks without a worker industry code is refused. Crosswalk version 2; digest re-pinned. Part of #719. Co-Authored-By: Claude Opus 5.5 --- .../us_runtime/acs_release_predictors.py | 389 +++++++++++++++++- .../tests/test_us_acs_release_predictors.py | 99 +++++ 2 files changed, 484 insertions(+), 4 deletions(-) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/acs_release_predictors.py b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_release_predictors.py index 98ba4e4d9f..97129d309b 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/acs_release_predictors.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_release_predictors.py @@ -23,7 +23,10 @@ * ``sipp_tips.CENSUS_OCCUPATION_CODE_TO_TTOC`` consumes detailed Census occupation codes directly; * ``org_wages.FLSA_OVERTIME_OCCUPATION_CODES`` and its EAP set consume the - 53-category CPS detailed occupation recode; and + 53-category CPS detailed occupation recode; +* ``org_wages.derive_us_org_occupation_inputs`` carries the CPS longest-job + industry recodes (``WEIND`` detailed, ``WEMIND`` major) and derives + ``worked_last_year`` from weeks worked (``WKSWORK``); and * ``sipp_vehicles._household_tenure_status`` consumes only mortgaged owner, outright owner, and non-owner tenure codes. @@ -46,6 +49,7 @@ import numpy as np import pandas as pd +from microcosm.build.us_runtime.asec_census_person_columns import WEIND_TO_WEMIND from microcosm.build.us_runtime.support_provenance import ( support_channel_column, support_clone_index_column, @@ -58,6 +62,7 @@ "ACS_2024_HOUSEHOLD_ZIP_SHA256", "ACS_2024_PERSON_ZIP_SHA256", "ACS_DIFFICULTY_TO_CPS", + "ACS_INDP_TO_WEIND", "ACS_OCCP_TO_POCCU2", "ACS_RAC1P_TO_CONSUMED_PRDTRACE", "ACS_RELEASE_PREDICTOR_CROSSWALK_SHA256", @@ -73,9 +78,9 @@ ACS_2024_HOUSEHOLD_ZIP_SHA256 = ( "8281008e53de98f0ef81e7a2ee5a8725991dda1ecfd2713ead73246425e515d0" ) -ACS_RELEASE_PREDICTOR_CROSSWALK_VERSION = 1 +ACS_RELEASE_PREDICTOR_CROSSWALK_VERSION = 2 ACS_RELEASE_PREDICTOR_CROSSWALK_SHA256 = ( - "1d4906242e9c73e31b3283659e5cad8242b8cbc42914ab6fa59547a10c8770e9" + "a9b8410493084f7002f1a617757dafba1ef9c451efdce200d9f2ef2c8541a194" ) _SHA256_PATTERN = re.compile(r"[0-9a-f]{64}") @@ -99,6 +104,8 @@ "RAC1P", "HISP", "OCCP", + "INDP", + "WKWN", "ESR", "SSIP", "ADJINC", @@ -686,6 +693,287 @@ 9920: 53, } +# 2022 Census industry codes (ACS 2024 INDP; CPS ASEC from survey year 2025) +# mapped to the 23 CPS longest-job detailed industry groups (WEIND). Every +# civilian code's group is read from the pinned 2025 ASEC person member +# (pppub25.csv), where INDUSTRY determines WEIND exactly; codes shared with +# the 2017-list 2023/2024 members keep the same group there. The ACS-only +# codes are reviewed: 9670-9870 are the Armed Forces branches (Army, Air +# Force, Navy, Marines, Coast Guard, branch not specified, Reserves or National +# Guard) that CPS collapses into 9890 (WEIND 22), and 9920 ("unemployed, no +# work experience in the last 5 years or never worked") is the CPS +# did-not-work group 23. WEMIND follows from WEIND by the pinned +# ``WEIND_TO_WEMIND`` nesting. +ACS_INDP_TO_WEIND: Mapping[int, int] = { + 170: 1, + 180: 1, + 190: 1, + 270: 1, + 280: 1, + 290: 1, + 370: 2, + 380: 2, + 390: 2, + 470: 2, + 480: 2, + 490: 2, + 570: 9, + 580: 9, + 590: 9, + 670: 9, + 680: 9, + 690: 9, + 770: 3, + 1070: 5, + 1080: 5, + 1090: 5, + 1170: 5, + 1180: 5, + 1190: 5, + 1270: 5, + 1280: 5, + 1290: 5, + 1370: 5, + 1390: 5, + 1470: 5, + 1480: 5, + 1490: 5, + 1570: 5, + 1590: 5, + 1670: 5, + 1691: 5, + 1770: 5, + 1790: 5, + 1870: 5, + 1880: 5, + 1890: 5, + 1990: 5, + 2070: 5, + 2090: 5, + 2170: 5, + 2180: 5, + 2190: 5, + 2270: 5, + 2280: 5, + 2290: 5, + 2370: 5, + 2380: 5, + 2390: 5, + 2470: 4, + 2480: 4, + 2490: 4, + 2570: 4, + 2590: 4, + 2670: 4, + 2680: 4, + 2690: 4, + 2770: 4, + 2780: 4, + 2790: 4, + 2870: 4, + 2880: 4, + 2890: 4, + 2970: 4, + 2980: 4, + 2990: 4, + 3070: 4, + 3080: 4, + 3095: 4, + 3170: 4, + 3180: 4, + 3291: 4, + 3365: 4, + 3370: 4, + 3380: 4, + 3390: 4, + 3470: 4, + 3490: 4, + 3570: 4, + 3580: 4, + 3590: 4, + 3670: 4, + 3680: 4, + 3690: 4, + 3770: 4, + 3780: 4, + 3790: 4, + 3875: 4, + 3895: 4, + 3960: 4, + 3970: 4, + 3980: 4, + 3990: 4, + 4070: 6, + 4080: 6, + 4090: 6, + 4170: 6, + 4180: 6, + 4195: 6, + 4265: 6, + 4270: 6, + 4280: 6, + 4290: 6, + 4370: 6, + 4380: 6, + 4390: 6, + 4470: 6, + 4480: 6, + 4490: 6, + 4560: 6, + 4570: 6, + 4580: 6, + 4585: 6, + 4590: 6, + 4670: 7, + 4681: 7, + 4691: 7, + 4771: 7, + 4796: 7, + 4871: 7, + 4881: 7, + 4891: 7, + 4971: 7, + 4973: 7, + 4981: 7, + 4991: 7, + 5071: 7, + 5081: 7, + 5090: 7, + 5171: 7, + 5181: 7, + 5191: 7, + 5276: 7, + 5281: 7, + 5296: 7, + 5371: 7, + 5382: 7, + 5392: 7, + 5471: 7, + 5481: 7, + 5491: 7, + 5571: 7, + 5581: 7, + 5680: 7, + 5791: 7, + 6070: 8, + 6080: 8, + 6090: 8, + 6170: 8, + 6180: 8, + 6190: 8, + 6270: 8, + 6280: 8, + 6290: 8, + 6370: 8, + 6380: 8, + 6390: 8, + 6471: 10, + 6481: 10, + 6490: 10, + 6570: 10, + 6590: 10, + 6671: 10, + 6680: 10, + 6690: 10, + 6695: 10, + 6770: 10, + 6781: 10, + 6871: 11, + 6881: 11, + 6890: 11, + 6970: 11, + 6991: 11, + 6992: 11, + 7071: 12, + 7072: 12, + 7080: 12, + 7181: 12, + 7190: 12, + 7270: 13, + 7280: 13, + 7290: 13, + 7370: 13, + 7380: 13, + 7390: 13, + 7460: 13, + 7470: 13, + 7480: 13, + 7490: 13, + 7570: 14, + 7580: 14, + 7590: 14, + 7670: 14, + 7680: 14, + 7690: 14, + 7770: 14, + 7780: 14, + 7790: 14, + 7860: 15, + 7870: 15, + 7880: 15, + 7890: 15, + 7970: 16, + 7980: 16, + 7990: 16, + 8070: 16, + 8080: 16, + 8090: 16, + 8170: 16, + 8180: 16, + 8191: 16, + 8192: 16, + 8270: 16, + 8290: 16, + 8370: 16, + 8380: 16, + 8390: 16, + 8470: 16, + 8561: 17, + 8562: 17, + 8563: 17, + 8564: 17, + 8570: 17, + 8580: 17, + 8590: 17, + 8660: 18, + 8670: 18, + 8680: 18, + 8690: 18, + 8770: 20, + 8780: 20, + 8790: 20, + 8870: 20, + 8891: 20, + 8970: 20, + 8980: 20, + 8990: 20, + 9070: 20, + 9080: 20, + 9090: 20, + 9160: 20, + 9170: 20, + 9180: 20, + 9190: 20, + 9290: 19, + 9370: 21, + 9380: 21, + 9390: 21, + 9470: 21, + 9480: 21, + 9490: 21, + 9570: 21, + 9590: 21, + 9670: 22, + 9680: 22, + 9690: 22, + 9770: 22, + 9780: 22, + 9790: 22, + 9870: 22, + 9890: 22, + 9920: 23, +} + ACS_TEN_TO_SPM_TENMORTSTATUS: Mapping[int, int] = { 1: 1, # owned with a mortgage or loan 2: 2, # owned free and clear @@ -706,7 +994,7 @@ "scf_auto_loans": ("PRDTRACE", "PRDTHSP"), "sipp_vehicles": ("SPM_TENMORTSTATUS",), "sipp_tips": ("PEIOOCC",), - "org_wages": ("PRDTRACE", "PRDTHSP", "POCCU2"), + "org_wages": ("PRDTRACE", "PRDTHSP", "POCCU2", "WEIND", "WEMIND", "WKSWORK"), } _OUTPUT_COLUMNS = tuple( dict.fromkeys( @@ -762,6 +1050,37 @@ def acs_release_predictor_crosswalk_payload() -> dict[str, Any]: ), }, }, + "industry": { + "INDP_to_WEIND": { + str(key): value for key, value in ACS_INDP_TO_WEIND.items() + }, + "blank_INDP_to_WEIND": { + "age_below_16": 0, + "age_16_plus": 23, + "age_15_source_target_universe_gap": ( + "ACS INDP is not asked; retain the out-of-universe sentinel " + "instead of fabricating CPS did-not-work code 23" + ), + }, + "WEIND_to_WEMIND": { + str(key): value for key, value in WEIND_TO_WEMIND.items() + }, + "observed_INDP_reference": ( + "most recent job in the past 5 years (CPS WEIND: longest job " + "last year)" + ), + }, + "work_experience": { + "WKWN_to_WKSWORK": { + "observed": "identity (weeks worked in the past 12 months, 1-52)", + "blank": 0, + "reference_period_gap": ( + "ACS past 12 months before interview; CPS WKSWORK prior " + "calendar year" + ), + }, + "invariant": "WKSWORK > 0 implies WEIND in 1..22", + }, "tenure": { "TEN_to_SPM_TENMORTSTATUS": { str(key): value for key, value in ACS_TEN_TO_SPM_TENMORTSTATUS.items() @@ -1452,6 +1771,68 @@ def _crosswalk_people(joined: pd.DataFrame) -> pd.DataFrame: # no-occupation / never-worked bin 53. poccu2.loc[occupation.isna()] = np.where(age.loc[occupation.isna()].ge(16), 53, 0) result["POCCU2"] = poccu2.to_numpy(dtype=np.int16) + + # INDP shares OCCP's universe (age 16+, worked in the past 5 years, or + # unemployed with code 9920); a blank from age 16 is admitted only for + # ESR=6 and maps to the CPS did-not-work group 23. + industry_raw = joined["INDP"] + industry = pd.to_numeric(industry_raw, errors="coerce") + invalid_industry = (industry_raw.notna() & industry.isna()) | ( + industry.notna() & age.lt(16) + ) + if invalid_industry.any(): + bad = joined.loc[invalid_industry, ["SERIALNO", "SPORDER", "AGEP", "INDP"]] + raise ValueError( + "ACS INDP code/universe mismatch (blank below age 16); examples=" + f"{bad.head().to_dict('records')}." + ) + blank_industry = industry.isna() & age.ge(16) & ~employment.eq(6) + if blank_industry.any(): + bad = joined.loc[ + blank_industry, ["SERIALNO", "SPORDER", "AGEP", "INDP", "ESR"] + ] + raise ValueError( + "ACS INDP is blank inside its observed employment universe; examples=" + f"{bad.head().to_dict('records')}." + ) + observed_industry = industry.loc[industry.notna()].to_numpy(dtype=np.float64) + if not np.equal(observed_industry, np.floor(observed_industry)).all(): + raise ValueError("ACS INDP contains non-integer code(s).") + unknown_industry = sorted( + set(industry.dropna().astype(np.int64)) - set(ACS_INDP_TO_WEIND) + ) + if unknown_industry: + raise ValueError(f"ACS INDP contains unsupported code(s): {unknown_industry}.") + weind = industry.map(ACS_INDP_TO_WEIND) + weind.loc[industry.isna()] = np.where(age.loc[industry.isna()].ge(16), 23, 0) + weind = weind.astype(np.int64) + + weeks_raw = joined["WKWN"] + weeks = pd.to_numeric(weeks_raw, errors="coerce") + invalid_weeks = ( + (weeks_raw.notna() & weeks.isna()) + | (weeks.notna() & age.lt(16)) + | (weeks.notna() & ~weeks.isin(range(1, 53))) + ) + if invalid_weeks.any(): + bad = joined.loc[invalid_weeks, ["SERIALNO", "SPORDER", "AGEP", "WKWN"]] + raise ValueError( + "ACS WKWN code/universe mismatch (blank below age 16; 1--52 weeks " + f"from age 16); examples={bad.head().to_dict('records')}." + ) + wkswork = weeks.fillna(0).astype(np.int64) + worked_without_industry = wkswork.gt(0) & ~weind.between(1, 22) + if worked_without_industry.any(): + bad = joined.loc[ + worked_without_industry, ["SERIALNO", "SPORDER", "AGEP", "INDP", "WKWN"] + ] + raise ValueError( + "ACS WKWN reports weeks worked without a worker INDP code; examples=" + f"{bad.head().to_dict('records')}." + ) + result["WEIND"] = weind.to_numpy(dtype=np.int16) + result["WEMIND"] = weind.map(WEIND_TO_WEMIND).to_numpy(dtype=np.int16) + result["WKSWORK"] = wkswork.to_numpy(dtype=np.int16) return result diff --git a/packages/microcosm-build/tests/test_us_acs_release_predictors.py b/packages/microcosm-build/tests/test_us_acs_release_predictors.py index 0c298aaea5..82fe3c407a 100644 --- a/packages/microcosm-build/tests/test_us_acs_release_predictors.py +++ b/packages/microcosm-build/tests/test_us_acs_release_predictors.py @@ -12,11 +12,13 @@ import microcosm.build.us_runtime.acs_release_predictors as module from microcosm.build.us_runtime.acs_release_predictors import ( + ACS_INDP_TO_WEIND, ACS_OCCP_TO_POCCU2, ACS_RELEASE_PREDICTOR_CROSSWALK_SHA256, acs_release_predictor_crosswalk_payload, join_acs_release_predictors, ) +from microcosm.build.us_runtime.asec_census_person_columns import WEIND_TO_WEMIND from microcosm.build.us_runtime.puf_support import clone_us_frame_for_puf_support from microcosm.build.us_runtime.spine_assembly import assemble_spines from microcosm.frame import US_SCHEMA, Frame, WeightKind, Weights @@ -33,10 +35,17 @@ "PRDTHSP", "PEIOOCC", "POCCU2", + "WEIND", + "WEMIND", + "WKSWORK", "SPM_TENMORTSTATUS", ) +#: ACS 2024 Armed Forces branch codes CPS collapses into industry 9890. +_ACS_ARMED_FORCES_INDP = (9670, 9680, 9690, 9770, 9780, 9790, 9870) + + def _source_frame(*, acs: bool) -> Frame: if acs: household_ids = np.asarray([1, 2], dtype=np.int64) @@ -87,6 +96,9 @@ def _source_frame(*, acs: bool) -> Frame: person["PRDTHSP"] = 0.0 person["PEIOOCC"] = 1005.0 person["POCCU2"] = 8.0 + person["WEIND"] = 13.0 + person["WEMIND"] = 9.0 + person["WKSWORK"] = 52.0 person["SPM_TENMORTSTATUS"] = 1.0 return Frame( @@ -125,6 +137,8 @@ def _raw_person() -> pd.DataFrame: "RAC1P": [1, 6, 2], "HISP": [1, 1, 2], "OCCP": [np.nan, 1005, 9800], + "INDP": [np.nan, 7380, 9670], + "WKWN": [np.nan, 52, 48], "ESR": [np.nan, 1, 4], "SSIP": [np.nan, 900.0, 0.0], "ADJINC": [1_000_000, 1_000_000, 1_000_000], @@ -210,8 +224,29 @@ def test_crosswalk_digest_and_every_consumed_occupation_bin_are_pinned() -> None assert ACS_OCCP_TO_POCCU2[6005] == 41 assert ACS_OCCP_TO_POCCU2[9800] == 52 assert ACS_OCCP_TO_POCCU2[9920] == 53 + assert len(ACS_INDP_TO_WEIND) == 267 + assert set(ACS_INDP_TO_WEIND.values()) == set(range(1, 24)) + assert ACS_INDP_TO_WEIND[170] == 1 + assert ACS_INDP_TO_WEIND[7380] == 13 + assert ACS_INDP_TO_WEIND[8191] == 16 + assert {ACS_INDP_TO_WEIND[code] for code in _ACS_ARMED_FORCES_INDP} == {22} + assert ACS_INDP_TO_WEIND[9920] == 23 payload = acs_release_predictor_crosswalk_payload() + assert payload["version"] == 2 + assert payload["industry"]["blank_INDP_to_WEIND"]["age_16_plus"] == 23 + assert payload["industry"]["WEIND_to_WEMIND"] == { + str(key): value for key, value in WEIND_TO_WEMIND.items() + } + assert payload["work_experience"]["WKWN_to_WKSWORK"]["blank"] == 0 + assert payload["model_predictors"]["org_wages"] == [ + "PRDTRACE", + "PRDTHSP", + "POCCU2", + "WEIND", + "WEMIND", + "WKSWORK", + ] assert payload["disability"]["DREM"]["minimum_question_age"] == 5 assert payload["disability"]["DOUT"]["minimum_question_age"] == 15 assert payload["disability"]["DOUT"]["codes"]["below_universe_blank"] == -1 @@ -257,6 +292,9 @@ def test_release_join_is_exact_total_clone_stable_and_receipted( "PRDTHSP", "PEIOOCC", "POCCU2", + "WEIND", + "WEMIND", + "WKSWORK", "SPM_TENMORTSTATUS", ], ].drop_duplicates("person_source_id") @@ -265,6 +303,9 @@ def test_release_join_is_exact_total_clone_stable_and_receipted( assert sorted(by_source["PRDTHSP"].tolist()) == [0.0, 0.0, 1.0] assert sorted(by_source["PEIOOCC"].tolist()) == [-1.0, 1005.0, 9800.0] assert sorted(by_source["POCCU2"].tolist()) == [0.0, 8.0, 52.0] + assert sorted(by_source["WEIND"].tolist()) == [0.0, 13.0, 22.0] + assert sorted(by_source["WEMIND"].tolist()) == [0.0, 9.0, 14.0] + assert sorted(by_source["WKSWORK"].tolist()) == [0.0, 48.0, 52.0] assert sorted(by_source["SPM_TENMORTSTATUS"].tolist()) == [1.0, 1.0, 3.0] for _, clones in person.loc[acs].groupby("person_source_id"): @@ -304,6 +345,8 @@ def test_occupation_crosswalk_preserves_age_15_source_universe_gap() -> None: rows["SPORDER"] = [1, 1] rows["AGEP"] = [15, 16] rows["OCCP"] = np.nan + rows["INDP"] = np.nan + rows["WKWN"] = np.nan rows["ESR"] = [np.nan, 6] rows["person_source_id"] = [1, 2] @@ -313,6 +356,62 @@ def test_occupation_crosswalk_preserves_age_15_source_universe_gap() -> None: assert mapped["POCCU2"].tolist() == [0, 53] +def test_industry_crosswalk_preserves_age_15_source_universe_gap() -> None: + rows = pd.concat([_raw_person().iloc[[1]]] * 2, ignore_index=True) + rows["SERIALNO"] = ["age15", "age16"] + rows["SPORDER"] = [1, 1] + rows["AGEP"] = [15, 16] + rows["OCCP"] = np.nan + rows["INDP"] = np.nan + rows["WKWN"] = np.nan + rows["ESR"] = [np.nan, 6] + rows["person_source_id"] = [1, 2] + + mapped = module._crosswalk_people(rows) + + assert mapped["WEIND"].tolist() == [0, 23] + assert mapped["WEMIND"].tolist() == [0, 15] + assert mapped["WKSWORK"].tolist() == [0, 0] + + +def test_industry_is_carried_for_a_job_held_in_the_past_five_years() -> None: + """ACS INDP covers the last 5 years; only the weeks say who worked lately.""" + + row = _raw_person().iloc[[1]].copy() + row["ESR"] = 6 + row["WKWN"] = np.nan + row["person_source_id"] = 1 + + mapped = module._crosswalk_people(row) + + assert mapped[["WEIND", "WEMIND", "WKSWORK"]].to_numpy().tolist() == [[13, 9, 0]] + + +@pytest.mark.parametrize( + ("changes", "match"), + [ + ({"INDP": np.nan}, "INDP is blank inside its observed employment universe"), + ({"INDP": 1}, r"INDP contains unsupported code\(s\): \[1\]"), + ({"INDP": 9920, "OCCP": 9920, "ESR": 3}, "weeks worked without a worker"), + ({"WKWN": 53}, "WKWN code/universe mismatch"), + ( + {"AGEP": 12, "DOUT": np.nan, "OCCP": np.nan, "INDP": np.nan, "ESR": np.nan}, + "WKWN code/universe mismatch", + ), + ], +) +def test_industry_crosswalk_refuses_incoherent_source_rows( + changes: dict[str, object], match: str +) -> None: + row = _raw_person().iloc[[1]].copy() + for column, value in changes.items(): + row[column] = value + row["person_source_id"] = 1 + + with pytest.raises(ValueError, match=match): + module._crosswalk_people(row) + + def test_occupation_crosswalk_refuses_malformed_esr_universe() -> None: row = _raw_person().iloc[[1]].copy() row["ESR"] = "not-a-code" From 1ba349761958beb61142a9881e2ebd1182f2547a Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 25 Sep 2026 15:22:53 -0400 Subject: [PATCH 03/10] Declare the work-experience outputs on org_wages and require them at release source_stages.json's org_wages stage names WEIND, WEMIND and WKSWORK as derive_from_asec inputs and detailed_industry_recode, major_industry_recode and worked_last_year as outputs (the recodes nonnegative). The three join the post-reference required release inputs with notes that today's published defaults lack them by design, and the coverage manifest regenerates (170 required). The bundle regenerates from the manifest authority (sources.yaml, calibration.yaml tail contracts) with the frozen source_stages digest re-pinned. Adds docs/us-work-experience-columns.md (sources, crosswalk, identities, real-data evidence on the pinned ASEC members and all 3,422,888 ACS 2024 persons) and the changelog fragment. Part of #719. Co-Authored-By: Claude Opus 5.5 --- .../719-work-experience-columns.added.md | 1 + docs/us-asec-census-person-columns.md | 6 + docs/us-work-experience-columns.md | 111 ++++++++++++++++++ .../us/release_input_coverage_manifest.json | 16 ++- .../src/microcosm/build/us/source_stages.json | 12 +- .../microcosm/build/us/spec/calibration.yaml | 3 + .../src/microcosm/build/us/spec/sources.yaml | 17 ++- .../us_runtime/release_input_coverage.py | 7 ++ .../tests/test_us_bundle_core_contracts.py | 2 +- .../tests/test_us_spec_bundle.py | 2 +- ...uild_us_release_input_coverage_manifest.py | 34 ++++++ tools/generate_us_bundle_from_constants.py | 2 +- 12 files changed, 204 insertions(+), 9 deletions(-) create mode 100644 changelog.d/719-work-experience-columns.added.md create mode 100644 docs/us-work-experience-columns.md diff --git a/changelog.d/719-work-experience-columns.added.md b/changelog.d/719-work-experience-columns.added.md new file mode 100644 index 0000000000..476e56a338 --- /dev/null +++ b/changelog.d/719-work-experience-columns.added.md @@ -0,0 +1 @@ +US release person records carry `detailed_industry_recode` (CPS `WEIND`), `major_industry_recode` (`WEMIND`) and `worked_last_year` (`WKSWORK > 0`) next to `detailed_occupation_recode`: ASEC vintages restore `WEIND`/`WEMIND` from their pinned Census person members before pooling, ACS-spine people get all three from native `INDP`/`WKWN` through the release predictor join (crosswalk version 2), and the `org_wages` carry emits them with the industry identities checked (#719). diff --git a/docs/us-asec-census-person-columns.md b/docs/us-asec-census-person-columns.md index 73592bac7a..60cd4ee5c7 100644 --- a/docs/us-asec-census-person-columns.md +++ b/docs/us-asec-census-person-columns.md @@ -109,6 +109,12 @@ the column. `CENSUS_TAX_ID` was never appended: the extractor derives it, and the Census files do not carry it. +Beyond that receipt, `WEIND` and `WEMIND` (industry of the longest job last +year) are restored for #719, read by `org_wages.derive_us_org_occupation_inputs`. +No H5 vintage carries them, so every vintage gets them from its member, and +each vintage is checked against the industry identities. See +[`us-work-experience-columns.md`](us-work-experience-columns.md). + ## Real-data evidence (2026-09-23, commit `39b8e7b63`) The real-data run used route A's exact base inputs and flags diff --git a/docs/us-work-experience-columns.md b/docs/us-work-experience-columns.md new file mode 100644 index 0000000000..4094442c0c --- /dev/null +++ b/docs/us-work-experience-columns.md @@ -0,0 +1,111 @@ +# Industry and worked-last-year on every US person record (#719) + +The county-file person schema carries a working indicator, industry and +occupation on every person record. Occupation already shipped +(`detailed_occupation_recode`, CPS ASEC `POCCU2`). This change adds three +PolicyEngine-US person inputs (defined in policyengine-us#9311, released in +1.820.2, locked at 2.2.1): + +| column | meaning | CPS ASEC source | ACS 2024 source | +|---|---|---|---| +| `detailed_industry_recode` | industry of the longest job last year, 23 detailed groups | `WEIND` | `INDP` via `ACS_INDP_TO_WEIND` | +| `major_industry_recode` | the same job, 15 major groups | `WEMIND` | `WEIND_TO_WEMIND` of the above | +| `worked_last_year` | worked at any time last year | `WKSWORK > 0` | `WKWN > 0` | + +Codes (`WEIND`): 0 = not in universe (under 15; ACS: under 16), 1–21 = +civilian industry groups, 22 = Armed Forces, 23 = age 15+ (ACS: 16+) and did +not work last year. `WEMIND` nests them: 0 = not in universe, 1–13 civilian +major groups, 14 = Armed Forces, 15 = did not work. + +## Why these CPS columns + +`WEIND`/`WEMIND` describe the longest job last year, the same job `POCCU2` +describes, and the same reference year as the income the file carries. The +survey-week recodes `A_DTIND`/`A_MJIND` describe the current job instead. +`worked_last_year` uses `WKSWORK > 0` rather than `WORKYN = 1`: in all three +pinned members `WEIND` carries a worker code exactly where `WKSWORK > 0` (zero +exceptions), while `WORKYN = 1` misses 642 / 572 / 594 allocation rows per +year that have weeks worked and an industry. + +## Where each spine gets them + +- **ASEC rows (and their PUF-support clones).** No `census_cps` H5 vintage + carries `WEIND`/`WEMIND`, not even 2024. `asec_census_person_columns` + (#720's restoration) appends both to every vintage from its pinned Census + person member by exact `PERIDNUM` join before pooling. They sit outside the + 2026-08-23 offline-fix receipt (`ASEC_CENSUS_PERSON_COLUMNS_BEYOND_OFFLINE_FIX`). + `WKSWORK` is already in every H5. +- **ACS rows.** The release predictor join (`acs_release_predictors`, + crosswalk version 2) materializes `WEIND`, `WEMIND` and `WKSWORK` for ACS + people from the pinned ACS 2024 person archive, the same boundary that maps + `OCCP` to `POCCU2`. +- **Every row.** The release-time `org_wages` carry + (`derive_us_org_occupation_inputs`) turns them into the three columns next + to `detailed_occupation_recode`. + +## The ACS crosswalk + +ACS 2024 `INDP` uses the 2022 Census industry code list, as does the 2025 ASEC +member (`pppub25.csv`), where `INDUSTRY` determines `WEIND` exactly (259 +codes). The 2023/2024 members use the 2017 list; the 225 codes the lists share +map to the same group in both. `ACS_INDP_TO_WEIND` takes every civilian code's +group from `pppub25.csv` and reviews the eight ACS-only codes: + +- 9670, 9680, 9690, 9770, 9780, 9790, 9870 (Army, Air Force, Navy, Marines, + Coast Guard, branch not specified, Reserves or National Guard) → 22. CPS + collapses them into 9890, which the members map to 22. +- 9920 ("unemployed, no work experience in the last 5 years or never + worked") → 23, as `OCCP` 9920 maps to `POCCU2` 53. + +Blank `INDP` follows the `OCCP` rule: 0 below age 16 (age 15 keeps the +out-of-universe sentinel rather than inventing CPS code 23), 23 from age 16, +where only ESR 6 may be blank. `WKSWORK` is `WKWN` (weeks worked in the past 12 +months) or 0. + +Reference periods differ, as they already do for occupation: an observed ACS +`INDP` describes the most recent job in the past five years, and `WKWN` counts +the 12 months before interview; the CPS columns describe the prior calendar +year. So on ACS rows an industry can sit on a person with +`worked_last_year = False` (a job held one to five years ago). + +## Checked identities + +| identity | ASEC rows | ACS rows | where checked | +|---|---|---|---| +| `WEMIND = WEIND_TO_WEMIND[WEIND]` | yes | yes | restoration (per vintage), ACS join (by construction), `org_wages` carry and gate | +| `WKSWORK > 0` ⇒ `WEIND` in 1–22 | yes | yes | restoration, ACS join, `org_wages` carry and gate | +| `WEIND` in 1–22 ⇒ `WKSWORK > 0` | yes | no (5-year `INDP`) | restoration only, where rows are known to be ASEC | + +## Real-data evidence (2026-09-24) + +ASEC members `pppub23/24/25.csv` (pins in `spm_role_source.ASEC_SPM_ROLE_SOURCES`): + +- `INDUSTRY` → `WEIND` and → `WEMIND` are functions in every vintage (263 / + 263 / 259 codes); `WEIND` → `WEMIND` is the same function in all three. +- `WEIND` in 1–22 iff `INDUSTRY > 0` iff `WKSWORK > 0`: zero violations in + each vintage. `WEIND = 0` exactly for ages 0–14; `WEIND = 23` only from 15. +- Weighted (`A_FNLWGT`) shares: nonzero `WEIND` 0.8217 / 0.8220 / 0.8244; + worked last year 0.5200 / 0.5232 / 0.5228. + +ACS 2024 one-year person archive (SHA-256 `afdc6d90…`, the release pin), all +3,422,888 persons run through `_crosswalk_people`: + +- No refusal fires; all 265 observed `INDP` codes are mapped. +- `INDP` and `OCCP` are blank on exactly the same rows; a blank at 16+ occurs + only with ESR 6. +- `WKWN > 0` iff `WKL = 1` (zero disagreements); every person with weeks + worked has a worker `WEIND` code. +- 264,806 persons carry an industry without weeks worked, exactly the count + with `WKL = 2` (last worked one to five years ago). +- Weighted (`PWGTP`) shares: nonzero `WEIND` 0.8123; worked last year 0.5385. + +The `org_wages` gate bands (`detailed_industry_recode` and +`major_industry_recode` nonzero 0.70–0.92, `worked_last_year` 0.40–0.62) +bracket both spines. + +## What this does not do + +It changes code and manifests only. Released files carry the columns once the +pooled base and multispine pool are rebuilt with the restoration and a new +release runs (#922 sequencing step 1); until then the release input-coverage +gate is red on these three columns by design. diff --git a/packages/microcosm-build/src/microcosm/build/us/release_input_coverage_manifest.json b/packages/microcosm-build/src/microcosm/build/us/release_input_coverage_manifest.json index 79218461f1..a0dfd28fde 100644 --- a/packages/microcosm-build/src/microcosm/build/us/release_input_coverage_manifest.json +++ b/packages/microcosm-build/src/microcosm/build/us/release_input_coverage_manifest.json @@ -56,6 +56,10 @@ "cps_race": { "status": "required" }, + "detailed_industry_recode": { + "note": "Industry of the longest job last year, 23 CPS detailed groups, carried by the org_wages release stage (org_wages.derive_us_org_occupation_inputs) from ASEC WEIND (restored before pooling by asec_census_person_columns) and, on ACS-spine people, from native INDP/WKWN through the release predictor join; a formula-less person input in PolicyEngine-US 2.2.1. Required with NO reviewed exclusion per PolicyEngine/microcosm#719: the county-file person schema carries a working indicator, industry and occupation on every record. Currently absent from every published US default (all predate #719) — this is the intended red gate until the next release is built with the carry.", + "status": "required" + }, "detailed_occupation_recode": { "status": "required" }, @@ -272,6 +276,10 @@ "long_term_capital_gains_on_collectibles": { "status": "required" }, + "major_industry_recode": { + "note": "Industry of the longest job last year, 15 CPS major groups, carried by the org_wages release stage (org_wages.derive_us_org_occupation_inputs) from ASEC WEMIND (restored before pooling by asec_census_person_columns) and, on ACS-spine people, from native INDP/WKWN through the release predictor join; a formula-less person input in PolicyEngine-US 2.2.1. Required with NO reviewed exclusion per PolicyEngine/microcosm#719: the county-file person schema carries a working indicator, industry and occupation on every record. Currently absent from every published US default (all predate #719) — this is the intended red gate until the next release is built with the carry.", + "status": "required" + }, "meets_ssi_disability_criteria": { "status": "required" }, @@ -542,6 +550,10 @@ "weeks_unemployed": { "status": "required" }, + "worked_last_year": { + "note": "Worked at any time last year, carried by the org_wages release stage (org_wages.derive_us_org_occupation_inputs) from ASEC WKSWORK > 0 (restored before pooling by asec_census_person_columns) and, on ACS-spine people, from native INDP/WKWN through the release predictor join; a formula-less person input in PolicyEngine-US 2.2.1. Required with NO reviewed exclusion per PolicyEngine/microcosm#719: the county-file person schema carries a working indicator, industry and occupation on every record. Currently absent from every published US default (all predate #719) — this is the intended red gate until the next release is built with the carry.", + "status": "required" + }, "workers_compensation": { "status": "required" }, @@ -550,9 +562,9 @@ } }, "counts": { - "required": 167, + "required": 170, "reviewed_exclusion": 7, - "total": 174 + "total": 177 }, "derivation": "Required surface = input columns in the pinned, sha-verified ecps_parity_reference.json populated layers, plus the documented post-reference fsla_overtime_premium, qualified_passenger_vehicle_loan_interest, five desired retirement-contribution inputs, meets_ssi_disability_criteria required by shipped validation probes, the #282 Schedule-D capital-gain-distributions route leg schedule_d_capital_gain_distributions (PolicyEngine/microcosm#462), the three ASEC reported-receipt inputs receives_wic, receives_snap, receives_tanf that the ACS local-area transfer requires in its donor (PolicyEngine/microcosm#978 option 1), and the engine's declared dataset source input is_spm_independent_minor_role (the spm_independence_role base-builder stage; PolicyEngine/microcosm#893). status='reviewed_exclusion' for ecps_parity_known_gaps.json entries (reason+issue from that register); EXCEPT every primary-source restoration pinned by RESTORED_REFERENCE_ECPS_REQUIRED_INPUTS (including the Section 199A QBI family), and the SSI countable-resource asset inputs (bank_account_assets, stock_assets, bond_assets), which are status='required' with NO exclusion per PolicyEngine/microcosm#368 so the gate fails on today's artifacts and asset restoration (Deliverable 2) turns it green. All other populated layers are 'required'. Regenerate with tools/build_us_release_input_coverage_manifest.py.", "description": "Declared full-coverage contract for a US release: every input column the reference eCPS exports must be persisted as a key with non-default signal, or carry a reviewed exclusion. Enforced as a hard release gate (microcosm.build.us_runtime.release_input_coverage) that generalizes assert_required_us_release_source_columns from 5 columns to the full eCPS input surface.", diff --git a/packages/microcosm-build/src/microcosm/build/us/source_stages.json b/packages/microcosm-build/src/microcosm/build/us/source_stages.json index 8b5d06c0c8..00b73bb37f 100644 --- a/packages/microcosm-build/src/microcosm/build/us/source_stages.json +++ b/packages/microcosm-build/src/microcosm/build/us/source_stages.json @@ -1702,7 +1702,10 @@ "inputs": [ "PRDTRACE", "PRDTHSP", - "POCCU2" + "POCCU2", + "WEIND", + "WEMIND", + "WKSWORK" ] }, { @@ -1726,6 +1729,9 @@ "cps_race", "is_hispanic", "detailed_occupation_recode", + "detailed_industry_recode", + "major_industry_recode", + "worked_last_year", "has_never_worked", "is_military", "is_computer_scientist", @@ -1739,10 +1745,12 @@ "nonnegative_outputs": [ "cps_race", "detailed_occupation_recode", + "detailed_industry_recode", + "major_industry_recode", "hourly_wage", "fsla_overtime_premium" ], - "notes": "Port of the final retired ORG/FLSA method at 9a823603e6b5fb916d65ec45d74c9c7eb0043db1: donor build, weighted QRF for hourly wage and hourly-pay status, archived deterministic BLS 2024 state-rate union assignment, and direct ASEC carries of CPS race, Hispanic status, and FLSA occupation groups are all exact — except the recipient-side QRF income feature, which is deliberately concept-aligned to the donor: the donor's employment_income is the annualized reference week (pternwa x 52), so recipients feed the full-year-equivalent of actual annual income (income x 52/weeks_worked, income untouched when weeks are zero or missing); without this, part-year workers matched low-wage full-year donors and the imputed wage distribution carried a severe sub-minimum tail (microcosm#529 receipts). The twelve official 2024 CPS basic-month files are transformed into the upstream 119,237-row cache; its canonical CSV content hash is pinned so a source reissue fails closed. Usual hours remain owned by hours_worked. fsla_overtime_premium keeps the upstream variable spelling and the archived annual-wage-share formula but deliberately deviates from the retired reference-week-only carrier gate: the retired construction annualized the March survey week (A_HRS1, outside the retrospective income year) and so collapsed incidence to one week's draw — 19.9M weighted carriers on certified Build N against Treasury's published '>29 million' TY2025 claimant floor (sb0517/sb0553, rounded floors, not mintable as exact-valued ledger facts). The two-signal estimator makes usual weekly hours worked last year (HRSWK) the persistent-overtime leg — usual>threshold carries with the usual-hours share as annualizer — while usual<=threshold retains the retired reference-week snapshot leg unchanged: representative for steady schedules, under- or over-recovering an occasional overtimer's annual premium depending on the sampled week (not an unbiased mass estimator), retained because it is the only measured signal for that population. On ASEC-channel rows HRSWK shares the retrospective year with employment_income and weeks_worked; on PUF-support-channel rows both hours signals are donor-ASEC schedule proxies attached to PUF income, the support design's property inherited equally by the reference-week leg. A frame whose ORG surface is already populated is fully re-imputed and compared on every entry: an identical surface passes through unchanged (the seeded QRF, deterministic union lottery, and pure derives make same-code re-entry byte-stable), and any stale construction of any output converges instead of silently surviving. The income-feature scaling also changes the deterministic union lottery's hash inputs, so which individuals carry union coverage reshuffles within unchanged state-level quotas — the assignment method itself remains the exact archived port. Exemption screens unchanged; both hours measures count all jobs while FLSA section 7 applies per employer (inherited upward concept edge). Adjudication and measurements: microcosm#451 item 4 overtime-incidence lane." + "notes": "Port of the final retired ORG/FLSA method at 9a823603e6b5fb916d65ec45d74c9c7eb0043db1: donor build, weighted QRF for hourly wage and hourly-pay status, archived deterministic BLS 2024 state-rate union assignment, and direct ASEC carries of CPS race, Hispanic status, and FLSA occupation groups are all exact — except the recipient-side QRF income feature, which is deliberately concept-aligned to the donor: the donor's employment_income is the annualized reference week (pternwa x 52), so recipients feed the full-year-equivalent of actual annual income (income x 52/weeks_worked, income untouched when weeks are zero or missing); without this, part-year workers matched low-wage full-year donors and the imputed wage distribution carried a severe sub-minimum tail (microcosm#529 receipts). The twelve official 2024 CPS basic-month files are transformed into the upstream 119,237-row cache; its canonical CSV content hash is pinned so a source reissue fails closed. Usual hours remain owned by hours_worked. fsla_overtime_premium keeps the upstream variable spelling and the archived annual-wage-share formula but deliberately deviates from the retired reference-week-only carrier gate: the retired construction annualized the March survey week (A_HRS1, outside the retrospective income year) and so collapsed incidence to one week's draw — 19.9M weighted carriers on certified Build N against Treasury's published '>29 million' TY2025 claimant floor (sb0517/sb0553, rounded floors, not mintable as exact-valued ledger facts). The two-signal estimator makes usual weekly hours worked last year (HRSWK) the persistent-overtime leg — usual>threshold carries with the usual-hours share as annualizer — while usual<=threshold retains the retired reference-week snapshot leg unchanged: representative for steady schedules, under- or over-recovering an occasional overtimer's annual premium depending on the sampled week (not an unbiased mass estimator), retained because it is the only measured signal for that population. On ASEC-channel rows HRSWK shares the retrospective year with employment_income and weeks_worked; on PUF-support-channel rows both hours signals are donor-ASEC schedule proxies attached to PUF income, the support design's property inherited equally by the reference-week leg. A frame whose ORG surface is already populated is fully re-imputed and compared on every entry: an identical surface passes through unchanged (the seeded QRF, deterministic union lottery, and pure derives make same-code re-entry byte-stable), and any stale construction of any output converges instead of silently surviving. The income-feature scaling also changes the deterministic union lottery's hash inputs, so which individuals carry union coverage reshuffles within unchanged state-level quotas — the assignment method itself remains the exact archived port. Exemption screens unchanged; both hours measures count all jobs while FLSA section 7 applies per employer (inherited upward concept edge). Adjudication and measurements: microcosm#451 item 4 overtime-incidence lane. Work-experience carries (microcosm#719): WEIND and WEMIND (industry of the longest job last year, restored before pooling from each vintage's pinned Census person member by the asec_census_person_columns restoration) become detailed_industry_recode and major_industry_recode, and worked_last_year is WKSWORK > 0; ACS-spine people get all three from native INDP/WKWN through the release predictor join. Checked on every spine: WEMIND is the major group of WEIND and positive weeks carry a worker industry code (1-22); the converse holds on ASEC rows and is checked before pooling." }, { "stage": "meps_esi_premiums", diff --git a/packages/microcosm-build/src/microcosm/build/us/spec/calibration.yaml b/packages/microcosm-build/src/microcosm/build/us/spec/calibration.yaml index 128371e097..146fc32131 100644 --- a/packages/microcosm-build/src/microcosm/build/us/spec/calibration.yaml +++ b/packages/microcosm-build/src/microcosm/build/us/spec/calibration.yaml @@ -267,6 +267,7 @@ tail_contracts: - charitable_cash_donations - charitable_non_cash_donations - cps_race + - detailed_industry_recode - detailed_occupation_recode - domestic_production_ald - educator_expense @@ -297,6 +298,7 @@ tail_contracts: - is_union_member_or_covered - long_term_capital_gains_before_response - long_term_capital_gains_on_collectibles + - major_industry_recode - meets_ssi_disability_criteria - miscellaneous_income - net_worth @@ -361,5 +363,6 @@ tail_contracts: - unrecaptured_section_1250_gain - unreimbursed_business_employee_expenses - w2_wages_from_qualified_business + - worked_last_year - would_file_taxes_voluntarily reviewed_exclusions_default: {} diff --git a/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml b/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml index b952fd60e4..1d46d78329 100644 --- a/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml +++ b/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml @@ -99,7 +99,7 @@ sources: stage_asset: id: source_stages path: microcosm.build.us/source_stages.json - sha256: 9f4f983091fb18dea8f4524cc100be20f9fca2b006435461592f8edf53c9a5d8 + sha256: 2074a1b918aec8a74d181ef9a946fa930936993d57acb23fc8a6babdf2e4afb5 stage_manifest: version: 1 country: us @@ -1720,6 +1720,9 @@ stages: - PRDTRACE - PRDTHSP - POCCU2 + - WEIND + - WEMIND + - WKSWORK - kind: derive method: derive_flsa_overtime_premium inputs: @@ -1737,6 +1740,9 @@ stages: - cps_race - is_hispanic - detailed_occupation_recode + - detailed_industry_recode + - major_industry_recode + - worked_last_year - has_never_worked - is_military - is_computer_scientist @@ -1749,6 +1755,8 @@ stages: nonnegative_outputs: - cps_race - detailed_occupation_recode + - detailed_industry_recode + - major_industry_recode - hourly_wage - fsla_overtime_premium notes: 'Port of the final retired ORG/FLSA method at 9a823603e6b5fb916d65ec45d74c9c7eb0043db1: donor @@ -1781,7 +1789,12 @@ stages: reshuffles within unchanged state-level quotas — the assignment method itself remains the exact archived port. Exemption screens unchanged; both hours measures count all jobs while FLSA section 7 applies per employer (inherited upward concept edge). Adjudication and measurements: microcosm#451 item 4 - overtime-incidence lane.' + overtime-incidence lane. Work-experience carries (microcosm#719): WEIND and WEMIND (industry of the + longest job last year, restored before pooling from each vintage''s pinned Census person member by + the asec_census_person_columns restoration) become detailed_industry_recode and major_industry_recode, + and worked_last_year is WKSWORK > 0; ACS-spine people get all three from native INDP/WKWN through + the release predictor join. Checked on every spine: WEMIND is the major group of WEIND and positive + weeks carry a worker industry code (1-22); the converse holds on ASEC rows and is checked before pooling.' - stage: meps_esi_premiums survey: MEPS-IC source: https://meps.ahrq.gov/mepsweb/survey_comp/Insurance.jsp diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/release_input_coverage.py b/packages/microcosm-build/src/microcosm/build/us_runtime/release_input_coverage.py index 516c535e6f..58985d848a 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/release_input_coverage.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/release_input_coverage.py @@ -188,6 +188,13 @@ # pinned Census ASEC person files; the certified default carries it # through the Build P source enrichment. *US_SPM_INDEPENDENCE_ROLE_OUTPUT_COLUMNS, + # PolicyEngine/microcosm#719: the county-file person schema carries a + # working indicator and industry with occupation on every record. + # Written by the org_wages release stage from ASEC WEIND/WEMIND/WKSWORK + # (ACS spine: native INDP/WKWN through the release predictor join). + "detailed_industry_recode", + "major_industry_recode", + "worked_last_year", } ) diff --git a/packages/microcosm-build/tests/test_us_bundle_core_contracts.py b/packages/microcosm-build/tests/test_us_bundle_core_contracts.py index 66569c76ad..dac1834c8d 100644 --- a/packages/microcosm-build/tests/test_us_bundle_core_contracts.py +++ b/packages/microcosm-build/tests/test_us_bundle_core_contracts.py @@ -129,7 +129,7 @@ def test_source_surface_classification_is_complete() -> None: } assert normative["stage_asset"] == { "id": "source_stages", - "sha256": "9f4f983091fb18dea8f4524cc100be20f9fca2b006435461592f8edf53c9a5d8", + "sha256": "2074a1b918aec8a74d181ef9a946fa930936993d57acb23fc8a6babdf2e4afb5", } assert operational["stage_asset"] == { "path": "microcosm.build.us/source_stages.json" diff --git a/packages/microcosm-build/tests/test_us_spec_bundle.py b/packages/microcosm-build/tests/test_us_spec_bundle.py index f47ae69bd4..6f13630e78 100644 --- a/packages/microcosm-build/tests/test_us_spec_bundle.py +++ b/packages/microcosm-build/tests/test_us_spec_bundle.py @@ -191,7 +191,7 @@ def _load_generator_module(): LEGACY_COMPATIBILITY_SHA256 = { "source_stages.json": ( - "9f4f983091fb18dea8f4524cc100be20f9fca2b006435461592f8edf53c9a5d8" + "2074a1b918aec8a74d181ef9a946fa930936993d57acb23fc8a6babdf2e4afb5" ), "support_spine.json": ( "68f37dc6ae6e0cde7ebccb53f88dd4a800e63456f838fa214ff98d1db8d815be" diff --git a/tools/build_us_release_input_coverage_manifest.py b/tools/build_us_release_input_coverage_manifest.py index ee62c43c8f..16f5f248d8 100644 --- a/tools/build_us_release_input_coverage_manifest.py +++ b/tools/build_us_release_input_coverage_manifest.py @@ -94,6 +94,12 @@ "receives_wic", "receives_snap", "receives_tanf", + # PolicyEngine/microcosm#719: the work-experience carries the county-file + # person schema commits to (industry and a working indicator, with + # occupation), written by the org_wages release stage. + "detailed_industry_recode", + "major_industry_recode", + "worked_last_year", ) # Shared tail of the three #978 reported-receipt notes: why each is a hard @@ -192,6 +198,34 @@ "cps_carried.derive_us_cps_carried_inputs; a formula-less monthly " "boolean spm_unit input in PolicyEngine-US 2.2.1. " + _RECEIPT_INPUT_NOTE_TAIL ), + **{ + column: ( + f"{description}, carried by the org_wages release stage " + "(org_wages.derive_us_org_occupation_inputs) from ASEC " + f"{source} (restored before pooling by asec_census_person_columns) " + "and, on ACS-spine people, from native INDP/WKWN through the " + "release predictor join; a formula-less person input in " + "PolicyEngine-US 2.2.1. Required with NO reviewed exclusion per " + "PolicyEngine/microcosm#719: the county-file person schema " + "carries a working indicator, industry and occupation on every " + "record. Currently absent from every published US default (all " + "predate #719) — this is the intended red gate until the next " + "release is built with the carry." + ) + for column, description, source in ( + ( + "detailed_industry_recode", + "Industry of the longest job last year, 23 CPS detailed groups", + "WEIND", + ), + ( + "major_industry_recode", + "Industry of the longest job last year, 15 CPS major groups", + "WEMIND", + ), + ("worked_last_year", "Worked at any time last year", "WKSWORK > 0"), + ) + }, } QBI_INPUTS = ( diff --git a/tools/generate_us_bundle_from_constants.py b/tools/generate_us_bundle_from_constants.py index 131991f695..4fe0521c9c 100644 --- a/tools/generate_us_bundle_from_constants.py +++ b/tools/generate_us_bundle_from_constants.py @@ -122,7 +122,7 @@ def _domain_builders() -> dict[str, Callable[[], object]]: # frozen files, is the forward YAML -> legacy-payload path. FROZEN_LEGACY_RESOURCE_SHA256 = { "source_stages.json": ( - "9f4f983091fb18dea8f4524cc100be20f9fca2b006435461592f8edf53c9a5d8" + "2074a1b918aec8a74d181ef9a946fa930936993d57acb23fc8a6babdf2e4afb5" ), "support_spine.json": ( "68f37dc6ae6e0cde7ebccb53f88dd4a800e63456f838fa214ff98d1db8d815be" From bc34948e719c1e74d532a5b56e734a7c875fde27 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 26 Sep 2026 08:28:47 -0400 Subject: [PATCH 04/10] Add work-experience property tests; re-cut the inventory vector Hypothesis property tests state and check the #719 invariants over generated inputs: WEMIND nests WEIND, positive weeks carry a worker industry code, the ASEC-only converse, the release carry's round trip, and the ACS crosswalk's universe rules. A differential test runs the same rows through the ASEC restoration and the release carry and requires each to accept exactly when its stated identities hold (the restoration is the stricter site by design). The spec-engine inventory vector moves with the org_wages manifest change: source-manifest, resource-semantics, full-checkpoint and tail digests, and 937 tail control fields (+3 tail-contract columns). Part of #719. Co-Authored-By: Claude Opus 5.5 --- .../build/spec_engine/inventory_coverage.py | 14 +- .../test_us_work_experience_properties.py | 227 ++++++++++++++++++ 2 files changed, 234 insertions(+), 7 deletions(-) create mode 100644 packages/microcosm-build/tests/test_us_work_experience_properties.py diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py index e3e5769c92..72ac275328 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py @@ -350,20 +350,20 @@ "acs_person_predictors": "878c788a6f037d7aca12b3586ea034eff04f3034ffa11935a736493042551f25", "authority": "9d4a9672a0f03039b1fe874b9fe21ed575be0d29f14afc396d03cdf5c809bdd2", "early_families": "4aa9f736fd76e83955477ad1667e58f48f264783f05bdc7f0102cd32d61323bd", - "full_checkpoint": "b1d5c65254b51897b27f0f5e44e344781ce9e01e305aaf1bfc042cc32bc417dc", + "full_checkpoint": "3bf611d9efdba2a2edd813f0d4a39739d397c4769e9d47767996fea941f05a42", "gap_fill_schedule": "1c31f9868f7884347cc19cf1ff65da43f950b9114941a715bab168246db414a7", "graph_nodes": "40cd51ffdfe2e9d9d08d48c08e8ded9de1e4b134783bab05c4abc6ad5c72ca1e", "geography_assignment": "f49425ca8734ac559c73cf44f6458d86d3162a48956b98a27e6e758959361585", "late_families": "d91f9ff0eb52f43e7b6eed3d5c58c37abe1620c3a11021da15dae9c10e16d382", - "late_resource_semantics": "6abbfad73014ce3bac9fbd549f16e4a5e645a3b0fbd450b56e186876f3f0dd95", + "late_resource_semantics": "397adfd4f4efb4450c17bec74155e0f56b55f2729198b0e1f792bfbeb1010cd7", "late_schedule": "e59c019d3d454eac99ac0ac209b6c5b6faaf9bdfcaeee18c36a25be19bf7da2f", "ownership": "5f64f0aac49e2313177564f71876bffc8c81b3ded4df701e70930e60e9c98356", "primary_tuples": "fdf23da429f8c501e198a2f5b719763d523a41565c2153fdf17b0344f21bcd7c", "seed_map": "dad44808366198b8826823b837c7b592c9ecee2299adbd9f8a20251dd6d0b00a", "seed_protocol": "5f6d3b527060bc6f15f1871c64a42994d4882ee5962f3350ea8a71a83ffcd6bf", - "source_manifest": "a87278445d928ea20e15935a8cc07312d6e819e34409b04e8c814f17910d828f", + "source_manifest": "ee7b7541a6925351d4acbfa0064f9586dd14c095e9208c38f492a83bb8a1afa1", "take_up": "9522ce40f7dea569312dd7a7beb474e5a5afadd2f1e10cb5876534c3ef623d35", - "tail": "ac92829c88a1a4fb6460d61190918d5d99c6c377fc8dd8f62f02b332d09bf59c", + "tail": "256947cd7c1bbd85f5d1fa3911a07a170ae87830f098110a40ed25d93f78347c", } INVENTORY_REPORT_SCHEMA_VERSION = 1 @@ -443,7 +443,7 @@ "stacked_checkpoint_full_components": 13, "stacked_checkpoint_pool_code_components": 19, "stacked_checkpoint_static_components": 10, - "tail_control_fields": 934, + "tail_control_fields": 937, "take_up_pipeline_steps": 28, "take_up_programs": 17, "typed_artifacts": 84, @@ -1411,7 +1411,7 @@ def add( add( "capital_gains_tail_contract_exact", clauses={ - "tail terminal field count differs": tail_count == 934, + "tail terminal field count differs": tail_count == 937, "resolved calibration tails differ": _json_equal( legacy.get("calibration_tail_contracts"), expected_legacy["calibration_tail_contracts"], @@ -1439,7 +1439,7 @@ def add( "legacy_adapter.stacked_checkpoint_static_components", ), observed={"terminal_fields": tail_count, "sha256": sha256_json(tail_bundle)}, - expected={"terminal_fields": 934, "sha256": EXPECTED_HASHES["tail"]}, + expected={"terminal_fields": 937, "sha256": EXPECTED_HASHES["tail"]}, ) add( diff --git a/packages/microcosm-build/tests/test_us_work_experience_properties.py b/packages/microcosm-build/tests/test_us_work_experience_properties.py new file mode 100644 index 0000000000..327cb2fc48 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_work_experience_properties.py @@ -0,0 +1,227 @@ +"""Property and differential tests for the #719 work-experience columns. + +Invariants, for every input the three producers accept: + +* **Nesting.** ``WEMIND == WEIND_TO_WEMIND[WEIND]``: the major industry group + is a function of the detailed one. +* **Forward worker identity.** ``WKSWORK > 0`` implies ``WEIND`` in 1--22 + (someone who worked last year has a worker industry code). Holds on every + spine. +* **Converse worker identity (ASEC only).** ``WEIND`` in 1--22 implies + ``WKSWORK > 0``. ACS ``INDP`` covers a job held in the past five years, so + the release carry deliberately does not check it (intended asymmetry). +* **Round trip.** The release carry emits the inputs unchanged as + ``detailed_industry_recode``/``major_industry_recode`` and + ``worked_last_year == (WKSWORK > 0)``. +* **ACS universe.** Below age 16 every ACS output is 0; a blank ``INDP`` from + age 16 is the did-not-work group 23; an observed ``INDP`` maps through + ``ACS_INDP_TO_WEIND``; ``WKSWORK`` is ``WKWN`` or 0. + +The differential tests run the same generated rows through every site that +enforces an identity and require the sites to agree. +""" + +from __future__ import annotations + +import numpy as np +import pandas as pd +import pytest +from hypothesis import given, settings +from hypothesis import strategies as st + +import microcosm.build.us_runtime.acs_release_predictors as acs +from microcosm.build.us_runtime.asec_census_person_columns import ( + WEIND_TO_WEMIND, + AsecCensusPersonColumnsError, + _check_work_experience_universe, +) +from microcosm.build.us_runtime.org_wages import derive_us_org_occupation_inputs + +_WORKER = range(1, 23) +_SETTINGS = settings(max_examples=200, deadline=None) + + +def _carry_frame(weind, wemind, wkswork) -> pd.DataFrame: + rows = len(weind) + return pd.DataFrame( + { + "PRDTRACE": np.ones(rows, dtype=np.int64), + "PRDTHSP": np.zeros(rows, dtype=np.int64), + "POCCU2": np.full(rows, 8, dtype=np.int64), + "WEIND": np.asarray(weind, dtype=np.int64), + "WEMIND": np.asarray(wemind, dtype=np.int64), + "WKSWORK": np.asarray(wkswork, dtype=np.int64), + } + ) + + +@st.composite +def _asec_rows(draw) -> tuple[list[int], list[int], list[int]]: + """Rows that satisfy both worker identities and the nesting.""" + + weind = draw(st.lists(st.integers(0, 23), min_size=1, max_size=40)) + weeks = [draw(st.integers(1, 52)) if code in _WORKER else 0 for code in weind] + return weind, [WEIND_TO_WEMIND[code] for code in weind], weeks + + +@st.composite +def _arbitrary_rows(draw) -> tuple[list[int], list[int], list[int]]: + """In-range codes with no identity promised.""" + + rows = draw(st.integers(1, 40)) + return ( + draw(st.lists(st.integers(0, 23), min_size=rows, max_size=rows)), + draw(st.lists(st.integers(0, 15), min_size=rows, max_size=rows)), + draw(st.lists(st.integers(0, 52), min_size=rows, max_size=rows)), + ) + + +def _nested(weind, wemind) -> bool: + return all(WEIND_TO_WEMIND[d] == m for d, m in zip(weind, wemind, strict=True)) + + +def _forward(weind, wkswork) -> bool: + return all(d in _WORKER for d, w in zip(weind, wkswork, strict=True) if w > 0) + + +def _converse(weind, wkswork) -> bool: + return all(w > 0 for d, w in zip(weind, wkswork, strict=True) if d in _WORKER) + + +def _carry_accepts(weind, wemind, wkswork) -> bool: + try: + derive_us_org_occupation_inputs(_carry_frame(weind, wemind, wkswork)) + except ValueError: + return False + return True + + +def _restore_accepts(weind, wemind, wkswork) -> bool: + try: + _check_work_experience_universe( + _carry_frame(weind, wemind, wkswork), "generated vintage" + ) + except AsecCensusPersonColumnsError: + return False + return True + + +@_SETTINGS +@given(_asec_rows()) +def test_release_carry_round_trips_coherent_rows(rows) -> None: + weind, wemind, wkswork = rows + out = derive_us_org_occupation_inputs(_carry_frame(weind, wemind, wkswork)) + assert out["detailed_industry_recode"].tolist() == weind + assert out["major_industry_recode"].tolist() == wemind + assert out["worked_last_year"].tolist() == [w > 0 for w in wkswork] + assert out["detailed_industry_recode"].dtype == np.int16 + assert out["worked_last_year"].dtype == bool + + +@_SETTINGS +@given(_arbitrary_rows()) +def test_restore_and_carry_accept_exactly_their_stated_identities(rows) -> None: + """Differential: each site accepts iff its stated identities hold.""" + + weind, wemind, wkswork = rows + nested = _nested(weind, wemind) + forward = _forward(weind, wkswork) + converse = _converse(weind, wkswork) + assert _restore_accepts(weind, wemind, wkswork) == (nested and forward and converse) + assert _carry_accepts(weind, wemind, wkswork) == (nested and forward) + # Intended asymmetry: the ASEC-only restoration is the stricter site. + if _restore_accepts(weind, wemind, wkswork): + assert _carry_accepts(weind, wemind, wkswork) + + +def _acs_template() -> dict[str, object]: + return { + "SERIALNO": "2024HU0000001", + "SPORDER": 1, + "DEAR": 2, + "DEYE": 2, + "DREM": 2, + "DPHY": 2, + "DDRS": 2, + "DOUT": 2, + "RAC1P": 1, + "HISP": 1, + "SSIP": 0.0, + "ADJINC": 1_000_000, + } + + +_OBSERVED_INDP = sorted(code for code in acs.ACS_INDP_TO_WEIND if code != 9920) + + +@st.composite +def _acs_person(draw) -> dict[str, object]: + """One ACS person inside the source universes the join admits.""" + + row = _acs_template() + age = draw(st.integers(0, 95)) + row["AGEP"] = age + if age < 16: + for column in ("DREM", "DPHY", "DDRS") if age < 5 else (): + row[column] = np.nan + if age < 15: + row["DOUT"] = np.nan + row.update(OCCP=np.nan, INDP=np.nan, WKWN=np.nan, ESR=np.nan) + return row + kind = draw(st.sampled_from(["worked", "past_five_years", "never", "unemployed"])) + if kind == "worked": + row.update( + INDP=draw(st.sampled_from(_OBSERVED_INDP)), + OCCP=1005, + ESR=draw(st.sampled_from([1, 2, 4, 5])), + WKWN=draw(st.integers(1, 52)), + ) + elif kind == "past_five_years": + row.update( + INDP=draw(st.sampled_from(_OBSERVED_INDP)), OCCP=1005, ESR=6, WKWN=np.nan + ) + elif kind == "never": + row.update(INDP=np.nan, OCCP=np.nan, ESR=6, WKWN=np.nan) + else: + row.update(INDP=9920, OCCP=9920, ESR=3, WKWN=np.nan) + return row + + +@_SETTINGS +@given(st.lists(_acs_person(), min_size=1, max_size=30)) +def test_acs_crosswalk_satisfies_every_stated_invariant(people) -> None: + joined = pd.DataFrame(people) + joined["person_source_id"] = np.arange(len(joined)) + mapped = acs._crosswalk_people(joined) + + weind = mapped["WEIND"].astype(int).tolist() + wemind = mapped["WEMIND"].astype(int).tolist() + wkswork = mapped["WKSWORK"].astype(int).tolist() + assert _nested(weind, wemind) + assert _forward(weind, wkswork) + for person, detailed, weeks in zip(people, weind, wkswork, strict=True): + if person["AGEP"] < 16: + assert (detailed, weeks) == (0, 0) + elif pd.isna(person["INDP"]): + assert detailed == 23 + else: + assert detailed == acs.ACS_INDP_TO_WEIND[int(person["INDP"])] + expected_weeks = 0 if pd.isna(person["WKWN"]) else int(person["WKWN"]) + assert weeks == expected_weeks + # The join's rows are admissible to the release carry that consumes them. + assert _carry_accepts(weind, wemind, wkswork) + + +def test_every_crosswalk_code_nests_into_its_major_group() -> None: + """Differential against the pinned nesting for all 267 table codes.""" + + for code, detailed in acs.ACS_INDP_TO_WEIND.items(): + assert detailed in _WORKER or detailed == 23, code + assert WEIND_TO_WEMIND[detailed] in range(1, 16), code + + +@pytest.mark.parametrize("weeks", [1, 52]) +def test_armed_forces_and_did_not_work_codes_bracket_the_worker_range(weeks) -> None: + assert _carry_accepts([22], [14], [weeks]) + assert not _carry_accepts([23], [15], [weeks]) + assert _carry_accepts([23], [15], [0]) From 4ddb87805a6a7b3738ce97633610773a8539874e Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 27 Sep 2026 16:40:11 -0400 Subject: [PATCH 05/10] Re-cut the field-usage attestation for the work-experience manifest fields +11 configuration/authored fields (3 derive inputs, 3 outputs, 2 nonnegative outputs, 3 tail-contract columns); the calibration (312 -> 315) and source_stages (1,712 -> 1,720) pointer claims and the pointer inventory digest move with them. Coverage evidence regenerated by tools/spec_engine_coverage.py: 42,191/42,191 fields, 41/41 inventory checks. Part of #719. Co-Authored-By: Claude Opus 5.5 --- docs/evidence/spec-engine/us-f0-coverage.json | 50 +++++++++---------- .../build/spec_engine/field_usage.py | 12 ++--- .../tests/test_spec_engine_coverage_tool.py | 6 +-- .../tests/test_spec_engine_field_usage.py | 6 +-- tools/spec_engine_coverage.py | 2 +- 5 files changed, 38 insertions(+), 38 deletions(-) diff --git a/docs/evidence/spec-engine/us-f0-coverage.json b/docs/evidence/spec-engine/us-f0-coverage.json index 75804e57cc..9075c33606 100644 --- a/docs/evidence/spec-engine/us-f0-coverage.json +++ b/docs/evidence/spec-engine/us-f0-coverage.json @@ -4,9 +4,9 @@ "version": 1 }, "country": "us", - "documentation_sha256": "911b21d13cc214601cfb537c538fed2851e5166e8f72de5ebfce5f2918af43d3", + "documentation_sha256": "4d8abb7d30a87d75e2c0b8ddeeba5b192299e5c26446ae0f5971aae0a5146fed", "field_usage": { - "authored_normative_field_count": 32400, + "authored_normative_field_count": 32411, "claim_count": 49, "claims": [ { @@ -108,8 +108,8 @@ ], "mode": "legacy_behavior", "pointer_class": "all", - "pointer_count": 312, - "pointer_sha256": "9bad55b4945af1c4213c510942e7b9d22d104bf75b0d9804e4a183e1656c8393", + "pointer_count": 315, + "pointer_sha256": "13227bf16227b7027c6ccd1220bf2b451b6c8be2b17dbe63b0c5500dc32433a2", "rationale": null, "relative_sink_prefix": "/calibration_contract", "source_prefix": "/authored/spec~1calibration.yaml", @@ -514,8 +514,8 @@ ], "mode": "legacy_behavior", "pointer_class": "all", - "pointer_count": 1712, - "pointer_sha256": "a46d5e657d919db23ee7752d844bd8bbe59c15321dec07102837452d7994d8dd", + "pointer_count": 1720, + "pointer_sha256": "7e187b42fb027a57d156f93992d455d6e28cc387d792be94058c08e796fb56a1", "rationale": null, "relative_sink_prefix": "/source_manifest/stages", "source_prefix": "/authored/spec~1sources.yaml/stages", @@ -772,20 +772,20 @@ "verifier": "vintages" } ], - "configuration_field_count": 42180, - "consumed_field_count": 42180, + "configuration_field_count": 42191, + "consumed_field_count": 42191, "generation0_effect_counts": { - "legacy_behavior": 38494, + "legacy_behavior": 38505, "no_generation0_effect": 3686 }, "mode_counts": { "compiler_semantic": 27717, "front_end_validation": 354, "identity_only": 103, - "legacy_behavior": 14006 + "legacy_behavior": 14017 }, "multiple_primary_use_field_count": 0, - "pointer_inventory_sha256": "21b6fe2f6b96307f8ec7d20d4162ee2e0702f363bb28ebd3fa47a4cf7fc12ed4", + "pointer_inventory_sha256": "b2ac963388957209c6d644d40a86d0f60f5d8592afb7f31e983ed76bf95df98c", "resolved_binding_field_count": 9780, "unused_field_count": 0 }, @@ -824,7 +824,7 @@ "stacked_checkpoint_full_components": 13, "stacked_checkpoint_pool_code_components": 19, "stacked_checkpoint_static_components": 10, - "tail_control_fields": 934, + "tail_control_fields": 937, "take_up_pipeline_steps": 28, "take_up_programs": 17, "typed_artifacts": 84, @@ -917,13 +917,13 @@ "legacy_adapter.stacked_checkpoint_static_components" ], "expected": { - "sha256": "ac92829c88a1a4fb6460d61190918d5d99c6c377fc8dd8f62f02b332d09bf59c", - "terminal_fields": 934 + "sha256": "256947cd7c1bbd85f5d1fa3911a07a170ae87830f098110a40ed25d93f78347c", + "terminal_fields": 937 }, "failures": [], "observed": { - "sha256": "ac92829c88a1a4fb6460d61190918d5d99c6c377fc8dd8f62f02b332d09bf59c", - "terminal_fields": 934 + "sha256": "256947cd7c1bbd85f5d1fa3911a07a170ae87830f098110a40ed25d93f78347c", + "terminal_fields": 937 }, "status": "covered" }, @@ -1361,12 +1361,12 @@ ], "expected": { "producer_count": 38, - "sha256": "6abbfad73014ce3bac9fbd549f16e4a5e645a3b0fbd450b56e186876f3f0dd95" + "sha256": "397adfd4f4efb4450c17bec74155e0f56b55f2729198b0e1f792bfbeb1010cd7" }, "failures": [], "observed": { "producer_count": 38, - "sha256": "6abbfad73014ce3bac9fbd549f16e4a5e645a3b0fbd450b56e186876f3f0dd95" + "sha256": "397adfd4f4efb4450c17bec74155e0f56b55f2729198b0e1f792bfbeb1010cd7" }, "status": "covered" }, @@ -1775,12 +1775,12 @@ "legacy_adapter.imputation.late_producer_resource_semantics" ], "expected": { - "sha256": "a87278445d928ea20e15935a8cc07312d6e819e34409b04e8c814f17910d828f", + "sha256": "ee7b7541a6925351d4acbfa0064f9586dd14c095e9208c38f492a83bb8a1afa1", "stages": 37 }, "failures": [], "observed": { - "sha256": "a87278445d928ea20e15935a8cc07312d6e819e34409b04e8c814f17910d828f", + "sha256": "ee7b7541a6925351d4acbfa0064f9586dd14c095e9208c38f492a83bb8a1afa1", "stages": 38 }, "status": "covered" @@ -1887,7 +1887,7 @@ "alpha", "zeta" ], - "sha256": "b1d5c65254b51897b27f0f5e44e344781ce9e01e305aaf1bfc042cc32bc417dc" + "sha256": "3bf611d9efdba2a2edd813f0d4a39739d397c4769e9d47767996fea941f05a42" }, "failures": [], "observed": { @@ -1911,7 +1911,7 @@ "alpha", "zeta" ], - "sha256": "b1d5c65254b51897b27f0f5e44e344781ce9e01e305aaf1bfc042cc32bc417dc" + "sha256": "3bf611d9efdba2a2edd813f0d4a39739d397c4769e9d47767996fea941f05a42" }, "status": "covered" }, @@ -1975,7 +1975,7 @@ "take_up_contract", "us_qbi_reconciliation_contract" ], - "sha256": "5b65545ed6f351b30e94ab1405e13d2c7f692ae0c8d4959badcfe9bb9d52d455" + "sha256": "56d2b08be5b745c04d43610c340da08406aa17c7aa49f09dfc6207c70d9b0cf3" }, "status": "covered" }, @@ -2599,7 +2599,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "f830589331dad583398485de5724dace0714d8526fb63b37f81cd783b1cb8f30" + "spec_sha256": "bbe92884034547e0cd9eb9be502fc539c33633e7552908406d3dabcfbcb96e3a" } }, "report_schema_version": 3, @@ -2609,7 +2609,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "f830589331dad583398485de5724dace0714d8526fb63b37f81cd783b1cb8f30" + "spec_sha256": "bbe92884034547e0cd9eb9be502fc539c33633e7552908406d3dabcfbcb96e3a" }, "status": "pass" } diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py b/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py index c70675d908..8a591404fa 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py @@ -26,9 +26,9 @@ ) from .schemas import load_schema_registry -EXPECTED_AUTHORED_FIELD_COUNT = 32_400 +EXPECTED_AUTHORED_FIELD_COUNT = 32_411 EXPECTED_RESOLVED_BINDING_FIELD_COUNT = 9_780 -EXPECTED_CONFIGURATION_FIELD_COUNT = 42_180 +EXPECTED_CONFIGURATION_FIELD_COUNT = 42_191 class FieldUsageError(AssertionError): @@ -361,8 +361,8 @@ def _path_inventory(rows: Sequence[tuple[str, object]]) -> tuple[int, str]: "36564747627fb0058bb99e8e8c4397e41d8ecc472a20c49d4b6d7df1003a80ec", ), "calibration": ( - 312, - "9bad55b4945af1c4213c510942e7b9d22d104bf75b0d9804e4a183e1656c8393", + 315, + "13227bf16227b7027c6ccd1220bf2b451b6c8be2b17dbe63b0c5500dc32433a2", ), "catalog_columns": ( 1_673, @@ -485,8 +485,8 @@ def _path_inventory(rows: Sequence[tuple[str, object]]) -> tuple[int, str]: "d6782c5de5bbed1bdc6bf653c4a6d4aadcad4ccc72d35e1092e130fcb04680a3", ), "source_stages": ( - 1_712, - "a46d5e657d919db23ee7752d844bd8bbe59c15321dec07102837452d7994d8dd", + 1_720, + "7e187b42fb027a57d156f93992d455d6e28cc387d792be94058c08e796fb56a1", ), "spine_assembly_household_mass_shares": ( 2, diff --git a/packages/microcosm-build/tests/test_spec_engine_coverage_tool.py b/packages/microcosm-build/tests/test_spec_engine_coverage_tool.py index b6f65a195a..85248d3d31 100644 --- a/packages/microcosm-build/tests/test_spec_engine_coverage_tool.py +++ b/packages/microcosm-build/tests/test_spec_engine_coverage_tool.py @@ -88,10 +88,10 @@ def test_us_coverage_is_exact_complete_and_honest( assert_coverage_complete(coverage_report) assert coverage_report["status"] == "pass" fields = coverage_report["field_usage"] - assert fields["configuration_field_count"] == 42_180 - assert fields["authored_normative_field_count"] == 32_400 + assert fields["configuration_field_count"] == 42_191 + assert fields["authored_normative_field_count"] == 32_411 assert fields["resolved_binding_field_count"] == 9_780 - assert fields["consumed_field_count"] == 42_180 + assert fields["consumed_field_count"] == 42_191 assert fields["unused_field_count"] == 0 assert fields["multiple_primary_use_field_count"] == 0 assert fields["claim_count"] == 49 diff --git a/packages/microcosm-build/tests/test_spec_engine_field_usage.py b/packages/microcosm-build/tests/test_spec_engine_field_usage.py index 4d468b5135..3e5fdd1964 100644 --- a/packages/microcosm-build/tests/test_spec_engine_field_usage.py +++ b/packages/microcosm-build/tests/test_spec_engine_field_usage.py @@ -92,9 +92,9 @@ def _mutated_bundle( def test_exact_complete_ledger_has_one_primary_mode_per_pointer(field_ledger) -> None: - assert len(field_ledger.fields) == EXPECTED_CONFIGURATION_FIELD_COUNT == 42_180 + assert len(field_ledger.fields) == EXPECTED_CONFIGURATION_FIELD_COUNT == 42_191 assert field_ledger.source_counts == { - "authored": 32_400, + "authored": 32_411, "resolved_bindings": 9_780, } assert field_ledger.mode_counts == { @@ -107,7 +107,7 @@ def test_exact_complete_ledger_has_one_primary_mode_per_pointer(field_ledger) -> "legacy_behavior": 38_494, "no_generation0_effect": 3_686, } - assert len({field.pointer for field in field_ledger.fields}) == 42_180 + assert len({field.pointer for field in field_ledger.fields}) == 42_191 def test_eligibility_concepts_are_validation_not_generation0_behavior( diff --git a/tools/spec_engine_coverage.py b/tools/spec_engine_coverage.py index 0ea30f3c74..136ba95ac4 100644 --- a/tools/spec_engine_coverage.py +++ b/tools/spec_engine_coverage.py @@ -41,7 +41,7 @@ REPORT_SCHEMA_VERSION = 3 EXPECTED_POINTER_INVENTORY_SHA256 = ( - "21b6fe2f6b96307f8ec7d20d4162ee2e0702f363bb28ebd3fa47a4cf7fc12ed4" + "b2ac963388957209c6d644d40a86d0f60f5d8592afb7f31e983ed76bf95df98c" ) DEFAULT_REPORT_PATH = ( Path(__file__).resolve().parents[1] From c78a2b731f458e5ba894bb3d825fed507aead5ce Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 24 Oct 2026 00:32:01 -0400 Subject: [PATCH 06/10] Restore the work-experience stage manifest and re-pin after the main merge The merge of main took main's sources.yaml, dropping this branch's work-experience entries (WEIND, WEMIND, WKSWORK inputs; industry and worked_last_year outputs; the stage note). Regenerating the bundle from the constants restores them exactly, and the documented re-pin sequence (tools/repin_us_spec_engine_contracts.py --legacy-only, the bundle generator, the re-pin, tools/spec_engine_coverage.py) moves the field counts to 42_228 configuration / 32_448 authored and re-pins the stage, calibration, tail and checkpoint digests. Both --check modes pass and the coverage tool reports 41/41 inventory checks. Co-Authored-By: Claude Opus 5.5 --- docs/evidence/spec-engine/us-f0-coverage.json | 50 +++++++++---------- .../build/spec_engine/field_usage.py | 12 ++--- .../build/spec_engine/inventory_coverage.py | 14 +++--- .../src/microcosm/build/us/spec/sources.yaml | 17 ++++++- .../us/test_spec_engine_coverage_tool.py | 12 ++--- .../test_spec_engine_calibration_semantics.py | 2 +- .../shared/test_spec_engine_field_usage.py | 10 ++-- .../us/test_us_bundle_core_contracts.py | 2 +- .../us/test_us_multispine_pool_tool.py | 2 +- .../spec_engine_inventory_coverage.py | 2 +- .../microcosm_build/us_spec_bundle.py | 2 +- tools/generate_us_bundle_from_constants.py | 2 +- tools/spec_engine_coverage.py | 2 +- 13 files changed, 71 insertions(+), 58 deletions(-) diff --git a/docs/evidence/spec-engine/us-f0-coverage.json b/docs/evidence/spec-engine/us-f0-coverage.json index 01318a3f01..5c84d518bd 100644 --- a/docs/evidence/spec-engine/us-f0-coverage.json +++ b/docs/evidence/spec-engine/us-f0-coverage.json @@ -4,9 +4,9 @@ "version": 1 }, "country": "us", - "documentation_sha256": "44f40b4e8aaa2e6be67a7bc9680cc29f88e185c10573ffb9c3f9dfbd3ad1081f", + "documentation_sha256": "801a99da54b8a4834a8c5e6c4b013fdae1e7576e0132de7e87cb1ef098f1ad8c", "field_usage": { - "authored_normative_field_count": 32437, + "authored_normative_field_count": 32448, "claim_count": 49, "claims": [ { @@ -108,8 +108,8 @@ ], "mode": "legacy_behavior", "pointer_class": "all", - "pointer_count": 312, - "pointer_sha256": "9bad55b4945af1c4213c510942e7b9d22d104bf75b0d9804e4a183e1656c8393", + "pointer_count": 315, + "pointer_sha256": "13227bf16227b7027c6ccd1220bf2b451b6c8be2b17dbe63b0c5500dc32433a2", "rationale": null, "relative_sink_prefix": "/calibration_contract", "source_prefix": "/authored/spec~1calibration.yaml", @@ -514,8 +514,8 @@ ], "mode": "legacy_behavior", "pointer_class": "all", - "pointer_count": 1749, - "pointer_sha256": "d0979722ea5153bd152e5e1c1201f5d530d11b21aecad18aca38473962caa062", + "pointer_count": 1757, + "pointer_sha256": "29c137db9d9e522c4280fbad42a3d3528411453853d2c7c5d1d347662e7b2076", "rationale": null, "relative_sink_prefix": "/source_manifest/stages", "source_prefix": "/authored/spec~1sources.yaml/stages", @@ -772,20 +772,20 @@ "verifier": "vintages" } ], - "configuration_field_count": 42217, - "consumed_field_count": 42217, + "configuration_field_count": 42228, + "consumed_field_count": 42228, "generation0_effect_counts": { - "legacy_behavior": 38531, + "legacy_behavior": 38542, "no_generation0_effect": 3686 }, "mode_counts": { "compiler_semantic": 27717, "front_end_validation": 354, "identity_only": 103, - "legacy_behavior": 14043 + "legacy_behavior": 14054 }, "multiple_primary_use_field_count": 0, - "pointer_inventory_sha256": "c1e2a118a0d53db0bb628b14357a2c86db2956332f5b359c9092764f9c451a95", + "pointer_inventory_sha256": "747ea61983bae6bc8a3977ed288176b9557d5cf58d623163d29bb8a05e6d913f", "resolved_binding_field_count": 9780, "unused_field_count": 0 }, @@ -824,7 +824,7 @@ "stacked_checkpoint_full_components": 13, "stacked_checkpoint_pool_code_components": 19, "stacked_checkpoint_static_components": 10, - "tail_control_fields": 934, + "tail_control_fields": 937, "take_up_pipeline_steps": 28, "take_up_programs": 17, "typed_artifacts": 84, @@ -917,13 +917,13 @@ "legacy_adapter.stacked_checkpoint_static_components" ], "expected": { - "sha256": "ac92829c88a1a4fb6460d61190918d5d99c6c377fc8dd8f62f02b332d09bf59c", - "terminal_fields": 934 + "sha256": "256947cd7c1bbd85f5d1fa3911a07a170ae87830f098110a40ed25d93f78347c", + "terminal_fields": 937 }, "failures": [], "observed": { - "sha256": "ac92829c88a1a4fb6460d61190918d5d99c6c377fc8dd8f62f02b332d09bf59c", - "terminal_fields": 934 + "sha256": "256947cd7c1bbd85f5d1fa3911a07a170ae87830f098110a40ed25d93f78347c", + "terminal_fields": 937 }, "status": "covered" }, @@ -1361,12 +1361,12 @@ ], "expected": { "producer_count": 38, - "sha256": "cbac7c8fbdbb82f0f06ab356d0c9f0d3645ca930550b44af2272d93315f2e934" + "sha256": "b555f811aaf1b65f588a3aa0a6cee2d311b1bc40b1735749c527749b5b3d0da1" }, "failures": [], "observed": { "producer_count": 38, - "sha256": "cbac7c8fbdbb82f0f06ab356d0c9f0d3645ca930550b44af2272d93315f2e934" + "sha256": "b555f811aaf1b65f588a3aa0a6cee2d311b1bc40b1735749c527749b5b3d0da1" }, "status": "covered" }, @@ -1775,12 +1775,12 @@ "legacy_adapter.imputation.late_producer_resource_semantics" ], "expected": { - "sha256": "629a0712659a26b2d6aa6c0e5672421f415d625e0599d6baa0a834d118953c6a", + "sha256": "87b1e124969ec11b836740c027d28dc2a9d319d279ed55be1d9fc1523390f634", "stages": 38 }, "failures": [], "observed": { - "sha256": "629a0712659a26b2d6aa6c0e5672421f415d625e0599d6baa0a834d118953c6a", + "sha256": "87b1e124969ec11b836740c027d28dc2a9d319d279ed55be1d9fc1523390f634", "stages": 38 }, "status": "covered" @@ -1887,7 +1887,7 @@ "alpha", "zeta" ], - "sha256": "e5da85693fab1f1cba9c37beba5cdf91477962d466bc21e9f29125f2f90b7e79" + "sha256": "9fb7350a66a0ec9f7a4dbc879ccafaf1e6cb51486722938b3a8e6f5c57ae9e06" }, "failures": [], "observed": { @@ -1911,7 +1911,7 @@ "alpha", "zeta" ], - "sha256": "e5da85693fab1f1cba9c37beba5cdf91477962d466bc21e9f29125f2f90b7e79" + "sha256": "9fb7350a66a0ec9f7a4dbc879ccafaf1e6cb51486722938b3a8e6f5c57ae9e06" }, "status": "covered" }, @@ -1975,7 +1975,7 @@ "take_up_contract", "us_qbi_reconciliation_contract" ], - "sha256": "fb863f18a79610dc1005089a32e1535c502c5b9f7e5ba589db0d6025b01e3c82" + "sha256": "b7669fd0f941d7ca68fbdf5966d5ae8751fda598dc64f6d0d7420c0c33b8e9e2" }, "status": "covered" }, @@ -2599,7 +2599,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "97f41a9fedba0ed1f26dd5f457997c10ffefe852c75f8fc18c793acdf21ebfd3" + "spec_sha256": "7f5a3a16722491c4dc4cb6defdf96b6a5409fc78e98eea57e4db3ab54a7d62c2" } }, "report_schema_version": 3, @@ -2609,7 +2609,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "97f41a9fedba0ed1f26dd5f457997c10ffefe852c75f8fc18c793acdf21ebfd3" + "spec_sha256": "7f5a3a16722491c4dc4cb6defdf96b6a5409fc78e98eea57e4db3ab54a7d62c2" }, "status": "pass" } diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py b/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py index f86db2dfb8..7cda1945c5 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py @@ -26,9 +26,9 @@ ) from .schemas import load_schema_registry -EXPECTED_AUTHORED_FIELD_COUNT = 32_437 +EXPECTED_AUTHORED_FIELD_COUNT = 32_448 EXPECTED_RESOLVED_BINDING_FIELD_COUNT = 9_780 -EXPECTED_CONFIGURATION_FIELD_COUNT = 42_217 +EXPECTED_CONFIGURATION_FIELD_COUNT = 42_228 class FieldUsageError(AssertionError): @@ -361,8 +361,8 @@ def _path_inventory(rows: Sequence[tuple[str, object]]) -> tuple[int, str]: "36564747627fb0058bb99e8e8c4397e41d8ecc472a20c49d4b6d7df1003a80ec", ), "calibration": ( - 312, - "9bad55b4945af1c4213c510942e7b9d22d104bf75b0d9804e4a183e1656c8393", + 315, + "13227bf16227b7027c6ccd1220bf2b451b6c8be2b17dbe63b0c5500dc32433a2", ), "catalog_columns": ( 1_673, @@ -485,8 +485,8 @@ def _path_inventory(rows: Sequence[tuple[str, object]]) -> tuple[int, str]: "d6782c5de5bbed1bdc6bf653c4a6d4aadcad4ccc72d35e1092e130fcb04680a3", ), "source_stages": ( - 1_749, - "d0979722ea5153bd152e5e1c1201f5d530d11b21aecad18aca38473962caa062", + 1_757, + "29c137db9d9e522c4280fbad42a3d3528411453853d2c7c5d1d347662e7b2076", ), "spine_assembly_household_mass_shares": ( 2, diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py index a463092b96..c6c5f7aa1a 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py @@ -350,12 +350,12 @@ "acs_person_predictors": "878c788a6f037d7aca12b3586ea034eff04f3034ffa11935a736493042551f25", "authority": "9d4a9672a0f03039b1fe874b9fe21ed575be0d29f14afc396d03cdf5c809bdd2", "early_families": "4aa9f736fd76e83955477ad1667e58f48f264783f05bdc7f0102cd32d61323bd", - "full_checkpoint": "e5da85693fab1f1cba9c37beba5cdf91477962d466bc21e9f29125f2f90b7e79", + "full_checkpoint": "9fb7350a66a0ec9f7a4dbc879ccafaf1e6cb51486722938b3a8e6f5c57ae9e06", "gap_fill_schedule": "1c31f9868f7884347cc19cf1ff65da43f950b9114941a715bab168246db414a7", "graph_nodes": "40cd51ffdfe2e9d9d08d48c08e8ded9de1e4b134783bab05c4abc6ad5c72ca1e", "geography_assignment": "44e7f5f6cd7ceeef85d532c3d46306d03de4f4ee811376e6d29920742746b65c", "late_families": "d91f9ff0eb52f43e7b6eed3d5c58c37abe1620c3a11021da15dae9c10e16d382", - "late_resource_semantics": "cbac7c8fbdbb82f0f06ab356d0c9f0d3645ca930550b44af2272d93315f2e934", + "late_resource_semantics": "b555f811aaf1b65f588a3aa0a6cee2d311b1bc40b1735749c527749b5b3d0da1", "late_schedule": "e59c019d3d454eac99ac0ac209b6c5b6faaf9bdfcaeee18c36a25be19bf7da2f", "ownership": "5f64f0aac49e2313177564f71876bffc8c81b3ded4df701e70930e60e9c98356", "primary_tuples": "fdf23da429f8c501e198a2f5b719763d523a41565c2153fdf17b0344f21bcd7c", @@ -371,9 +371,9 @@ # manifest. "seed_map": "3cf71bdd3e6df567a4f9eb8d182bea160108cc9941e6ff08b8a0eee4e9fc8ba2", "seed_protocol": "15e9b110999c37a0a78ecd001d6b79bbd860b408cf08a1e0ea435690f0a5f041", - "source_manifest": "629a0712659a26b2d6aa6c0e5672421f415d625e0599d6baa0a834d118953c6a", + "source_manifest": "87b1e124969ec11b836740c027d28dc2a9d319d279ed55be1d9fc1523390f634", "take_up": "9522ce40f7dea569312dd7a7beb474e5a5afadd2f1e10cb5876534c3ef623d35", - "tail": "ac92829c88a1a4fb6460d61190918d5d99c6c377fc8dd8f62f02b332d09bf59c", + "tail": "256947cd7c1bbd85f5d1fa3911a07a170ae87830f098110a40ed25d93f78347c", } INVENTORY_REPORT_SCHEMA_VERSION = 1 @@ -453,7 +453,7 @@ "stacked_checkpoint_full_components": 13, "stacked_checkpoint_pool_code_components": 19, "stacked_checkpoint_static_components": 10, - "tail_control_fields": 934, + "tail_control_fields": 937, "take_up_pipeline_steps": 28, "take_up_programs": 17, "typed_artifacts": 84, @@ -1421,7 +1421,7 @@ def add( add( "capital_gains_tail_contract_exact", clauses={ - "tail terminal field count differs": tail_count == 934, + "tail terminal field count differs": tail_count == 937, "resolved calibration tails differ": _json_equal( legacy.get("calibration_tail_contracts"), expected_legacy["calibration_tail_contracts"], @@ -1449,7 +1449,7 @@ def add( "legacy_adapter.stacked_checkpoint_static_components", ), observed={"terminal_fields": tail_count, "sha256": sha256_json(tail_bundle)}, - expected={"terminal_fields": 934, "sha256": EXPECTED_HASHES["tail"]}, + expected={"terminal_fields": 937, "sha256": EXPECTED_HASHES["tail"]}, ) add( diff --git a/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml b/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml index 3a0bc01f62..6198319651 100644 --- a/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml +++ b/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml @@ -99,7 +99,7 @@ sources: stage_asset: id: source_stages path: microcosm.build.us/source_stages.json - sha256: 559062a1bd10efdd89b8c83da9294fea43062b8b6760524f4350dd84d63ef383 + sha256: dc3b72982675b95d30cfabdeb5f7f4c492e6cd3788009ef3094ab4d48bfd9093 stage_manifest: version: 1 country: us @@ -1720,6 +1720,9 @@ stages: - PRDTRACE - PRDTHSP - POCCU2 + - WEIND + - WEMIND + - WKSWORK - kind: derive method: derive_flsa_overtime_premium inputs: @@ -1737,6 +1740,9 @@ stages: - cps_race - is_hispanic - detailed_occupation_recode + - detailed_industry_recode + - major_industry_recode + - worked_last_year - has_never_worked - is_military - is_computer_scientist @@ -1749,6 +1755,8 @@ stages: nonnegative_outputs: - cps_race - detailed_occupation_recode + - detailed_industry_recode + - major_industry_recode - hourly_wage - fsla_overtime_premium notes: 'Port of the final retired ORG/FLSA method at 9a823603e6b5fb916d65ec45d74c9c7eb0043db1: donor @@ -1781,7 +1789,12 @@ stages: reshuffles within unchanged state-level quotas — the assignment method itself remains the exact archived port. Exemption screens unchanged; both hours measures count all jobs while FLSA section 7 applies per employer (inherited upward concept edge). Adjudication and measurements: microcosm#451 item 4 - overtime-incidence lane.' + overtime-incidence lane. Work-experience carries (microcosm#719): WEIND and WEMIND (industry of the + longest job last year, restored before pooling from each vintage''s pinned Census person member by + the asec_census_person_columns restoration) become detailed_industry_recode and major_industry_recode, + and worked_last_year is WKSWORK > 0; ACS-spine people get all three from native INDP/WKWN through + the release predictor join. Checked on every spine: WEMIND is the major group of WEIND and positive + weeks carry a worker industry code (1-22); the converse holds on ASEC rows and is checked before pooling.' - stage: meps_esi_premiums survey: MEPS-IC + CPS ASEC current employment-based coverage source: https://meps.ahrq.gov/mepsweb/survey_comp/Insurance.jsp diff --git a/packages/microcosm-build/tests/engine_contract/us/test_spec_engine_coverage_tool.py b/packages/microcosm-build/tests/engine_contract/us/test_spec_engine_coverage_tool.py index 3a0dd0e3d9..79bc5db821 100644 --- a/packages/microcosm-build/tests/engine_contract/us/test_spec_engine_coverage_tool.py +++ b/packages/microcosm-build/tests/engine_contract/us/test_spec_engine_coverage_tool.py @@ -82,21 +82,21 @@ def test_us_coverage_is_exact_complete_and_honest( assert_coverage_complete(coverage_report) assert coverage_report["status"] == "pass" fields = coverage_report["field_usage"] - assert fields["configuration_field_count"] == 42_217 - assert fields["authored_normative_field_count"] == 32_437 + assert fields["configuration_field_count"] == 42_228 + assert fields["authored_normative_field_count"] == 32_448 assert fields["resolved_binding_field_count"] == 9_780 - assert fields["consumed_field_count"] == 42_217 + assert fields["consumed_field_count"] == 42_228 assert fields["unused_field_count"] == 0 assert fields["multiple_primary_use_field_count"] == 0 assert fields["claim_count"] == 49 assert fields["mode_counts"] == { - "legacy_behavior": 14_043, + "legacy_behavior": 14054, "compiler_semantic": 27_717, "front_end_validation": 354, "identity_only": 103, } assert fields["generation0_effect_counts"] == { - "legacy_behavior": 38_531, + "legacy_behavior": 38542, "no_generation0_effect": 3_686, } @@ -107,7 +107,7 @@ def test_us_coverage_is_exact_complete_and_honest( assert inventory["missing_items"] == [] assert inventory["counts"]["producer_inputs"] == 2_744 assert inventory["counts"]["ownership_rows"] == 18 - assert inventory["counts"]["tail_control_fields"] == 934 + assert inventory["counts"]["tail_control_fields"] == 937 assert inventory["counts"]["seed_owner_bindings"] == 112 diff --git a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_calibration_semantics.py b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_calibration_semantics.py index ccec9ee148..eb01f7213b 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_calibration_semantics.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_calibration_semantics.py @@ -31,7 +31,7 @@ ROOT = _TEST_PATHS.repository US_ROOT = ROOT / "packages/microcosm-build/src/microcosm/build/us" CANONICAL_LEGACY_PROJECTION_SHA256 = ( - "05cb099ee8bd0234ae94586b76dc20ab8a5bd0735c00d4e82e67f912e0b33823" + "080ec47985e271d799b174774977ef5a58df311fc9808992dc531cbaf693c079" ) diff --git a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_field_usage.py b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_field_usage.py index 79d903c887..7f6c7b8c39 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_field_usage.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_field_usage.py @@ -95,22 +95,22 @@ def _mutated_bundle( def test_exact_complete_ledger_has_one_primary_mode_per_pointer(field_ledger) -> None: - assert len(field_ledger.fields) == EXPECTED_CONFIGURATION_FIELD_COUNT == 42_217 + assert len(field_ledger.fields) == EXPECTED_CONFIGURATION_FIELD_COUNT == 42_228 assert field_ledger.source_counts == { - "authored": 32_437, + "authored": 32_448, "resolved_bindings": 9_780, } assert field_ledger.mode_counts == { - "legacy_behavior": 14_043, + "legacy_behavior": 14054, "compiler_semantic": 27_717, "front_end_validation": 354, "identity_only": 103, } assert field_ledger.generation0_effect_counts == { - "legacy_behavior": 38_531, + "legacy_behavior": 38542, "no_generation0_effect": 3_686, } - assert len({field.pointer for field in field_ledger.fields}) == 42_217 + assert len({field.pointer for field in field_ledger.fields}) == 42_228 def test_eligibility_concepts_are_validation_not_generation0_behavior( diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_bundle_core_contracts.py b/packages/microcosm-build/tests/engine_free/us/test_us_bundle_core_contracts.py index 89d96e39ef..ee8e0a62d4 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_bundle_core_contracts.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_bundle_core_contracts.py @@ -25,7 +25,7 @@ def test_source_surface_classification_is_complete() -> None: } assert normative["stage_asset"] == { "id": "source_stages", - "sha256": "559062a1bd10efdd89b8c83da9294fea43062b8b6760524f4350dd84d63ef383", + "sha256": "dc3b72982675b95d30cfabdeb5f7f4c492e6cd3788009ef3094ab4d48bfd9093", } assert operational["stage_asset"] == { "path": "microcosm.build.us/source_stages.json" diff --git a/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py b/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py index d74db8e007..c70ff7749f 100644 --- a/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py +++ b/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py @@ -344,7 +344,7 @@ def capture_equality(expected: object, actual: object) -> None: # move it. It covers microcosm.fit.qrf's attested bytes too, so # fit_draw_next's caller-owned uniforms (microcosm#1332) move it, # and so does microcosm#454's meps_esi_premiums stage. - "spec_sha256": "97f41a9fedba0ed1f26dd5f457997c10ffefe852c75f8fc18c793acdf21ebfd3", + "spec_sha256": "7f5a3a16722491c4dc4cb6defdf96b6a5409fc78e98eea57e4db3ab54a7d62c2", }, } diff --git a/test_support/microcosm_build/spec_engine_inventory_coverage.py b/test_support/microcosm_build/spec_engine_inventory_coverage.py index 1d051f5c4d..1764417b85 100644 --- a/test_support/microcosm_build/spec_engine_inventory_coverage.py +++ b/test_support/microcosm_build/spec_engine_inventory_coverage.py @@ -133,7 +133,7 @@ def binding() -> dict[str, object]: "stacked_checkpoint_full_components": 13, "stacked_checkpoint_pool_code_components": 19, "stacked_checkpoint_static_components": 10, - "tail_control_fields": 934, + "tail_control_fields": 937, "take_up_pipeline_steps": 28, "take_up_programs": 17, "typed_artifacts": 84, diff --git a/test_support/microcosm_build/us_spec_bundle.py b/test_support/microcosm_build/us_spec_bundle.py index 0f3cd267be..906c3a7440 100644 --- a/test_support/microcosm_build/us_spec_bundle.py +++ b/test_support/microcosm_build/us_spec_bundle.py @@ -195,7 +195,7 @@ def _load_generator_module(): LEGACY_COMPATIBILITY_SHA256 = { "source_stages.json": ( - "559062a1bd10efdd89b8c83da9294fea43062b8b6760524f4350dd84d63ef383" + "dc3b72982675b95d30cfabdeb5f7f4c492e6cd3788009ef3094ab4d48bfd9093" ), "support_spine.json": ( "68f37dc6ae6e0cde7ebccb53f88dd4a800e63456f838fa214ff98d1db8d815be" diff --git a/tools/generate_us_bundle_from_constants.py b/tools/generate_us_bundle_from_constants.py index 1eda413959..2454909041 100644 --- a/tools/generate_us_bundle_from_constants.py +++ b/tools/generate_us_bundle_from_constants.py @@ -122,7 +122,7 @@ def _domain_builders() -> dict[str, Callable[[], object]]: # frozen files, is the forward YAML -> legacy-payload path. FROZEN_LEGACY_RESOURCE_SHA256 = { "source_stages.json": ( - "559062a1bd10efdd89b8c83da9294fea43062b8b6760524f4350dd84d63ef383" + "dc3b72982675b95d30cfabdeb5f7f4c492e6cd3788009ef3094ab4d48bfd9093" ), "support_spine.json": ( "68f37dc6ae6e0cde7ebccb53f88dd4a800e63456f838fa214ff98d1db8d815be" diff --git a/tools/spec_engine_coverage.py b/tools/spec_engine_coverage.py index 16a8e339db..2c5cbd018b 100644 --- a/tools/spec_engine_coverage.py +++ b/tools/spec_engine_coverage.py @@ -41,7 +41,7 @@ REPORT_SCHEMA_VERSION = 3 EXPECTED_POINTER_INVENTORY_SHA256 = ( - "c1e2a118a0d53db0bb628b14357a2c86db2956332f5b359c9092764f9c451a95" + "747ea61983bae6bc8a3977ed288176b9557d5cf58d623163d29bb8a05e6d913f" ) DEFAULT_REPORT_PATH = ( Path(__file__).resolve().parents[1] From 558d5b693aee446488dd043889245ca298570786 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 24 Oct 2026 01:58:07 -0400 Subject: [PATCH 07/10] Pin the calibration projection from the fixture's raw contract The repinner hashed the compiled bundle's calibration, but the loader writes integral floats as integers (5.0 becomes 5), so its digest (080ec479...) never matched the semantics test, which hashes the constants builder's raw contract (aa65317d..., failing engine-free CI). The repinner now builds the contract exactly as that fixture does, and a new test holds its derivation to the committed pin. Stripping this branch's three new tail names reproduces main's pin (05cb099e...) from the raw contract, not from the bundle. The work-experience differential test now also draws nested industry pairs with any weeks, so the forward and converse worker checks decide the outcome; removing either check in either site now fails it. The docs and crosswalk comment say nonzero INDUSTRY determines WEIND: in the pinned 2025 member INDUSTRY = 0 maps to both WEIND 0 and 23. Co-Authored-By: Claude Opus 5.5 --- docs/us-work-experience-columns.md | 9 ++++--- .../us_runtime/acs_release_predictors.py | 2 +- .../test_spec_engine_calibration_semantics.py | 2 +- .../us/test_us_spec_engine_repin.py | 23 +++++++++++++++++ .../us/test_us_work_experience_properties.py | 17 ++++++++++++- tools/repin_us_spec_engine_contracts.py | 25 +++++++++++++++++-- 6 files changed, 69 insertions(+), 9 deletions(-) diff --git a/docs/us-work-experience-columns.md b/docs/us-work-experience-columns.md index 725c03efa8..9301f12391 100644 --- a/docs/us-work-experience-columns.md +++ b/docs/us-work-experience-columns.md @@ -49,8 +49,8 @@ year that have weeks worked and an industry. ## The ACS crosswalk ACS 2024 `INDP` uses the 2022 Census industry code list, as does the 2025 ASEC -member (`pppub25.csv`), where `INDUSTRY` determines `WEIND` exactly (259 -codes). The 2023/2024 members use the 2017 list; the 225 codes the lists share +member (`pppub25.csv`), where nonzero `INDUSTRY` determines `WEIND` exactly +(259 codes). The 2023/2024 members use the 2017 list; the 225 codes the lists share map to the same group in both. `ACS_INDP_TO_WEIND` takes every civilian code's group from `pppub25.csv` and reviews the eight ACS-only codes: @@ -83,8 +83,9 @@ year. So on ACS rows an industry can sit on a person with ASEC members `pppub23/24/25.csv` (pins in `spm_role_source.ASEC_SPM_ROLE_SOURCES`): -- `INDUSTRY` → `WEIND` and → `WEMIND` are functions in every vintage (263 / - 263 / 259 codes); `WEIND` → `WEMIND` is the same function in all three. +- Nonzero `INDUSTRY` → `WEIND` and → `WEMIND` are functions in every vintage + (263 / 263 / 259 codes); `INDUSTRY = 0` covers both `WEIND = 0` and the + did-not-work group 23; `WEIND` → `WEMIND` is the same function in all three. - `WEIND` in 1–22 iff `INDUSTRY > 0` iff `WKSWORK > 0`: zero violations in each vintage. `WEIND = 0` exactly for ages 0–14; `WEIND = 23` only from 15. - Weighted (`A_FNLWGT`) shares: nonzero `WEIND` 0.8217 / 0.8220 / 0.8244; diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/acs_release_predictors.py b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_release_predictors.py index b70ae52660..e26f6d83a3 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/acs_release_predictors.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_release_predictors.py @@ -700,7 +700,7 @@ # 2022 Census industry codes (ACS 2024 INDP; CPS ASEC from survey year 2025) # mapped to the 23 CPS longest-job detailed industry groups (WEIND). Every # civilian code's group is read from the pinned 2025 ASEC person member -# (pppub25.csv), where INDUSTRY determines WEIND exactly; codes shared with +# (pppub25.csv), where nonzero INDUSTRY determines WEIND exactly; codes shared with # the 2017-list 2023/2024 members keep the same group there. The ACS-only # codes are reviewed: 9670-9870 are the Armed Forces branches (Army, Air # Force, Navy, Marines, Coast Guard, branch not specified, Reserves or National diff --git a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_calibration_semantics.py b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_calibration_semantics.py index eb01f7213b..a18d6c535e 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_calibration_semantics.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_calibration_semantics.py @@ -31,7 +31,7 @@ ROOT = _TEST_PATHS.repository US_ROOT = ROOT / "packages/microcosm-build/src/microcosm/build/us" CANONICAL_LEGACY_PROJECTION_SHA256 = ( - "080ec47985e271d799b174774977ef5a58df311fc9808992dc531cbaf693c079" + "aa65317d84f808e03d133f9f8156cbdfca18578a8e5169d200a70990bb372b19" ) diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_spec_engine_repin.py b/packages/microcosm-build/tests/engine_free/us/test_us_spec_engine_repin.py index ca7f4a6f11..10a8784933 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_spec_engine_repin.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_spec_engine_repin.py @@ -1,5 +1,7 @@ """The F0 pin writer preserves surrounding source and refuses unknown pins.""" +import ast + import pytest from tools.repin_us_spec_engine_contracts import ( @@ -114,3 +116,24 @@ def test_nested_receipt_pin_refuses_duplicate_assertions_or_dictionary_keys() -> ("sha256",), "new", ) + + +def test_calibration_pin_hashes_the_fixture_contract_not_the_loaded_bundle() -> None: + # The loader writes integral floats as integers (5.0 -> 5), so a digest of + # the compiled bundle's calibration never matches the golden test's pin. + from microcosm.build.spec_engine import project_legacy_calibration_contract + from microcosm.build.spec_engine.canonical import sha256_json + from tools.repin_us_spec_engine_contracts import ( + PACKAGE, + _assignment, + fixture_calibration_contract, + ) + + path = ( + PACKAGE / "tests/engine_free/shared/test_spec_engine_calibration_semantics.py" + ) + pinned = ast.literal_eval( + _assignment(path.read_text(), "CANONICAL_LEGACY_PROJECTION_SHA256") + ) + projected = project_legacy_calibration_contract(fixture_calibration_contract()) + assert sha256_json(projected) == pinned diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_work_experience_properties.py b/packages/microcosm-build/tests/engine_free/us/test_us_work_experience_properties.py index 327cb2fc48..1cd453413e 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_work_experience_properties.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_work_experience_properties.py @@ -76,6 +76,21 @@ def _arbitrary_rows(draw) -> tuple[list[int], list[int], list[int]]: ) +@st.composite +def _nested_rows(draw) -> tuple[list[int], list[int], list[int]]: + """Nested industry pairs with any weeks, so the worker identities decide. + + Independent codes almost always fail the nesting first, which would leave + the forward and converse checks unexercised. + """ + + rows = draw(st.integers(1, 40)) + weind = draw(st.lists(st.integers(0, 23), min_size=rows, max_size=rows)) + weeks = st.one_of(st.just(0), st.integers(1, 52)) + wkswork = draw(st.lists(weeks, min_size=rows, max_size=rows)) + return weind, [WEIND_TO_WEMIND[code] for code in weind], wkswork + + def _nested(weind, wemind) -> bool: return all(WEIND_TO_WEMIND[d] == m for d, m in zip(weind, wemind, strict=True)) @@ -119,7 +134,7 @@ def test_release_carry_round_trips_coherent_rows(rows) -> None: @_SETTINGS -@given(_arbitrary_rows()) +@given(st.one_of(_arbitrary_rows(), _nested_rows())) def test_restore_and_carry_accept_exactly_their_stated_identities(rows) -> None: """Differential: each site accepts iff its stated identities hold.""" diff --git a/tools/repin_us_spec_engine_contracts.py b/tools/repin_us_spec_engine_contracts.py index 0831b4f6d5..4b5d094f0a 100644 --- a/tools/repin_us_spec_engine_contracts.py +++ b/tools/repin_us_spec_engine_contracts.py @@ -217,12 +217,33 @@ def _replace_asserted_dictionary_value( return source if old == value else _replace_node(source, node, value) +def fixture_calibration_contract() -> dict: + """Build the calibration contract exactly as its semantics test fixture does. + + The golden projection hashes the constants builder's raw contract. The + compiled bundle's ``to_wire()`` has been through the loader, which writes + integral floats as integers (``5.0`` becomes ``5``), so its canonical bytes + and digest differ even where every value agrees. + """ + + from microcosm.build.spec_engine import scoped_take_up_manifest_program_bindings + + frozen = json.loads((BUILD / "us/take_up_contract.json").read_text()) + bindings = tuple( + (row["variable"], row["entity"], row["populace_treatment"]) + for row in frozen["programs"] + ) + with scoped_take_up_manifest_program_bindings(bindings): + from tools.us_bundle_generation.contracts import build_calibration_contract + + return build_calibration_contract() + + def _semantic_test_pins(spec, observed_hashes: dict, rendered: dict) -> None: """Derive the existing test vectors through their production projections.""" from microcosm.build.spec_engine import ( KernelRegistry, - ResourceKind, project_legacy_calibration_contract, ) from microcosm.build.spec_engine.canonical import sha256_json @@ -258,7 +279,7 @@ def _semantic_test_pins(spec, observed_hashes: dict, rendered: dict) -> None: path.read_text(), {"first.spec_sha256": minimal.spec_sha256} ) - calibration = spec.domain(ResourceKind.CALIBRATION).to_wire() + calibration = fixture_calibration_contract() path = ( PACKAGE / "tests/engine_free/shared/test_spec_engine_calibration_semantics.py" ) From 42802622ed4728fa04a55be71b090bee53f0b3db Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 24 Oct 2026 02:36:05 -0400 Subject: [PATCH 08/10] Rewrite asserted count breakdowns by key in the repinner The repinner moved the field-usage tests' mode and generation-0 counts by value transition, taking each old value from the committed coverage JSON. A merge can break that: in a trial merge of #1376 into this branch, git auto-merged #1376's compiler_semantic 27718 into the JSON while the conflicted test kept 27_717. The tool then saw no transition, left the stale literal, and --check passed while the test failed. The five asserted count dictionaries (source, mode and generation-0 counts in the engine-free field-usage test; mode and generation-0 counts in the coverage-tool test) are now rewritten by key from the ledger. The keys must match the observed ledger exactly. Totals still move by value transition from the module's own pins. On this branch --check is unchanged. On the trial merge the tool now flags and fixes the stale count, and both tests pass (21 passed). Co-Authored-By: Claude Opus 5.5 --- .../us/test_us_spec_engine_repin.py | 39 ++++++++++ tools/repin_us_spec_engine_contracts.py | 78 ++++++++++++++++--- 2 files changed, 106 insertions(+), 11 deletions(-) diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_spec_engine_repin.py b/packages/microcosm-build/tests/engine_free/us/test_us_spec_engine_repin.py index 10a8784933..fb5a260282 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_spec_engine_repin.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_spec_engine_repin.py @@ -6,6 +6,7 @@ from tools.repin_us_spec_engine_contracts import ( _add_count_replacement, + _replace_asserted_counts, _replace_asserted_dictionary_value, _replace_assignment, _replace_dictionary, @@ -118,6 +119,44 @@ def test_nested_receipt_pin_refuses_duplicate_assertions_or_dictionary_keys() -> ) +def test_count_breakdowns_are_rewritten_by_key_not_by_old_value() -> None: + # A merge can move the evidence JSON to the new count while a conflicted + # test keeps the old one; a value transition taken from the JSON would then + # see no change. Keying by name refreshes the stale literal regardless. + source = ( + "assert ledger.mode_counts == {\n" + ' "legacy_behavior": 14_054,\n' + ' "compiler_semantic": 27_717,\n' + "}\n" + "# 27_717 is historical\n" + ) + observed = {"legacy_behavior": 14_054, "compiler_semantic": 27_718} + assert _replace_asserted_counts(source, "ledger.mode_counts", observed) == ( + source.replace('"compiler_semantic": 27_717', '"compiler_semantic": 27718') + ) + assert _replace_asserted_counts( + source.replace("27_717,", "27_718,"), "ledger.mode_counts", observed + ) == source.replace("27_717,", "27_718,") + subscripted = source.replace("ledger.mode_counts", "fields['mode_counts']") + assert "27718" in _replace_asserted_counts( + subscripted, "fields['mode_counts']", observed + ) + + +def test_count_breakdowns_refuse_missing_extra_or_duplicate_keys() -> None: + source = 'assert ledger.counts == {"a": 1, "b": 2}\n' + with pytest.raises(ValueError, match="differ from the observed"): + _replace_asserted_counts(source, "ledger.counts", {"a": 1}) + with pytest.raises(ValueError, match="differ from the observed"): + _replace_asserted_counts(source, "ledger.counts", {"a": 1, "b": 2, "c": 3}) + with pytest.raises(ValueError, match="differ from the observed"): + _replace_asserted_counts( + 'assert ledger.counts == {"a": 1, "a": 2}\n', "ledger.counts", {"a": 1} + ) + with pytest.raises(ValueError, match="expected one asserted count dictionary"): + _replace_asserted_counts(source + source, "ledger.counts", {"a": 1, "b": 2}) + + def test_calibration_pin_hashes_the_fixture_contract_not_the_loaded_bundle() -> None: # The loader writes integral floats as integers (5.0 -> 5), so a digest of # the compiled bundle's calibration never matches the golden test's pin. diff --git a/tools/repin_us_spec_engine_contracts.py b/tools/repin_us_spec_engine_contracts.py index 4b5d094f0a..850141992d 100644 --- a/tools/repin_us_spec_engine_contracts.py +++ b/tools/repin_us_spec_engine_contracts.py @@ -217,6 +217,49 @@ def _replace_asserted_dictionary_value( return source if old == value else _replace_node(source, node, value) +def _replace_asserted_counts(source: str, target: str, observed: dict) -> str: + """Refresh every entry of one asserted count dictionary by its key. + + Keying by name rather than by the old value keeps a merge that already + moved the evidence JSON, but left a conflicted test on the other side, + from hiding a stale literal. The asserted keys must match the observed + ledger exactly. + """ + + matching = [ + node.test.comparators[0] + for node in ast.walk(ast.parse(source)) + if isinstance(node, ast.Assert) + and isinstance(node.test, ast.Compare) + and len(node.test.ops) == 1 + and isinstance(node.test.ops[0], ast.Eq) + and len(node.test.comparators) == 1 + and ast.unparse(node.test.left) == target + ] + if len(matching) != 1 or not isinstance(matching[0], ast.Dict): + raise ValueError(f"expected one asserted count dictionary: {target}") + node = matching[0] + if None in node.keys: + raise ValueError(f"{target}: dictionary unpacking is not a count pin") + keys = [ast.literal_eval(key) for key in node.keys] + if len(set(keys)) != len(keys) or set(keys) != set(observed): + raise ValueError( + f"{target}: asserted keys {sorted(keys)} differ from the observed " + f"{sorted(observed)}" + ) + stale = [ + (value, observed[key]) + for key, value in zip(keys, node.values, strict=True) + if ast.literal_eval(value) != observed[key] + ] + # Rewrite from the end so earlier offsets stay valid. + for value, new in sorted( + stale, key=lambda item: (item[0].lineno, item[0].col_offset), reverse=True + ): + source = _replace_node(source, value, new) + return source + + def fixture_calibration_contract() -> dict: """Build the calibration contract exactly as its semantics test fixture does. @@ -456,23 +499,36 @@ def _compiler_pins(*, check: bool) -> None: # Tests deliberately pin literal counts, so derive their corresponding # expectations from the same validated ledger rather than weakening them. - old_report = json.loads( - (ROOT / "docs/evidence/spec-engine/us-f0-coverage.json").read_text() - ) + # Totals move by value transition from the module's own pins; the asserted + # breakdowns are rewritten by key from the ledger. replacements = {} for name, value in counts.items(): _add_count_replacement(replacements, old_counts[name], value) - for group, observed in ( - ("mode_counts", ledger.mode_counts), - ("generation0_effect_counts", ledger.generation0_effect_counts), - ): - for key, value in observed.items(): - _add_count_replacement( - replacements, old_report["field_usage"][group][key], value - ) _add_count_replacement(replacements, old_tail, new_tail) for path in (FIELD_TEST, COVERAGE_TEST): rendered[path] = _replace_literals(path.read_text(), replacements) + source_counts = { + "authored": counts["EXPECTED_AUTHORED_FIELD_COUNT"], + "resolved_bindings": counts["EXPECTED_RESOLVED_BINDING_FIELD_COUNT"], + } + for path, prefix, breakdowns in ( + ( + FIELD_TEST, + "field_ledger.", + {"source_counts": source_counts}, + ), + (COVERAGE_TEST, "fields", {}), + ): + breakdowns = { + **breakdowns, + "mode_counts": dict(ledger.mode_counts), + "generation0_effect_counts": dict(ledger.generation0_effect_counts), + } + for name, observed in breakdowns.items(): + target = ( + f"{prefix}{name}" if prefix.endswith(".") else f"{prefix}[{name!r}]" + ) + rendered[path] = _replace_asserted_counts(rendered[path], target, observed) observed_counts = report["counts"] column_count = observed_counts["typed_columns"] artifact_count = observed_counts["typed_artifacts"] From f44c3239b016d29b9e647ccd50f454d7f547ea4f Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 24 Oct 2026 02:56:32 -0400 Subject: [PATCH 09/10] Find every field-usage test pin by its asserted expression Round-3 review noted the scalar totals still moved by value transition from the module's own pins, so a merge that had already moved EXPECTED_CONFIGURATION_FIELD_COUNT could leave a stale 42_228 in a test that --check would not see. The engine-free field-usage test and the coverage-tool test are now refreshed only by asserted expression: - `len(field_ledger.fields)` and the distinct-pointer count; - the coverage report's field totals and claim count; - the inventory item counts and the four asserted inventory counts; - the keyed breakdowns. Planting stale values in three of these makes --check fail and the tool restore them. The value-transition helper, now unused, is removed with its test. Co-Authored-By: Claude Opus 5.5 --- .../us/test_us_spec_engine_repin.py | 34 ++++++--- tools/repin_us_spec_engine_contracts.py | 74 +++++++++++++++---- 2 files changed, 84 insertions(+), 24 deletions(-) diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_spec_engine_repin.py b/packages/microcosm-build/tests/engine_free/us/test_us_spec_engine_repin.py index fb5a260282..1801cea525 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_spec_engine_repin.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_spec_engine_repin.py @@ -5,9 +5,9 @@ import pytest from tools.repin_us_spec_engine_contracts import ( - _add_count_replacement, _replace_asserted_counts, _replace_asserted_dictionary_value, + _replace_asserted_scalar, _replace_assignment, _replace_dictionary, _replace_length_assertions, @@ -38,14 +38,6 @@ def test_repin_refuses_unknown_assignments_and_dictionary_entries() -> None: _replace_dictionary('PINS = {"known": 1}\n', "PINS", {"unknown": 2}) -def test_repin_refuses_colliding_count_transitions() -> None: - replacements = {} - _add_count_replacement(replacements, 176, 179) - _add_count_replacement(replacements, 176, 179) - with pytest.raises(ValueError, match="ambiguous count replacement"): - _add_count_replacement(replacements, 176, 180) - - def test_inventory_count_updates_only_the_reviewed_length_target() -> None: source = ( "assert len(columns) == 176\nassert len(unrelated) == 176\n" @@ -157,6 +149,30 @@ def test_count_breakdowns_refuse_missing_extra_or_duplicate_keys() -> None: _replace_asserted_counts(source + source, "ledger.counts", {"a": 1, "b": 2}) +def test_scalar_pins_are_found_by_asserted_expression_not_old_value() -> None: + # The module pin may already read 42_229 after a merge while the test + # still asserts 42_228; matching the expression refreshes it anyway. + source = ( + "assert len(ledger.fields) == EXPECTED == 42_228\n" + 'assert fields["claim_count"] == 49\n' + "assert other == 42_228\n" + ) + refreshed = _replace_asserted_scalar(source, "len(ledger.fields)", 42_229) + assert refreshed == source.replace("EXPECTED == 42_228", "EXPECTED == 42229") + assert _replace_asserted_scalar(source, "fields['claim_count']", 49) == source + + +def test_scalar_pins_refuse_missing_duplicate_or_nonliteral_assertions() -> None: + with pytest.raises(ValueError, match="expected one scalar assertion"): + _replace_asserted_scalar("assert x == 1\n", "y", 2) + with pytest.raises(ValueError, match="expected one scalar assertion"): + _replace_asserted_scalar("assert x == 1\nassert x == 1\n", "x", 2) + with pytest.raises(ValueError, match="not an integer literal"): + _replace_asserted_scalar("assert x == EXPECTED\n", "x", 2) + with pytest.raises(ValueError, match="not an integer literal"): + _replace_asserted_scalar("assert x == True\n", "x", 2) + + def test_calibration_pin_hashes_the_fixture_contract_not_the_loaded_bundle() -> None: # The loader writes integral floats as integers (5.0 -> 5), so a digest of # the compiled bundle's calibration never matches the golden test's pin. diff --git a/tools/repin_us_spec_engine_contracts.py b/tools/repin_us_spec_engine_contracts.py index 850141992d..cbd6798048 100644 --- a/tools/repin_us_spec_engine_contracts.py +++ b/tools/repin_us_spec_engine_contracts.py @@ -113,12 +113,6 @@ def _save(path: Path, source: str, *, check: bool) -> None: print(f"updated {relative}") -def _add_count_replacement(replacements: dict, old: int, new: int) -> None: - if old in replacements and replacements[old] != new: - raise ValueError(f"ambiguous count replacement for {old}") - replacements[old] = new - - def _replace_length_assertions(source: str, expected: dict[str, int]) -> str: matches = {} for assertion in ast.walk(ast.parse(source)): @@ -260,6 +254,29 @@ def _replace_asserted_counts(source: str, target: str, observed: dict) -> str: return source +def _replace_asserted_scalar(source: str, target: str, value: int) -> str: + """Refresh the integer that ``assert == ... == `` pins. + + Matching the asserted expression, not the old number, refreshes a stale + literal even when the module pin it was copied from has already moved. + """ + + matching = [ + node.test + for node in ast.walk(ast.parse(source)) + if isinstance(node, ast.Assert) + and isinstance(node.test, ast.Compare) + and all(isinstance(op, ast.Eq) for op in node.test.ops) + and ast.unparse(node.test.left) == target + ] + if len(matching) != 1: + raise ValueError(f"expected one scalar assertion: {target}") + pin = matching[0].comparators[-1] + if not (isinstance(pin, ast.Constant) and type(pin.value) is int): + raise ValueError(f"{target}: the asserted pin is not an integer literal") + return source if pin.value == value else _replace_node(source, pin, value) + + def fixture_calibration_contract() -> dict: """Build the calibration contract exactly as its semantics test fixture does. @@ -434,7 +451,6 @@ def _compiler_pins(*, check: bool) -> None: ), "EXPECTED_CONFIGURATION_FIELD_COUNT": len(pointers), } - old_counts = {name: getattr(field_usage, name) for name in counts} source = FIELD_USAGE.read_text() source = _replace_dictionary(source, "_PINS", expansions) for name, value in counts.items(): @@ -499,14 +515,42 @@ def _compiler_pins(*, check: bool) -> None: # Tests deliberately pin literal counts, so derive their corresponding # expectations from the same validated ledger rather than weakening them. - # Totals move by value transition from the module's own pins; the asserted - # breakdowns are rewritten by key from the ledger. - replacements = {} - for name, value in counts.items(): - _add_count_replacement(replacements, old_counts[name], value) - _add_count_replacement(replacements, old_tail, new_tail) - for path in (FIELD_TEST, COVERAGE_TEST): - rendered[path] = _replace_literals(path.read_text(), replacements) + # Every pin is found by its asserted expression, never by its old value, + # so a merge that moved one copy of a count cannot hide a stale other. + total = counts["EXPECTED_CONFIGURATION_FIELD_COUNT"] + scalars = { + FIELD_TEST: { + "len(field_ledger.fields)": total, + "len({field.pointer for field in field_ledger.fields})": total, + }, + COVERAGE_TEST: { + "fields['configuration_field_count']": total, + "fields['authored_normative_field_count']": counts[ + "EXPECTED_AUTHORED_FIELD_COUNT" + ], + "fields['resolved_binding_field_count']": counts[ + "EXPECTED_RESOLVED_BINDING_FIELD_COUNT" + ], + "fields['consumed_field_count']": total, + "fields['claim_count']": len(ledger.claims), + "inventory['required_item_count']": report["required_item_count"], + "inventory['covered_item_count']": report["covered_item_count"], + **{ + f"inventory['counts'][{key!r}]": report["counts"][key] + for key in ( + "producer_inputs", + "ownership_rows", + "tail_control_fields", + "seed_owner_bindings", + ) + }, + }, + } + for path, expected in scalars.items(): + source = path.read_text() + for target, value in expected.items(): + source = _replace_asserted_scalar(source, target, value) + rendered[path] = source source_counts = { "authored": counts["EXPECTED_AUTHORED_FIELD_COUNT"], "resolved_bindings": counts["EXPECTED_RESOLVED_BINDING_FIELD_COUNT"], From 7ab3a7737422391e560a109b43eb499292f73cf9 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 24 Oct 2026 03:54:11 -0400 Subject: [PATCH 10/10] Refresh the unclaimed-field expectation and restore #1376's seed note Round-4 review found that the identity test's `r"87 unclaimed normative field"` regex has no refresh path. Every field has exactly one primary claim, so dropping the selection claim leaves exactly its observed expansion (87) unclaimed. The repinner now rewrites that one expectation from the expansion it already computes. With a stale 86 planted, --check fails and the tool restores 87. The main merge also dropped #1376's note that the puma_county_vintage draw module joins the seed protocol's attested kernel modules; it is restored above the seed hashes. Co-Authored-By: Claude Opus 5.5 --- .../build/spec_engine/inventory_coverage.py | 3 +- .../us/test_us_spec_engine_repin.py | 13 +++++++ tools/repin_us_spec_engine_contracts.py | 34 +++++++++++++++++++ 3 files changed, 49 insertions(+), 1 deletion(-) diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py index 0480f57965..88fe1f6a47 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py @@ -368,7 +368,8 @@ # keeps the US bracket, and the budget search, which the US solve does # not run, changes its probes, so again no US solve changes. #454: the # meps_esi_premiums stage joins the attested sources and the source - # manifest. + # manifest. #1376: the puma_county_vintage draw module joins the seed + # protocol's attested kernel modules. "seed_map": "58741b7c6c8ca8f0ead476ec9893355419c9550d9bdb9287bec57928ce345afb", "seed_protocol": "b4c450a618c5ca3116f806d1ae270ae6fb410517404c204e505d10418bc1181f", "source_manifest": "87b1e124969ec11b836740c027d28dc2a9d319d279ed55be1d9fc1523390f634", diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_spec_engine_repin.py b/packages/microcosm-build/tests/engine_free/us/test_us_spec_engine_repin.py index 1801cea525..d928b269cb 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_spec_engine_repin.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_spec_engine_repin.py @@ -13,6 +13,7 @@ _replace_length_assertions, _replace_literal_assertions, _replace_literals, + _replace_unclaimed_count, ) @@ -173,6 +174,18 @@ def test_scalar_pins_refuse_missing_duplicate_or_nonliteral_assertions() -> None _replace_asserted_scalar("assert x == True\n", "x", 2) +def test_unclaimed_count_expectation_follows_the_observed_expansion() -> None: + source = 'with raises(E, match=r"87 unclaimed normative field"):\n# 87 stays\n' + assert _replace_unclaimed_count(source, 90) == source.replace( + 'r"87 unclaimed', 'r"90 unclaimed' + ) + assert _replace_unclaimed_count(source, 87) == source + with pytest.raises(ValueError, match="found 0"): + _replace_unclaimed_count("x = 1\n", 90) + with pytest.raises(ValueError, match="found 2"): + _replace_unclaimed_count(source + source, 90) + + def test_calibration_pin_hashes_the_fixture_contract_not_the_loaded_bundle() -> None: # The loader writes integral floats as integers (5.0 -> 5), so a digest of # the compiled bundle's calibration never matches the golden test's pin. diff --git a/tools/repin_us_spec_engine_contracts.py b/tools/repin_us_spec_engine_contracts.py index cbd6798048..cf706781e0 100644 --- a/tools/repin_us_spec_engine_contracts.py +++ b/tools/repin_us_spec_engine_contracts.py @@ -22,6 +22,7 @@ import ast import hashlib import json +import re import sys import tokenize from io import StringIO @@ -277,6 +278,34 @@ def _replace_asserted_scalar(source: str, target: str, value: int) -> str: return source if pin.value == value else _replace_node(source, pin, value) +def _replace_unclaimed_count(source: str, count: int) -> str: + """Refresh the one ``" unclaimed normative field"`` expectation. + + Every field has exactly one primary claim, so dropping a claim leaves + exactly its observed expansion unclaimed. + """ + + pattern = re.compile(r"\d+ unclaimed normative field") + hits = [] + for token in tokenize.generate_tokens(StringIO(source).readline): + if token.type != tokenize.STRING: + continue + try: + text = ast.literal_eval(token.string) + except (ValueError, SyntaxError): + continue + if isinstance(text, str) and pattern.fullmatch(text): + hits.append(token) + if len(hits) != 1: + raise ValueError(f"expected one unclaimed-field expectation, found {len(hits)}") + (token,) = hits + lines = source.splitlines(keepends=True) + start = sum(map(len, lines[: token.start[0] - 1])) + token.start[1] + end = sum(map(len, lines[: token.end[0] - 1])) + token.end[1] + literal = re.sub(r"\d+", str(count), token.string, count=1) + return source[:start] + literal + source[end:] + + def fixture_calibration_contract() -> dict: """Build the calibration contract exactly as its semantics test fixture does. @@ -573,6 +602,11 @@ def _compiler_pins(*, check: bool) -> None: f"{prefix}{name}" if prefix.endswith(".") else f"{prefix}[{name!r}]" ) rendered[path] = _replace_asserted_counts(rendered[path], target, observed) + # The identity test drops the selection claim and expects its fields back + # as unclaimed. + rendered[FIELD_TEST] = _replace_unclaimed_count( + rendered[FIELD_TEST], expansions["selection"][0] + ) observed_counts = report["counts"] column_count = observed_counts["typed_columns"] artifact_count = observed_counts["typed_artifacts"]