diff --git a/changelog.d/719-work-experience-columns.added.md b/changelog.d/719-work-experience-columns.added.md new file mode 100644 index 0000000000..64a161e82d --- /dev/null +++ b/changelog.d/719-work-experience-columns.added.md @@ -0,0 +1 @@ +US release person records carry `detailed_industry_recode` (CPS `WEIND`), `major_industry_recode` (`WEMIND`) and `worked_last_year` (`WKSWORK > 0`) next to `detailed_occupation_recode`: ASEC vintages restore `WEIND`/`WEMIND` by exact `PERIDNUM` join before pooling, and the `org_wages` carry checks their industry identities. Both the release predictor join and native ACS input mapper reuse the same `INDP`/`WKWN` crosswalk (version 2). ACS local staging and packaging refuse incomplete canonical job inputs on donor or final person rows (#719). diff --git a/docs/agent-guide.md b/docs/agent-guide.md index 2fe1078b74..455aad6cbb 100644 --- a/docs/agent-guide.md +++ b/docs/agent-guide.md @@ -185,6 +185,14 @@ module or dependency. CI tests the merge ref, so merge main and re-pin rather than hunting for an environment leak. Editable installs hide packaging breaks; if you touch packaging, build wheels locally before pushing. +For an intentional US source-stage change, regenerate the release input +coverage manifest with `tools/build_us_release_input_coverage_manifest.py`, +then follow the bundle/pin/coverage sequence documented in +`tools/repin_us_spec_engine_contracts.py`. The repinner derives frozen source +hashes, compiler field inventories and their test expectations; its `--check` +mode refuses drift. Full bundle generation needs the US engine environment. +These commands do not rebuild survey data or certify a release. + The `integration-uk` job in `.github/workflows/test.yml` runs the `integration/uk/` directory serially on Python 3.13 whenever the unified changed-path selection includes the UK, and on every push to `main`. It skips @@ -335,6 +343,11 @@ and the ACS local-area `stored_inputs` gate, each naming the built-with engine on which an SPM unit the built-with engine measures has no classified adult, and binds the `spm_measurement_composition` gate the publisher requires (microcosm#1263; `microcosm.build.us_runtime.spm_measurement_gate`). +The same package stage requires complete occupation, industry and +worked-last-year inputs on every final person row and binds that verdict to +the H5 hash. ACS staging checks its preserved ASEC donor before acquisition; +historical donors and staging files missing these inputs must be rebuilt +through the exact ASEC restoration/fiscal carry and native ACS mapping. US household ladders carry both `county_fips` (2020 counties for the locked policyengine-us rules) and `county_fips_2023` (3,144 county-equivalents for calibration, including Connecticut's nine planning regions). The latter is diff --git a/docs/evidence/spec-engine/us-f0-coverage.json b/docs/evidence/spec-engine/us-f0-coverage.json index 7faef0faae..5998136af6 100644 --- a/docs/evidence/spec-engine/us-f0-coverage.json +++ b/docs/evidence/spec-engine/us-f0-coverage.json @@ -4,9 +4,9 @@ "version": 1 }, "country": "us", - "documentation_sha256": "44f40b4e8aaa2e6be67a7bc9680cc29f88e185c10573ffb9c3f9dfbd3ad1081f", + "documentation_sha256": "801a99da54b8a4834a8c5e6c4b013fdae1e7576e0132de7e87cb1ef098f1ad8c", "field_usage": { - "authored_normative_field_count": 32437, + "authored_normative_field_count": 32448, "claim_count": 49, "claims": [ { @@ -108,8 +108,8 @@ ], "mode": "legacy_behavior", "pointer_class": "all", - "pointer_count": 312, - "pointer_sha256": "9bad55b4945af1c4213c510942e7b9d22d104bf75b0d9804e4a183e1656c8393", + "pointer_count": 315, + "pointer_sha256": "13227bf16227b7027c6ccd1220bf2b451b6c8be2b17dbe63b0c5500dc32433a2", "rationale": null, "relative_sink_prefix": "/calibration_contract", "source_prefix": "/authored/spec~1calibration.yaml", @@ -514,8 +514,8 @@ ], "mode": "legacy_behavior", "pointer_class": "all", - "pointer_count": 1749, - "pointer_sha256": "d0979722ea5153bd152e5e1c1201f5d530d11b21aecad18aca38473962caa062", + "pointer_count": 1757, + "pointer_sha256": "29c137db9d9e522c4280fbad42a3d3528411453853d2c7c5d1d347662e7b2076", "rationale": null, "relative_sink_prefix": "/source_manifest/stages", "source_prefix": "/authored/spec~1sources.yaml/stages", @@ -772,20 +772,20 @@ "verifier": "vintages" } ], - "configuration_field_count": 42218, - "consumed_field_count": 42218, + "configuration_field_count": 42229, + "consumed_field_count": 42229, "generation0_effect_counts": { - "legacy_behavior": 38531, + "legacy_behavior": 38542, "no_generation0_effect": 3687 }, "mode_counts": { "compiler_semantic": 27718, "front_end_validation": 354, "identity_only": 103, - "legacy_behavior": 14043 + "legacy_behavior": 14054 }, "multiple_primary_use_field_count": 0, - "pointer_inventory_sha256": "a0da3562ad5559cbbcdf5ee14ebd21db2f067c90e7e470e24d4e3762d7b90848", + "pointer_inventory_sha256": "0a2951f4484a0dd9459a474505b5a0caa0216b80ad13009a286dd85e2165e7e8", "resolved_binding_field_count": 9781, "unused_field_count": 0 }, @@ -824,7 +824,7 @@ "stacked_checkpoint_full_components": 13, "stacked_checkpoint_pool_code_components": 19, "stacked_checkpoint_static_components": 10, - "tail_control_fields": 934, + "tail_control_fields": 937, "take_up_pipeline_steps": 28, "take_up_programs": 17, "typed_artifacts": 84, @@ -917,13 +917,13 @@ "legacy_adapter.stacked_checkpoint_static_components" ], "expected": { - "sha256": "ac92829c88a1a4fb6460d61190918d5d99c6c377fc8dd8f62f02b332d09bf59c", - "terminal_fields": 934 + "sha256": "256947cd7c1bbd85f5d1fa3911a07a170ae87830f098110a40ed25d93f78347c", + "terminal_fields": 937 }, "failures": [], "observed": { - "sha256": "ac92829c88a1a4fb6460d61190918d5d99c6c377fc8dd8f62f02b332d09bf59c", - "terminal_fields": 934 + "sha256": "256947cd7c1bbd85f5d1fa3911a07a170ae87830f098110a40ed25d93f78347c", + "terminal_fields": 937 }, "status": "covered" }, @@ -1361,12 +1361,12 @@ ], "expected": { "producer_count": 38, - "sha256": "cbac7c8fbdbb82f0f06ab356d0c9f0d3645ca930550b44af2272d93315f2e934" + "sha256": "b555f811aaf1b65f588a3aa0a6cee2d311b1bc40b1735749c527749b5b3d0da1" }, "failures": [], "observed": { "producer_count": 38, - "sha256": "cbac7c8fbdbb82f0f06ab356d0c9f0d3645ca930550b44af2272d93315f2e934" + "sha256": "b555f811aaf1b65f588a3aa0a6cee2d311b1bc40b1735749c527749b5b3d0da1" }, "status": "covered" }, @@ -1775,12 +1775,12 @@ "legacy_adapter.imputation.late_producer_resource_semantics" ], "expected": { - "sha256": "629a0712659a26b2d6aa6c0e5672421f415d625e0599d6baa0a834d118953c6a", + "sha256": "87b1e124969ec11b836740c027d28dc2a9d319d279ed55be1d9fc1523390f634", "stages": 38 }, "failures": [], "observed": { - "sha256": "629a0712659a26b2d6aa6c0e5672421f415d625e0599d6baa0a834d118953c6a", + "sha256": "87b1e124969ec11b836740c027d28dc2a9d319d279ed55be1d9fc1523390f634", "stages": 38 }, "status": "covered" @@ -1887,7 +1887,7 @@ "alpha", "zeta" ], - "sha256": "e5da85693fab1f1cba9c37beba5cdf91477962d466bc21e9f29125f2f90b7e79" + "sha256": "9fb7350a66a0ec9f7a4dbc879ccafaf1e6cb51486722938b3a8e6f5c57ae9e06" }, "failures": [], "observed": { @@ -1911,7 +1911,7 @@ "alpha", "zeta" ], - "sha256": "e5da85693fab1f1cba9c37beba5cdf91477962d466bc21e9f29125f2f90b7e79" + "sha256": "9fb7350a66a0ec9f7a4dbc879ccafaf1e6cb51486722938b3a8e6f5c57ae9e06" }, "status": "covered" }, @@ -1975,7 +1975,7 @@ "take_up_contract", "us_qbi_reconciliation_contract" ], - "sha256": "fb863f18a79610dc1005089a32e1535c502c5b9f7e5ba589db0d6025b01e3c82" + "sha256": "b7669fd0f941d7ca68fbdf5966d5ae8751fda598dc64f6d0d7420c0c33b8e9e2" }, "status": "covered" }, @@ -2599,7 +2599,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "77df608c9b343219cd2717aa08af5198c0ba69b70c61691fb2459f7a886528ba" + "spec_sha256": "53fb8de63585a27fc86a10a94eeae4ca546a4ded2d528514c64e4df574b430f8" } }, "report_schema_version": 3, @@ -2609,7 +2609,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "77df608c9b343219cd2717aa08af5198c0ba69b70c61691fb2459f7a886528ba" + "spec_sha256": "53fb8de63585a27fc86a10a94eeae4ca546a4ded2d528514c64e4df574b430f8" }, "status": "pass" } diff --git a/docs/us-asec-census-person-columns.md b/docs/us-asec-census-person-columns.md index 11cd9aeefc..96bb84b1fc 100644 --- a/docs/us-asec-census-person-columns.md +++ b/docs/us-asec-census-person-columns.md @@ -115,6 +115,12 @@ the column. `CENSUS_TAX_ID` was never appended: the extractor derives it, and the Census files do not carry it. +Beyond that receipt, `WEIND` and `WEMIND` (industry of the longest job last +year) are restored for #719, read by `org_wages.derive_us_org_occupation_inputs`. +No H5 vintage carries them, so every vintage gets them from its member, and +each vintage is checked against the industry identities. See +[`us-work-experience-columns.md`](us-work-experience-columns.md). + ## Real-data evidence (2026-09-23, commit `39b8e7b63`) The real-data run used route A's exact base inputs and flags diff --git a/docs/us-work-experience-columns.md b/docs/us-work-experience-columns.md new file mode 100644 index 0000000000..9301f12391 --- /dev/null +++ b/docs/us-work-experience-columns.md @@ -0,0 +1,133 @@ +# Industry and worked-last-year on every US person record (#719) + +The county-file person schema carries a working indicator, industry and +occupation on every person record. Occupation already shipped +(`detailed_occupation_recode`, CPS ASEC `POCCU2`). This change adds three +PolicyEngine-US person inputs (defined in policyengine-us#9311, released in +1.820.2, locked at 2.2.1): + +| column | meaning | CPS ASEC source | ACS 2024 source | +|---|---|---|---| +| `detailed_industry_recode` | industry of the longest job last year, 23 detailed groups | `WEIND` | `INDP` via `ACS_INDP_TO_WEIND` | +| `major_industry_recode` | the same job, 15 major groups | `WEMIND` | `WEIND_TO_WEMIND` of the above | +| `worked_last_year` | worked at any time last year | `WKSWORK > 0` | `WKWN > 0` | + +Codes (`WEIND`): 0 = not in universe (under 15; ACS: under 16), 1–21 = +civilian industry groups, 22 = Armed Forces, 23 = age 15+ (ACS: 16+) and did +not work last year. `WEMIND` nests them: 0 = not in universe, 1–13 civilian +major groups, 14 = Armed Forces, 15 = did not work. + +## Why these CPS columns + +`WEIND`/`WEMIND` describe the longest job last year, the same job `POCCU2` +describes, and the same reference year as the income the file carries. The +survey-week recodes `A_DTIND`/`A_MJIND` describe the current job instead. +`worked_last_year` uses `WKSWORK > 0` rather than `WORKYN = 1`: in all three +pinned members `WEIND` carries a worker code exactly where `WKSWORK > 0` (zero +exceptions), while `WORKYN = 1` misses 642 / 572 / 594 allocation rows per +year that have weeks worked and an industry. + +## Where each spine gets them + +- **ASEC rows (and their PUF-support clones).** No `census_cps` H5 vintage + carries `WEIND`/`WEMIND`, not even 2024. `asec_census_person_columns` + (#720's restoration) appends both to every vintage from its pinned Census + person member by exact `PERIDNUM` join before pooling. They sit outside the + 2026-08-23 offline-fix receipt (`ASEC_CENSUS_PERSON_COLUMNS_BEYOND_OFFLINE_FIX`). + `WKSWORK` is already in every H5. +- **ACS rows.** The release predictor join (`acs_release_predictors`, + crosswalk version 2) materializes `WEIND`, `WEMIND` and `WKSWORK` for ACS + people from the pinned ACS 2024 person archive, the same boundary that maps + `OCCP` to `POCCU2`. The newer ACS local-area native-input mapper uses the + same `crosswalk_acs_occupation_and_work_experience` helper and writes the + canonical occupation, industry and worked-last-year inputs directly. Its + source loader retains the required `OCCP`, `INDP`, `WKWN` and `ESR` fields. +- **Pooled release rows.** The release-time `org_wages` carry + (`derive_us_org_occupation_inputs`) turns them into the three columns next + to `detailed_occupation_recode`. + +## The ACS crosswalk + +ACS 2024 `INDP` uses the 2022 Census industry code list, as does the 2025 ASEC +member (`pppub25.csv`), where nonzero `INDUSTRY` determines `WEIND` exactly +(259 codes). The 2023/2024 members use the 2017 list; the 225 codes the lists share +map to the same group in both. `ACS_INDP_TO_WEIND` takes every civilian code's +group from `pppub25.csv` and reviews the eight ACS-only codes: + +- 9670, 9680, 9690, 9770, 9780, 9790, 9870 (Army, Air Force, Navy, Marines, + Coast Guard, branch not specified, Reserves or National Guard) → 22. CPS + collapses them into 9890, which the members map to 22. +- 9920 ("unemployed, no work experience in the last 5 years or never + worked") → 23, as `OCCP` 9920 maps to `POCCU2` 53. + +Blank `INDP` follows the `OCCP` rule: 0 below age 16 (age 15 keeps the +out-of-universe sentinel rather than inventing CPS code 23), 23 from age 16, +where only ESR 6 may be blank. `WKSWORK` is `WKWN` (weeks worked in the past 12 +months) or 0. + +Reference periods differ, as they already do for occupation: an observed ACS +`INDP` describes the most recent job in the past five years, and `WKWN` counts +the 12 months before interview; the CPS columns describe the prior calendar +year. So on ACS rows an industry can sit on a person with +`worked_last_year = False` (a job held one to five years ago). + +## Checked identities + +| identity | ASEC rows | ACS rows | where checked | +|---|---|---|---| +| `WEMIND = WEIND_TO_WEMIND[WEIND]` | yes | yes | restoration (per vintage), ACS join (by construction), `org_wages` carry and gate | +| `WKSWORK > 0` ⇒ `WEIND` in 1–22 | yes | yes | restoration, ACS join, `org_wages` carry and gate | +| `WEIND` in 1–22 ⇒ `WKSWORK > 0` | yes | no (5-year `INDP`) | restoration only, where rows are known to be ASEC | + +## Real-data evidence (2026-09-24) + +ASEC members `pppub23/24/25.csv` (pins in `spm_role_source.ASEC_SPM_ROLE_SOURCES`): + +- Nonzero `INDUSTRY` → `WEIND` and → `WEMIND` are functions in every vintage + (263 / 263 / 259 codes); `INDUSTRY = 0` covers both `WEIND = 0` and the + did-not-work group 23; `WEIND` → `WEMIND` is the same function in all three. +- `WEIND` in 1–22 iff `INDUSTRY > 0` iff `WKSWORK > 0`: zero violations in + each vintage. `WEIND = 0` exactly for ages 0–14; `WEIND = 23` only from 15. +- Weighted (`A_FNLWGT`) shares: nonzero `WEIND` 0.8217 / 0.8220 / 0.8244; + worked last year 0.5200 / 0.5232 / 0.5228. + +ACS 2024 one-year person archive (SHA-256 `afdc6d90…`, the release pin), all +3,422,888 persons run through `_crosswalk_people`: + +- No refusal fires; all 265 observed `INDP` codes are mapped. +- `INDP` and `OCCP` are blank on exactly the same rows; a blank at 16+ occurs + only with ESR 6. +- `WKWN > 0` iff `WKL = 1` (zero disagreements); every person with weeks + worked has a worker `WEIND` code. +- 264,806 persons carry an industry without weeks worked, exactly the count + with `WKL = 2` (last worked one to five years ago). +- Weighted (`PWGTP`) shares: nonzero `WEIND` 0.8123; worked last year 0.5385. + +The `org_wages` gate bands (`detailed_industry_recode` and +`major_industry_recode` nonzero 0.70–0.92, `worked_last_year` 0.40–0.62) +bracket both spines. + +## What this does not do + +It changes code and manifests only. Released files carry the columns once the +pooled base and multispine pool are rebuilt with the restoration and a new +release runs (#922 sequencing step 1); until then the release input-coverage +gate is red on these three columns by design. + +The ACS local-area staging lane also preserves its ASEC donor rows. It now +refuses a donor missing complete canonical occupation, industry or +worked-last-year inputs before acquiring ACS data. Historical donors need +the exact ASEC restoration and fiscal `org_wages` carry before that lane can +produce a complete stacked export. Packaging rechecks the final H5 person +rows and binds the result to its hash, so an older green staging summary +cannot bypass this requirement. The assembly-first nullable pool remains +an intermediate input; its fiscal release applies that carry to every row. + +The 2026-10-23 branch refresh preserves main's current stage and graph +structure, including its six ESI source fields alongside the industry +restoration. Main's independent refusal of mixed ACS inputs with null ESI +premiums remains in place; this refresh adds no ESI producer. + +The real-data measurements above are historical evidence; the +refresh validates synthetic contracts without rebuilding ASEC, ACS or a +release, so it does not certify any existing or newly built data artifact. diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py b/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py index 45da55cb35..115d58f049 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py @@ -26,9 +26,9 @@ ) from .schemas import load_schema_registry -EXPECTED_AUTHORED_FIELD_COUNT = 32_437 -EXPECTED_RESOLVED_BINDING_FIELD_COUNT = 9_781 -EXPECTED_CONFIGURATION_FIELD_COUNT = 42_218 +EXPECTED_AUTHORED_FIELD_COUNT = 32_448 +EXPECTED_RESOLVED_BINDING_FIELD_COUNT = 9781 +EXPECTED_CONFIGURATION_FIELD_COUNT = 42229 class FieldUsageError(AssertionError): @@ -361,8 +361,8 @@ def _path_inventory(rows: Sequence[tuple[str, object]]) -> tuple[int, str]: "36564747627fb0058bb99e8e8c4397e41d8ecc472a20c49d4b6d7df1003a80ec", ), "calibration": ( - 312, - "9bad55b4945af1c4213c510942e7b9d22d104bf75b0d9804e4a183e1656c8393", + 315, + "13227bf16227b7027c6ccd1220bf2b451b6c8be2b17dbe63b0c5500dc32433a2", ), "catalog_columns": ( 1_673, @@ -485,8 +485,8 @@ def _path_inventory(rows: Sequence[tuple[str, object]]) -> tuple[int, str]: "d6782c5de5bbed1bdc6bf653c4a6d4aadcad4ccc72d35e1092e130fcb04680a3", ), "source_stages": ( - 1_749, - "d0979722ea5153bd152e5e1c1201f5d530d11b21aecad18aca38473962caa062", + 1_757, + "29c137db9d9e522c4280fbad42a3d3528411453853d2c7c5d1d347662e7b2076", ), "spine_assembly_household_mass_shares": ( 2, diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py index 0b31e123ea..88fe1f6a47 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py @@ -350,12 +350,12 @@ "acs_person_predictors": "878c788a6f037d7aca12b3586ea034eff04f3034ffa11935a736493042551f25", "authority": "9d4a9672a0f03039b1fe874b9fe21ed575be0d29f14afc396d03cdf5c809bdd2", "early_families": "4aa9f736fd76e83955477ad1667e58f48f264783f05bdc7f0102cd32d61323bd", - "full_checkpoint": "e5da85693fab1f1cba9c37beba5cdf91477962d466bc21e9f29125f2f90b7e79", + "full_checkpoint": "9fb7350a66a0ec9f7a4dbc879ccafaf1e6cb51486722938b3a8e6f5c57ae9e06", "gap_fill_schedule": "1c31f9868f7884347cc19cf1ff65da43f950b9114941a715bab168246db414a7", "graph_nodes": "40cd51ffdfe2e9d9d08d48c08e8ded9de1e4b134783bab05c4abc6ad5c72ca1e", "geography_assignment": "44e7f5f6cd7ceeef85d532c3d46306d03de4f4ee811376e6d29920742746b65c", "late_families": "d91f9ff0eb52f43e7b6eed3d5c58c37abe1620c3a11021da15dae9c10e16d382", - "late_resource_semantics": "cbac7c8fbdbb82f0f06ab356d0c9f0d3645ca930550b44af2272d93315f2e934", + "late_resource_semantics": "b555f811aaf1b65f588a3aa0a6cee2d311b1bc40b1735749c527749b5b3d0da1", "late_schedule": "e59c019d3d454eac99ac0ac209b6c5b6faaf9bdfcaeee18c36a25be19bf7da2f", "ownership": "5f64f0aac49e2313177564f71876bffc8c81b3ded4df701e70930e60e9c98356", "primary_tuples": "fdf23da429f8c501e198a2f5b719763d523a41565c2153fdf17b0344f21bcd7c", @@ -372,9 +372,9 @@ # protocol's attested kernel modules. "seed_map": "58741b7c6c8ca8f0ead476ec9893355419c9550d9bdb9287bec57928ce345afb", "seed_protocol": "b4c450a618c5ca3116f806d1ae270ae6fb410517404c204e505d10418bc1181f", - "source_manifest": "629a0712659a26b2d6aa6c0e5672421f415d625e0599d6baa0a834d118953c6a", + "source_manifest": "87b1e124969ec11b836740c027d28dc2a9d319d279ed55be1d9fc1523390f634", "take_up": "9522ce40f7dea569312dd7a7beb474e5a5afadd2f1e10cb5876534c3ef623d35", - "tail": "ac92829c88a1a4fb6460d61190918d5d99c6c377fc8dd8f62f02b332d09bf59c", + "tail": "256947cd7c1bbd85f5d1fa3911a07a170ae87830f098110a40ed25d93f78347c", } INVENTORY_REPORT_SCHEMA_VERSION = 1 @@ -454,7 +454,7 @@ "stacked_checkpoint_full_components": 13, "stacked_checkpoint_pool_code_components": 19, "stacked_checkpoint_static_components": 10, - "tail_control_fields": 934, + "tail_control_fields": 937, "take_up_pipeline_steps": 28, "take_up_programs": 17, "typed_artifacts": 84, @@ -1422,7 +1422,7 @@ def add( add( "capital_gains_tail_contract_exact", clauses={ - "tail terminal field count differs": tail_count == 934, + "tail terminal field count differs": tail_count == 937, "resolved calibration tails differ": _json_equal( legacy.get("calibration_tail_contracts"), expected_legacy["calibration_tail_contracts"], @@ -1450,7 +1450,7 @@ def add( "legacy_adapter.stacked_checkpoint_static_components", ), observed={"terminal_fields": tail_count, "sha256": sha256_json(tail_bundle)}, - expected={"terminal_fields": 934, "sha256": EXPECTED_HASHES["tail"]}, + expected={"terminal_fields": 937, "sha256": EXPECTED_HASHES["tail"]}, ) add( diff --git a/packages/microcosm-build/src/microcosm/build/us/release_input_coverage_manifest.json b/packages/microcosm-build/src/microcosm/build/us/release_input_coverage_manifest.json index 6555a7a3e2..1edacd1464 100644 --- a/packages/microcosm-build/src/microcosm/build/us/release_input_coverage_manifest.json +++ b/packages/microcosm-build/src/microcosm/build/us/release_input_coverage_manifest.json @@ -56,6 +56,10 @@ "cps_race": { "status": "required" }, + "detailed_industry_recode": { + "note": "Industry of the longest job last year, 23 CPS detailed groups, carried from ASEC WEIND (restored before pooling by asec_census_person_columns) by the org_wages release stage (org_wages.derive_us_org_occupation_inputs). ACS derives the same input from native INDP/WKWN through the shared crosswalk in the fiscal release predictor join and native ACS mapper; a formula-less person input in PolicyEngine-US 2.2.1. Required with NO reviewed exclusion per PolicyEngine/microcosm#719: the county-file person schema carries a working indicator, industry and occupation on every record. Currently absent from every published US default (all predate #719) — this is the intended red gate until the next release is built with the carry.", + "status": "required" + }, "detailed_occupation_recode": { "status": "required" }, @@ -271,6 +275,10 @@ "long_term_capital_gains_on_collectibles": { "status": "required" }, + "major_industry_recode": { + "note": "Industry of the longest job last year, 15 CPS major groups, carried from ASEC WEMIND (restored before pooling by asec_census_person_columns) by the org_wages release stage (org_wages.derive_us_org_occupation_inputs). ACS derives the same input from native INDP/WKWN through the shared crosswalk in the fiscal release predictor join and native ACS mapper; a formula-less person input in PolicyEngine-US 2.2.1. Required with NO reviewed exclusion per PolicyEngine/microcosm#719: the county-file person schema carries a working indicator, industry and occupation on every record. Currently absent from every published US default (all predate #719) — this is the intended red gate until the next release is built with the carry.", + "status": "required" + }, "meets_ssi_disability_criteria": { "status": "required" }, @@ -541,6 +549,10 @@ "weeks_unemployed": { "status": "required" }, + "worked_last_year": { + "note": "Worked at any time last year, carried from ASEC WKSWORK > 0 by the org_wages release stage (org_wages.derive_us_org_occupation_inputs). ACS derives the same input from native INDP/WKWN through the shared crosswalk in the fiscal release predictor join and native ACS mapper; a formula-less person input in PolicyEngine-US 2.2.1. Required with NO reviewed exclusion per PolicyEngine/microcosm#719: the county-file person schema carries a working indicator, industry and occupation on every record. Currently absent from every published US default (all predate #719) — this is the intended red gate until the next release is built with the carry.", + "status": "required" + }, "workers_compensation": { "status": "required" }, @@ -549,11 +561,11 @@ } }, "counts": { - "required": 168, + "required": 171, "reviewed_exclusion": 6, - "total": 174 + "total": 177 }, - "derivation": "Required surface = input columns in the pinned, sha-verified ecps_parity_reference.json populated layers, plus the documented post-reference fsla_overtime_premium, qualified_passenger_vehicle_loan_interest, five desired retirement-contribution inputs, meets_ssi_disability_criteria required by shipped validation probes, the #282 Schedule-D capital-gain-distributions route leg schedule_d_capital_gain_distributions (PolicyEngine/microcosm#462), the three ASEC reported-receipt inputs receives_wic, receives_snap, receives_tanf that the ACS local-area transfer requires in its donor (PolicyEngine/microcosm#978 option 1), and the engine's declared dataset source input is_spm_independent_minor_role (the spm_independence_role base-builder stage; PolicyEngine/microcosm#893). status='reviewed_exclusion' for ecps_parity_known_gaps.json entries (reason+issue from that register); EXCEPT every primary-source restoration pinned by RESTORED_REFERENCE_ECPS_REQUIRED_INPUTS (including the Section 199A QBI family), and the SSI countable-resource asset inputs (bank_account_assets, stock_assets, bond_assets), which are status='required' with NO exclusion per PolicyEngine/microcosm#368 so the gate fails on today's artifacts and asset restoration (Deliverable 2) turns it green. All other populated layers are 'required'. Regenerate with tools/build_us_release_input_coverage_manifest.py.", + "derivation": "Required surface = input columns in the pinned, sha-verified ecps_parity_reference.json populated layers, plus the documented post-reference fsla_overtime_premium, qualified_passenger_vehicle_loan_interest, five desired retirement-contribution inputs, meets_ssi_disability_criteria required by shipped validation probes, the #282 Schedule-D capital-gain-distributions route leg schedule_d_capital_gain_distributions (PolicyEngine/microcosm#462), the three ASEC reported-receipt inputs receives_wic, receives_snap, receives_tanf that the ACS local-area transfer requires in its donor (PolicyEngine/microcosm#978 option 1), the work-experience inputs detailed_industry_recode, major_industry_recode, worked_last_year (PolicyEngine/microcosm#719), and the engine's declared dataset source input is_spm_independent_minor_role (the spm_independence_role base-builder stage; PolicyEngine/microcosm#893). status='reviewed_exclusion' for ecps_parity_known_gaps.json entries (reason+issue from that register); EXCEPT every primary-source restoration pinned by RESTORED_REFERENCE_ECPS_REQUIRED_INPUTS (including the Section 199A QBI family), and the SSI countable-resource asset inputs (bank_account_assets, stock_assets, bond_assets), which are status='required' with NO exclusion per PolicyEngine/microcosm#368 so the gate fails on today's artifacts and asset restoration (Deliverable 2) turns it green. All other populated layers are 'required'. Regenerate with tools/build_us_release_input_coverage_manifest.py.", "description": "Declared full-coverage contract for a US release: every input column the reference eCPS exports must be persisted as a key with non-default signal, or carry a reviewed exclusion. Enforced as a hard release gate (microcosm.build.us_runtime.release_input_coverage) that generalizes assert_required_us_release_source_columns from 5 columns to the full eCPS input surface.", "issue": "PolicyEngine/microcosm#368", "reference": { diff --git a/packages/microcosm-build/src/microcosm/build/us/source_stages.json b/packages/microcosm-build/src/microcosm/build/us/source_stages.json index eee6592225..15f72026d4 100644 --- a/packages/microcosm-build/src/microcosm/build/us/source_stages.json +++ b/packages/microcosm-build/src/microcosm/build/us/source_stages.json @@ -1702,7 +1702,10 @@ "inputs": [ "PRDTRACE", "PRDTHSP", - "POCCU2" + "POCCU2", + "WEIND", + "WEMIND", + "WKSWORK" ] }, { @@ -1726,6 +1729,9 @@ "cps_race", "is_hispanic", "detailed_occupation_recode", + "detailed_industry_recode", + "major_industry_recode", + "worked_last_year", "has_never_worked", "is_military", "is_computer_scientist", @@ -1739,10 +1745,12 @@ "nonnegative_outputs": [ "cps_race", "detailed_occupation_recode", + "detailed_industry_recode", + "major_industry_recode", "hourly_wage", "fsla_overtime_premium" ], - "notes": "Port of the final retired ORG/FLSA method at 9a823603e6b5fb916d65ec45d74c9c7eb0043db1: donor build, weighted QRF for hourly wage and hourly-pay status, archived deterministic BLS 2024 state-rate union assignment, and direct ASEC carries of CPS race, Hispanic status, and FLSA occupation groups are all exact — except the recipient-side QRF income feature, which is deliberately concept-aligned to the donor: the donor's employment_income is the annualized reference week (pternwa x 52), so recipients feed the full-year-equivalent of actual annual income (income x 52/weeks_worked, income untouched when weeks are zero or missing); without this, part-year workers matched low-wage full-year donors and the imputed wage distribution carried a severe sub-minimum tail (microcosm#529 receipts). The twelve official 2024 CPS basic-month files are transformed into the upstream 119,237-row cache; its canonical CSV content hash is pinned so a source reissue fails closed. Usual hours remain owned by hours_worked. fsla_overtime_premium keeps the upstream variable spelling and the archived annual-wage-share formula but deliberately deviates from the retired reference-week-only carrier gate: the retired construction annualized the March survey week (A_HRS1, outside the retrospective income year) and so collapsed incidence to one week's draw — 19.9M weighted carriers on certified Build N against Treasury's published '>29 million' TY2025 claimant floor (sb0517/sb0553, rounded floors, not mintable as exact-valued ledger facts). The two-signal estimator makes usual weekly hours worked last year (HRSWK) the persistent-overtime leg — usual>threshold carries with the usual-hours share as annualizer — while usual<=threshold retains the retired reference-week snapshot leg unchanged: representative for steady schedules, under- or over-recovering an occasional overtimer's annual premium depending on the sampled week (not an unbiased mass estimator), retained because it is the only measured signal for that population. On ASEC-channel rows HRSWK shares the retrospective year with employment_income and weeks_worked; on PUF-support-channel rows both hours signals are donor-ASEC schedule proxies attached to PUF income, the support design's property inherited equally by the reference-week leg. A frame whose ORG surface is already populated is fully re-imputed and compared on every entry: an identical surface passes through unchanged (the seeded QRF, deterministic union lottery, and pure derives make same-code re-entry byte-stable), and any stale construction of any output converges instead of silently surviving. The income-feature scaling also changes the deterministic union lottery's hash inputs, so which individuals carry union coverage reshuffles within unchanged state-level quotas — the assignment method itself remains the exact archived port. Exemption screens unchanged; both hours measures count all jobs while FLSA section 7 applies per employer (inherited upward concept edge). Adjudication and measurements: microcosm#451 item 4 overtime-incidence lane." + "notes": "Port of the final retired ORG/FLSA method at 9a823603e6b5fb916d65ec45d74c9c7eb0043db1: donor build, weighted QRF for hourly wage and hourly-pay status, archived deterministic BLS 2024 state-rate union assignment, and direct ASEC carries of CPS race, Hispanic status, and FLSA occupation groups are all exact — except the recipient-side QRF income feature, which is deliberately concept-aligned to the donor: the donor's employment_income is the annualized reference week (pternwa x 52), so recipients feed the full-year-equivalent of actual annual income (income x 52/weeks_worked, income untouched when weeks are zero or missing); without this, part-year workers matched low-wage full-year donors and the imputed wage distribution carried a severe sub-minimum tail (microcosm#529 receipts). The twelve official 2024 CPS basic-month files are transformed into the upstream 119,237-row cache; its canonical CSV content hash is pinned so a source reissue fails closed. Usual hours remain owned by hours_worked. fsla_overtime_premium keeps the upstream variable spelling and the archived annual-wage-share formula but deliberately deviates from the retired reference-week-only carrier gate: the retired construction annualized the March survey week (A_HRS1, outside the retrospective income year) and so collapsed incidence to one week's draw — 19.9M weighted carriers on certified Build N against Treasury's published '>29 million' TY2025 claimant floor (sb0517/sb0553, rounded floors, not mintable as exact-valued ledger facts). The two-signal estimator makes usual weekly hours worked last year (HRSWK) the persistent-overtime leg — usual>threshold carries with the usual-hours share as annualizer — while usual<=threshold retains the retired reference-week snapshot leg unchanged: representative for steady schedules, under- or over-recovering an occasional overtimer's annual premium depending on the sampled week (not an unbiased mass estimator), retained because it is the only measured signal for that population. On ASEC-channel rows HRSWK shares the retrospective year with employment_income and weeks_worked; on PUF-support-channel rows both hours signals are donor-ASEC schedule proxies attached to PUF income, the support design's property inherited equally by the reference-week leg. A frame whose ORG surface is already populated is fully re-imputed and compared on every entry: an identical surface passes through unchanged (the seeded QRF, deterministic union lottery, and pure derives make same-code re-entry byte-stable), and any stale construction of any output converges instead of silently surviving. The income-feature scaling also changes the deterministic union lottery's hash inputs, so which individuals carry union coverage reshuffles within unchanged state-level quotas — the assignment method itself remains the exact archived port. Exemption screens unchanged; both hours measures count all jobs while FLSA section 7 applies per employer (inherited upward concept edge). Adjudication and measurements: microcosm#451 item 4 overtime-incidence lane. Work-experience carries (microcosm#719): WEIND and WEMIND (industry of the longest job last year, restored before pooling from each vintage's pinned Census person member by the asec_census_person_columns restoration) become detailed_industry_recode and major_industry_recode, and worked_last_year is WKSWORK > 0; ACS-spine people get all three from native INDP/WKWN through the release predictor join. Checked on every spine: WEMIND is the major group of WEIND and positive weeks carry a worker industry code (1-22); the converse holds on ASEC rows and is checked before pooling." }, { "stage": "meps_esi_premiums", diff --git a/packages/microcosm-build/src/microcosm/build/us/spec/calibration.yaml b/packages/microcosm-build/src/microcosm/build/us/spec/calibration.yaml index 128371e097..146fc32131 100644 --- a/packages/microcosm-build/src/microcosm/build/us/spec/calibration.yaml +++ b/packages/microcosm-build/src/microcosm/build/us/spec/calibration.yaml @@ -267,6 +267,7 @@ tail_contracts: - charitable_cash_donations - charitable_non_cash_donations - cps_race + - detailed_industry_recode - detailed_occupation_recode - domestic_production_ald - educator_expense @@ -297,6 +298,7 @@ tail_contracts: - is_union_member_or_covered - long_term_capital_gains_before_response - long_term_capital_gains_on_collectibles + - major_industry_recode - meets_ssi_disability_criteria - miscellaneous_income - net_worth @@ -361,5 +363,6 @@ tail_contracts: - unrecaptured_section_1250_gain - unreimbursed_business_employee_expenses - w2_wages_from_qualified_business + - worked_last_year - would_file_taxes_voluntarily reviewed_exclusions_default: {} diff --git a/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml b/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml index 3a0bc01f62..6198319651 100644 --- a/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml +++ b/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml @@ -99,7 +99,7 @@ sources: stage_asset: id: source_stages path: microcosm.build.us/source_stages.json - sha256: 559062a1bd10efdd89b8c83da9294fea43062b8b6760524f4350dd84d63ef383 + sha256: dc3b72982675b95d30cfabdeb5f7f4c492e6cd3788009ef3094ab4d48bfd9093 stage_manifest: version: 1 country: us @@ -1720,6 +1720,9 @@ stages: - PRDTRACE - PRDTHSP - POCCU2 + - WEIND + - WEMIND + - WKSWORK - kind: derive method: derive_flsa_overtime_premium inputs: @@ -1737,6 +1740,9 @@ stages: - cps_race - is_hispanic - detailed_occupation_recode + - detailed_industry_recode + - major_industry_recode + - worked_last_year - has_never_worked - is_military - is_computer_scientist @@ -1749,6 +1755,8 @@ stages: nonnegative_outputs: - cps_race - detailed_occupation_recode + - detailed_industry_recode + - major_industry_recode - hourly_wage - fsla_overtime_premium notes: 'Port of the final retired ORG/FLSA method at 9a823603e6b5fb916d65ec45d74c9c7eb0043db1: donor @@ -1781,7 +1789,12 @@ stages: reshuffles within unchanged state-level quotas — the assignment method itself remains the exact archived port. Exemption screens unchanged; both hours measures count all jobs while FLSA section 7 applies per employer (inherited upward concept edge). Adjudication and measurements: microcosm#451 item 4 - overtime-incidence lane.' + overtime-incidence lane. Work-experience carries (microcosm#719): WEIND and WEMIND (industry of the + longest job last year, restored before pooling from each vintage''s pinned Census person member by + the asec_census_person_columns restoration) become detailed_industry_recode and major_industry_recode, + and worked_last_year is WKSWORK > 0; ACS-spine people get all three from native INDP/WKWN through + the release predictor join. Checked on every spine: WEMIND is the major group of WEIND and positive + weeks carry a worker industry code (1-22); the converse holds on ASEC rows and is checked before pooling.' - stage: meps_esi_premiums survey: MEPS-IC + CPS ASEC current employment-based coverage source: https://meps.ahrq.gov/mepsweb/survey_comp/Insurance.jsp diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/acs_inputs.py b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_inputs.py index d7dae0c5cb..481e517f14 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/acs_inputs.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_inputs.py @@ -18,6 +18,9 @@ import numpy as np import pandas as pd +from microcosm.build.us_runtime.acs_release_predictors import ( + crosswalk_acs_occupation_and_work_experience, +) from microcosm.build.us_runtime.spm_role_source import ( NATIVE_SPM_ROLE, SPM_ADULT_RULE, @@ -29,6 +32,7 @@ "ACS_SPM_ROLE_PARTITION", "ACS_SPM_ROLE_RULE", "ACS_UNRESOLVED_PARENT_ID_MAPPINGS", + "ACS_WORK_EXPERIENCE_NATIVE_MAPPINGS", "ACS_GROUP_QUARTERS_CHILD_RULE", "AcsGroupQuartersChildrenResult", "AcsNativeInputResult", @@ -75,6 +79,33 @@ ("parent_2_id", "PEPAR2"), ) +#: Native work-experience receipts shared with the pre-assembly boundary. +#: The same reviewed crosswalk serves fiscal-refresh and fresh ACS releases. +ACS_WORK_EXPERIENCE_NATIVE_MAPPINGS: Mapping[ + str, tuple[str, tuple[str, ...], str] +] = { + "detailed_occupation_recode": ( + "POCCU2", + ("OCCP", "AGEP", "ESR"), + "OCCP via ACS_OCCP_TO_POCCU2; source-universe sentinels", + ), + "detailed_industry_recode": ( + "WEIND", + ("INDP", "AGEP", "ESR"), + "INDP via ACS_INDP_TO_WEIND; source-universe sentinels", + ), + "major_industry_recode": ( + "WEMIND", + ("INDP", "AGEP", "ESR"), + "WEIND_TO_WEMIND of INDP via ACS_INDP_TO_WEIND", + ), + "worked_last_year": ( + "WKSWORK", + ("WKWN", "AGEP"), + "WKWN > 0; blank outside past-year work universe becomes False", + ), +} + _FORMULA_OWNED_AGGREGATES = frozenset( { "employment_income", @@ -186,6 +217,7 @@ def map_acs_native_inputs(frame: Frame) -> AcsNativeInputResult: ) _map_usual_hours(person, register=native) + _map_work_experience(person, register=native) _map_adjusted_person_amount( person, @@ -612,6 +644,32 @@ def _refuse_units(offending: pd.Index, problem: str) -> None: ) +def _map_work_experience( + person: pd.DataFrame, *, register: dict[str, Mapping[str, Any]] +) -> None: + # Sparse in-memory frames may omit this entire source surface. The PUMS + # loader requires it on every production archive; a partial surface never + # silently drops the contract. + if not {"OCCP", "INDP", "WKWN"}.intersection(person.columns): + return + mapped = crosswalk_acs_occupation_and_work_experience(person) + for output, (source, columns, transformation) in ( + ACS_WORK_EXPERIENCE_NATIVE_MAPPINGS.items() + ): + values = mapped[source].to_numpy() + if output == "worked_last_year": + values = values > 0 + _add_native( + person, + output, + values, + entity="person", + source_columns=columns, + transformation=transformation, + register=register, + ) + + def _map_usual_hours( person: pd.DataFrame, *, register: dict[str, Mapping[str, Any]] ) -> None: diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/acs_pums.py b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_pums.py index 6e237a0f58..3dc1ed8bee 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/acs_pums.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_pums.py @@ -84,6 +84,10 @@ "RETP", "INTP", "PWGTP", + "OCCP", + "INDP", + "WKWN", + "ESR", ) # Preserve source hours and their universe/allocation evidence when supplied. # Older/minimal source fixtures remain loadable; absence is not a zero. diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/acs_release_predictors.py b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_release_predictors.py index 98ba4e4d9f..e26f6d83a3 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/acs_release_predictors.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_release_predictors.py @@ -23,7 +23,10 @@ * ``sipp_tips.CENSUS_OCCUPATION_CODE_TO_TTOC`` consumes detailed Census occupation codes directly; * ``org_wages.FLSA_OVERTIME_OCCUPATION_CODES`` and its EAP set consume the - 53-category CPS detailed occupation recode; and + 53-category CPS detailed occupation recode; +* ``org_wages.derive_us_org_occupation_inputs`` carries the CPS longest-job + industry recodes (``WEIND`` detailed, ``WEMIND`` major) and derives + ``worked_last_year`` from weeks worked (``WKSWORK``); and * ``sipp_vehicles._household_tenure_status`` consumes only mortgaged owner, outright owner, and non-owner tenure codes. @@ -46,6 +49,10 @@ import numpy as np import pandas as pd +from microcosm.build.us_runtime.asec_census_person_columns import ( + WEIND_TO_WEMIND, + WORKER_INDUSTRY_CODES, +) from microcosm.build.us_runtime.support_provenance import ( support_channel_column, support_clone_index_column, @@ -58,12 +65,14 @@ "ACS_2024_HOUSEHOLD_ZIP_SHA256", "ACS_2024_PERSON_ZIP_SHA256", "ACS_DIFFICULTY_TO_CPS", + "ACS_INDP_TO_WEIND", "ACS_OCCP_TO_POCCU2", "ACS_RAC1P_TO_CONSUMED_PRDTRACE", "ACS_RELEASE_PREDICTOR_CROSSWALK_SHA256", "ACS_TEN_TO_SPM_TENMORTSTATUS", "AcsReleasePredictorJoinResult", "acs_release_predictor_crosswalk_payload", + "crosswalk_acs_occupation_and_work_experience", "join_acs_release_predictors", ] @@ -73,9 +82,9 @@ ACS_2024_HOUSEHOLD_ZIP_SHA256 = ( "8281008e53de98f0ef81e7a2ee5a8725991dda1ecfd2713ead73246425e515d0" ) -ACS_RELEASE_PREDICTOR_CROSSWALK_VERSION = 1 +ACS_RELEASE_PREDICTOR_CROSSWALK_VERSION = 2 ACS_RELEASE_PREDICTOR_CROSSWALK_SHA256 = ( - "1d4906242e9c73e31b3283659e5cad8242b8cbc42914ab6fa59547a10c8770e9" + "a9b8410493084f7002f1a617757dafba1ef9c451efdce200d9f2ef2c8541a194" ) _SHA256_PATTERN = re.compile(r"[0-9a-f]{64}") @@ -99,6 +108,8 @@ "RAC1P", "HISP", "OCCP", + "INDP", + "WKWN", "ESR", "SSIP", "ADJINC", @@ -686,6 +697,287 @@ 9920: 53, } +# 2022 Census industry codes (ACS 2024 INDP; CPS ASEC from survey year 2025) +# mapped to the 23 CPS longest-job detailed industry groups (WEIND). Every +# civilian code's group is read from the pinned 2025 ASEC person member +# (pppub25.csv), where nonzero INDUSTRY determines WEIND exactly; codes shared with +# the 2017-list 2023/2024 members keep the same group there. The ACS-only +# codes are reviewed: 9670-9870 are the Armed Forces branches (Army, Air +# Force, Navy, Marines, Coast Guard, branch not specified, Reserves or National +# Guard) that CPS collapses into 9890 (WEIND 22), and 9920 ("unemployed, no +# work experience in the last 5 years or never worked") is the CPS +# did-not-work group 23. WEMIND follows from WEIND by the pinned +# ``WEIND_TO_WEMIND`` nesting. +ACS_INDP_TO_WEIND: Mapping[int, int] = { + 170: 1, + 180: 1, + 190: 1, + 270: 1, + 280: 1, + 290: 1, + 370: 2, + 380: 2, + 390: 2, + 470: 2, + 480: 2, + 490: 2, + 570: 9, + 580: 9, + 590: 9, + 670: 9, + 680: 9, + 690: 9, + 770: 3, + 1070: 5, + 1080: 5, + 1090: 5, + 1170: 5, + 1180: 5, + 1190: 5, + 1270: 5, + 1280: 5, + 1290: 5, + 1370: 5, + 1390: 5, + 1470: 5, + 1480: 5, + 1490: 5, + 1570: 5, + 1590: 5, + 1670: 5, + 1691: 5, + 1770: 5, + 1790: 5, + 1870: 5, + 1880: 5, + 1890: 5, + 1990: 5, + 2070: 5, + 2090: 5, + 2170: 5, + 2180: 5, + 2190: 5, + 2270: 5, + 2280: 5, + 2290: 5, + 2370: 5, + 2380: 5, + 2390: 5, + 2470: 4, + 2480: 4, + 2490: 4, + 2570: 4, + 2590: 4, + 2670: 4, + 2680: 4, + 2690: 4, + 2770: 4, + 2780: 4, + 2790: 4, + 2870: 4, + 2880: 4, + 2890: 4, + 2970: 4, + 2980: 4, + 2990: 4, + 3070: 4, + 3080: 4, + 3095: 4, + 3170: 4, + 3180: 4, + 3291: 4, + 3365: 4, + 3370: 4, + 3380: 4, + 3390: 4, + 3470: 4, + 3490: 4, + 3570: 4, + 3580: 4, + 3590: 4, + 3670: 4, + 3680: 4, + 3690: 4, + 3770: 4, + 3780: 4, + 3790: 4, + 3875: 4, + 3895: 4, + 3960: 4, + 3970: 4, + 3980: 4, + 3990: 4, + 4070: 6, + 4080: 6, + 4090: 6, + 4170: 6, + 4180: 6, + 4195: 6, + 4265: 6, + 4270: 6, + 4280: 6, + 4290: 6, + 4370: 6, + 4380: 6, + 4390: 6, + 4470: 6, + 4480: 6, + 4490: 6, + 4560: 6, + 4570: 6, + 4580: 6, + 4585: 6, + 4590: 6, + 4670: 7, + 4681: 7, + 4691: 7, + 4771: 7, + 4796: 7, + 4871: 7, + 4881: 7, + 4891: 7, + 4971: 7, + 4973: 7, + 4981: 7, + 4991: 7, + 5071: 7, + 5081: 7, + 5090: 7, + 5171: 7, + 5181: 7, + 5191: 7, + 5276: 7, + 5281: 7, + 5296: 7, + 5371: 7, + 5382: 7, + 5392: 7, + 5471: 7, + 5481: 7, + 5491: 7, + 5571: 7, + 5581: 7, + 5680: 7, + 5791: 7, + 6070: 8, + 6080: 8, + 6090: 8, + 6170: 8, + 6180: 8, + 6190: 8, + 6270: 8, + 6280: 8, + 6290: 8, + 6370: 8, + 6380: 8, + 6390: 8, + 6471: 10, + 6481: 10, + 6490: 10, + 6570: 10, + 6590: 10, + 6671: 10, + 6680: 10, + 6690: 10, + 6695: 10, + 6770: 10, + 6781: 10, + 6871: 11, + 6881: 11, + 6890: 11, + 6970: 11, + 6991: 11, + 6992: 11, + 7071: 12, + 7072: 12, + 7080: 12, + 7181: 12, + 7190: 12, + 7270: 13, + 7280: 13, + 7290: 13, + 7370: 13, + 7380: 13, + 7390: 13, + 7460: 13, + 7470: 13, + 7480: 13, + 7490: 13, + 7570: 14, + 7580: 14, + 7590: 14, + 7670: 14, + 7680: 14, + 7690: 14, + 7770: 14, + 7780: 14, + 7790: 14, + 7860: 15, + 7870: 15, + 7880: 15, + 7890: 15, + 7970: 16, + 7980: 16, + 7990: 16, + 8070: 16, + 8080: 16, + 8090: 16, + 8170: 16, + 8180: 16, + 8191: 16, + 8192: 16, + 8270: 16, + 8290: 16, + 8370: 16, + 8380: 16, + 8390: 16, + 8470: 16, + 8561: 17, + 8562: 17, + 8563: 17, + 8564: 17, + 8570: 17, + 8580: 17, + 8590: 17, + 8660: 18, + 8670: 18, + 8680: 18, + 8690: 18, + 8770: 20, + 8780: 20, + 8790: 20, + 8870: 20, + 8891: 20, + 8970: 20, + 8980: 20, + 8990: 20, + 9070: 20, + 9080: 20, + 9090: 20, + 9160: 20, + 9170: 20, + 9180: 20, + 9190: 20, + 9290: 19, + 9370: 21, + 9380: 21, + 9390: 21, + 9470: 21, + 9480: 21, + 9490: 21, + 9570: 21, + 9590: 21, + 9670: 22, + 9680: 22, + 9690: 22, + 9770: 22, + 9780: 22, + 9790: 22, + 9870: 22, + 9890: 22, + 9920: 23, +} + ACS_TEN_TO_SPM_TENMORTSTATUS: Mapping[int, int] = { 1: 1, # owned with a mortgage or loan 2: 2, # owned free and clear @@ -706,7 +998,7 @@ "scf_auto_loans": ("PRDTRACE", "PRDTHSP"), "sipp_vehicles": ("SPM_TENMORTSTATUS",), "sipp_tips": ("PEIOOCC",), - "org_wages": ("PRDTRACE", "PRDTHSP", "POCCU2"), + "org_wages": ("PRDTRACE", "PRDTHSP", "POCCU2", "WEIND", "WEMIND", "WKSWORK"), } _OUTPUT_COLUMNS = tuple( dict.fromkeys( @@ -762,6 +1054,36 @@ def acs_release_predictor_crosswalk_payload() -> dict[str, Any]: ), }, }, + "industry": { + "INDP_to_WEIND": { + str(key): value for key, value in ACS_INDP_TO_WEIND.items() + }, + "blank_INDP_to_WEIND": { + "age_below_16": 0, + "age_16_plus": 23, + "age_15_source_target_universe_gap": ( + "ACS INDP is not asked; retain the out-of-universe sentinel " + "instead of fabricating CPS did-not-work code 23" + ), + }, + "WEIND_to_WEMIND": { + str(key): value for key, value in WEIND_TO_WEMIND.items() + }, + "observed_INDP_reference": ( + "most recent job in the past 5 years (CPS WEIND: longest job last year)" + ), + }, + "work_experience": { + "WKWN_to_WKSWORK": { + "observed": "identity (weeks worked in the past 12 months, 1-52)", + "blank": 0, + "reference_period_gap": ( + "ACS past 12 months before interview; CPS WKSWORK prior " + "calendar year" + ), + }, + "invariant": "WKSWORK > 0 implies WEIND in 1..22", + }, "tenure": { "TEN_to_SPM_TENMORTSTATUS": { str(key): value for key, value in ACS_TEN_TO_SPM_TENMORTSTATUS.items() @@ -1391,6 +1713,30 @@ def _crosswalk_people(joined: pd.DataFrame) -> pd.DataFrame: raise ValueError(f"ACS HISP contains unsupported code(s): {unknown_hisp}.") result["PRDTHSP"] = hisp.map(_ACS_HISP_TO_CONSUMED_PRDTHSP).to_numpy(dtype=np.int16) + work_experience = crosswalk_acs_occupation_and_work_experience(joined) + return pd.concat([result, work_experience], axis=1) + + +def crosswalk_acs_occupation_and_work_experience(joined: pd.DataFrame) -> pd.DataFrame: + """Map native ACS job fields to the reviewed CPS occupation/work recodes. + + Both ACS input mapping and the authenticated release predictor join use + this crosswalk. Required source columns are AGEP, ESR, OCCP, INDP and + WKWN; optional SERIALNO/SPORDER identify refused source rows. The returned + PEIOOCC, POCCU2, WEIND, WEMIND and WKSWORK retain the input's row index. + """ + + required = ("AGEP", "ESR", "OCCP", "INDP", "WKWN") + missing = [column for column in required if column not in joined] + if missing: + raise ValueError( + f"ACS work-experience crosswalk needs source column(s): {missing}." + ) + age = _required_integral(joined["AGEP"], label="ACS AGEP", minimum=0) + identity_columns = [ + column for column in ("SERIALNO", "SPORDER") if column in joined + ] + result = pd.DataFrame(index=joined.index) occupation_raw = joined["OCCP"] occupation = pd.to_numeric(occupation_raw, errors="coerce") employment_raw = joined["ESR"] @@ -1401,7 +1747,7 @@ def _crosswalk_people(joined: pd.DataFrame) -> pd.DataFrame: | (age.ge(16) & ~employment.isin(range(1, 7))) ) if invalid_employment.any(): - bad = joined.loc[invalid_employment, ["SERIALNO", "SPORDER", "AGEP", "ESR"]] + bad = joined.loc[invalid_employment, [*identity_columns, "AGEP", "ESR"]] raise ValueError( "ACS ESR code/universe mismatch (blank below age 16; codes 1--6 " "from age 16); examples=" @@ -1414,7 +1760,7 @@ def _crosswalk_people(joined: pd.DataFrame) -> pd.DataFrame: bad_mask = invalid_occupation | invalid_observed_universe bad = joined.loc[ bad_mask, - ["SERIALNO", "SPORDER", "AGEP", "OCCP", "ESR"], + [*identity_columns, "AGEP", "OCCP", "ESR"], ] raise ValueError( "ACS OCCP code/universe mismatch (blank below age 16); examples=" @@ -1423,7 +1769,7 @@ def _crosswalk_people(joined: pd.DataFrame) -> pd.DataFrame: if invalid_blank.any(): bad = joined.loc[ invalid_blank, - ["SERIALNO", "SPORDER", "AGEP", "OCCP", "ESR"], + [*identity_columns, "AGEP", "OCCP", "ESR"], ] raise ValueError( "ACS OCCP is blank inside its observed employment universe; examples=" @@ -1452,6 +1798,66 @@ def _crosswalk_people(joined: pd.DataFrame) -> pd.DataFrame: # no-occupation / never-worked bin 53. poccu2.loc[occupation.isna()] = np.where(age.loc[occupation.isna()].ge(16), 53, 0) result["POCCU2"] = poccu2.to_numpy(dtype=np.int16) + + # INDP shares OCCP's universe (age 16+, worked in the past 5 years, or + # unemployed with code 9920); a blank from age 16 is admitted only for + # ESR=6 and maps to the CPS did-not-work group 23. + industry_raw = joined["INDP"] + industry = pd.to_numeric(industry_raw, errors="coerce") + invalid_industry = (industry_raw.notna() & industry.isna()) | ( + industry.notna() & age.lt(16) + ) + if invalid_industry.any(): + bad = joined.loc[invalid_industry, [*identity_columns, "AGEP", "INDP"]] + raise ValueError( + "ACS INDP code/universe mismatch (blank below age 16); examples=" + f"{bad.head().to_dict('records')}." + ) + blank_industry = industry.isna() & age.ge(16) & ~employment.eq(6) + if blank_industry.any(): + bad = joined.loc[blank_industry, [*identity_columns, "AGEP", "INDP", "ESR"]] + raise ValueError( + "ACS INDP is blank inside its observed employment universe; examples=" + f"{bad.head().to_dict('records')}." + ) + observed_industry = industry.loc[industry.notna()].to_numpy(dtype=np.float64) + if not np.equal(observed_industry, np.floor(observed_industry)).all(): + raise ValueError("ACS INDP contains non-integer code(s).") + unknown_industry = sorted( + set(industry.dropna().astype(np.int64)) - set(ACS_INDP_TO_WEIND) + ) + if unknown_industry: + raise ValueError(f"ACS INDP contains unsupported code(s): {unknown_industry}.") + weind = industry.map(ACS_INDP_TO_WEIND) + weind.loc[industry.isna()] = np.where(age.loc[industry.isna()].ge(16), 23, 0) + weind = weind.astype(np.int64) + + weeks_raw = joined["WKWN"] + weeks = pd.to_numeric(weeks_raw, errors="coerce") + invalid_weeks = ( + (weeks_raw.notna() & weeks.isna()) + | (weeks.notna() & age.lt(16)) + | (weeks.notna() & ~weeks.isin(range(1, 53))) + ) + if invalid_weeks.any(): + bad = joined.loc[invalid_weeks, [*identity_columns, "AGEP", "WKWN"]] + raise ValueError( + "ACS WKWN code/universe mismatch (blank below age 16; 1--52 weeks " + f"from age 16); examples={bad.head().to_dict('records')}." + ) + wkswork = weeks.fillna(0).astype(np.int64) + worked_without_industry = wkswork.gt(0) & ~weind.isin(WORKER_INDUSTRY_CODES) + if worked_without_industry.any(): + bad = joined.loc[ + worked_without_industry, [*identity_columns, "AGEP", "INDP", "WKWN"] + ] + raise ValueError( + "ACS WKWN reports weeks worked without a worker INDP code; examples=" + f"{bad.head().to_dict('records')}." + ) + result["WEIND"] = weind.to_numpy(dtype=np.int16) + result["WEMIND"] = weind.map(WEIND_TO_WEMIND).to_numpy(dtype=np.int16) + result["WKSWORK"] = wkswork.to_numpy(dtype=np.int16) return result @@ -1497,8 +1903,8 @@ def _validate_canonical_ssi_reporter_values( "canonical raw join." ) by_source = pd.Series(reported.to_numpy(), index=native[source_id_column]) - pool_aligned = canonical["person_source_id"].map(by_source).to_numpy( - dtype=np.float64 + pool_aligned = ( + canonical["person_source_id"].map(by_source).to_numpy(dtype=np.float64) ) raw_ssip = pd.to_numeric(joined["SSIP"], errors="coerce") diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_census_person_columns.py b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_census_person_columns.py index d0187f65d3..1787241d60 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_census_person_columns.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_census_person_columns.py @@ -57,11 +57,25 @@ ``A_FTPT`` same; ``eligibility_inputs`` ``derive_us_eligibility_inputs_from_manifest`` (``is_full_time_college_student``) +``WEIND`` ``org_wages.derive_us_org_occupation_inputs`` -> + ``detailed_industry_recode`` (#719) +``WEMIND`` same -> ``major_industry_recode`` (#719) ====================== ====================================================== The other thirteen columns the 2026-08-23 offline fix appended are not restored; :data:`ASEC_CENSUS_PERSON_COLUMNS_NOT_RESTORED` records why. +``WEIND`` and ``WEMIND`` (industry of the longest job last year, detailed and +major groups) were never in that offline fix: no H5 vintage carries them, not +even 2024, so every vintage gets them from its member. +:data:`ASEC_CENSUS_PERSON_COLUMNS_BEYOND_OFFLINE_FIX` records that. Once they +are restored, two identities that hold in all three pinned members are +checked on the vintage (:func:`_check_work_experience_universe`): ``WEMIND`` +is the function :data:`WEIND_TO_WEMIND` of ``WEIND``, and ``WEIND`` carries a +worker code (1--22) exactly where ``WKSWORK > 0``. The second identity is +ASEC-only: an ACS person may carry an industry from a job held one to five +years ago, so the release-time consumer can check only its forward half. + Microcosm #454 adds six columns that every pinned H5 lacks (2024 included), read by the ``meps_esi_premiums`` stage (:mod:`.esi_premiums`): @@ -102,9 +116,12 @@ __all__ = [ "ASEC_CENSUS_PERSON_COLUMNS", + "ASEC_CENSUS_PERSON_COLUMNS_BEYOND_OFFLINE_FIX", "ASEC_CENSUS_PERSON_COLUMNS_NOT_RESTORED", "ASEC_CENSUS_PERSON_COLUMN_NAMES", "ASEC_CENSUS_PERSON_IDENTITY_COLUMNS", + "WEIND_TO_WEMIND", + "WORKER_INDUSTRY_CODES", "AsecCensusPersonColumn", "AsecCensusPersonColumnsError", "restore_asec_census_person_columns", @@ -128,6 +145,49 @@ class AsecCensusPersonColumn: _NOW_YES_NO = frozenset({1, 2}) _ENROLLMENT = frozenset({0, 1, 2}) +#: ``WEIND``: 0 = not in universe (under 15), 1--21 = civilian industry of the +#: longest job last year, 22 = Armed Forces, 23 = did not work last year (age +#: 15+). ``WEMIND`` aggregates it into 15 major groups (0 = not in universe). +_DETAILED_INDUSTRY_CODES = frozenset(range(24)) +_MAJOR_INDUSTRY_CODES = frozenset(range(16)) +_WORK_EXPERIENCE_READER = ( + "microcosm.build.us_runtime.org_wages.derive_us_org_occupation_inputs" +) + +#: ``WEMIND`` as a function of ``WEIND``, identical in all three pinned person +#: members (pppub23/24/25, 2026-09-24: every ``WEIND`` value co-occurs with +#: exactly one ``WEMIND`` value) and nested as the codebook's major and +#: detailed industry groups are. +WEIND_TO_WEMIND: MappingProxyType[int, int] = MappingProxyType( + { + 0: 0, + 1: 1, + 2: 2, + 3: 3, + 4: 4, + 5: 4, + 6: 5, + 7: 5, + 8: 6, + 9: 6, + 10: 7, + 11: 8, + 12: 8, + 13: 9, + 14: 9, + 15: 10, + 16: 10, + 17: 11, + 18: 11, + 19: 12, + 20: 12, + 21: 13, + 22: 14, + 23: 15, + } +) +#: Civilian WEIND industry groups 1--21 and Armed Forces group 22. +WORKER_INDUSTRY_CODES: tuple[int, ...] = tuple(range(1, 23)) _ESI_READER = "microcosm.build.us_runtime.esi_premiums._esi_person_codes" _HEALTH_COVERAGE_READER = ( @@ -229,6 +289,16 @@ class AsecCensusPersonColumn: "(is_full_time_college_student <- A_HSCOL == 2 & A_FTPT == 1)", ), ), + AsecCensusPersonColumn( + "WEIND", + _DETAILED_INDUSTRY_CODES, + (f"{_WORK_EXPERIENCE_READER} -> detailed_industry_recode",), + ), + AsecCensusPersonColumn( + "WEMIND", + _MAJOR_INDUSTRY_CODES, + (f"{_WORK_EXPERIENCE_READER} -> major_industry_recode",), + ), # Microcosm #454: the meps_esi_premiums stage's universe, tier, payment # status, employment and employer-size fields. AsecCensusPersonColumn( @@ -304,6 +374,22 @@ class AsecCensusPersonColumn: } ) +#: Reviewed columns that the 2026-08-23 offline fix never appended, and why +#: they are restored anyway. +ASEC_CENSUS_PERSON_COLUMNS_BEYOND_OFFLINE_FIX: MappingProxyType[str, str] = ( + MappingProxyType( + { + column: ( + "Industry of the longest job last year (#719): the county-file " + "person schema carries industry with occupation (POCCU2) on " + "every record. No H5 vintage carries it, not even 2024, so " + "every vintage restores it from its pinned member." + ) + for column in ("WEIND", "WEMIND") + } + ) +) + #: Census identity carried by both the H5 and the person member. After the #: PERIDNUM join each must agree row for row, so a join that pairs the wrong #: people cannot pass on a coincidence of keys. @@ -425,6 +511,40 @@ def _value_summary(spec: AsecCensusPersonColumn, values: pd.Series) -> dict[str, return {str(code): int(counts.get(code, 0)) for code in sorted(spec.domain)} +def _check_work_experience_universe( + table: pd.DataFrame, h5_label: str +) -> dict[str, object]: + """Check the ASEC work-experience identities on one restored vintage.""" + + detailed = table["WEIND"].to_numpy(dtype=np.int64) + major = table["WEMIND"].to_numpy(dtype=np.int64) + expected_major = np.array([WEIND_TO_WEMIND[code] for code in detailed]) + mismatch = major != expected_major + if mismatch.any(): + _refuse( + f"{h5_label} WEMIND disagrees with the major group of WEIND on " + f"{int(mismatch.sum())} rows; the join paired the wrong people or " + "the member changed its industry recodes." + ) + record: dict[str, object] = {"weind_to_wemind_rows_verified": len(table)} + if "WKSWORK" not in table.columns: + record["worker_code_iff_weeks_worked"] = "not checked: H5 lacks WKSWORK" + return record + weeks = pd.to_numeric(table["WKSWORK"], errors="coerce") + if weeks.isna().any(): + _refuse(f"{h5_label} WKSWORK has missing or non-numeric values.") + worker = np.isin(detailed, WORKER_INDUSTRY_CODES) + disagree = worker != weeks.gt(0).to_numpy() + if disagree.any(): + _refuse( + f"{h5_label} WEIND carries a worker code (1--22) where WKSWORK is " + f"not positive, or the reverse, on {int(disagree.sum())} rows." + ) + record["worker_code_iff_weeks_worked"] = "verified" + record["worker_rows"] = int(worker.sum()) + return record + + def restore_asec_census_person_columns( person: pd.DataFrame, *, @@ -448,7 +568,10 @@ def restore_asec_census_person_columns( person has no member row, or a member person is absent from the H5; when ``PH_SEQ``/``P_SEQ``/``A_LINENO``/``A_AGE`` disagree after the join; and when a reviewed column the H5 already carries is not integer or differs - from the member on any row. A column the H5 carries is never overwritten. + from the member on any row; and when ``WEMIND`` is not the major group of + ``WEIND``, or (where the H5 carries ``WKSWORK``) ``WEIND`` holds a worker + code on a row without positive weeks worked or the reverse. A column the + H5 carries is never overwritten. New columns are appended as ``int64`` after the H5's own, in reviewed order; every H5 column keeps its position, dtype and values (checked). @@ -539,6 +662,7 @@ def restore_asec_census_person_columns( ) if not unchanged: _refuse(f"Restoring Census columns changed an existing {h5_label} column.") + work_experience = _check_work_experience_universe(result, h5_label) return result, { "operation": "exact_source_join", @@ -560,4 +684,5 @@ def restore_asec_census_person_columns( "columns_added": list(added), "columns_verified_equal": verified, "member_values": values, + "work_experience_universe": work_experience, } diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/operator_boundary.py b/packages/microcosm-build/src/microcosm/build/us_runtime/operator_boundary.py index 646b04d968..470e8e016c 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/operator_boundary.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/operator_boundary.py @@ -17,6 +17,7 @@ from microcosm.build.us_runtime.acs_inputs import ( _ACS_UNRESOLVED_PARENT_POINTER_TRANSFORMATION, ACS_UNRESOLVED_PARENT_ID_MAPPINGS, + ACS_WORK_EXPERIENCE_NATIVE_MAPPINGS, ) from microcosm.build.us_runtime.acs_transfer import ACS_DERIVED_TRANSFER_INPUTS from microcosm.build.us_runtime.adult_care import US_ADULT_CARE_OUTPUT_COLUMNS @@ -136,6 +137,12 @@ str, tuple[str, tuple[str, ...], str], ] = { + **{ + output: ("person", source_columns, transformation) + for output, (_, source_columns, transformation) in ( + ACS_WORK_EXPERIENCE_NATIVE_MAPPINGS.items() + ) + }, "age": ("person", ("AGEP",), "identity"), "is_female": ("person", ("SEX",), "SEX == 2"), "is_household_head": ("person", ("RELSHIPP",), "RELSHIPP == 20"), diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/org_wages.py b/packages/microcosm-build/src/microcosm/build/us_runtime/org_wages.py index d309067756..66744d5404 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/org_wages.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/org_wages.py @@ -53,6 +53,10 @@ from microcosm.build.gates import GateResult from microcosm.build.source_manifest import SourceStageSpec, load_source_manifest +from microcosm.build.us_runtime.asec_census_person_columns import ( + WEIND_TO_WEMIND, + WORKER_INDUSTRY_CODES, +) from microcosm.frame import Frame from microcosm.frame.units import US_SCHEMA @@ -63,6 +67,7 @@ "ORG_2024_DONOR_CONTENT_SHA256", "ORG_2024_DONOR_FILENAME", "ORG_PREDICTORS", + "US_ORG_OCCUPATION_AND_WORK_EXPERIENCE_OUTPUT_COLUMNS", "US_ORG_WAGES_NONCONSTANT_PERSON_COLUMNS", "US_ORG_WAGES_OUTPUT_COLUMNS", "US_ORG_WAGES_REQUIRED_SOURCE_COLUMNS", @@ -72,6 +77,7 @@ "fetch_org_2024_donor", "impute_us_org_wages", "load_org_2024_donor", + "require_us_occupation_and_work_experience_inputs", "us_org_wages_signal_gate", "us_org_wages_stage_spec", "us_org_wages_summary", @@ -115,10 +121,16 @@ _DONOR_WEIGHT_COLUMN = "sample_weight" _DEFAULT_N_ESTIMATORS = 100 +US_ORG_OCCUPATION_AND_WORK_EXPERIENCE_OUTPUT_COLUMNS: tuple[str, ...] = ( + "detailed_occupation_recode", + "detailed_industry_recode", + "major_industry_recode", + "worked_last_year", +) US_ORG_WAGES_OUTPUT_COLUMNS: tuple[str, ...] = ( "cps_race", "is_hispanic", - "detailed_occupation_recode", + *US_ORG_OCCUPATION_AND_WORK_EXPERIENCE_OUTPUT_COLUMNS, "has_never_worked", "is_military", "is_computer_scientist", @@ -137,6 +149,9 @@ "PRDTRACE", "PRDTHSP", "POCCU2", + "WEIND", + "WEMIND", + "WKSWORK", "employment_income_before_lsr", "weekly_hours_worked_before_lsr", "hours_worked_last_week", @@ -307,6 +322,12 @@ "cps_race": (0.95, 1.0), "is_hispanic": (0.05, 0.30), "detailed_occupation_recode": (0.65, 0.95), + # Measured on the pinned inputs: 0.822-0.824 of weighted ASEC persons (all + # of age 15+) and 0.80 of ACS persons (age 16+) carry a nonzero industry + # recode; 0.520-0.523 (ASEC) and 0.52 (ACS) worked last year. + "detailed_industry_recode": (0.70, 0.92), + "major_industry_recode": (0.70, 0.92), + "worked_last_year": (0.40, 0.62), "has_never_worked": (0.15, 0.35), "is_military": (0.0005, 0.01), "is_computer_scientist": (0.005, 0.05), @@ -532,16 +553,94 @@ def load_org_2024_donor( return result +def require_us_occupation_and_work_experience_inputs( + person: pd.DataFrame, *, label: str +) -> None: + """Require complete canonical job inputs on a donor or an existing staging H5. + + The local ACS lane preserves its ASEC donor rows and can package an + existing staging file. Both boundaries must refuse a historical file + which lacks the source-derived occupation, industry or working inputs. + """ + + missing = [ + column + for column in US_ORG_OCCUPATION_AND_WORK_EXPERIENCE_OUTPUT_COLUMNS + if column not in person + ] + if missing: + raise ValueError( + f"{label} lacks canonical work-experience input(s) {missing}." + ) + invalid_counts = {} + for column in US_ORG_OCCUPATION_AND_WORK_EXPERIENCE_OUTPUT_COLUMNS: + values = pd.to_numeric(person[column], errors="coerce").to_numpy( + dtype=np.float64, na_value=np.nan + ) + invalid = ~np.isfinite(values) + if invalid.any(): + invalid_counts[column] = int(invalid.sum()) + if invalid_counts: + raise ValueError( + f"{label} has missing or nonfinite canonical work-experience " + f"inputs: {invalid_counts}." + ) + + +def _complete_codes(person: pd.DataFrame, column: str, upper: int) -> np.ndarray: + values = pd.to_numeric(person[column], errors="coerce").to_numpy(dtype=np.float64) + invalid = ~np.isfinite(values) | (values != np.floor(values)) + invalid |= (values < 0) | (values > upper) + if invalid.any(): + raise ValueError( + f"ORG work-experience carry needs {column} as complete integer codes " + f"in [0, {upper}]; {int(invalid.sum())} row(s) are missing or outside." + ) + return values.astype(np.int64) + + def derive_us_org_occupation_inputs(person: pd.DataFrame) -> pd.DataFrame: - """Carry CPS race/ethnicity and derive the exact POCCU2 FLSA flags.""" + """Carry CPS race/ethnicity, occupation, industry and worked-last-year. + + Occupation derives the exact POCCU2 FLSA flags. The work-experience + carries (#719) are the longest-job industry recodes ``WEIND`` and + ``WEMIND`` and ``worked_last_year = WKSWORK > 0``; on ACS rows the release + predictor join supplies all three from native ``INDP``/``WKWN`` before this + runs. Both spines satisfy the checked identities: ``WEMIND`` is the major + group of ``WEIND``, and positive weeks worked carry a worker industry code. + The converse holds on ASEC rows only (an ACS industry can come from a job + held one to five years ago) and is checked before pooling, where the rows + are known to be ASEC (``asec_census_person_columns``). + """ missing = [ - column for column in ("PRDTRACE", "PRDTHSP", "POCCU2") if column not in person + column + for column in ("PRDTRACE", "PRDTHSP", "POCCU2", "WEIND", "WEMIND", "WKSWORK") + if column not in person ] if missing: raise ValueError( f"ORG occupation derivation needs person column(s): {missing}." ) + detailed_industry = _complete_codes(person, "WEIND", 23) + major_industry = _complete_codes(person, "WEMIND", 15) + weeks_worked = _complete_codes(person, "WKSWORK", 52) + expected_major = np.array([WEIND_TO_WEMIND[code] for code in detailed_industry]) + mismatched_major = int((major_industry != expected_major).sum()) + if mismatched_major: + raise ValueError( + f"ORG work-experience carry: WEMIND is not the major group of WEIND " + f"on {mismatched_major} row(s)." + ) + worked = weeks_worked > 0 + worked_without_industry = int( + (worked & ~np.isin(detailed_industry, WORKER_INDUSTRY_CODES)).sum() + ) + if worked_without_industry: + raise ValueError( + "ORG work-experience carry: positive WKSWORK without a worker WEIND " + f"code (1--22) on {worked_without_industry} row(s)." + ) occupation = pd.to_numeric(person["POCCU2"], errors="coerce").fillna(0).astype(int) result = pd.DataFrame(index=person.index) result["cps_race"] = ( @@ -551,6 +650,9 @@ def derive_us_org_occupation_inputs(person: pd.DataFrame) -> pd.DataFrame: pd.to_numeric(person["PRDTHSP"], errors="coerce").fillna(0).ne(0) ) result["detailed_occupation_recode"] = occupation.astype(np.int16) + result["detailed_industry_recode"] = detailed_industry.astype(np.int16) + result["major_industry_recode"] = major_industry.astype(np.int16) + result["worked_last_year"] = worked for variable, code in FLSA_OVERTIME_OCCUPATION_CODES.items(): result[variable] = occupation.eq(code) result["is_executive_administrative_professional"] = occupation.isin( @@ -993,6 +1095,27 @@ def share(column: str) -> float: "positive_without_weeks": int(((premium > 0) & (weeks <= 0)).sum()), "positive_always_exempt": int(((premium > 0) & (never | military)).sum()), } + detailed_industry = pd.to_numeric( + person["detailed_industry_recode"], errors="coerce" + ).to_numpy(dtype=np.float64) + major_industry = pd.to_numeric( + person["major_industry_recode"], errors="coerce" + ).to_numpy(dtype=np.float64) + worked = person["worked_last_year"].fillna(False).astype(bool).to_numpy() + known_detailed = np.isin(detailed_industry, list(WEIND_TO_WEMIND)) + expected_major = np.full(detailed_industry.shape, np.nan) + expected_major[known_detailed] = [ + WEIND_TO_WEMIND[int(code)] for code in detailed_industry[known_detailed] + ] + work_experience_violations = { + "unknown_industry_code": int((~known_detailed).sum()), + "major_group_mismatch": int( + (known_detailed & (major_industry != expected_major)).sum() + ), + "worked_without_worker_industry": int( + (worked & ~np.isin(detailed_industry, WORKER_INDUSTRY_CODES)).sum() + ), + } return { "nonzero_shares": {column: share(column) for column in _SHARE_BANDS}, "share_bands": {column: list(band) for column, band in _SHARE_BANDS.items()}, @@ -1002,6 +1125,7 @@ def share(column: str) -> float: }, "fsla_overtime_premium_weighted_total": float(np.nansum(premium * weights)), "constraint_violations": violations, + "work_experience_violations": work_experience_violations, } @@ -1032,6 +1156,9 @@ def us_org_wages_signal_gate(frame: Frame) -> GateResult: for name, count in summary["constraint_violations"].items(): if count: failures.append(f"fsla_overtime_premium: {count} {name} violation(s).") + for name, count in summary["work_experience_violations"].items(): + if count: + failures.append(f"industry/worked_last_year: {count} {name} row(s).") return GateResult( name="org_wages_signal", passed=not failures, diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/release_input_coverage.py b/packages/microcosm-build/src/microcosm/build/us_runtime/release_input_coverage.py index 38b4ecf8dc..00f88d9a7a 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/release_input_coverage.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/release_input_coverage.py @@ -190,6 +190,13 @@ # pinned Census ASEC person files; the certified default carries it # through the Build P source enrichment. *US_SPM_INDEPENDENCE_ROLE_OUTPUT_COLUMNS, + # PolicyEngine/microcosm#719: the county-file person schema carries a + # working indicator and industry with occupation on every record. + # Written by the org_wages release stage from ASEC WEIND/WEMIND/WKSWORK + # (ACS spine: native INDP/WKWN through the release predictor join). + "detailed_industry_recode", + "major_industry_recode", + "worked_last_year", } ) diff --git a/packages/microcosm-build/tests/engine_contract/us/test_spec_engine_coverage_tool.py b/packages/microcosm-build/tests/engine_contract/us/test_spec_engine_coverage_tool.py index 646d57b651..ab9fab5824 100644 --- a/packages/microcosm-build/tests/engine_contract/us/test_spec_engine_coverage_tool.py +++ b/packages/microcosm-build/tests/engine_contract/us/test_spec_engine_coverage_tool.py @@ -82,22 +82,22 @@ def test_us_coverage_is_exact_complete_and_honest( assert_coverage_complete(coverage_report) assert coverage_report["status"] == "pass" fields = coverage_report["field_usage"] - assert fields["configuration_field_count"] == 42_218 - assert fields["authored_normative_field_count"] == 32_437 - assert fields["resolved_binding_field_count"] == 9_781 - assert fields["consumed_field_count"] == 42_218 + assert fields["configuration_field_count"] == 42229 + assert fields["authored_normative_field_count"] == 32_448 + assert fields["resolved_binding_field_count"] == 9781 + assert fields["consumed_field_count"] == 42229 assert fields["unused_field_count"] == 0 assert fields["multiple_primary_use_field_count"] == 0 assert fields["claim_count"] == 49 assert fields["mode_counts"] == { - "legacy_behavior": 14_043, - "compiler_semantic": 27_718, + "legacy_behavior": 14054, + "compiler_semantic": 27718, "front_end_validation": 354, "identity_only": 103, } assert fields["generation0_effect_counts"] == { - "legacy_behavior": 38_531, - "no_generation0_effect": 3_687, + "legacy_behavior": 38542, + "no_generation0_effect": 3687, } inventory = coverage_report["inventory_coverage"] @@ -107,7 +107,7 @@ def test_us_coverage_is_exact_complete_and_honest( assert inventory["missing_items"] == [] assert inventory["counts"]["producer_inputs"] == 2_744 assert inventory["counts"]["ownership_rows"] == 18 - assert inventory["counts"]["tail_control_fields"] == 934 + assert inventory["counts"]["tail_control_fields"] == 937 assert inventory["counts"]["seed_owner_bindings"] == 112 diff --git a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_calibration_semantics.py b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_calibration_semantics.py index ccec9ee148..a18d6c535e 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_calibration_semantics.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_calibration_semantics.py @@ -31,7 +31,7 @@ ROOT = _TEST_PATHS.repository US_ROOT = ROOT / "packages/microcosm-build/src/microcosm/build/us" CANONICAL_LEGACY_PROJECTION_SHA256 = ( - "05cb099ee8bd0234ae94586b76dc20ab8a5bd0735c00d4e82e67f912e0b33823" + "aa65317d84f808e03d133f9f8156cbdfca18578a8e5169d200a70990bb372b19" ) diff --git a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_field_usage.py b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_field_usage.py index 5132ba2e09..aa43254b68 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_field_usage.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_field_usage.py @@ -95,22 +95,22 @@ def _mutated_bundle( def test_exact_complete_ledger_has_one_primary_mode_per_pointer(field_ledger) -> None: - assert len(field_ledger.fields) == EXPECTED_CONFIGURATION_FIELD_COUNT == 42_218 + assert len(field_ledger.fields) == EXPECTED_CONFIGURATION_FIELD_COUNT == 42229 assert field_ledger.source_counts == { - "authored": 32_437, - "resolved_bindings": 9_781, + "authored": 32_448, + "resolved_bindings": 9781, } assert field_ledger.mode_counts == { - "legacy_behavior": 14_043, - "compiler_semantic": 27_718, + "legacy_behavior": 14054, + "compiler_semantic": 27718, "front_end_validation": 354, "identity_only": 103, } assert field_ledger.generation0_effect_counts == { - "legacy_behavior": 38_531, - "no_generation0_effect": 3_687, + "legacy_behavior": 38542, + "no_generation0_effect": 3687, } - assert len({field.pointer for field in field_ledger.fields}) == 42_218 + assert len({field.pointer for field in field_ledger.fields}) == 42229 def test_eligibility_concepts_are_validation_not_generation0_behavior( diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_acs_inputs.py b/packages/microcosm-build/tests/engine_free/us/test_us_acs_inputs.py index 8ae28e141c..1341cdae66 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_acs_inputs.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_acs_inputs.py @@ -84,6 +84,33 @@ def _hours_frame(**columns) -> Frame: ) +def test_acs_native_work_experience_preserves_source_rows_and_records_provenance(): + before = _hours_frame( + OCCP=[1010, np.nan], INDP=[7380, np.nan], WKWN=[13, np.nan], ESR=[1, np.nan] + ) + result = map_acs_native_inputs(before) + person = result.frame.person + + assert person["detailed_occupation_recode"].tolist() == [8, 0] + assert person["detailed_industry_recode"].tolist() == [13, 0] + assert person["major_industry_recode"].tolist() == [9, 0] + assert person["worked_last_year"].tolist() == [True, False] + assert person["worked_last_year"].dtype == bool + for column in ("OCCP", "INDP", "WKWN", "ESR"): + pd.testing.assert_series_equal(person[column], before.person[column]) + assert result.native_inputs["detailed_industry_recode"]["source_columns"] == [ + "INDP", + "AGEP", + "ESR", + ] + assert result.native_inputs["worked_last_year"]["missing_rows"] == 0 + + +def test_acs_partial_work_experience_surface_fails_closed(): + with pytest.raises(ValueError, match="needs.*INDP.*WKWN"): + map_acs_native_inputs(_hours_frame(OCCP=[1010, np.nan], ESR=[1, np.nan])) + + def test_acs_usual_hours_preserve_forty_and_allocation_despite_current_unemployment(): before = _hours_frame(WKHP=[40, np.nan], WKL=[1, np.nan], FWKHP=[1, 0], ESR=[3, 0]) result = map_acs_native_inputs(before) diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_acs_local_release_tool.py b/packages/microcosm-build/tests/engine_free/us/test_us_acs_local_release_tool.py index 839e45fe78..50a1a804bc 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_acs_local_release_tool.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_acs_local_release_tool.py @@ -16,6 +16,9 @@ from microcosm.build.gates import GateResult from microcosm.build.us_runtime import spm_measurement_gate +from microcosm.build.us_runtime.org_wages import ( + US_ORG_OCCUPATION_AND_WORK_EXPERIENCE_OUTPUT_COLUMNS, +) from microcosm.data import stored_inputs from test_support.paths import paths_for @@ -55,6 +58,7 @@ "weekly_hours_worked_before_lsr", "hours_worked_last_week", "takes_up_wic_if_eligible", + *US_ORG_OCCUPATION_AND_WORK_EXPERIENCE_OUTPUT_COLUMNS, } ), ) @@ -601,7 +605,9 @@ def _package_evidence_args(module, tmp_path: Path, monkeypatch, *, hours_report) for name, payload in evidence.items(): (ckpt / name).write_text(json.dumps(payload)) (tmp_path / "out.summary.json").write_text(json.dumps({"simulation_ready": True})) - monkeypatch.setattr(module, "_load_staging_frame", lambda *_a, **_k: object()) + monkeypatch.setattr( + module, "_load_staging_frame", lambda *_a, **_k: _plausible_hours_frame() + ) monkeypatch.setattr( module, "acs_local_hours_signal_gate", @@ -693,7 +699,7 @@ def test_package_refuses_when_the_packaged_bytes_fail_the_hours_gate( def load_frame(path, *_a, **_k): loaded.append(Path(path)) - return object() + return _plausible_hours_frame() monkeypatch.setattr(module, "_load_staging_frame", load_frame) monkeypatch.setattr( @@ -1158,6 +1164,10 @@ def _staging_frame_with_hours(weekly: list[float], last_week: list[float]): "person_marital_unit_id": ids, "weekly_hours_worked_before_lsr": np.asarray(weekly, dtype=float), "hours_worked_last_week": np.asarray(last_week, dtype=float), + "detailed_occupation_recode": np.where(np.asarray(weekly) > 0, 8, 53), + "detailed_industry_recode": np.where(np.asarray(weekly) > 0, 13, 23), + "major_industry_recode": np.where(np.asarray(weekly) > 0, 9, 15), + "worked_last_year": np.asarray(weekly) > 0, } ) tables = { @@ -1597,6 +1607,70 @@ def test_package_accepts_passing_hours_gate_bound_to_the_packaged_bytes( copied_sha = module._sha256(Path(result["root_artifact"]["local_path"])) assert gate["passed"] is True assert gate["artifact_sha256"] == result["root_artifact"]["sha256"] == copied_sha + for name in ("build_manifest.json", "gate_summary.json"): + work_gate = json.loads((release_dir / name).read_text())["gates"][ + "work_experience_inputs" + ] + assert work_gate == { + "passed": True, + "failures": [], + "columns": list(US_ORG_OCCUPATION_AND_WORK_EXPERIENCE_OUTPUT_COLUMNS), + "person_rows": 8, + "artifact_sha256": copied_sha, + "checked_at_stage": "package", + } + + +@requires_pytables +@pytest.mark.parametrize("defect", ["missing", "asec_null", "acs_null", "nonfinite"]) +def test_package_refuses_historical_staging_with_incomplete_work_experience( + tmp_path, monkeypatch, defect +): + """A green historical staging/finalize summary cannot stand in for inputs.""" + from microcosm.frame import Frame + + module = _load_tool_module() + args = _package_args_with_hours(module, tmp_path, monkeypatch, gate_state="passed") + frame = _plausible_hours_frame() + person = frame.person.copy() + spines = ["asec_puf"] * 4 + ["acs_2024_1yr"] * 4 + person["person_spine"] = spines + if defect == "missing": + person = person.drop( + columns=[ + column + for column in US_ORG_OCCUPATION_AND_WORK_EXPERIENCE_OUTPUT_COLUMNS + if column != "detailed_occupation_recode" + ] + ) + else: + person["major_industry_recode"] = person["major_industry_recode"].astype(float) + row = 7 if defect == "acs_null" else 0 + person.loc[row, "major_industry_recode"] = ( + np.inf if defect == "nonfinite" else np.nan + ) + incomplete = Frame( + { + **{entity: frame.table(entity) for entity in frame.entities}, + "person": person, + "household": frame.table("household").assign(household_spine=spines), + }, + frame.schema, + {"household": frame.weights_for("household")}, + ) + _write_frame_h5(args.out_h5, incomplete) + args.staging_h5.write_bytes(args.out_h5.read_bytes()) + identity_path = args.checkpoint_dir / "run_identity.json" + identity = json.loads(identity_path.read_text()) + identity["staging_sha256"] = module._sha256(args.staging_h5) + identity_path.write_text(json.dumps(identity)) + _refresh_package_evidence_bindings(module, args) + + with pytest.raises(SystemExit, match="Refusing to package:.*work-experience"): + module.do_package(args) + assert not (args.out / "releases").exists() + assert not (args.out / "package_result.json").exists() + assert not (args.out / module.ARTIFACT_FILENAME).exists() @requires_pytables diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_acs_multispine_legacy_builder.py b/packages/microcosm-build/tests/engine_free/us/test_us_acs_multispine_legacy_builder.py index 59d4531679..44c9135fa1 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_acs_multispine_legacy_builder.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_acs_multispine_legacy_builder.py @@ -53,6 +53,10 @@ def _frame( "person_marital_unit_id": [1, 2], "age": [35, 67], "is_female": [False, True], + "detailed_occupation_recode": [8, 53], + "detailed_industry_recode": [13, 23], + "major_industry_recode": [9, 15], + "worked_last_year": [True, False], } ) if benefit_participation: @@ -1225,6 +1229,70 @@ def test_donor_release_identity_requires_one_microdata_artifact( module._donor_release_identity(manifest, "a" * 64) +@pytest.mark.parametrize( + "column", + [ + "detailed_occupation_recode", + "detailed_industry_recode", + "major_industry_recode", + "worked_last_year", + ], +) +def test_donor_without_work_experience_fails_before_fetch( + monkeypatch: pytest.MonkeyPatch, + column: str, +) -> None: + builder = _load_builder_module() + donor = _frame() + donor.person.drop(columns=[column], inplace=True) + monkeypatch.setattr( + builder.acs_sources, "load_acs_source_manifest", lambda path: _manifest() + ) + monkeypatch.setattr(builder, "_load_base_frame", lambda path: donor) + monkeypatch.setattr(builder, "_sha256", lambda path: "0" * 64) + + def must_not_fetch(*_args, **_kwargs): + pytest.fail("source fetch must not run for an incomplete work-experience donor") + + monkeypatch.setattr(builder.acs_sources, "fetch_acs_pums_sources", must_not_fetch) + + with pytest.raises(SystemExit, match=f"lacks canonical work-experience.*{column}"): + builder.main(["--base-h5", "donor.h5", "--out-h5", "combined.h5"]) + + +@pytest.mark.parametrize( + "column", + [ + "detailed_occupation_recode", + "detailed_industry_recode", + "major_industry_recode", + "worked_last_year", + ], +) +@pytest.mark.parametrize("invalid", [None, np.inf, "invalid"]) +def test_donor_work_experience_rejects_null_or_nonfinite_values( + column, invalid +) -> None: + builder = _load_builder_module() + donor = _frame() + donor.person[column] = pd.Series( + [donor.person[column].iloc[0], invalid], dtype=object + ) + + with pytest.raises(SystemExit, match=f"missing or nonfinite.*{column}"): + builder._require_donor_work_experience(donor) + + +def test_complete_donor_work_experience_passes_without_changing_rows() -> None: + builder = _load_builder_module() + donor = _frame() + before = donor.person.copy(deep=True) + + builder._require_donor_work_experience(donor) + + pd.testing.assert_frame_equal(donor.person, before) + + def test_donor_without_the_spm_role_fails_before_fetch( monkeypatch: pytest.MonkeyPatch, ) -> None: diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_acs_pums.py b/packages/microcosm-build/tests/engine_free/us/test_us_acs_pums.py index 30e3f8eddf..961f599ce7 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_acs_pums.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_acs_pums.py @@ -8,12 +8,16 @@ import pytest from microcosm.build.serialization_dtypes import CANONICAL_STRING_DTYPE +from microcosm.build.us_runtime.acs_inputs import map_acs_native_inputs from microcosm.build.us_runtime.acs_pums import ( ACS_2024_1YR_SPINE, AcsPumsSource, build_acs_pums_unit_frame, load_acs_pums_tables, ) +from microcosm.build.us_runtime.operator_boundary import ( + assert_operator_free_source_frame, +) from microcosm.build.us_runtime.spine_assembly import assemble_spines from microcosm.frame import US_SCHEMA, Frame, WeightKind, Weights @@ -68,6 +72,14 @@ def _person( "PWGTP": 10, } row.update(overrides) + in_work_universe = int(row["AGEP"]) >= 16 + for column, value in ( + ("OCCP", 1010), + ("INDP", 7380), + ("WKWN", 52), + ("ESR", 1), + ): + row.setdefault(column, value if in_work_universe else None) return row @@ -146,6 +158,62 @@ def test_acs_loader_preserves_hours_and_allocation_without_filling_blanks(tmp_pa assert "weekly_hours_worked_before_lsr" not in tables["person"] +def test_acs_archive_to_native_inputs_carries_work_experience_on_every_row(tmp_path): + household_zip = tmp_path / "work-hh.zip" + person_zip = tmp_path / "work-person.zip" + _write_csv_zip(household_zip, {"psam_husa.csv": [_household("work", NP=4)]}) + _write_csv_zip( + person_zip, + { + "psam_pusa.csv": [ + _person("work", 1, 20, WKWN=13), + # ACS retains the most recent job for up to five years. + _person("work", 2, 21, ESR=6, WKWN=None), + _person( + "work", 3, 28, AGEP=16, ESR=6, OCCP=None, INDP=None, WKWN=None + ), + _person("work", 4, 25, AGEP=15, MAR=5), + ] + }, + ) + frame, _ = build_acs_pums_unit_frame(AcsPumsSource(household_zip, person_zip)) + result = map_acs_native_inputs(frame) + person = result.frame.person.sort_values("source_person_id") + + assert person["detailed_occupation_recode"].tolist() == [8, 8, 53, 0] + assert person["detailed_industry_recode"].tolist() == [13, 13, 23, 0] + assert person["major_industry_recode"].tolist() == [9, 9, 15, 0] + assert person["worked_last_year"].tolist() == [True, False, False, False] + assert person["worked_last_year"].dtype == bool + assert person["detailed_industry_recode"].dtype == np.int16 + for column in ( + "detailed_occupation_recode", + "detailed_industry_recode", + "major_industry_recode", + "worked_last_year", + ): + assert result.native_inputs[column]["observed_rows"] == 4 + assert result.native_inputs[column]["missing_rows"] == 0 + assert_operator_free_source_frame( + result.frame, + label="ACS native work-experience fixture", + native_inputs=result.native_inputs, + ) + + +@pytest.mark.parametrize("column", ["OCCP", "INDP", "WKWN", "ESR"]) +def test_acs_loader_refuses_missing_work_experience_source_columns(tmp_path, column): + household_zip = tmp_path / "missing-work-hh.zip" + person_zip = tmp_path / "missing-work-person.zip" + _write_csv_zip(household_zip, {"psam_husa.csv": [_household("work")]}) + row = _person("work", 1, 20) + del row[column] + _write_csv_zip(person_zip, {"psam_pusa.csv": [row]}) + + with pytest.raises(ValueError, match=f"missing required column.*{column}"): + load_acs_pums_tables(AcsPumsSource(household_zip, person_zip)) + + def _asec_shaped_frame() -> Frame: """Return one ASEC-like row sharing only structural and lineage fields.""" diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_acs_release_predictors.py b/packages/microcosm-build/tests/engine_free/us/test_us_acs_release_predictors.py index 0c298aaea5..c463e64401 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_acs_release_predictors.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_acs_release_predictors.py @@ -12,11 +12,14 @@ import microcosm.build.us_runtime.acs_release_predictors as module from microcosm.build.us_runtime.acs_release_predictors import ( + ACS_INDP_TO_WEIND, ACS_OCCP_TO_POCCU2, ACS_RELEASE_PREDICTOR_CROSSWALK_SHA256, acs_release_predictor_crosswalk_payload, join_acs_release_predictors, ) +from microcosm.build.us_runtime.asec_census_person_columns import WEIND_TO_WEMIND +from microcosm.build.us_runtime.org_wages import derive_us_org_occupation_inputs from microcosm.build.us_runtime.puf_support import clone_us_frame_for_puf_support from microcosm.build.us_runtime.spine_assembly import assemble_spines from microcosm.frame import US_SCHEMA, Frame, WeightKind, Weights @@ -33,10 +36,17 @@ "PRDTHSP", "PEIOOCC", "POCCU2", + "WEIND", + "WEMIND", + "WKSWORK", "SPM_TENMORTSTATUS", ) +#: ACS 2024 Armed Forces branch codes CPS collapses into industry 9890. +_ACS_ARMED_FORCES_INDP = (9670, 9680, 9690, 9770, 9780, 9790, 9870) + + def _source_frame(*, acs: bool) -> Frame: if acs: household_ids = np.asarray([1, 2], dtype=np.int64) @@ -87,6 +97,9 @@ def _source_frame(*, acs: bool) -> Frame: person["PRDTHSP"] = 0.0 person["PEIOOCC"] = 1005.0 person["POCCU2"] = 8.0 + person["WEIND"] = 13.0 + person["WEMIND"] = 9.0 + person["WKSWORK"] = 52.0 person["SPM_TENMORTSTATUS"] = 1.0 return Frame( @@ -125,6 +138,8 @@ def _raw_person() -> pd.DataFrame: "RAC1P": [1, 6, 2], "HISP": [1, 1, 2], "OCCP": [np.nan, 1005, 9800], + "INDP": [np.nan, 7380, 9670], + "WKWN": [np.nan, 52, 48], "ESR": [np.nan, 1, 4], "SSIP": [np.nan, 900.0, 0.0], "ADJINC": [1_000_000, 1_000_000, 1_000_000], @@ -210,8 +225,29 @@ def test_crosswalk_digest_and_every_consumed_occupation_bin_are_pinned() -> None assert ACS_OCCP_TO_POCCU2[6005] == 41 assert ACS_OCCP_TO_POCCU2[9800] == 52 assert ACS_OCCP_TO_POCCU2[9920] == 53 + assert len(ACS_INDP_TO_WEIND) == 267 + assert set(ACS_INDP_TO_WEIND.values()) == set(range(1, 24)) + assert ACS_INDP_TO_WEIND[170] == 1 + assert ACS_INDP_TO_WEIND[7380] == 13 + assert ACS_INDP_TO_WEIND[8191] == 16 + assert {ACS_INDP_TO_WEIND[code] for code in _ACS_ARMED_FORCES_INDP} == {22} + assert ACS_INDP_TO_WEIND[9920] == 23 payload = acs_release_predictor_crosswalk_payload() + assert payload["version"] == 2 + assert payload["industry"]["blank_INDP_to_WEIND"]["age_16_plus"] == 23 + assert payload["industry"]["WEIND_to_WEMIND"] == { + str(key): value for key, value in WEIND_TO_WEMIND.items() + } + assert payload["work_experience"]["WKWN_to_WKSWORK"]["blank"] == 0 + assert payload["model_predictors"]["org_wages"] == [ + "PRDTRACE", + "PRDTHSP", + "POCCU2", + "WEIND", + "WEMIND", + "WKSWORK", + ] assert payload["disability"]["DREM"]["minimum_question_age"] == 5 assert payload["disability"]["DOUT"]["minimum_question_age"] == 15 assert payload["disability"]["DOUT"]["codes"]["below_universe_blank"] == -1 @@ -257,6 +293,9 @@ def test_release_join_is_exact_total_clone_stable_and_receipted( "PRDTHSP", "PEIOOCC", "POCCU2", + "WEIND", + "WEMIND", + "WKSWORK", "SPM_TENMORTSTATUS", ], ].drop_duplicates("person_source_id") @@ -265,12 +304,29 @@ def test_release_join_is_exact_total_clone_stable_and_receipted( assert sorted(by_source["PRDTHSP"].tolist()) == [0.0, 0.0, 1.0] assert sorted(by_source["PEIOOCC"].tolist()) == [-1.0, 1005.0, 9800.0] assert sorted(by_source["POCCU2"].tolist()) == [0.0, 8.0, 52.0] + assert sorted(by_source["WEIND"].tolist()) == [0.0, 13.0, 22.0] + assert sorted(by_source["WEMIND"].tolist()) == [0.0, 9.0, 14.0] + assert sorted(by_source["WKSWORK"].tolist()) == [0.0, 48.0, 52.0] assert sorted(by_source["SPM_TENMORTSTATUS"].tolist()) == [1.0, 1.0, 3.0] for _, clones in person.loc[acs].groupby("person_source_id"): for column in module._OUTPUT_COLUMNS: assert clones[column].nunique(dropna=False) == 1 + carried = derive_us_org_occupation_inputs(person) + np.testing.assert_array_equal(carried["detailed_industry_recode"], person["WEIND"]) + np.testing.assert_array_equal(carried["major_industry_recode"], person["WEMIND"]) + np.testing.assert_array_equal(carried["worked_last_year"], person["WKSWORK"].gt(0)) + assert carried.loc[asec, "worked_last_year"].all() + assert sorted(carried.loc[acs, "worked_last_year"].tolist()) == [ + False, + False, + True, + True, + True, + True, + ] + receipt = result.receipt assert receipt["crosswalk"]["sha256"] == ACS_RELEASE_PREDICTOR_CROSSWALK_SHA256 assert receipt["join"] == { @@ -304,6 +360,8 @@ def test_occupation_crosswalk_preserves_age_15_source_universe_gap() -> None: rows["SPORDER"] = [1, 1] rows["AGEP"] = [15, 16] rows["OCCP"] = np.nan + rows["INDP"] = np.nan + rows["WKWN"] = np.nan rows["ESR"] = [np.nan, 6] rows["person_source_id"] = [1, 2] @@ -313,6 +371,62 @@ def test_occupation_crosswalk_preserves_age_15_source_universe_gap() -> None: assert mapped["POCCU2"].tolist() == [0, 53] +def test_industry_crosswalk_preserves_age_15_source_universe_gap() -> None: + rows = pd.concat([_raw_person().iloc[[1]]] * 2, ignore_index=True) + rows["SERIALNO"] = ["age15", "age16"] + rows["SPORDER"] = [1, 1] + rows["AGEP"] = [15, 16] + rows["OCCP"] = np.nan + rows["INDP"] = np.nan + rows["WKWN"] = np.nan + rows["ESR"] = [np.nan, 6] + rows["person_source_id"] = [1, 2] + + mapped = module._crosswalk_people(rows) + + assert mapped["WEIND"].tolist() == [0, 23] + assert mapped["WEMIND"].tolist() == [0, 15] + assert mapped["WKSWORK"].tolist() == [0, 0] + + +def test_industry_is_carried_for_a_job_held_in_the_past_five_years() -> None: + """ACS INDP covers the last 5 years; only the weeks say who worked lately.""" + + row = _raw_person().iloc[[1]].copy() + row["ESR"] = 6 + row["WKWN"] = np.nan + row["person_source_id"] = 1 + + mapped = module._crosswalk_people(row) + + assert mapped[["WEIND", "WEMIND", "WKSWORK"]].to_numpy().tolist() == [[13, 9, 0]] + + +@pytest.mark.parametrize( + ("changes", "match"), + [ + ({"INDP": np.nan}, "INDP is blank inside its observed employment universe"), + ({"INDP": 1}, r"INDP contains unsupported code\(s\): \[1\]"), + ({"INDP": 9920, "OCCP": 9920, "ESR": 3}, "weeks worked without a worker"), + ({"WKWN": 53}, "WKWN code/universe mismatch"), + ( + {"AGEP": 12, "DOUT": np.nan, "OCCP": np.nan, "INDP": np.nan, "ESR": np.nan}, + "WKWN code/universe mismatch", + ), + ], +) +def test_industry_crosswalk_refuses_incoherent_source_rows( + changes: dict[str, object], match: str +) -> None: + row = _raw_person().iloc[[1]].copy() + for column, value in changes.items(): + row[column] = value + row["person_source_id"] = 1 + + with pytest.raises(ValueError, match=match): + module._crosswalk_people(row) + + def test_occupation_crosswalk_refuses_malformed_esr_universe() -> None: row = _raw_person().iloc[[1]].copy() row["ESR"] = "not-a-code" diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_asec_census_person_columns.py b/packages/microcosm-build/tests/engine_free/us/test_us_asec_census_person_columns.py index 2fad301645..2a7f33e5f5 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_asec_census_person_columns.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_asec_census_person_columns.py @@ -17,8 +17,10 @@ from microcosm.build.us_runtime.asec_census_person_columns import ( ASEC_CENSUS_PERSON_COLUMN_NAMES, ASEC_CENSUS_PERSON_COLUMNS, + ASEC_CENSUS_PERSON_COLUMNS_BEYOND_OFFLINE_FIX, ASEC_CENSUS_PERSON_COLUMNS_NOT_RESTORED, ASEC_CENSUS_PERSON_IDENTITY_COLUMNS, + WEIND_TO_WEMIND, AsecCensusPersonColumnsError, restore_asec_census_person_columns, ) @@ -77,6 +79,12 @@ #: H5 row order differs from the member's, so a positional copy would fail. _H5_ORDER = [3, 0, 7, 5, 1, 6, 2, 4] +#: Member-order industry codes: six workers (one in the Armed Forces) and two +#: people 15+ who did not work last year. Every member person is 16+. +_WEIND = [7, 16, 23, 22, 23, 9, 1, 21] +#: Weeks worked last year consistent with ``_WEIND`` (positive iff 1--22). +_WKSWORK = [52, 40, 0, 52, 0, 12, 26, 3] + def _digest(path: Path) -> str: return hashlib.sha256(path.read_bytes()).hexdigest() @@ -102,6 +110,9 @@ def _member() -> pd.DataFrame: else: codes = sorted(spec.domain) member[spec.name] = [codes[(row * 3) % len(codes)] for row in range(8)] + # The industry pair is rigid: WEMIND is the major group of WEIND. + member["WEIND"] = _WEIND + member["WEMIND"] = [WEIND_TO_WEMIND[code] for code in _WEIND] return member @@ -371,14 +382,98 @@ def test_default_pins_are_the_shared_census_person_pins(sources): def test_the_review_places_every_offline_fix_column_exactly_once(): restored = set(ASEC_CENSUS_PERSON_COLUMN_NAMES) not_restored = set(ASEC_CENSUS_PERSON_COLUMNS_NOT_RESTORED) + beyond = set(ASEC_CENSUS_PERSON_COLUMNS_BEYOND_OFFLINE_FIX) assert len(restored) == len(ASEC_CENSUS_PERSON_COLUMN_NAMES) assert restored.isdisjoint(not_restored) - assert restored | not_restored == _RECEIPT_720_ADDED | _ISSUE_454_ADDED + assert beyond == {"WEIND", "WEMIND"} + assert beyond <= restored + assert beyond.isdisjoint(_RECEIPT_720_ADDED) + assert (restored - beyond) | not_restored == _RECEIPT_720_ADDED | _ISSUE_454_ADDED assert _ISSUE_454_ADDED <= restored assert _ISSUE_454_ADDED.isdisjoint(_RECEIPT_720_ADDED) assert not restored & set(ASEC_CENSUS_PERSON_IDENTITY_COLUMNS) +def test_weind_to_wemind_is_the_codebook_nesting(): + """Pinned from all three members; every major group nests detailed ones.""" + + assert dict(WEIND_TO_WEMIND) == { + 0: 0, + 1: 1, + 2: 2, + 3: 3, + 4: 4, + 5: 4, + 6: 5, + 7: 5, + 8: 6, + 9: 6, + 10: 7, + 11: 8, + 12: 8, + 13: 9, + 14: 9, + 15: 10, + 16: 10, + 17: 11, + 18: 11, + 19: 12, + 20: 12, + 21: 13, + 22: 14, + 23: 15, + } + specs = {spec.name: spec for spec in ASEC_CENSUS_PERSON_COLUMNS} + assert set(WEIND_TO_WEMIND) == specs["WEIND"].domain + assert set(WEIND_TO_WEMIND.values()) == specs["WEMIND"].domain + majors = list(WEIND_TO_WEMIND.values()) + assert majors == sorted(majors) + + +def test_restores_industry_and_verifies_the_worker_universe(sources): + member, csv, _, pin = sources + person = _h5_person(member) + person["WKSWORK"] = np.asarray(_WKSWORK, dtype=np.int64)[_H5_ORDER] + result, record = _restore(person, csv, pin) + joined = member.set_index("PERIDNUM").loc[person["PERIDNUM"].astype(str)] + assert result["WEIND"].tolist() == joined["WEIND"].tolist() + assert result["WEMIND"].tolist() == joined["WEMIND"].tolist() + assert record["work_experience_universe"] == { + "weind_to_wemind_rows_verified": 8, + "worker_code_iff_weeks_worked": "verified", + "worker_rows": 6, + } + + +def test_records_that_the_worker_universe_needs_wkswork(sources): + member, csv, _, pin = sources + _, record = _restore(_h5_person(member), csv, pin) + assert record["work_experience_universe"] == { + "weind_to_wemind_rows_verified": 8, + "worker_code_iff_weeks_worked": "not checked: H5 lacks WKSWORK", + } + + +@pytest.mark.parametrize("weeks", [0, 30]) +def test_refuses_a_worker_code_that_disagrees_with_weeks_worked(sources, weeks): + member, csv, _, pin = sources + person = _h5_person(member) + wkswork = np.asarray(_WKSWORK, dtype=np.int64)[_H5_ORDER] + # H5 row 1 is member row 0 (a worker); H5 row 6 is member row 2 (no work). + wkswork[1 if weeks == 0 else 6] = weeks + person["WKSWORK"] = wkswork + with pytest.raises(AsecCensusPersonColumnsError, match="worker code"): + _restore(person, csv, pin) + + +def test_refuses_a_major_group_that_is_not_the_detailed_groups_parent(tmp_path): + member = _member() + member.loc[0, "WEMIND"] = 6 # WEIND 7 nests in major group 5 + csv, _, pin = _write(tmp_path, member) + with pytest.raises(AsecCensusPersonColumnsError, match="major group of WEIND"): + _restore(_h5_person(member), csv, pin) + + def _cited_objects(reader: str): for dotted in re.findall(r"\b(?:microcosm|microunit)(?:\.\w+)+", reader): parts = dotted.split(".") diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_bundle_core_contracts.py b/packages/microcosm-build/tests/engine_free/us/test_us_bundle_core_contracts.py index 89d96e39ef..ee8e0a62d4 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_bundle_core_contracts.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_bundle_core_contracts.py @@ -25,7 +25,7 @@ def test_source_surface_classification_is_complete() -> None: } assert normative["stage_asset"] == { "id": "source_stages", - "sha256": "559062a1bd10efdd89b8c83da9294fea43062b8b6760524f4350dd84d63ef383", + "sha256": "dc3b72982675b95d30cfabdeb5f7f4c492e6cd3788009ef3094ab4d48bfd9093", } assert operational["stage_asset"] == { "path": "microcosm.build.us/source_stages.json" diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_l0_refit_export.py b/packages/microcosm-build/tests/engine_free/us/test_us_l0_refit_export.py index 7240abe15f..5aa49ccc9b 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_l0_refit_export.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_l0_refit_export.py @@ -130,6 +130,9 @@ def _us_frame(**person_extra: object) -> Frame: "cps_race": [1, 2, 4], "is_hispanic": [False, True, False], "detailed_occupation_recode": [1, 20, 53], + "detailed_industry_recode": [13, 7, 23], + "major_industry_recode": [9, 5, 15], + "worked_last_year": [True, True, False], "has_never_worked": [False, False, True], "is_military": [False, True, False], "is_computer_scientist": [False, True, False], diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_org_wages.py b/packages/microcosm-build/tests/engine_free/us/test_us_org_wages.py index b5c262def0..2355d4b7f2 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_org_wages.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_org_wages.py @@ -145,11 +145,25 @@ def test_occupation_carries_match_retired_poccu2_codes() -> None: "PRDTRACE": [1, 2, 4, 1, 1, 1, 1], "PRDTHSP": [0, 1, 0, 0, 0, 0, 0], "POCCU2": [53, 52, 8, 41, 1, 50, 20], + "WEIND": [23, 22, 13, 1, 13, 7, 16], + "WEMIND": [15, 14, 9, 1, 9, 5, 10], + "WKSWORK": [0, 52, 52, 26, 52, 40, 12], } ) result = derive_us_org_occupation_inputs(person) assert result["cps_race"].tolist() == [1, 2, 4, 1, 1, 1, 1] + assert result["detailed_industry_recode"].tolist() == person["WEIND"].tolist() + assert result["major_industry_recode"].tolist() == person["WEMIND"].tolist() + assert result["worked_last_year"].tolist() == [ + False, + True, + True, + True, + True, + True, + True, + ] assert result["is_hispanic"].tolist() == [ False, True, diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_spec_engine_repin.py b/packages/microcosm-build/tests/engine_free/us/test_us_spec_engine_repin.py new file mode 100644 index 0000000000..d928b269cb --- /dev/null +++ b/packages/microcosm-build/tests/engine_free/us/test_us_spec_engine_repin.py @@ -0,0 +1,207 @@ +"""The F0 pin writer preserves surrounding source and refuses unknown pins.""" + +import ast + +import pytest + +from tools.repin_us_spec_engine_contracts import ( + _replace_asserted_counts, + _replace_asserted_dictionary_value, + _replace_asserted_scalar, + _replace_assignment, + _replace_dictionary, + _replace_length_assertions, + _replace_literal_assertions, + _replace_literals, + _replace_unclaimed_count, +) + + +def test_repin_replaces_only_exact_literals() -> None: + source = 'count = 934\nother = 1934\nlabel = "934"\n# 934 is historical\n' + assert _replace_literals(source, {934: 937}) == ( + 'count = 937\nother = 1934\nlabel = "934"\n# 934 is historical\n' + ) + + +def test_repin_preserves_unchanged_pin_bytes_and_unrelated_comments() -> None: + source = 'PINS = {\n "keep": (2, "abc"), # reviewed\n "move": 3,\n}\n' + assert _replace_dictionary(source, "PINS", {"keep": (2, "abc")}) == source + assert _replace_dictionary(source, "PINS", {"move": 4}) == source.replace( + '"move": 3', '"move": 4' + ) + + +def test_repin_refuses_unknown_assignments_and_dictionary_entries() -> None: + with pytest.raises(ValueError, match="missing assignment"): + _replace_assignment("KNOWN = 1\n", "UNKNOWN", 2) + with pytest.raises(ValueError, match="expected one key"): + _replace_dictionary('PINS = {"known": 1}\n', "PINS", {"unknown": 2}) + + +def test_inventory_count_updates_only_the_reviewed_length_target() -> None: + source = ( + "assert len(columns) == 176\nassert len(unrelated) == 176\n" + "if len(columns) > 176:\n pass\n" + ) + assert _replace_length_assertions(source, {"columns": 179}) == ( + "assert len(columns) == 179\nassert len(unrelated) == 176\n" + "if len(columns) > 176:\n pass\n" + ) + with pytest.raises(ValueError, match="targets changed"): + _replace_length_assertions(source, {"missing": 179}) + + +def test_inventory_count_update_refuses_duplicate_targets() -> None: + source = "assert len(columns) == 176\nassert len(columns) == 176\n" + with pytest.raises(ValueError, match="duplicate inventory assertion"): + _replace_length_assertions(source, {"columns": 179}) + + +def test_receipt_pin_updates_only_one_exact_literal_assertion() -> None: + source = ( + 'assert first.spec_sha256 == ("old")\n' + 'assert other.spec_sha256 == "old"\n' + "# old is historical\n" + ) + assert _replace_literal_assertions(source, {"first.spec_sha256": "new"}) == ( + "assert first.spec_sha256 == ('new')\n" + 'assert other.spec_sha256 == "old"\n' + "# old is historical\n" + ) + with pytest.raises(ValueError, match="targets changed"): + _replace_literal_assertions(source, {"missing": "new"}) + + +def test_receipt_pin_refuses_duplicates_and_nonliteral_expectations() -> None: + source = 'assert receipt == "old"\nassert receipt == "old"\n' + with pytest.raises(ValueError, match="duplicate receipt assertion"): + _replace_literal_assertions(source, {"receipt": "new"}) + with pytest.raises(ValueError): + _replace_literal_assertions("assert receipt == expected\n", {"receipt": "new"}) + with pytest.raises(ValueError, match="not a literal string"): + _replace_literal_assertions("assert receipt == 123\n", {"receipt": "new"}) + + +def test_nested_receipt_pin_preserves_the_complete_dictionary_assertion() -> None: + source = ( + 'assert run_config == {"spec_binding": {"sha256": "old", "keep": "old"}}\n' + "# old is historical\n" + ) + assert _replace_asserted_dictionary_value( + source, "run_config", ("spec_binding", "sha256"), "new" + ) == source.replace('"sha256": "old"', "\"sha256\": 'new'") + with pytest.raises(ValueError, match="expected one receipt dictionary key"): + _replace_asserted_dictionary_value( + source, "run_config", ("spec_binding", "missing"), "new" + ) + + +def test_nested_receipt_pin_refuses_duplicate_assertions_or_dictionary_keys() -> None: + source = 'assert run_config == {"sha256": "old"}\n' + with pytest.raises(ValueError, match="expected one asserted dictionary"): + _replace_asserted_dictionary_value( + source + source, "run_config", ("sha256",), "new" + ) + with pytest.raises(ValueError, match="expected one receipt dictionary key"): + _replace_asserted_dictionary_value( + 'assert run_config == {"sha256": "old", "sha256": "other"}\n', + "run_config", + ("sha256",), + "new", + ) + + +def test_count_breakdowns_are_rewritten_by_key_not_by_old_value() -> None: + # A merge can move the evidence JSON to the new count while a conflicted + # test keeps the old one; a value transition taken from the JSON would then + # see no change. Keying by name refreshes the stale literal regardless. + source = ( + "assert ledger.mode_counts == {\n" + ' "legacy_behavior": 14_054,\n' + ' "compiler_semantic": 27_717,\n' + "}\n" + "# 27_717 is historical\n" + ) + observed = {"legacy_behavior": 14_054, "compiler_semantic": 27_718} + assert _replace_asserted_counts(source, "ledger.mode_counts", observed) == ( + source.replace('"compiler_semantic": 27_717', '"compiler_semantic": 27718') + ) + assert _replace_asserted_counts( + source.replace("27_717,", "27_718,"), "ledger.mode_counts", observed + ) == source.replace("27_717,", "27_718,") + subscripted = source.replace("ledger.mode_counts", "fields['mode_counts']") + assert "27718" in _replace_asserted_counts( + subscripted, "fields['mode_counts']", observed + ) + + +def test_count_breakdowns_refuse_missing_extra_or_duplicate_keys() -> None: + source = 'assert ledger.counts == {"a": 1, "b": 2}\n' + with pytest.raises(ValueError, match="differ from the observed"): + _replace_asserted_counts(source, "ledger.counts", {"a": 1}) + with pytest.raises(ValueError, match="differ from the observed"): + _replace_asserted_counts(source, "ledger.counts", {"a": 1, "b": 2, "c": 3}) + with pytest.raises(ValueError, match="differ from the observed"): + _replace_asserted_counts( + 'assert ledger.counts == {"a": 1, "a": 2}\n', "ledger.counts", {"a": 1} + ) + with pytest.raises(ValueError, match="expected one asserted count dictionary"): + _replace_asserted_counts(source + source, "ledger.counts", {"a": 1, "b": 2}) + + +def test_scalar_pins_are_found_by_asserted_expression_not_old_value() -> None: + # The module pin may already read 42_229 after a merge while the test + # still asserts 42_228; matching the expression refreshes it anyway. + source = ( + "assert len(ledger.fields) == EXPECTED == 42_228\n" + 'assert fields["claim_count"] == 49\n' + "assert other == 42_228\n" + ) + refreshed = _replace_asserted_scalar(source, "len(ledger.fields)", 42_229) + assert refreshed == source.replace("EXPECTED == 42_228", "EXPECTED == 42229") + assert _replace_asserted_scalar(source, "fields['claim_count']", 49) == source + + +def test_scalar_pins_refuse_missing_duplicate_or_nonliteral_assertions() -> None: + with pytest.raises(ValueError, match="expected one scalar assertion"): + _replace_asserted_scalar("assert x == 1\n", "y", 2) + with pytest.raises(ValueError, match="expected one scalar assertion"): + _replace_asserted_scalar("assert x == 1\nassert x == 1\n", "x", 2) + with pytest.raises(ValueError, match="not an integer literal"): + _replace_asserted_scalar("assert x == EXPECTED\n", "x", 2) + with pytest.raises(ValueError, match="not an integer literal"): + _replace_asserted_scalar("assert x == True\n", "x", 2) + + +def test_unclaimed_count_expectation_follows_the_observed_expansion() -> None: + source = 'with raises(E, match=r"87 unclaimed normative field"):\n# 87 stays\n' + assert _replace_unclaimed_count(source, 90) == source.replace( + 'r"87 unclaimed', 'r"90 unclaimed' + ) + assert _replace_unclaimed_count(source, 87) == source + with pytest.raises(ValueError, match="found 0"): + _replace_unclaimed_count("x = 1\n", 90) + with pytest.raises(ValueError, match="found 2"): + _replace_unclaimed_count(source + source, 90) + + +def test_calibration_pin_hashes_the_fixture_contract_not_the_loaded_bundle() -> None: + # The loader writes integral floats as integers (5.0 -> 5), so a digest of + # the compiled bundle's calibration never matches the golden test's pin. + from microcosm.build.spec_engine import project_legacy_calibration_contract + from microcosm.build.spec_engine.canonical import sha256_json + from tools.repin_us_spec_engine_contracts import ( + PACKAGE, + _assignment, + fixture_calibration_contract, + ) + + path = ( + PACKAGE / "tests/engine_free/shared/test_spec_engine_calibration_semantics.py" + ) + pinned = ast.literal_eval( + _assignment(path.read_text(), "CANONICAL_LEGACY_PROJECTION_SHA256") + ) + projected = project_legacy_calibration_contract(fixture_calibration_contract()) + assert sha256_json(projected) == pinned diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_spine_blindness.py b/packages/microcosm-build/tests/engine_free/us/test_us_spine_blindness.py index 0e0decad2f..f9987bb678 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_spine_blindness.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_spine_blindness.py @@ -3506,19 +3506,21 @@ def test_pool_build_tool_import_graph_is_source_spine_blind() -> None: for tool in _SPINE_BLIND_BUILD_TOOLS: runtime_graph, missing_modules = _us_runtime_import_graph(tool) - # 75 = main's 70 plus spm_independence_role.py, spm_role_source.py and + # 77 = main's 70 plus spm_independence_role.py, spm_role_source.py and # spm_composition.py, reached because the pool's engine-input # projection names the SPM role as a required source input (#893), # plus asec_sources.py, whose default ASEC pool the education sidecar # loader now defaults to, plus esi_premiums.py, whose output family # operator_boundary registers so a raw source frame that already - # carries it is refused (#454). 78 adds county_vintage.py and - # puma_county_vintage.py, which write county_fips_2023, and - # block_ladder_sources.py, whose CT_STATE_FIPS they import (#1376). - # All are classified in _OTHER_US_RUNTIME_MODULES and scanned below - # like every other reached module. - assert len(runtime_graph) == 78, ( - f"{tool.name} must reach the pinned 78-module runtime graph; " + # carries it is refused (#454), plus acs_release_predictors.py and + # asec_census_person_columns.py, reached through the shared native ACS + # occupation/work-experience crosswalk (#719), and county_vintage.py and + # puma_county_vintage.py, which write county_fips_2023, with + # block_ladder_sources.py, whose CT_STATE_FIPS they import (#1376). All + # are classified by the runtime registries and scanned below like every + # reached module. + assert len(runtime_graph) == 80, ( + f"{tool.name} must reach the pinned 80-module runtime graph; " f"reached {len(runtime_graph)}" ) assert not missing_modules, ( diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_work_experience_properties.py b/packages/microcosm-build/tests/engine_free/us/test_us_work_experience_properties.py new file mode 100644 index 0000000000..1cd453413e --- /dev/null +++ b/packages/microcosm-build/tests/engine_free/us/test_us_work_experience_properties.py @@ -0,0 +1,242 @@ +"""Property and differential tests for the #719 work-experience columns. + +Invariants, for every input the three producers accept: + +* **Nesting.** ``WEMIND == WEIND_TO_WEMIND[WEIND]``: the major industry group + is a function of the detailed one. +* **Forward worker identity.** ``WKSWORK > 0`` implies ``WEIND`` in 1--22 + (someone who worked last year has a worker industry code). Holds on every + spine. +* **Converse worker identity (ASEC only).** ``WEIND`` in 1--22 implies + ``WKSWORK > 0``. ACS ``INDP`` covers a job held in the past five years, so + the release carry deliberately does not check it (intended asymmetry). +* **Round trip.** The release carry emits the inputs unchanged as + ``detailed_industry_recode``/``major_industry_recode`` and + ``worked_last_year == (WKSWORK > 0)``. +* **ACS universe.** Below age 16 every ACS output is 0; a blank ``INDP`` from + age 16 is the did-not-work group 23; an observed ``INDP`` maps through + ``ACS_INDP_TO_WEIND``; ``WKSWORK`` is ``WKWN`` or 0. + +The differential tests run the same generated rows through every site that +enforces an identity and require the sites to agree. +""" + +from __future__ import annotations + +import numpy as np +import pandas as pd +import pytest +from hypothesis import given, settings +from hypothesis import strategies as st + +import microcosm.build.us_runtime.acs_release_predictors as acs +from microcosm.build.us_runtime.asec_census_person_columns import ( + WEIND_TO_WEMIND, + AsecCensusPersonColumnsError, + _check_work_experience_universe, +) +from microcosm.build.us_runtime.org_wages import derive_us_org_occupation_inputs + +_WORKER = range(1, 23) +_SETTINGS = settings(max_examples=200, deadline=None) + + +def _carry_frame(weind, wemind, wkswork) -> pd.DataFrame: + rows = len(weind) + return pd.DataFrame( + { + "PRDTRACE": np.ones(rows, dtype=np.int64), + "PRDTHSP": np.zeros(rows, dtype=np.int64), + "POCCU2": np.full(rows, 8, dtype=np.int64), + "WEIND": np.asarray(weind, dtype=np.int64), + "WEMIND": np.asarray(wemind, dtype=np.int64), + "WKSWORK": np.asarray(wkswork, dtype=np.int64), + } + ) + + +@st.composite +def _asec_rows(draw) -> tuple[list[int], list[int], list[int]]: + """Rows that satisfy both worker identities and the nesting.""" + + weind = draw(st.lists(st.integers(0, 23), min_size=1, max_size=40)) + weeks = [draw(st.integers(1, 52)) if code in _WORKER else 0 for code in weind] + return weind, [WEIND_TO_WEMIND[code] for code in weind], weeks + + +@st.composite +def _arbitrary_rows(draw) -> tuple[list[int], list[int], list[int]]: + """In-range codes with no identity promised.""" + + rows = draw(st.integers(1, 40)) + return ( + draw(st.lists(st.integers(0, 23), min_size=rows, max_size=rows)), + draw(st.lists(st.integers(0, 15), min_size=rows, max_size=rows)), + draw(st.lists(st.integers(0, 52), min_size=rows, max_size=rows)), + ) + + +@st.composite +def _nested_rows(draw) -> tuple[list[int], list[int], list[int]]: + """Nested industry pairs with any weeks, so the worker identities decide. + + Independent codes almost always fail the nesting first, which would leave + the forward and converse checks unexercised. + """ + + rows = draw(st.integers(1, 40)) + weind = draw(st.lists(st.integers(0, 23), min_size=rows, max_size=rows)) + weeks = st.one_of(st.just(0), st.integers(1, 52)) + wkswork = draw(st.lists(weeks, min_size=rows, max_size=rows)) + return weind, [WEIND_TO_WEMIND[code] for code in weind], wkswork + + +def _nested(weind, wemind) -> bool: + return all(WEIND_TO_WEMIND[d] == m for d, m in zip(weind, wemind, strict=True)) + + +def _forward(weind, wkswork) -> bool: + return all(d in _WORKER for d, w in zip(weind, wkswork, strict=True) if w > 0) + + +def _converse(weind, wkswork) -> bool: + return all(w > 0 for d, w in zip(weind, wkswork, strict=True) if d in _WORKER) + + +def _carry_accepts(weind, wemind, wkswork) -> bool: + try: + derive_us_org_occupation_inputs(_carry_frame(weind, wemind, wkswork)) + except ValueError: + return False + return True + + +def _restore_accepts(weind, wemind, wkswork) -> bool: + try: + _check_work_experience_universe( + _carry_frame(weind, wemind, wkswork), "generated vintage" + ) + except AsecCensusPersonColumnsError: + return False + return True + + +@_SETTINGS +@given(_asec_rows()) +def test_release_carry_round_trips_coherent_rows(rows) -> None: + weind, wemind, wkswork = rows + out = derive_us_org_occupation_inputs(_carry_frame(weind, wemind, wkswork)) + assert out["detailed_industry_recode"].tolist() == weind + assert out["major_industry_recode"].tolist() == wemind + assert out["worked_last_year"].tolist() == [w > 0 for w in wkswork] + assert out["detailed_industry_recode"].dtype == np.int16 + assert out["worked_last_year"].dtype == bool + + +@_SETTINGS +@given(st.one_of(_arbitrary_rows(), _nested_rows())) +def test_restore_and_carry_accept_exactly_their_stated_identities(rows) -> None: + """Differential: each site accepts iff its stated identities hold.""" + + weind, wemind, wkswork = rows + nested = _nested(weind, wemind) + forward = _forward(weind, wkswork) + converse = _converse(weind, wkswork) + assert _restore_accepts(weind, wemind, wkswork) == (nested and forward and converse) + assert _carry_accepts(weind, wemind, wkswork) == (nested and forward) + # Intended asymmetry: the ASEC-only restoration is the stricter site. + if _restore_accepts(weind, wemind, wkswork): + assert _carry_accepts(weind, wemind, wkswork) + + +def _acs_template() -> dict[str, object]: + return { + "SERIALNO": "2024HU0000001", + "SPORDER": 1, + "DEAR": 2, + "DEYE": 2, + "DREM": 2, + "DPHY": 2, + "DDRS": 2, + "DOUT": 2, + "RAC1P": 1, + "HISP": 1, + "SSIP": 0.0, + "ADJINC": 1_000_000, + } + + +_OBSERVED_INDP = sorted(code for code in acs.ACS_INDP_TO_WEIND if code != 9920) + + +@st.composite +def _acs_person(draw) -> dict[str, object]: + """One ACS person inside the source universes the join admits.""" + + row = _acs_template() + age = draw(st.integers(0, 95)) + row["AGEP"] = age + if age < 16: + for column in ("DREM", "DPHY", "DDRS") if age < 5 else (): + row[column] = np.nan + if age < 15: + row["DOUT"] = np.nan + row.update(OCCP=np.nan, INDP=np.nan, WKWN=np.nan, ESR=np.nan) + return row + kind = draw(st.sampled_from(["worked", "past_five_years", "never", "unemployed"])) + if kind == "worked": + row.update( + INDP=draw(st.sampled_from(_OBSERVED_INDP)), + OCCP=1005, + ESR=draw(st.sampled_from([1, 2, 4, 5])), + WKWN=draw(st.integers(1, 52)), + ) + elif kind == "past_five_years": + row.update( + INDP=draw(st.sampled_from(_OBSERVED_INDP)), OCCP=1005, ESR=6, WKWN=np.nan + ) + elif kind == "never": + row.update(INDP=np.nan, OCCP=np.nan, ESR=6, WKWN=np.nan) + else: + row.update(INDP=9920, OCCP=9920, ESR=3, WKWN=np.nan) + return row + + +@_SETTINGS +@given(st.lists(_acs_person(), min_size=1, max_size=30)) +def test_acs_crosswalk_satisfies_every_stated_invariant(people) -> None: + joined = pd.DataFrame(people) + joined["person_source_id"] = np.arange(len(joined)) + mapped = acs._crosswalk_people(joined) + + weind = mapped["WEIND"].astype(int).tolist() + wemind = mapped["WEMIND"].astype(int).tolist() + wkswork = mapped["WKSWORK"].astype(int).tolist() + assert _nested(weind, wemind) + assert _forward(weind, wkswork) + for person, detailed, weeks in zip(people, weind, wkswork, strict=True): + if person["AGEP"] < 16: + assert (detailed, weeks) == (0, 0) + elif pd.isna(person["INDP"]): + assert detailed == 23 + else: + assert detailed == acs.ACS_INDP_TO_WEIND[int(person["INDP"])] + expected_weeks = 0 if pd.isna(person["WKWN"]) else int(person["WKWN"]) + assert weeks == expected_weeks + # The join's rows are admissible to the release carry that consumes them. + assert _carry_accepts(weind, wemind, wkswork) + + +def test_every_crosswalk_code_nests_into_its_major_group() -> None: + """Differential against the pinned nesting for all 267 table codes.""" + + for code, detailed in acs.ACS_INDP_TO_WEIND.items(): + assert detailed in _WORKER or detailed == 23, code + assert WEIND_TO_WEMIND[detailed] in range(1, 16), code + + +@pytest.mark.parametrize("weeks", [1, 52]) +def test_armed_forces_and_did_not_work_codes_bracket_the_worker_range(weeks) -> None: + assert _carry_accepts([22], [14], [weeks]) + assert not _carry_accepts([23], [15], [weeks]) + assert _carry_accepts([23], [15], [0]) diff --git a/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py b/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py index 611a4c3df2..761ff54056 100644 --- a/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py +++ b/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py @@ -345,7 +345,7 @@ def capture_equality(expected: object, actual: object) -> None: # fit_draw_next's caller-owned uniforms (microcosm#1332) move it, # and so do microcosm#454's meps_esi_premiums stage and # microcosm#1376's puma_county_vintage draw module. - "spec_sha256": "77df608c9b343219cd2717aa08af5198c0ba69b70c61691fb2459f7a886528ba", + "spec_sha256": "53fb8de63585a27fc86a10a94eeae4ca546a4ded2d528514c64e4df574b430f8", }, } diff --git a/test_support/microcosm_build/spec_engine_inventory_coverage.py b/test_support/microcosm_build/spec_engine_inventory_coverage.py index 1d051f5c4d..1764417b85 100644 --- a/test_support/microcosm_build/spec_engine_inventory_coverage.py +++ b/test_support/microcosm_build/spec_engine_inventory_coverage.py @@ -133,7 +133,7 @@ def binding() -> dict[str, object]: "stacked_checkpoint_full_components": 13, "stacked_checkpoint_pool_code_components": 19, "stacked_checkpoint_static_components": 10, - "tail_control_fields": 934, + "tail_control_fields": 937, "take_up_pipeline_steps": 28, "take_up_programs": 17, "typed_artifacts": 84, diff --git a/test_support/microcosm_build/us_org_wages.py b/test_support/microcosm_build/us_org_wages.py index 8b2c2b654c..6ce719e7f1 100644 --- a/test_support/microcosm_build/us_org_wages.py +++ b/test_support/microcosm_build/us_org_wages.py @@ -26,6 +26,7 @@ us_org_wages_stage_spec, ) from microcosm.build.us_runtime import org_wages as module +from microcosm.build.us_runtime.asec_census_person_columns import WEIND_TO_WEMIND from microcosm.frame import US_SCHEMA, Frame, WeightKind, Weights from microcosm.frame.adapters.policyengine_us import ( PolicyEngineUSVariableMetadataIndex, @@ -58,6 +59,9 @@ def _person(n: int = 1_000) -> pd.DataFrame: occupation[447:737] = 1 employment = np.zeros(n) employment[400:] = 52_000.0 + industry = np.where(occupation == 0, 0, 23).astype(np.int16) + industry[employment > 0] = 13 + industry[occupation == 52] = 22 return pd.DataFrame( { "person_id": np.arange(1, n + 1, dtype=np.int64), @@ -71,6 +75,9 @@ def _person(n: int = 1_000) -> pd.DataFrame: "PRDTRACE": np.resize(np.asarray([1, 1, 2, 4]), n), "PRDTHSP": np.arange(n) % 7 == 0, "POCCU2": occupation, + "WEIND": industry, + "WEMIND": np.asarray([WEIND_TO_WEMIND[code] for code in industry]), + "WKSWORK": np.where(employment > 0, 52, 0).astype(np.int16), "employment_income_before_lsr": employment, "self_employment_income_before_lsr": 0.0, "weekly_hours_worked_before_lsr": np.where(employment > 0, 40.0, 0.0), diff --git a/test_support/microcosm_build/us_spec_bundle.py b/test_support/microcosm_build/us_spec_bundle.py index 0f3cd267be..906c3a7440 100644 --- a/test_support/microcosm_build/us_spec_bundle.py +++ b/test_support/microcosm_build/us_spec_bundle.py @@ -195,7 +195,7 @@ def _load_generator_module(): LEGACY_COMPATIBILITY_SHA256 = { "source_stages.json": ( - "559062a1bd10efdd89b8c83da9294fea43062b8b6760524f4350dd84d63ef383" + "dc3b72982675b95d30cfabdeb5f7f4c492e6cd3788009ef3094ab4d48bfd9093" ), "support_spine.json": ( "68f37dc6ae6e0cde7ebccb53f88dd4a800e63456f838fa214ff98d1db8d815be" diff --git a/tools/_legacy/build_us_acs_multispine_base.py b/tools/_legacy/build_us_acs_multispine_base.py index e68800f319..ff3c4dfe5d 100644 --- a/tools/_legacy/build_us_acs_multispine_base.py +++ b/tools/_legacy/build_us_acs_multispine_base.py @@ -59,6 +59,9 @@ refuse_denied_frame, refuse_denied_pool_h5, ) +from microcosm.build.us_runtime.org_wages import ( + require_us_occupation_and_work_experience_inputs, +) from microcosm.build.us_runtime.puma_ladder import ( UsPumaLadder, load_us_puma_ladder, @@ -246,6 +249,7 @@ def main(argv: list[str] | None = None) -> int: base = _load_base_frame(args.base_h5) _require_benefit_participation_inputs(base) _require_donor_spm_independence_role(base) + _require_donor_work_experience(base) transfer_plan = declared_acs_transfer_target_families() _require_dense_donor_coverage( base, @@ -407,6 +411,26 @@ def _require_donor_spm_independence_role(base: Frame) -> None: raise SystemExit(f"Dense ASEC-by-PUF donor {NATIVE_SPM_ROLE} {failure}") +def _require_donor_work_experience(base: Frame) -> None: + """Keep historical donors from leaving ASEC job inputs null after pooling. + + The local lane appends the donor's original rows to native ACS rows; it + never runs the fiscal ORG carry on those donor rows. The donor must + already carry the canonical occupation, industry and working inputs. + """ + + try: + require_us_occupation_and_work_experience_inputs( + base.table("person"), label="Dense ASEC-by-PUF donor" + ) + except ValueError as error: + raise SystemExit( + f"{error} Build a fresh donor through the exact ASEC " + "WEIND/WEMIND restoration and fiscal org_wages carry first; " + "the ACS local lane preserves donor rows and cannot default these inputs." + ) from error + + def _drop_acs_group_quarters_children(engine_variables=None) -> bool: """Whether to drop the ACS group-quarters children (microcosm#1263, d1484). diff --git a/tools/build_us_acs_local_release.py b/tools/build_us_acs_local_release.py index 370aa9cdc8..4c91ce6147 100644 --- a/tools/build_us_acs_local_release.py +++ b/tools/build_us_acs_local_release.py @@ -52,7 +52,9 @@ ``microcosm.data.stored_inputs``), or on which that engine's SPM measurement would refuse a unit with no classified adult (microcosm#1263; - ``microcosm.build.us_runtime.spm_measurement_gate``), then + ``microcosm.build.us_runtime.spm_measurement_gate``), or that + lacks complete occupation, industry and worked-last-year + inputs on any person (microcosm#719), then assemble releases// with the non-default local-area manifest shape (dataset_role ``non_default_local_area``, namespace ``buildo_acs_local``, donor identity chain, the @@ -84,6 +86,10 @@ from microcosm.build.us_runtime import target_loss_weights as loss_weighting from microcosm.build.us_runtime.acs_local_hours import acs_local_hours_signal_gate from microcosm.build.us_runtime.esi_premiums import refuse_unassigned_us_esi_premiums +from microcosm.build.us_runtime.org_wages import ( + US_ORG_OCCUPATION_AND_WORK_EXPERIENCE_OUTPUT_COLUMNS, + require_us_occupation_and_work_experience_inputs, +) from microcosm.data.contract import ( US_LOCAL_AREA_SPM_MEASUREMENT_STAGE, US_LOCAL_AREA_STORED_INPUTS_STAGE, @@ -3592,6 +3598,18 @@ def do_package(args) -> dict: "finalize." ) artifact_frame = _load_staging_frame(calibrated_h5) + # Historical staging can predate the donor/native work-experience gates. + # Grade the actual final rows, even when its old summary is green. + try: + require_us_occupation_and_work_experience_inputs( + artifact_frame.table("person"), label="ACS local-area release" + ) + except ValueError as error: + raise SystemExit( + f"Refusing to package: {error} Rebuild staging with a complete " + "ASEC donor and the native ACS occupation/work-experience mapping." + ) from error + work_experience_rows = artifact_frame.n("person") # microcosm#1263: the bytes must also be measurable by the engine the # release records as built-with. spm_measurement_gate = _require_spm_measurement(artifact_frame) @@ -3606,6 +3624,14 @@ def do_package(args) -> dict: **gate_report, "gates": { **gate_report.get("gates", {}), + "work_experience_inputs": { + "passed": True, + "failures": [], + "columns": list(US_ORG_OCCUPATION_AND_WORK_EXPERIENCE_OUTPUT_COLUMNS), + "person_rows": work_experience_rows, + "artifact_sha256": h5_sha, + "checked_at_stage": "package", + }, "acs_local_hours_signal": { "passed": True, "failures": [], diff --git a/tools/build_us_release_input_coverage_manifest.py b/tools/build_us_release_input_coverage_manifest.py index b3bc625863..0744073ac1 100644 --- a/tools/build_us_release_input_coverage_manifest.py +++ b/tools/build_us_release_input_coverage_manifest.py @@ -94,6 +94,12 @@ "receives_wic", "receives_snap", "receives_tanf", + # PolicyEngine/microcosm#719: the work-experience carries the county-file + # person schema commits to (industry and a working indicator, with + # occupation), written by the org_wages release stage. + "detailed_industry_recode", + "major_industry_recode", + "worked_last_year", ) # Shared tail of the three #978 reported-receipt notes: why each is a hard @@ -206,6 +212,40 @@ "cps_carried.derive_us_cps_carried_inputs; a formula-less monthly " "boolean spm_unit input in PolicyEngine-US 2.2.1. " + _RECEIPT_INPUT_NOTE_TAIL ), + **{ + column: ( + f"{description}, carried from ASEC {source}" + + ( + " (restored before pooling by asec_census_person_columns)" + if source in {"WEIND", "WEMIND"} + else "" + ) + + " by the org_wages release stage " + "(org_wages.derive_us_org_occupation_inputs). ACS derives the " + "same input from native INDP/WKWN through the shared crosswalk " + "in the fiscal release predictor join and native ACS mapper; " + "a formula-less person input in " + "PolicyEngine-US 2.2.1. Required with NO reviewed exclusion per " + "PolicyEngine/microcosm#719: the county-file person schema " + "carries a working indicator, industry and occupation on every " + "record. Currently absent from every published US default (all " + "predate #719) — this is the intended red gate until the next " + "release is built with the carry." + ) + for column, description, source in ( + ( + "detailed_industry_recode", + "Industry of the longest job last year, 23 CPS detailed groups", + "WEIND", + ), + ( + "major_industry_recode", + "Industry of the longest job last year, 15 CPS major groups", + "WEMIND", + ), + ("worked_last_year", "Worked at any time last year", "WKSWORK > 0"), + ) + }, } QBI_INPUTS = ( @@ -1667,7 +1707,9 @@ def build_manifest() -> dict: "(PolicyEngine/microcosm#462), the three ASEC reported-receipt " "inputs receives_wic, receives_snap, receives_tanf that the ACS " "local-area transfer requires in its donor " - "(PolicyEngine/microcosm#978 option 1), and the engine's declared " + "(PolicyEngine/microcosm#978 option 1), the work-experience inputs " + "detailed_industry_recode, major_industry_recode, worked_last_year " + "(PolicyEngine/microcosm#719), and the engine's declared " "dataset source input is_spm_independent_minor_role (the " "spm_independence_role base-builder stage; " "PolicyEngine/microcosm#893). " diff --git a/tools/generate_us_bundle_from_constants.py b/tools/generate_us_bundle_from_constants.py index 1eda413959..2454909041 100644 --- a/tools/generate_us_bundle_from_constants.py +++ b/tools/generate_us_bundle_from_constants.py @@ -122,7 +122,7 @@ def _domain_builders() -> dict[str, Callable[[], object]]: # frozen files, is the forward YAML -> legacy-payload path. FROZEN_LEGACY_RESOURCE_SHA256 = { "source_stages.json": ( - "559062a1bd10efdd89b8c83da9294fea43062b8b6760524f4350dd84d63ef383" + "dc3b72982675b95d30cfabdeb5f7f4c492e6cd3788009ef3094ab4d48bfd9093" ), "support_spine.json": ( "68f37dc6ae6e0cde7ebccb53f88dd4a800e63456f838fa214ff98d1db8d815be" diff --git a/tools/repin_us_spec_engine_contracts.py b/tools/repin_us_spec_engine_contracts.py new file mode 100644 index 0000000000..cf706781e0 --- /dev/null +++ b/tools/repin_us_spec_engine_contracts.py @@ -0,0 +1,664 @@ +#!/usr/bin/env python3 +"""Recompute the reviewed US F0 migration pins after an intentional change. + +Run ``--legacy-only`` before regenerating the US bundle, then run without it +before regenerating ``tools/spec_engine_coverage.py``. Only observed hashes, +field counts, and their exact receipt and semantic test vectors are rewritten. Compiler +routes and inventory clauses remain authoritative: the final coverage command +validates the full inventory after pins are written. No microdata, build, +publication, or remote access is involved. + + uv run --no-sync python tools/repin_us_spec_engine_contracts.py --legacy-only + uv run --no-sync python tools/generate_us_bundle_from_constants.py + uv run --no-sync python tools/repin_us_spec_engine_contracts.py + uv run --no-sync python tools/spec_engine_coverage.py + +Use ``--check`` to refuse stale pins without modifying files. +""" + +from __future__ import annotations + +import argparse +import ast +import hashlib +import json +import re +import sys +import tokenize +from io import StringIO +from pathlib import Path +from tempfile import TemporaryDirectory + +ROOT = Path(__file__).resolve().parents[1] +PACKAGE = ROOT / "packages/microcosm-build" +BUILD = PACKAGE / "src/microcosm/build" +FIELD_USAGE = BUILD / "spec_engine/field_usage.py" +INVENTORY = BUILD / "spec_engine/inventory_coverage.py" +FIELD_TEST = PACKAGE / "tests/engine_free/shared/test_spec_engine_field_usage.py" +COVERAGE_TEST = PACKAGE / "tests/engine_contract/us/test_spec_engine_coverage_tool.py" + + +def _assignment(source: str, name: str) -> ast.expr: + for node in ast.parse(source).body: + if isinstance(node, ast.Assign) and any( + isinstance(target, ast.Name) and target.id == name + for target in node.targets + ): + return node.value + if isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name): + if node.target.id == name and node.value is not None: + return node.value + raise ValueError(f"missing assignment: {name}") + + +def _replace_node(source: str, node: ast.AST, value: object) -> str: + lines = source.splitlines(keepends=True) + start = sum(map(len, lines[: node.lineno - 1])) + node.col_offset + end = sum(map(len, lines[: node.end_lineno - 1])) + node.end_col_offset + return source[:start] + repr(value) + source[end:] + + +def _replace_assignment(source: str, name: str, value: object) -> str: + node = _assignment(source, name) + if ast.literal_eval(node) == value: + return source + return _replace_node(source, node, value) + + +def _replace_dictionary(source: str, name: str, values: dict) -> str: + for key, value in values.items(): + node = _assignment(source, name) + if not isinstance(node, ast.Dict): + raise TypeError(f"{name} is not a literal dictionary") + matching = [ + child + for candidate, child in zip(node.keys, node.values, strict=True) + if candidate is not None and ast.literal_eval(candidate) == key + ] + if len(matching) != 1: + raise ValueError(f"{name}: expected one key {key!r}") + if ast.literal_eval(matching[0]) == value: + continue + source = _replace_node(source, matching[0], value) + return source + + +def _replace_literals(source: str, replacements: dict) -> str: + """Replace exact Python literals, leaving comments and substrings alone.""" + + edits = [] + lines = source.splitlines(keepends=True) + for token in tokenize.generate_tokens(StringIO(source).readline): + if token.type not in {tokenize.NUMBER, tokenize.STRING}: + continue + try: + old = ast.literal_eval(token.string) + except (ValueError, SyntaxError): + continue + if old in replacements and replacements[old] != old: + start = sum(map(len, lines[: token.start[0] - 1])) + token.start[1] + end = sum(map(len, lines[: token.end[0] - 1])) + token.end[1] + edits.append((start, end, repr(replacements[old]))) + for start, end, value in reversed(edits): + source = source[:start] + value + source[end:] + return source + + +def _save(path: Path, source: str, *, check: bool) -> None: + if source == path.read_text(): + return + relative = path.relative_to(ROOT) + if check: + raise SystemExit(f"stale US F0 pins: {relative}") + path.write_text(source) + print(f"updated {relative}") + + +def _replace_length_assertions(source: str, expected: dict[str, int]) -> str: + matches = {} + for assertion in ast.walk(ast.parse(source)): + if not isinstance(assertion, ast.Assert): + continue + node = assertion.test + if not ( + isinstance(node, ast.Compare) + and len(node.ops) == 1 + and isinstance(node.ops[0], ast.Eq) + and isinstance(node.left, ast.Call) + and isinstance(node.left.func, ast.Name) + and node.left.func.id == "len" + and len(node.left.args) == 1 + and len(node.comparators) == 1 + ): + continue + target = ast.unparse(node.left.args[0]) + if target in expected: + if target in matches: + raise ValueError(f"duplicate inventory assertion: {target}") + matches[target] = node.comparators[0] + if set(matches) != set(expected): + raise ValueError("typed inventory assertion targets changed") + for target, node in sorted( + matches.items(), key=lambda row: row[1].lineno, reverse=True + ): + if ast.literal_eval(node) != expected[target]: + source = _replace_node(source, node, expected[target]) + return source + + +def _replace_literal_assertions(source: str, expected: dict[str, str]) -> str: + """Refresh each exact receipt expression once, retaining literal assertions.""" + + matches = {} + for node in ast.walk(ast.parse(source)): + if not ( + isinstance(node, ast.Assert) + and isinstance(node.test, ast.Compare) + and len(node.test.ops) == 1 + and isinstance(node.test.ops[0], ast.Eq) + and len(node.test.comparators) == 1 + ): + continue + target = ast.unparse(node.test.left) + if target in expected: + if target in matches: + raise ValueError(f"duplicate receipt assertion: {target}") + matches[target] = node.test.comparators[0] + if set(matches) != set(expected): + raise ValueError("receipt assertion targets changed") + for target, node in sorted( + matches.items(), key=lambda row: row[1].lineno, reverse=True + ): + old = ast.literal_eval(node) + if not isinstance(old, str): + raise ValueError(f"receipt assertion is not a literal string: {target}") + if old != expected[target]: + source = _replace_node(source, node, expected[target]) + return source + + +def _replace_asserted_dictionary_value( + source: str, target: str, keys: tuple[str, ...], value: str +) -> str: + """Refresh a nested literal pin inside one complete dictionary assertion.""" + + matching = [ + node.test.comparators[0] + for node in ast.walk(ast.parse(source)) + if isinstance(node, ast.Assert) + and isinstance(node.test, ast.Compare) + and len(node.test.ops) == 1 + and isinstance(node.test.ops[0], ast.Eq) + and len(node.test.comparators) == 1 + and ast.unparse(node.test.left) == target + ] + if len(matching) != 1: + raise ValueError(f"expected one asserted dictionary: {target}") + node = matching[0] + for key in keys: + if not isinstance(node, ast.Dict): + raise ValueError(f"receipt key is not in a literal dictionary: {key}") + matching = [ + child + for candidate, child in zip(node.keys, node.values, strict=True) + if candidate is not None and ast.literal_eval(candidate) == key + ] + if len(matching) != 1: + raise ValueError(f"expected one receipt dictionary key: {key}") + node = matching[0] + old = ast.literal_eval(node) + if not isinstance(old, str): + raise ValueError("receipt dictionary pin is not a literal string") + return source if old == value else _replace_node(source, node, value) + + +def _replace_asserted_counts(source: str, target: str, observed: dict) -> str: + """Refresh every entry of one asserted count dictionary by its key. + + Keying by name rather than by the old value keeps a merge that already + moved the evidence JSON, but left a conflicted test on the other side, + from hiding a stale literal. The asserted keys must match the observed + ledger exactly. + """ + + matching = [ + node.test.comparators[0] + for node in ast.walk(ast.parse(source)) + if isinstance(node, ast.Assert) + and isinstance(node.test, ast.Compare) + and len(node.test.ops) == 1 + and isinstance(node.test.ops[0], ast.Eq) + and len(node.test.comparators) == 1 + and ast.unparse(node.test.left) == target + ] + if len(matching) != 1 or not isinstance(matching[0], ast.Dict): + raise ValueError(f"expected one asserted count dictionary: {target}") + node = matching[0] + if None in node.keys: + raise ValueError(f"{target}: dictionary unpacking is not a count pin") + keys = [ast.literal_eval(key) for key in node.keys] + if len(set(keys)) != len(keys) or set(keys) != set(observed): + raise ValueError( + f"{target}: asserted keys {sorted(keys)} differ from the observed " + f"{sorted(observed)}" + ) + stale = [ + (value, observed[key]) + for key, value in zip(keys, node.values, strict=True) + if ast.literal_eval(value) != observed[key] + ] + # Rewrite from the end so earlier offsets stay valid. + for value, new in sorted( + stale, key=lambda item: (item[0].lineno, item[0].col_offset), reverse=True + ): + source = _replace_node(source, value, new) + return source + + +def _replace_asserted_scalar(source: str, target: str, value: int) -> str: + """Refresh the integer that ``assert == ... == `` pins. + + Matching the asserted expression, not the old number, refreshes a stale + literal even when the module pin it was copied from has already moved. + """ + + matching = [ + node.test + for node in ast.walk(ast.parse(source)) + if isinstance(node, ast.Assert) + and isinstance(node.test, ast.Compare) + and all(isinstance(op, ast.Eq) for op in node.test.ops) + and ast.unparse(node.test.left) == target + ] + if len(matching) != 1: + raise ValueError(f"expected one scalar assertion: {target}") + pin = matching[0].comparators[-1] + if not (isinstance(pin, ast.Constant) and type(pin.value) is int): + raise ValueError(f"{target}: the asserted pin is not an integer literal") + return source if pin.value == value else _replace_node(source, pin, value) + + +def _replace_unclaimed_count(source: str, count: int) -> str: + """Refresh the one ``" unclaimed normative field"`` expectation. + + Every field has exactly one primary claim, so dropping a claim leaves + exactly its observed expansion unclaimed. + """ + + pattern = re.compile(r"\d+ unclaimed normative field") + hits = [] + for token in tokenize.generate_tokens(StringIO(source).readline): + if token.type != tokenize.STRING: + continue + try: + text = ast.literal_eval(token.string) + except (ValueError, SyntaxError): + continue + if isinstance(text, str) and pattern.fullmatch(text): + hits.append(token) + if len(hits) != 1: + raise ValueError(f"expected one unclaimed-field expectation, found {len(hits)}") + (token,) = hits + lines = source.splitlines(keepends=True) + start = sum(map(len, lines[: token.start[0] - 1])) + token.start[1] + end = sum(map(len, lines[: token.end[0] - 1])) + token.end[1] + literal = re.sub(r"\d+", str(count), token.string, count=1) + return source[:start] + literal + source[end:] + + +def fixture_calibration_contract() -> dict: + """Build the calibration contract exactly as its semantics test fixture does. + + The golden projection hashes the constants builder's raw contract. The + compiled bundle's ``to_wire()`` has been through the loader, which writes + integral floats as integers (``5.0`` becomes ``5``), so its canonical bytes + and digest differ even where every value agrees. + """ + + from microcosm.build.spec_engine import scoped_take_up_manifest_program_bindings + + frozen = json.loads((BUILD / "us/take_up_contract.json").read_text()) + bindings = tuple( + (row["variable"], row["entity"], row["populace_treatment"]) + for row in frozen["programs"] + ) + with scoped_take_up_manifest_program_bindings(bindings): + from tools.us_bundle_generation.contracts import build_calibration_contract + + return build_calibration_contract() + + +def _semantic_test_pins(spec, observed_hashes: dict, rendered: dict) -> None: + """Derive the existing test vectors through their production projections.""" + + from microcosm.build.spec_engine import ( + KernelRegistry, + project_legacy_calibration_contract, + ) + from microcosm.build.spec_engine.canonical import sha256_json + from microcosm.build.spec_engine.legacy_adapter import compile_to_legacy_payload + from microcosm.build.spec_engine.loader import load_bundle + from test_support.microcosm_build.spec_engine_loader import ( + SELECTION_KERNEL_IDS, + _rich_minimal, + ) + + legacy = compile_to_legacy_payload(spec) + imputation = legacy["imputation"] + schedule = imputation["late_producer_schedule_receipt"] + receipts = { + "authority": legacy["stacked_authority_receipt"]["sha256"], + "late_schedule": schedule["schedule_sha256"], + "ownership": imputation["overlap_ownership"]["sha256"], + "take_up": legacy["take_up_contract_identity"]["resource_sha256"], + } + for key, value in receipts.items(): + if value != observed_hashes[key]: + raise ValueError(f"{key}: compiled and inventory receipt digests differ") + + # Use the identical fixture builder, country, note, store and selected + # registry as the golden test; no test assertion or constant is consulted. + with TemporaryDirectory(prefix="microcosm-spec-pin-") as directory: + root = _rich_minimal(Path(directory) / "xx", note="first", store="local:a") + minimal = load_bundle( + root, kernel_registry=KernelRegistry.from_ids(SELECTION_KERNEL_IDS) + ) + path = PACKAGE / "tests/engine_free/shared/test_spec_engine_loader.py" + rendered[path] = _replace_literal_assertions( + path.read_text(), {"first.spec_sha256": minimal.spec_sha256} + ) + + calibration = fixture_calibration_contract() + path = ( + PACKAGE / "tests/engine_free/shared/test_spec_engine_calibration_semantics.py" + ) + rendered[path] = _replace_assignment( + path.read_text(), + "CANONICAL_LEGACY_PROJECTION_SHA256", + sha256_json(project_legacy_calibration_contract(calibration)), + ) + + path = PACKAGE / "tests/engine_free/shared/test_spec_engine_compiler_ir.py" + rendered[path] = _replace_assignment( + rendered.get(path, path.read_text()), + "US_SCHEDULE_SHA256", + receipts["late_schedule"], + ) + assertion_files = { + "engine_free/shared/test_spec_engine_legacy_adapter.py": { + "legacy_payload['stacked_authority_receipt']['sha256']": receipts[ + "authority" + ], + "imputation['late_producer_schedule_receipt']['schedule_sha256']": receipts[ + "late_schedule" + ], + "imputation['overlap_ownership']['sha256']": receipts["ownership"], + "legacy_payload['take_up_contract_identity']['resource_sha256']": receipts[ + "take_up" + ], + }, + "engine_free/shared/test_spec_engine_imputation_semantics.py": { + "projected['late_producer_schedule_receipt']['schedule_sha256']": receipts[ + "late_schedule" + ], + "projected['overlap_ownership']['sha256']": receipts["ownership"], + }, + "engine_free/us/test_us_spec_bundle.py": { + "compiled_schedule['schedule_sha256']": receipts["late_schedule"], + "compiled_schedule['payload_sha256']": schedule["payload_sha256"], + }, + } + for relative, expected in assertion_files.items(): + path = PACKAGE / "tests" / relative + rendered[path] = _replace_literal_assertions( + rendered.get(path, path.read_text()), expected + ) + path = PACKAGE / "tests/engine_workflow/us/test_us_multispine_pool_tool.py" + rendered[path] = _replace_asserted_dictionary_value( + path.read_text(), + "run_config", + ("spec_binding", "spec_sha256"), + spec.spec_sha256, + ) + + +def _legacy_pins(*, check: bool) -> None: + generator = ROOT / "tools/generate_us_bundle_from_constants.py" + observed = hashlib.sha256( + (BUILD / "us/source_stages.json").read_bytes() + ).hexdigest() + for path, name in ( + (generator, "FROZEN_LEGACY_RESOURCE_SHA256"), + ( + ROOT / "test_support/microcosm_build/us_spec_bundle.py", + "LEGACY_COMPATIBILITY_SHA256", + ), + ): + source = _replace_dictionary( + path.read_text(), name, {"source_stages.json": observed} + ) + _save(path, source, check=check) + path = PACKAGE / "tests/engine_free/us/test_us_bundle_core_contracts.py" + source = path.read_text() + pins = [] + for node in ast.walk(ast.parse(source)): + if not isinstance(node, ast.Dict): + continue + pairs = { + key.value: value + for key, value in zip(node.keys, node.values, strict=True) + if isinstance(key, ast.Constant) and isinstance(key.value, str) + } + identifier = pairs.get("id") + if isinstance(identifier, ast.Constant) and identifier.value == "source_stages": + pins.append(pairs["sha256"]) + if len(pins) != 1: + raise ValueError("expected one source-stage asset test pin") + if ast.literal_eval(pins[0]) != observed: + source = _replace_node(source, pins[0], observed) + _save(path, source, check=check) + + +def _compiler_pins(*, check: bool) -> None: + # Imports follow the legacy-only bootstrap so stale generation-0 pins cannot + # prevent the bootstrap itself from reading and hashing its local resource. + sys.path.insert(0, str(ROOT)) + from microcosm.build.spec_engine import field_usage, inventory_coverage + from microcosm.build.spec_engine.canonical import sha256_json + from microcosm.build.spec_engine.loader import load_bundle + + spec = load_bundle("us") + expansions = field_usage.claim_expansion_inventory(spec) + sources = field_usage.configuration_sources(spec) + pointers = [path for path, _ in field_usage._terminal_rows(sources, path="")] + counts = { + "EXPECTED_AUTHORED_FIELD_COUNT": sum( + p.startswith("/authored/") for p in pointers + ), + "EXPECTED_RESOLVED_BINDING_FIELD_COUNT": sum( + p.startswith("/resolved/") for p in pointers + ), + "EXPECTED_CONFIGURATION_FIELD_COUNT": len(pointers), + } + source = FIELD_USAGE.read_text() + source = _replace_dictionary(source, "_PINS", expansions) + for name, value in counts.items(): + source = _replace_assignment(source, name, value) + rendered = {FIELD_USAGE: source} + + # Derive an independently validated ledger using the observed expansions. + # Every verifier and exact single-owner check still runs; only stale pin + # values and totals are supplied from the source just inspected above. + from dataclasses import replace + + claims = tuple( + replace( + claim, + expected_pointer_count=expansions[claim.id][0], + expected_pointer_sha256=expansions[claim.id][1], + ) + for claim in field_usage.default_usage_claims() + ) + ledger = field_usage.build_field_usage_ledger( + spec, claims=claims, enforce_expected_total=False + ) + coverage_tool = ROOT / "tools/spec_engine_coverage.py" + source = _replace_assignment( + coverage_tool.read_text(), + "EXPECTED_POINTER_INVENTORY_SHA256", + sha256_json(sorted(pointers)), + ) + rendered[coverage_tool] = source + + report = inventory_coverage.build_inventory_coverage(spec) + observed_hashes = {} + # Match each named pin through its exact expected receipt field. Pins may + # appear in multiple receipts; all independently observed values must agree. + for key, old in inventory_coverage.EXPECTED_HASHES.items(): + observed = set() + for item in report["items"].values(): + expected = item["expected"] + if not isinstance(expected, dict): + continue + for field, value in expected.items(): + if value == old: + actual_field = ( + "families_sha256" + if field == "sha256" and "families_sha256" in item["observed"] + else field + ) + actual = item["observed"].get(actual_field) + if isinstance(actual, str): + observed.add(actual) + if len(observed) != 1: + raise ValueError(f"{key}: no unambiguous observed digest: {observed}") + observed_hashes[key] = observed.pop() + source = _replace_dictionary( + INVENTORY.read_text(), "EXPECTED_HASHES", observed_hashes + ) + source = _replace_dictionary(source, "EXPECTED_INVENTORY_COUNTS", report["counts"]) + old_tail = inventory_coverage.EXPECTED_INVENTORY_COUNTS["tail_control_fields"] + new_tail = report["counts"]["tail_control_fields"] + source = _replace_literals(source, {old_tail: new_tail}) + rendered[INVENTORY] = source + + # Tests deliberately pin literal counts, so derive their corresponding + # expectations from the same validated ledger rather than weakening them. + # Every pin is found by its asserted expression, never by its old value, + # so a merge that moved one copy of a count cannot hide a stale other. + total = counts["EXPECTED_CONFIGURATION_FIELD_COUNT"] + scalars = { + FIELD_TEST: { + "len(field_ledger.fields)": total, + "len({field.pointer for field in field_ledger.fields})": total, + }, + COVERAGE_TEST: { + "fields['configuration_field_count']": total, + "fields['authored_normative_field_count']": counts[ + "EXPECTED_AUTHORED_FIELD_COUNT" + ], + "fields['resolved_binding_field_count']": counts[ + "EXPECTED_RESOLVED_BINDING_FIELD_COUNT" + ], + "fields['consumed_field_count']": total, + "fields['claim_count']": len(ledger.claims), + "inventory['required_item_count']": report["required_item_count"], + "inventory['covered_item_count']": report["covered_item_count"], + **{ + f"inventory['counts'][{key!r}]": report["counts"][key] + for key in ( + "producer_inputs", + "ownership_rows", + "tail_control_fields", + "seed_owner_bindings", + ) + }, + }, + } + for path, expected in scalars.items(): + source = path.read_text() + for target, value in expected.items(): + source = _replace_asserted_scalar(source, target, value) + rendered[path] = source + source_counts = { + "authored": counts["EXPECTED_AUTHORED_FIELD_COUNT"], + "resolved_bindings": counts["EXPECTED_RESOLVED_BINDING_FIELD_COUNT"], + } + for path, prefix, breakdowns in ( + ( + FIELD_TEST, + "field_ledger.", + {"source_counts": source_counts}, + ), + (COVERAGE_TEST, "fields", {}), + ): + breakdowns = { + **breakdowns, + "mode_counts": dict(ledger.mode_counts), + "generation0_effect_counts": dict(ledger.generation0_effect_counts), + } + for name, observed in breakdowns.items(): + target = ( + f"{prefix}{name}" if prefix.endswith(".") else f"{prefix}[{name!r}]" + ) + rendered[path] = _replace_asserted_counts(rendered[path], target, observed) + # The identity test drops the selection claim and expects its fields back + # as unclaimed. + rendered[FIELD_TEST] = _replace_unclaimed_count( + rendered[FIELD_TEST], expansions["selection"][0] + ) + observed_counts = report["counts"] + column_count = observed_counts["typed_columns"] + artifact_count = observed_counts["typed_artifacts"] + scope_count = observed_counts["typed_scopes"] + assertion_files = { + "engine_free/us/test_us_spec_bundle.py": { + "catalogs['columns']": column_count, + "resolved_us_spec.columns": column_count, + "resolved_us_spec.scopes": scope_count, + }, + "engine_free/shared/test_spec_engine_compiler_ir.py": { + "inventory['entities']": observed_counts["typed_entities"], + "inventory['columns']": column_count, + "inventory['artifacts']": artifact_count, + "inventory['scopes']": scope_count, + "inventory['references']": observed_counts["resolved_references"], + }, + "engine_contract/us/test_spec_engine_typed_closure.py": { + "typed_result.columns": column_count, + "typed_result.artifacts": artifact_count, + "typed_result.scopes": scope_count, + }, + "engine_contract/us/test_us_bundle_core_contracts.py": { + "columns": column_count, + "{row['key'] for row in columns}": column_count, + "resolved.columns": column_count, + }, + } + for relative, expected in assertion_files.items(): + path = PACKAGE / "tests" / relative + rendered[path] = _replace_length_assertions(path.read_text(), expected) + path = ROOT / "test_support/microcosm_build/spec_engine_inventory_coverage.py" + rendered[path] = _replace_dictionary( + path.read_text(), "EXPECTED_COUNTS", observed_counts + ) + _semantic_test_pins(spec, observed_hashes, rendered) + # Finish all derivations before writing so a failed compiler route or an + # ambiguous inventory observation cannot leave partially refreshed pins. + for path, source in rendered.items(): + _save(path, source, check=check) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--legacy-only", action="store_true") + parser.add_argument("--check", action="store_true") + args = parser.parse_args() + _legacy_pins(check=args.check) + if not args.legacy_only: + _compiler_pins(check=args.check) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/spec_engine_coverage.py b/tools/spec_engine_coverage.py index 557999f172..7f5e867905 100644 --- a/tools/spec_engine_coverage.py +++ b/tools/spec_engine_coverage.py @@ -41,7 +41,7 @@ REPORT_SCHEMA_VERSION = 3 EXPECTED_POINTER_INVENTORY_SHA256 = ( - "a0da3562ad5559cbbcdf5ee14ebd21db2f067c90e7e470e24d4e3762d7b90848" + "0a2951f4484a0dd9459a474505b5a0caa0216b80ad13009a286dd85e2165e7e8" ) DEFAULT_REPORT_PATH = ( Path(__file__).resolve().parents[1]