Retire the legacy target store on the Thesis ledger branch

.github/workflows/ci.yml

@@ -88,42 +88,23 @@ jobs:
           arch
           tests/test_arch_bundle.py
           db/cli.py
-          db/etl_snap.py
-          db/etl_aca_enrollment.py
-          db/etl_medicaid.py
-          db/etl_ssi.py
           db/pe_source_inventory.py
           db/schema.py
           db/supabase_client.py
           db/source_files.py
           micro/us/census/download_cps.py
           micro/us/entities.py
-          micro/us/hierarchy_validation.py
-          micro/us/hierarchical_pipeline.py
-          micro/us/pipeline.py
           micro/us/policyengine.py
-          micro/us/targets.py
-          micro/us/validation_dashboard.py
-          calibration/constraints.py
-          calibration/targets.py
           scripts/build_series_catalog.py
           tests/test_build_series_catalog.py
           tests/test_arch_facts.py
           tests/test_arch_namespace.py
           tests/test_arch_normalization.py
-          tests/test_etl_snap.py
-          tests/test_etl_soi.py
-          tests/test_etl_aca_enrollment.py
-          tests/test_etl_medicaid.py
-          tests/test_etl_ssi.py
+          tests/test_legacy_targets_retired.py
           tests/test_source_files.py
           tests/test_microplex_entities.py
-          tests/test_microplex_hierarchy_validation.py
-          tests/test_microplex_target_adapter.py
-          tests/test_microplex_pipeline.py
           tests/test_microplex_policyengine.py
           tests/test_download_cps.py
-          tests/test_hierarchical_pipeline.py
           tests/test_supabase_client.py
           tests/test_policyengine_ledger.py
           tests/test_release_chain.py
@@ -140,31 +121,21 @@ jobs:
           tests/test_arch_facts.py
           tests/test_arch_namespace.py
           tests/test_arch_normalization.py
-          tests/test_etl_snap.py
-          tests/test_etl_soi.py
-          tests/test_etl_aca_enrollment.py
-          tests/test_etl_medicaid.py
-          tests/test_etl_ssi.py
+          tests/test_legacy_targets_retired.py
           tests/test_source_files.py
-          tests/test_calibration.py
           tests/test_microplex_entities.py
-          tests/test_microplex_hierarchy_validation.py
-          tests/test_microplex_target_adapter.py
-          tests/test_microplex_pipeline.py
           tests/test_microplex_policyengine.py
           tests/test_download_cps.py
-          tests/test_hierarchical_pipeline.py
           tests/test_supabase_client.py
           tests/test_policyengine_ledger.py
           tests/test_release_chain.py
           tests/test_thesis_append_adversarial.py
           -q
 
-      - name: Build target input database
+      - name: Smoke-test the source-artifact database CLI
         run: |
-          uv run arch --db /tmp/arch-targets-ci.db init
-          uv run arch --db /tmp/arch-targets-ci.db load all
-          uv run arch --db /tmp/arch-targets-ci.db stats
+          uv run arch --db /tmp/arch-source-ci.db init
+          uv run arch --db /tmp/arch-source-ci.db source-stats
 
       - name: Build wheel
         run: |
@@ -178,18 +149,28 @@ jobs:
           uv pip install --python /tmp/arch-wheel-smoke/bin/python "$wheel_path"
           cd /tmp
           /tmp/arch-wheel-smoke/bin/python - <<'PY'
+          import importlib.util
+
           import arch
-          import arch.targets
-          from micro.us.district import DistrictMicroplex
-          from micro.us.pipeline import run_pipeline
-          from micro.us.targets import load_microplex_targets
+          from arch.jurisdictions.us.soi import AGI_BRACKETS
+          from arch.normalization import convert_units
+          from micro.us.entities import build_microplex_entities
           from microplex import ConditionalMAF
 
           assert arch.__name__ == "arch"
-          assert DistrictMicroplex is not None
-          assert run_pipeline is not None
-          assert load_microplex_targets is not None
+          assert len(AGI_BRACKETS) == 19
+          assert convert_units is not None
+          assert build_microplex_entities is not None
           assert ConditionalMAF is not None
+          # The legacy target store is not shipped.
+          for retired in (
+              "arch.targets",
+              "calibration",
+              "db.etl_soi",
+              "micro.us.pipeline",
+              "micro.us.targets",
+          ):
+              assert importlib.util.find_spec(retired) is None, retired
           PY
           /tmp/arch-wheel-smoke/bin/arch --help >/dev/null
 

README.md

@@ -91,22 +91,20 @@ arch/
 │   ├── sources/             # Source lineage helpers
 │   ├── facts/               # Source-backed facts
 │   ├── normalization/       # Low-assumption representation helpers
-│   ├── targets/             # Target input schema, client, loaders
 │   ├── microdata/           # Microdata registry, ingestion, queries
 │   ├── jurisdictions/       # Temporary in-repo jurisdiction source prototypes
-├── db/                      # SQLModel persistence and source loaders
-│   ├── schema.py            # SQLModel: Target, Stratum, StratumConstraint
+├── db/                      # Source-artifact store and packaged source files
+│   ├── schema.py            # SQLModel: SourceArtifact, SourceTable, SourceRow
 │   ├── supabase_client.py   # Supabase client helpers
-│   └── etl_*.py             # Source-specific ETL pipelines
-├── micro/                   # Microplex consumers of Arch records
-├── calibration/             # Calibration target adapters and constraints
+│   └── data/                # Publisher files that source packages read
+├── micro/                   # Microplex microdata utilities
 ├── data/                    # Cached data files
 └── docs/                    # Architecture and source documentation
 ```
 
 New code should prefer `arch.sources`, `arch.facts`, `arch.normalization`,
-`arch.targets`, and `arch.microdata`. Microplex-specific target composition
-and calibration code belongs under `micro/`.
+and `arch.microdata`. Target composition and calibration belong in Microplex,
+not in this repository.
 
 ## Quick Start
 
@@ -120,15 +118,7 @@ cd arch
 pip install -e ".[dev]"
 ```
 
-### 2. Initialize and Load Legacy Target Inputs
-
-```bash
-arch init
-arch load soi --years 2021
-arch stats
-```
-
-### 3. Validate Fixture Facts
+### 2. Validate Fixture Facts
 
 The standalone Arch fact harness validates JSONL aggregate facts and emits a
 JSON report with fact counts, QA counts, warnings, and validation errors:
@@ -429,7 +419,7 @@ same `--axiom-cli` and `--axiom-root` flags, plus
 `--require-axiom-validation` when skipped concept checks should fail agent
 acceptance.
 
-### 4. Run Arch Explorer
+### 3. Run Arch Explorer
 
 Arch Explorer is a Next/Tailwind app that reads the fixture fact JSONL and
 source-cell JSONL, then shows aggregate facts, source-cell lineage, and
@@ -451,37 +441,27 @@ By default, the workbench reads the current local suite outputs at
 ARCH_EXPLORER_DATA_DIRS=/tmp/arch-build-a,/tmp/arch-build-b npm run dev -- --port 3090
 ```
 
-### 5. Query Target Inputs in Python
-
-```python
-from arch.targets import DataSource, Target, TargetType
-from calibration.targets import get_targets
-
-target_inputs = get_targets(
-    jurisdiction="us",
-    year=2021,
-    sources=["irs-soi"],
-)
-```
-
-### 6. Query Microdata
+### 4. Query Microdata
 
 ```python
 from arch.microdata import query_cps_asec
 
 persons = query_cps_asec(year=2024, table_type="person", limit=10_000)
 ```
 
-## Target Input Schema
-
-Target inputs use a three-table schema:
-
-- **strata**: Population subgroups, such as California filers with AGI between
-  $50k and $75k.
-- **stratum_constraints**: Rules defining each stratum.
-- **targets**: Source-published aggregate values linked to strata.
-
-These are inputs to Microplex target composition. Microplex owns the active,
+## Retired legacy target tables
+
+This branch used to load hand-written ETLs (`db/etl_*.py`) into a local SQLite
+`targets` / `strata` / `stratum_constraints` schema (`arch load`, `arch stats`,
+`arch query`), read a Supabase copy through `arch.targets.query_targets`, and
+calibrate microdata to those rows in `calibration/` and `micro/us/pipeline.py`.
+Nothing outside that stack read the tables, and the audit in
+PolicyEngine/chronicle#348 (`docs/us-legacy-etl-cleanup.md` there) found that
+several of the US loaders wrote invented or untraceable values, so the whole
+stack was removed. Main dropped `calibration/` and `micro/` in #68 and the UK
+loaders in #181 and #186, and is retiring the US loaders and target tables in
+#348 and its follow-on. Source-backed facts reach consumers through source
+packages and the bundle (`arch build-bundle`). Microplex owns the active,
 reconciled, aged target sets used for calibration.
 
 ## Arch Facts And Microplex Targets
@@ -498,8 +478,8 @@ adjusted or projected series.
 
 ```python
 from arch.facts import SourceFact
-from arch.targets import DataSource, Jurisdiction, TargetType
-from arch.normalization import as_target, convert_units
+from arch.normalization import convert_units
+from db.schema import DataSource, Jurisdiction
 
 fact = SourceFact(
     name="snap_households",
@@ -510,11 +490,7 @@ fact = SourceFact(
     jurisdiction=Jurisdiction.US,
 )
 
-target_input = as_target(
-    convert_units(fact, 1000, "count"),
-    target_type=TargetType.COUNT,
-    stratum_name="US SNAP Households",
-)
+normalized_fact = convert_units(fact, 1000, "count")
 ```
 
 ## Current Coverage
@@ -527,18 +503,30 @@ target_input = as_target(
 | US IRS PUF | 33 | Tax return sample |
 | UK FRS | 29 | DWP household survey |
 
-### Aggregate Facts And Target Inputs
-
-| Source | Coverage | Description |
-|--------|----------|-------------|
-| IRS SOI | National, state, AGI brackets | Tax return aggregates |
-| Census | Demographics, poverty, districts | Population statistics |
-| BLS | Labor market and price data | Employment and index series |
-| CBO | Federal projections | Budget and economic projections |
-| SSA/SSI | National and state programs | Social Security data |
-| SNAP | State-level | Food assistance |
-| CMS | Medicaid and ACA enrollment | Health coverage |
-| HMRC/ONS/OBR | UK tax, population, projections | UK official statistics |
+### Aggregate facts
+
+Facts in the default 2023 bundle (`arch build-bundle --year 2023`), by source,
+as `coverage.json` reports them and `tests/test_arch_bundle.py` pins them:
+
+| Source | Facts |
+|--------|------:|
+| `irs_soi` | 6,911 |
+| `census_pep` | 988 |
+| `cms_medicaid` | 515 |
+| `census_acs` | 468 |
+| `bea` | 443 |
+| `ssa` | 422 |
+| `usda_snap` | 216 |
+| `hhs_acf_tanf` | 110 |
+| `census_population_projections` | 86 |
+| `kff` | 52 |
+| `census_stc` | 46 |
+| `cbo` | 6 |
+| `hhs_acf_liheap` | 2 |
+| `cms_medicare` | 1 |
+| `cms_nhe` | 1 |
+| `federal_reserve` | 1 |
+| **Total** | **10,268** |
 
 ## Boundaries
 

arch/__init__.py

@@ -23,5 +23,4 @@
     "store",
     "sources",
     "suite",
-    "targets",
 ]

arch/cli.py

@@ -8,7 +8,7 @@
 
 
 def main() -> None:
-    """Dispatch Arch-owned harness commands or the legacy DB CLI."""
+    """Dispatch Arch-owned harness commands or the source-artifact DB CLI."""
     if sys.argv[1:2] in (
         ["bootstrap-r2"],
         ["build-bundle"],

arch/jurisdictions/uk/__init__.py

@@ -1,9 +1,7 @@
-"""United Kingdom Arch target loaders."""
+"""United Kingdom Arch source builders.
 
-from arch.targets.loaders import (
-    load_hmrc_targets,
-    load_obr_targets,
-    load_ons_targets,
-)
+The legacy HMRC, OBR and ONS DB target loaders were retired, so this module
+exports no loaders.
+"""
 
-__all__ = ["load_hmrc_targets", "load_obr_targets", "load_ons_targets"]
+__all__: list[str] = []

arch/jurisdictions/us/__init__.py

@@ -1,35 +1,7 @@
-"""United States Arch target loaders."""
+"""United States Arch source builders.
 
-from arch.targets.loaders import (
-    load_aca_enrollment_targets,
-    load_bls_targets,
-    load_cbo_targets,
-    load_census_targets,
-    load_cps_targets,
-    load_medicaid_targets,
-    load_snap_targets,
-    load_soi_targets,
-    load_soi_credits_targets,
-    load_soi_deductions_targets,
-    load_soi_income_sources_targets,
-    load_soi_state_targets,
-    load_ssa_targets,
-    load_ssi_targets,
-)
+US measures are source packages. The legacy DB target loaders were retired, so
+this module exports no loaders.
+"""
 
-__all__ = [
-    "load_aca_enrollment_targets",
-    "load_bls_targets",
-    "load_cbo_targets",
-    "load_census_targets",
-    "load_cps_targets",
-    "load_medicaid_targets",
-    "load_snap_targets",
-    "load_soi_targets",
-    "load_soi_credits_targets",
-    "load_soi_deductions_targets",
-    "load_soi_income_sources_targets",
-    "load_soi_state_targets",
-    "load_ssa_targets",
-    "load_ssi_targets",
-]
+__all__: list[str] = []