Add FSA portfolio and ACS S1501 attainment source packages

changelog.d/310.added.md

@@ -0,0 +1 @@
+Add Federal Student Aid portfolio source packages for chronicle#310: portfolio summary, loan type and repayment plan (FY2026 Q1 to Q3), borrower age and debt size (FY2026 Q1 and Q2), and borrower location (March 31, 2026), 461 administrative facts stored at their published levels; and Census ACS 2024 1-year S1501 educational attainment by state for the 18 to 24 and 25 and over universes (676 facts), with a parser for the data.census.gov table API.

chronicle/source_package.py

@@ -48,6 +48,7 @@
     SourceRow,
     source_cells_from_source_rows,
     source_rows_from_census_acs_s0101_age_json,
+    source_rows_from_census_acs_s1501_attainment_json,
     source_rows_from_census_acs_s2201_snap_json,
     source_rows_from_census_b01001_female_age_json,
     source_rows_from_cdc_vsrr_live_births_json,
@@ -792,6 +793,7 @@
     ),
     "census-acs-s0101-national-age-2024": Path("census/acs_s0101_national_2024"),
     "census-acs-s0101-state-age-2024": Path("census/acs_s0101_state_2024"),
+    "census-acs-s1501-state-attainment-2024": Path("census/acs_s1501_state_2024"),
     "census-acs-s2201-congressional-district-snap-2024": Path(
         "census/acs_s2201_district_2024"
     ),
@@ -893,6 +895,20 @@
     "hmrc-hydrocarbon-oils-quantities-june-2026": Path(
         "hmrc/hydrocarbon_oils_quantities_june_2026"
     ),
+    "ed-fsa-portfolio-summary-fy2026-q3": Path("ed_fsa/portfolio_summary_fy2026_q3"),
+    "ed-fsa-portfolio-by-location-fy2026-q2": Path(
+        "ed_fsa/portfolio_by_location_fy2026_q2"
+    ),
+    "ed-fsa-portfolio-by-age-fy2026-q2": Path("ed_fsa/portfolio_by_age_fy2026_q2"),
+    "ed-fsa-portfolio-by-debt-size-fy2026-q2": Path(
+        "ed_fsa/portfolio_by_debt_size_fy2026_q2"
+    ),
+    "ed-fsa-portfolio-by-loan-type-fy2026-q3": Path(
+        "ed_fsa/portfolio_by_loan_type_fy2026_q3"
+    ),
+    "ed-fsa-dl-portfolio-by-repayment-plan-fy2026-q3": Path(
+        "ed_fsa/dl_portfolio_by_repayment_plan_fy2026_q3"
+    ),
     "eurostat-gov-10a-taxag": Path("eurostat/gov_10a_taxag"),
     "eurostat-gov-10a-taxag-2025": Path("eurostat/gov_10a_taxag_2025"),
     "eurostat-spr-exp-func": Path("eurostat/spr_exp_func"),
@@ -1118,6 +1134,12 @@ def build_source_rows(self, year: int) -> list[SourceRow]:
                 artifact,
                 sheet_name=self._sheet_name(filename, year=year),
             )
+        if self.parser == "census_acs_s1501_attainment_json_rows":
+            return source_rows_from_census_acs_s1501_attainment_json(
+                content,
+                artifact,
+                sheet_name=self._sheet_name(filename, year=year),
+            )
         if self.parser == "census_b01001_female_age_json_rows":
             return source_rows_from_census_b01001_female_age_json(
                 content,
@@ -1414,6 +1436,26 @@ def build_source_cells(
                     for row in self.selected_rows
                 ),
             )
+        if self.parser == "census_acs_s1501_attainment_json_rows":
+            rows = (
+                source_rows
+                if source_rows is not None
+                else source_rows_from_census_acs_s1501_attainment_json(
+                    content,
+                    artifact,
+                    sheet_name=self._sheet_name(filename, year=year),
+                )
+            )
+            return source_cells_from_source_rows(
+                rows,
+                selected_rows=tuple(
+                    {
+                        key: str(_render_value(value, year=year))
+                        for key, value in row.items()
+                    }
+                    for row in self.selected_rows
+                ),
+            )
         if self.parser == "cdc_vsrr_live_births_json_rows":
             rows = (
                 source_rows

chronicle/sources/rows.py

@@ -689,6 +689,169 @@ def source_rows_from_census_acs_s2201_snap_json(
     return rows
 
 
+# Census ACS S1501 "Total" estimate columns for the two age universes the table
+# splits attainment by. Labels are the Census variable labels from
+# https://api.census.gov/data/{year}/acs/acs1/subject/groups/S1501.json; the
+# age wording is normalized to the "Aged ..." form the age-band evidence check
+# reads, and the attainment ids are slugs of the publisher's category labels.
+# The 18 to 24 universe publishes four categories (associate's degrees are
+# inside "Some college or associate's degree"); the 25 and over universe
+# publishes seven, separating associate's, bachelor's and graduate degrees.
+S1501_ATTAINMENT_COLUMNS: tuple[tuple[str, str, str, str], ...] = (
+    (
+        "S1501_C01_001E",
+        "Aged 18 to 24",
+        "all",
+        "Estimate!!Total!!AGE BY EDUCATIONAL ATTAINMENT!!Population 18 to 24 years",
+    ),
+    (
+        "S1501_C01_002E",
+        "Aged 18 to 24",
+        "less_than_high_school_graduate",
+        "Estimate!!Total!!AGE BY EDUCATIONAL ATTAINMENT!!Population 18 to 24 years"
+        "!!Less than high school graduate",
+    ),
+    (
+        "S1501_C01_003E",
+        "Aged 18 to 24",
+        "high_school_graduate_includes_equivalency",
+        "Estimate!!Total!!AGE BY EDUCATIONAL ATTAINMENT!!Population 18 to 24 years"
+        "!!High school graduate (includes equivalency)",
+    ),
+    (
+        "S1501_C01_004E",
+        "Aged 18 to 24",
+        "some_college_or_associates_degree",
+        "Estimate!!Total!!AGE BY EDUCATIONAL ATTAINMENT!!Population 18 to 24 years"
+        "!!Some college or associate's degree",
+    ),
+    (
+        "S1501_C01_005E",
+        "Aged 18 to 24",
+        "bachelors_degree_or_higher",
+        "Estimate!!Total!!AGE BY EDUCATIONAL ATTAINMENT!!Population 18 to 24 years"
+        "!!Bachelor's degree or higher",
+    ),
+    (
+        "S1501_C01_006E",
+        "Aged 25 and over",
+        "all",
+        "Estimate!!Total!!AGE BY EDUCATIONAL ATTAINMENT!!Population 25 years and over",
+    ),
+    (
+        "S1501_C01_007E",
+        "Aged 25 and over",
+        "less_than_9th_grade",
+        "Estimate!!Total!!AGE BY EDUCATIONAL ATTAINMENT!!Population 25 years and over"
+        "!!Less than 9th grade",
+    ),
+    (
+        "S1501_C01_008E",
+        "Aged 25 and over",
+        "9th_to_12th_grade_no_diploma",
+        "Estimate!!Total!!AGE BY EDUCATIONAL ATTAINMENT!!Population 25 years and over"
+        "!!9th to 12th grade, no diploma",
+    ),
+    (
+        "S1501_C01_009E",
+        "Aged 25 and over",
+        "high_school_graduate_includes_equivalency",
+        "Estimate!!Total!!AGE BY EDUCATIONAL ATTAINMENT!!Population 25 years and over"
+        "!!High school graduate (includes equivalency)",
+    ),
+    (
+        "S1501_C01_010E",
+        "Aged 25 and over",
+        "some_college_no_degree",
+        "Estimate!!Total!!AGE BY EDUCATIONAL ATTAINMENT!!Population 25 years and over"
+        "!!Some college, no degree",
+    ),
+    (
+        "S1501_C01_011E",
+        "Aged 25 and over",
+        "associates_degree",
+        "Estimate!!Total!!AGE BY EDUCATIONAL ATTAINMENT!!Population 25 years and over"
+        "!!Associate's degree",
+    ),
+    (
+        "S1501_C01_012E",
+        "Aged 25 and over",
+        "bachelors_degree",
+        "Estimate!!Total!!AGE BY EDUCATIONAL ATTAINMENT!!Population 25 years and over"
+        "!!Bachelor's degree",
+    ),
+    (
+        "S1501_C01_013E",
+        "Aged 25 and over",
+        "graduate_or_professional_degree",
+        "Estimate!!Total!!AGE BY EDUCATIONAL ATTAINMENT!!Population 25 years and over"
+        "!!Graduate or professional degree",
+    ),
+)
+
+
+def _census_table_json_content(content: bytes) -> bytes:
+    """Return the Census array-of-arrays table from either Census API.
+
+    ``api.census.gov`` returns the array itself. ``data.census.gov``'s
+    ``/api/access/data/table`` endpoint wraps the same array, header row first,
+    in ``{"response": {"data": [...]}}``.
+    """
+    data = json.loads(content.decode("utf-8"))
+    if isinstance(data, list):
+        return content
+    response = data.get("response") if isinstance(data, dict) else None
+    table = response.get("data") if isinstance(response, dict) else None
+    if not isinstance(table, list):
+        raise ValueError(
+            "Census table JSON must be an array of arrays or a data.census.gov "
+            '{"response": {"data": [...]}} payload.'
+        )
+    return json.dumps(table).encode("utf-8")
+
+
+def source_rows_from_census_acs_s1501_attainment_json(
+    content: bytes,
+    artifact: SourceArtifactMetadata,
+    *,
+    sheet_name: str,
+) -> list[SourceRow]:
+    """Unpivot Census ACS S1501 table rows into age-by-attainment source rows."""
+    table_rows = source_rows_from_json_table(
+        _census_table_json_content(content),
+        artifact,
+        sheet_name=sheet_name,
+    )
+    rows: list[SourceRow] = []
+    for source_table_row in table_rows:
+        missing = [
+            variable
+            for variable, _age, _attainment, _label in S1501_ATTAINMENT_COLUMNS
+            if variable not in source_table_row.values
+        ]
+        if missing:
+            raise ValueError(f"Census S1501 table is missing columns {missing}.")
+        for variable, age_label, attainment, source_concept in S1501_ATTAINMENT_COLUMNS:
+            rows.append(
+                SourceRow(
+                    artifact=artifact,
+                    sheet_name=sheet_name,
+                    row_number=len(rows) + 1,
+                    values={
+                        "GEO_ID": source_table_row.values.get("GEO_ID"),
+                        "NAME": source_table_row.values.get("NAME"),
+                        "age": age_label,
+                        "educational_attainment": attainment,
+                        "source_column_id": variable,
+                        "source_concept": source_concept,
+                        "value": source_table_row.values.get(variable),
+                        "source_table_row_number": source_table_row.row_number,
+                    },
+                )
+            )
+    return rows
+
+
 def source_rows_from_census_b01001_female_age_json(
     content: bytes,
     artifact: SourceArtifactMetadata,

db/data/census/acs_s1501_state_2024/manifest.yaml

@@ -0,0 +1,21 @@
+source_id: census_acs
+package_id: census-acs-s1501-state-attainment-2024
+dataset: census_acs_s1501_state_attainment_2024
+source_page: https://data.census.gov/table/ACSST1Y2024.S1501
+vintage: 2024 ACS 1-year subject tables
+table: S1501 Educational Attainment
+files:
+  2024:
+    filename: acs_S1501_state_2024.json
+    source_url: https://data.census.gov/api/access/data/table?id=ACSST1Y2024.S1501&g=010XX00US$0400000
+    sha256: 84fff08c9d2793a90f2326eb68ae8ba4d5e18957899fc8cdda015d26520f73a4
+    size_bytes: 575536
+    fetched_at: '2026-10-12T14:09:26Z'
+    source_table: ACS 2024 1-year subject table S1501 state educational attainment
+    year: 2024
+    storage:
+      r2:
+        provider: r2
+        bucket: ledger-raw
+        key: raw/census_acs/census-acs-s1501-state-attainment-2024/2024/84fff08c9d2793a90f2326eb68ae8ba4d5e18957899fc8cdda015d26520f73a4/acs_S1501_state_2024.json
+        uri: r2://ledger-raw/raw/census_acs/census-acs-s1501-state-attainment-2024/2024/84fff08c9d2793a90f2326eb68ae8ba4d5e18957899fc8cdda015d26520f73a4/acs_S1501_state_2024.json

db/data/ed_fsa/dl_portfolio_by_repayment_plan_fy2026_q3/manifest.yaml

@@ -0,0 +1,21 @@
+source_id: ed_fsa
+package_id: ed-fsa-dl-portfolio-by-repayment-plan-fy2026-q3
+dataset: ed_fsa_dl_portfolio_by_repayment_plan_fy2026_q3
+source_page: https://studentaid.gov/data-center/student/portfolio
+table: Direct Loan Portfolio by Repayment Plan (DLPortfoliobyRepaymentPlan.xls), through FY2026 Q3
+files:
+  2026:
+    filename: DLPortfoliobyRepaymentPlan.xls
+    source_url: https://studentaid.gov/sites/default/files/fsawg/datacenter/library/DLPortfoliobyRepaymentPlan.xls
+    sha256: 5cea1958115574270c825ccc34dd656658d58bff6e439a21fb1f3aafe7bf3ca0
+    size_bytes: 165376
+    fetched_at: '2026-10-12T14:09:17Z'
+    source_table: Direct Loan Portfolio by Repayment Plan (DLPortfoliobyRepaymentPlan.xls), through FY2026
+      Q3
+    year: 2026
+    storage:
+      r2:
+        provider: r2
+        bucket: ledger-raw
+        key: raw/ed_fsa/ed-fsa-dl-portfolio-by-repayment-plan-fy2026-q3/2026/5cea1958115574270c825ccc34dd656658d58bff6e439a21fb1f3aafe7bf3ca0/DLPortfoliobyRepaymentPlan.xls
+        uri: r2://ledger-raw/raw/ed_fsa/ed-fsa-dl-portfolio-by-repayment-plan-fy2026-q3/2026/5cea1958115574270c825ccc34dd656658d58bff6e439a21fb1f3aafe7bf3ca0/DLPortfoliobyRepaymentPlan.xls

db/data/ed_fsa/portfolio_by_age_fy2026_q2/manifest.yaml

@@ -0,0 +1,21 @@
+source_id: ed_fsa
+package_id: ed-fsa-portfolio-by-age-fy2026-q2
+dataset: ed_fsa_portfolio_by_age_fy2026_q2
+source_page: https://studentaid.gov/data-center/student/portfolio
+table: Federal Student Loan Portfolio by Borrower Age (Portfolio-by-Age.xls), through FY2026 Q2
+files:
+  2026:
+    filename: Portfolio-by-Age.xls
+    source_url: https://studentaid.gov/sites/default/files/fsawg/datacenter/library/Portfolio-by-Age.xls
+    sha256: c2027d277bb88e31c5a8c0f8fc6d81422f45d60e0306f79e63a0c25c7286e2ef
+    size_bytes: 61952
+    fetched_at: '2026-10-12T14:09:15Z'
+    source_table: Federal Student Loan Portfolio by Borrower Age (Portfolio-by-Age.xls), through FY2026
+      Q2
+    year: 2026
+    storage:
+      r2:
+        provider: r2
+        bucket: ledger-raw
+        key: raw/ed_fsa/ed-fsa-portfolio-by-age-fy2026-q2/2026/c2027d277bb88e31c5a8c0f8fc6d81422f45d60e0306f79e63a0c25c7286e2ef/Portfolio-by-Age.xls
+        uri: r2://ledger-raw/raw/ed_fsa/ed-fsa-portfolio-by-age-fy2026-q2/2026/c2027d277bb88e31c5a8c0f8fc6d81422f45d60e0306f79e63a0c25c7286e2ef/Portfolio-by-Age.xls

db/data/ed_fsa/portfolio_by_debt_size_fy2026_q2/manifest.yaml

@@ -0,0 +1,22 @@
+source_id: ed_fsa
+package_id: ed-fsa-portfolio-by-debt-size-fy2026-q2
+dataset: ed_fsa_portfolio_by_debt_size_fy2026_q2
+source_page: https://studentaid.gov/data-center/student/portfolio
+table: Federal Student Loan Portfolio by Borrower Debt Size (Portfolio-by-Debt-Size.xls), through FY2026
+  Q2
+files:
+  2026:
+    filename: Portfolio-by-Debt-Size.xls
+    source_url: https://studentaid.gov/sites/default/files/fsawg/datacenter/library/Portfolio-by-Debt-Size.xls
+    sha256: 881bd8bbd257da5f5b906b0f0923cf246de6e3d28ee35e8be1cd23d841e88a4d
+    size_bytes: 71680
+    fetched_at: '2026-10-12T14:09:15Z'
+    source_table: Federal Student Loan Portfolio by Borrower Debt Size (Portfolio-by-Debt-Size.xls), through
+      FY2026 Q2
+    year: 2026
+    storage:
+      r2:
+        provider: r2
+        bucket: ledger-raw
+        key: raw/ed_fsa/ed-fsa-portfolio-by-debt-size-fy2026-q2/2026/881bd8bbd257da5f5b906b0f0923cf246de6e3d28ee35e8be1cd23d841e88a4d/Portfolio-by-Debt-Size.xls
+        uri: r2://ledger-raw/raw/ed_fsa/ed-fsa-portfolio-by-debt-size-fy2026-q2/2026/881bd8bbd257da5f5b906b0f0923cf246de6e3d28ee35e8be1cd23d841e88a4d/Portfolio-by-Debt-Size.xls

db/data/ed_fsa/portfolio_by_loan_type_fy2026_q3/manifest.yaml

@@ -0,0 +1,20 @@
+source_id: ed_fsa
+package_id: ed-fsa-portfolio-by-loan-type-fy2026-q3
+dataset: ed_fsa_portfolio_by_loan_type_fy2026_q3
+source_page: https://studentaid.gov/data-center/student/portfolio
+table: Portfolio by Loan Type (PortfoliobyLoanType.xls), through FY2026 Q3
+files:
+  2026:
+    filename: PortfoliobyLoanType.xls
+    source_url: https://studentaid.gov/sites/default/files/fsawg/datacenter/library/PortfoliobyLoanType.xls
+    sha256: e64e9d2945dfe703d42d383a7f65fbd4b044d134565ea77a0bd194f366becd33
+    size_bytes: 58368
+    fetched_at: '2026-10-12T14:09:16Z'
+    source_table: Portfolio by Loan Type (PortfoliobyLoanType.xls), through FY2026 Q3
+    year: 2026
+    storage:
+      r2:
+        provider: r2
+        bucket: ledger-raw
+        key: raw/ed_fsa/ed-fsa-portfolio-by-loan-type-fy2026-q3/2026/e64e9d2945dfe703d42d383a7f65fbd4b044d134565ea77a0bd194f366becd33/PortfoliobyLoanType.xls
+        uri: r2://ledger-raw/raw/ed_fsa/ed-fsa-portfolio-by-loan-type-fy2026-q3/2026/e64e9d2945dfe703d42d383a7f65fbd4b044d134565ea77a0bd194f366becd33/PortfoliobyLoanType.xls

db/data/ed_fsa/portfolio_by_location_fy2026_q2/manifest.yaml

@@ -0,0 +1,22 @@
+source_id: ed_fsa
+package_id: ed-fsa-portfolio-by-location-fy2026-q2
+dataset: ed_fsa_portfolio_by_location_fy2026_q2
+source_page: https://studentaid.gov/data-center/student/portfolio
+table: Federal Student Loan Portfolio by Borrower Location (Portfolio-by-Location.xls), data as of Mar.
+  31, 2026
+files:
+  2026:
+    filename: Portfolio-by-Location.xls
+    source_url: https://studentaid.gov/sites/default/files/fsawg/datacenter/library/Portfolio-by-Location.xls
+    sha256: d9e8608a82fa1e130c6a80bc5870d25d6f47c626ed200bd33b8f45a43e262dba
+    size_bytes: 51200
+    fetched_at: '2026-10-12T14:09:14Z'
+    source_table: Federal Student Loan Portfolio by Borrower Location (Portfolio-by-Location.xls), data
+      as of Mar. 31, 2026
+    year: 2026
+    storage:
+      r2:
+        provider: r2
+        bucket: ledger-raw
+        key: raw/ed_fsa/ed-fsa-portfolio-by-location-fy2026-q2/2026/d9e8608a82fa1e130c6a80bc5870d25d6f47c626ed200bd33b8f45a43e262dba/Portfolio-by-Location.xls
+        uri: r2://ledger-raw/raw/ed_fsa/ed-fsa-portfolio-by-location-fy2026-q2/2026/d9e8608a82fa1e130c6a80bc5870d25d6f47c626ed200bd33b8f45a43e262dba/Portfolio-by-Location.xls