fix(holdings): reject broken fresh holdings with the shared holdings guard

scripts/update-data.test.ts

@@ -18,6 +18,7 @@ import {
   withYieldBasis,
   expenseRatios,
   fetchWithRetry,
+  holdingsGuard,
   indicatedYield,
   installSystemCa,
   isCertError,
@@ -268,6 +269,21 @@ describe('parsing', () => {
     expect(bond.table.rows[0][5]).toBe('77000000');
   });
 
+  test('holdings guard: header-only, renamed weight column, broken weights, a bad weight sum and a halved sheet are rejected', () => {
+    const headers = ['Name', 'Ticker', 'Weight'];
+    const rows = [{ Name: 'A', Ticker: 'A', Weight: '60%' }, { Name: 'B', Ticker: 'B', Weight: '40%' }];
+    expect(holdingsGuard(headers, rows, 2)).toBeNull();
+    expect(holdingsGuard(headers, [['A', 'A', '60'], ['B', 'B', '40']], null)).toBeNull();
+    expect(holdingsGuard([], [], 0)).toBeNull(); // nothing published before: an empty sheet is honest
+    expect(holdingsGuard(headers, [], 505)).toContain('no holdings rows');
+    expect(holdingsGuard(['Name', 'Ticker', 'Wt'], rows, 2)).toContain('no weight column');
+    expect(holdingsGuard(headers, rows.map((row) => ({ ...row, Weight: '6\u200B0' })), 2)).toContain('numeric weight');
+    const allocation = [{ Name: 'Cash&Collateral', Ticker: '-', Weight: '100' }, { Name: 'Equities', Ticker: '-', Weight: '476' }];
+    expect(holdingsGuard(headers, allocation, 2)).toContain('outside 50..150');
+    expect(holdingsGuard(headers, allocation, 2, ['sum'])).toBeNull();
+    expect(holdingsGuard(headers, rows, 5)).toContain('less than half');
+  });
+
   test('NAV and premium/discount history workbooks (blank rows dropped, registered mark stripped)', () => {
     const nav = table(buildXlsx([
       ['Fund Name:', 'SPDR Gold MiniShares'], ['Ticker Symbol:', 'GLDM®'], ['Date', 'NAV', 'Shares Outstanding', 'Total Net Assets'],
@@ -573,6 +589,30 @@ describe('pipeline', () => {
     expect(listFiles(dir).filter((f) => f.endsWith('.tmp'))).toEqual([]);
   });
 
+  test('broken fresh holdings (renamed Weight header, header-only sheet) keep the fund exactly as published and report it failed', async () => {
+    const dir = tempRoot();
+    installMock();
+    await runMain();
+    const snap = () => [JSON.stringify(row(dir, 'SPY')), ...listFiles(dir, 'funds/SPY/').map((f) => `${f}:${readFileSync(join(dir, f), 'utf8')}`)];
+    const before = snap();
+    expect(readJson(dir, 'funds/SPY/meta.json').holdings.totalRows).toBe(1);
+    const sheets: Record<string, string[][]> = {
+      renamed: holdingsSheet('SPY').map((r) => r.map((cell) => (cell === 'Weight' ? 'Weighting' : cell))),
+      headerOnly: holdingsSheet('SPY').slice(0, 4),
+    };
+    for (const [name, sheet] of Object.entries(sheets)) {
+      installMock({ nav: { SPY: 777, XLK: 200, GLD: 300 } });
+      const good = globalThis.fetch;
+      globalThis.fetch = (async (input: unknown, init?: RequestInit) => (String(input).includes('holdings-daily-us-en-spy')
+        ? new Response(new Uint8Array(buildXlsx(sheet)))
+        : good(input as string, init))) as unknown as typeof fetch;
+      const lines: string[] = [];
+      await runMain({}, lines); // XLK and GLD still update, so the run itself succeeds
+      expect([name, snap()]).toEqual([name, before]);
+      expect(lines.join('\n')).toMatch(/SPY +failed .*reason=holdings guard: /);
+    }
+  });
+
   test('a fund whose source failed keeps its previous index row and meta exactly as published', async () => {
     const dir = tempRoot();
     installMock();

scripts/update-data.ts

@@ -200,6 +200,8 @@ export function setApiRoot(root: URL): void {
 }
 
 const HOLDINGS_PAGE_SIZE_FALLBACK = 250;
+/** Funds whose holdings legitimately fail a holdingsGuard check (geared, futures, look-through, top-N lists): ticker, or '*' for every fund -> relaxed checks. */
+const HOLDINGS_GUARD_RELAX: Record<string, string[]> = {};
 const HISTORY_PAGE_SIZE_FALLBACK = 1000;
 const CONCURRENCY_FALLBACK = 2;
 const REQUEST_SLEEP_FALLBACK = 1;
@@ -1485,16 +1487,22 @@ async function processFund(
     const holdingsResponse = await fetchWithRetry(holdingsUrl, `[fetch  ] ${ticker} holdings`);
     let holdingsBytes: Uint8Array | null = null;
     let holdings: { meta: JsonRecord; table: SheetTable } = { meta: {}, table: { headers: [], rows: [] } };
+    const readPreviousMeta = () => readFile(decodeURIComponent(new URL('meta.json', fundDir).pathname), 'utf8').then(JSON.parse).catch(() => null);
     if (holdingsResponse.status === 404) {
       // Commodity trusts (GLD, GLDM, ...) hold metal, not securities, and publish no holdings workbook: valid, status "empty".
       // But a 404 for a fund that published holdings before is treated as a failure so published rows are kept.
-      const previousMeta = await readFile(decodeURIComponent(new URL('meta.json', fundDir).pathname), 'utf8').then(JSON.parse).catch(() => null);
+      const previousMeta = await readPreviousMeta();
       if ((previousMeta?.holdings?.totalRows ?? 0) > 0) throw new Error('holdings: 404 for a fund with published holdings, keeping the previous data');
     } else if (!holdingsResponse.ok) {
       throw new Error(`holdings: ${holdingsResponse.status} ${holdingsResponse.statusText}`);
     } else {
       holdingsBytes = new Uint8Array(await holdingsResponse.arrayBuffer());
       holdings = sheetToTable(parseXlsxSheet(holdingsBytes, loadSharedStrings(holdingsBytes)), 'holdings');
+      // Freshly fetched holdings that look broken (no weight column, non-numeric weights, a bad weight sum, half the rows gone)
+      // never replace the published ones: the fund fails here, before anything of it is written, and keeps its previous state.
+      const previousMeta = await readPreviousMeta();
+      const holdingsProblem = holdingsGuard(holdings.table.headers, holdings.table.rows, previousMeta?.holdings?.totalRows, HOLDINGS_GUARD_RELAX[ticker] ?? HOLDINGS_GUARD_RELAX['*'] ?? []);
+      if (holdingsProblem) throw new Error(`holdings guard: ${holdingsProblem}`);
     }
 
     const history = await fetchXlsx(historyUrl, `[fetch  ] ${ticker} navhist`);
@@ -1939,3 +1947,49 @@ if (import.meta.main) {
     });
   }
 }
+
+// <holdings-guard> shared block of .claude/tools/etfs/etf-std/holdings-guard.block.ts, applied by apply-holdings-guard.ts: never edit it in one repo
+/**
+ * Sanity check of a fund's new holdings before they replace the published ones. Returns why they must be rejected
+ * (the caller then keeps the fund's previous state and reports it as failed), or null when they may be written.
+ * An empty sheet passes only when nothing was published before. Otherwise a Name and a weight column (Weight,
+ * Weight (%) or Market Weight) are required, >= 80% of the rows carry a numeric weight, the weights add up to
+ * 50..150 and the sheet keeps >= 50% of the previously published rows. `relax` switches off the checks a fund
+ * legitimately fails: 'numeric' and 'sum' (geared, inverse, futures, look-through, top-N lists) and 'count'.
+ * Rows are arrays in header order or objects keyed by the header text.
+ */
+export function holdingsGuard(headers: ReadonlyArray<string>, rows: ReadonlyArray<unknown>, previousRows: number | null | undefined, relax: ReadonlyArray<string> = []): string | null {
+  const previous = Number(previousRows) > 0 ? Number(previousRows) : 0;
+  if (!rows.length) return previous ? `no holdings rows (${previous} published)` : null;
+  const names = headers.map((header) => String(header).trim().toLowerCase());
+  const nameAt = names.indexOf('name');
+  let weightAt = names.indexOf('weight');
+  if (weightAt < 0) weightAt = names.indexOf('weight (%)');
+  if (weightAt < 0) weightAt = names.indexOf('market weight');
+  if (nameAt < 0 || weightAt < 0) return `no ${nameAt < 0 ? 'Name' : 'weight'} column (headers: ${headers.join(', ') || 'none'})`;
+  let numeric = 0;
+  let sum = 0;
+  for (let i = 0; i < rows.length; i++) {
+    const row: any = rows[i];
+    const weight = holdingsGuardNumber(Array.isArray(row) ? row[weightAt] : row && typeof row === 'object' ? row[headers[weightAt]] : undefined);
+    if (weight !== null) {
+      numeric++;
+      sum += weight;
+    }
+  }
+  if (relax.indexOf('numeric') < 0 && numeric < rows.length * 0.8) return `only ${numeric} of ${rows.length} rows have a numeric weight`;
+  if (relax.indexOf('sum') < 0 && (sum < 50 || sum > 150)) return `weights add up to ${sum.toFixed(2)}, outside 50..150`;
+  if (relax.indexOf('count') < 0 && previous && rows.length < previous * 0.5) return `${rows.length} rows, less than half of the ${previous} published`;
+  return null;
+}
+
+/** A weight cell as a number: %, $, commas and spaces are ignored, (x) is negative; null when it is no plain number (a zero-width character inside counts as broken). */
+function holdingsGuardNumber(value: unknown): number | null {
+  if (typeof value === 'number') return Number.isFinite(value) ? value : null;
+  let text = String(value === null || value === undefined ? '' : value).replace(/[ \t ,%$]/g, '').replace(/−/g, '-');
+  const negative = /^\(.*\)$/.test(text);
+  if (negative) text = text.slice(1, -1);
+  if (!/^[-+]?(\d+\.?\d*|\.\d+)(e[-+]?\d+)?$/i.test(text)) return null;
+  return negative ? -Number(text) : Number(text);
+}
+// </holdings-guard>