From 5ccd4d97c1ee868cec0ae45b92966a640d748ea9 Mon Sep 17 00:00:00 2001 From: Maksim Kostromin Date: Sat, 24 Oct 2026 23:51:42 -0400 Subject: [PATCH] fix(holdings): reject broken fresh holdings with the shared holdings guard Broken holdings (renamed Weight header, header-only sheet, non-numeric weights, bad weight sum, half the rows gone) no longer overwrite the published sheet: the fund fails and keeps its previous state (ETF-H1) --- scripts/update-data.test.ts | 40 ++++++++++++++++++++++++++ scripts/update-data.ts | 56 ++++++++++++++++++++++++++++++++++++- 2 files changed, 95 insertions(+), 1 deletion(-) diff --git a/scripts/update-data.test.ts b/scripts/update-data.test.ts index 7e9d4a9d..9f5c9b7f 100644 --- a/scripts/update-data.test.ts +++ b/scripts/update-data.test.ts @@ -18,6 +18,7 @@ import { withYieldBasis, expenseRatios, fetchWithRetry, + holdingsGuard, indicatedYield, installSystemCa, isCertError, @@ -268,6 +269,21 @@ describe('parsing', () => { expect(bond.table.rows[0][5]).toBe('77000000'); }); + test('holdings guard: header-only, renamed weight column, broken weights, a bad weight sum and a halved sheet are rejected', () => { + const headers = ['Name', 'Ticker', 'Weight']; + const rows = [{ Name: 'A', Ticker: 'A', Weight: '60%' }, { Name: 'B', Ticker: 'B', Weight: '40%' }]; + expect(holdingsGuard(headers, rows, 2)).toBeNull(); + expect(holdingsGuard(headers, [['A', 'A', '60'], ['B', 'B', '40']], null)).toBeNull(); + expect(holdingsGuard([], [], 0)).toBeNull(); // nothing published before: an empty sheet is honest + expect(holdingsGuard(headers, [], 505)).toContain('no holdings rows'); + expect(holdingsGuard(['Name', 'Ticker', 'Wt'], rows, 2)).toContain('no weight column'); + expect(holdingsGuard(headers, rows.map((row) => ({ ...row, Weight: '6\u200B0' })), 2)).toContain('numeric weight'); + const allocation = [{ Name: 'Cash&Collateral', Ticker: '-', Weight: '100' }, { Name: 'Equities', Ticker: '-', Weight: '476' }]; + expect(holdingsGuard(headers, allocation, 2)).toContain('outside 50..150'); + expect(holdingsGuard(headers, allocation, 2, ['sum'])).toBeNull(); + expect(holdingsGuard(headers, rows, 5)).toContain('less than half'); + }); + test('NAV and premium/discount history workbooks (blank rows dropped, registered mark stripped)', () => { const nav = table(buildXlsx([ ['Fund Name:', 'SPDR Gold MiniShares'], ['Ticker Symbol:', 'GLDM®'], ['Date', 'NAV', 'Shares Outstanding', 'Total Net Assets'], @@ -573,6 +589,30 @@ describe('pipeline', () => { expect(listFiles(dir).filter((f) => f.endsWith('.tmp'))).toEqual([]); }); + test('broken fresh holdings (renamed Weight header, header-only sheet) keep the fund exactly as published and report it failed', async () => { + const dir = tempRoot(); + installMock(); + await runMain(); + const snap = () => [JSON.stringify(row(dir, 'SPY')), ...listFiles(dir, 'funds/SPY/').map((f) => `${f}:${readFileSync(join(dir, f), 'utf8')}`)]; + const before = snap(); + expect(readJson(dir, 'funds/SPY/meta.json').holdings.totalRows).toBe(1); + const sheets: Record = { + renamed: holdingsSheet('SPY').map((r) => r.map((cell) => (cell === 'Weight' ? 'Weighting' : cell))), + headerOnly: holdingsSheet('SPY').slice(0, 4), + }; + for (const [name, sheet] of Object.entries(sheets)) { + installMock({ nav: { SPY: 777, XLK: 200, GLD: 300 } }); + const good = globalThis.fetch; + globalThis.fetch = (async (input: unknown, init?: RequestInit) => (String(input).includes('holdings-daily-us-en-spy') + ? new Response(new Uint8Array(buildXlsx(sheet))) + : good(input as string, init))) as unknown as typeof fetch; + const lines: string[] = []; + await runMain({}, lines); // XLK and GLD still update, so the run itself succeeds + expect([name, snap()]).toEqual([name, before]); + expect(lines.join('\n')).toMatch(/SPY +failed .*reason=holdings guard: /); + } + }); + test('a fund whose source failed keeps its previous index row and meta exactly as published', async () => { const dir = tempRoot(); installMock(); diff --git a/scripts/update-data.ts b/scripts/update-data.ts index daae5666..77185b16 100755 --- a/scripts/update-data.ts +++ b/scripts/update-data.ts @@ -200,6 +200,8 @@ export function setApiRoot(root: URL): void { } const HOLDINGS_PAGE_SIZE_FALLBACK = 250; +/** Funds whose holdings legitimately fail a holdingsGuard check (geared, futures, look-through, top-N lists): ticker, or '*' for every fund -> relaxed checks. */ +const HOLDINGS_GUARD_RELAX: Record = {}; const HISTORY_PAGE_SIZE_FALLBACK = 1000; const CONCURRENCY_FALLBACK = 2; const REQUEST_SLEEP_FALLBACK = 1; @@ -1485,16 +1487,22 @@ async function processFund( const holdingsResponse = await fetchWithRetry(holdingsUrl, `[fetch ] ${ticker} holdings`); let holdingsBytes: Uint8Array | null = null; let holdings: { meta: JsonRecord; table: SheetTable } = { meta: {}, table: { headers: [], rows: [] } }; + const readPreviousMeta = () => readFile(decodeURIComponent(new URL('meta.json', fundDir).pathname), 'utf8').then(JSON.parse).catch(() => null); if (holdingsResponse.status === 404) { // Commodity trusts (GLD, GLDM, ...) hold metal, not securities, and publish no holdings workbook: valid, status "empty". // But a 404 for a fund that published holdings before is treated as a failure so published rows are kept. - const previousMeta = await readFile(decodeURIComponent(new URL('meta.json', fundDir).pathname), 'utf8').then(JSON.parse).catch(() => null); + const previousMeta = await readPreviousMeta(); if ((previousMeta?.holdings?.totalRows ?? 0) > 0) throw new Error('holdings: 404 for a fund with published holdings, keeping the previous data'); } else if (!holdingsResponse.ok) { throw new Error(`holdings: ${holdingsResponse.status} ${holdingsResponse.statusText}`); } else { holdingsBytes = new Uint8Array(await holdingsResponse.arrayBuffer()); holdings = sheetToTable(parseXlsxSheet(holdingsBytes, loadSharedStrings(holdingsBytes)), 'holdings'); + // Freshly fetched holdings that look broken (no weight column, non-numeric weights, a bad weight sum, half the rows gone) + // never replace the published ones: the fund fails here, before anything of it is written, and keeps its previous state. + const previousMeta = await readPreviousMeta(); + const holdingsProblem = holdingsGuard(holdings.table.headers, holdings.table.rows, previousMeta?.holdings?.totalRows, HOLDINGS_GUARD_RELAX[ticker] ?? HOLDINGS_GUARD_RELAX['*'] ?? []); + if (holdingsProblem) throw new Error(`holdings guard: ${holdingsProblem}`); } const history = await fetchXlsx(historyUrl, `[fetch ] ${ticker} navhist`); @@ -1939,3 +1947,49 @@ if (import.meta.main) { }); } } + +// shared block of .claude/tools/etfs/etf-std/holdings-guard.block.ts, applied by apply-holdings-guard.ts: never edit it in one repo +/** + * Sanity check of a fund's new holdings before they replace the published ones. Returns why they must be rejected + * (the caller then keeps the fund's previous state and reports it as failed), or null when they may be written. + * An empty sheet passes only when nothing was published before. Otherwise a Name and a weight column (Weight, + * Weight (%) or Market Weight) are required, >= 80% of the rows carry a numeric weight, the weights add up to + * 50..150 and the sheet keeps >= 50% of the previously published rows. `relax` switches off the checks a fund + * legitimately fails: 'numeric' and 'sum' (geared, inverse, futures, look-through, top-N lists) and 'count'. + * Rows are arrays in header order or objects keyed by the header text. + */ +export function holdingsGuard(headers: ReadonlyArray, rows: ReadonlyArray, previousRows: number | null | undefined, relax: ReadonlyArray = []): string | null { + const previous = Number(previousRows) > 0 ? Number(previousRows) : 0; + if (!rows.length) return previous ? `no holdings rows (${previous} published)` : null; + const names = headers.map((header) => String(header).trim().toLowerCase()); + const nameAt = names.indexOf('name'); + let weightAt = names.indexOf('weight'); + if (weightAt < 0) weightAt = names.indexOf('weight (%)'); + if (weightAt < 0) weightAt = names.indexOf('market weight'); + if (nameAt < 0 || weightAt < 0) return `no ${nameAt < 0 ? 'Name' : 'weight'} column (headers: ${headers.join(', ') || 'none'})`; + let numeric = 0; + let sum = 0; + for (let i = 0; i < rows.length; i++) { + const row: any = rows[i]; + const weight = holdingsGuardNumber(Array.isArray(row) ? row[weightAt] : row && typeof row === 'object' ? row[headers[weightAt]] : undefined); + if (weight !== null) { + numeric++; + sum += weight; + } + } + if (relax.indexOf('numeric') < 0 && numeric < rows.length * 0.8) return `only ${numeric} of ${rows.length} rows have a numeric weight`; + if (relax.indexOf('sum') < 0 && (sum < 50 || sum > 150)) return `weights add up to ${sum.toFixed(2)}, outside 50..150`; + if (relax.indexOf('count') < 0 && previous && rows.length < previous * 0.5) return `${rows.length} rows, less than half of the ${previous} published`; + return null; +} + +/** A weight cell as a number: %, $, commas and spaces are ignored, (x) is negative; null when it is no plain number (a zero-width character inside counts as broken). */ +function holdingsGuardNumber(value: unknown): number | null { + if (typeof value === 'number') return Number.isFinite(value) ? value : null; + let text = String(value === null || value === undefined ? '' : value).replace(/[ \t ,%$]/g, '').replace(/−/g, '-'); + const negative = /^\(.*\)$/.test(text); + if (negative) text = text.slice(1, -1); + if (!/^[-+]?(\d+\.?\d*|\.\d+)(e[-+]?\d+)?$/i.test(text)) return null; + return negative ? -Number(text) : Number(text); +} +//