From 426b1344ae7d012ae2840acb79ba01799236dadc Mon Sep 17 00:00:00 2001 From: Super <20788648+supertj@users.noreply.github.com> Date: Sat, 7 Nov 2026 19:36:55 +0800 Subject: [PATCH 1/4] feat: drill and restore compare every table's row count with the backup exactly (2.11.0) The manifest's row counts are n_live_tup estimates taken before pg_dump, so the drill could only check that tables with rows didn't come back empty. The backup's exact count now comes from the dump itself: pg_restore --data-only over the archive, counting each COPY section. Nothing extra runs against the source database. One row more or less fails the check; if the dump can't be counted, the check fails rather than falling back to the estimate. --- README.md | 2 + examples/reproducible-drill/README.md | 45 ++++---- package.json | 2 +- src/drill.ts | 83 ++++++++++++++- src/manifest.ts | 3 +- src/restore-engine.ts | 115 ++++++++++++++++++++ test/copy-row-counter.test.js | 132 +++++++++++++++++++++++ test/drill-row-counts.test.js | 148 ++++++++++++++++++++++++++ 8 files changed, 505 insertions(+), 25 deletions(-) create mode 100644 test/copy-row-counter.test.js create mode 100644 test/drill-row-counts.test.js diff --git a/README.md b/README.md index 21ca8d5..e634112 100644 --- a/README.md +++ b/README.md @@ -179,6 +179,8 @@ backupdrill drill # latest snapshot; add --snapshot sign in on the restored project ✓ user1@example.com signed in with the password it had before the backup (HTTP 200, access token issued) ✓ the same account with a wrong password is refused (HTTP 400) @@ -163,8 +166,8 @@ output; they are cut here for length. ## What is and isn't identical when you rerun this -We ran it three times in July 2026, again in October, and again in November -after adding the Auth users. Worth knowing before you compare: +We ran it three times in July 2026, again in October, and twice in November: +after adding the Auth users, and after adding the row count check. Worth knowing before you compare: | Value | Stable across runs? | Why | |---|---|---| @@ -173,13 +176,17 @@ after adding the Auth users. Worth knowing before you compare: | Dump and `auth.dump` sha256 | **No** | Two reasons: the random uuid keys, `now()` stamps and bcrypt salts above mean the bytes genuinely differ, and `pg_dump` writes a creation timestamp into the archive header, so even identical data would not hash the same | | `pg_restore` seconds | **No** | Hardware | -The first two rows are the reason the drill measures instead of trusting the -manifest: on one of our runs the manifest claimed 250,000 more rows than the -database actually held, and the drill's count was right anyway. The manifest's -row counts are estimates, which is exactly why the check is "every table that -had rows came back non-empty" rather than "the row counts match". - -**Restore time is hardware, not a product claim.** 1.9 s is a 21.6 MB dump +The first two rows are the reason the drill never compares against the +manifest's row counts: on one of our runs the manifest claimed 250,000 more rows +than the database actually held. "Row counts match the backup" counts the rows +of each table in the dump file itself and compares them with the restored +tables, one table at a time. 12 of the 14 tables are compared that way; the +partitioned parent `events` has no rows of its own in the dump, and the +materialized view `order_totals` is rebuilt by `REFRESH`, so those two are only +checked for being non-empty. That is why 1,476,250 rows are compared out of the +1,496,250 restored: the difference is the view's 20,000 rows. + +**Restore time is hardware, not a product claim.** 1.8 s is a 21.6 MB dump restored into a container on the same laptop. Your number depends on dump size, disk, and network — which is the point of measuring it on a schedule instead of guessing. diff --git a/package.json b/package.json index 0a2b23a..0391aa1 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "backupdrill", - "version": "2.10.1", + "version": "2.11.0", "description": "Open-source CLI to back up a Supabase project — Postgres database and Storage files — to your own S3/R2/B2 bucket, with a checksummed manifest.", "type": "module", "bin": { diff --git a/src/drill.ts b/src/drill.ts index 12b74d0..e42ff48 100644 --- a/src/drill.ts +++ b/src/drill.ts @@ -18,9 +18,10 @@ import { restoreDatabaseArtifact, installSandboxShim, tableDataInDump, + countRowsInDump, } from "./restore-engine.js"; import { connectPg } from "./supabase-ca.js"; -import { orphanedForeignKeysCheck, type OrphanedForeignKey } from "./orphaned-foreign-keys.js"; +import { orphanedForeignKeysCheck, tableKey, type OrphanedForeignKey } from "./orphaned-foreign-keys.js"; import { grantsCheck, type GrantRestore, type GrantedObject } from "./grants.js"; import { localPgRestoreMajor, restoreToolBreaksOnTarget, sandboxMajorFor } from "./pgbin.js"; import { log } from "./log.js"; @@ -448,13 +449,74 @@ async function pgRestore( // ── 校验 ──────────────────────────────────────────────────────────── +/** 恢复出的一张表(verifyRestored 里查到的口径) */ +interface RestoredTable { + schema: string; + name: string; + kind: string; + extension_member: boolean; +} + +/** + * 第 4 项检查:恢复后每张表的行数和备份文件里的确切行数逐表相等。 + * 只比普通表(含分区的叶子):matview 的数据是 REFRESH、分区父表不存行,dump 里都没有行可数; + * 扩展建的表 dump 里只有一部分行(见 verifyRestored 的 extension_member)。 + * 备份文件数不出来、某张表在备份文件里找不到数据段,都判失败:比不了就是没验证过,不拿估算顶上。 + */ +export function rowCountsCheck( + inBackup: Map | null, + manifest: Manifest, + restoredTables: RestoredTable[], + restoredRows: Map +): DrillCheck { + const name = "row counts match the backup"; + if (!inBackup) { + return { name, pass: false, detail: "couldn't count the rows in the backup file, so row counts weren't compared" }; + } + const inManifest = new Set(manifest.database.tables.map((t) => tableKey(t))); + const format = (n: number) => n.toLocaleString("en-US"); + const differ: string[] = []; + const notInFile: string[] = []; + let compared = 0; + let rows = 0; + for (const t of restoredTables) { + if (t.kind !== "r" || t.extension_member || !inManifest.has(tableKey(t))) continue; + const label = `${t.schema}.${t.name}`; + const expected = inBackup.get(tableKey(t)); + if (expected === undefined) { + notInFile.push(label); + continue; + } + compared++; + rows += expected; + const restored = restoredRows.get(label) ?? 0; + if (restored !== expected) differ.push(`${label} has ${format(restored)}, the backup has ${format(expected)}`); + } + const problems = [...differ, ...notInFile.map((label) => `${label} has no data section in the backup file`)]; + if (problems.length > 0) { + const shown = problems.slice(0, 5).join("; "); + const more = problems.length > 5 ? `; and ${problems.length - 5} more` : ""; + return { name, pass: false, detail: `${shown}${more}` }; + } + return { + name, + pass: true, + detail: + compared === 0 + ? "no tables to compare" + : `all ${format(compared)} tables match the backup exactly (${format(rows)} rows)`, + }; +} + /** 导出:真实恢复(restore.ts)复用同一套表级验证——两条路径对同一快照必须同一结论。 */ export async function verifyRestored( connString: string, manifest: Manifest, - /** 恢复用的那份 dump:恢复出空表时回查它,见第 3 项检查 */ + /** 恢复用的那份 dump:逐表行数从它里面数(第 4 项检查),恢复出空表时也回查它(第 3 项) */ dumpPath?: string ): Promise<{ checks: DrillCheck[]; tableCount: number; rowTotal: number }> { + // 备份文件里的确切行数只读本地文件、不连库,和下面逐表 count(*) 同时跑,省掉一段等待 + const backupRowsP = dumpPath ? countRowsInDump(dumpPath) : null; // pgConnectOptions:真实 Supabase 目标(restore 复用本函数)需要打包根 CA; // 沙箱的 docker 连接串非 Supabase 主机,原样透传,行为不变 const client = await connectPg(connString); @@ -469,9 +531,14 @@ export async function verifyRestored( name: string; kind: string; populated: boolean; + extension_member: boolean; }>( + // extension_member:扩展建的表(比如装在 public 里的 PostGIS spatial_ref_sys)。CREATE EXTENSION 会自带 + // 初始行,dump 里只有过了扩展过滤条件的那部分,恢复后的行数天然比 dump 多,不做逐行比对 `select n.nspname as schema, c.relname as name, - c.relkind as kind, c.relispopulated as populated + c.relkind as kind, c.relispopulated as populated, + exists (select 1 from pg_depend d + where d.classid = 'pg_class'::regclass and d.objid = c.oid and d.deptype = 'e') as extension_member from pg_class c join pg_namespace n on n.oid = c.relnamespace where c.relkind in ('r', 'p', 'm') @@ -497,8 +564,11 @@ export async function verifyRestored( } // 名字来自恢复出的库(= 来自 dump = 桶里的数据):必须走标识符转义, // 内嵌引号的合法名不炸,构造名不注入 + // 普通表用 only:老式继承的父表 count(*) 会连子表的行一起算,dump 里父表只有它自己的行。 + // 分区父表本身不存行,照旧数全部分区(只用于"有没有数据",不参与逐表比对) + const only = t.kind === "r" ? "only " : ""; const c = await client.query<{ n: string }>( - `select count(*)::bigint as n from ${quoteIdent(t.schema)}.${quoteIdent(t.name)}` + `select count(*)::bigint as n from ${only}${quoteIdent(t.schema)}.${quoteIdent(t.name)}` ); const n = Number(c.rows[0].n); counts.set(key, n); @@ -562,6 +632,11 @@ export async function verifyRestored( : "no populated table restored empty", }); + // 4. 逐表行数和备份文件里一模一样。manifest 的行数是统计估算,只能管"别变空";确切的备份行数 + // 从 dump 本身数(dump 是 pg_dump 在一个一致快照里导出的,它的行数就是这次备份下来的行数), + // 不去源库 count(*):那要在用户的库上全表扫一遍。数不出来就判失败,不退回估算 + if (backupRowsP) checks.push(rowCountsCheck(await backupRowsP, manifest, restoredTables, counts)); + return { checks, tableCount: restoredTables.length, rowTotal }; } finally { await client.end(); diff --git a/src/manifest.ts b/src/manifest.ts index 2742f27..f52aa35 100644 --- a/src/manifest.ts +++ b/src/manifest.ts @@ -11,7 +11,8 @@ export interface TableStat { schema: string; name: string; // 来自 n_live_tup:是估算值(planner 统计),不是精确 count(*);分区父表恒为 0。 - // 对"体积/行数骤降"这类异常检测足够;需要精确值时云版会在演练里重新计数。 + // 只用于展示、异常检测和演练的"有行的表别变空"。演练的逐表确切比对不用它: + // 确切行数从备份文件本身数(restore-engine countRowsInDump),不在源库上 count(*)。 estimatedRows: number; // ---- 以下为验证升级包预留字段(恢复闭环 PRD §5.1.2 / §0.1 D3):随 v2 一并定稿 // 以免二次升版,但本期不生成。只存聚合结果,绝不落抽样行原始值或 PII。 diff --git a/src/restore-engine.ts b/src/restore-engine.ts index 4a4768e..d5b77aa 100644 --- a/src/restore-engine.ts +++ b/src/restore-engine.ts @@ -1352,3 +1352,118 @@ export function tableDataInDump( }); }); } + +/** + * 逐块读 `pg_restore --data-only -f -` 的整份输出,数出每个 COPY 段的数据行数(键 = tableKey)。 + * 返回喂数据的 feed 和收尾的 finish;读不懂的 COPY 头、超长的头、没收尾的 COPY 段都让 finish 返回 null: + * 数不准就不给数,绝不拿半截结果去比。纯函数,方便单测。 + * + * COPY 文本格式里一行就是一条记录(值里的换行被转义成 \n),所以数 \n 就是数行;结束符是只有 \. 的一行, + * 数据里以 \. 开头的值会被转义成 \\.。数据行只记前三个字符,不缓存整行:大 bytea 行能有几百 MB。 + * COPY 头的多行拼接和解析与 copyDataProbe 相同(带引号的名字里可以有换行)。 + */ +export function copyRowCounter(): { feed(chunk: string): void; finish(): Map | null } { + const counts = new Map(); + let broken = false; + // 不在数据段时:当前行已读到的内容(只在它可能是 COPY 头时才攒),以及攒到一半的多行 COPY 头 + let line = ""; + let skippingLine = false; + let header: string | null = null; + // 在数据段时:正在数的表、已数的行数、当前行的前三个字符 + let current: string | null = null; + let rows = 0; + let linePrefix = ""; + + const endOfHeaderLine = (complete: string) => { + header = header === null ? complete : `${header}\n${complete}`; + const balanced = (header.split('"').length - 1) % 2 === 0; + if (!(balanced && header.endsWith(" FROM stdin;"))) { + if (header.length > MAX_COPY_HEADER) broken = true; + return; + } + const parsed = parseCopyHeader(header); + header = null; + if (!parsed) { + broken = true; + return; + } + current = tableKey(parsed.table); + rows = 0; + linePrefix = ""; + }; + + return { + feed(chunk) { + let at = 0; + while (!broken && at < chunk.length) { + const newline = chunk.indexOf("\n", at); + const end = newline < 0 ? chunk.length : newline; + if (current !== null) { + if (linePrefix.length < 3) linePrefix += chunk.slice(at, Math.min(end, at + 3 - linePrefix.length)); + if (newline < 0) break; + if (linePrefix === "\\.") { + counts.set(current, (counts.get(current) ?? 0) + rows); + current = null; + } else { + rows++; + } + linePrefix = ""; + } else if (skippingLine) { + if (newline < 0) break; + skippingLine = false; + } else { + line += chunk.slice(at, end); + // 不在拼 COPY 头、这一行也不以 COPY 开头:SET、注释、setval、大对象数据,整行跳过不攒 + if (header === null && line.length >= 5 && !line.startsWith("COPY ")) { + line = ""; + skippingLine = newline < 0; + if (newline < 0) break; + } else if ((header?.length ?? 0) + line.length > MAX_COPY_HEADER) { + broken = true; + break; + } else if (newline >= 0) { + const complete = line; + line = ""; + if (header !== null || complete.startsWith("COPY ")) endOfHeaderLine(complete); + } + } + at = end + 1; + } + }, + finish() { + // 输出停在 COPY 段或 COPY 头中间 = pg_restore 没写完,不给数 + if (broken || current !== null || header !== null) return null; + return counts; + }, + }; +} + +/** + * 备份文件里每张表的确切行数:让 pg_restore 把整份 dump 的数据段写成 SQL,边读边数(copyRowCounter)。 + * 不连库,也不碰源库。matview 的数据是 REFRESH、分区父表没有自己的数据,都不会出现在结果里。 + * pg_restore 起不来、出错退出、被中断,或者输出读不懂,都返回 null,由调用方按"数不出来"处理。 + */ +export function countRowsInDump(dumpPath: string): Promise | null> { + if (interrupted()) return Promise.resolve(null); + return new Promise((resolve) => { + // 和 pgRestoreOutput 一样钉 C locale + const proc = spawn(resolvePgRestoreBin(), ["--data-only", "--file=-", dumpPath], { + stdio: ["ignore", "pipe", "ignore"], + env: libpqChildEnv(), + }); + const forget = stopOnInterrupt(proc); + const counter = copyRowCounter(); + // 按流解码:多字节字符可能被拆在两块里 + proc.stdout.setEncoding("utf8"); + proc.stdout.on("data", (chunk: string) => counter.feed(chunk)); + let settled = false; + const settle = (result: Map | null) => { + if (settled) return; + settled = true; + forget(); + resolve(result); + }; + proc.on("error", () => settle(null)); + proc.on("close", (code) => settle(code === 0 ? counter.finish() : null)); + }); +} diff --git a/test/copy-row-counter.test.js b/test/copy-row-counter.test.js new file mode 100644 index 0000000..9de8b70 --- /dev/null +++ b/test/copy-row-counter.test.js @@ -0,0 +1,132 @@ +// copyRowCounter:逐块读 pg_restore --data-only 的整份输出,数出每张表的确切行数; +// rowCountsCheck:恢复后的行数和这份数逐表比。真 dump 的端到端见 drill-row-counts.test.js。 +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { copyRowCounter } from "../dist/restore-engine.js"; +import { rowCountsCheck } from "../dist/drill.js"; + +const key = (schema, name) => `${schema}\u0000${name}`; + +/** 按给定切点逐块喂,返回 finish() 的结果,Map 转成普通对象方便比较 */ +function count(...chunks) { + const counter = copyRowCounter(); + for (const chunk of chunks) counter.feed(chunk); + const result = counter.finish(); + return result === null ? null : Object.fromEntries([...result].map(([k, v]) => [k.replace("\u0000", "."), v])); +} + +const OUTPUT = + "--\n-- PostgreSQL database dump\n--\n\nSET statement_timeout = 0;\n" + + "--\n-- Data for Name: a; Type: TABLE DATA; Schema: public; Owner: -\n--\n\n" + + "COPY public.a (id, v) FROM stdin;\n1\tx\n2\t\\\\.starts with an escaped terminator\n3\tline\\nbreak\n\\.\n\n" + + "COPY public.empty (id) FROM stdin;\n\\.\n\n" + + // 没有列的表:每行数据是空行 + "COPY public.nocols FROM stdin;\n\n\n\n\\.\n\n" + + 'COPY "my schema"."T" ("a""b") FROM stdin;\n1\n\\.\n\n' + + // 值本身就是 \.:COPY 里写成 \\.,不是结束符 + "COPY public.esc (v) FROM stdin;\n\\\\.\n\\\\.x\n\\.\n\n" + + "SELECT pg_catalog.setval('public.a_id_seq', 3, true);\n"; + +const EXPECTED = { "public.a": 3, "public.empty": 0, "public.nocols": 3, "my schema.T": 1, "public.esc": 2 }; + +test("copyRowCounter: counts every COPY section exactly", () => { + assert.deepEqual(count(OUTPUT), EXPECTED); +}); + +test("copyRowCounter: the same answer however the output is split into chunks", () => { + for (let cut = 1; cut < OUTPUT.length; cut++) { + assert.deepEqual(count(OUTPUT.slice(0, cut), OUTPUT.slice(cut)), EXPECTED, `cut at ${cut}`); + } + assert.deepEqual(count(...OUTPUT.split("")), EXPECTED, "one character at a time"); +}); + +test("copyRowCounter: a header that spans lines inside quoted names is read whole", () => { + // 列名里有 " FROM stdin;"、换行和 \.:只看单行会在第一行"结束"头,再把 \. 当成结束符 + const tricky = 'COPY public.tricky (id, "x FROM stdin;\n\\.\ny") FROM stdin;\n1\ta\n2\tb\n\\.\n'; + assert.deepEqual(count(tricky), { "public.tricky": 2 }); + assert.deepEqual(count('COPY public."line\nbreak" (id) FROM stdin;\n1\n\\.\n'), { "public.line\nbreak": 1 }); +}); + +test("copyRowCounter: a huge row is counted without holding the whole line", () => { + const counter = copyRowCounter(); + counter.feed("COPY public.big (id, blob) FROM stdin;\n1\t\\\\x"); + for (let i = 0; i < 50; i++) counter.feed("ab".repeat(65_536)); + counter.feed("\n\\.\n"); + assert.deepEqual(Object.fromEntries(counter.finish()), { [key("public", "big")]: 1 }); +}); + +test("copyRowCounter: anything it can't read exactly gives no counts at all", () => { + // 读不懂的 COPY 头 + assert.equal(count("COPY public.t WITH garbage FROM stdin;\n1\n\\.\n"), null); + // 头超长 + assert.equal(count('COPY public."' + "x".repeat(100_001)), null); + // 输出停在数据段中间、或者停在多行头中间(pg_restore 没写完) + assert.equal(count("COPY public.t (id) FROM stdin;\n1\n2\n"), null); + assert.equal(count('COPY public."half\n'), null); + // 没有 COPY 段:空结果,不是 null(由比对那一步决定缺数据段的表怎么办) + assert.deepEqual(count("REFRESH MATERIALIZED VIEW public.mv;\n"), {}); +}); + +// ── rowCountsCheck ── + +const manifest = (names) => ({ + database: { tables: names.map((name) => ({ schema: "public", name, estimatedRows: 0 })) }, +}); +const table = (name, kind = "r", extension_member = false) => ({ schema: "public", name, kind, extension_member }); + +test("rowCountsCheck: every table equal passes, with the totals", () => { + const check = rowCountsCheck( + new Map([[key("public", "a"), 1000], [key("public", "b"), 0]]), + manifest(["a", "b"]), + [table("a"), table("b")], + new Map([["public.a", 1000], ["public.b", 0]]) + ); + assert.deepEqual(check, { + name: "row counts match the backup", + pass: true, + detail: "all 2 tables match the backup exactly (1,000 rows)", + }); +}); + +test("rowCountsCheck: one row short fails and names the table", () => { + const check = rowCountsCheck( + new Map([[key("public", "orders"), 10_000]]), + manifest(["orders"]), + [table("orders")], + new Map([["public.orders", 9_990]]) + ); + assert.equal(check.pass, false); + assert.equal(check.detail, "public.orders has 9,990, the backup has 10,000"); +}); + +test("rowCountsCheck: matviews, partition parents and extension tables aren't compared", () => { + const check = rowCountsCheck( + new Map([[key("public", "spatial_ref_sys"), 2]]), + manifest(["mv", "parted", "spatial_ref_sys"]), + [table("mv", "m"), table("parted", "p"), table("spatial_ref_sys", "r", true)], + new Map([["public.mv", 5], ["public.parted", 7], ["public.spatial_ref_sys", 8_500]]) + ); + assert.deepEqual(check, { name: "row counts match the backup", pass: true, detail: "no tables to compare" }); +}); + +test("rowCountsCheck: no counts from the backup file, or a table missing from it, fails", () => { + const none = rowCountsCheck(null, manifest(["a"]), [table("a")], new Map([["public.a", 1]])); + assert.equal(none.pass, false); + assert.match(none.detail, /couldn't count the rows in the backup file/); + + const missing = rowCountsCheck(new Map(), manifest(["a"]), [table("a")], new Map([["public.a", 1]])); + assert.equal(missing.pass, false); + assert.equal(missing.detail, "public.a has no data section in the backup file"); +}); + +test("rowCountsCheck: lists at most five problems", () => { + const names = ["t1", "t2", "t3", "t4", "t5", "t6", "t7"]; + const check = rowCountsCheck( + new Map(names.map((n) => [key("public", n), 2])), + manifest(names), + names.map((n) => table(n)), + new Map(names.map((n) => [`public.${n}`, 1])) + ); + assert.equal(check.pass, false); + assert.match(check.detail, /^public\.t1 has 1, the backup has 2; .*public\.t5 has 1, the backup has 2; and 2 more$/); +}); diff --git a/test/drill-row-counts.test.js b/test/drill-row-counts.test.js new file mode 100644 index 0000000..328780b --- /dev/null +++ b/test/drill-row-counts.test.js @@ -0,0 +1,148 @@ +// 逐表确切行数比对(2.11.0):演练从备份文件本身数出每张表的行数,和恢复后的 count(*) 逐表比, +// 少一行也判失败。manifest 里的行数是统计估算,只能管"别变空"。 +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; +import { createHash } from "node:crypto"; +import { readFileSync, mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { drillDump, verifyRestored } from "../dist/drill.js"; +import { countRowsInDump } from "../dist/restore-engine.js"; + +const x = promisify(execFile); +const pgDump = process.env.BACKUPDRILL_PG_DUMP || "pg_dump"; +const pgRestore = process.env.BACKUPDRILL_PG_RESTORE || "pg_restore"; + +async function available(cmd, args) { + try { + await x(cmd, args); + return true; + } catch { + return false; + } +} +const canRun = (await available("docker", ["version"])) && (await available(pgDump, ["--version"])); + +/** 起一个一次性 Postgres,等它在 TCP 上就绪,返回容器 id 和连接串。 */ +async function startPostgres() { + const { stdout } = await x("docker", [ + "run", "-d", "--rm", "-e", "POSTGRES_PASSWORD=seed", "-p", "127.0.0.1:0:5432", "postgres:17-alpine", + ]); + const id = stdout.trim(); + const port = (await x("docker", ["port", id, "5432/tcp"])).stdout.trim().split(":").pop().trim(); + for (let i = 0; i < 60; i++) { + try { + // -h 强制 TCP:镜像初始化阶段的临时服务只听 socket(见 drill.test.js) + await x("docker", ["exec", id, "pg_isready", "-h", "127.0.0.1", "-U", "postgres", "-q"]); + break; + } catch { + await new Promise((r) => setTimeout(r, 500)); + } + } + return { id, conn: `postgresql://postgres:seed@127.0.0.1:${port}/postgres` }; +} + +const psql = (id, sql) => x("docker", ["exec", id, "psql", "-U", "postgres", "-v", "ON_ERROR_STOP=1", "-c", sql]); + +test( + "drill: row counts are compared exactly against the backup file, one missing row fails", + { skip: canRun ? false : "requires Docker + pg_dump", timeout: 300_000 }, + async () => { + const tmp = mkdtempSync(join(tmpdir(), "bd-row-counts-")); + const dumpPath = join(tmp, "dump.pgcustom"); + const source = await startPostgres(); + let target = null; + try { + await psql( + source.id, + // orders:值里有换行、制表符、反斜杠、以 \. 开头的文本,COPY 里都转义在一行之内 + "create table orders(id int primary key, note text); " + + "insert into orders select g, case g % 4 when 0 then E'two\\nlines' when 1 then E'tab\\there' " + + "when 2 then E'\\\\.looks like the end' else 'plain' end from generate_series(1, 1000) g; " + + // 老式继承:父表自己 2 行、子表 3 行。父表的 count(*) 会算上子表,dump 里父表只有自己的 2 行 + "create table animals(id int, name text); create table dogs() inherits (animals); " + + "insert into animals values (1, 'cat'), (2, 'owl'); insert into dogs values (3, 'a'), (4, 'b'), (5, 'c'); " + + // 分区表:数据在两个分区上,父表本身不比 + "create table events(id int, at date) partition by range (at); " + + "create table events_2025 partition of events for values from ('2025-01-01') to ('2026-01-01'); " + + "create table events_2026 partition of events for values from ('2026-01-01') to ('2027-01-01'); " + + "insert into events select g, date '2025-06-01' + g from generate_series(1, 400) g; " + + // 没有列的表:每行数据是空行 + "create table nocols(); insert into nocols select from generate_series(1, 3); " + + "create table empty_table(id int); " + + 'create table "Odd Name"(id int); insert into "Odd Name" values (1), (2); ' + + "create materialized view orders_mv as select id from orders where id <= 10;" + ); + await x(pgDump, [ + "--format=custom", "--no-owner", "--no-privileges", "--schema=public", "--dbname", source.conn, "-f", dumpPath, + ]); + + // 1. 备份文件里的确切行数 + const inBackup = await countRowsInDump(dumpPath); + assert.deepEqual( + Object.fromEntries([...inBackup].map(([k, v]) => [k.replace("\u0000", "."), v]).sort()), + { + "public.Odd Name": 2, + "public.animals": 2, + "public.dogs": 3, + "public.empty_table": 0, + "public.events_2025": 213, + "public.events_2026": 187, + "public.nocols": 3, + "public.orders": 1000, + } + ); + assert.equal(await countRowsInDump(join(tmp, "missing.pgcustom")), null); + + const names = ["Odd Name", "animals", "dogs", "empty_table", "events", "events_2025", "events_2026", "nocols", "orders", "orders_mv"]; + const buf = readFileSync(dumpPath); + const manifest = { + tool: "backupdrill-cli", toolVersion: "test", createdAt: "2026-11-07T00:00:00.000Z", + projectName: "test", + database: { + serverVersion: "17.4", pgDumpVersion: "test", schemas: ["public"], + tableCount: names.length, estimatedRowTotal: 0, + // 估算全写 0:逐表比对只认备份文件里数出来的数,和 manifest 的估算无关 + tables: names.map((name) => ({ schema: "public", name, estimatedRows: 0 })), + }, + dump: { + key: "seed/dump.pgcustom", format: "custom", bytes: buf.length, + sha256: createHash("sha256").update(buf).digest("hex"), + }, + storage: null, + }; + + // 2. 完整演练:每张表都对上 + const report = await drillDump(dumpPath, manifest, "row-counts"); + const rows = report.checks.find((c) => c.name === "row counts match the backup"); + assert.deepEqual(rows, { + name: "row counts match the backup", + pass: true, + detail: "all 8 tables match the backup exactly (1,410 rows)", + }); + assert.equal(report.pass, true, JSON.stringify(report.checks)); + + // 3. 恢复后少一行、子表多一行:都要抓到,并且不是靠"表变空"那项 + target = await startPostgres(); + // dump 自带 CREATE SCHEMA public,新库里已有同名 schema 会让 pg_restore 退出码非 0 + await psql(target.id, "drop schema public cascade;"); + await x(pgRestore, ["--no-owner", "--no-privileges", "--dbname", target.conn, dumpPath]); + await psql(target.id, "delete from orders where id = 500; insert into dogs values (6, 'd');"); + const verified = await verifyRestored(target.conn, manifest, dumpPath); + const check = verified.checks.find((c) => c.name === "row counts match the backup"); + assert.equal(check.pass, false); + assert.equal(check.detail, "public.dogs has 4, the backup has 3; public.orders has 999, the backup has 1,000"); + assert.equal(verified.checks.find((c) => c.name === "populated tables came back").pass, true); + + // 4. 不给 dump 路径(老调用方式):没有可比的确切行数,这一项不出现 + const withoutDump = await verifyRestored(target.conn, manifest); + assert.equal(withoutDump.checks.find((c) => c.name === "row counts match the backup"), undefined); + } finally { + await x("docker", ["rm", "-f", "-v", source.id]).catch(() => {}); + if (target) await x("docker", ["rm", "-f", "-v", target.id]).catch(() => {}); + rmSync(tmp, { recursive: true, force: true }); + } + } +); From 690feb719d2268c7e6f9f2cc2d710504d7b01393 Mon Sep 17 00:00:00 2001 From: Super <20788648+supertj@users.noreply.github.com> Date: Sat, 7 Nov 2026 19:45:39 +0800 Subject: [PATCH 2/4] fix: key restored counts by schema and name, count only TABLE DATA entries, allow headers up to 1 MB Dot-joined keys made public."a.b" and "public.a".b one table, so a row missing from one could pass. setval and large-object SQL can hold a line that starts with COPY inside a quoted name; pg_restore now writes only the TABLE DATA entries for counting. A valid 1600-column COPY header can pass 100,000 characters. --- src/drill.ts | 12 ++++--- src/dump-sql.ts | 8 +++-- src/restore-engine.ts | 65 ++++++++++++++++++++++------------- test/copy-data-probe.test.js | 2 +- test/copy-row-counter.test.js | 14 ++++---- test/drill-row-counts.test.js | 60 +++++++++++++++++++++++--------- 6 files changed, 106 insertions(+), 55 deletions(-) diff --git a/src/drill.ts b/src/drill.ts index e42ff48..c5fac95 100644 --- a/src/drill.ts +++ b/src/drill.ts @@ -467,6 +467,7 @@ export function rowCountsCheck( inBackup: Map | null, manifest: Manifest, restoredTables: RestoredTable[], + /** 键 = tableKey */ restoredRows: Map ): DrillCheck { const name = "row counts match the backup"; @@ -489,7 +490,7 @@ export function rowCountsCheck( } compared++; rows += expected; - const restored = restoredRows.get(label) ?? 0; + const restored = restoredRows.get(tableKey(t)) ?? 0; if (restored !== expected) differ.push(`${label} has ${format(restored)}, the backup has ${format(expected)}`); } const problems = [...differ, ...notInFile.map((label) => `${label} has no data section in the backup file`)]; @@ -553,7 +554,8 @@ export async function verifyRestored( const unpopulatedMatviews = new Set(); let rowTotal = 0; for (const t of restoredTables) { - const key = `${t.schema}.${t.name}`; + // tableKey(NUL 分隔),不用点号拼:public."a.b" 和 "public.a".b 拼出来都是 public.a.b,会串表 + const key = tableKey(t); // 未填充的 matview 无法 count(*)(Postgres 直接报错)。它要么备份时就没数据 // (dump 里本来就没有 REFRESH),要么 REFRESH 在 post-data 阶段失败——后者已由 // post-data 检查如实记录,这里按 0 行处理即可。 @@ -589,7 +591,7 @@ export async function verifyRestored( // 2. manifest 里的每张表都还原出来了 const restoredSet = new Set(counts.keys()); const missing = manifest.database.tables.filter( - (t) => !restoredSet.has(`${t.schema}.${t.name}`) + (t) => !restoredSet.has(tableKey(t)) ); checks.push({ name: "no missing tables", @@ -607,8 +609,8 @@ export async function verifyRestored( const restoredEmpty = manifest.database.tables.filter( (t) => t.estimatedRows > 0 && - !unpopulatedMatviews.has(`${t.schema}.${t.name}`) && - (counts.get(`${t.schema}.${t.name}`) ?? 0) === 0 + !unpopulatedMatviews.has(tableKey(t)) && + (counts.get(tableKey(t)) ?? 0) === 0 ); // manifest 的行数是 pg_dump 之前另开连接取的 n_live_tup(统计估算),和 dump 不是同一个快照。 // 队列、会话、一次性密钥这类表,取统计时有行、dump 时已经删光,恢复出 0 行是对的 diff --git a/src/dump-sql.ts b/src/dump-sql.ts index 3acab8a..a403c9c 100644 --- a/src/dump-sql.ts +++ b/src/dump-sql.ts @@ -157,8 +157,12 @@ export function parseGeneratedColumns(sql: string): Map> { return tables; } -/** 一个 COPY 头最长读这么多字符;超过就当作读不了,免得坏归档把内存吃满。 */ -export const MAX_COPY_HEADER = 100_000; +/** + * 一个 COPY 头最长读这么多字符;超过就当作读不了,免得坏归档把内存吃满。 + * 合法的头可以很长:1600 列(上限)、每个列名 63 字节且带引号要转义时,光列清单就有二十多万字符。 + * 1 MB 留足余量,仍然挡得住坏归档。 + */ +export const MAX_COPY_HEADER = 1_000_000; /** * 数据段的 COPY 头(dumpTableData_copy):"COPY <表> <列清单> FROM stdin;"。没有列的表列清单是空串, diff --git a/src/restore-engine.ts b/src/restore-engine.ts index d5b77aa..7564b55 100644 --- a/src/restore-engine.ts +++ b/src/restore-engine.ts @@ -1439,31 +1439,50 @@ export function copyRowCounter(): { feed(chunk: string): void; finish(): Map | null> { - if (interrupted()) return Promise.resolve(null); - return new Promise((resolve) => { - // 和 pgRestoreOutput 一样钉 C locale - const proc = spawn(resolvePgRestoreBin(), ["--data-only", "--file=-", dumpPath], { - stdio: ["ignore", "pipe", "ignore"], - env: libpqChildEnv(), +export async function countRowsInDump(dumpPath: string): Promise | null> { + if (interrupted()) return null; + const bin = resolvePgRestoreBin(); + let listing: { code: number | null; stdout: string }; + try { + listing = await pgRestoreOutput(bin, ["--list", dumpPath]); + } catch { + return null; + } + if (listing.code !== 0) return null; + const dataEntries = listing.stdout.split("\n").filter((line) => /^\d+; 0 \d+ TABLE DATA /.test(line)); + if (dataEntries.length === 0) return new Map(); + const listDir = await mkdtemp(join(tmpdir(), "backupdrill-rowcount-")); + try { + const listFile = join(listDir, "table-data.list"); + await writeFile(listFile, dataEntries.join("\n") + "\n"); + return await new Promise((resolve) => { + // 和 pgRestoreOutput 一样钉 C locale + const proc = spawn(bin, ["--data-only", `--use-list=${listFile}`, "--file=-", dumpPath], { + stdio: ["ignore", "pipe", "ignore"], + env: libpqChildEnv(), + }); + const forget = stopOnInterrupt(proc); + const counter = copyRowCounter(); + // 按流解码:多字节字符可能被拆在两块里 + proc.stdout.setEncoding("utf8"); + proc.stdout.on("data", (chunk: string) => counter.feed(chunk)); + let settled = false; + const settle = (result: Map | null) => { + if (settled) return; + settled = true; + forget(); + resolve(result); + }; + proc.on("error", () => settle(null)); + proc.on("close", (code) => settle(code === 0 ? counter.finish() : null)); }); - const forget = stopOnInterrupt(proc); - const counter = copyRowCounter(); - // 按流解码:多字节字符可能被拆在两块里 - proc.stdout.setEncoding("utf8"); - proc.stdout.on("data", (chunk: string) => counter.feed(chunk)); - let settled = false; - const settle = (result: Map | null) => { - if (settled) return; - settled = true; - forget(); - resolve(result); - }; - proc.on("error", () => settle(null)); - proc.on("close", (code) => settle(code === 0 ? counter.finish() : null)); - }); + } finally { + await rm(listDir, { recursive: true, force: true }); + } } diff --git a/test/copy-data-probe.test.js b/test/copy-data-probe.test.js index 092ac4e..f00b28b 100644 --- a/test/copy-data-probe.test.js +++ b/test/copy-data-probe.test.js @@ -54,7 +54,7 @@ test("copyDataProbe: quoted names may hold CR, CRLF or a whole fake header", () test("copyDataProbe: another table's header or an unreadable one is unknown, never a verdict", () => { assert.equal(feed("public", "t", "COPY public.other (id) FROM stdin;\n\\.\n"), "unknown"); assert.equal(feed("public", "t", "COPY public.t WITH garbage FROM stdin;\n\\.\n"), "unknown"); - assert.equal(feed("public", "t", 'COPY public."' + "x".repeat(100_001)), "unknown"); + assert.equal(feed("public", "t", 'COPY public."' + "x".repeat(1_000_001)), "unknown"); // 输出里没有 COPY 段(比如 matview 只有 REFRESH):喂完也没有结论,调用方按查不清处理 assert.equal(feed("public", "mv", PREAMBLE + "REFRESH MATERIALIZED VIEW public.mv;\n"), null); }); diff --git a/test/copy-row-counter.test.js b/test/copy-row-counter.test.js index 9de8b70..8a319bb 100644 --- a/test/copy-row-counter.test.js +++ b/test/copy-row-counter.test.js @@ -59,7 +59,7 @@ test("copyRowCounter: anything it can't read exactly gives no counts at all", () // 读不懂的 COPY 头 assert.equal(count("COPY public.t WITH garbage FROM stdin;\n1\n\\.\n"), null); // 头超长 - assert.equal(count('COPY public."' + "x".repeat(100_001)), null); + assert.equal(count('COPY public."' + "x".repeat(1_000_001)), null); // 输出停在数据段中间、或者停在多行头中间(pg_restore 没写完) assert.equal(count("COPY public.t (id) FROM stdin;\n1\n2\n"), null); assert.equal(count('COPY public."half\n'), null); @@ -79,7 +79,7 @@ test("rowCountsCheck: every table equal passes, with the totals", () => { new Map([[key("public", "a"), 1000], [key("public", "b"), 0]]), manifest(["a", "b"]), [table("a"), table("b")], - new Map([["public.a", 1000], ["public.b", 0]]) + new Map([[key("public", "a"), 1000], [key("public", "b"), 0]]) ); assert.deepEqual(check, { name: "row counts match the backup", @@ -93,7 +93,7 @@ test("rowCountsCheck: one row short fails and names the table", () => { new Map([[key("public", "orders"), 10_000]]), manifest(["orders"]), [table("orders")], - new Map([["public.orders", 9_990]]) + new Map([[key("public", "orders"), 9_990]]) ); assert.equal(check.pass, false); assert.equal(check.detail, "public.orders has 9,990, the backup has 10,000"); @@ -104,17 +104,17 @@ test("rowCountsCheck: matviews, partition parents and extension tables aren't co new Map([[key("public", "spatial_ref_sys"), 2]]), manifest(["mv", "parted", "spatial_ref_sys"]), [table("mv", "m"), table("parted", "p"), table("spatial_ref_sys", "r", true)], - new Map([["public.mv", 5], ["public.parted", 7], ["public.spatial_ref_sys", 8_500]]) + new Map([[key("public", "mv"), 5], [key("public", "parted"), 7], [key("public", "spatial_ref_sys"), 8_500]]) ); assert.deepEqual(check, { name: "row counts match the backup", pass: true, detail: "no tables to compare" }); }); test("rowCountsCheck: no counts from the backup file, or a table missing from it, fails", () => { - const none = rowCountsCheck(null, manifest(["a"]), [table("a")], new Map([["public.a", 1]])); + const none = rowCountsCheck(null, manifest(["a"]), [table("a")], new Map([[key("public", "a"), 1]])); assert.equal(none.pass, false); assert.match(none.detail, /couldn't count the rows in the backup file/); - const missing = rowCountsCheck(new Map(), manifest(["a"]), [table("a")], new Map([["public.a", 1]])); + const missing = rowCountsCheck(new Map(), manifest(["a"]), [table("a")], new Map([[key("public", "a"), 1]])); assert.equal(missing.pass, false); assert.equal(missing.detail, "public.a has no data section in the backup file"); }); @@ -125,7 +125,7 @@ test("rowCountsCheck: lists at most five problems", () => { new Map(names.map((n) => [key("public", n), 2])), manifest(names), names.map((n) => table(n)), - new Map(names.map((n) => [`public.${n}`, 1])) + new Map(names.map((n) => [key("public", n), 1])) ); assert.equal(check.pass, false); assert.match(check.detail, /^public\.t1 has 1, the backup has 2; .*public\.t5 has 1, the backup has 2; and 2 more$/); diff --git a/test/drill-row-counts.test.js b/test/drill-row-counts.test.js index 328780b..245c9b5 100644 --- a/test/drill-row-counts.test.js +++ b/test/drill-row-counts.test.js @@ -75,37 +75,62 @@ test( 'create table "Odd Name"(id int); insert into "Odd Name" values (1), (2); ' + "create materialized view orders_mv as select id from orders where id <= 10;" ); + // 交叉审查复现的三种: + // 1. 点号拼键会串表:public."a.b" 和 "public.a".b 拼出来都是 public.a.b + // 2. 序列名里带换行和 "COPY ...":setval 的字符串字面量里会出现一行以 COPY 开头的假头 + // 3. 1600 列、列名 63 字节带引号:COPY 头超过十万字符,仍是合法的 + const wideColumns = Array.from({ length: 1600 }, (_, i) => `"${'"'.repeat(20)}${String(i).padStart(4, "0")}${"c".repeat(39)}" int`); + await psql( + source.id, + 'create table "a.b"(id int); insert into "a.b" values (1), (2), (3); ' + + 'create schema "public.a"; create table "public.a".b(id int); insert into "public.a".b values (1); ' + + 'create table fake_target(id int); insert into fake_target values (1), (2); ' + + 'create sequence "s\nCOPY public.fake_target (id) FROM stdin;\n9"; ' + + `create table wide(${wideColumns.join(", ")}); insert into wide ("${'"'.repeat(20)}0000${"c".repeat(39)}") values (1), (2);` + ); await x(pgDump, [ - "--format=custom", "--no-owner", "--no-privileges", "--schema=public", "--dbname", source.conn, "-f", dumpPath, + "--format=custom", "--no-owner", "--no-privileges", "--schema=public", '--schema="public.a"', "--dbname", source.conn, "-f", dumpPath, ]); // 1. 备份文件里的确切行数 const inBackup = await countRowsInDump(dumpPath); assert.deepEqual( - Object.fromEntries([...inBackup].map(([k, v]) => [k.replace("\u0000", "."), v]).sort()), + // 用 | 拼,不用点号:测的就是点号会串表 + Object.fromEntries([...inBackup].map(([k, v]) => [k.replace("\u0000", "|"), v])), { - "public.Odd Name": 2, - "public.animals": 2, - "public.dogs": 3, - "public.empty_table": 0, - "public.events_2025": 213, - "public.events_2026": 187, - "public.nocols": 3, - "public.orders": 1000, + "public|Odd Name": 2, + "public|a.b": 3, + "public.a|b": 1, + "public|animals": 2, + "public|dogs": 3, + "public|empty_table": 0, + "public|events_2025": 213, + "public|events_2026": 187, + "public|fake_target": 2, + "public|nocols": 3, + "public|orders": 1000, + "public|wide": 2, } ); assert.equal(await countRowsInDump(join(tmp, "missing.pgcustom")), null); - const names = ["Odd Name", "animals", "dogs", "empty_table", "events", "events_2025", "events_2026", "nocols", "orders", "orders_mv"]; + const names = [ + "Odd Name", "a.b", "animals", "dogs", "empty_table", "events", "events_2025", "events_2026", + "fake_target", "nocols", "orders", "orders_mv", "wide", + ]; + const tables = [ + ...names.map((name) => ({ schema: "public", name, estimatedRows: 0 })), + { schema: "public.a", name: "b", estimatedRows: 0 }, + ]; const buf = readFileSync(dumpPath); const manifest = { tool: "backupdrill-cli", toolVersion: "test", createdAt: "2026-11-07T00:00:00.000Z", projectName: "test", database: { - serverVersion: "17.4", pgDumpVersion: "test", schemas: ["public"], - tableCount: names.length, estimatedRowTotal: 0, + serverVersion: "17.4", pgDumpVersion: "test", schemas: ["public", "public.a"], + tableCount: tables.length, estimatedRowTotal: 0, // 估算全写 0:逐表比对只认备份文件里数出来的数,和 manifest 的估算无关 - tables: names.map((name) => ({ schema: "public", name, estimatedRows: 0 })), + tables, }, dump: { key: "seed/dump.pgcustom", format: "custom", bytes: buf.length, @@ -120,7 +145,7 @@ test( assert.deepEqual(rows, { name: "row counts match the backup", pass: true, - detail: "all 8 tables match the backup exactly (1,410 rows)", + detail: "all 12 tables match the backup exactly (1,418 rows)", }); assert.equal(report.pass, true, JSON.stringify(report.checks)); @@ -129,11 +154,12 @@ test( // dump 自带 CREATE SCHEMA public,新库里已有同名 schema 会让 pg_restore 退出码非 0 await psql(target.id, "drop schema public cascade;"); await x(pgRestore, ["--no-owner", "--no-privileges", "--dbname", target.conn, dumpPath]); - await psql(target.id, "delete from orders where id = 500; insert into dogs values (6, 'd');"); + // "a.b" 少一行、"public.a".b 不动:键串了的话两张表会被当成一张,这一行就漏掉 + await psql(target.id, "delete from orders where id = 500; insert into dogs values (6, 'd'); delete from \"a.b\" where id = 1;"); const verified = await verifyRestored(target.conn, manifest, dumpPath); const check = verified.checks.find((c) => c.name === "row counts match the backup"); assert.equal(check.pass, false); - assert.equal(check.detail, "public.dogs has 4, the backup has 3; public.orders has 999, the backup has 1,000"); + assert.equal(check.detail, "public.a.b has 2, the backup has 3; public.dogs has 4, the backup has 3; public.orders has 999, the backup has 1,000"); assert.equal(verified.checks.find((c) => c.name === "populated tables came back").pass, true); // 4. 不给 dump 路径(老调用方式):没有可比的确切行数,这一项不出现 From d8cf4954b52fce9d2d7b9a51e89e24363d3ea0a8 Mon Sep 17 00:00:00 2001 From: Super <20788648+supertj@users.noreply.github.com> Date: Sat, 7 Nov 2026 19:50:17 +0800 Subject: [PATCH 3/4] fix: counting the backup's rows never rejects; README names extension tables among the exclusions --- README.md | 2 +- src/restore-engine.ts | 10 ++++++++++ test/drill-row-counts.test.js | 9 +++++++++ 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index e634112..6debcd4 100644 --- a/README.md +++ b/README.md @@ -179,7 +179,7 @@ backupdrill drill # latest snapshot; add --snapshot | null> { + // 绝不抛错:调用方是在逐表 count(*) 之前就把它挂在后台的,临时目录建不了、写不进、删不掉, + // 都只能落成"数不出来",不能变成没人接的 rejection 把演练进程带崩 + try { + return await countRowsInDumpOrThrow(dumpPath); + } catch { + return null; + } +} + +async function countRowsInDumpOrThrow(dumpPath: string): Promise | null> { if (interrupted()) return null; const bin = resolvePgRestoreBin(); let listing: { code: number | null; stdout: string }; diff --git a/test/drill-row-counts.test.js b/test/drill-row-counts.test.js index 245c9b5..43bafc2 100644 --- a/test/drill-row-counts.test.js +++ b/test/drill-row-counts.test.js @@ -113,6 +113,15 @@ test( } ); assert.equal(await countRowsInDump(join(tmp, "missing.pgcustom")), null); + // 临时目录建不了:返回 null(演练判"数不出来"),不能抛出去变成没人接的 rejection + const savedTmp = process.env.TMPDIR; + process.env.TMPDIR = join(tmp, "no-such-dir"); + try { + assert.equal(await countRowsInDump(dumpPath), null); + } finally { + if (savedTmp === undefined) delete process.env.TMPDIR; + else process.env.TMPDIR = savedTmp; + } const names = [ "Odd Name", "a.b", "animals", "dogs", "empty_table", "events", "events_2025", "events_2026", From 4cd60b99c1962749d9d85581034887ae63486534 Mon Sep 17 00:00:00 2001 From: Super <20788648+supertj@users.noreply.github.com> Date: Sat, 7 Nov 2026 21:07:41 +0800 Subject: [PATCH 4/4] fix: stop counting the backup's rows when verification fails partway --- src/drill.ts | 9 +++++++-- src/restore-engine.ts | 19 +++++++++++++------ test/drill-row-counts.test.js | 15 ++++++++++++++- 3 files changed, 34 insertions(+), 9 deletions(-) diff --git a/src/drill.ts b/src/drill.ts index c5fac95..1b118c9 100644 --- a/src/drill.ts +++ b/src/drill.ts @@ -516,11 +516,13 @@ export async function verifyRestored( /** 恢复用的那份 dump:逐表行数从它里面数(第 4 项检查),恢复出空表时也回查它(第 3 项) */ dumpPath?: string ): Promise<{ checks: DrillCheck[]; tableCount: number; rowTotal: number }> { - // 备份文件里的确切行数只读本地文件、不连库,和下面逐表 count(*) 同时跑,省掉一段等待 - const backupRowsP = dumpPath ? countRowsInDump(dumpPath) : null; // pgConnectOptions:真实 Supabase 目标(restore 复用本函数)需要打包根 CA; // 沙箱的 docker 连接串非 Supabase 主机,原样透传,行为不变 const client = await connectPg(connString); + // 备份文件里的确切行数只读本地文件、不连库,和下面逐表 count(*) 同时跑,省掉一段等待。 + // 连上库之后才开始;这一步中途失败时由 finally 停掉它,不让它在演练收尾之后还在后台读整份 dump + const stopCounting = new AbortController(); + const backupRowsP = dumpPath ? countRowsInDump(dumpPath, stopCounting.signal) : null; try { const schemas = manifest.database.schemas; // 口径必须与 manifest 统计端(backup.ts inspectDatabase)完全一致: @@ -641,6 +643,9 @@ export async function verifyRestored( return { checks, tableCount: restoredTables.length, rowTotal }; } finally { + // 正常走完时它早已结束,这两步什么都不做 + stopCounting.abort(); + await backupRowsP; await client.end(); } } diff --git a/src/restore-engine.ts b/src/restore-engine.ts index 533172f..1f11431 100644 --- a/src/restore-engine.ts +++ b/src/restore-engine.ts @@ -1445,28 +1445,29 @@ export function copyRowCounter(): { feed(chunk: string): void; finish(): Map | null> { +export async function countRowsInDump(dumpPath: string, signal?: AbortSignal): Promise | null> { // 绝不抛错:调用方是在逐表 count(*) 之前就把它挂在后台的,临时目录建不了、写不进、删不掉, // 都只能落成"数不出来",不能变成没人接的 rejection 把演练进程带崩 try { - return await countRowsInDumpOrThrow(dumpPath); + return await countRowsInDumpOrThrow(dumpPath, signal); } catch { return null; } } -async function countRowsInDumpOrThrow(dumpPath: string): Promise | null> { - if (interrupted()) return null; +async function countRowsInDumpOrThrow(dumpPath: string, signal?: AbortSignal): Promise | null> { + if (interrupted() || signal?.aborted) return null; const bin = resolvePgRestoreBin(); let listing: { code: number | null; stdout: string }; try { - listing = await pgRestoreOutput(bin, ["--list", dumpPath]); + listing = await pgRestoreOutput(bin, ["--list", dumpPath], { signal }); } catch { return null; } if (listing.code !== 0) return null; const dataEntries = listing.stdout.split("\n").filter((line) => /^\d+; 0 \d+ TABLE DATA /.test(line)); if (dataEntries.length === 0) return new Map(); + if (signal?.aborted) return null; const listDir = await mkdtemp(join(tmpdir(), "backupdrill-rowcount-")); try { const listFile = join(listDir, "table-data.list"); @@ -1489,8 +1490,14 @@ async function countRowsInDumpOrThrow(dumpPath: string): Promise proc.kill("SIGTERM"); + signal?.addEventListener("abort", stop, { once: true }); proc.on("error", () => settle(null)); - proc.on("close", (code) => settle(code === 0 ? counter.finish() : null)); + proc.on("close", (code) => { + signal?.removeEventListener("abort", stop); + settle(code === 0 && !signal?.aborted ? counter.finish() : null); + }); }); } finally { await rm(listDir, { recursive: true, force: true }); diff --git a/test/drill-row-counts.test.js b/test/drill-row-counts.test.js index 43bafc2..5bff566 100644 --- a/test/drill-row-counts.test.js +++ b/test/drill-row-counts.test.js @@ -113,6 +113,11 @@ test( } ); assert.equal(await countRowsInDump(join(tmp, "missing.pgcustom")), null); + // 叫停:返回 null,不读完整份 dump + const stop = new AbortController(); + const stopped = countRowsInDump(dumpPath, stop.signal); + stop.abort(); + assert.equal(await stopped, null); // 临时目录建不了:返回 null(演练判"数不出来"),不能抛出去变成没人接的 rejection const savedTmp = process.env.TMPDIR; process.env.TMPDIR = join(tmp, "no-such-dir"); @@ -171,7 +176,15 @@ test( assert.equal(check.detail, "public.a.b has 2, the backup has 3; public.dogs has 4, the backup has 3; public.orders has 999, the backup has 1,000"); assert.equal(verified.checks.find((c) => c.name === "populated tables came back").pass, true); - // 4. 不给 dump 路径(老调用方式):没有可比的确切行数,这一项不出现 + // 4. 校验中途失败(这个角色读不了 orders,count(*) 报错):verifyRestored 照样抛错, + // 但后台数 dump 的 pg_restore 要一起停掉,不能在演练收尾之后还在读 + await psql(target.id, "create role limited login password 'lim'; grant usage on schema public to limited; grant select on all tables in schema public to limited; revoke select on orders from limited;"); + const limitedConn = target.conn.replace("postgres:seed@", "limited:lim@"); + await assert.rejects(verifyRestored(limitedConn, manifest, dumpPath), /permission denied/); + const leftover = await x("pgrep", ["-f", dumpPath]).then((r) => r.stdout.trim(), () => ""); + assert.equal(leftover, "", "no pg_restore still reading the dump"); + + // 5. 不给 dump 路径(老调用方式):没有可比的确切行数,这一项不出现 const withoutDump = await verifyRestored(target.conn, manifest); assert.equal(withoutDump.checks.find((c) => c.name === "row counts match the backup"), undefined); } finally {