From b75adcc85e9e6bc781c7b8b6d93511098e65a935 Mon Sep 17 00:00:00 2001 From: Jacob Cable Date: Wed, 2 Sep 2026 20:18:03 +0100 Subject: [PATCH 1/8] feat(firestore-bigquery-export): reinstate the Cloud Tasks write buffer Port the extension's syncBigQuery queue into the kit (option D, #3031): a failed inline write enqueues the serialized change onto a task queue (5 attempts, 60s min backoff, MAX_DISPATCHES_PER_SECOND throttle) instead of replaying through Eventarc for 24h. The task handler self-heals (ensureInitialized) before re-attempting and rethrows so Cloud Tasks retries; tracker 2.1.0 backs the row up to BACKUP_COLLECTION on every terminal insert failure. The trigger keeps retry: true so a failed enqueue - logged at error level and rethrown, unlike the extension's silent swallow - is redelivered rather than dropped. Enqueue targets the bare function name on firebase-admin ^14.2.0, which resolves the kit-- prefix from FIREBASE_KIT_INSTANCE_ID (Firebase CLI 15.28.0+); queue region derives from DATABASE_REGION with FUNCTION_REGION as fallback. MAX_DISPATCHES_PER_SECOND (default 100) and MAX_ENQUEUE_ATTEMPTS (default 3) keep their extension names so migrated .env values carry over. --- kits/firestore-bigquery-export/CHANGELOG.md | 1 + kits/firestore-bigquery-export/README.md | 133 ++- .../npm-shrinkwrap.json | 914 ++++++++++++++++-- kits/firestore-bigquery-export/package.json | 2 +- kits/firestore-bigquery-export/src/config.ts | 37 +- .../src/export-config.ts | 17 + .../firestore-bigquery-export/src/handlers.ts | 111 ++- kits/firestore-bigquery-export/src/index.ts | 46 +- kits/firestore-bigquery-export/src/lib.ts | 2 + kits/firestore-bigquery-export/src/logs.ts | 4 +- kits/firestore-bigquery-export/src/tasks.ts | 98 ++ .../tests/config.test.ts | 35 + .../tests/handlers.test.ts | 138 ++- .../tests/index.test.ts | 37 +- .../tests/tasks.test.ts | 142 +++ 15 files changed, 1572 insertions(+), 145 deletions(-) create mode 100644 kits/firestore-bigquery-export/src/tasks.ts create mode 100644 kits/firestore-bigquery-export/tests/tasks.test.ts diff --git a/kits/firestore-bigquery-export/CHANGELOG.md b/kits/firestore-bigquery-export/CHANGELOG.md index d4a4a7407a..76f5c70c7d 100644 --- a/kits/firestore-bigquery-export/CHANGELOG.md +++ b/kits/firestore-bigquery-export/CHANGELOG.md @@ -1,3 +1,4 @@ +- feat: reinstate the extension's Cloud Tasks write buffer. A failed inline BigQuery write now enqueues onto a new `syncBigQuery` task queue (5 attempts, 60s minimum backoff, throttled by the restored `MAX_DISPATCHES_PER_SECOND` param, default 100) instead of replaying the Firestore event through Eventarc redelivery for up to 24 hours; `MAX_ENQUEUE_ATTEMPTS` (default 3) is also back. The `onSuccess` event returns with the queue handler. Two behavior changes against earlier release candidates: a row that exhausts the queue is dropped unless `BACKUP_COLLECTION` is set (extension parity - the tracker backs the row up on every terminal insert failure, so configure a backup collection), and deleting or moving the functions can leave the Cloud Tasks queue behind. One deliberate fix over the extension: a failed enqueue is logged at error level and rethrown so the trigger's retry redelivers the event, where the extension silently dropped it. Export the new `syncBigQuery` function from your codebase entry, and deploy with Firebase CLI 15.28.0+ so the trigger can address its own queue (`FIREBASE_KIT_INSTANCE_ID`); requires firebase-admin 14.2.0+. - fix: restore explicit function placement from `DATABASE_REGION`, now with the Firestore-location-to-Cloud-Run-region mapping. The `DATABASE_REGION` parameter is back and all three functions deploy to the region derived from it: regional locations pass through unchanged, and the multi-region locations map to a Cloud Run region (`nam5`/`nam7` to `us-central1`, `eur3` to `europe-west1`) instead of failing the deploy. With the parameter unset the functions still declare no region and the CLI falls back as before (`us-central1` by default, `FIREBASE_FUNCTIONS_DEFAULT_REGION` to override). Placement requires firebase-tools >= 15.28.0 (older CLIs do not load `.env` at discovery and keep the fallback). If your `.env` already carries `DATABASE_REGION` from an extension migration, upgrading to this version moves the functions to the mapped region on your next deploy, which deletes and recreates them. - fix: stop deploying functions to the `DATABASE_REGION` value. Firestore multi-region locations (`eur3`, `nam5`, `nam7`) are not Cloud Run regions, so any multi-region database made every deploy fail. The `DATABASE_REGION` parameter is removed; the functions now declare no region and deploy to `us-central1` by default (set `FIREBASE_FUNCTIONS_DEFAULT_REGION` when deploying to choose another region), while the Firestore trigger is always pinned to the database's own region. `ExportConfig.location` is removed from the library surface. - Initial release of kit, see README for differences between the legacy extension and this kit diff --git a/kits/firestore-bigquery-export/README.md b/kits/firestore-bigquery-export/README.md index 8ffb4e27d2..99ceefaf3e 100644 --- a/kits/firestore-bigquery-export/README.md +++ b/kits/firestore-bigquery-export/README.md @@ -5,9 +5,10 @@ BigQuery Firebase Extension as an npm package you add to your own Firebase Functions codebase and deploy. It listens for document writes on a collection, serializes each change, and -writes it to a BigQuery changelog table. Failed writes are retried through a -Firebase Functions runtime retry policy. The functions run in your own Firebase -project; there is no hosted version, so you deploy them yourself. +writes it to a BigQuery changelog table. Failed writes buffer through a Cloud +Tasks queue (`syncBigQuery`), which retries them on its own throttled schedule. +The functions run in your own Firebase project; there is no hosted version, so +you deploy them yourself. ## Install @@ -30,6 +31,7 @@ conflicts with that automatic setup. | `roles/datastore.user` | write failed-row records back to Firestore (only if you configure a backup collection) | | `roles/eventarc.eventReceiver` | receive Gen2 Firestore trigger events | | `roles/run.invoker` | allow Eventarc to invoke the Gen2 Cloud Run service | +| `roles/cloudtasks.enqueuer` | enqueue failed writes onto the kit's own `syncBigQuery` task queue | | `bigquery.googleapis.com` | mirror Firestore collection changes in BigQuery | If the dataset lives in a different project (`BIGQUERY_PROJECT_ID`), grant the @@ -38,12 +40,13 @@ CMEK dataset, also grant the BigQuery service account access to your KMS key. ## Usage -Export the three functions from your functions codebase entry: +Export the four functions from your functions codebase entry: ```ts // functions/src/index.ts export { fsexportbigquery, + syncBigQuery, initBigQuerySync, setupBigQuerySync, } from "@firebase-function-kits/firestore-bigquery-export"; @@ -59,6 +62,7 @@ DATABASE_REGION=europe-west2 ``` - `fsexportbigquery` is the Firestore trigger. +- `syncBigQuery` is the write-buffer task queue that retries failed writes. - `initBigQuerySync` is the first-deploy provisioning lifecycle task. - `setupBigQuerySync` is the reconfigure provisioning lifecycle task. @@ -87,8 +91,14 @@ later, behind the `kits` experiment): `instances` maps each instance id to the directory (relative to `firebase.json`) holding that instance's `.env`. The CLI prefixes every function and task queue name with `kit--`, so the functions above -deploy as `kit-default-fsexportbigquery`, `kit-default-initBigQuerySync`, and -`kit-default-setupBigQuerySync`. +deploy as `kit-default-fsexportbigquery`, `kit-default-syncBigQuery`, +`kit-default-initBigQuerySync`, and `kit-default-setupBigQuerySync`. + +Deploy with Firebase CLI 15.28.0 or later: it sets the +`FIREBASE_KIT_INSTANCE_ID` env var on the deployed functions, which the trigger +needs to address its own `syncBigQuery` queue. On functions deployed with an +older CLI, enqueues fail (loudly - the event is redelivered, not lost) until +you redeploy with a newer CLI. ```sh firebase experiments:enable kits @@ -112,7 +122,9 @@ loads them at deploy time and prompts for any required values that are missing. | `datasetLocation` | `DATASET_LOCATION` | no | `us` | BigQuery dataset location | | `database` | `DATABASE` | no | `(default)` | Firestore database id | | `bigqueryProjectId` | `BIGQUERY_PROJECT_ID` | no | project id | Dataset project, if different | -| `backupCollection` | `BACKUP_COLLECTION` | no | (empty) | Firestore collection for failed rows | +| `backupCollection` | `BACKUP_COLLECTION` | no | (empty) | Strongly recommended: collection for rows the queue gave up on | +| `maxDispatchesPerSecond` | `MAX_DISPATCHES_PER_SECOND` | no | `100` | `syncBigQuery` queue dispatch rate (1-500) | +| `maxEnqueueAttempts` | `MAX_ENQUEUE_ATTEMPTS` | no | `3` | In-process enqueue attempts before rethrowing (1-10) | | `transformFunction` | `TRANSFORM_FUNCTION` | no | (empty) | Optional transform Cloud Function | | `tablePartitioning` | `TABLE_PARTITIONING` | no | `NONE` | Table partitioning strategy | | `timePartitioningField` | `TIME_PARTITIONING_FIELD` | no | (empty) | Time-partitioning column name | @@ -154,11 +166,11 @@ the instances cannot collide. ## Events -When `EVENTARC_CHANNEL` is configured, the function publishes `onStart` and -`onError` lifecycle events under -`firebase.extensions.firestore-bigquery-export.v1.*`. The extension's -`onSuccess` event is not published; see the events entry under -"Differences from the Stream Firestore to BigQuery extension" below. +When `EVENTARC_CHANNEL` is configured, the functions publish lifecycle events +under `firebase.extensions.firestore-bigquery-export.v1.*`: `onStart` and +`onError` from the write path, and `onSuccess` from the `syncBigQuery` task +when a buffered write lands (matching the extension, which only emitted +`onSuccess` from its queue handler). ## Provisioning @@ -216,10 +228,51 @@ curl -fsS -X POST -H "Content-Type: application/json" -d '{"data":{}}' \ ``` The Firestore write path never provisions on the hot path. If resources are -missing when a write arrives, the inline write fails, the handler calls -`ensureInitialized()` once as a self-heal and retries the write, and a remaining -failure is surfaced to the function runtime retry policy (`retry: true` on -`fsexportbigquery`). +missing when a write arrives, the inline write fails and the change buffers +through the `syncBigQuery` queue, whose handler calls `ensureInitialized()` as +a self-heal before re-attempting the write. + +## Failure handling + +The write path mirrors the extension's Cloud Tasks buffer: + +1. The trigger attempts the BigQuery insert inline. On success, done. +2. On failure, it enqueues the serialized change onto the `syncBigQuery` queue + (up to `MAX_ENQUEUE_ATTEMPTS` in-process attempts with backoff) and the + execution succeeds - the Firestore event is not redelivered. +3. `syncBigQuery` re-attempts the write on the queue's schedule: 5 attempts, + 60 seconds minimum backoff, throttled to `MAX_DISPATCHES_PER_SECOND` + dispatches per second (500 concurrent max). +4. On every terminal insert failure the tracker writes the row to + `BACKUP_COLLECTION` (when configured), keyed by the event id, before the + task fails. After the fifth attempt the task is dropped. **Without a backup + collection, the row is dropped with it** - configure `BACKUP_COLLECTION`. +5. If the enqueue itself fails (BigQuery AND Cloud Tasks both failing), the + trigger logs at error level and rethrows, so the Firestore event is + redelivered by the runtime retry policy (`retry: true`) instead of being + lost. The extension silently dropped the event in this window; this kit + does not. + +### Recovering parked rows + +Rows in `BACKUP_COLLECTION` are changelog-shaped documents, not plain document +snapshots, so `fs-bq-import-collection` cannot consume them. Treat them as +"possibly failed": a transient failure that later succeeded on retry also +leaves one behind, and nothing cleans them up. To recover after an outage, +load the backup docs' fields into a temp table and `MERGE` them into the +changelog table with a `WHEN NOT MATCHED` condition on `event_id` (the +anti-join is mandatory because of those stale rows). See +[firebase/extensions#3031](https://github.com/firebase/extensions/issues/3031) +for the full recipe. + +### Known limits + +- A task queue is a project-level resource created for each task function. + Deleting the functions (or moving them to another region) can leave the old + queue behind; delete it in the Cloud Tasks console if it lingers. +- Rows that exhaust the queue with no `BACKUP_COLLECTION` configured are gone. + This matches the extension; it is the reason the backup collection is + strongly recommended. ## Differences from the Stream Firestore to BigQuery extension @@ -234,23 +287,27 @@ boolean params, and only the literal string `true` enables them. The extension used `yes` / `no` for the last two, so copying an old config across leaves them silently disabled. Change any `yes` to `true` in your `.env`. -### Failed writes retry differently +### Failed writes: same buffer, one fix -The extension pushed a failed BigQuery write onto a Cloud Tasks queue -(`syncBigQuery`) and retried it from there. This kit has no task queue on the -write path. A failed write is retried once in place, and anything still failing -is handed to the Cloud Functions runtime retry policy, which redelivers the -Firestore event. +The kit keeps the extension's write-path architecture: a failed BigQuery write +buffers through the `syncBigQuery` Cloud Tasks queue, with the same shape (5 +attempts, 60s minimum backoff, `MAX_DISPATCHES_PER_SECOND` throttling) and the +same knobs (`MAX_DISPATCHES_PER_SECOND`, `MAX_ENQUEUE_ATTEMPTS`) - your +migrated `.env` values carry over unchanged. -The practical effects: retries no longer show up as a separate function or -queue in the console, and the two knobs that tuned that queue, -`MAX_DISPATCHES_PER_SECOND` and `MAX_ENQUEUE_ATTEMPTS`, no longer exist. +One deliberate fix: when the enqueue itself failed, the extension swallowed the +error and dropped the event with no trace. The kit logs it at error level and +rethrows so the Firestore event is redelivered (`retry: true` on the trigger). +You only pay that redelivery cost in the window where BigQuery and Cloud Tasks +are failing at the same time. -### Events +Earlier release candidates of this kit had no queue: they retried every failed +write through Eventarc redelivery for up to 24 hours and never lost a row +inside that window. That property is gone by design - a row that exhausts the +queue without a configured `BACKUP_COLLECTION` is dropped, exactly as in the +extension. Set `BACKUP_COLLECTION`. -`onSuccess` is no longer published. The extension emitted it from the task -queue handler, which is gone, so the kit publishes `onStart` and `onError` -only. +### Events Events are published under `firebase.extensions.firestore-bigquery-export.v1.*` only. The extension also published a duplicate copy of every event under @@ -321,14 +378,16 @@ BigQuery changelog table, so they still work against data this kit writes. ## API surface - **Main entry** (`@firebase-function-kits/firestore-bigquery-export`): exports - `fsexportbigquery`, `initBigQuerySync`, and `setupBigQuerySync`, and - registers the first-deploy / redeploy provisioning hooks. Runtime config is - resolved lazily on first invocation. Use this entry from Firebase - deploy/emulator/runtime. For your own triggers, import from `./lib` instead. -- **Library entry** (`./lib`): `handleDocumentWrite`, the raw handler for owning - trigger registration yourself, plus the config types and helpers - (`ExportConfig`, `resolveExportConfig`, `toTrackerConfig`) for building its - injected `HandlerContext`. Safe to import anywhere. + `fsexportbigquery`, `syncBigQuery`, `initBigQuerySync`, and + `setupBigQuerySync`, and registers the first-deploy / redeploy provisioning + hooks. Runtime config is resolved lazily on first invocation. Use this entry + from Firebase deploy/emulator/runtime. For your own triggers, import from + `./lib` instead. +- **Library entry** (`./lib`): `handleDocumentWrite` and + `handleSyncBigQueryTask`, the raw handlers for owning trigger registration + yourself, plus the config types and helpers (`ExportConfig`, + `resolveExportConfig`, `toTrackerConfig`, `SerializedDocumentChange`) for + building their injected `HandlerContext`. Safe to import anywhere. The change-tracker engine is an internal dependency and is not exported. diff --git a/kits/firestore-bigquery-export/npm-shrinkwrap.json b/kits/firestore-bigquery-export/npm-shrinkwrap.json index 90d3b25216..6c2a65714d 100644 --- a/kits/firestore-bigquery-export/npm-shrinkwrap.json +++ b/kits/firestore-bigquery-export/npm-shrinkwrap.json @@ -11,7 +11,7 @@ "dependencies": { "@firebaseextensions/firestore-bigquery-change-tracker": "^2.1.0", "@google-cloud/bigquery": "^7.6.0", - "firebase-admin": "^13.6.0", + "firebase-admin": "^14.2.0", "firebase-functions": "^7.3.3-rc.0", "generate-schema": "^2.6.0", "lodash": "^4.17.14" @@ -603,6 +603,23 @@ "traverse": "^0.6.6" } }, + "node_modules/@firebaseextensions/firestore-bigquery-change-tracker/node_modules/@google-cloud/firestore": { + "version": "7.11.6", + "resolved": "https://registry.npmjs.org/@google-cloud/firestore/-/firestore-7.11.6.tgz", + "integrity": "sha512-EW/O8ktzwLfyWBOsNuhRoMi8lrC3clHM5LVFhGvO1HCsLozCOOXRAlHrYBoE6HL42Sc8yYMuCb2XqcnJ4OOEpw==", + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "@opentelemetry/api": "^1.3.0", + "fast-deep-equal": "^3.1.1", + "functional-red-black-tree": "^1.0.1", + "google-gax": "^4.3.3", + "protobufjs": "^7.2.6" + }, + "engines": { + "node": ">=14.0.0" + } + }, "node_modules/@firebaseextensions/firestore-bigquery-change-tracker/node_modules/@types/express": { "version": "4.17.25", "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.25.tgz", @@ -782,6 +799,29 @@ "node": ">= 0.8" } }, + "node_modules/@firebaseextensions/firestore-bigquery-change-tracker/node_modules/firebase-admin": { + "version": "13.10.0", + "resolved": "https://registry.npmjs.org/firebase-admin/-/firebase-admin-13.10.0.tgz", + "integrity": "sha512-rbuCrJvYRwqBqvbccMS8fj/x2zsaMisdf5RQbRzQzr14Rbq9r2UlpuBHqWAwrO6c9dIRF56xF/xoepXsD5yDuQ==", + "license": "Apache-2.0", + "dependencies": { + "@fastify/busboy": "^3.0.0", + "@firebase/database-compat": "^2.0.0", + "@firebase/database-types": "^1.0.6", + "farmhash-modern": "^1.1.0", + "fast-deep-equal": "^3.1.1", + "google-auth-library": "^10.6.1", + "jsonwebtoken": "^9.0.0", + "jwks-rsa": "^3.1.0" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@google-cloud/firestore": "^7.11.0", + "@google-cloud/storage": "^7.19.0" + } + }, "node_modules/@firebaseextensions/firestore-bigquery-change-tracker/node_modules/firebase-functions": { "version": "6.6.0", "resolved": "https://registry.npmjs.org/firebase-functions/-/firebase-functions-6.6.0.tgz", @@ -813,6 +853,60 @@ "node": ">= 0.6" } }, + "node_modules/@firebaseextensions/firestore-bigquery-change-tracker/node_modules/gaxios": { + "version": "7.3.1", + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-7.3.1.tgz", + "integrity": "sha512-kB3rzJV7d9juLZh8/56QTXCwQfxyhdOMdyYk1HdQKFtF8TJTDTZQJtixWIwXdE9Jji91mC41DUNpjleo4L4eAQ==", + "license": "Apache-2.0", + "dependencies": { + "extend": "^3.0.2", + "https-proxy-agent": "^7.0.1", + "node-fetch": "^3.3.2" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@firebaseextensions/firestore-bigquery-change-tracker/node_modules/gcp-metadata": { + "version": "8.1.2", + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-8.1.2.tgz", + "integrity": "sha512-zV/5HKTfCeKWnxG0Dmrw51hEWFGfcF2xiXqcA3+J90WDuP0SvoiSO5ORvcBsifmx/FoIjgQN3oNOGaQ5PhLFkg==", + "license": "Apache-2.0", + "dependencies": { + "gaxios": "^7.0.0", + "google-logging-utils": "^1.0.0", + "json-bigint": "^1.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@firebaseextensions/firestore-bigquery-change-tracker/node_modules/google-auth-library": { + "version": "10.9.1", + "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-10.9.1.tgz", + "integrity": "sha512-i1ydyHrqcIxXkWh/uBmVkzCvIuq5yiK2ATndIe5XxKholrG/MTYP9xGYka4sQhrbIAgGjL2B6NOE7rFaiF3fXw==", + "license": "Apache-2.0", + "dependencies": { + "base64-js": "^1.3.0", + "ecdsa-sig-formatter": "^1.0.11", + "gaxios": "^7.1.4", + "gcp-metadata": "8.1.2", + "google-logging-utils": "1.1.3", + "jws": "^4.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@firebaseextensions/firestore-bigquery-change-tracker/node_modules/google-logging-utils": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/google-logging-utils/-/google-logging-utils-1.1.3.tgz", + "integrity": "sha512-eAmLkjDjAFCVXg7A1unxHsLf961m6y17QFqXqAXGj/gVkKFrEICfStRfwUlGNfeCEjNRa32JEWOUTlYXPyyKvA==", + "license": "Apache-2.0", + "engines": { + "node": ">=14" + } + }, "node_modules/@firebaseextensions/firestore-bigquery-change-tracker/node_modules/iconv-lite": { "version": "0.4.24", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", @@ -825,6 +919,70 @@ "node": ">=0.10.0" } }, + "node_modules/@firebaseextensions/firestore-bigquery-change-tracker/node_modules/jose": { + "version": "4.15.9", + "resolved": "https://registry.npmjs.org/jose/-/jose-4.15.9.tgz", + "integrity": "sha512-1vUQX+IdDMVPj4k8kOxgUqlcK518yluMuGZwqlr44FS1ppZB/5GWh4rZG89erpOBOJjU/OBsnCVFfapsRz6nEA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/panva" + } + }, + "node_modules/@firebaseextensions/firestore-bigquery-change-tracker/node_modules/jwks-rsa": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/jwks-rsa/-/jwks-rsa-3.2.2.tgz", + "integrity": "sha512-BqTyEDV+lS8F2trk3A+qJnxV5Q9EqKCBJOPti3W97r7qTympCZjb7h2X6f2kc+0K3rsSTY1/6YG2eaXKoj497w==", + "license": "MIT", + "dependencies": { + "@types/jsonwebtoken": "^9.0.4", + "debug": "^4.3.4", + "jose": "^4.15.4", + "limiter": "^1.1.5", + "lru-memoizer": "^2.2.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/@firebaseextensions/firestore-bigquery-change-tracker/node_modules/jwks-rsa/node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/@firebaseextensions/firestore-bigquery-change-tracker/node_modules/lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "license": "ISC", + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/@firebaseextensions/firestore-bigquery-change-tracker/node_modules/lru-memoizer": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/lru-memoizer/-/lru-memoizer-2.3.0.tgz", + "integrity": "sha512-GXn7gyHAMhO13WSKrIiNfztwxodVsP8IoZ3XfrJV4yH2x0/OeTO/FIaAHTY5YekdGgW94njfuKmyyt1E0mR6Ug==", + "license": "MIT", + "dependencies": { + "lodash.clonedeep": "^4.5.0", + "lru-cache": "6.0.0" + } + }, "node_modules/@firebaseextensions/firestore-bigquery-change-tracker/node_modules/media-typer": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", @@ -885,6 +1043,24 @@ "node": ">= 0.6" } }, + "node_modules/@firebaseextensions/firestore-bigquery-change-tracker/node_modules/node-fetch": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", + "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", + "license": "MIT", + "dependencies": { + "data-uri-to-buffer": "^4.0.0", + "fetch-blob": "^3.1.4", + "formdata-polyfill": "^4.0.10" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/node-fetch" + } + }, "node_modules/@firebaseextensions/firestore-bigquery-change-tracker/node_modules/path-to-regexp": { "version": "0.1.13", "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.13.tgz", @@ -1026,20 +1202,227 @@ } }, "node_modules/@google-cloud/firestore": { - "version": "7.11.6", - "resolved": "https://registry.npmjs.org/@google-cloud/firestore/-/firestore-7.11.6.tgz", - "integrity": "sha512-EW/O8ktzwLfyWBOsNuhRoMi8lrC3clHM5LVFhGvO1HCsLozCOOXRAlHrYBoE6HL42Sc8yYMuCb2XqcnJ4OOEpw==", + "version": "8.7.1", + "resolved": "https://registry.npmjs.org/@google-cloud/firestore/-/firestore-8.7.1.tgz", + "integrity": "sha512-Hp/WI8sH569ANitsko6RNvCUkzTCYudHoINOkQjiJq2FBG6kKnofBAW7PtS823cu6RMxHqK2RxRcspdjrRpUFA==", "license": "Apache-2.0", "optional": true, "dependencies": { - "@opentelemetry/api": "^1.3.0", - "fast-deep-equal": "^3.1.1", + "@opentelemetry/api": "^1.9.0", + "fast-deep-equal": "^3.1.3", "functional-red-black-tree": "^1.0.1", - "google-gax": "^4.3.3", - "protobufjs": "^7.2.6" + "google-gax": "^5.0.1", + "protobufjs": "^7.5.3" }, "engines": { - "node": ">=14.0.0" + "node": ">=18" + } + }, + "node_modules/@google-cloud/firestore/node_modules/@grpc/proto-loader": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.8.1.tgz", + "integrity": "sha512-wtF6h+DY6M3YaDBPAmvuuA6jV8Sif9MjtOI5euKFWRgCDl5PeDpPsHR9u2l6St5ceY8AZgoNDww5+HvEsXFsGg==", + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "lodash.camelcase": "^4.3.0", + "long": "^5.0.0", + "protobufjs": "^7.5.5", + "yargs": "^17.7.2" + }, + "bin": { + "proto-loader-gen-types": "build/bin/proto-loader-gen-types.js" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/@google-cloud/firestore/node_modules/gaxios": { + "version": "7.3.1", + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-7.3.1.tgz", + "integrity": "sha512-kB3rzJV7d9juLZh8/56QTXCwQfxyhdOMdyYk1HdQKFtF8TJTDTZQJtixWIwXdE9Jji91mC41DUNpjleo4L4eAQ==", + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "extend": "^3.0.2", + "https-proxy-agent": "^7.0.1", + "node-fetch": "^3.3.2" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@google-cloud/firestore/node_modules/gcp-metadata": { + "version": "8.1.4", + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-8.1.4.tgz", + "integrity": "sha512-iJ9KMsiu+xKtNRX0PmGLSaIU3bUBAyzWTyqKemKPzNPsmmsBCQYmlNg+brEbES7IHSXtdVwzBPzx1vz3FAaipw==", + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "gaxios": "7.1.3", + "google-logging-utils": "1.1.3", + "json-bigint": "^1.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@google-cloud/firestore/node_modules/gcp-metadata/node_modules/gaxios": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-7.1.3.tgz", + "integrity": "sha512-YGGyuEdVIjqxkxVH1pUTMY/XtmmsApXrCVv5EU25iX6inEPbV+VakJfLealkBtJN69AQmh1eGOdCl9Sm1UP6XQ==", + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "extend": "^3.0.2", + "https-proxy-agent": "^7.0.1", + "node-fetch": "^3.3.2", + "rimraf": "^5.0.1" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@google-cloud/firestore/node_modules/google-auth-library": { + "version": "10.5.0", + "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-10.5.0.tgz", + "integrity": "sha512-7ABviyMOlX5hIVD60YOfHw4/CxOfBhyduaYB+wbFWCWoni4N7SLcV46hrVRktuBbZjFC9ONyqamZITN7q3n32w==", + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "base64-js": "^1.3.0", + "ecdsa-sig-formatter": "^1.0.11", + "gaxios": "^7.0.0", + "gcp-metadata": "^8.0.0", + "google-logging-utils": "^1.0.0", + "gtoken": "^8.0.0", + "jws": "^4.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@google-cloud/firestore/node_modules/google-gax": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/google-gax/-/google-gax-5.0.8.tgz", + "integrity": "sha512-M4vpZcXQIC1gqIVGQ7eaU3jXQA6zecStyTXu514TYfThlgSurYJOxHZo9fzU6hAgwPWvuEynAVHWyaIk80VeEA==", + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "@grpc/grpc-js": "^1.12.6", + "@grpc/proto-loader": "^0.8.0", + "duplexify": "^4.1.3", + "google-auth-library": "10.5.0", + "google-logging-utils": "1.1.3", + "node-fetch": "^3.3.2", + "object-hash": "^3.0.0", + "proto3-json-serializer": "3.0.4", + "protobufjs": "^7.5.4", + "retry-request": "^8.0.2", + "rimraf": "^5.0.1" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@google-cloud/firestore/node_modules/google-logging-utils": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/google-logging-utils/-/google-logging-utils-1.1.3.tgz", + "integrity": "sha512-eAmLkjDjAFCVXg7A1unxHsLf961m6y17QFqXqAXGj/gVkKFrEICfStRfwUlGNfeCEjNRa32JEWOUTlYXPyyKvA==", + "license": "Apache-2.0", + "optional": true, + "engines": { + "node": ">=14" + } + }, + "node_modules/@google-cloud/firestore/node_modules/gtoken": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-8.0.0.tgz", + "integrity": "sha512-+CqsMbHPiSTdtSO14O51eMNlrp9N79gmeqmXeouJOhfucAedHw9noVe/n5uJk3tbKE6a+6ZCQg3RPhVhHByAIw==", + "license": "MIT", + "optional": true, + "dependencies": { + "gaxios": "^7.0.0", + "jws": "^4.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@google-cloud/firestore/node_modules/http-proxy-agent": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", + "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==", + "license": "MIT", + "optional": true, + "dependencies": { + "agent-base": "^7.1.0", + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/@google-cloud/firestore/node_modules/node-fetch": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", + "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", + "license": "MIT", + "optional": true, + "dependencies": { + "data-uri-to-buffer": "^4.0.0", + "fetch-blob": "^3.1.4", + "formdata-polyfill": "^4.0.10" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/node-fetch" + } + }, + "node_modules/@google-cloud/firestore/node_modules/proto3-json-serializer": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/proto3-json-serializer/-/proto3-json-serializer-3.0.4.tgz", + "integrity": "sha512-E1sbAYg3aEbXrq0n1ojJkRHQJGE1kaE/O6GLA94y8rnJBfgvOPTOd1b9hOceQK1FFZI9qMh1vBERCyO2ifubcw==", + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "protobufjs": "^7.4.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@google-cloud/firestore/node_modules/retry-request": { + "version": "8.0.4", + "resolved": "https://registry.npmjs.org/retry-request/-/retry-request-8.0.4.tgz", + "integrity": "sha512-pI6/7eabUYkZxamkOq0g0uMxKLLGnjzhefY+vL8bVXag5rto4OU2YBTPytWLuHH7aEKD6fn7kJycQfid0Mwnkw==", + "license": "MIT", + "optional": true, + "dependencies": { + "extend": "^3.0.2", + "teeny-request": "^10.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@google-cloud/firestore/node_modules/teeny-request": { + "version": "10.1.4", + "resolved": "https://registry.npmjs.org/teeny-request/-/teeny-request-10.1.4.tgz", + "integrity": "sha512-R1Cg4Vu0UULeDfHL/kjABLaTW++9yD/B6n2g48y5dJ04hsEaxcfmAqbNDzNsbqAYJyIpZafjklLG9YxRu9uzOg==", + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "http-proxy-agent": "^7.0.0", + "https-proxy-agent": "^7.0.1", + "node-fetch": "^3.3.2", + "stream-events": "^1.0.5" + }, + "engines": { + "node": ">=18" } }, "node_modules/@google-cloud/paginator": { @@ -1130,43 +1513,146 @@ "@js-sdsl/ordered-map": "^4.4.2" }, "engines": { - "node": ">=12.10.0" + "node": ">=12.10.0" + } + }, + "node_modules/@grpc/grpc-js/node_modules/@grpc/proto-loader": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.8.1.tgz", + "integrity": "sha512-wtF6h+DY6M3YaDBPAmvuuA6jV8Sif9MjtOI5euKFWRgCDl5PeDpPsHR9u2l6St5ceY8AZgoNDww5+HvEsXFsGg==", + "license": "Apache-2.0", + "dependencies": { + "lodash.camelcase": "^4.3.0", + "long": "^5.0.0", + "protobufjs": "^7.5.5", + "yargs": "^17.7.2" + }, + "bin": { + "proto-loader-gen-types": "build/bin/proto-loader-gen-types.js" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/@grpc/proto-loader": { + "version": "0.7.15", + "resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.7.15.tgz", + "integrity": "sha512-tMXdRCfYVixjuFK+Hk0Q1s38gV9zDiDJfWL3h1rv4Qc39oILCu1TRTDt7+fGUI8K4G1Fj125Hx/ru3azECWTyQ==", + "license": "Apache-2.0", + "dependencies": { + "lodash.camelcase": "^4.3.0", + "long": "^5.0.0", + "protobufjs": "^7.2.5", + "yargs": "^17.7.2" + }, + "bin": { + "proto-loader-gen-types": "build/bin/proto-loader-gen-types.js" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/@isaacs/cliui": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", + "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==", + "license": "ISC", + "optional": true, + "dependencies": { + "string-width": "^5.1.2", + "string-width-cjs": "npm:string-width@^4.2.0", + "strip-ansi": "^7.0.1", + "strip-ansi-cjs": "npm:strip-ansi@^6.0.1", + "wrap-ansi": "^8.1.0", + "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/@isaacs/cliui/node_modules/ansi-regex": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.3.0.tgz", + "integrity": "sha512-WpDfL7NO6j7tH88IDBNVdUJxDh9nmCteAVW9dsep846XdwF4naCBK+/tGLX3KJgcpgMRXCFlTM2hKGoK9FsdrQ==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-regex?sponsor=1" + } + }, + "node_modules/@isaacs/cliui/node_modules/ansi-styles": { + "version": "6.2.3", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.3.tgz", + "integrity": "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/@isaacs/cliui/node_modules/emoji-regex": { + "version": "9.2.2", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", + "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==", + "license": "MIT", + "optional": true + }, + "node_modules/@isaacs/cliui/node_modules/string-width": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", + "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", + "license": "MIT", + "optional": true, + "dependencies": { + "eastasianwidth": "^0.2.0", + "emoji-regex": "^9.2.2", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/@grpc/grpc-js/node_modules/@grpc/proto-loader": { - "version": "0.8.1", - "resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.8.1.tgz", - "integrity": "sha512-wtF6h+DY6M3YaDBPAmvuuA6jV8Sif9MjtOI5euKFWRgCDl5PeDpPsHR9u2l6St5ceY8AZgoNDww5+HvEsXFsGg==", - "license": "Apache-2.0", + "node_modules/@isaacs/cliui/node_modules/strip-ansi": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.2.0.tgz", + "integrity": "sha512-yDPMNjp4WyfYBkHnjIRLfca1i6KMyGCtsVgoKe/z1+6vukgaENdgGBZt+ZmKPc4gavvEZ5OgHfHdrazhgNyG7w==", + "license": "MIT", + "optional": true, "dependencies": { - "lodash.camelcase": "^4.3.0", - "long": "^5.0.0", - "protobufjs": "^7.5.5", - "yargs": "^17.7.2" - }, - "bin": { - "proto-loader-gen-types": "build/bin/proto-loader-gen-types.js" + "ansi-regex": "^6.2.2" }, "engines": { - "node": ">=6" + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/strip-ansi?sponsor=1" } }, - "node_modules/@grpc/proto-loader": { - "version": "0.7.15", - "resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.7.15.tgz", - "integrity": "sha512-tMXdRCfYVixjuFK+Hk0Q1s38gV9zDiDJfWL3h1rv4Qc39oILCu1TRTDt7+fGUI8K4G1Fj125Hx/ru3azECWTyQ==", - "license": "Apache-2.0", + "node_modules/@isaacs/cliui/node_modules/wrap-ansi": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", + "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", + "license": "MIT", + "optional": true, "dependencies": { - "lodash.camelcase": "^4.3.0", - "long": "^5.0.0", - "protobufjs": "^7.2.5", - "yargs": "^17.7.2" - }, - "bin": { - "proto-loader-gen-types": "build/bin/proto-loader-gen-types.js" + "ansi-styles": "^6.1.0", + "string-width": "^5.0.1", + "strip-ansi": "^7.0.1" }, "engines": { - "node": ">=6" + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" } }, "node_modules/@jridgewell/sourcemap-codec": { @@ -1229,6 +1715,16 @@ "node": ">=8.0.0" } }, + "node_modules/@pkgjs/parseargs": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", + "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=14" + } + }, "node_modules/@protobufjs/aspromise": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", @@ -2153,6 +2649,13 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "license": "MIT", + "optional": true + }, "node_modules/base64-js": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", @@ -2232,6 +2735,16 @@ "url": "https://opencollective.com/express" } }, + "node_modules/brace-expansion": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.4.tgz", + "integrity": "sha512-hGfVzPxthbf3+2yjg/RBs60cB0FhqBS/zvdV/4wn4/BmN0bNMMHPc4V/BbFieqf1TKAGGAHnY4eSjajCl0f2Xg==", + "license": "MIT", + "optional": true, + "dependencies": { + "balanced-match": "^1.0.0" + } + }, "node_modules/buffer-equal-constant-time": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", @@ -2438,6 +2951,21 @@ "url": "https://opencollective.com/express" } }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "license": "MIT", + "optional": true, + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, "node_modules/data-uri-to-buffer": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", @@ -2613,6 +3141,13 @@ "stream-shift": "^1.0.2" } }, + "node_modules/eastasianwidth": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", + "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==", + "license": "MIT", + "optional": true + }, "node_modules/ecdsa-sig-formatter": { "version": "1.0.11", "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", @@ -3085,26 +3620,25 @@ } }, "node_modules/firebase-admin": { - "version": "13.10.0", - "resolved": "https://registry.npmjs.org/firebase-admin/-/firebase-admin-13.10.0.tgz", - "integrity": "sha512-rbuCrJvYRwqBqvbccMS8fj/x2zsaMisdf5RQbRzQzr14Rbq9r2UlpuBHqWAwrO6c9dIRF56xF/xoepXsD5yDuQ==", + "version": "14.3.0", + "resolved": "https://registry.npmjs.org/firebase-admin/-/firebase-admin-14.3.0.tgz", + "integrity": "sha512-FUxr5HI9C0RNJW3B+CKyJOnIt5uXn/XHheyCECVTLQGSJ/MaE1Zcwf+dCaaG+xx0+VX1A4sZ1t+hc35bu73dVw==", "license": "Apache-2.0", "dependencies": { "@fastify/busboy": "^3.0.0", - "@firebase/database-compat": "^2.0.0", - "@firebase/database-types": "^1.0.6", - "farmhash-modern": "^1.1.0", + "@firebase/database-compat": "^2.1.4", + "@firebase/database-types": "^1.0.20", "fast-deep-equal": "^3.1.1", - "google-auth-library": "^10.6.1", + "google-auth-library": "^10.6.2", "jsonwebtoken": "^9.0.0", - "jwks-rsa": "^3.1.0" + "jwks-rsa": "^4.0.1" }, "engines": { - "node": ">=18" + "node": ">=22" }, "optionalDependencies": { - "@google-cloud/firestore": "^7.11.0", - "@google-cloud/storage": "^7.19.0" + "@google-cloud/firestore": "^8.7.1", + "@google-cloud/storage": "^7.22.0" } }, "node_modules/firebase-admin/node_modules/gaxios": { @@ -3230,6 +3764,23 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/foreground-child": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.3.1.tgz", + "integrity": "sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw==", + "license": "ISC", + "optional": true, + "dependencies": { + "cross-spawn": "^7.0.6", + "signal-exit": "^4.0.1" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/form-data": { "version": "2.5.6", "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.5.6.tgz", @@ -3476,6 +4027,28 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/glob": { + "version": "10.5.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-10.5.0.tgz", + "integrity": "sha512-DfXN8DfhJ7NH3Oe7cFmu3NCu1wKbkReJ8TorzSAFbSKrlNaQSKfIzqYqVY8zlbs2NLBbWpRiU52GX2PbaBVNkg==", + "deprecated": "Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me", + "license": "ISC", + "optional": true, + "dependencies": { + "foreground-child": "^3.1.0", + "jackspeak": "^3.1.2", + "minimatch": "^9.0.4", + "minipass": "^7.1.2", + "package-json-from-dist": "^1.0.0", + "path-scurry": "^1.11.1" + }, + "bin": { + "glob": "dist/esm/bin.mjs" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/globalthis": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/globalthis/-/globalthis-1.0.4.tgz", @@ -4162,10 +4735,33 @@ "integrity": "sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw==", "license": "MIT" }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "license": "ISC", + "optional": true + }, + "node_modules/jackspeak": { + "version": "3.4.3", + "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-3.4.3.tgz", + "integrity": "sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==", + "license": "BlueOak-1.0.0", + "optional": true, + "dependencies": { + "@isaacs/cliui": "^8.0.2" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + }, + "optionalDependencies": { + "@pkgjs/parseargs": "^0.11.0" + } + }, "node_modules/jose": { - "version": "4.15.9", - "resolved": "https://registry.npmjs.org/jose/-/jose-4.15.9.tgz", - "integrity": "sha512-1vUQX+IdDMVPj4k8kOxgUqlcK518yluMuGZwqlr44FS1ppZB/5GWh4rZG89erpOBOJjU/OBsnCVFfapsRz6nEA==", + "version": "6.2.10", + "resolved": "https://registry.npmjs.org/jose/-/jose-6.2.10.tgz", + "integrity": "sha512-iiW7J9qRFlGxvCOIBDBDxFePQSn7ZMAnrYGhrrOo6siO/MIqwfyilLR27pkfDgUk+raLuzADS8A3S/KLBisc0g==", "license": "MIT", "funding": { "url": "https://github.com/sponsors/panva" @@ -4221,19 +4817,20 @@ } }, "node_modules/jwks-rsa": { - "version": "3.2.2", - "resolved": "https://registry.npmjs.org/jwks-rsa/-/jwks-rsa-3.2.2.tgz", - "integrity": "sha512-BqTyEDV+lS8F2trk3A+qJnxV5Q9EqKCBJOPti3W97r7qTympCZjb7h2X6f2kc+0K3rsSTY1/6YG2eaXKoj497w==", + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/jwks-rsa/-/jwks-rsa-4.1.0.tgz", + "integrity": "sha512-sbkByqyATKYJP5F4RXj03N5TUNC0QLTjCAZvwTzC4BwJZ8e0/cWxN8YROnyUth2g1/ONWi4eSFHeu6oYalrc3Q==", "license": "MIT", "dependencies": { "@types/jsonwebtoken": "^9.0.4", "debug": "^4.3.4", - "jose": "^4.15.4", + "jose": "^6.1.3", "limiter": "^1.1.5", - "lru-memoizer": "^2.2.0" + "lru-cache": "^11.0.0", + "lru-memoizer": "^3.0.0" }, "engines": { - "node": ">=14" + "node": "^20.19.0 || ^22.12.0 || >= 23.0.0" } }, "node_modules/jws": { @@ -4325,25 +4922,22 @@ "license": "MIT" }, "node_modules/lru-cache": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", - "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", - "license": "ISC", - "dependencies": { - "yallist": "^4.0.0" - }, + "version": "11.5.2", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.5.2.tgz", + "integrity": "sha512-4pfM1Ff0x50o0tQwb5ucw/RzNyD0/YJME6IVcStalZuMWxdt3sR3huStTtxz4PUmvZfRguvDejasvQ2kifR11g==", + "license": "BlueOak-1.0.0", "engines": { - "node": ">=10" + "node": "20 || >=22" } }, "node_modules/lru-memoizer": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/lru-memoizer/-/lru-memoizer-2.3.0.tgz", - "integrity": "sha512-GXn7gyHAMhO13WSKrIiNfztwxodVsP8IoZ3XfrJV4yH2x0/OeTO/FIaAHTY5YekdGgW94njfuKmyyt1E0mR6Ug==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/lru-memoizer/-/lru-memoizer-3.0.0.tgz", + "integrity": "sha512-m83w/cYXLdUIboKSPxzPAGfYnk+vqeDYXuoSrQRw1q+yVEd8IXhvMufN8Q5TIPe7e2jyX4SRNrDJI2Skw1yznQ==", "license": "MIT", "dependencies": { "lodash.clonedeep": "^4.5.0", - "lru-cache": "6.0.0" + "lru-cache": "^11.0.1" } }, "node_modules/magic-string": { @@ -4437,6 +5031,32 @@ "url": "https://opencollective.com/express" } }, + "node_modules/minimatch": { + "version": "9.0.9", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.9.tgz", + "integrity": "sha512-OBwBN9AL4dqmETlpS2zasx+vTeWclWzkblfZk7KTA5j3jeOONz/tRCnZomUyvNg83wL5Zv9Ss6HMJXAgL8R2Yg==", + "license": "ISC", + "optional": true, + "dependencies": { + "brace-expansion": "^2.0.2" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/minipass": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.3.tgz", + "integrity": "sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==", + "license": "BlueOak-1.0.0", + "optional": true, + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, "node_modules/ms": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", @@ -4645,6 +5265,13 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/package-json-from-dist": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/package-json-from-dist/-/package-json-from-dist-1.0.1.tgz", + "integrity": "sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==", + "license": "BlueOak-1.0.0", + "optional": true + }, "node_modules/parseurl": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", @@ -4670,6 +5297,40 @@ "node": ">=14.0.0" } }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/path-scurry": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.11.1.tgz", + "integrity": "sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA==", + "license": "BlueOak-1.0.0", + "optional": true, + "dependencies": { + "lru-cache": "^10.2.0", + "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0" + }, + "engines": { + "node": ">=16 || 14 >=14.18" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/path-scurry/node_modules/lru-cache": { + "version": "10.4.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.4.3.tgz", + "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==", + "license": "ISC", + "optional": true + }, "node_modules/path-to-regexp": { "version": "8.4.2", "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-8.4.2.tgz", @@ -4936,6 +5597,22 @@ "node": ">=14" } }, + "node_modules/rimraf": { + "version": "5.0.10", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-5.0.10.tgz", + "integrity": "sha512-l0OE8wL34P4nJH/H2ffoaniAokM2qSmrtXHmlpvYr5AVVX8msAyW0l8NVJFDxlSK4u3Uh/f41cQheDVdnYijwQ==", + "license": "ISC", + "optional": true, + "dependencies": { + "glob": "^10.3.7" + }, + "bin": { + "rimraf": "dist/esm/bin.mjs" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/rollup": { "version": "4.63.1", "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.63.1.tgz", @@ -5185,6 +5862,29 @@ "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", "license": "ISC" }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "license": "MIT", + "optional": true, + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=8" + } + }, "node_modules/side-channel": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.1.tgz", @@ -5264,6 +5964,19 @@ "dev": true, "license": "ISC" }, + "node_modules/signal-exit": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", + "license": "ISC", + "optional": true, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/source-map-js": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", @@ -5357,6 +6070,22 @@ "node": ">=8" } }, + "node_modules/string-width-cjs": { + "name": "string-width", + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "license": "MIT", + "optional": true, + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/string.prototype.trim": { "version": "1.2.11", "resolved": "https://registry.npmjs.org/string.prototype.trim/-/string.prototype.trim-1.2.11.tgz", @@ -5426,6 +6155,20 @@ "node": ">=8" } }, + "node_modules/strip-ansi-cjs": { + "name": "strip-ansi", + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", + "optional": true, + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/strip-literal": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/strip-literal/-/strip-literal-3.1.0.tgz", @@ -6041,6 +6784,22 @@ "webidl-conversions": "^3.0.0" } }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "license": "ISC", + "optional": true, + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, "node_modules/which-boxed-primitive": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/which-boxed-primitive/-/which-boxed-primitive-1.1.1.tgz", @@ -6160,6 +6919,25 @@ "url": "https://github.com/chalk/wrap-ansi?sponsor=1" } }, + "node_modules/wrap-ansi-cjs": { + "name": "wrap-ansi", + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "license": "MIT", + "optional": true, + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, "node_modules/wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", diff --git a/kits/firestore-bigquery-export/package.json b/kits/firestore-bigquery-export/package.json index 135f3ef50f..5d197509a1 100644 --- a/kits/firestore-bigquery-export/package.json +++ b/kits/firestore-bigquery-export/package.json @@ -33,7 +33,7 @@ "dependencies": { "@firebaseextensions/firestore-bigquery-change-tracker": "^2.1.0", "@google-cloud/bigquery": "^7.6.0", - "firebase-admin": "^13.6.0", + "firebase-admin": "^14.2.0", "firebase-functions": "^7.3.3-rc.0", "generate-schema": "^2.6.0", "lodash": "^4.17.14" diff --git a/kits/firestore-bigquery-export/src/config.ts b/kits/firestore-bigquery-export/src/config.ts index 08e4b1d23a..23e1f92cf3 100644 --- a/kits/firestore-bigquery-export/src/config.ts +++ b/kits/firestore-bigquery-export/src/config.ts @@ -23,6 +23,7 @@ import { LogLevel } from "@firebaseextensions/firestore-bigquery-change-tracker" import type { Expression } from "firebase-functions/params"; import { defineBoolean, + defineInt, defineString, projectID, select, @@ -102,6 +103,7 @@ export interface ConfigExpressions { datasetId: ConfigExpression; tableId: ConfigExpression; database: ConfigExpression; + maxDispatchesPerSecond: ConfigExpression; } /** @@ -287,9 +289,39 @@ const params = { backupCollection: defineString("BACKUP_COLLECTION", { label: "Backup Collection Name", description: - "This (optional) parameter will allow you to specify a collection for which failed BigQuery updates will be written to.", + "Strongly recommended. The Firestore collection where rows that still fail after the sync queue's five attempts are written; without it, those rows are dropped. See the README for how to reconcile backed-up rows into BigQuery.", default: "", }), + maxDispatchesPerSecond: defineInt("MAX_DISPATCHES_PER_SECOND", { + label: "Maximum number of synced documents per second", + description: + "This parameter will set the maximum number of synchronized documents per second with BQ. Please note, any other external updates to a Big Query table will be included within this quota. Ensure that you have set a low enough number to compensate. Defaults to 100.", + + default: 100, + input: { + text: { + example: "100", + + validationRegex: /^([1-9]|[1-9][0-9]|[1-4][0-9]{2}|500)$/, + validationErrorMessage: "Please select a number between 1 and 500", + }, + }, + }), + maxEnqueueAttempts: defineInt("MAX_ENQUEUE_ATTEMPTS", { + label: "Maximum number of enqueue attempts", + description: + "This parameter will set the maximum number of attempts to enqueue a document to cloud tasks for export to BigQuery.", + + default: 3, + input: { + text: { + example: "3", + + validationRegex: /^(10|[1-9])$/, + validationErrorMessage: "Please select an integer between 1 and 10", + }, + }, + }), transformFunction: defineString("TRANSFORM_FUNCTION", { label: "Transform function URL", description: @@ -452,6 +484,7 @@ export const CONFIG_EXPRESSIONS: ConfigExpressions = { datasetId: params.datasetId, tableId: params.tableId, database: params.database, + maxDispatchesPerSecond: params.maxDispatchesPerSecond, }; function timePartitioning( @@ -646,5 +679,7 @@ export function configFromEnv(): ExportConfig { transformFunction: optional(params.transformFunction.value()), kmsKeyName: optional(params.kmsKeyName.value()), logLevel: normalizeLogLevel(params.logLevel.value()), + maxDispatchesPerSecond: params.maxDispatchesPerSecond.value(), + maxEnqueueAttempts: params.maxEnqueueAttempts.value(), }; } diff --git a/kits/firestore-bigquery-export/src/export-config.ts b/kits/firestore-bigquery-export/src/export-config.ts index f735786a9a..84d768e6f8 100644 --- a/kits/firestore-bigquery-export/src/export-config.ts +++ b/kits/firestore-bigquery-export/src/export-config.ts @@ -86,6 +86,17 @@ export interface ExportConfig { /** Log verbosity. Defaults to `info`. */ logLevel?: ConfigValue; + + /** + * Cloud Tasks dispatch rate for the `syncBigQuery` queue, in tasks per + * second. Defaults to `100`. + */ + maxDispatchesPerSecond?: ConfigValue; + /** + * How many times the trigger tries to enqueue a failed write onto the + * `syncBigQuery` queue before giving up and rethrowing. Defaults to `3`. + */ + maxEnqueueAttempts?: ConfigValue; } /** {@link ExportConfig} with all defaults applied. */ @@ -109,6 +120,8 @@ export interface ResolvedExportConfig { transformFunction?: string; kmsKeyName?: string; logLevel: TrackerLogLevel; + maxDispatchesPerSecond: number; + maxEnqueueAttempts: number; } function isExpression( @@ -165,6 +178,10 @@ export function resolveExportConfig( transformFunction: resolveOptionalConfigValue(config.transformFunction), kmsKeyName: resolveOptionalConfigValue(config.kmsKeyName), logLevel: (logLevel as TrackerLogLevel) ?? "info", + maxDispatchesPerSecond: + resolveOptionalConfigValue(config.maxDispatchesPerSecond) ?? 100, + maxEnqueueAttempts: + resolveOptionalConfigValue(config.maxEnqueueAttempts) ?? 3, }; } diff --git a/kits/firestore-bigquery-export/src/handlers.ts b/kits/firestore-bigquery-export/src/handlers.ts index 7510f3c8ee..e8f38a38fe 100644 --- a/kits/firestore-bigquery-export/src/handlers.ts +++ b/kits/firestore-bigquery-export/src/handlers.ts @@ -24,13 +24,18 @@ import type { DocumentSnapshot, FirestoreEvent, } from "firebase-functions/firestore"; +import type { Request } from "firebase-functions/tasks"; import * as events from "./events"; import type { ResolvedExportConfig } from "./export-config"; import * as logs from "./logs"; import { getChangeType, getDocumentId } from "./util"; -/** Serialized Firestore change ready to write to BigQuery. */ -interface SerializedDocumentChange { +/** + * Serialized Firestore change ready to write to BigQuery. Also the + * `syncBigQuery` task payload: it is built from already-serialized data, so it + * survives the JSON round trip through Cloud Tasks unchanged. + */ +export interface SerializedDocumentChange { timestamp: string; eventId: string; fullResourceName: string; @@ -55,11 +60,17 @@ export interface HandlerContext { tracker: FirestoreBigQueryEventHistoryTracker; config: ResolvedExportConfig; /** - * Provisions the BigQuery dataset/table/views once per instance. Only called - * after an inline write failure as a self-heal; the hot path relies on - * out-of-band provisioning (`initBigQuerySync` / `setupBigQuerySync`). + * Provisions the BigQuery dataset/table/views once per instance. Called by + * the `syncBigQuery` task as a self-heal before re-attempting a write; the + * hot path relies on out-of-band provisioning + * (`initBigQuerySync` / `setupBigQuerySync`). */ ensureInitialized: () => Promise; + /** + * Enqueues a failed change onto the `syncBigQuery` task queue. Rejects with + * the enqueue error once its own retry budget is exhausted. + */ + enqueue: (change: SerializedDocumentChange) => Promise; } /** @@ -87,38 +98,39 @@ async function recordEventToBigQuery( } /** - * Gives a failed inline write one self-heal attempt before surfacing it to the - * Firestore trigger retry policy. + * Buffers a failed inline write through the `syncBigQuery` task queue. A + * terminal enqueue failure is logged, recorded, and rethrown so the trigger + * retry policy covers the window where both BigQuery and Cloud Tasks fail; + * swallowing it here would drop the event with no durable copy. * - * @param change - The serialized change to write. + * @param change - The serialized change to enqueue. * @param ctx - The handler context. */ -async function retryAfterSelfHeal( +async function enqueueForSync( change: SerializedDocumentChange, ctx: HandlerContext ): Promise { try { - await ctx.ensureInitialized(); - await recordEventToBigQuery(change, ctx.tracker); - } catch (retryErr) { - await events.recordErrorEvent(retryErr as Error); + await ctx.enqueue(change); + } catch (enqueueErr) { + await events.recordErrorEvent(enqueueErr as Error); logs.logFailedEventAction( - "Failed to write event to BigQuery from onWrite handler after self-heal", + "Failed to enqueue event to Cloud Tasks from onWrite handler", change.fullResourceName, change.eventId, change.changeType, - retryErr as Error + enqueueErr as Error ); - throw retryErr; + throw enqueueErr; } } /** * Handles a Firestore document write: serializes the change and writes it to - * BigQuery. Failed writes are surfaced to the trigger retry policy after one - * self-heal attempt. + * BigQuery. A failed inline write is buffered through the `syncBigQuery` task + * queue; only a failed enqueue surfaces to the trigger retry policy. * * @param event - The Firestore document-write event. * @param ctx - The handler context. @@ -134,8 +146,8 @@ export async function handleDocumentWrite( // No provisioning on the hot path: BigQuery resources are provisioned // out-of-band (afterFirstDeploy / afterRedeploy tasks). If they are missing, - // the inline write fails, self-heals once, then falls back to the trigger - // retry policy. + // the inline write fails and the change buffers through the syncBigQuery + // queue, whose handler self-heals before re-attempting. const { config, tracker } = ctx; const changeType = getChangeType(data); const documentId = getDocumentId(data); @@ -204,8 +216,65 @@ export async function handleDocumentWrite( await recordEventToBigQuery(change, tracker); } catch (err) { logs.failedToWriteToBigQueryImmediately(err as Error); - await retryAfterSelfHeal(change, ctx); + await enqueueForSync(change, ctx); } logs.complete(); } + +/** + * Handles a `syncBigQuery` task: re-attempts a buffered write. Provisioning + * runs first as a self-heal (memoized, a no-op after the first success), so a + * write that failed only because the BigQuery resources were missing succeeds + * on the first task attempt. A failed write rethrows so Cloud Tasks retries + * on the queue's schedule; the tracker has already written the row to the + * backup collection (when one is configured) before each terminal rethrow. + * + * @param req - The dispatched task request carrying the serialized change. + * @param ctx - The handler context. + */ +export async function handleSyncBigQueryTask( + req: Request, + ctx: HandlerContext +): Promise { + const change = req.data; + + logs.logEventAction( + "Firestore event received by onDispatch trigger", + change.fullResourceName, + change.eventId, + change.changeType + ); + + try { + await ctx.ensureInitialized(); + await recordEventToBigQuery(change, ctx.tracker); + + await events.recordSuccessEvent({ + subject: change.documentId, + data: { + timestamp: change.timestamp, + operation: change.changeType, + documentName: change.fullResourceName, + documentId: change.documentId, + pathParams: change.params, + eventId: change.eventId, + data: change.data, + oldData: change.oldData, + }, + }); + + logs.complete(); + } catch (err) { + logs.logFailedEventAction( + "Failed to write event to BigQuery from onDispatch handler", + change.fullResourceName, + change.eventId, + change.changeType, + err as Error, + req.retryCount + ); + + throw err; + } +} diff --git a/kits/firestore-bigquery-export/src/index.ts b/kits/firestore-bigquery-export/src/index.ts index fb4cbc6f50..eb76d083ee 100644 --- a/kits/firestore-bigquery-export/src/index.ts +++ b/kits/firestore-bigquery-export/src/index.ts @@ -17,8 +17,8 @@ /** * Main entry point. Exports the wired functions with deploy-time param * expressions, then resolves concrete config lazily at runtime. Re-export - * `fsexportbigquery` and `initBigQuerySync` from your own functions codebase - * entry; configuration comes from a `.env` (or + * `fsexportbigquery`, `syncBigQuery`, and `initBigQuerySync` from your own + * functions codebase entry; configuration comes from a `.env` (or * `.env.`), which the Firebase CLI loads at deploy. * * Because this module initializes runtime dependencies lazily, deploy discovery @@ -41,10 +41,16 @@ import { import { CONFIG_EXPRESSIONS, configFromEnv } from "./config"; import * as events from "./events"; import { resolveExportConfig, toTrackerConfig } from "./export-config"; -import { type HandlerContext, handleDocumentWrite } from "./handlers"; +import { + type HandlerContext, + type SerializedDocumentChange, + handleDocumentWrite, + handleSyncBigQueryTask, +} from "./handlers"; import { createEnsureInitialized } from "./init"; import * as logs from "./logs"; import { firestoreLocationToFunctionRegion } from "./region"; +import { enqueueSyncTask } from "./tasks"; // Re-export the side-effect-free library surface (handlers and config types). export * from "./lib"; @@ -55,6 +61,11 @@ const LIFECYCLE_RETRY_CONFIG = { maxAttempts: 15, minBackoffSeconds: 60, } as const; +const SYNC_RETRY_CONFIG = { + maxAttempts: 5, + minBackoffSeconds: 60, +} as const; +const SYNC_MAX_CONCURRENT_DISPATCHES = 500; const REQUIRED_ROLES: ReadonlyArray = [ "roles/bigquery.dataEditor", "roles/datastore.user", @@ -62,6 +73,8 @@ const REQUIRED_ROLES: ReadonlyArray = [ // Gen2 Firestore triggers need Eventarc receive and run.invoker on the function SA. "roles/eventarc.eventReceiver", "roles/run.invoker", + // The trigger enqueues failed writes onto its own syncBigQuery task queue. + "roles/cloudtasks.enqueuer", ]; const REQUIRED_APIS = [ { @@ -112,6 +125,8 @@ function getHandlerContext(): HandlerContext { tracker, config, ensureInitialized, + enqueue: (change: SerializedDocumentChange) => + enqueueSyncTask(change, config.maxEnqueueAttempts), }; return ctx; @@ -131,8 +146,10 @@ const functionRegion = firestoreLocationToFunctionRegion( /** * Firestore trigger: streams document writes on the watched collection into the - * BigQuery changelog table. Failed executions are retried by the Firebase - * Functions runtime. + * BigQuery changelog table. A failed inline write buffers through the + * `syncBigQuery` queue and the execution still succeeds; `retry: true` stays on + * so an event whose enqueue ALSO failed (rethrown by the handler) is + * redelivered instead of dropped. */ export const fsexportbigquery = onDocumentWritten( { @@ -144,6 +161,25 @@ export const fsexportbigquery = onDocumentWritten( (event) => handleDocumentWrite(event, getHandlerContext()) ); +/** + * Write-buffer task queue: re-attempts writes that failed inline, on Cloud + * Tasks' schedule (5 attempts, 60s minimum backoff, dispatch-throttled by + * `MAX_DISPATCHES_PER_SECOND`). After the last attempt the task is dropped; + * by then the tracker has written the row to `BACKUP_COLLECTION` on every + * terminal insert failure, when that collection is configured. + */ +export const syncBigQuery = onTaskDispatched( + { + ...(functionRegion ? { region: functionRegion } : {}), + retryConfig: SYNC_RETRY_CONFIG, + rateLimits: { + maxConcurrentDispatches: SYNC_MAX_CONCURRENT_DISPATCHES, + maxDispatchesPerSecond: CONFIG_EXPRESSIONS.maxDispatchesPerSecond, + }, + }, + (req) => handleSyncBigQueryTask(req, getHandlerContext()) +); + async function handleBigQuerySyncInitialization(): Promise { try { await getHandlerContext().ensureInitialized(); diff --git a/kits/firestore-bigquery-export/src/lib.ts b/kits/firestore-bigquery-export/src/lib.ts index c17127998b..8e5939f514 100644 --- a/kits/firestore-bigquery-export/src/lib.ts +++ b/kits/firestore-bigquery-export/src/lib.ts @@ -51,5 +51,7 @@ export { export { type DocumentWriteEvent, type HandlerContext, + type SerializedDocumentChange, handleDocumentWrite, + handleSyncBigQueryTask, } from "./handlers"; diff --git a/kits/firestore-bigquery-export/src/logs.ts b/kits/firestore-bigquery-export/src/logs.ts index 12ee2c4885..bb17b72c14 100644 --- a/kits/firestore-bigquery-export/src/logs.ts +++ b/kits/firestore-bigquery-export/src/logs.ts @@ -208,7 +208,8 @@ export const logFailedEventAction = ( document_name: string, event_id: string, operation: ChangeType, - error: Error + error: Error, + retry_count?: number ) => { const changeTypeMap = { 0: "CREATE", @@ -222,6 +223,7 @@ export const logFailedEventAction = ( event_id, operation: changeTypeMap[operation], error, + ...(retry_count === undefined ? {} : { retry_count }), }); }; diff --git a/kits/firestore-bigquery-export/src/tasks.ts b/kits/firestore-bigquery-export/src/tasks.ts new file mode 100644 index 0000000000..982c80c693 --- /dev/null +++ b/kits/firestore-bigquery-export/src/tasks.ts @@ -0,0 +1,98 @@ +/* + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { getFunctions } from "firebase-admin/functions"; +import { firestoreLocationToFunctionRegion } from "./region"; + +/** Export name of the write-buffer task function. */ +export const SYNC_BIGQUERY_FUNCTION = "syncBigQuery"; + +const MAX_BACKOFF_MS = 5000; +const BACKOFF_BASE_MS = 100; +const JITTER_MS = 100; + +/** + * Resolves the queue resource path for a task function of this kit instance. + * + * The name is deliberately unprefixed: firebase-admin >= 14.2.0 resolves the + * deployed `kit--` prefix itself from the + * `FIREBASE_KIT_INSTANCE_ID` env var, which the CLI sets on every deployed kit + * function. All functions of a kit instance deploy to one region, so the + * enqueuing function's own region (`DATABASE_REGION`-derived, with the + * CLI-set `FUNCTION_REGION` as fallback) is also the queue's region. + * + * @param functionName - The export name of the task function. + * @returns The queue resource path, `locations//functions/`. + * @throws If no region can be resolved. + */ +export function syncQueuePath( + functionName: string = SYNC_BIGQUERY_FUNCTION +): string { + const region = + firestoreLocationToFunctionRegion(process.env.DATABASE_REGION) ?? + process.env.FUNCTION_REGION; + + if (!region) { + throw new Error( + "A region is required to resolve the syncBigQuery task queue. " + + "Set DATABASE_REGION, or deploy with the Firebase CLI so FUNCTION_REGION is set." + ); + } + + return `locations/${region}/functions/${functionName}`; +} + +function backoffMs(attempt: number, jitter: number): number { + return ( + Math.min(Math.pow(2, attempt) * BACKOFF_BASE_MS, MAX_BACKOFF_MS) + jitter + ); +} + +/** + * Enqueues a payload onto the `syncBigQuery` queue, retrying transient enqueue + * failures in-process with exponential backoff and jitter. + * + * @param payload - The task payload. + * @param maxAttempts - How many enqueue attempts to make before giving up. + * @throws The last enqueue error, once every attempt has failed. + */ +export async function enqueueSyncTask( + payload: object, + maxAttempts: number +): Promise { + const queue = getFunctions().taskQueue(syncQueuePath()); + + const jitter = Math.random() * JITTER_MS; + let attempts = 0; + + while (attempts < maxAttempts) { + if (attempts > 0) { + await new Promise((resolve) => + setTimeout(resolve, backoffMs(attempts, jitter)) + ); + } + + attempts++; + try { + await queue.enqueue(payload); + return; + } catch (enqueueErr) { + if (attempts >= maxAttempts) { + throw enqueueErr; + } + } + } +} diff --git a/kits/firestore-bigquery-export/tests/config.test.ts b/kits/firestore-bigquery-export/tests/config.test.ts index 40508c3c68..c875409c1e 100644 --- a/kits/firestore-bigquery-export/tests/config.test.ts +++ b/kits/firestore-bigquery-export/tests/config.test.ts @@ -171,11 +171,46 @@ describe("configFromEnv", () => { expect(resolveExportConfig(config)).not.toHaveProperty("location"); }); + test("queue params keep the extension's defaults", () => { + const config = configFromEnv(); + expect(config.maxDispatchesPerSecond).toBe(100); + expect(config.maxEnqueueAttempts).toBe(3); + }); + test("exposes deploy-time expressions for trigger metadata", () => { expect(CONFIG_EXPRESSIONS.collectionPath.toString()).toBe( "params.COLLECTION_PATH" ); expect(CONFIG_EXPRESSIONS.database.toString()).toBe("params.DATABASE"); + expect(CONFIG_EXPRESSIONS.maxDispatchesPerSecond.toString()).toBe( + "params.MAX_DISPATCHES_PER_SECOND" + ); expect(CONFIG_EXPRESSIONS).not.toHaveProperty("location"); }); }); + +describe("resolveExportConfig queue defaults", () => { + test("applies the extension's queue defaults when unset", () => { + const resolved = resolveExportConfig({ + collectionPath: "users", + datasetId: "ds", + tableId: "tbl", + projectId: "p", + }); + expect(resolved.maxDispatchesPerSecond).toBe(100); + expect(resolved.maxEnqueueAttempts).toBe(3); + }); + + test("passes explicit queue values through", () => { + const resolved = resolveExportConfig({ + collectionPath: "users", + datasetId: "ds", + tableId: "tbl", + projectId: "p", + maxDispatchesPerSecond: 250, + maxEnqueueAttempts: 5, + }); + expect(resolved.maxDispatchesPerSecond).toBe(250); + expect(resolved.maxEnqueueAttempts).toBe(5); + }); +}); diff --git a/kits/firestore-bigquery-export/tests/handlers.test.ts b/kits/firestore-bigquery-export/tests/handlers.test.ts index b37f675090..bb607b5500 100644 --- a/kits/firestore-bigquery-export/tests/handlers.test.ts +++ b/kits/firestore-bigquery-export/tests/handlers.test.ts @@ -15,12 +15,15 @@ */ import { ChangeType } from "@firebaseextensions/firestore-bigquery-change-tracker"; +import type { Request } from "firebase-functions/tasks"; import { beforeEach, describe, expect, test, vi } from "vitest"; import { resolveExportConfig } from "../src/export-config"; import { type DocumentWriteEvent, type HandlerContext, + type SerializedDocumentChange, handleDocumentWrite, + handleSyncBigQueryTask, } from "../src/handlers"; vi.mock("../src/events"); @@ -70,6 +73,33 @@ function makeCtx( tracker: tracker as unknown as HandlerContext["tracker"], config: config as HandlerContext["config"], ensureInitialized: vi.fn().mockResolvedValue(undefined), + enqueue: vi.fn().mockResolvedValue(undefined), + }; +} + +/** Fake dispatched task request carrying a serialized change. */ +function taskRequest( + change: SerializedDocumentChange, + retryCount = 0 +): Request { + return { data: change, retryCount } as Request; +} + +/** A serialized change as it would arrive in a task payload. */ +function serializedChange( + overrides: Partial = {} +): SerializedDocumentChange { + return { + timestamp: "2026-01-01T00:00:00Z", + eventId: "evt-1", + fullResourceName: + "projects/test-project/databases/(default)/documents/users/doc1", + changeType: ChangeType.CREATE, + documentId: "doc1", + params: null, + data: { a: 1 }, + oldData: undefined, + ...overrides, }; } @@ -170,7 +200,7 @@ describe("handleDocumentWrite", () => { expect(recordedWithout[0].pathParams).toBeNull(); }); - test("self-heals and retries the write when the inline write fails", async () => { + test("a failed inline write buffers through the queue and the execution succeeds", async () => { const ctx = makeCtx(); (ctx.tracker.record as ReturnType).mockRejectedValueOnce( new Error("bq down") @@ -181,23 +211,65 @@ describe("handleDocumentWrite", () => { ctx ); - expect(ctx.ensureInitialized).toHaveBeenCalledTimes(1); - expect(ctx.tracker.record).toHaveBeenCalledTimes(2); + expect(ctx.tracker.record).toHaveBeenCalledTimes(1); + expect(ctx.enqueue).toHaveBeenCalledTimes(1); + expect(ctx.ensureInitialized).not.toHaveBeenCalled(); + }); + + test("a successful inline write enqueues nothing", async () => { + const ctx = makeCtx(); + + await handleDocumentWrite( + writeEvent(snap(false, "doc1"), snap(true, "doc1", { a: 1 })), + ctx + ); + + expect(ctx.enqueue).not.toHaveBeenCalled(); + }); + + test("the enqueued change equals what the inline path tried to write", async () => { + const ctx = makeCtx(); + (ctx.tracker.record as ReturnType).mockRejectedValueOnce( + new Error("bq down") + ); + + await handleDocumentWrite( + writeEvent(snap(true, "doc1", { a: 1 }), snap(true, "doc1", { a: 2 })), + ctx + ); + + const [[recorded]] = (ctx.tracker.record as ReturnType).mock + .calls; + const [[enqueued]] = (ctx.enqueue as ReturnType).mock.calls; + expect(enqueued).toMatchObject({ + timestamp: recorded[0].timestamp, + eventId: recorded[0].eventId, + fullResourceName: recorded[0].documentName, + changeType: recorded[0].operation, + documentId: recorded[0].documentId, + params: recorded[0].pathParams, + data: recorded[0].data, + oldData: recorded[0].oldData, + }); + // The payload must survive the JSON round trip through Cloud Tasks. + expect(JSON.parse(JSON.stringify(enqueued))).toEqual(enqueued); }); - test("rethrows when self-heal retry fails so runtime retry can replay", async () => { + test("a failed enqueue is recorded and rethrown, never swallowed", async () => { const ctx = makeCtx(); - (ctx.tracker.record as ReturnType) - .mockRejectedValueOnce(new Error("bq down")) - .mockRejectedValueOnce(new Error("still down")); + (ctx.tracker.record as ReturnType).mockRejectedValueOnce( + new Error("bq down") + ); + (ctx.enqueue as ReturnType).mockRejectedValueOnce( + new Error("tasks down") + ); await expect( handleDocumentWrite( writeEvent(snap(false, "doc1"), snap(true, "doc1", { a: 1 })), ctx ) - ).rejects.toThrow("still down"); - expect(ctx.ensureInitialized).toHaveBeenCalledTimes(1); + ).rejects.toThrow("tasks down"); expect(events.recordErrorEvent).toHaveBeenCalled(); }); @@ -218,3 +290,51 @@ describe("handleDocumentWrite", () => { expect(ctx.tracker.record).not.toHaveBeenCalled(); }); }); + +describe("handleSyncBigQueryTask", () => { + beforeEach(() => vi.clearAllMocks()); + + test("self-heals, records the buffered change, and emits a success event", async () => { + const ctx = makeCtx(); + const change = serializedChange(); + + await handleSyncBigQueryTask(taskRequest(change), ctx); + + expect(ctx.ensureInitialized).toHaveBeenCalledTimes(1); + const [[recorded]] = (ctx.tracker.record as ReturnType).mock + .calls; + expect(recorded[0]).toMatchObject({ + timestamp: change.timestamp, + operation: change.changeType, + documentName: change.fullResourceName, + documentId: change.documentId, + eventId: change.eventId, + data: change.data, + }); + expect(events.recordSuccessEvent).toHaveBeenCalledTimes(1); + }); + + test("rethrows a failed write so Cloud Tasks retries", async () => { + const ctx = makeCtx(); + (ctx.tracker.record as ReturnType).mockRejectedValueOnce( + new Error("still down") + ); + + await expect( + handleSyncBigQueryTask(taskRequest(serializedChange(), 2), ctx) + ).rejects.toThrow("still down"); + expect(events.recordSuccessEvent).not.toHaveBeenCalled(); + }); + + test("rethrows a failed self-heal without attempting the write", async () => { + const ctx = makeCtx(); + (ctx.ensureInitialized as ReturnType).mockRejectedValueOnce( + new Error("no dataset") + ); + + await expect( + handleSyncBigQueryTask(taskRequest(serializedChange()), ctx) + ).rejects.toThrow("no dataset"); + expect(ctx.tracker.record).not.toHaveBeenCalled(); + }); +}); diff --git a/kits/firestore-bigquery-export/tests/index.test.ts b/kits/firestore-bigquery-export/tests/index.test.ts index f5748c4280..e2276fead4 100644 --- a/kits/firestore-bigquery-export/tests/index.test.ts +++ b/kits/firestore-bigquery-export/tests/index.test.ts @@ -35,6 +35,7 @@ type FunctionOptions = Record; interface ExportedOptions { trigger: FunctionOptions; + /** Options of syncBigQuery, initBigQuerySync, setupBigQuerySync, in order. */ tasks: FunctionOptions[]; } @@ -66,10 +67,10 @@ async function loadExportedOptions( const taskCalls = vi.mocked(onTaskDispatched).mock.calls; const trigger = triggerCalls[triggerCalls.length - 1][0] as FunctionOptions; const tasks = taskCalls - .slice(-2) + .slice(-3) .map((call) => call[0] as unknown as FunctionOptions); - expect(tasks).toHaveLength(2); + expect(tasks).toHaveLength(3); return { trigger, tasks }; } @@ -123,4 +124,36 @@ describe("exported function options", () => { const document = trigger.document as { toCEL(): string }; expect(document.toCEL()).toContain("params.COLLECTION_PATH"); }); + + test("the trigger keeps retry enabled so a rethrown enqueue failure is redelivered", async () => { + const { trigger } = await loadExportedOptions(); + expect(trigger.retry).toBe(true); + }); + + test("syncBigQuery pins the extension's queue shape", async () => { + const { tasks } = await loadExportedOptions(); + const [syncTask] = tasks; + + expect(syncTask.retryConfig).toEqual({ + maxAttempts: 5, + minBackoffSeconds: 60, + }); + + const rateLimits = syncTask.rateLimits as Record; + expect(rateLimits.maxConcurrentDispatches).toBe(500); + expect(String(rateLimits.maxDispatchesPerSecond)).toBe( + "params.MAX_DISPATCHES_PER_SECOND" + ); + }); + + test("the lifecycle tasks keep their own retry config", async () => { + const { tasks } = await loadExportedOptions(); + for (const opts of tasks.slice(1)) { + expect(opts.retryConfig).toEqual({ + maxAttempts: 15, + minBackoffSeconds: 60, + }); + expect(opts).not.toHaveProperty("rateLimits"); + } + }); }); diff --git a/kits/firestore-bigquery-export/tests/tasks.test.ts b/kits/firestore-bigquery-export/tests/tasks.test.ts new file mode 100644 index 0000000000..b541f175a0 --- /dev/null +++ b/kits/firestore-bigquery-export/tests/tasks.test.ts @@ -0,0 +1,142 @@ +/** + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; + +vi.mock("firebase-admin/functions", () => ({ + getFunctions: vi.fn(), +})); + +import { getFunctions } from "firebase-admin/functions"; +import { enqueueSyncTask, syncQueuePath } from "../src/tasks"; + +const ENV_KEYS = ["DATABASE_REGION", "FUNCTION_REGION"] as const; +const originalEnv: Record = {}; + +beforeEach(() => { + for (const key of ENV_KEYS) { + originalEnv[key] = process.env[key]; + delete process.env[key]; + } +}); + +afterEach(() => { + for (const key of ENV_KEYS) { + if (originalEnv[key] === undefined) { + delete process.env[key]; + } else { + process.env[key] = originalEnv[key]; + } + } + vi.useRealTimers(); + vi.clearAllMocks(); +}); + +describe("syncQueuePath", () => { + test("derives the region from DATABASE_REGION, mapping multi-regions", () => { + process.env.DATABASE_REGION = "nam5"; + expect(syncQueuePath()).toBe( + "locations/us-central1/functions/syncBigQuery" + ); + }); + + test("passes a regional DATABASE_REGION through", () => { + process.env.DATABASE_REGION = "europe-west2"; + expect(syncQueuePath()).toBe( + "locations/europe-west2/functions/syncBigQuery" + ); + }); + + test("falls back to FUNCTION_REGION when DATABASE_REGION is unset", () => { + process.env.FUNCTION_REGION = "us-central1"; + expect(syncQueuePath()).toBe( + "locations/us-central1/functions/syncBigQuery" + ); + }); + + test("prefers DATABASE_REGION over FUNCTION_REGION", () => { + process.env.DATABASE_REGION = "eur3"; + process.env.FUNCTION_REGION = "us-central1"; + expect(syncQueuePath()).toBe( + "locations/europe-west1/functions/syncBigQuery" + ); + }); + + test("throws when no region is resolvable", () => { + expect(() => syncQueuePath()).toThrow(/region/i); + }); +}); + +describe("enqueueSyncTask", () => { + function mockQueue(enqueue: ReturnType) { + const taskQueue = vi.fn(() => ({ enqueue })); + vi.mocked(getFunctions).mockReturnValue({ + taskQueue, + } as unknown as ReturnType); + return taskQueue; + } + + beforeEach(() => { + process.env.FUNCTION_REGION = "us-central1"; + }); + + test("targets the bare function name; the admin SDK adds the kit prefix", async () => { + const enqueue = vi.fn().mockResolvedValue(undefined); + const taskQueue = mockQueue(enqueue); + + await enqueueSyncTask({ eventId: "evt-1" }, 3); + + expect(taskQueue).toHaveBeenCalledWith( + "locations/us-central1/functions/syncBigQuery" + ); + expect(enqueue).toHaveBeenCalledTimes(1); + expect(enqueue).toHaveBeenCalledWith({ eventId: "evt-1" }); + }); + + test("retries a failed enqueue after a backoff and then succeeds", async () => { + vi.useFakeTimers(); + const enqueue = vi + .fn() + .mockRejectedValueOnce(new Error("blip")) + .mockResolvedValueOnce(undefined); + mockQueue(enqueue); + + const pending = enqueueSyncTask({}, 3); + await vi.runAllTimersAsync(); + await pending; + + expect(enqueue).toHaveBeenCalledTimes(2); + }); + + test("throws the last error once every attempt fails", async () => { + vi.useFakeTimers(); + const enqueue = vi + .fn() + .mockRejectedValueOnce(new Error("first")) + .mockRejectedValueOnce(new Error("second")) + .mockRejectedValue(new Error("last")); + mockQueue(enqueue); + + const pending = enqueueSyncTask({}, 3); + // Attach the rejection expectation before advancing timers so the + // rejection is never unhandled. + const assertion = expect(pending).rejects.toThrow("last"); + await vi.runAllTimersAsync(); + await assertion; + + expect(enqueue).toHaveBeenCalledTimes(3); + }); +}); From 5bf4e639ef682539dff4804f972bc7c2f136f18f Mon Sep 17 00:00:00 2001 From: Jacob Cable Date: Wed, 2 Sep 2026 20:38:45 +0100 Subject: [PATCH 2/8] fix(firestore-bigquery-export): clamp the enqueue attempt budget and pin buffer safety properties A library consumer passing maxEnqueueAttempts <= 0 made enqueueSyncTask resolve without enqueueing, so the trigger logged success for an event buffered nowhere; the budget is now clamped to at least one attempt. New tests pin the backup wiring (toTrackerConfig.backupTableId carries backupCollectionId - the property that keeps queue exhaustion durable) and that the syncBigQuery task never re-enqueues (the trigger-queue loop seed). --- kits/firestore-bigquery-export/src/tasks.ts | 7 +++++-- .../tests/export-config.test.ts | 10 ++++++++++ kits/firestore-bigquery-export/tests/handlers.test.ts | 4 ++++ kits/firestore-bigquery-export/tests/tasks.test.ts | 9 +++++++++ 4 files changed, 28 insertions(+), 2 deletions(-) diff --git a/kits/firestore-bigquery-export/src/tasks.ts b/kits/firestore-bigquery-export/src/tasks.ts index 982c80c693..b6b501cadc 100644 --- a/kits/firestore-bigquery-export/src/tasks.ts +++ b/kits/firestore-bigquery-export/src/tasks.ts @@ -67,6 +67,8 @@ function backoffMs(attempt: number, jitter: number): number { * * @param payload - The task payload. * @param maxAttempts - How many enqueue attempts to make before giving up. + * Clamped to at least 1: resolving without an enqueue would report success + * for an event that was never buffered anywhere. * @throws The last enqueue error, once every attempt has failed. */ export async function enqueueSyncTask( @@ -75,10 +77,11 @@ export async function enqueueSyncTask( ): Promise { const queue = getFunctions().taskQueue(syncQueuePath()); + const attemptBudget = Math.max(1, maxAttempts); const jitter = Math.random() * JITTER_MS; let attempts = 0; - while (attempts < maxAttempts) { + while (attempts < attemptBudget) { if (attempts > 0) { await new Promise((resolve) => setTimeout(resolve, backoffMs(attempts, jitter)) @@ -90,7 +93,7 @@ export async function enqueueSyncTask( await queue.enqueue(payload); return; } catch (enqueueErr) { - if (attempts >= maxAttempts) { + if (attempts >= attemptBudget) { throw enqueueErr; } } diff --git a/kits/firestore-bigquery-export/tests/export-config.test.ts b/kits/firestore-bigquery-export/tests/export-config.test.ts index c3cf1b7914..e88d22fc9e 100644 --- a/kits/firestore-bigquery-export/tests/export-config.test.ts +++ b/kits/firestore-bigquery-export/tests/export-config.test.ts @@ -107,4 +107,14 @@ describe("toTrackerConfig", () => { ); expect(tracker.bqProjectId).toBe("analytics-project"); }); + + test("wires the backup collection into the tracker (queue exhaustion durability)", () => { + const tracker = toTrackerConfig( + resolveExportConfig({ ...base, backupCollectionId: "bq_failures" }) + ); + expect(tracker.backupTableId).toBe("bq_failures"); + + const withoutBackup = toTrackerConfig(resolveExportConfig(base)); + expect(withoutBackup.backupTableId).toBeUndefined(); + }); }); diff --git a/kits/firestore-bigquery-export/tests/handlers.test.ts b/kits/firestore-bigquery-export/tests/handlers.test.ts index bb607b5500..c8cd5796a7 100644 --- a/kits/firestore-bigquery-export/tests/handlers.test.ts +++ b/kits/firestore-bigquery-export/tests/handlers.test.ts @@ -312,6 +312,7 @@ describe("handleSyncBigQueryTask", () => { data: change.data, }); expect(events.recordSuccessEvent).toHaveBeenCalledTimes(1); + expect(ctx.enqueue).not.toHaveBeenCalled(); }); test("rethrows a failed write so Cloud Tasks retries", async () => { @@ -324,6 +325,9 @@ describe("handleSyncBigQueryTask", () => { handleSyncBigQueryTask(taskRequest(serializedChange(), 2), ctx) ).rejects.toThrow("still down"); expect(events.recordSuccessEvent).not.toHaveBeenCalled(); + // Re-enqueueing from the task would seed a trigger-queue loop; retries + // belong to Cloud Tasks alone. + expect(ctx.enqueue).not.toHaveBeenCalled(); }); test("rethrows a failed self-heal without attempting the write", async () => { diff --git a/kits/firestore-bigquery-export/tests/tasks.test.ts b/kits/firestore-bigquery-export/tests/tasks.test.ts index b541f175a0..9308eda44c 100644 --- a/kits/firestore-bigquery-export/tests/tasks.test.ts +++ b/kits/firestore-bigquery-export/tests/tasks.test.ts @@ -106,6 +106,15 @@ describe("enqueueSyncTask", () => { expect(enqueue).toHaveBeenCalledWith({ eventId: "evt-1" }); }); + test("a non-positive attempt budget still enqueues once", async () => { + const enqueue = vi.fn().mockResolvedValue(undefined); + mockQueue(enqueue); + + await enqueueSyncTask({ eventId: "evt-1" }, 0); + + expect(enqueue).toHaveBeenCalledTimes(1); + }); + test("retries a failed enqueue after a backoff and then succeeds", async () => { vi.useFakeTimers(); const enqueue = vi From 0f30873c710913fec1a568d1121b9eccb89938f9 Mon Sep 17 00:00:00 2001 From: Jacob Cable Date: Mon, 7 Sep 2026 09:36:13 +0100 Subject: [PATCH 3/8] fix(firestore-bigquery-export): keep buffered rows recoverable on every failure path - Attempt the buffered write even when provisioning fails. The tracker only parks a row in BACKUP_COLLECTION from its insert failure path, so throwing before the write dropped the row instead of backing it up. - Publish the success event outside the insert try. Rethrowing after the row had landed made Cloud Tasks retry past the insertId dedupe window and duplicate it. - Enqueue with a task id derived from the event id and treat task-already-exists as success, so an Eventarc redelivery after a failed enqueue cannot buffer the same event twice. - Drop the FUNCTION_REGION fallback. It is a gen1 variable and is not set on gen2 runtimes; resolve the region from DATABASE_REGION only. - Route MAX_ENQUEUE_ATTEMPTS and MAX_DISPATCHES_PER_SECOND through optionalInt so an unset param reaches resolveExportConfig as undefined and gets the documented default rather than 0. --- kits/firestore-bigquery-export/README.md | 18 +++-- kits/firestore-bigquery-export/src/config.ts | 19 ++++- .../firestore-bigquery-export/src/handlers.ts | 51 +++++++++---- kits/firestore-bigquery-export/src/tasks.ts | 33 ++++++--- .../tests/config.test.ts | 32 +++++++- .../tests/handlers.test.ts | 35 ++++++++- .../tests/tasks.test.ts | 73 +++++++++++++------ 7 files changed, 200 insertions(+), 61 deletions(-) diff --git a/kits/firestore-bigquery-export/README.md b/kits/firestore-bigquery-export/README.md index 99ceefaf3e..05fcbea5c3 100644 --- a/kits/firestore-bigquery-export/README.md +++ b/kits/firestore-bigquery-export/README.md @@ -230,7 +230,9 @@ curl -fsS -X POST -H "Content-Type: application/json" -d '{"data":{}}' \ The Firestore write path never provisions on the hot path. If resources are missing when a write arrives, the inline write fails and the change buffers through the `syncBigQuery` queue, whose handler calls `ensureInitialized()` as -a self-heal before re-attempting the write. +a self-heal before re-attempting the write. Provisioning is memoized once it +succeeds, so the self-heal covers resources that were never created, not +resources deleted out from under a warm instance. ## Failure handling @@ -238,15 +240,21 @@ The write path mirrors the extension's Cloud Tasks buffer: 1. The trigger attempts the BigQuery insert inline. On success, done. 2. On failure, it enqueues the serialized change onto the `syncBigQuery` queue - (up to `MAX_ENQUEUE_ATTEMPTS` in-process attempts with backoff) and the - execution succeeds - the Firestore event is not redelivered. + (up to `MAX_ENQUEUE_ATTEMPTS` in-process attempts with backoff, keyed by + event id so a retried enqueue cannot buffer the same event twice) and the + execution succeeds. A failed inline write on its own does not redeliver the + Firestore event. Failures _before_ the write is attempted (serializing the + change, publishing the `onStart` event) do rethrow, and `retry: true` means + those are redelivered by the runtime. 3. `syncBigQuery` re-attempts the write on the queue's schedule: 5 attempts, 60 seconds minimum backoff, throttled to `MAX_DISPATCHES_PER_SECOND` dispatches per second (500 concurrent max). 4. On every terminal insert failure the tracker writes the row to `BACKUP_COLLECTION` (when configured), keyed by the event id, before the - task fails. After the fifth attempt the task is dropped. **Without a backup - collection, the row is dropped with it** - configure `BACKUP_COLLECTION`. + task fails. A failed provisioning attempt is logged and the write is tried + anyway, so it still reaches that path. After the fifth attempt the task is + dropped. **Without a backup collection, the row is dropped with it** - + configure `BACKUP_COLLECTION`. 5. If the enqueue itself fails (BigQuery AND Cloud Tasks both failing), the trigger logs at error level and rethrows, so the Firestore event is redelivered by the runtime retry policy (`retry: true`) instead of being diff --git a/kits/firestore-bigquery-export/src/config.ts b/kits/firestore-bigquery-export/src/config.ts index 23e1f92cf3..d67ddc121f 100644 --- a/kits/firestore-bigquery-export/src/config.ts +++ b/kits/firestore-bigquery-export/src/config.ts @@ -20,7 +20,7 @@ import type { TimePartitioningGranularity, } from "@firebaseextensions/firestore-bigquery-change-tracker"; import { LogLevel } from "@firebaseextensions/firestore-bigquery-change-tracker"; -import type { Expression } from "firebase-functions/params"; +import type { Expression, IntParam } from "firebase-functions/params"; import { defineBoolean, defineInt, @@ -637,6 +637,19 @@ function optional(value: string): string | undefined { return value.length > 0 ? value : undefined; } +/** + * Reads an int param, reporting a missing or blank env var as `undefined`. + * + * `IntParam.value()` is `parseInt(env || "0", 10) || 0` and never consults the + * declared default, so an unset param has to reach `resolveExportConfig` as + * `undefined` for the documented default to apply. An explicit `0` is a real + * setting and is preserved. + */ +function optionalInt(param: IntParam): number | undefined { + const raw = process.env[param.name]?.trim(); + return raw === undefined || raw === "" ? undefined : param.value(); +} + /** * Resolves all deploy-time params into an {@link ExportConfig}. * @@ -679,7 +692,7 @@ export function configFromEnv(): ExportConfig { transformFunction: optional(params.transformFunction.value()), kmsKeyName: optional(params.kmsKeyName.value()), logLevel: normalizeLogLevel(params.logLevel.value()), - maxDispatchesPerSecond: params.maxDispatchesPerSecond.value(), - maxEnqueueAttempts: params.maxEnqueueAttempts.value(), + maxDispatchesPerSecond: optionalInt(params.maxDispatchesPerSecond), + maxEnqueueAttempts: optionalInt(params.maxEnqueueAttempts), }; } diff --git a/kits/firestore-bigquery-export/src/handlers.ts b/kits/firestore-bigquery-export/src/handlers.ts index e8f38a38fe..02b52299ae 100644 --- a/kits/firestore-bigquery-export/src/handlers.ts +++ b/kits/firestore-bigquery-export/src/handlers.ts @@ -226,9 +226,10 @@ export async function handleDocumentWrite( * Handles a `syncBigQuery` task: re-attempts a buffered write. Provisioning * runs first as a self-heal (memoized, a no-op after the first success), so a * write that failed only because the BigQuery resources were missing succeeds - * on the first task attempt. A failed write rethrows so Cloud Tasks retries - * on the queue's schedule; the tracker has already written the row to the - * backup collection (when one is configured) before each terminal rethrow. + * on the first task attempt. A failed provision is logged and the write is + * attempted anyway, so the tracker still parks the row in the backup + * collection. A failed write rethrows so Cloud Tasks retries on the queue's + * schedule. * * @param req - The dispatched task request carrying the serialized change. * @param ctx - The handler context. @@ -247,9 +248,34 @@ export async function handleSyncBigQueryTask( ); try { - await ctx.ensureInitialized(); + try { + await ctx.ensureInitialized(); + } catch (initErr) { + // Fall through to the write regardless: the tracker only parks a row in + // BACKUP_COLLECTION from its insert failure path, so throwing here would + // drop the row instead of backing it up. + logs.error( + false, + "Failed to provision BigQuery resources before a buffered write", + initErr as Error + ); + } + await recordEventToBigQuery(change, ctx.tracker); + } catch (err) { + logs.logFailedEventAction( + "Failed to write event to BigQuery from onDispatch handler", + change.fullResourceName, + change.eventId, + change.changeType, + err as Error, + req.retryCount + ); + throw err; + } + + try { await events.recordSuccessEvent({ subject: change.documentId, data: { @@ -263,18 +289,11 @@ export async function handleSyncBigQueryTask( oldData: change.oldData, }, }); - - logs.complete(); } catch (err) { - logs.logFailedEventAction( - "Failed to write event to BigQuery from onDispatch handler", - change.fullResourceName, - change.eventId, - change.changeType, - err as Error, - req.retryCount - ); - - throw err; + // The row is already in BigQuery. Rethrowing would have Cloud Tasks retry + // the insert past the dedupe window and duplicate it. + logs.error(false, "Failed to record success event", err as Error); } + + logs.complete(); } diff --git a/kits/firestore-bigquery-export/src/tasks.ts b/kits/firestore-bigquery-export/src/tasks.ts index b6b501cadc..91e921ccea 100644 --- a/kits/firestore-bigquery-export/src/tasks.ts +++ b/kits/firestore-bigquery-export/src/tasks.ts @@ -15,6 +15,7 @@ */ import { getFunctions } from "firebase-admin/functions"; +import type { SerializedDocumentChange } from "./handlers"; import { firestoreLocationToFunctionRegion } from "./region"; /** Export name of the write-buffer task function. */ @@ -24,6 +25,13 @@ const MAX_BACKOFF_MS = 5000; const BACKOFF_BASE_MS = 100; const JITTER_MS = 100; +/** Cloud Tasks accepts `[A-Za-z0-9_-]{1,500}` as a task id. */ +const TASK_ID_DISALLOWED = /[^A-Za-z0-9_-]/g; + +function taskIdFor(change: SerializedDocumentChange): string { + return change.eventId.replace(TASK_ID_DISALLOWED, "-").slice(0, 500); +} + /** * Resolves the queue resource path for a task function of this kit instance. * @@ -31,8 +39,8 @@ const JITTER_MS = 100; * deployed `kit--` prefix itself from the * `FIREBASE_KIT_INSTANCE_ID` env var, which the CLI sets on every deployed kit * function. All functions of a kit instance deploy to one region, so the - * enqueuing function's own region (`DATABASE_REGION`-derived, with the - * CLI-set `FUNCTION_REGION` as fallback) is also the queue's region. + * enqueuing function's own `DATABASE_REGION`-derived region is also the + * queue's region. * * @param functionName - The export name of the task function. * @returns The queue resource path, `locations//functions/`. @@ -41,14 +49,12 @@ const JITTER_MS = 100; export function syncQueuePath( functionName: string = SYNC_BIGQUERY_FUNCTION ): string { - const region = - firestoreLocationToFunctionRegion(process.env.DATABASE_REGION) ?? - process.env.FUNCTION_REGION; + const region = firestoreLocationToFunctionRegion(process.env.DATABASE_REGION); if (!region) { throw new Error( "A region is required to resolve the syncBigQuery task queue. " + - "Set DATABASE_REGION, or deploy with the Firebase CLI so FUNCTION_REGION is set." + "Set DATABASE_REGION." ); } @@ -65,17 +71,21 @@ function backoffMs(attempt: number, jitter: number): number { * Enqueues a payload onto the `syncBigQuery` queue, retrying transient enqueue * failures in-process with exponential backoff and jitter. * - * @param payload - The task payload. + * The task id is derived from the event id, so a retried enqueue of an event + * that already reached Cloud Tasks is rejected rather than buffered twice. + * + * @param payload - The serialized change to enqueue. * @param maxAttempts - How many enqueue attempts to make before giving up. * Clamped to at least 1: resolving without an enqueue would report success * for an event that was never buffered anywhere. * @throws The last enqueue error, once every attempt has failed. */ export async function enqueueSyncTask( - payload: object, + payload: SerializedDocumentChange, maxAttempts: number ): Promise { const queue = getFunctions().taskQueue(syncQueuePath()); + const id = taskIdFor(payload); const attemptBudget = Math.max(1, maxAttempts); const jitter = Math.random() * JITTER_MS; @@ -90,9 +100,14 @@ export async function enqueueSyncTask( attempts++; try { - await queue.enqueue(payload); + await queue.enqueue(payload, { id }); return; } catch (enqueueErr) { + // The event is already buffered; a second task would double-write the row. + if ((enqueueErr as { code?: string })?.code === "task-already-exists") { + return; + } + if (attempts >= attemptBudget) { throw enqueueErr; } diff --git a/kits/firestore-bigquery-export/tests/config.test.ts b/kits/firestore-bigquery-export/tests/config.test.ts index c875409c1e..191a85c5e8 100644 --- a/kits/firestore-bigquery-export/tests/config.test.ts +++ b/kits/firestore-bigquery-export/tests/config.test.ts @@ -44,8 +44,11 @@ vi.mock("firebase-functions/params", () => ({ : opts?.default?.value() ?? "", toString: () => `params.${_name}`, }), + // Mirrors the real IntParam: a missing or blank env var resolves to 0, and + // the declared default never reaches runtime. defineInt: (_name: string, opts?: { default?: number }) => ({ - value: () => opts?.default ?? 0, + name: _name, + value: () => Number.parseInt(process.env[_name] || "0", 10) || 0, toString: () => `params.${_name}`, }), defineBoolean: (_name: string, opts?: { default?: boolean }) => ({ @@ -171,10 +174,31 @@ describe("configFromEnv", () => { expect(resolveExportConfig(config)).not.toHaveProperty("location"); }); - test("queue params keep the extension's defaults", () => { + test("reports unset queue params as undefined so the documented defaults apply", () => { + // IntParam.value() resolves an unset var to 0, which would defeat the + // `?? 100` / `?? 3` fallbacks in resolveExportConfig. const config = configFromEnv(); - expect(config.maxDispatchesPerSecond).toBe(100); - expect(config.maxEnqueueAttempts).toBe(3); + expect(config.maxDispatchesPerSecond).toBeUndefined(); + expect(config.maxEnqueueAttempts).toBeUndefined(); + + const resolved = resolveExportConfig(config); + expect(resolved.maxDispatchesPerSecond).toBe(100); + expect(resolved.maxEnqueueAttempts).toBe(3); + }); + + test("reports a blank queue param as undefined", () => { + vi.stubEnv("MAX_ENQUEUE_ATTEMPTS", " "); + expect(configFromEnv().maxEnqueueAttempts).toBeUndefined(); + vi.unstubAllEnvs(); + }); + + test("passes an explicit queue param through", () => { + vi.stubEnv("MAX_ENQUEUE_ATTEMPTS", "7"); + vi.stubEnv("MAX_DISPATCHES_PER_SECOND", "250"); + const config = configFromEnv(); + expect(config.maxEnqueueAttempts).toBe(7); + expect(config.maxDispatchesPerSecond).toBe(250); + vi.unstubAllEnvs(); }); test("exposes deploy-time expressions for trigger metadata", () => { diff --git a/kits/firestore-bigquery-export/tests/handlers.test.ts b/kits/firestore-bigquery-export/tests/handlers.test.ts index c8cd5796a7..e2293e61fd 100644 --- a/kits/firestore-bigquery-export/tests/handlers.test.ts +++ b/kits/firestore-bigquery-export/tests/handlers.test.ts @@ -330,15 +330,44 @@ describe("handleSyncBigQueryTask", () => { expect(ctx.enqueue).not.toHaveBeenCalled(); }); - test("rethrows a failed self-heal without attempting the write", async () => { + test("attempts the write even when the self-heal fails", async () => { + // The tracker only parks a row in BACKUP_COLLECTION from its insert + // failure path, so skipping the write would drop the row instead. const ctx = makeCtx(); (ctx.ensureInitialized as ReturnType).mockRejectedValueOnce( new Error("no dataset") ); + await handleSyncBigQueryTask(taskRequest(serializedChange()), ctx); + + expect(ctx.tracker.record).toHaveBeenCalledTimes(1); + }); + + test("surfaces the write error when the self-heal also failed", async () => { + const ctx = makeCtx(); + (ctx.ensureInitialized as ReturnType).mockRejectedValueOnce( + new Error("no dataset") + ); + (ctx.tracker.record as ReturnType).mockRejectedValueOnce( + new Error("no table") + ); + await expect( handleSyncBigQueryTask(taskRequest(serializedChange()), ctx) - ).rejects.toThrow("no dataset"); - expect(ctx.tracker.record).not.toHaveBeenCalled(); + ).rejects.toThrow("no table"); + }); + + test("does not rethrow when the success event fails after the row lands", async () => { + // The row is already in BigQuery; a Cloud Tasks retry would land past the + // insertId dedupe window and duplicate it. + const ctx = makeCtx(); + ( + events.recordSuccessEvent as ReturnType + ).mockRejectedValueOnce(new Error("channel down")); + + await expect( + handleSyncBigQueryTask(taskRequest(serializedChange()), ctx) + ).resolves.toBeUndefined(); + expect(ctx.tracker.record).toHaveBeenCalledTimes(1); }); }); diff --git a/kits/firestore-bigquery-export/tests/tasks.test.ts b/kits/firestore-bigquery-export/tests/tasks.test.ts index 9308eda44c..5aafe2fecd 100644 --- a/kits/firestore-bigquery-export/tests/tasks.test.ts +++ b/kits/firestore-bigquery-export/tests/tasks.test.ts @@ -21,9 +21,26 @@ vi.mock("firebase-admin/functions", () => ({ })); import { getFunctions } from "firebase-admin/functions"; +import type { SerializedDocumentChange } from "../src/handlers"; import { enqueueSyncTask, syncQueuePath } from "../src/tasks"; -const ENV_KEYS = ["DATABASE_REGION", "FUNCTION_REGION"] as const; +function makeChange( + overrides: Partial = {} +): SerializedDocumentChange { + return { + timestamp: "2026-01-01T00:00:00.000Z", + eventId: "evt-1", + fullResourceName: "projects/p/databases/(default)/documents/c/d", + changeType: "CREATE", + documentId: "d", + params: null, + data: { a: 1 }, + oldData: undefined, + ...overrides, + } as SerializedDocumentChange; +} + +const ENV_KEYS = ["DATABASE_REGION"] as const; const originalEnv: Record = {}; beforeEach(() => { @@ -60,22 +77,12 @@ describe("syncQueuePath", () => { ); }); - test("falls back to FUNCTION_REGION when DATABASE_REGION is unset", () => { - process.env.FUNCTION_REGION = "us-central1"; - expect(syncQueuePath()).toBe( - "locations/us-central1/functions/syncBigQuery" - ); - }); - - test("prefers DATABASE_REGION over FUNCTION_REGION", () => { - process.env.DATABASE_REGION = "eur3"; - process.env.FUNCTION_REGION = "us-central1"; - expect(syncQueuePath()).toBe( - "locations/europe-west1/functions/syncBigQuery" - ); + test("throws when DATABASE_REGION is unset", () => { + expect(() => syncQueuePath()).toThrow(/region/i); }); - test("throws when no region is resolvable", () => { + test("throws when DATABASE_REGION is an empty string", () => { + process.env.DATABASE_REGION = ""; expect(() => syncQueuePath()).toThrow(/region/i); }); }); @@ -90,27 +97,51 @@ describe("enqueueSyncTask", () => { } beforeEach(() => { - process.env.FUNCTION_REGION = "us-central1"; + process.env.DATABASE_REGION = "us-central1"; }); test("targets the bare function name; the admin SDK adds the kit prefix", async () => { const enqueue = vi.fn().mockResolvedValue(undefined); const taskQueue = mockQueue(enqueue); - await enqueueSyncTask({ eventId: "evt-1" }, 3); + const change = makeChange(); + await enqueueSyncTask(change, 3); expect(taskQueue).toHaveBeenCalledWith( "locations/us-central1/functions/syncBigQuery" ); expect(enqueue).toHaveBeenCalledTimes(1); - expect(enqueue).toHaveBeenCalledWith({ eventId: "evt-1" }); + expect(enqueue).toHaveBeenCalledWith(change, { id: "evt-1" }); + }); + + test("derives the task id from the event id so a retry cannot double-buffer", async () => { + const enqueue = vi.fn().mockResolvedValue(undefined); + mockQueue(enqueue); + + await enqueueSyncTask(makeChange({ eventId: "a/b:c d" }), 3); + + expect(enqueue).toHaveBeenCalledWith(expect.anything(), { + id: "a-b-c-d", + }); + }); + + test("treats an already-enqueued task as success", async () => { + const enqueue = vi + .fn() + .mockRejectedValue( + Object.assign(new Error("exists"), { code: "task-already-exists" }) + ); + mockQueue(enqueue); + + await expect(enqueueSyncTask(makeChange(), 3)).resolves.toBeUndefined(); + expect(enqueue).toHaveBeenCalledTimes(1); }); test("a non-positive attempt budget still enqueues once", async () => { const enqueue = vi.fn().mockResolvedValue(undefined); mockQueue(enqueue); - await enqueueSyncTask({ eventId: "evt-1" }, 0); + await enqueueSyncTask(makeChange(), 0); expect(enqueue).toHaveBeenCalledTimes(1); }); @@ -123,7 +154,7 @@ describe("enqueueSyncTask", () => { .mockResolvedValueOnce(undefined); mockQueue(enqueue); - const pending = enqueueSyncTask({}, 3); + const pending = enqueueSyncTask(makeChange(), 3); await vi.runAllTimersAsync(); await pending; @@ -139,7 +170,7 @@ describe("enqueueSyncTask", () => { .mockRejectedValue(new Error("last")); mockQueue(enqueue); - const pending = enqueueSyncTask({}, 3); + const pending = enqueueSyncTask(makeChange(), 3); // Attach the rejection expectation before advancing timers so the // rejection is never unhandled. const assertion = expect(pending).rejects.toThrow("last"); From a2781a7d08718e91efcab3572569ca1a06ae85cc Mon Sep 17 00:00:00 2001 From: Jacob Cable Date: Mon, 7 Sep 2026 09:42:26 +0100 Subject: [PATCH 4/8] fix(firestore-bigquery-export): match the prefixed task-already-exists code firebase-admin reports the duplicate-task error as functions/task-already-exists, so the bare comparison never matched and a redelivered enqueue rethrew instead of resolving. The test mock now carries the prefixed code as the SDK does. --- kits/firestore-bigquery-export/src/tasks.ts | 6 +++++- kits/firestore-bigquery-export/tests/tasks.test.ts | 11 ++++++----- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/kits/firestore-bigquery-export/src/tasks.ts b/kits/firestore-bigquery-export/src/tasks.ts index 91e921ccea..1e9cdb5b8c 100644 --- a/kits/firestore-bigquery-export/src/tasks.ts +++ b/kits/firestore-bigquery-export/src/tasks.ts @@ -104,7 +104,11 @@ export async function enqueueSyncTask( return; } catch (enqueueErr) { // The event is already buffered; a second task would double-write the row. - if ((enqueueErr as { code?: string })?.code === "task-already-exists") { + // firebase-admin prefixes its codes: `functions/task-already-exists`. + if ( + (enqueueErr as { code?: string })?.code === + "functions/task-already-exists" + ) { return; } diff --git a/kits/firestore-bigquery-export/tests/tasks.test.ts b/kits/firestore-bigquery-export/tests/tasks.test.ts index 5aafe2fecd..6b8bde938b 100644 --- a/kits/firestore-bigquery-export/tests/tasks.test.ts +++ b/kits/firestore-bigquery-export/tests/tasks.test.ts @@ -126,11 +126,12 @@ describe("enqueueSyncTask", () => { }); test("treats an already-enqueued task as success", async () => { - const enqueue = vi - .fn() - .mockRejectedValue( - Object.assign(new Error("exists"), { code: "task-already-exists" }) - ); + const enqueue = vi.fn().mockRejectedValue( + // Shaped like firebase-admin's PrefixedFirebaseError: `/`. + Object.assign(new Error("exists"), { + code: "functions/task-already-exists", + }) + ); mockQueue(enqueue); await expect(enqueueSyncTask(makeChange(), 3)).resolves.toBeUndefined(); From 4b1b9ba40e834d8767f84092f83506aaa95a5faf Mon Sep 17 00:00:00 2001 From: Jacob Cable Date: Mon, 7 Sep 2026 10:06:29 +0100 Subject: [PATCH 5/8] fix(firestore-bigquery-export): resolve the queue region from FUNCTION_REGION first The Firebase CLI sets FUNCTION_REGION on every deployed gen2 function (cloudfunctionsv2.js sets it from endpoint.region), so it is the region the queue actually lives in. Dropping it in 0f30873c was based on a wrong claim that the variable is never set. With DATABASE_REGION empty, which the README documents as supported, every failed inline write threw before the first enqueue attempt and went back to Eventarc for redelivery; on a first interactive deploy the queue path named the DATABASE_REGION target while the functions were in us-central1. Also clamp a NaN or non-integer attempt budget to one attempt: Math.max(1, NaN) is NaN and skipped the enqueue loop entirely. --- kits/firestore-bigquery-export/src/tasks.ts | 18 +++++++--- .../tests/tasks.test.ts | 35 +++++++++++++++++-- 2 files changed, 45 insertions(+), 8 deletions(-) diff --git a/kits/firestore-bigquery-export/src/tasks.ts b/kits/firestore-bigquery-export/src/tasks.ts index 1e9cdb5b8c..558168788f 100644 --- a/kits/firestore-bigquery-export/src/tasks.ts +++ b/kits/firestore-bigquery-export/src/tasks.ts @@ -39,8 +39,12 @@ function taskIdFor(change: SerializedDocumentChange): string { * deployed `kit--` prefix itself from the * `FIREBASE_KIT_INSTANCE_ID` env var, which the CLI sets on every deployed kit * function. All functions of a kit instance deploy to one region, so the - * enqueuing function's own `DATABASE_REGION`-derived region is also the - * queue's region. + * enqueuing function's own region is the queue's region. + * + * The CLI-set `FUNCTION_REGION` wins because it is the region the function + * was actually deployed to; a `DATABASE_REGION`-derived region can disagree + * with it on a first deploy or when the variable is unset, and is only the + * fallback for local runs where the CLI has not populated the environment. * * @param functionName - The export name of the task function. * @returns The queue resource path, `locations//functions/`. @@ -49,12 +53,14 @@ function taskIdFor(change: SerializedDocumentChange): string { export function syncQueuePath( functionName: string = SYNC_BIGQUERY_FUNCTION ): string { - const region = firestoreLocationToFunctionRegion(process.env.DATABASE_REGION); + const region = + process.env.FUNCTION_REGION || + firestoreLocationToFunctionRegion(process.env.DATABASE_REGION); if (!region) { throw new Error( "A region is required to resolve the syncBigQuery task queue. " + - "Set DATABASE_REGION." + "Deploy with the Firebase CLI (which sets FUNCTION_REGION) or set DATABASE_REGION." ); } @@ -87,7 +93,9 @@ export async function enqueueSyncTask( const queue = getFunctions().taskQueue(syncQueuePath()); const id = taskIdFor(payload); - const attemptBudget = Math.max(1, maxAttempts); + // Math.max(1, NaN) is NaN and would skip the loop entirely. + const attemptBudget = + Number.isInteger(maxAttempts) && maxAttempts >= 1 ? maxAttempts : 1; const jitter = Math.random() * JITTER_MS; let attempts = 0; diff --git a/kits/firestore-bigquery-export/tests/tasks.test.ts b/kits/firestore-bigquery-export/tests/tasks.test.ts index 6b8bde938b..612d28c207 100644 --- a/kits/firestore-bigquery-export/tests/tasks.test.ts +++ b/kits/firestore-bigquery-export/tests/tasks.test.ts @@ -40,7 +40,7 @@ function makeChange( } as SerializedDocumentChange; } -const ENV_KEYS = ["DATABASE_REGION"] as const; +const ENV_KEYS = ["DATABASE_REGION", "FUNCTION_REGION"] as const; const originalEnv: Record = {}; beforeEach(() => { @@ -77,12 +77,31 @@ describe("syncQueuePath", () => { ); }); - test("throws when DATABASE_REGION is unset", () => { + test("prefers the CLI-set FUNCTION_REGION, the region the function is deployed in", () => { + // On a first interactive deploy the functions land in us-central1 while + // DATABASE_REGION already names the target region; the queue is where + // the functions are. + process.env.FUNCTION_REGION = "us-central1"; + process.env.DATABASE_REGION = "europe-west2"; + expect(syncQueuePath()).toBe( + "locations/us-central1/functions/syncBigQuery" + ); + }); + + test("falls back to DATABASE_REGION when FUNCTION_REGION is unset", () => { + process.env.DATABASE_REGION = "europe-west2"; + expect(syncQueuePath()).toBe( + "locations/europe-west2/functions/syncBigQuery" + ); + }); + + test("throws when neither region variable is set", () => { expect(() => syncQueuePath()).toThrow(/region/i); }); - test("throws when DATABASE_REGION is an empty string", () => { + test("throws when both region variables are empty strings", () => { process.env.DATABASE_REGION = ""; + process.env.FUNCTION_REGION = ""; expect(() => syncQueuePath()).toThrow(/region/i); }); }); @@ -147,6 +166,16 @@ describe("enqueueSyncTask", () => { expect(enqueue).toHaveBeenCalledTimes(1); }); + test("a NaN or non-integer attempt budget still enqueues once", async () => { + const enqueue = vi.fn().mockResolvedValue(undefined); + mockQueue(enqueue); + + await enqueueSyncTask(makeChange(), NaN); + await enqueueSyncTask(makeChange(), Infinity); + + expect(enqueue).toHaveBeenCalledTimes(2); + }); + test("retries a failed enqueue after a backoff and then succeeds", async () => { vi.useFakeTimers(); const enqueue = vi From da45faf702fd6b6b1f7e0c6474c569aad1752e55 Mon Sep 17 00:00:00 2001 From: Jacob Cable Date: Mon, 7 Sep 2026 10:09:13 +0100 Subject: [PATCH 6/8] fix(firestore-bigquery-export): match the extension's retry behaviour on the trigger The extension declares no retry policy on fsexportbigquery and, when the enqueue itself fails, logs at error level, publishes onError, and drops the event. The kit kept retry: true from before the buffer landed and rethrew on enqueue exhaustion, which redelivered the event through Eventarc for up to 24 hours; it also redelivered on every failure before the write was attempted (serialization, onStart publish), which the PR text did not say. Drop retry: true and stop rethrowing on enqueue exhaustion so the failure behaviour and cost profile match the extension. --- kits/firestore-bigquery-export/CHANGELOG.md | 2 +- kits/firestore-bigquery-export/README.md | 32 +++++++++---------- .../src/export-config.ts | 2 +- .../firestore-bigquery-export/src/handlers.ts | 10 +++--- kits/firestore-bigquery-export/src/index.ts | 7 ++-- .../tests/handlers.test.ts | 8 +++-- .../tests/index.test.ts | 4 +-- 7 files changed, 31 insertions(+), 34 deletions(-) diff --git a/kits/firestore-bigquery-export/CHANGELOG.md b/kits/firestore-bigquery-export/CHANGELOG.md index 76f5c70c7d..a3a12579e5 100644 --- a/kits/firestore-bigquery-export/CHANGELOG.md +++ b/kits/firestore-bigquery-export/CHANGELOG.md @@ -1,4 +1,4 @@ -- feat: reinstate the extension's Cloud Tasks write buffer. A failed inline BigQuery write now enqueues onto a new `syncBigQuery` task queue (5 attempts, 60s minimum backoff, throttled by the restored `MAX_DISPATCHES_PER_SECOND` param, default 100) instead of replaying the Firestore event through Eventarc redelivery for up to 24 hours; `MAX_ENQUEUE_ATTEMPTS` (default 3) is also back. The `onSuccess` event returns with the queue handler. Two behavior changes against earlier release candidates: a row that exhausts the queue is dropped unless `BACKUP_COLLECTION` is set (extension parity - the tracker backs the row up on every terminal insert failure, so configure a backup collection), and deleting or moving the functions can leave the Cloud Tasks queue behind. One deliberate fix over the extension: a failed enqueue is logged at error level and rethrown so the trigger's retry redelivers the event, where the extension silently dropped it. Export the new `syncBigQuery` function from your codebase entry, and deploy with Firebase CLI 15.28.0+ so the trigger can address its own queue (`FIREBASE_KIT_INSTANCE_ID`); requires firebase-admin 14.2.0+. +- feat: reinstate the extension's Cloud Tasks write buffer. A failed inline BigQuery write now enqueues onto a new `syncBigQuery` task queue (5 attempts, 60s minimum backoff, throttled by the restored `MAX_DISPATCHES_PER_SECOND` param, default 100) instead of replaying the Firestore event through Eventarc redelivery for up to 24 hours; `MAX_ENQUEUE_ATTEMPTS` (default 3) is also back. The `onSuccess` event returns with the queue handler. Two behavior changes against earlier release candidates: a row that exhausts the queue is dropped unless `BACKUP_COLLECTION` is set (extension parity - the tracker backs the row up on every terminal insert failure, so configure a backup collection), and deleting or moving the functions can leave the Cloud Tasks queue behind. A failed enqueue is logged at error level, published as an `onError` event, and dropped, as in the extension; the trigger no longer declares `retry: true`, so nothing is redelivered through Eventarc. Export the new `syncBigQuery` function from your codebase entry, and deploy with Firebase CLI 15.28.0+ so the trigger can address its own queue (`FIREBASE_KIT_INSTANCE_ID`); requires firebase-admin 14.2.0+. - fix: restore explicit function placement from `DATABASE_REGION`, now with the Firestore-location-to-Cloud-Run-region mapping. The `DATABASE_REGION` parameter is back and all three functions deploy to the region derived from it: regional locations pass through unchanged, and the multi-region locations map to a Cloud Run region (`nam5`/`nam7` to `us-central1`, `eur3` to `europe-west1`) instead of failing the deploy. With the parameter unset the functions still declare no region and the CLI falls back as before (`us-central1` by default, `FIREBASE_FUNCTIONS_DEFAULT_REGION` to override). Placement requires firebase-tools >= 15.28.0 (older CLIs do not load `.env` at discovery and keep the fallback). If your `.env` already carries `DATABASE_REGION` from an extension migration, upgrading to this version moves the functions to the mapped region on your next deploy, which deletes and recreates them. - fix: stop deploying functions to the `DATABASE_REGION` value. Firestore multi-region locations (`eur3`, `nam5`, `nam7`) are not Cloud Run regions, so any multi-region database made every deploy fail. The `DATABASE_REGION` parameter is removed; the functions now declare no region and deploy to `us-central1` by default (set `FIREBASE_FUNCTIONS_DEFAULT_REGION` when deploying to choose another region), while the Firestore trigger is always pinned to the database's own region. `ExportConfig.location` is removed from the library surface. - Initial release of kit, see README for differences between the legacy extension and this kit diff --git a/kits/firestore-bigquery-export/README.md b/kits/firestore-bigquery-export/README.md index 05fcbea5c3..cc17477293 100644 --- a/kits/firestore-bigquery-export/README.md +++ b/kits/firestore-bigquery-export/README.md @@ -97,8 +97,9 @@ deploy as `kit-default-fsexportbigquery`, `kit-default-syncBigQuery`, Deploy with Firebase CLI 15.28.0 or later: it sets the `FIREBASE_KIT_INSTANCE_ID` env var on the deployed functions, which the trigger needs to address its own `syncBigQuery` queue. On functions deployed with an -older CLI, enqueues fail (loudly - the event is redelivered, not lost) until -you redeploy with a newer CLI. +older CLI, enqueues fail (logged at error level and published as an `onError` +event; the event is dropped, as in the extension) until you redeploy with a +newer CLI. ```sh firebase experiments:enable kits @@ -124,7 +125,7 @@ loads them at deploy time and prompts for any required values that are missing. | `bigqueryProjectId` | `BIGQUERY_PROJECT_ID` | no | project id | Dataset project, if different | | `backupCollection` | `BACKUP_COLLECTION` | no | (empty) | Strongly recommended: collection for rows the queue gave up on | | `maxDispatchesPerSecond` | `MAX_DISPATCHES_PER_SECOND` | no | `100` | `syncBigQuery` queue dispatch rate (1-500) | -| `maxEnqueueAttempts` | `MAX_ENQUEUE_ATTEMPTS` | no | `3` | In-process enqueue attempts before rethrowing (1-10) | +| `maxEnqueueAttempts` | `MAX_ENQUEUE_ATTEMPTS` | no | `3` | In-process enqueue attempts before giving up (1-10) | | `transformFunction` | `TRANSFORM_FUNCTION` | no | (empty) | Optional transform Cloud Function | | `tablePartitioning` | `TABLE_PARTITIONING` | no | `NONE` | Table partitioning strategy | | `timePartitioningField` | `TIME_PARTITIONING_FIELD` | no | (empty) | Time-partitioning column name | @@ -242,10 +243,10 @@ The write path mirrors the extension's Cloud Tasks buffer: 2. On failure, it enqueues the serialized change onto the `syncBigQuery` queue (up to `MAX_ENQUEUE_ATTEMPTS` in-process attempts with backoff, keyed by event id so a retried enqueue cannot buffer the same event twice) and the - execution succeeds. A failed inline write on its own does not redeliver the - Firestore event. Failures _before_ the write is attempted (serializing the - change, publishing the `onStart` event) do rethrow, and `retry: true` means - those are redelivered by the runtime. + execution succeeds. The trigger declares no retry policy, as in the + extension: a failure _before_ the write is attempted (serializing the + change, publishing the `onStart` event) fails the execution once and the + event is not redelivered. 3. `syncBigQuery` re-attempts the write on the queue's schedule: 5 attempts, 60 seconds minimum backoff, throttled to `MAX_DISPATCHES_PER_SECOND` dispatches per second (500 concurrent max). @@ -256,10 +257,9 @@ The write path mirrors the extension's Cloud Tasks buffer: dropped. **Without a backup collection, the row is dropped with it** - configure `BACKUP_COLLECTION`. 5. If the enqueue itself fails (BigQuery AND Cloud Tasks both failing), the - trigger logs at error level and rethrows, so the Firestore event is - redelivered by the runtime retry policy (`retry: true`) instead of being - lost. The extension silently dropped the event in this window; this kit - does not. + trigger logs at error level, publishes an `onError` event, and the + execution succeeds: the event is dropped, exactly as the extension did in + this window. ### Recovering parked rows @@ -295,7 +295,7 @@ boolean params, and only the literal string `true` enables them. The extension used `yes` / `no` for the last two, so copying an old config across leaves them silently disabled. Change any `yes` to `true` in your `.env`. -### Failed writes: same buffer, one fix +### Failed writes: same buffer The kit keeps the extension's write-path architecture: a failed BigQuery write buffers through the `syncBigQuery` Cloud Tasks queue, with the same shape (5 @@ -303,11 +303,9 @@ attempts, 60s minimum backoff, `MAX_DISPATCHES_PER_SECOND` throttling) and the same knobs (`MAX_DISPATCHES_PER_SECOND`, `MAX_ENQUEUE_ATTEMPTS`) - your migrated `.env` values carry over unchanged. -One deliberate fix: when the enqueue itself failed, the extension swallowed the -error and dropped the event with no trace. The kit logs it at error level and -rethrows so the Firestore event is redelivered (`retry: true` on the trigger). -You only pay that redelivery cost in the window where BigQuery and Cloud Tasks -are failing at the same time. +When the enqueue itself fails, the kit does what the extension does: logs at +error level, publishes an `onError` event, and drops the event. The trigger +declares no retry policy, so nothing is redelivered through Eventarc. Earlier release candidates of this kit had no queue: they retried every failed write through Eventarc redelivery for up to 24 hours and never lost a row diff --git a/kits/firestore-bigquery-export/src/export-config.ts b/kits/firestore-bigquery-export/src/export-config.ts index 84d768e6f8..e8af202881 100644 --- a/kits/firestore-bigquery-export/src/export-config.ts +++ b/kits/firestore-bigquery-export/src/export-config.ts @@ -94,7 +94,7 @@ export interface ExportConfig { maxDispatchesPerSecond?: ConfigValue; /** * How many times the trigger tries to enqueue a failed write onto the - * `syncBigQuery` queue before giving up and rethrowing. Defaults to `3`. + * `syncBigQuery` queue before giving up. Defaults to `3`. */ maxEnqueueAttempts?: ConfigValue; } diff --git a/kits/firestore-bigquery-export/src/handlers.ts b/kits/firestore-bigquery-export/src/handlers.ts index 02b52299ae..199b52db74 100644 --- a/kits/firestore-bigquery-export/src/handlers.ts +++ b/kits/firestore-bigquery-export/src/handlers.ts @@ -99,9 +99,9 @@ async function recordEventToBigQuery( /** * Buffers a failed inline write through the `syncBigQuery` task queue. A - * terminal enqueue failure is logged, recorded, and rethrown so the trigger - * retry policy covers the window where both BigQuery and Cloud Tasks fail; - * swallowing it here would drop the event with no durable copy. + * terminal enqueue failure is logged and published as an `onError` event, then + * dropped, exactly as the extension did: with no retry policy on the trigger a + * rethrow would only fail the execution once and drop it anyway. * * @param change - The serialized change to enqueue. * @param ctx - The handler context. @@ -122,15 +122,13 @@ async function enqueueForSync( change.changeType, enqueueErr as Error ); - - throw enqueueErr; } } /** * Handles a Firestore document write: serializes the change and writes it to * BigQuery. A failed inline write is buffered through the `syncBigQuery` task - * queue; only a failed enqueue surfaces to the trigger retry policy. + * queue; a failed enqueue is logged and dropped. * * @param event - The Firestore document-write event. * @param ctx - The handler context. diff --git a/kits/firestore-bigquery-export/src/index.ts b/kits/firestore-bigquery-export/src/index.ts index eb76d083ee..99338a80a9 100644 --- a/kits/firestore-bigquery-export/src/index.ts +++ b/kits/firestore-bigquery-export/src/index.ts @@ -147,16 +147,15 @@ const functionRegion = firestoreLocationToFunctionRegion( /** * Firestore trigger: streams document writes on the watched collection into the * BigQuery changelog table. A failed inline write buffers through the - * `syncBigQuery` queue and the execution still succeeds; `retry: true` stays on - * so an event whose enqueue ALSO failed (rethrown by the handler) is - * redelivered instead of dropped. + * `syncBigQuery` queue and the execution still succeeds. No runtime retry + * policy, as in the extension: a failure before the write is attempted fails + * the execution once, and a failed enqueue is logged and dropped. */ export const fsexportbigquery = onDocumentWritten( { ...(functionRegion ? { region: functionRegion } : {}), document: expr`${CONFIG_EXPRESSIONS.collectionPath}/{documentId}`, database: CONFIG_EXPRESSIONS.database, - retry: true, }, (event) => handleDocumentWrite(event, getHandlerContext()) ); diff --git a/kits/firestore-bigquery-export/tests/handlers.test.ts b/kits/firestore-bigquery-export/tests/handlers.test.ts index e2293e61fd..37d880fdfe 100644 --- a/kits/firestore-bigquery-export/tests/handlers.test.ts +++ b/kits/firestore-bigquery-export/tests/handlers.test.ts @@ -255,7 +255,9 @@ describe("handleDocumentWrite", () => { expect(JSON.parse(JSON.stringify(enqueued))).toEqual(enqueued); }); - test("a failed enqueue is recorded and rethrown, never swallowed", async () => { + test("a failed enqueue is recorded and logged, then the execution succeeds", async () => { + // Extension parity: no retry policy on the trigger, so a rethrow would + // only fail the execution once and drop the event anyway. const ctx = makeCtx(); (ctx.tracker.record as ReturnType).mockRejectedValueOnce( new Error("bq down") @@ -269,8 +271,8 @@ describe("handleDocumentWrite", () => { writeEvent(snap(false, "doc1"), snap(true, "doc1", { a: 1 })), ctx ) - ).rejects.toThrow("tasks down"); - expect(events.recordErrorEvent).toHaveBeenCalled(); + ).resolves.toBeUndefined(); + expect(events.recordErrorEvent).toHaveBeenCalledTimes(1); }); test("rethrows when serialization fails", async () => { diff --git a/kits/firestore-bigquery-export/tests/index.test.ts b/kits/firestore-bigquery-export/tests/index.test.ts index e2276fead4..967aa7bc43 100644 --- a/kits/firestore-bigquery-export/tests/index.test.ts +++ b/kits/firestore-bigquery-export/tests/index.test.ts @@ -125,9 +125,9 @@ describe("exported function options", () => { expect(document.toCEL()).toContain("params.COLLECTION_PATH"); }); - test("the trigger keeps retry enabled so a rethrown enqueue failure is redelivered", async () => { + test("the trigger declares no retry policy, matching the extension", async () => { const { trigger } = await loadExportedOptions(); - expect(trigger.retry).toBe(true); + expect(trigger.retry).toBeUndefined(); }); test("syncBigQuery pins the extension's queue shape", async () => { From 747e6a13f6d8791219872cedadbb8c22aa2428d3 Mon Sep 17 00:00:00 2001 From: Jacob Cable Date: Mon, 7 Sep 2026 11:52:35 +0100 Subject: [PATCH 7/8] docs(firestore-bigquery-export): scope BACKUP_COLLECTION to insert failures Transform-function failures throw before the insert and are never backed up, on the kit and the extension alike. --- kits/firestore-bigquery-export/README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/kits/firestore-bigquery-export/README.md b/kits/firestore-bigquery-export/README.md index cc17477293..fd314e513e 100644 --- a/kits/firestore-bigquery-export/README.md +++ b/kits/firestore-bigquery-export/README.md @@ -281,6 +281,10 @@ for the full recipe. - Rows that exhaust the queue with no `BACKUP_COLLECTION` configured are gone. This matches the extension; it is the reason the backup collection is strongly recommended. +- `BACKUP_COLLECTION` captures rows whose BigQuery insert fails. A failure + earlier in the tracker, such as a `TRANSFORM_FUNCTION` endpoint that is down + or returns malformed JSON, throws before the insert and is not backed up. + Same as the extension. ## Differences from the Stream Firestore to BigQuery extension From 615460bc0d94bb8eabce73742072e36982c4303a Mon Sep 17 00:00:00 2001 From: Jacob Cable Date: Mon, 7 Sep 2026 12:15:42 +0100 Subject: [PATCH 8/8] fix(firestore-bigquery-export): match the extension on the queue handler's hot path Drop provisioning from handleSyncBigQueryTask. The extension's queue handler goes straight to the write with skipInit on the tracker; the kit ran tracker.initialize() before every buffered write on a cold instance, which under a recovery burst fanned six to eight BigQuery metadata calls, and with TABLE_PARTITIONING set on an unpartitioned table a table.setMetadata, across up to 500 concurrent instances. Provisioning stays in the lifecycle tasks. Give syncBigQuery maxInstances equal to its maxConcurrentDispatches. The extension's handler is gen1 with no instance cap; gen2 defaults to 100, so dispatches beyond that would 429 at the cap and consume queue attempts. --- kits/firestore-bigquery-export/README.md | 15 ++++---- .../firestore-bigquery-export/src/handlers.ts | 34 +++++-------------- kits/firestore-bigquery-export/src/index.ts | 3 ++ kits/firestore-bigquery-export/src/init.ts | 2 +- .../tests/handlers.test.ts | 32 ++--------------- .../tests/index.test.ts | 2 ++ 6 files changed, 25 insertions(+), 63 deletions(-) diff --git a/kits/firestore-bigquery-export/README.md b/kits/firestore-bigquery-export/README.md index fd314e513e..ef00e9b6c0 100644 --- a/kits/firestore-bigquery-export/README.md +++ b/kits/firestore-bigquery-export/README.md @@ -230,10 +230,10 @@ curl -fsS -X POST -H "Content-Type: application/json" -d '{"data":{}}' \ The Firestore write path never provisions on the hot path. If resources are missing when a write arrives, the inline write fails and the change buffers -through the `syncBigQuery` queue, whose handler calls `ensureInitialized()` as -a self-heal before re-attempting the write. Provisioning is memoized once it -succeeds, so the self-heal covers resources that were never created, not -resources deleted out from under a warm instance. +through the `syncBigQuery` queue, which re-attempts the write on Cloud Tasks' +schedule. The queue handler does not provision, as in the extension: if the +resources are still missing the retries fail and the row lands in +`BACKUP_COLLECTION`; run the lifecycle task (redeploy) to recreate them. ## Failure handling @@ -249,12 +249,11 @@ The write path mirrors the extension's Cloud Tasks buffer: event is not redelivered. 3. `syncBigQuery` re-attempts the write on the queue's schedule: 5 attempts, 60 seconds minimum backoff, throttled to `MAX_DISPATCHES_PER_SECOND` - dispatches per second (500 concurrent max). + dispatches per second (500 concurrent max, and the function allows 500 + instances so that ceiling is reachable; gen2 would otherwise cap at 100). 4. On every terminal insert failure the tracker writes the row to `BACKUP_COLLECTION` (when configured), keyed by the event id, before the - task fails. A failed provisioning attempt is logged and the write is tried - anyway, so it still reaches that path. After the fifth attempt the task is - dropped. **Without a backup collection, the row is dropped with it** - + task fails. After the fifth attempt the task is dropped. **Without a backup collection, the row is dropped with it** - configure `BACKUP_COLLECTION`. 5. If the enqueue itself fails (BigQuery AND Cloud Tasks both failing), the trigger logs at error level, publishes an `onError` event, and the diff --git a/kits/firestore-bigquery-export/src/handlers.ts b/kits/firestore-bigquery-export/src/handlers.ts index 199b52db74..bff815d659 100644 --- a/kits/firestore-bigquery-export/src/handlers.ts +++ b/kits/firestore-bigquery-export/src/handlers.ts @@ -60,10 +60,8 @@ export interface HandlerContext { tracker: FirestoreBigQueryEventHistoryTracker; config: ResolvedExportConfig; /** - * Provisions the BigQuery dataset/table/views once per instance. Called by - * the `syncBigQuery` task as a self-heal before re-attempting a write; the - * hot path relies on out-of-band provisioning - * (`initBigQuerySync` / `setupBigQuerySync`). + * Provisions the BigQuery resources. Used by the lifecycle tasks only; the + * write paths never call it. */ ensureInitialized: () => Promise; /** @@ -145,7 +143,7 @@ export async function handleDocumentWrite( // No provisioning on the hot path: BigQuery resources are provisioned // out-of-band (afterFirstDeploy / afterRedeploy tasks). If they are missing, // the inline write fails and the change buffers through the syncBigQuery - // queue, whose handler self-heals before re-attempting. + // queue, whose handler re-attempts the write on Cloud Tasks' schedule. const { config, tracker } = ctx; const changeType = getChangeType(data); const documentId = getDocumentId(data); @@ -221,13 +219,12 @@ export async function handleDocumentWrite( } /** - * Handles a `syncBigQuery` task: re-attempts a buffered write. Provisioning - * runs first as a self-heal (memoized, a no-op after the first success), so a - * write that failed only because the BigQuery resources were missing succeeds - * on the first task attempt. A failed provision is logged and the write is - * attempted anyway, so the tracker still parks the row in the backup - * collection. A failed write rethrows so Cloud Tasks retries on the queue's - * schedule. + * Handles a `syncBigQuery` task: re-attempts a buffered write. No provisioning + * runs here, as in the extension: that stays in the lifecycle tasks, so a + * recovery burst does not fan `initialize()` out across every cold instance. + * A failed write rethrows so Cloud Tasks retries on the queue's schedule; the + * tracker parks the row in the backup collection before each terminal + * rethrow. * * @param req - The dispatched task request carrying the serialized change. * @param ctx - The handler context. @@ -246,19 +243,6 @@ export async function handleSyncBigQueryTask( ); try { - try { - await ctx.ensureInitialized(); - } catch (initErr) { - // Fall through to the write regardless: the tracker only parks a row in - // BACKUP_COLLECTION from its insert failure path, so throwing here would - // drop the row instead of backing it up. - logs.error( - false, - "Failed to provision BigQuery resources before a buffered write", - initErr as Error - ); - } - await recordEventToBigQuery(change, ctx.tracker); } catch (err) { logs.logFailedEventAction( diff --git a/kits/firestore-bigquery-export/src/index.ts b/kits/firestore-bigquery-export/src/index.ts index 99338a80a9..3fc5421a39 100644 --- a/kits/firestore-bigquery-export/src/index.ts +++ b/kits/firestore-bigquery-export/src/index.ts @@ -171,6 +171,9 @@ export const syncBigQuery = onTaskDispatched( { ...(functionRegion ? { region: functionRegion } : {}), retryConfig: SYNC_RETRY_CONFIG, + // The extension's queue handler is gen1 with no instance cap; gen2 defaults + // to 100, which would 429 dispatches beyond it and burn queue attempts. + maxInstances: SYNC_MAX_CONCURRENT_DISPATCHES, rateLimits: { maxConcurrentDispatches: SYNC_MAX_CONCURRENT_DISPATCHES, maxDispatchesPerSecond: CONFIG_EXPRESSIONS.maxDispatchesPerSecond, diff --git a/kits/firestore-bigquery-export/src/init.ts b/kits/firestore-bigquery-export/src/init.ts index 530b2b3543..06c57f1b66 100644 --- a/kits/firestore-bigquery-export/src/init.ts +++ b/kits/firestore-bigquery-export/src/init.ts @@ -18,7 +18,7 @@ import type { FirestoreBigQueryEventHistoryTracker } from "@firebaseextensions/f /** * Builds the provisioning guard used by the `initBigQuerySync` endpoint and the - * retry-path self-heal. The hot write path never calls it. + * lifecycle tasks. The write paths never call it. * * The returned function runs `tracker.initialize()` at most once per instance: * concurrent invocations on a cold instance share a single in-flight promise. A diff --git a/kits/firestore-bigquery-export/tests/handlers.test.ts b/kits/firestore-bigquery-export/tests/handlers.test.ts index 37d880fdfe..8e88f0fcc9 100644 --- a/kits/firestore-bigquery-export/tests/handlers.test.ts +++ b/kits/firestore-bigquery-export/tests/handlers.test.ts @@ -296,13 +296,14 @@ describe("handleDocumentWrite", () => { describe("handleSyncBigQueryTask", () => { beforeEach(() => vi.clearAllMocks()); - test("self-heals, records the buffered change, and emits a success event", async () => { + test("records the buffered change and emits a success event", async () => { const ctx = makeCtx(); const change = serializedChange(); await handleSyncBigQueryTask(taskRequest(change), ctx); - expect(ctx.ensureInitialized).toHaveBeenCalledTimes(1); + // Extension parity: no provisioning on the write path. + expect(ctx.ensureInitialized).not.toHaveBeenCalled(); const [[recorded]] = (ctx.tracker.record as ReturnType).mock .calls; expect(recorded[0]).toMatchObject({ @@ -332,33 +333,6 @@ describe("handleSyncBigQueryTask", () => { expect(ctx.enqueue).not.toHaveBeenCalled(); }); - test("attempts the write even when the self-heal fails", async () => { - // The tracker only parks a row in BACKUP_COLLECTION from its insert - // failure path, so skipping the write would drop the row instead. - const ctx = makeCtx(); - (ctx.ensureInitialized as ReturnType).mockRejectedValueOnce( - new Error("no dataset") - ); - - await handleSyncBigQueryTask(taskRequest(serializedChange()), ctx); - - expect(ctx.tracker.record).toHaveBeenCalledTimes(1); - }); - - test("surfaces the write error when the self-heal also failed", async () => { - const ctx = makeCtx(); - (ctx.ensureInitialized as ReturnType).mockRejectedValueOnce( - new Error("no dataset") - ); - (ctx.tracker.record as ReturnType).mockRejectedValueOnce( - new Error("no table") - ); - - await expect( - handleSyncBigQueryTask(taskRequest(serializedChange()), ctx) - ).rejects.toThrow("no table"); - }); - test("does not rethrow when the success event fails after the row lands", async () => { // The row is already in BigQuery; a Cloud Tasks retry would land past the // insertId dedupe window and duplicate it. diff --git a/kits/firestore-bigquery-export/tests/index.test.ts b/kits/firestore-bigquery-export/tests/index.test.ts index 967aa7bc43..dbaaa57b8c 100644 --- a/kits/firestore-bigquery-export/tests/index.test.ts +++ b/kits/firestore-bigquery-export/tests/index.test.ts @@ -141,6 +141,8 @@ describe("exported function options", () => { const rateLimits = syncTask.rateLimits as Record; expect(rateLimits.maxConcurrentDispatches).toBe(500); + // gen2 defaults to 100 instances; the queue must be able to use its ceiling. + expect(syncTask.maxInstances).toBe(500); expect(String(rateLimits.maxDispatchesPerSecond)).toBe( "params.MAX_DISPATCHES_PER_SECOND" );