diff --git a/.github/workflows/db-saturation-probe.yml b/.github/workflows/db-saturation-probe.yml new file mode 100644 index 00000000..48f0cf0a --- /dev/null +++ b/.github/workflows/db-saturation-probe.yml @@ -0,0 +1,84 @@ +# TC-1601: sustained DB saturation alert for PlanetScale databases. +# +# WHAT: every 5 minutes, read CPU and connection-pool metrics for each target in +# test/probe/db-saturation-targets.json and send one Better Stack heartbeat +# decision per target. Breach -> POST the heartbeat's /fail URL (pages team@). +# Healthy -> GET the heartbeat (ping). Unknown or transient API error -> no call, +# so the missed-heartbeat alert covers it. +# +# A target is skipped with a notice until its two secrets are set, so this +# workflow stays green while unconfigured. +# +# SECRETS (Settings -> Secrets and variables -> Actions): +# PLANETSCALE_METRICS_TOKEN_NODE_DB PlanetScale service token ID:TOKEN, +# read_branch on all branches of tinycloud-node-production. +# BETTERSTACK_HEARTBEAT_URL_NODE_DB "Node DB saturation" heartbeat URL. +# PLANETSCALE_METRICS_TOKEN_OPENKEY_DB Added after the 2026-10-22 DB move. +# BETTERSTACK_HEARTBEAT_URL_OPENKEY_DB "OpenKey DB saturation" heartbeat URL. +# +# PROOF RUNS: gh workflow run db-saturation-probe.yml -f threshold_pct=1 +# expect /fail (pages team@). Then with no inputs: expect a ping. +# Dispatch only from the default branch, and warn on-call first. +# +# Adding a target: add its entry to db-saturation-targets.json and set its two +# secrets. Secrets must also appear in the env block below, because Actions only +# passes secrets that are named here. + +name: DB Saturation Probe +run-name: >- + ${{ github.event_name == 'workflow_dispatch' && inputs.threshold_pct != '' && format('DB saturation probe (manual CPU threshold {0}%)', inputs.threshold_pct) || 'DB saturation probe' }} + +on: + schedule: + - cron: "*/5 * * * *" + workflow_dispatch: + inputs: + threshold_pct: + description: "Proof runs only: manual CPU threshold (0-100). Empty uses the targets file (85)." + required: false + default: "" + type: string + target: + description: "Target label from db-saturation-targets.json. Empty runs every target." + required: false + default: "" + type: string + +permissions: + contents: read + +concurrency: + group: db-saturation-probe + cancel-in-progress: false + +jobs: + check: + runs-on: ubuntu-latest + timeout-minutes: 3 + defaults: + run: + working-directory: test/probe + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: "20" + cache: npm + cache-dependency-path: test/probe/package-lock.json + + - name: Install probe dependencies + run: npm ci --no-audit --no-fund + + - name: Type-check probe + run: npm run typecheck + + - name: Run saturation check + env: + DB_TARGET: ${{ inputs.target }} + THRESHOLD_PCT: ${{ inputs.threshold_pct }} + PLANETSCALE_METRICS_TOKEN_NODE_DB: ${{ secrets.PLANETSCALE_METRICS_TOKEN_NODE_DB }} + BETTERSTACK_HEARTBEAT_URL_NODE_DB: ${{ secrets.BETTERSTACK_HEARTBEAT_URL_NODE_DB }} + PLANETSCALE_METRICS_TOKEN_OPENKEY_DB: ${{ secrets.PLANETSCALE_METRICS_TOKEN_OPENKEY_DB }} + BETTERSTACK_HEARTBEAT_URL_OPENKEY_DB: ${{ secrets.BETTERSTACK_HEARTBEAT_URL_OPENKEY_DB }} + run: npm run probe:db diff --git a/test/probe/README.md b/test/probe/README.md index b17d483e..a6d77257 100644 --- a/test/probe/README.md +++ b/test/probe/README.md @@ -198,3 +198,30 @@ PROBE_PRIVATE_KEY=0x... PROBE_MODE=dense npm run probe # promote watch (~66 min) ``` Without `BETTERSTACK_HEARTBEAT_URL` nothing is reported to Better Stack. + +## DB saturation probe (TC-1601) + +`db-saturation-run.ts` checks each target in `db-saturation-targets.json` once +per run (`db-saturation-probe.yml`, every 5 min). It finds the database's production +branch from the PlanetScale branch list each run, reads that branch's metrics, and +sends one Better Stack heartbeat decision per target. + +- **CPU:** breach when at least 13 of the last 15 one-minute points of + `planetscale_primary_pods_cpu_util_percentages` are above `cpu_pct` (85). + Points newer than 90 s are ignored for ingestion lag. +- **Pool:** same counting on the target's `pool` rule. The shipped target pins + direct Postgres, port 5432: active connections >= 15 + (`TINYCLOUD_DATABASE__MAX_CONNECTIONS`). `pool: null` is a config fault. A + PgBouncer port (6432) would use `planetscale_pgbouncer_pools_client` waiting clients > 0. +- **Outcomes:** breach or config fault -> `/fail` with numeric counts. Healthy + -> ping. Unknown (fewer than max(`min_points`, `min_breach_points`) valid points, + no series, or not exactly one production branch) or transient API error + (429, 5xx, timeout) -> no call, so the missed-heartbeat alert fires. +- **Secrets:** each target reads the env names in its `token_secret` and + `heartbeat_secret`. A target whose secrets are unset is skipped. The token is + a PlanetScale service token (`ID:TOKEN`) with `read_branch` on all branches of that database. + +Proof runs (default branch only): `gh workflow run db-saturation-probe.yml -f threshold_pct=1` +should send `/fail`; a run with no inputs should send a ping. + +Local: `npm run probe:db` (skips when secrets are unset). Tests: `npm test`. diff --git a/test/probe/db-saturation-run.ts b/test/probe/db-saturation-run.ts new file mode 100644 index 00000000..23272e33 --- /dev/null +++ b/test/probe/db-saturation-run.ts @@ -0,0 +1,117 @@ +/** + * TC-1601 runner: one heartbeat decision per target in db-saturation-targets.json. + * + * tsx db-saturation-run.ts + * + * Env: + * DB_TARGET optional target label; default runs every target + * THRESHOLD_PCT optional manual CPU threshold (0-100). Proof runs only; the + * label in the log and the /fail body says so. + * PlanetScale token, ID:TOKEN, read_branch on all branches of that database + * Better Stack heartbeat URL for that target + * + * Heartbeat: ok -> ping; breach or config fault -> /fail; unknown or transient + * API error -> nothing (the missed-heartbeat alert handles it). A target whose + * token or heartbeat env is unset is skipped, like write-probe. Logs never + * contain the token, the heartbeat URL, or query text. + */ +import { readFileSync } from "node:fs"; +import { classify, detail, heartbeatUrl, redact, report, type Delivery, type Fetch } from "./probe-core"; +import { ConfigFault, branchesUrl, evaluateTarget, loadTargets, metricsUrl, parseSeries, productionBranch, type Target, type Verdict } from "./db-saturation"; + +const TIMEOUT_MS = 10_000; +const TARGETS_FILE = new URL("./db-saturation-targets.json", import.meta.url); + +/** GET a PlanetScale endpoint. 429 and 5xx are transient; any other non-2xx is a token, scope or path fault. */ +async function getJson(url: string, token: string, fetchFn: Fetch): Promise { + const res = await fetchFn(url, { + headers: { authorization: token, accept: "application/json" }, + signal: AbortSignal.timeout(TIMEOUT_MS), + }); + // 429 and 5xx are transient: no heartbeat call, the missed-heartbeat alert covers it. + if (res.status === 429 || res.status >= 500) throw new Error(`planetscale HTTP ${res.status}`); + // Any other non-2xx is a token, scope or path fault and must page now. + if (!res.ok) throw new ConfigFault(`planetscale HTTP ${res.status}`); + return res.json(); +} + +function parseOverride(raw: string | undefined): number | undefined { + if (raw === undefined || raw.trim() === "") return undefined; + const n = Number(raw); + if (!Number.isFinite(n) || n < 0 || n > 100) throw new ConfigFault(`THRESHOLD_PCT must be 0-100, got ${raw}`); + return n; +} + +function failBody(label: string, verdict: Verdict, now: Date): string { + const at = now.toISOString().replace(/\.\d{3}Z$/, "Z"); + return `TC-1601 db saturation ${verdict.status}: target=${label} ${verdict.detail} at=${at}`; +} + +let targetSecretNames = new Set(); + +async function main(): Promise { + const only = process.env.DB_TARGET?.trim() || undefined; + const override = parseOverride(process.env.THRESHOLD_PCT); + const targets = loadTargets(JSON.parse(readFileSync(TARGETS_FILE, "utf8"))); + targetSecretNames = new Set(targets.flatMap((t) => [t.token_secret, t.heartbeat_secret])); + const selected = only ? targets.filter((t) => t.label === only) : targets; + if (selected.length === 0) throw new ConfigFault(`no target named ${only}`); + + let deliveryFailed = false; + for (const target of selected) { + const token = process.env[target.token_secret]?.trim(); + const heartbeatRaw = process.env[target.heartbeat_secret]?.trim(); + if (!token || !heartbeatRaw) { + const missing = !token ? target.token_secret : target.heartbeat_secret; + console.log(`${target.label}: skipped (${missing} not set)`); + continue; + } + let heartbeat: URL | undefined; + try { + heartbeat = heartbeatUrl(heartbeatRaw); + } catch { + console.log(`${target.label}: config fault, ${target.heartbeat_secret} is not a Better Stack heartbeat URL`); + deliveryFailed = true; + continue; + } + const t = override === undefined ? target : { ...target, cpu_pct: override }; + const label = override === undefined ? target.label : `${target.label} (manual cpu threshold ${override}%)`; + const now = new Date(); + + let verdict: Verdict; + try { + const branch = productionBranch(await getJson(branchesUrl(t), token, fetch)); + verdict = branch === null + ? { status: "unknown", detail: "no single production branch in the branch list" } + : evaluateTarget(t, parseSeries(await getJson(metricsUrl(t, branch), token, fetch)), Math.floor(now.getTime() / 1000)); + } catch (error) { + if (!(error instanceof ConfigFault)) { + console.log(`${label}: check skipped, no heartbeat: ${redact(classify(error), [token])}`); + continue; + } + verdict = { status: "config", detail: error.message }; + } + + console.log(`${label}: ${verdict.status} ${redact(verdict.detail, [token], 600)}`); + if (verdict.status === "unknown") continue; + const action = verdict.status === "ok" ? "ping" : "fail"; + const body = action === "fail" ? redact(failBody(label, verdict, now), [token], 400) : undefined; + const delivery: Delivery = await report(action, heartbeat, body, fetch); + console.log(delivery.line); + if (!delivery.ok) deliveryFailed = true; + } + return deliveryFailed ? 1 : 0; +} + +main().then( + (code) => { + process.exitCode = code; + }, + (err: unknown) => { + const secrets = Object.entries(process.env) + .filter(([k, v]) => v && (targetSecretNames.has(k) || /^(PLANETSCALE_METRICS_TOKEN|BETTERSTACK_HEARTBEAT_URL)/.test(k))) + .map(([, v]) => v as string); + console.error(`[db-saturation] crashed: ${detail(err, secrets)}`); + process.exitCode = 1; + }, +); diff --git a/test/probe/db-saturation-targets.json b/test/probe/db-saturation-targets.json new file mode 100644 index 00000000..6289057c --- /dev/null +++ b/test/probe/db-saturation-targets.json @@ -0,0 +1,19 @@ +[ + { + "label": "node-db", + "org": "tinycloudlabs", + "database": "tinycloud-node-production", + "token_secret": "PLANETSCALE_METRICS_TOKEN_NODE_DB", + "heartbeat_secret": "BETTERSTACK_HEARTBEAT_URL_NODE_DB", + "cpu_pct": 85, + "window_min": 15, + "min_points": 13, + "min_breach_points": 13, + "pool": { + "metric": "planetscale_primary_postgres_connection_state", + "series_labels": { "state": "active" }, + "op": "gte", + "threshold": 15 + } + } +] diff --git a/test/probe/db-saturation.test.ts b/test/probe/db-saturation.test.ts new file mode 100644 index 00000000..ffbf9944 --- /dev/null +++ b/test/probe/db-saturation.test.ts @@ -0,0 +1,219 @@ +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { test } from "node:test"; +import { + CPU_METRIC, + ConfigFault, + LAG_SEC, + evaluateTarget, + loadTargets, + branchesUrl, + metricsUrl, + parseSeries, + productionBranch, + type Point, + type Series, + type Target, +} from "./db-saturation"; + +const NOW = 1_800_000_000; +const POOL_METRIC = "planetscale_pgbouncer_pools_client"; + +const base: Target = { + label: "node-db", + org: "tinycloudlabs", + database: "tinycloud-node-production", + token_secret: "PLANETSCALE_METRICS_TOKEN_NODE_DB", + heartbeat_secret: "BETTERSTACK_HEARTBEAT_URL_NODE_DB", + cpu_pct: 85, + window_min: 15, + min_points: 12, + min_breach_points: 13, + pool: { metric: POOL_METRIC, series_labels: {}, op: "gt", threshold: 0 }, +}; + +/** values[0] is the newest counted point (NOW - LAG_SEC); each older value is one minute earlier. */ +function points(values: (number | null)[]): Point[] { + return values.map((v, k): Point => [NOW - LAG_SEC - 60 * k, v]); +} + +function cpu(values: (number | null)[]): Series { + return { metric: CPU_METRIC, labels: {}, points: points(values) }; +} + +function pool(values: number[], labels: Record = {}): Series { + return { metric: POOL_METRIC, labels, points: points(values) }; +} + +const idlePool = pool(Array(15).fill(0)); +const idleCpu = cpu(Array(15).fill(50)); + +test("CPU breaches when 13 of the 15 window points exceed the threshold", () => { + const verdict = evaluateTarget(base, [cpu([...Array(13).fill(90), 10, 10]), idlePool], NOW); + assert.equal(verdict.status, "breach"); +}); + +test("CPU does not breach at 12 of 15 points, so two dips are tolerated", () => { + const verdict = evaluateTarget(base, [cpu([...Array(12).fill(90), 10, 10, 10]), idlePool], NOW); + assert.equal(verdict.status, "ok"); +}); + +test("points inside the lag window are ignored", () => { + const lagSpike: Point = [NOW - 30, 99]; + const series: Series = { metric: CPU_METRIC, labels: {}, points: [lagSpike, ...points([...Array(12).fill(90), 10, 10, 10])] }; + assert.equal(evaluateTarget(base, [series, idlePool], NOW).status, "ok"); +}); + +test("a point older than the window is ignored", () => { + const boundary: Point = [NOW - LAG_SEC - 15 * 60, 99]; + const series: Series = { metric: CPU_METRIC, labels: {}, points: [...points([...Array(12).fill(90), 10, 10, 10]), boundary] }; + assert.equal(evaluateTarget(base, [series, idlePool], NOW).status, "ok"); +}); + +test("fewer than min_points valid points is unknown even when every point breaches", () => { + const verdict = evaluateTarget(base, [cpu(Array(11).fill(99)), idlePool], NOW); + assert.equal(verdict.status, "unknown"); +}); + +test("null samples do not count as valid points", () => { + const verdict = evaluateTarget(base, [cpu([...Array(11).fill(99), null, null, null, null]), idlePool], NOW); + assert.equal(verdict.status, "unknown"); +}); + +test("no CPU series is unknown, not ok", () => { + assert.equal(evaluateTarget(base, [idlePool], NOW).status, "unknown"); +}); + +test("a breach on one rule wins over unknown on the other", () => { + const verdict = evaluateTarget(base, [cpu(Array(13).fill(99).concat(Array(2).fill(10))), pool([1, 1, 1])], NOW); + assert.equal(verdict.status, "breach"); +}); + +test("pool rule breaches when waiting clients persist in 13 of 15 points", () => { + const verdict = evaluateTarget(base, [idleCpu, pool([...Array(13).fill(2), 0, 0])], NOW); + assert.equal(verdict.status, "breach"); +}); + +test("pool rule only reads series whose labels match the rule", () => { + const rule = { ...base, pool: { metric: POOL_METRIC, series_labels: { state: "waiting" }, op: "gt" as const, threshold: 0 } }; + const verdict = evaluateTarget(rule, [idleCpu, pool(Array(15).fill(9), { state: "active" }), pool(Array(15).fill(0), { state: "waiting" })], NOW); + assert.equal(verdict.status, "ok"); +}); + +test("gte compares inclusively and gt does not", () => { + const at = pool(Array(15).fill(15)); + const gte = { ...base, pool: { metric: POOL_METRIC, series_labels: {}, op: "gte" as const, threshold: 15 } }; + const gt = { ...base, pool: { metric: POOL_METRIC, series_labels: {}, op: "gt" as const, threshold: 15 } }; + assert.equal(evaluateTarget(gte, [idleCpu, at], NOW).status, "breach"); + assert.equal(evaluateTarget(gt, [idleCpu, at], NOW).status, "ok"); +}); + +test("a target without a pool rule is a config fault", () => { + assert.equal(evaluateTarget({ ...base, pool: null }, [idleCpu, idlePool], NOW).status, "config"); +}); + +test("a manual CPU threshold of 1% breaches on normal load", () => { + assert.equal(evaluateTarget({ ...base, cpu_pct: 1 }, [idleCpu, idlePool], NOW).status, "breach"); +}); + +test("loadTargets rejects a target missing a required field", () => { + const { label: _label, ...missing } = base; + assert.throws(() => loadTargets([missing]), ConfigFault); +}); + +test("loadTargets rejects a window smaller than the breach count", () => { + assert.throws(() => loadTargets([{ ...base, window_min: 10 }]), ConfigFault); +}); + +test("metrics request asks for the 1h period, a value PlanetScale accepts", () => { + assert.match(metricsUrl(base, "main"), /[?&]period=1h(&|$)/); +}); + +test("12 valid points that all breach are unknown when 13 must breach", () => { + const t = { ...base, min_points: 12, min_breach_points: 13 }; + assert.equal(evaluateTarget(t, [cpu(Array(12).fill(100)), idlePool], NOW).status, "unknown"); +}); + +test("a repeated timestamp counts once, so duplicates cannot fake a breach", () => { + const t = { ...base, min_points: 13, min_breach_points: 13 }; + const dup: Series = { metric: CPU_METRIC, labels: {}, points: Array.from({ length: 13 }, (): Point => [NOW - LAG_SEC - 60, 100]) }; + assert.equal(evaluateTarget(t, [dup, idlePool], NOW).status, "unknown"); +}); + +test("loadTargets rejects zero, fractional, or oversize integers", () => { + assert.throws(() => loadTargets([{ ...base, min_points: 0 }]), ConfigFault); + assert.throws(() => loadTargets([{ ...base, min_breach_points: 2.5 }]), ConfigFault); + assert.throws(() => loadTargets([{ ...base, window_min: 59 }]), ConfigFault); +}); + +test("loadTargets accepts a null pool and rejects duplicate labels", () => { + assert.equal(loadTargets([{ ...base, pool: null }])[0].pool, null); + assert.throws(() => loadTargets([base, base]), ConfigFault); +}); + +test("loadTargets rejects a pool threshold of 0 under gte, or a negative one", () => { + const pool = (op: string, threshold: number) => ({ metric: POOL_METRIC, series_labels: {}, op, threshold }); + assert.throws(() => loadTargets([{ ...base, pool: pool("gte", 0) }]), ConfigFault); + assert.throws(() => loadTargets([{ ...base, pool: pool("gte", -1) }]), ConfigFault); + assert.throws(() => loadTargets([{ ...base, pool: pool("gt", -1) }]), ConfigFault); + assert.equal(loadTargets([{ ...base, pool: pool("gt", 0) }])[0].pool?.threshold, 0); +}); + +test("the shipped targets file passes loadTargets, so a bad target fails CI", () => { + loadTargets(JSON.parse(readFileSync(new URL("./db-saturation-targets.json", import.meta.url), "utf8"))); +}); + +test("metrics requests go only to the branch metrics endpoint", () => { + const url = new URL(metricsUrl(base, "main")); + assert.equal(url.hostname, "api.planetscale.com"); + assert.ok(url.pathname.endsWith("/branches/main/metrics")); + assert.ok(url.searchParams.get("metrics")?.split(",").includes(CPU_METRIC)); +}); + +test("branch list requests go to the database's branches endpoint", () => { + assert.ok(new URL(branchesUrl(base)).pathname.endsWith("/databases/tinycloud-node-production/branches")); +}); + +test("branch list asks for 100 per page so the production branch is not paged out", () => { + assert.equal(new URL(branchesUrl(base)).searchParams.get("per_page"), "100"); +}); + +test("the production branch is the one flagged production, whatever its name", () => { + const branches = [ + { name: "main", production: false }, + { name: "us-west", production: true }, + { name: "dev", production: false }, + ]; + assert.equal(productionBranch({ data: branches }), "us-west"); +}); + +test("no production branch, two, or an unnamed one fails closed to null", () => { + assert.equal(productionBranch({ data: [{ name: "dev", production: false }] }), null); + assert.equal(productionBranch({ data: [{ name: "a", production: true }, { name: "b", production: true }] }), null); + assert.equal(productionBranch({ data: [{ production: true }] }), null); + assert.equal(productionBranch({ data: [] }), null); +}); + +test("the production branch is read from the data field of the list body", () => { + assert.equal(productionBranch({ type: "list", data: [{ name: "main", production: true }, { name: "dev", production: false }] }), "main"); +}); + +test("a body without a data array, or a bare array, fails closed to null", () => { + assert.equal(productionBranch([{ name: "main", production: true }]), null); + assert.equal(productionBranch({ branches: [{ name: "main", production: true }] }), null); + assert.equal(productionBranch(null), null); +}); + +test("parseSeries rejects a body without a series array", () => { + assert.throws(() => parseSeries({ type: "MetricSeries" }), ConfigFault); +}); + +test("parseSeries keeps numeric samples and turns non-numeric ones into null", () => { + const [series] = parseSeries({ + series: [{ metric: CPU_METRIC, labels: {}, points: [[NOW, 42], [NOW - 60, "x"]] }], + }); + assert.deepEqual(series.points, [ + [NOW, 42], + [NOW - 60, null], + ]); +}); diff --git a/test/probe/db-saturation.ts b/test/probe/db-saturation.ts new file mode 100644 index 00000000..9f762e82 --- /dev/null +++ b/test/probe/db-saturation.ts @@ -0,0 +1,237 @@ +/** + * TC-1601: sustained DB saturation check for a PlanetScale database's production branch. + * + * Pure logic only. The PlanetScale fetch and the heartbeat call live in + * db-saturation-run.ts. Logs numeric metric values only, never query text. + * + * RULES + * ----- + * Points are one-minute samples. The window is the `window_min` minutes that + * end LAG_SEC seconds ago, so ingestion lag never counts. + * - CPU: breach when at least `min_breach_points` window points are above + * `cpu_pct` (13 of 15 by default, so one or two dips do not reset it). + * - Pool: the same counting, using the target's `pool` comparator. + * - Fewer than max(`min_points`, `min_breach_points`) valid points in a series, + * or no series at all, is UNKNOWN: the heartbeat gets no ping, so the + * missed-heartbeat alert fires. + * - A breach on any rule wins over UNKNOWN on another rule. + * - A target with no `pool` rule is a CONFIG fault and fails the heartbeat. + */ + +export class ConfigFault extends Error {} + +export const CPU_METRIC = "planetscale_primary_pods_cpu_util_percentages"; +/** Points newer than this are dropped for ingestion lag. */ +export const LAG_SEC = 90; + +export type PoolRule = { + metric: string; + /** Every key must equal the series label; {} matches every series of the metric. */ + series_labels: Record; + op: "gt" | "gte"; + threshold: number; +}; + +export type Target = { + label: string; + org: string; + database: string; + /** Name of the env var holding the PlanetScale token (ID:TOKEN). */ + token_secret: string; + /** Name of the env var holding the Better Stack heartbeat URL. */ + heartbeat_secret: string; + cpu_pct: number; + window_min: number; + min_points: number; + min_breach_points: number; + /** null until the connection path is confirmed; null fails the target as CONFIG. */ + pool: PoolRule | null; +}; + +/** [unix seconds, value or null when PlanetScale has no sample for that minute]. */ +export type Point = [number, number | null]; +export type Series = { metric: string; labels: Record; points: Point[] }; + +export type Status = "ok" | "breach" | "unknown" | "config"; +export type Verdict = { status: Status; detail: string }; + +type Obj = Record; +const isObj = (x: unknown): x is Obj => typeof x === "object" && x !== null && !Array.isArray(x); + +function str(o: Obj, key: string, at: string): string { + const v = o[key]; + if (typeof v !== "string" || v.trim() === "") throw new ConfigFault(`${at}: ${key} must be a non-empty string`); + return v; +} + +function num(o: Obj, key: string, at: string): number { + const v = o[key]; + if (typeof v !== "number" || !Number.isFinite(v)) throw new ConfigFault(`${at}: ${key} must be a finite number`); + return v; +} + +function parsePool(v: unknown, at: string): PoolRule { + if (!isObj(v)) throw new ConfigFault(`${at}: pool must be an object or null`); + const labels = v.series_labels; + if (!isObj(labels) || Object.values(labels).some((x) => typeof x !== "string")) { + throw new ConfigFault(`${at}: pool.series_labels must map strings to strings`); + } + if (v.op !== "gt" && v.op !== "gte") throw new ConfigFault(`${at}: pool.op must be "gt" or "gte"`); + const threshold = num(v, "threshold", `${at}.pool`); + if (threshold < 0 || (v.op === "gte" && threshold === 0)) { + throw new ConfigFault(`${at}: pool.threshold must be >= 0, and > 0 under gte (0 breaches an idle database)`); + } + return { + metric: str(v, "metric", `${at}.pool`), + series_labels: labels as Record, + op: v.op, + threshold, + }; +} + +/** Validates the targets file. Throws ConfigFault on any malformed entry. */ +export function loadTargets(raw: unknown): Target[] { + if (!Array.isArray(raw) || raw.length === 0) throw new ConfigFault("targets file must be a non-empty array"); + const labels = new Set(); + return raw.map((entry, i) => { + const at = `target ${i}`; + if (!isObj(entry)) throw new ConfigFault(`${at} must be an object`); + const label = str(entry, "label", at); + if (labels.has(label)) throw new ConfigFault(`${at}: duplicate label ${label}`); + labels.add(label); + const target: Target = { + label, + org: str(entry, "org", at), + database: str(entry, "database", at), + token_secret: str(entry, "token_secret", at), + heartbeat_secret: str(entry, "heartbeat_secret", at), + cpu_pct: num(entry, "cpu_pct", at), + window_min: num(entry, "window_min", at), + min_points: num(entry, "min_points", at), + min_breach_points: num(entry, "min_breach_points", at), + pool: entry.pool === null ? null : parsePool(entry.pool, at), + }; + if (target.cpu_pct < 0 || target.cpu_pct > 100) throw new ConfigFault(`${at}: cpu_pct out of range`); + if (!Number.isInteger(target.window_min) || target.window_min < 1 || target.window_min * 60 + LAG_SEC > 3600) { + throw new ConfigFault(`${at}: window_min must be an integer from 1 to 58`); + } + for (const key of ["min_points", "min_breach_points"] as const) { + if (!Number.isInteger(target[key]) || target[key] < 1) throw new ConfigFault(`${at}: ${key} must be a positive integer`); + } + if (target.min_breach_points > target.window_min || target.min_points > target.window_min) { + throw new ConfigFault(`${at}: min_points and min_breach_points must not exceed window_min`); + } + return target; + }); +} + +/** Branch list endpoint. Called once per run to find the production branch. */ +export function branchesUrl(t: Target): string { + const enc = encodeURIComponent; + return `https://api.planetscale.com/v1/organizations/${enc(t.org)}/databases/${enc(t.database)}/branches?per_page=100`; +} + +/** The name of the one branch flagged production, or null when there is not exactly one. */ +export function productionBranch(body: unknown): string | null { + if (!isObj(body) || !Array.isArray(body.data)) return null; + const names = body.data.flatMap((b): string[] => (isObj(b) && b.production === true && typeof b.name === "string" ? [b.name] : [])); + return names.length === 1 ? names[0] : null; +} + +/** Metrics endpoint for one branch. Only this endpoint is ever called. */ +export function metricsUrl(t: Target, branch: string): string { + const metrics = [CPU_METRIC, ...(t.pool ? [t.pool.metric] : [])]; + const enc = encodeURIComponent; + // 1h covers window_min plus lag; evaluateRule trims to the window. + return ( + `https://api.planetscale.com/v1/organizations/${enc(t.org)}/databases/${enc(t.database)}` + + `/branches/${enc(branch)}/metrics?metrics=${metrics.join(",")}&period=1h` + ); +} + +/** Parses a PlanetScale MetricSeries body. Shape changes throw ConfigFault. */ +export function parseSeries(body: unknown): Series[] { + if (!isObj(body) || !Array.isArray(body.series)) throw new ConfigFault("metrics response has no series array"); + return body.series.map((s: unknown, i: number): Series => { + const at = `series ${i}`; + if (!isObj(s) || typeof s.metric !== "string" || !isObj(s.labels) || !Array.isArray(s.points)) { + throw new ConfigFault(`${at} is malformed`); + } + const points = s.points.map((p: unknown): Point => { + if (!Array.isArray(p) || p.length !== 2 || typeof p[0] !== "number") throw new ConfigFault(`${at} has a malformed point`); + const value = p[1]; + return [p[0], typeof value === "number" && Number.isFinite(value) ? value : null]; + }); + const labels = Object.fromEntries(Object.entries(s.labels).map(([k, v]) => [k, String(v)])); + return { metric: s.metric, labels, points }; + }); +} + +type RuleResult = { status: "ok" | "breach" | "unknown"; detail: string }; + +function labelText(s: Series): string { + const entries = Object.entries(s.labels); + return entries.length === 0 ? "all" : entries.map(([k, v]) => `${k}=${v}`).join(","); +} + +function evaluateRule(args: { + name: string; + series: Series[]; + above: (v: number) => boolean; + t: Target; + nowSec: number; +}): RuleResult { + const { name, series, above, t, nowSec } = args; + if (series.length === 0) return { status: "unknown", detail: `${name}: no series` }; + const hi = nowSec - LAG_SEC; + const lo = hi - t.window_min * 60; + // Fewer valid points than this can prove neither a breach nor health. + const needed = Math.max(t.min_points, t.min_breach_points); + let breach = false; + let unknown = false; + const parts: string[] = []; + for (const s of series) { + // Duplicate timestamps collapse to the last value; only the newest window_min points count. + const byTime = new Map(); + for (const [ts, v] of s.points) if (ts > lo && ts <= hi && v !== null) byTime.set(ts, v); + const values = [...byTime] + .sort(([a], [b]) => a - b) + .slice(-t.window_min) + .map(([, v]) => v); + const count = values.filter(above).length; + parts.push(`[${labelText(s)}] ${count}/${values.length}`); + if (values.length < needed) unknown = true; + else if (count >= t.min_breach_points) breach = true; + } + const detail = `${name} ${parts.join(" ")} (need ${t.min_breach_points} of ${t.window_min} for breach, ${needed} valid to judge)`; + if (breach) return { status: "breach", detail }; + if (unknown) return { status: "unknown", detail }; + return { status: "ok", detail }; +} + +/** Judges one target from its metric series. Pure: `nowSec` is passed in. */ +export function evaluateTarget(t: Target, series: Series[], nowSec: number): Verdict { + if (!t.pool) return { status: "config", detail: "pool rule not set in targets file" }; + const pool = t.pool; + const cpu = evaluateRule({ + name: `cpu>${t.cpu_pct}%`, + series: series.filter((s) => s.metric === CPU_METRIC), + above: (v) => v > t.cpu_pct, + t, + nowSec, + }); + const comparator = pool.op === "gt" ? ">" : ">="; + const poolRes = evaluateRule({ + name: `pool${comparator}${pool.threshold}`, + series: series.filter( + (s) => s.metric === pool.metric && Object.entries(pool.series_labels).every(([k, v]) => s.labels[k] === v), + ), + above: (v) => (pool.op === "gt" ? v > pool.threshold : v >= pool.threshold), + t, + nowSec, + }); + const detail = `${cpu.detail} | ${poolRes.detail}`; + if (cpu.status === "breach" || poolRes.status === "breach") return { status: "breach", detail }; + if (cpu.status === "unknown" || poolRes.status === "unknown") return { status: "unknown", detail }; + return { status: "ok", detail }; +} diff --git a/test/probe/package.json b/test/probe/package.json index ac5fc051..651577e0 100644 --- a/test/probe/package.json +++ b/test/probe/package.json @@ -7,7 +7,8 @@ "scripts": { "typecheck": "tsc --noEmit", "probe": "tsx write-probe.ts", - "test": "tsx --test probe-core.test.ts" + "probe:db": "tsx db-saturation-run.ts", + "test": "tsx --test probe-core.test.ts db-saturation.test.ts" }, "dependencies": { "@tinycloud/node-sdk": "^2.10.0" diff --git a/test/probe/tsconfig.json b/test/probe/tsconfig.json index 767fe2cf..256138d8 100644 --- a/test/probe/tsconfig.json +++ b/test/probe/tsconfig.json @@ -22,6 +22,9 @@ "write-probe.ts", "probe-core.ts", "probe-core.test.ts", + "db-saturation.ts", + "db-saturation-run.ts", + "db-saturation.test.ts", "personal-node-check.ts" ] }