From 847c1eaee6bc6699341ec50ca33de444cb1a3d27 Mon Sep 17 00:00:00 2001 From: iza <59828082+izadoesdev@users.noreply.github.com> Date: Mon, 5 Oct 2026 02:04:47 +0300 Subject: [PATCH 1/9] feat(db): one command for a reproducible local workspace bun run workspace [--reset] [--anomaly] [--events N] [--website ] creates a local login, website, read:data key and seeded analytics, and refuses non-local data hosts. seed.ts becomes a seeded, deterministic library shared with the dashboard e2e route; db:seed and its silent random-id fallback are removed. An insights integration test proves the anomaly fixture produces a signal. --- .../references/codebase-map.md | 2 +- AGENTS.md | 2 +- CLAUDE.md | 2 +- CONTRIBUTING.md | 6 +- .../app/api/test/e2e/clickhouse/route.ts | 99 +--- apps/insights/package.json | 2 +- .../src/detection.integration.test.ts | 42 ++ bun.lock | 1 - package.json | 2 +- packages/db/package.json | 6 +- packages/db/src/e2e-db-lifecycle.ts | 68 ++- packages/db/src/seed.ts | 548 +++++++----------- packages/test/src/auth.ts | 16 +- packages/test/src/db.ts | 39 +- packages/test/src/setup.ts | 144 ++++- setup.ts | 2 +- 16 files changed, 530 insertions(+), 451 deletions(-) create mode 100644 apps/insights/src/detection.integration.test.ts diff --git a/.agents/skills/databuddy-internal/references/codebase-map.md b/.agents/skills/databuddy-internal/references/codebase-map.md index ac477c021a..a7b9473773 100644 --- a/.agents/skills/databuddy-internal/references/codebase-map.md +++ b/.agents/skills/databuddy-internal/references/codebase-map.md @@ -188,7 +188,7 @@ cd apps/api && bun test cd apps/basket && bun test cd packages/sdk && bun test cd packages/tracker && bun run test:unit -cd packages/db && bun run db:seed +bun run workspace --reset ``` ## Search Patterns diff --git a/AGENTS.md b/AGENTS.md index 26b3672bc8..28181dc73c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -32,7 +32,7 @@ bun run test:watch bun run db:push # Apply schema changes (no migration files) bun run db:migrate # Run migration files bun run db:studio # Open Drizzle Studio GUI -bun run db:seed [EVENT_COUNT] # Seed sample analytics data +bun run workspace [--reset] [--anomaly] [--events N] [--website ] # Local login, website and seeded analytics # SDK (must build before dev if SDK changed) bun run sdk:build diff --git a/CLAUDE.md b/CLAUDE.md index 0b13a5cb5a..68a0c4dc09 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -32,7 +32,7 @@ bun run test:watch bun run db:push # Apply schema changes (no migration files) bun run db:migrate # Run migration files bun run db:studio # Open Drizzle Studio GUI -bun run db:seed [EVENT_COUNT] # Seed sample analytics data +bun run workspace [--reset] [--anomaly] [--events N] [--website ] # Local login, website and seeded analytics # SDK (must build before dev if SDK changed) bun run sdk:build diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4de55f1a7a..18e4bb55e8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -39,8 +39,10 @@ Open [localhost:3000](http://localhost:3000). The dev command builds the SDK and devtools for you. Add provider keys only for the features you're working on; see [optional services](README.md#optional-services). -To explore with sample data, create a website, copy its ID from its settings, -and run `bun run db:seed YOUR_WEBSITE_ID 1000`. +To explore with sample data, run `bun run workspace --reset`. It targets the +local Docker services only and prints a login, a website and an API key. Add +`--anomaly` for a traffic spike and error surge that Insights will flag, or +`--website YOUR_WEBSITE_ID` to seed a website you already made. ## Check your changes diff --git a/apps/dashboard/app/api/test/e2e/clickhouse/route.ts b/apps/dashboard/app/api/test/e2e/clickhouse/route.ts index eb378fb96b..ab2b7ed553 100644 --- a/apps/dashboard/app/api/test/e2e/clickhouse/route.ts +++ b/apps/dashboard/app/api/test/e2e/clickhouse/route.ts @@ -1,12 +1,15 @@ -import { clickHouse, TABLE_NAMES } from "@databuddy/db/clickhouse"; +import { clickHouse } from "@databuddy/db/clickhouse"; +import { + EVENTS_PER_SESSION, + generateAnalytics, + seedAnalytics, +} from "@databuddy/db/seed"; import { readBooleanEnv } from "@databuddy/env/app"; export const runtime = "nodejs"; export const dynamic = "force-dynamic"; const TEST_KEY_HEADER = "x-e2e-test-key"; -const PATHS = ["/", "/pricing", "/docs", "/dashboard", "/settings"]; -const REFERRERS = [null, "https://google.com", "https://github.com"]; interface SeedBody { eventCount?: unknown; @@ -43,14 +46,6 @@ function normalizeEventCount(value: unknown): number { return Math.min(Math.max(Math.floor(parsed), 1), 5000); } -function clickHouseDate(date: Date): string { - return date.toISOString().replace("T", " ").replace("Z", ""); -} - -function pageTitle(path: string): string { - return path === "/" ? "Home" : path.slice(1).replaceAll("-", " "); -} - export async function POST(request: Request): Promise { const denied = assertE2EAccess(request); if (denied) { @@ -62,51 +57,17 @@ export async function POST(request: Request): Promise { return Response.json({ error: "websiteId is required" }, { status: 400 }); } - const eventCount = normalizeEventCount(body.eventCount); - - const now = Date.now(); - const users = Array.from( - { length: Math.max(3, Math.ceil(eventCount / 40)) }, - () => crypto.randomUUID() - ); - const sessions = Array.from( - { length: Math.max(5, Math.ceil(eventCount / 12)) }, - (_, index) => ({ - anonymousId: users[index % users.length], - sessionId: crypto.randomUUID(), - start: now - (eventCount - index) * 60_000, - }) - ); - - const events = Array.from({ length: eventCount }, (_, index) => { - const session = sessions[index % sessions.length]; - const path = PATHS[index % PATHS.length]; - const timestamp = new Date(session.start + index * 30_000); - return { - id: crypto.randomUUID(), - client_id: body.websiteId as string, - event_name: index % 4 === 0 ? "page_exit" : "screen_view", - anonymous_id: session.anonymousId, - time: clickHouseDate(timestamp), - session_id: session.sessionId, - session_start_time: clickHouseDate(new Date(session.start)), - timestamp: clickHouseDate(timestamp), - referrer: REFERRERS[index % REFERRERS.length], - url: `https://e2e.databuddy.local${path}`, - path, - title: pageTitle(path), - ip: "127.0.0.1", - user_agent: "Databuddy E2E", - browser_name: index % 3 === 0 ? "Firefox" : "Chrome", - os_name: index % 2 === 0 ? "macOS" : "Linux", - device_type: index % 5 === 0 ? "mobile" : "desktop", - country: index % 2 === 0 ? "US" : "DE", - properties: "{}", - created_at: clickHouseDate(new Date(now)), - }; + const rows = generateAnalytics({ + clientId: body.websiteId, + dailySessions: Math.ceil( + normalizeEventCount(body.eventCount) / EVENTS_PER_SESSION + ), + days: 1, + domain: "e2e.databuddy.local", }); + await seedAnalytics(clickHouse, rows); - const screenViewEvents = events.filter( + const screenViewEvents = rows.events.filter( (event) => event.event_name === "screen_view" ); const screenViewsByCountry = screenViewEvents.reduce>( @@ -124,35 +85,9 @@ export async function POST(request: Request): Promise { {} ); - const outgoingLinks = sessions - .slice(0, Math.ceil(eventCount / 20)) - .map((session) => ({ - id: crypto.randomUUID(), - client_id: body.websiteId as string, - anonymous_id: session.anonymousId, - session_id: session.sessionId, - href: "https://github.com/databuddy-analytics/Databuddy", - text: "Databuddy GitHub", - properties: "{}", - timestamp: clickHouseDate(new Date(session.start + 45_000)), - })); - - await Promise.all([ - clickHouse.insert({ - format: "JSONEachRow", - table: TABLE_NAMES.events, - values: events, - }), - clickHouse.insert({ - format: "JSONEachRow", - table: TABLE_NAMES.outgoing_links, - values: outgoingLinks, - }), - ]); - return Response.json({ - events: events.length, - outgoingLinks: outgoingLinks.length, + events: rows.events.length, + outgoingLinks: rows.outgoingLinks.length, screenViews: screenViewEvents.length, screenViewsByCountry, screenViewsByPath, diff --git a/apps/insights/package.json b/apps/insights/package.json index bbf279f1d1..7641265ea3 100644 --- a/apps/insights/package.json +++ b/apps/insights/package.json @@ -6,7 +6,7 @@ "scripts": { "dev": "dotenv -e ../../.env -- bun --watch run src/index.ts", "test": "bun test src --path-ignore-patterns='**/*.integration.test.ts'", - "test:integration": "INSIGHTS_INTEGRATION_TESTS=true BULLMQ_REDIS_URL=\"${BULLMQ_REDIS_URL:-redis://localhost:6379/4}\" bun test --isolate src/scheduler.integration.test.ts src/idempotency.integration.test.ts src/selection-billing.integration.test.ts src/investigation-billing.integration.test.ts src/generation-billing.integration.test.ts src/resume-clarification.integration.test.ts src/business-context.integration.test.ts src/retention-retirement.integration.test.ts", + "test:integration": "INSIGHTS_INTEGRATION_TESTS=true BULLMQ_REDIS_URL=\"${BULLMQ_REDIS_URL:-redis://localhost:6379/4}\" bun test --isolate src/scheduler.integration.test.ts src/idempotency.integration.test.ts src/selection-billing.integration.test.ts src/investigation-billing.integration.test.ts src/generation-billing.integration.test.ts src/resume-clarification.integration.test.ts src/business-context.integration.test.ts src/retention-retirement.integration.test.ts src/detection.integration.test.ts", "check-types": "tsc --noEmit" }, "dependencies": { diff --git a/apps/insights/src/detection.integration.test.ts b/apps/insights/src/detection.integration.test.ts new file mode 100644 index 0000000000..e2193dbe4e --- /dev/null +++ b/apps/insights/src/detection.integration.test.ts @@ -0,0 +1,42 @@ +import "@databuddy/test/env"; +import { afterAll, describe, expect, it } from "bun:test"; +import { clickHouse } from "@databuddy/db/clickhouse"; +import { + deleteAnalytics, + generateAnalytics, + seedAnalytics, +} from "@databuddy/db/seed"; +import { detectSignals } from "./detection"; +import { TRAFFIC_METRICS } from "./investigation"; + +const describeIntegration = + process.env.INSIGHTS_INTEGRATION_TESTS === "true" ? describe : describe.skip; + +const websiteId = `detection-fixture-${crypto.randomUUID()}`; + +describeIntegration("detectSignals on the workspace fixture", () => { + afterAll(() => deleteAnalytics(clickHouse, websiteId), 30_000); + + it("flags the seeded anomaly", async () => { + await seedAnalytics( + clickHouse, + generateAnalytics({ + anomaly: true, + clientId: websiteId, + domain: "localhost", + }) + ); + + const signals = await detectSignals({ + lookbackDays: 7, + timezone: "UTC", + websiteId, + }); + + const rising = signals + .filter((signal) => signal.direction === "up") + .map((signal) => signal.metric); + expect(rising.some((metric) => TRAFFIC_METRICS.has(metric))).toBe(true); + expect(rising).toContain("error_count"); + }, 30_000); +}); diff --git a/bun.lock b/bun.lock index af96f68afa..d5e1aacedf 100644 --- a/bun.lock +++ b/bun.lock @@ -436,7 +436,6 @@ }, "devDependencies": { "@types/pg": "^8.16.0", - "tsx": "^4.21.0", }, }, "packages/devtools": { diff --git a/package.json b/package.json index a761b7ca58..dd163f2962 100644 --- a/package.json +++ b/package.json @@ -52,7 +52,7 @@ "db:push": "dotenv -- sh -c 'NODE_ENV=${NODE_ENV:-development} turbo run db:push \"$@\"' --", "db:migrate": "dotenv -- turbo run db:migrate", "db:deploy": "dotenv -- turbo run db:deploy", - "db:seed": "dotenv -- sh -c 'cd packages/db && bun run db:seed \"$@\"' --", + "workspace": "NODE_ENV=development bun --no-env-file packages/test/src/setup.ts", "email:dev": "dotenv -- sh -c 'cd packages/email && bun run dev'", "email:segments": "dotenv -- sh -c 'cd packages/db && bun run email:segments \"$@\"' --", "sdk:build": "turbo run build --filter @databuddy/sdk", diff --git a/packages/db/package.json b/packages/db/package.json index 5add2ed39d..e6305c7ed9 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -10,10 +10,12 @@ "./schema": "./src/drizzle/schema/index.ts", "./schema/relations": "./src/drizzle/schema/relations.ts", "./clickhouse": "./src/clickhouse/index.ts", + "./clickhouse/apply": "./src/clickhouse/apply.ts", "./clickhouse/dql": "./src/clickhouse/dql.ts", "./clickhouse/schema": "./src/clickhouse/schema.ts", "./clickhouse/tables": "./src/clickhouse/schema/tables.generated.ts", "./e2e-db-lifecycle": "./src/e2e-db-lifecycle.ts", + "./seed": "./src/seed.ts", "./test-env": "./src/test-env.ts" }, "scripts": { @@ -23,7 +25,6 @@ "ch:check": "bun run generate && git diff --exit-code src/clickhouse/schema/tables.generated.ts", "db:push": "drizzle-kit push", "db:studio": "drizzle-kit studio", - "db:seed": "tsx src/seed.ts", "email:segments": "bun src/scripts/export-user-email-segments.ts", "clickhouse:init": "bun src/clickhouse/setup.ts", "test": "bun test src", @@ -41,7 +42,6 @@ "pg": "^8.16.3" }, "devDependencies": { - "@types/pg": "^8.16.0", - "tsx": "^4.21.0" + "@types/pg": "^8.16.0" } } diff --git a/packages/db/src/e2e-db-lifecycle.ts b/packages/db/src/e2e-db-lifecycle.ts index aa4a5abfb8..82060dadd2 100644 --- a/packages/db/src/e2e-db-lifecycle.ts +++ b/packages/db/src/e2e-db-lifecycle.ts @@ -1,5 +1,6 @@ -import { isLoopbackHost } from "@databuddy/env/app"; -import { Client } from "pg"; +import { dataUrl, isLocalHost, isLoopbackHost } from "@databuddy/env/app"; +import { drizzle } from "drizzle-orm/node-postgres"; +import { Client, Pool } from "pg"; const DEFAULT_E2E_DB_PREFIX = "databuddy_e2e"; const INVALID_DB_IDENTIFIER_PARTS = /[^A-Za-z0-9_]+/g; @@ -236,6 +237,69 @@ export async function dropLifecycleDatabase( return { dbDsn: config.dbDsn, dbName: config.dbName }; } +type Env = Record; + +function localUrl( + name: "CLICKHOUSE_URL" | "DATABASE_URL" | "REDIS_URL", + env: Env +): string { + const url = dataUrl(name, env); + if (url && URL.canParse(url) && isLocalHost(url)) { + return url; + } + const host = url && URL.canParse(url) ? new URL(url).hostname : "unset"; + throw new Error( + `Refusing to run against a non-local database; ${name} host is "${host}"` + ); +} + +export function assertLocalTargets(env: Env = process.env) { + return { + clickhouseUrl: localUrl("CLICKHOUSE_URL", env), + databaseUrl: localUrl("DATABASE_URL", env), + redisUrl: localUrl("REDIS_URL", env), + }; +} + +export async function resetLocalDatabase(databaseUrl: string): Promise { + const url = normalizeDatabaseUrl(databaseUrl); + const dbName = decodeURIComponent(url.pathname.slice(1)); + if (!(dbName && isLocalHost(url.href))) { + throw new Error( + `Refusing to reset database "${dbName}" on ${url.hostname}` + ); + } + const config = { + adminDsn: deriveAdminDatabaseUrl(url).toString(), + dbDsn: url.toString(), + dbName, + }; + await dropLifecycleDatabase(config); + await createLifecycleDatabase(config); +} + +export async function applyPostgresSchema(databaseUrl: string): Promise { + const [{ pushSchema }, schema] = await Promise.all([ + import("drizzle-kit/api-postgres"), + import("./drizzle/schema"), + ]); + const pool = new Pool({ connectionString: databaseUrl, max: 1 }); + try { + const { apply, hints } = await pushSchema( + schema, + drizzle({ client: pool }) as unknown as Parameters[1] + ); + if (hints.length > 0) { + throw new Error( + `Schema drift needs a decision (${hints.map(({ hint }) => hint).join("; ")}); rerun with --reset` + ); + } + await apply(); + } finally { + await pool.end(); + } +} + export async function runLifecycleCommand( argv: readonly string[] ): Promise { diff --git a/packages/db/src/seed.ts b/packages/db/src/seed.ts index 6f1557e7bd..b0fbd27559 100644 --- a/packages/db/src/seed.ts +++ b/packages/db/src/seed.ts @@ -1,379 +1,259 @@ -import { dataUrl, isLoopbackHost } from "@databuddy/env/app"; -import { faker } from "@faker-js/faker"; -import { clickHouse, TABLE_NAMES } from "./clickhouse/client"; -import { db } from "./client"; +import type { ClickHouseClient } from "@clickhouse/client"; +import { en, Faker } from "@faker-js/faker"; +import { TABLE_NAMES } from "./clickhouse/client"; -for (const name of ["CLICKHOUSE_URL", "DATABASE_URL"] as const) { - const url = dataUrl(name); - const hostname = url && URL.canParse(url) ? new URL(url).hostname : ""; - if (!(url && hostname && isLoopbackHost(url))) { - throw new Error( - `db:seed only runs against local databases; ${name} host is "${hostname || "unset"}"` - ); - } -} +export const EVENTS_PER_SESSION = 6; -const clientId = process.argv[2] || faker.string.uuid(); -const eventCount = Number(process.argv[3]) || 10_000; +const DAY_MS = 86_400_000; +const SESSION_WINDOW_MS = DAY_MS - 3_600_000; +const ANOMALY_TRAFFIC_MULTIPLIER = 3; +const ANOMALY_ERROR_MULTIPLIER = 4; +const ANOMALY_ERROR_DAYS = 7; +const ERRORS_PER_SESSION = 0.05; +const WEEKEND_TRAFFIC = 0.7; const PATHS = [ "/", - "/home", "/pricing", "/features", "/docs", - "/login", - "/signup", - "/dashboard", - "/settings", - "/profile", "/blog", "/about", - "/contact", + "/signup", + "/dashboard", ]; - const REFERRERS = [ - "direct", + null, + null, "https://google.com", - "https://facebook.com", - "https://twitter.com", "https://github.com", + "https://twitter.com", +]; +const COUNTRIES = ["US", "US", "DE", "GB", "FR", "CA", "IN", "BR"]; +const BROWSERS = ["Chrome", "Chrome", "Safari", "Firefox", "Edge"]; +const OPERATING_SYSTEMS = ["macOS", "Windows", "iOS", "Android", "Linux"]; +const DEVICES = ["desktop", "desktop", "mobile", "tablet"]; +const ERROR_TYPES = ["Error", "TypeError", "ReferenceError"]; +const ERROR_MESSAGES = [ + "Cannot read properties of undefined (reading 'id')", + "Failed to fetch", + "Unexpected token in JSON", +]; +const VITALS = [ + { max: 4000, min: 800, name: "LCP" }, + { max: 2500, min: 300, name: "FCP" }, + { max: 400, min: 40, name: "INP" }, + { max: 800, min: 50, name: "TTFB" }, + { max: 0.3, min: 0, name: "CLS" }, ]; -const UNIQUE_USERS = Math.max(10, Math.floor(eventCount / 8)); -const TOTAL_SESSIONS = Math.floor(UNIQUE_USERS * 2.5); - -const USER_POOL = Array.from({ length: UNIQUE_USERS }, () => ({ - anonymousId: `anon_${faker.string.uuid()}`, - country: faker.location.countryCode(), - region: faker.location.state(), - city: faker.location.city(), - timezone: faker.helpers.arrayElement([ - "America/New_York", - "Europe/London", - "Asia/Tokyo", - ]), - language: faker.helpers.arrayElement(["en-US", "en-GB", "fr-FR", "de-DE"]), - deviceType: faker.helpers.arrayElement(["desktop", "mobile", "tablet"]), - browser: faker.helpers.arrayElement(["Chrome", "Firefox", "Safari", "Edge"]), - os: faker.helpers.arrayElement([ - "Windows", - "macOS", - "Linux", - "Android", - "iOS", - ]), - viewportSize: `${faker.number.int({ min: 800, max: 1920 })}x${faker.number.int({ min: 600, max: 1080 })}`, -})); - -const SESSION_POOL = Array.from({ length: TOTAL_SESSIONS }, () => { - const user = faker.helpers.arrayElement(USER_POOL); - return { - sessionId: `sess_${faker.string.uuid()}`, - anonymousId: user.anonymousId, - sessionStartTime: faker.date.recent({ days: 30 }).getTime(), - user, - referrer: faker.helpers.arrayElement(REFERRERS), - }; -}); - -function sessionFor(index: number, total: number) { - const sessionIndex = Math.floor(index / (total / TOTAL_SESSIONS)); - const session = SESSION_POOL[Math.min(sessionIndex, SESSION_POOL.length - 1)]; - if (!session) { - throw new Error("Seed session pool is empty"); - } - return session; +export interface GenerateAnalyticsOptions { + anomaly?: boolean; + clientId: string; + dailySessions?: number; + days?: number; + domain: string; + seed?: number; } -function generatePageTitle(path: string): string { - if (path === "/") { - return "Home"; - } - const name = path.slice(1).replace(/-/g, " "); - return name.charAt(0).toUpperCase() + name.slice(1) || "Page"; +function clickHouseTime(ms: number): string { + return new Date(ms).toISOString().replace("T", " ").replace("Z", ""); } -(async () => { - const website = await db.query.websites.findFirst({ - where: { id: clientId }, - columns: { domain: true }, - }); - - const domain = website?.domain || "example.com"; - - const events = Array.from({ length: eventCount }, (_, index) => { - const session = sessionFor(index, eventCount); - const user = session.user; - - const maxSessionDuration = 2 * 60 * 60 * 1000; - const sessionProgress = - (index % Math.ceil(eventCount / TOTAL_SESSIONS)) / - Math.ceil(eventCount / TOTAL_SESSIONS); - const baseTime = - session.sessionStartTime + sessionProgress * maxSessionDuration; - - const path = faker.helpers.arrayElement(PATHS); - const isLastEvent = - sessionProgress > 0.8 || faker.datatype.boolean({ probability: 0.2 }); - const eventName = - isLastEvent && faker.datatype.boolean({ probability: 0.8 }) - ? "page_exit" - : "screen_view"; - const isPageExit = eventName === "page_exit"; - const fullUrl = `https://${domain}${path}`; +function pageTitle(path: string): string { + return path === "/" ? "Home" : path.slice(1); +} +export function generateAnalytics({ + anomaly = false, + clientId, + dailySessions = 150, + days = 28, + domain, + seed = 42, +}: GenerateAnalyticsOptions) { + const faker = new Faker({ locale: [en], seed }); + const now = Date.now(); + const todayStart = Math.floor(now / DAY_MS) * DAY_MS; + const visitors = Array.from({ length: dailySessions * 4 }, () => ({ + anonymousId: `anon_${faker.string.uuid()}`, + browser: faker.helpers.arrayElement(BROWSERS), + country: faker.helpers.arrayElement(COUNTRIES), + device: faker.helpers.arrayElement(DEVICES), + os: faker.helpers.arrayElement(OPERATING_SYSTEMS), + })); + + const dailyTraffic = Array.from({ length: days }, (_, index) => { + const daysAgo = days - index; + const dayStart = todayStart - daysAgo * DAY_MS; + const weekday = new Date(dayStart).getUTCDay(); + const traffic = + (weekday === 0 || weekday === 6 ? WEEKEND_TRAFFIC : 1) * + faker.number.float({ max: 1.1, min: 0.9 }) * + (anomaly && daysAgo === 1 ? ANOMALY_TRAFFIC_MULTIPLIER : 1); + const sessions = Array.from( + { length: Math.round(dailySessions * traffic) }, + () => ({ + pages: Array.from( + { length: faker.number.int({ max: 5, min: 1 }) }, + () => ({ + path: faker.helpers.arrayElement(PATHS), + seconds: faker.number.int({ max: 180, min: 3 }), + }) + ), + referrer: faker.helpers.arrayElement(REFERRERS), + sessionId: `sess_${faker.string.uuid()}`, + start: dayStart + faker.number.int({ max: SESSION_WINDOW_MS, min: 0 }), + visitor: faker.helpers.arrayElement(visitors), + }) + ); + const errorMultiplier = + anomaly && daysAgo <= ANOMALY_ERROR_DAYS ? ANOMALY_ERROR_MULTIPLIER : 1; return { - id: faker.string.uuid(), - client_id: clientId, - event_name: eventName, - anonymous_id: session.anonymousId, - time: baseTime, - session_id: session.sessionId, - event_type: "track", - event_id: faker.string.uuid(), - session_start_time: session.sessionStartTime, - timestamp: baseTime, - referrer: session.referrer === "direct" ? undefined : session.referrer, - url: fullUrl, - path: fullUrl, - title: generatePageTitle(path), - ip: "", - user_agent: "", - browser_name: user.browser, - browser_version: faker.system.semver(), - os_name: user.os, - os_version: faker.system.semver(), - device_type: user.deviceType, - device_brand: - user.deviceType === "mobile" - ? faker.helpers.arrayElement(["Apple", "Samsung", "Google"]) - : undefined, - device_model: - user.deviceType === "mobile" ? faker.phone.imei() : undefined, - country: user.country, - region: user.region, - city: user.city, - screen_resolution: undefined, - viewport_size: user.viewportSize, - language: user.language, - timezone: user.timezone, - connection_type: undefined, - rtt: undefined, - downlink: undefined, - time_on_page: isPageExit - ? Math.round( - faker.number.float({ min: 5, max: 600, fractionDigits: 1 }) - ) - : undefined, - scroll_depth: isPageExit - ? faker.number.float({ min: 10, max: 100, fractionDigits: 1 }) - : undefined, - interaction_count: isPageExit - ? faker.number.int({ min: 0, max: 50 }) - : undefined, - page_count: - eventName === "screen_view" || isPageExit - ? faker.number.int({ min: 1, max: 10 }) - : 1, - utm_source: faker.helpers.maybe( - () => faker.helpers.arrayElement(["google", "facebook", "twitter"]), - { probability: 0.3 } - ), - utm_medium: faker.helpers.maybe( - () => faker.helpers.arrayElement(["cpc", "organic", "social"]), - { probability: 0.3 } + errorCount: Math.round( + sessions.length * ERRORS_PER_SESSION * errorMultiplier ), - utm_campaign: faker.helpers.maybe(() => faker.lorem.slug(), { - probability: 0.2, - }), - utm_term: faker.helpers.maybe(() => faker.lorem.word(), { - probability: 0.15, - }), - utm_content: faker.helpers.maybe( - () => faker.lorem.words({ min: 1, max: 3 }), - { probability: 0.15 } - ), - load_time: undefined, - dom_ready_time: undefined, - dom_interactive: undefined, - ttfb: undefined, - connection_time: undefined, - request_time: undefined, - render_time: undefined, - redirect_time: undefined, - domain_lookup_time: undefined, - properties: "{}", - created_at: Date.now(), + sessions, }; }); - - events.sort((a, b) => a.time - b.time); - - const outgoingLinks = Array.from( - { length: Math.floor(eventCount / 10) }, - (_, index) => { - const session = sessionFor(index, Math.floor(eventCount / 10)); - - const maxSessionDuration = 2 * 60 * 60 * 1000; - const sessionProgress = - (index % Math.ceil(Math.floor(eventCount / 10) / TOTAL_SESSIONS)) / - Math.ceil(Math.floor(eventCount / 10) / TOTAL_SESSIONS); - const timestamp = - session.sessionStartTime + sessionProgress * maxSessionDuration; - - return { - id: faker.string.uuid(), + const sessions = dailyTraffic.flatMap((day) => day.sessions); + + const events = sessions.flatMap((session) => { + let time = session.start; + return session.pages.flatMap((page, index) => { + const viewedAt = time; + time += (page.seconds + 1) * 1000; + const shared = { + anonymous_id: session.visitor.anonymousId, + browser_name: session.visitor.browser, client_id: clientId, - anonymous_id: session.anonymousId, - session_id: session.sessionId, - href: faker.internet.url(), - text: faker.helpers.maybe(() => faker.lorem.words({ min: 1, max: 4 }), { - probability: 0.7, - }), + country: session.visitor.country, + created_at: clickHouseTime(now), + device_type: session.visitor.device, + ip: "", + os_name: session.visitor.os, + path: page.path, properties: "{}", - timestamp, - }; - } - ); - - outgoingLinks.sort((a, b) => a.timestamp - b.timestamp); - - const errors = Array.from( - { length: Math.floor(eventCount / 20) }, - (_, index) => { - const session = sessionFor(index, Math.floor(eventCount / 20)); - - const maxSessionDuration = 2 * 60 * 60 * 1000; - const sessionProgress = - (index % Math.ceil(Math.floor(eventCount / 20) / TOTAL_SESSIONS)) / - Math.ceil(Math.floor(eventCount / 20) / TOTAL_SESSIONS); - const timestamp = - session.sessionStartTime + sessionProgress * maxSessionDuration; - const pathname = faker.helpers.arrayElement(PATHS); - const errorType = faker.helpers.arrayElement([ - "Error", - "TypeError", - "ReferenceError", - "UnhandledRejection", - ]); - - return { - client_id: clientId, - anonymous_id: session.anonymousId, + referrer: index === 0 ? session.referrer : null, session_id: session.sessionId, - timestamp, - path: pathname, - message: faker.helpers.arrayElement([ - "Cannot read property 'x' of undefined", - "Unexpected token in JSON", - "Network request failed", - "Maximum call stack size exceeded", - "Failed to fetch", - ]), - filename: faker.helpers.maybe( - () => `https://${domain}${pathname}/app.js`, - { probability: 0.8 } - ), - lineno: faker.helpers.maybe( - () => faker.number.int({ min: 1, max: 1000 }), - { probability: 0.7 } - ), - colno: faker.helpers.maybe( - () => faker.number.int({ min: 1, max: 100 }), - { probability: 0.6 } - ), - stack: faker.helpers.maybe( - () => - `Error: ${errorType}\n at function (app.js:${faker.number.int({ min: 1, max: 100 })}:${faker.number.int({ min: 1, max: 50 })})`, - { probability: 0.8 } - ), - error_type: errorType, + title: pageTitle(page.path), + url: `https://${domain}${page.path}`, + user_agent: "", }; - } - ); - - errors.sort((a, b) => a.timestamp - b.timestamp); - - const webVitals = Array.from( - { length: Math.floor(eventCount / 5) }, - (_, index) => { - const session = sessionFor(index, Math.floor(eventCount / 5)); - - const maxSessionDuration = 2 * 60 * 60 * 1000; - const sessionProgress = - (index % Math.ceil(Math.floor(eventCount / 5) / TOTAL_SESSIONS)) / - Math.ceil(Math.floor(eventCount / 5) / TOTAL_SESSIONS); - const timestamp = - session.sessionStartTime + sessionProgress * maxSessionDuration; - const pathname = faker.helpers.arrayElement(PATHS); - const metricName = faker.helpers.arrayElement([ - "FCP", - "LCP", - "CLS", - "INP", - "TTFB", - "FPS", - ]); + return [ + { + ...shared, + event_name: "screen_view", + id: faker.string.uuid(), + time: clickHouseTime(viewedAt), + }, + { + ...shared, + event_name: "page_exit", + id: faker.string.uuid(), + interaction_count: faker.number.int({ max: 20, min: 0 }), + scroll_depth: faker.number.int({ max: 100, min: 10 }), + time: clickHouseTime(viewedAt + page.seconds * 1000), + time_on_page: page.seconds, + }, + ]; + }); + }); - let metricValue: number; - if (metricName === "CLS") { - metricValue = faker.number.float({ - min: 0, - max: 0.5, - fractionDigits: 3, - }); - } else if (metricName === "FPS") { - metricValue = faker.number.int({ min: 30, max: 60 }); - } else { - metricValue = faker.number.int({ min: 100, max: 5000 }); - } + const webVitals = sessions.flatMap((session) => + VITALS.map((vital) => ({ + anonymous_id: session.visitor.anonymousId, + client_id: clientId, + metric_name: vital.name, + metric_value: faker.number.float({ + fractionDigits: 3, + max: vital.max, + min: vital.min, + }), + path: session.pages[0]?.path ?? "/", + session_id: session.sessionId, + timestamp: clickHouseTime(session.start + 1000), + })) + ); + const outgoingLinks = sessions + .filter(() => faker.datatype.boolean({ probability: 0.1 })) + .map((session) => ({ + anonymous_id: session.visitor.anonymousId, + client_id: clientId, + href: "https://github.com/databuddy-analytics/Databuddy", + id: faker.string.uuid(), + properties: "{}", + session_id: session.sessionId, + text: "Databuddy GitHub", + timestamp: clickHouseTime(session.start + 5000), + })); + + const errors = dailyTraffic.flatMap((day) => + Array.from({ length: day.errorCount }, () => { + const session = faker.helpers.arrayElement(day.sessions); + const errorType = faker.helpers.arrayElement(ERROR_TYPES); return { + anonymous_id: session.visitor.anonymousId, client_id: clientId, - anonymous_id: session.anonymousId, + colno: faker.number.int({ max: 80, min: 1 }), + error_type: errorType, + filename: `https://${domain}/_next/static/chunks/app.js`, + lineno: faker.number.int({ max: 900, min: 1 }), + message: faker.helpers.arrayElement(ERROR_MESSAGES), + path: faker.helpers.arrayElement(PATHS), session_id: session.sessionId, - timestamp, - path: pathname, - metric_name: metricName, - metric_value: metricValue, + stack: `${errorType}\n at app.js`, + timestamp: clickHouseTime(session.start + 2000), }; - } + }) ); - webVitals.sort((a, b) => a.timestamp - b.timestamp); + return { errors, events, outgoingLinks, webVitals }; +} - console.log( - `Generating seed data for client: ${clientId} on domain: ${domain}` - ); - console.log( - `Creating ${UNIQUE_USERS} users across ${TOTAL_SESSIONS} sessions` - ); +export type AnalyticsRows = ReturnType; +const SEEDED_TABLES = [ + TABLE_NAMES.events, + TABLE_NAMES.outgoing_links, + TABLE_NAMES.error_spans, + TABLE_NAMES.web_vitals_spans, +]; + +export async function seedAnalytics( + client: ClickHouseClient, + rows: AnalyticsRows +): Promise { + const format = "JSONEachRow"; await Promise.all([ - clickHouse.insert({ - table: TABLE_NAMES.events, - format: "JSONEachRow", - values: events, - }), - clickHouse.insert({ - table: "analytics.outgoing_links", - format: "JSONEachRow", - values: outgoingLinks, + client.insert({ format, table: TABLE_NAMES.events, values: rows.events }), + client.insert({ + format, + table: TABLE_NAMES.outgoing_links, + values: rows.outgoingLinks, }), - clickHouse.insert({ - table: "analytics.error_spans", - format: "JSONEachRow", - values: errors, + client.insert({ + format, + table: TABLE_NAMES.error_spans, + values: rows.errors, }), - clickHouse.insert({ - table: "analytics.web_vitals_spans", - format: "JSONEachRow", - values: webVitals, + client.insert({ + format, + table: TABLE_NAMES.web_vitals_spans, + values: rows.webVitals, }), ]); +} - console.log( - `Inserted ${events.length} events, ${outgoingLinks.length} outgoing links, ${errors.length} errors, ${webVitals.length} web vitals for client ${clientId}` - ); -})(); +export async function deleteAnalytics( + client: ClickHouseClient, + clientId: string +): Promise { + for (const table of SEEDED_TABLES) { + await client.command({ + query: `DELETE FROM ${table} WHERE client_id = {clientId:String}`, + query_params: { clientId }, + }); + } +} diff --git a/packages/test/src/auth.ts b/packages/test/src/auth.ts index 091e44a070..193964b996 100644 --- a/packages/test/src/auth.ts +++ b/packages/test/src/auth.ts @@ -1,5 +1,6 @@ import { auth } from "@databuddy/auth"; -import { member as memberTable } from "@databuddy/db/schema"; +import { member as memberTable, user as userTable } from "@databuddy/db/schema"; +import { eq } from "drizzle-orm"; import { db } from "./db"; import { nextId } from "./factories/id"; @@ -10,7 +11,12 @@ export interface AuthUser { } export async function signUp( - overrides: { email?: string; name?: string; password?: string } = {} + overrides: { + email?: string; + name?: string; + password?: string; + verified?: boolean; + } = {} ): Promise { const id = nextId("auth"); const email = overrides.email ?? `${id}@test.local`; @@ -18,6 +24,12 @@ export async function signUp( const password = overrides.password ?? "test-password-123!"; await auth.api.signUpEmail({ body: { email, name, password } }); + if (overrides.verified) { + await db() + .update(userTable) + .set({ emailVerified: true }) + .where(eq(userTable.email, email)); + } const res = await auth.api.signInEmail({ body: { email, password }, diff --git a/packages/test/src/db.ts b/packages/test/src/db.ts index 4d3ad4137d..340bb184bf 100644 --- a/packages/test/src/db.ts +++ b/packages/test/src/db.ts @@ -1,5 +1,6 @@ import { setPgTimingFn } from "@databuddy/db"; import { relations } from "@databuddy/db/schema/relations"; +import { dataUrl } from "@databuddy/env/app"; import { sql } from "drizzle-orm"; import { drizzle, type NodePgDatabase } from "drizzle-orm/node-postgres"; import { Pool } from "pg"; @@ -8,7 +9,7 @@ const DEFAULT_DATABASE_URL = "postgres://databuddy:databuddy_dev_password@localhost:5432/databuddy_test"; function databaseUrl(): string { - return process.env.DATABASE_URL ?? DEFAULT_DATABASE_URL; + return dataUrl("DATABASE_URL") ?? DEFAULT_DATABASE_URL; } export type DB = NodePgDatabase; @@ -16,24 +17,26 @@ export type DB = NodePgDatabase; let pool: Pool | null = null; let instance: DB | null = null; -export const hasTestDb = await (async () => { - const p = new Pool({ connectionString: databaseUrl(), max: 1 }); - try { - const c = await p.connect(); - c.release(); - return true; - } catch (error) { - if (process.env.CI) { - throw new Error( - "Integration tests could not reach the test database; CI must not skip them.", - { cause: error } - ); +export const hasTestDb = + process.env.NODE_ENV === "test" && + (await (async () => { + const p = new Pool({ connectionString: databaseUrl(), max: 1 }); + try { + const c = await p.connect(); + c.release(); + return true; + } catch (error) { + if (process.env.CI) { + throw new Error( + "Integration tests could not reach the test database; CI must not skip them.", + { cause: error } + ); + } + return false; + } finally { + await p.end(); } - return false; - } finally { - await p.end(); - } -})(); + })()); export function db(): DB { if (!instance) { diff --git a/packages/test/src/setup.ts b/packages/test/src/setup.ts index d02577dc59..7de0231397 100644 --- a/packages/test/src/setup.ts +++ b/packages/test/src/setup.ts @@ -1,5 +1,21 @@ +import { parseArgs } from "node:util"; +import { clickHouse } from "@databuddy/db/clickhouse"; +import { applyClickHouseSchema } from "@databuddy/db/clickhouse/apply"; +import { + applyPostgresSchema, + assertLocalTargets, + resetLocalDatabase, +} from "@databuddy/db/e2e-db-lifecycle"; +import { + deleteAnalytics, + EVENTS_PER_SESSION, + generateAnalytics, + seedAnalytics, +} from "@databuddy/db/seed"; +import { signUp } from "./auth"; import { closeClickHouse } from "./clickhouse"; -import { closePostgres, truncatePostgres } from "./db"; +import { closePostgres, db, truncatePostgres } from "./db"; +import { insertApiKey, insertWebsite } from "./factories"; import { closeRedis, flushRedis } from "./redis"; export async function reset() { @@ -9,3 +25,129 @@ export async function reset() { export async function cleanup() { await Promise.all([closePostgres(), closeRedis(), closeClickHouse()]); } + +const WORKSPACE = { + days: 28, + domain: "localhost", + email: "dev@databuddy.local", + password: "databuddy-dev", + websiteId: "local-website", +}; + +export interface WorkspaceOptions { + anomaly?: boolean; + events?: number; + reset?: boolean; + websiteId?: string; +} + +async function existingWebsite(websiteId: string) { + const website = await db().query.websites.findFirst({ + where: { id: websiteId }, + }); + if (!website) { + throw new Error(`Website "${websiteId}" does not exist`); + } + return { apiKey: null, website }; +} + +async function workspaceWebsite() { + const existing = await db().query.websites.findFirst({ + where: { id: WORKSPACE.websiteId }, + }); + if (existing) { + return { apiKey: null, website: existing }; + } + const user = await signUp({ + email: WORKSPACE.email, + name: "Local Dev", + password: WORKSPACE.password, + verified: true, + }); + const membership = await db().query.member.findFirst({ + where: { userId: user.id }, + }); + if (!membership) { + throw new Error("Sign-up did not provision an organization"); + } + const website = await insertWebsite({ + domain: WORKSPACE.domain, + id: WORKSPACE.websiteId, + name: "Localhost", + organizationId: membership.organizationId, + }); + if (!website) { + throw new Error("Website insert returned no row"); + } + const apiKey = await insertApiKey({ + name: "Local workspace", + organizationId: membership.organizationId, + scopes: ["read:data"], + }); + return { apiKey: apiKey.secret, website }; +} + +export async function bootstrapWorkspace(options: WorkspaceOptions = {}) { + const { databaseUrl } = assertLocalTargets(); + if (options.reset) { + await resetLocalDatabase(databaseUrl); + } + await Promise.all([ + applyPostgresSchema(databaseUrl), + applyClickHouseSchema(), + ]); + const { apiKey, website } = options.websiteId + ? await existingWebsite(options.websiteId) + : await workspaceWebsite(); + const rows = generateAnalytics({ + anomaly: options.anomaly, + clientId: website.id, + dailySessions: options.events + ? Math.ceil(options.events / WORKSPACE.days / EVENTS_PER_SESSION) + : undefined, + days: WORKSPACE.days, + domain: website.domain, + }); + await deleteAnalytics(clickHouse, website.id); + await seedAnalytics(clickHouse, rows); + return { apiKey, rows, website }; +} + +if (import.meta.main) { + try { + const { values } = parseArgs({ + options: { + anomaly: { type: "boolean" }, + events: { type: "string" }, + reset: { type: "boolean" }, + website: { type: "string" }, + }, + }); + const events = values.events ? Number(values.events) : undefined; + if (events !== undefined && !(events > 0)) { + throw new Error("--events must be a positive number"); + } + const { apiKey, rows, website } = await bootstrapWorkspace({ + anomaly: values.anomaly, + events, + reset: values.reset, + websiteId: values.website, + }); + console.info( + [ + `Seeded ${rows.events.length} events, ${rows.errors.length} errors, ${rows.webVitals.length} web vitals and ${rows.outgoingLinks.length} outgoing links`, + values.website + ? null + : `Login: ${WORKSPACE.email} / ${WORKSPACE.password}`, + `Website: ${website.id} (${website.domain})`, + apiKey ? `API key: ${apiKey} (shown once)` : null, + ] + .filter(Boolean) + .join("\n") + ); + process.exit(0); + } catch (error) { + console.error(error instanceof Error ? error.message : error); + process.exit(1); + } +} diff --git a/setup.ts b/setup.ts index 03943cbc18..7ca1f33cc0 100755 --- a/setup.ts +++ b/setup.ts @@ -310,7 +310,7 @@ async function main() { log(chalk.cyan(" 2. Run 'bun run dev' to start development servers")); log( chalk.cyan( - " 3. (Optional) Run 'bun run db:seed [DOMAIN] [EVENT_COUNT]' to seed sample data" + " 3. (Optional) Run 'bun run workspace --reset' for a local login and a seeded website" ) ); log(chalk.bold("\nHappy coding! 🎉")); From df451aa14139c86ba699ecf21ae3de41ccd822c5 Mon Sep 17 00:00:00 2001 From: iza <59828082+izadoesdev@users.noreply.github.com> Date: Mon, 5 Oct 2026 02:29:14 +0300 Subject: [PATCH 2/9] refactor(db): trim the workspace seed generator generateAnalytics takes a target event count, one loop writes every table, and the workspace CLI and dashboard e2e route share it without adapters; the e2e route parses its body with zod instead of a cast. --- .../app/api/test/e2e/clickhouse/route.ts | 92 ++++++------------- packages/db/src/e2e-db-lifecycle.ts | 17 ++-- packages/db/src/seed.ts | 81 +++++++--------- packages/test/src/setup.ts | 69 ++++---------- 4 files changed, 82 insertions(+), 177 deletions(-) diff --git a/apps/dashboard/app/api/test/e2e/clickhouse/route.ts b/apps/dashboard/app/api/test/e2e/clickhouse/route.ts index ab2b7ed553..8bd81f271c 100644 --- a/apps/dashboard/app/api/test/e2e/clickhouse/route.ts +++ b/apps/dashboard/app/api/test/e2e/clickhouse/route.ts @@ -1,97 +1,57 @@ import { clickHouse } from "@databuddy/db/clickhouse"; -import { - EVENTS_PER_SESSION, - generateAnalytics, - seedAnalytics, -} from "@databuddy/db/seed"; +import { generateAnalytics, seedAnalytics } from "@databuddy/db/seed"; import { readBooleanEnv } from "@databuddy/env/app"; +import { z } from "zod"; export const runtime = "nodejs"; export const dynamic = "force-dynamic"; -const TEST_KEY_HEADER = "x-e2e-test-key"; +const seedBody = z.object({ + eventCount: z.coerce.number().catch(250), + websiteId: z.string().min(1), +}); -interface SeedBody { - eventCount?: unknown; - websiteId?: unknown; -} - -function isE2EModeEnabled(): boolean { - return readBooleanEnv("DATABUDDY_E2E_MODE"); -} - -function notFound(): Response { - return Response.json({ error: "Not found" }, { status: 404 }); +function countBy(items: T[], key: (item: T) => string) { + const counts: Record = {}; + for (const item of items) { + counts[key(item)] = (counts[key(item)] ?? 0) + 1; + } + return counts; } -function assertE2EAccess(request: Request): Response | null { - if (!isE2EModeEnabled()) { - return notFound(); - } +export async function POST(request: Request): Promise { const key = process.env.DATABUDDY_E2E_TEST_KEY; - if (!key) { - return notFound(); + if (!(readBooleanEnv("DATABUDDY_E2E_MODE") && key)) { + return Response.json({ error: "Not found" }, { status: 404 }); } - if (request.headers.get(TEST_KEY_HEADER) !== key) { + if (request.headers.get("x-e2e-test-key") !== key) { return Response.json({ error: "Unauthorized" }, { status: 401 }); } - return null; -} -function normalizeEventCount(value: unknown): number { - const parsed = typeof value === "number" ? value : Number(value); - if (!Number.isFinite(parsed)) { - return 250; - } - return Math.min(Math.max(Math.floor(parsed), 1), 5000); -} - -export async function POST(request: Request): Promise { - const denied = assertE2EAccess(request); - if (denied) { - return denied; - } - - const body = (await request.json().catch(() => ({}))) as SeedBody; - if (typeof body.websiteId !== "string" || !body.websiteId) { + const body = seedBody.safeParse(await request.json().catch(() => null)); + if (!body.success) { return Response.json({ error: "websiteId is required" }, { status: 400 }); } + const { eventCount, websiteId } = body.data; const rows = generateAnalytics({ - clientId: body.websiteId, - dailySessions: Math.ceil( - normalizeEventCount(body.eventCount) / EVENTS_PER_SESSION - ), + clientId: websiteId, days: 1, domain: "e2e.databuddy.local", + events: Math.min(Math.max(Math.floor(eventCount), 1), 5000), }); await seedAnalytics(clickHouse, rows); - const screenViewEvents = rows.events.filter( + const screenViews = rows.events.filter( (event) => event.event_name === "screen_view" ); - const screenViewsByCountry = screenViewEvents.reduce>( - (acc, event) => { - acc[event.country] = (acc[event.country] ?? 0) + 1; - return acc; - }, - {} - ); - const screenViewsByPath = screenViewEvents.reduce>( - (acc, event) => { - acc[event.path] = (acc[event.path] ?? 0) + 1; - return acc; - }, - {} - ); - return Response.json({ events: rows.events.length, outgoingLinks: rows.outgoingLinks.length, - screenViews: screenViewEvents.length, - screenViewsByCountry, - screenViewsByPath, + screenViews: screenViews.length, + screenViewsByCountry: countBy(screenViews, (event) => event.country), + screenViewsByPath: countBy(screenViews, (event) => event.path), seeded: true, - websiteId: body.websiteId, + websiteId, }); } diff --git a/packages/db/src/e2e-db-lifecycle.ts b/packages/db/src/e2e-db-lifecycle.ts index 82060dadd2..93f349014f 100644 --- a/packages/db/src/e2e-db-lifecycle.ts +++ b/packages/db/src/e2e-db-lifecycle.ts @@ -237,13 +237,10 @@ export async function dropLifecycleDatabase( return { dbDsn: config.dbDsn, dbName: config.dbName }; } -type Env = Record; - function localUrl( - name: "CLICKHOUSE_URL" | "DATABASE_URL" | "REDIS_URL", - env: Env + name: "CLICKHOUSE_URL" | "DATABASE_URL" | "REDIS_URL" ): string { - const url = dataUrl(name, env); + const url = dataUrl(name); if (url && URL.canParse(url) && isLocalHost(url)) { return url; } @@ -253,12 +250,10 @@ function localUrl( ); } -export function assertLocalTargets(env: Env = process.env) { - return { - clickhouseUrl: localUrl("CLICKHOUSE_URL", env), - databaseUrl: localUrl("DATABASE_URL", env), - redisUrl: localUrl("REDIS_URL", env), - }; +export function assertLocalTargets(): string { + localUrl("CLICKHOUSE_URL"); + localUrl("REDIS_URL"); + return localUrl("DATABASE_URL"); } export async function resetLocalDatabase(databaseUrl: string): Promise { diff --git a/packages/db/src/seed.ts b/packages/db/src/seed.ts index b0fbd27559..440fecb098 100644 --- a/packages/db/src/seed.ts +++ b/packages/db/src/seed.ts @@ -2,8 +2,7 @@ import type { ClickHouseClient } from "@clickhouse/client"; import { en, Faker } from "@faker-js/faker"; import { TABLE_NAMES } from "./clickhouse/client"; -export const EVENTS_PER_SESSION = 6; - +const EVENTS_PER_SESSION = 6; const DAY_MS = 86_400_000; const SESSION_WINDOW_MS = DAY_MS - 3_600_000; const ANOMALY_TRAFFIC_MULTIPLIER = 3; @@ -47,32 +46,25 @@ const VITALS = [ { max: 0.3, min: 0, name: "CLS" }, ]; -export interface GenerateAnalyticsOptions { - anomaly?: boolean; - clientId: string; - dailySessions?: number; - days?: number; - domain: string; - seed?: number; -} - function clickHouseTime(ms: number): string { return new Date(ms).toISOString().replace("T", " ").replace("Z", ""); } -function pageTitle(path: string): string { - return path === "/" ? "Home" : path.slice(1); -} - export function generateAnalytics({ anomaly = false, clientId, - dailySessions = 150, days = 28, domain, - seed = 42, -}: GenerateAnalyticsOptions) { - const faker = new Faker({ locale: [en], seed }); + events: eventCount = days * 150 * EVENTS_PER_SESSION, +}: { + anomaly?: boolean; + clientId: string; + days?: number; + domain: string; + events?: number; +}) { + const faker = new Faker({ locale: [en], seed: 42 }); + const dailySessions = Math.ceil(eventCount / days / EVENTS_PER_SESSION); const now = Date.now(); const todayStart = Math.floor(now / DAY_MS) * DAY_MS; const visitors = Array.from({ length: dailySessions * 4 }, () => ({ @@ -136,7 +128,7 @@ export function generateAnalytics({ properties: "{}", referrer: index === 0 ? session.referrer : null, session_id: session.sessionId, - title: pageTitle(page.path), + title: page.path === "/" ? "Home" : page.path.slice(1), url: `https://${domain}${page.path}`, user_agent: "", }; @@ -212,45 +204,34 @@ export function generateAnalytics({ return { errors, events, outgoingLinks, webVitals }; } -export type AnalyticsRows = ReturnType; - -const SEEDED_TABLES = [ - TABLE_NAMES.events, - TABLE_NAMES.outgoing_links, - TABLE_NAMES.error_spans, - TABLE_NAMES.web_vitals_spans, -]; - export async function seedAnalytics( client: ClickHouseClient, - rows: AnalyticsRows + rows: ReturnType ): Promise { - const format = "JSONEachRow"; - await Promise.all([ - client.insert({ format, table: TABLE_NAMES.events, values: rows.events }), - client.insert({ - format, - table: TABLE_NAMES.outgoing_links, - values: rows.outgoingLinks, - }), - client.insert({ - format, - table: TABLE_NAMES.error_spans, - values: rows.errors, - }), - client.insert({ - format, - table: TABLE_NAMES.web_vitals_spans, - values: rows.webVitals, - }), - ]); + await Promise.all( + ( + [ + [TABLE_NAMES.events, rows.events], + [TABLE_NAMES.outgoing_links, rows.outgoingLinks], + [TABLE_NAMES.error_spans, rows.errors], + [TABLE_NAMES.web_vitals_spans, rows.webVitals], + ] as const + ).map(([table, values]) => + client.insert({ format: "JSONEachRow", table, values }) + ) + ); } export async function deleteAnalytics( client: ClickHouseClient, clientId: string ): Promise { - for (const table of SEEDED_TABLES) { + for (const table of [ + TABLE_NAMES.events, + TABLE_NAMES.outgoing_links, + TABLE_NAMES.error_spans, + TABLE_NAMES.web_vitals_spans, + ]) { await client.command({ query: `DELETE FROM ${table} WHERE client_id = {clientId:String}`, query_params: { clientId }, diff --git a/packages/test/src/setup.ts b/packages/test/src/setup.ts index 7de0231397..faf558bd8a 100644 --- a/packages/test/src/setup.ts +++ b/packages/test/src/setup.ts @@ -8,7 +8,6 @@ import { } from "@databuddy/db/e2e-db-lifecycle"; import { deleteAnalytics, - EVENTS_PER_SESSION, generateAnalytics, seedAnalytics, } from "@databuddy/db/seed"; @@ -27,37 +26,22 @@ export async function cleanup() { } const WORKSPACE = { - days: 28, domain: "localhost", email: "dev@databuddy.local", password: "databuddy-dev", websiteId: "local-website", }; -export interface WorkspaceOptions { - anomaly?: boolean; - events?: number; - reset?: boolean; - websiteId?: string; -} - -async function existingWebsite(websiteId: string) { - const website = await db().query.websites.findFirst({ - where: { id: websiteId }, - }); - if (!website) { - throw new Error(`Website "${websiteId}" does not exist`); - } - return { apiKey: null, website }; -} - -async function workspaceWebsite() { +async function workspaceWebsite(websiteId?: string) { const existing = await db().query.websites.findFirst({ - where: { id: WORKSPACE.websiteId }, + where: { id: websiteId ?? WORKSPACE.websiteId }, }); if (existing) { return { apiKey: null, website: existing }; } + if (websiteId) { + throw new Error(`Website "${websiteId}" does not exist`); + } const user = await signUp({ email: WORKSPACE.email, name: "Local Dev", @@ -87,32 +71,6 @@ async function workspaceWebsite() { return { apiKey: apiKey.secret, website }; } -export async function bootstrapWorkspace(options: WorkspaceOptions = {}) { - const { databaseUrl } = assertLocalTargets(); - if (options.reset) { - await resetLocalDatabase(databaseUrl); - } - await Promise.all([ - applyPostgresSchema(databaseUrl), - applyClickHouseSchema(), - ]); - const { apiKey, website } = options.websiteId - ? await existingWebsite(options.websiteId) - : await workspaceWebsite(); - const rows = generateAnalytics({ - anomaly: options.anomaly, - clientId: website.id, - dailySessions: options.events - ? Math.ceil(options.events / WORKSPACE.days / EVENTS_PER_SESSION) - : undefined, - days: WORKSPACE.days, - domain: website.domain, - }); - await deleteAnalytics(clickHouse, website.id); - await seedAnalytics(clickHouse, rows); - return { apiKey, rows, website }; -} - if (import.meta.main) { try { const { values } = parseArgs({ @@ -127,12 +85,23 @@ if (import.meta.main) { if (events !== undefined && !(events > 0)) { throw new Error("--events must be a positive number"); } - const { apiKey, rows, website } = await bootstrapWorkspace({ + const databaseUrl = assertLocalTargets(); + if (values.reset) { + await resetLocalDatabase(databaseUrl); + } + await Promise.all([ + applyPostgresSchema(databaseUrl), + applyClickHouseSchema(), + ]); + const { apiKey, website } = await workspaceWebsite(values.website); + const rows = generateAnalytics({ anomaly: values.anomaly, + clientId: website.id, + domain: website.domain, events, - reset: values.reset, - websiteId: values.website, }); + await deleteAnalytics(clickHouse, website.id); + await seedAnalytics(clickHouse, rows); console.info( [ `Seeded ${rows.events.length} events, ${rows.errors.length} errors, ${rows.webVitals.length} web vitals and ${rows.outgoingLinks.length} outgoing links`, From ff4725f16915150779354bc5914557ffb490df97 Mon Sep 17 00:00:00 2001 From: iza <59828082+izadoesdev@users.noreply.github.com> Date: Mon, 5 Oct 2026 12:55:13 +0300 Subject: [PATCH 3/9] fix(db): require loopback before workspace writes and resets --- packages/db/src/e2e-db-lifecycle.test.ts | 82 +++++++++++++++++++++++- packages/db/src/e2e-db-lifecycle.ts | 6 +- 2 files changed, 84 insertions(+), 4 deletions(-) diff --git a/packages/db/src/e2e-db-lifecycle.test.ts b/packages/db/src/e2e-db-lifecycle.test.ts index 3163955e83..3ab3ed1dd6 100644 --- a/packages/db/src/e2e-db-lifecycle.test.ts +++ b/packages/db/src/e2e-db-lifecycle.test.ts @@ -1,10 +1,21 @@ -import { describe, expect, it } from "bun:test"; import { + afterEach, + beforeEach, + describe, + expect, + it, + mock, + spyOn, +} from "bun:test"; +import { Client } from "pg"; +import { + assertLocalTargets, deriveAdminDatabaseUrl, deriveDatabaseUrl, parseLifecycleArgs, resolveE2EDatabaseName, resolveLifecycleConfig, + resetLocalDatabase, sanitizeDbIdentifierPart, toShellAssignments, } from "./e2e-db-lifecycle"; @@ -159,3 +170,72 @@ describe("e2e db lifecycle helpers", () => { ).toBe("DATABASE_URL='postgres://u:p@localhost/db'\"'\"'quoted'"); }); }); + +describe("workspace target guards", () => { + const originalEnv = process.env; + + beforeEach(() => { + process.env = { ...originalEnv, NODE_ENV: "development" }; + delete process.env.CLICKHOUSE_URL; + delete process.env.DATABASE_URL; + delete process.env.REDIS_URL; + }); + + afterEach(() => { + process.env = originalEnv; + mock.restore(); + }); + + it("uses the local development defaults when service URLs are unset", () => { + expect(assertLocalTargets()).toBe( + "postgres://databuddy:databuddy_dev_password@localhost:5432/databuddy" + ); + }); + + it.each([ + "localhost", + "127.0.0.1", + "[::1]", + ])("allows explicit loopback targets before workspace writes: %s", (host) => { + process.env.CLICKHOUSE_URL = `http://${host}:8123`; + process.env.REDIS_URL = `redis://${host}:6379`; + process.env.DATABASE_URL = `postgres://u:p@${host}:5432/databuddy`; + expect(assertLocalTargets()).toBe(process.env.DATABASE_URL); + }); + + it.each([ + ["CLICKHOUSE_URL", "http://clickhouse:8123"], + ["DATABASE_URL", "postgres://u:p@staging:5432/databuddy"], + ["REDIS_URL", "redis://redis:6379"], + ])("refuses single-label %s before workspace writes", (name, url) => { + process.env[name] = url; + expect(() => assertLocalTargets()).toThrow( + `Refusing to run against a non-local database; ${name}` + ); + }); + + it.each([ + "production", + "test", + ])("still refuses unset workspace service URLs outside development: %s", (nodeEnv) => { + process.env.NODE_ENV = nodeEnv; + expect(() => assertLocalTargets()).toThrow('host is "unset"'); + }); + + it.each([ + "staging", + "0.0.0.0", + "db.example.com", + "[2001:db8::1]", + ])("refuses a non-loopback reset before opening a connection: %s", async (host) => { + const connect = spyOn(Client.prototype, "connect").mockImplementation( + () => { + throw new Error("Unexpected database connection in guard test"); + } + ); + await expect( + resetLocalDatabase(`postgres://u:p@${host}:5432/databuddy`) + ).rejects.toThrow("Refusing to reset database"); + expect(connect).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/db/src/e2e-db-lifecycle.ts b/packages/db/src/e2e-db-lifecycle.ts index 93f349014f..d2c8893492 100644 --- a/packages/db/src/e2e-db-lifecycle.ts +++ b/packages/db/src/e2e-db-lifecycle.ts @@ -1,4 +1,4 @@ -import { dataUrl, isLocalHost, isLoopbackHost } from "@databuddy/env/app"; +import { dataUrl, isLoopbackHost } from "@databuddy/env/app"; import { drizzle } from "drizzle-orm/node-postgres"; import { Client, Pool } from "pg"; @@ -241,7 +241,7 @@ function localUrl( name: "CLICKHOUSE_URL" | "DATABASE_URL" | "REDIS_URL" ): string { const url = dataUrl(name); - if (url && URL.canParse(url) && isLocalHost(url)) { + if (url && URL.canParse(url) && isLoopbackHost(url)) { return url; } const host = url && URL.canParse(url) ? new URL(url).hostname : "unset"; @@ -259,7 +259,7 @@ export function assertLocalTargets(): string { export async function resetLocalDatabase(databaseUrl: string): Promise { const url = normalizeDatabaseUrl(databaseUrl); const dbName = decodeURIComponent(url.pathname.slice(1)); - if (!(dbName && isLocalHost(url.href))) { + if (!(dbName && isLoopbackHost(url.href))) { throw new Error( `Refusing to reset database "${dbName}" on ${url.hostname}` ); From 454b4552e15780d725a4194a195d1d7e339ea513 Mon Sep 17 00:00:00 2001 From: iza <59828082+izadoesdev@users.noreply.github.com> Date: Tue, 6 Oct 2026 13:23:40 +0300 Subject: [PATCH 4/9] fix(db): reject workspace PostgreSQL host overrides --- packages/db/src/e2e-db-lifecycle.test.ts | 31 ++++++++++++++++++++++++ packages/db/src/e2e-db-lifecycle.ts | 10 ++++++++ 2 files changed, 41 insertions(+) diff --git a/packages/db/src/e2e-db-lifecycle.test.ts b/packages/db/src/e2e-db-lifecycle.test.ts index 3ab3ed1dd6..478e83b465 100644 --- a/packages/db/src/e2e-db-lifecycle.test.ts +++ b/packages/db/src/e2e-db-lifecycle.test.ts @@ -222,6 +222,37 @@ describe("workspace target guards", () => { expect(() => assertLocalTargets()).toThrow('host is "unset"'); }); + it.each([ + "host=remote.example", + "%68ost=remote.example", + "host=localhost&host=remote.example", + "host=%2Ftmp%2Fpostgres", + ])("refuses workspace PostgreSQL host overrides before writes: %s", (query) => { + process.env.DATABASE_URL = `postgres://u:p@localhost:5432/databuddy?${query}`; + expect(() => assertLocalTargets()).toThrow( + "Refusing to run the workspace with a PostgreSQL host override" + ); + }); + + it.each([ + "host=remote.example", + "%68ost=remote.example", + "host=localhost&host=remote.example", + "host=%2Ftmp%2Fpostgres", + ])("refuses PostgreSQL reset host overrides before connecting: %s", async (query) => { + const connect = spyOn(Client.prototype, "connect").mockImplementation( + () => { + throw new Error("Unexpected database connection in guard test"); + } + ); + await expect( + resetLocalDatabase(`postgres://u:p@localhost:5432/databuddy?${query}`) + ).rejects.toThrow( + "Refusing to reset database with a PostgreSQL host override" + ); + expect(connect).not.toHaveBeenCalled(); + }); + it.each([ "staging", "0.0.0.0", diff --git a/packages/db/src/e2e-db-lifecycle.ts b/packages/db/src/e2e-db-lifecycle.ts index d2c8893492..5dffef327d 100644 --- a/packages/db/src/e2e-db-lifecycle.ts +++ b/packages/db/src/e2e-db-lifecycle.ts @@ -242,6 +242,11 @@ function localUrl( ): string { const url = dataUrl(name); if (url && URL.canParse(url) && isLoopbackHost(url)) { + if (name === "DATABASE_URL" && new URL(url).searchParams.has("host")) { + throw new Error( + "Refusing to run the workspace with a PostgreSQL host override" + ); + } return url; } const host = url && URL.canParse(url) ? new URL(url).hostname : "unset"; @@ -258,6 +263,11 @@ export function assertLocalTargets(): string { export async function resetLocalDatabase(databaseUrl: string): Promise { const url = normalizeDatabaseUrl(databaseUrl); + if (url.searchParams.has("host")) { + throw new Error( + "Refusing to reset database with a PostgreSQL host override" + ); + } const dbName = decodeURIComponent(url.pathname.slice(1)); if (!(dbName && isLoopbackHost(url.href))) { throw new Error( From ae13cf199ca3f0fb1fb2eeeeb874baa22de55c69 Mon Sep 17 00:00:00 2001 From: iza <59828082+izadoesdev@users.noreply.github.com> Date: Tue, 6 Oct 2026 13:35:12 +0300 Subject: [PATCH 5/9] fix(db): require finite workspace event counts --- packages/test/src/setup.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/test/src/setup.ts b/packages/test/src/setup.ts index faf558bd8a..222ac13b54 100644 --- a/packages/test/src/setup.ts +++ b/packages/test/src/setup.ts @@ -82,7 +82,7 @@ if (import.meta.main) { }, }); const events = values.events ? Number(values.events) : undefined; - if (events !== undefined && !(events > 0)) { + if (events !== undefined && !(Number.isFinite(events) && events > 0)) { throw new Error("--events must be a positive number"); } const databaseUrl = assertLocalTargets(); From 43dc59cf10f19535431494bf21babea0fca0cd77 Mon Sep 17 00:00:00 2001 From: iza <59828082+izadoesdev@users.noreply.github.com> Date: Tue, 6 Oct 2026 14:13:29 +0300 Subject: [PATCH 6/9] fix(db): replace workspace totals and reuse the local user --- packages/db/src/seed.ts | 2 ++ packages/test/src/setup.ts | 18 +++++++++++------- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/packages/db/src/seed.ts b/packages/db/src/seed.ts index 440fecb098..90c3fc77ca 100644 --- a/packages/db/src/seed.ts +++ b/packages/db/src/seed.ts @@ -228,11 +228,13 @@ export async function deleteAnalytics( ): Promise { for (const table of [ TABLE_NAMES.events, + "analytics.daily_pageviews", TABLE_NAMES.outgoing_links, TABLE_NAMES.error_spans, TABLE_NAMES.web_vitals_spans, ]) { await client.command({ + clickhouse_settings: { lightweight_deletes_sync: "1" }, query: `DELETE FROM ${table} WHERE client_id = {clientId:String}`, query_params: { clientId }, }); diff --git a/packages/test/src/setup.ts b/packages/test/src/setup.ts index 222ac13b54..c4d96cbbeb 100644 --- a/packages/test/src/setup.ts +++ b/packages/test/src/setup.ts @@ -42,17 +42,21 @@ async function workspaceWebsite(websiteId?: string) { if (websiteId) { throw new Error(`Website "${websiteId}" does not exist`); } - const user = await signUp({ - email: WORKSPACE.email, - name: "Local Dev", - password: WORKSPACE.password, - verified: true, - }); + const user = + (await db().query.user.findFirst({ + where: { email: WORKSPACE.email }, + })) ?? + (await signUp({ + email: WORKSPACE.email, + name: "Local Dev", + password: WORKSPACE.password, + verified: true, + })); const membership = await db().query.member.findFirst({ where: { userId: user.id }, }); if (!membership) { - throw new Error("Sign-up did not provision an organization"); + throw new Error("Local dev user has no organization"); } const website = await insertWebsite({ domain: WORKSPACE.domain, From 1f703be2ac948ff975687bd3017f39654ca93ca1 Mon Sep 17 00:00:00 2001 From: iza <59828082+izadoesdev@users.noreply.github.com> Date: Tue, 6 Oct 2026 14:13:43 +0300 Subject: [PATCH 7/9] fix(db): bound workspace analytics fixture generation --- CONTRIBUTING.md | 5 ++++- packages/db/src/seed.ts | 18 ++++++++++++++++++ packages/test/src/setup.ts | 13 ++++++++++--- 3 files changed, 32 insertions(+), 4 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 18e4bb55e8..1c1d11a9b4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -42,7 +42,10 @@ see [optional services](README.md#optional-services). To explore with sample data, run `bun run workspace --reset`. It targets the local Docker services only and prints a login, a website and an API key. Add `--anomaly` for a traffic spike and error surge that Insights will flag, or -`--website YOUR_WEBSITE_ID` to seed a website you already made. +`--website YOUR_WEBSITE_ID` to seed a website you already made. The optional +`--events N` sets an approximate event target of up to 100,000 across 28 days; +session sizes, weekend traffic and anomaly variation affect the number written. +The command prints the actual inserted counts. ## Check your changes diff --git a/packages/db/src/seed.ts b/packages/db/src/seed.ts index 90c3fc77ca..721467bbbf 100644 --- a/packages/db/src/seed.ts +++ b/packages/db/src/seed.ts @@ -2,6 +2,8 @@ import type { ClickHouseClient } from "@clickhouse/client"; import { en, Faker } from "@faker-js/faker"; import { TABLE_NAMES } from "./clickhouse/client"; +export const MAX_ANALYTICS_EVENTS = 100_000; + const EVENTS_PER_SESSION = 6; const DAY_MS = 86_400_000; const SESSION_WINDOW_MS = DAY_MS - 3_600_000; @@ -50,6 +52,8 @@ function clickHouseTime(ms: number): string { return new Date(ms).toISOString().replace("T", " ").replace("Z", ""); } +// Events are an approximate target: session sizes, weekends and anomaly mode +// vary the rows generated within the bounded day range. export function generateAnalytics({ anomaly = false, clientId, @@ -63,6 +67,20 @@ export function generateAnalytics({ domain: string; events?: number; }) { + if (!(Number.isInteger(days) && days > 0 && days <= 365)) { + throw new Error("days must be a positive integer no greater than 365"); + } + if ( + !( + Number.isSafeInteger(eventCount) && + eventCount > 0 && + eventCount <= MAX_ANALYTICS_EVENTS + ) + ) { + throw new Error( + `events must be a positive integer no greater than ${MAX_ANALYTICS_EVENTS}` + ); + } const faker = new Faker({ locale: [en], seed: 42 }); const dailySessions = Math.ceil(eventCount / days / EVENTS_PER_SESSION); const now = Date.now(); diff --git a/packages/test/src/setup.ts b/packages/test/src/setup.ts index c4d96cbbeb..61c78d010b 100644 --- a/packages/test/src/setup.ts +++ b/packages/test/src/setup.ts @@ -9,6 +9,7 @@ import { import { deleteAnalytics, generateAnalytics, + MAX_ANALYTICS_EVENTS, seedAnalytics, } from "@databuddy/db/seed"; import { signUp } from "./auth"; @@ -85,9 +86,15 @@ if (import.meta.main) { website: { type: "string" }, }, }); - const events = values.events ? Number(values.events) : undefined; - if (events !== undefined && !(Number.isFinite(events) && events > 0)) { - throw new Error("--events must be a positive number"); + const events = + values.events === undefined ? undefined : Number(values.events); + if (events !== undefined && !(Number.isSafeInteger(events) && events > 0)) { + throw new Error("--events must be a positive integer"); + } + if (events !== undefined && events > MAX_ANALYTICS_EVENTS) { + throw new Error( + `--events must be no greater than ${MAX_ANALYTICS_EVENTS}` + ); } const databaseUrl = assertLocalTargets(); if (values.reset) { From 3b60bc07434c948b177de964a3fecb1944f50dd4 Mon Sep 17 00:00:00 2001 From: iza <59828082+izadoesdev@users.noreply.github.com> Date: Tue, 6 Oct 2026 14:36:14 +0300 Subject: [PATCH 8/9] fix(db): preserve workspace defaults and login guidance --- CONTRIBUTING.md | 7 +++++-- packages/db/src/seed.ts | 5 ++++- packages/test/src/setup.ts | 22 +++++++++++++--------- 3 files changed, 22 insertions(+), 12 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1c1d11a9b4..d47a40382f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -40,8 +40,11 @@ devtools for you. Add provider keys only for the features you're working on; see [optional services](README.md#optional-services). To explore with sample data, run `bun run workspace --reset`. It targets the -local Docker services only and prints a login, a website and an API key. Add -`--anomaly` for a traffic spike and error surge that Insights will flag, or +local Docker services only and prints the website. Without `--website`, it +prints a password for a newly created default account or directs you to use +the existing account's password. It prints an API key only when it creates +the default website. Add `--anomaly` for a traffic spike and error surge that +Insights will flag, or `--website YOUR_WEBSITE_ID` to seed a website you already made. The optional `--events N` sets an approximate event target of up to 100,000 across 28 days; session sizes, weekend traffic and anomaly variation affect the number written. diff --git a/packages/db/src/seed.ts b/packages/db/src/seed.ts index 721467bbbf..1f60005668 100644 --- a/packages/db/src/seed.ts +++ b/packages/db/src/seed.ts @@ -59,7 +59,10 @@ export function generateAnalytics({ clientId, days = 28, domain, - events: eventCount = days * 150 * EVENTS_PER_SESSION, + events: eventCount = Math.min( + days * 150 * EVENTS_PER_SESSION, + MAX_ANALYTICS_EVENTS + ), }: { anomaly?: boolean; clientId: string; diff --git a/packages/test/src/setup.ts b/packages/test/src/setup.ts index 61c78d010b..b7c986779e 100644 --- a/packages/test/src/setup.ts +++ b/packages/test/src/setup.ts @@ -38,15 +38,16 @@ async function workspaceWebsite(websiteId?: string) { where: { id: websiteId ?? WORKSPACE.websiteId }, }); if (existing) { - return { apiKey: null, website: existing }; + return { apiKey: null, createdUser: false, website: existing }; } if (websiteId) { throw new Error(`Website "${websiteId}" does not exist`); } + const existingUser = await db().query.user.findFirst({ + where: { email: WORKSPACE.email }, + }); const user = - (await db().query.user.findFirst({ - where: { email: WORKSPACE.email }, - })) ?? + existingUser ?? (await signUp({ email: WORKSPACE.email, name: "Local Dev", @@ -73,7 +74,7 @@ async function workspaceWebsite(websiteId?: string) { organizationId: membership.organizationId, scopes: ["read:data"], }); - return { apiKey: apiKey.secret, website }; + return { apiKey: apiKey.secret, createdUser: !existingUser, website }; } if (import.meta.main) { @@ -104,7 +105,9 @@ if (import.meta.main) { applyPostgresSchema(databaseUrl), applyClickHouseSchema(), ]); - const { apiKey, website } = await workspaceWebsite(values.website); + const { apiKey, createdUser, website } = await workspaceWebsite( + values.website + ); const rows = generateAnalytics({ anomaly: values.anomaly, clientId: website.id, @@ -113,12 +116,13 @@ if (import.meta.main) { }); await deleteAnalytics(clickHouse, website.id); await seedAnalytics(clickHouse, rows); + const loginMessage = createdUser + ? `Login: ${WORKSPACE.email} / ${WORKSPACE.password}` + : `Login: ${WORKSPACE.email} (use your existing password)`; console.info( [ `Seeded ${rows.events.length} events, ${rows.errors.length} errors, ${rows.webVitals.length} web vitals and ${rows.outgoingLinks.length} outgoing links`, - values.website - ? null - : `Login: ${WORKSPACE.email} / ${WORKSPACE.password}`, + values.website ? null : loginMessage, `Website: ${website.id} (${website.domain})`, apiKey ? `API key: ${apiKey} (shown once)` : null, ] From 131bcb331205de08d6ab01fcb5ac773bd38d6f9b Mon Sep 17 00:00:00 2001 From: iza <59828082+izadoesdev@users.noreply.github.com> Date: Wed, 7 Oct 2026 12:44:29 +0300 Subject: [PATCH 9/9] fix(db): initialize workspace auth after local target validation --- packages/db/src/e2e-db-lifecycle.test.ts | 43 ++++++++++++++++++++++++ packages/test/src/setup.ts | 5 +-- 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/packages/db/src/e2e-db-lifecycle.test.ts b/packages/db/src/e2e-db-lifecycle.test.ts index 478e83b465..37b9512023 100644 --- a/packages/db/src/e2e-db-lifecycle.test.ts +++ b/packages/db/src/e2e-db-lifecycle.test.ts @@ -7,6 +7,9 @@ import { mock, spyOn, } from "bun:test"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import { Client } from "pg"; import { assertLocalTargets, @@ -214,6 +217,46 @@ describe("workspace target guards", () => { ); }); + it("rejects remote workspace targets before initializing auth", () => { + const directory = mkdtempSync(join(tmpdir(), "workspace-auth-guard-")); + const preload = join(directory, "preload.ts"); + try { + writeFileSync( + preload, + `Bun.plugin({ name: "auth-startup-guard", setup(build) { + build.onLoad({ filter: /\\/packages\\/test\\/src\\/auth\\.ts$/ }, () => ({ + loader: "js", + contents: 'throw new Error("Auth initialized before target validation"); export const signUp = () => {};', + })); + } });` + ); + const result = Bun.spawnSync( + [ + process.execPath, + "--no-env-file", + "--preload", + preload, + join(import.meta.dir, "../../test/src/setup.ts"), + ], + { + env: { + CLICKHOUSE_URL: "http://localhost:8123", + DATABASE_URL: "postgres://u:p@localhost:5432/databuddy", + NODE_ENV: "development", + REDIS_URL: "redis://remote.example:6379", + }, + timeout: 10_000, + } + ); + expect(result.exitCode).toBe(1); + expect(result.stderr.toString()).toContain( + 'Refusing to run against a non-local database; REDIS_URL host is "remote.example"' + ); + } finally { + rmSync(directory, { force: true, recursive: true }); + } + }); + it.each([ "production", "test", diff --git a/packages/test/src/setup.ts b/packages/test/src/setup.ts index b7c986779e..82ad296ffc 100644 --- a/packages/test/src/setup.ts +++ b/packages/test/src/setup.ts @@ -12,7 +12,6 @@ import { MAX_ANALYTICS_EVENTS, seedAnalytics, } from "@databuddy/db/seed"; -import { signUp } from "./auth"; import { closeClickHouse } from "./clickhouse"; import { closePostgres, db, truncatePostgres } from "./db"; import { insertApiKey, insertWebsite } from "./factories"; @@ -48,7 +47,9 @@ async function workspaceWebsite(websiteId?: string) { }); const user = existingUser ?? - (await signUp({ + (await ( + await import("./auth") + ).signUp({ email: WORKSPACE.email, name: "Local Dev", password: WORKSPACE.password,