diff --git a/.env.example b/.env.example index c258a70..ec5eec9 100644 --- a/.env.example +++ b/.env.example @@ -13,7 +13,7 @@ POSTGRES_PASSWORD=bufferdash_password # Auth SESSION_SECRET=change_this_to_a_long_random_secret ADMIN_EMAIL=admin@example.com -# Prefer ADMIN_PASSWORD_HASH. Generate with: node -e "const bcrypt=require('bcryptjs'); bcrypt.hash(process.argv[1], 12).then(console.log)" 'your-password' +# Prefer ADMIN_PASSWORD_HASH. See README for an interactive generation command. ADMIN_PASSWORD_HASH= # Development fallback only. Do not use plain-text passwords in production. ADMIN_PASSWORD=change_this_password @@ -25,11 +25,22 @@ ANONYMIZE_IP=true TRUST_PROXY=true ENFORCE_TRACKING_ORIGIN=true +# GeoIP (optional). Trusted Cloudflare/Vercel headers work without a token. +# Lite provides country/ASN; Core also provides city/region. +IPINFO_TOKEN= +IPINFO_TIER=lite + +# Optional structured events from a trusted reverse proxy or host security agent. +ENABLE_LOG_INGESTION=false +INGESTION_SECRET= + # Security RATE_LIMIT_TRACKING_PER_MINUTE=120 RATE_LIMIT_ADMIN_PER_MINUTE=60 # Server Monitoring Optional ENABLE_SERVER_METRICS=false +METRICS_INTERVAL_SECONDS=60 +CLEANUP_INTERVAL_HOURS=24 DATA_RETENTION_DAYS=90 FILTER_BOTS=false diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c156dd8..f096dee 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,6 +10,23 @@ on: jobs: validate: runs-on: ubuntu-latest + services: + postgres: + image: postgres:16 + env: + POSTGRES_DB: bufferdash_test + POSTGRES_USER: bufferdash + POSTGRES_PASSWORD: test_password + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U bufferdash -d bufferdash_test" + --health-interval 5s + --health-timeout 5s + --health-retries 10 + env: + DATABASE_URL: postgresql://bufferdash:test_password@127.0.0.1:5432/bufferdash_test + TEST_DATABASE_URL: postgresql://bufferdash:test_password@127.0.0.1:5432/bufferdash_test steps: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 @@ -17,6 +34,7 @@ jobs: node-version: 22 cache: npm - run: npm ci + - run: npx prisma migrate deploy - run: npm run lint - run: npm run typecheck - run: npm test diff --git a/.gitignore b/.gitignore index aa069da..4264b65 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,8 @@ node_modules/ .next/ dist/ coverage/ +output/playwright/ +.playwright-cli/ prisma/dev.db tsconfig.tsbuildinfo bufferdash_build_plan.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..fa1ba19 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,22 @@ +# Contributing + +## Development + +Run PostgreSQL, copy `.env.example` to `.env`, and use non-production development values. + +```bash +npm ci +npx prisma migrate dev +npm run dev +``` + +Before submitting a change: + +```bash +npm run validate +docker compose config --quiet +``` + +Changes to `prisma/schema.prisma` must include a reviewed migration. Security-sensitive changes should include focused tests. Never commit `.env`, logs, database dumps, visitor data, production hostnames, or credentials. + +Keep public ingestion endpoints small and bounded. Analytics reads and operational mutations must remain authenticated and server-side. diff --git a/DEPLOYMENT.md b/DEPLOYMENT.md new file mode 100644 index 0000000..a14b54f --- /dev/null +++ b/DEPLOYMENT.md @@ -0,0 +1,75 @@ +# Production Deployment + +## Prerequisites + +- A Linux VPS with current security updates +- Docker Engine with the Compose plugin +- A DNS record such as `dash.example.com` pointing to the VPS +- Caddy or Nginx terminating HTTPS + +## Configure + +```bash +git clone https://github.com/1337lean/bufferdash.git +cd bufferdash +cp .env.example .env +chmod 600 .env +``` + +Generate separate database, session, tracking, and optional ingestion secrets. Never reuse a secret and never commit `.env`. + +Use a hexadecimal database password so it can safely appear in `DATABASE_URL`. Keep the bcrypt admin hash single-quoted in `.env` so its `$` characters remain literal. + +Recommended privacy defaults: + +```env +ANONYMIZE_IP=true +TRUST_PROXY=true +ENFORCE_TRACKING_ORIGIN=true +FILTER_BOTS=false +ENABLE_SERVER_METRICS=true +DATA_RETENTION_DAYS=90 +``` + +## Start and verify + +```bash +docker compose up -d --build +docker compose ps +curl -fsS http://127.0.0.1:3000/health +``` + +Both `app` and `worker` should become healthy, `migrate` should exit successfully, and `postgres` should remain healthy. BufferDash binds only to `127.0.0.1:3000`; PostgreSQL has no host port. + +## Caddy + +```caddy +dash.example.com { + encode zstd gzip + reverse_proxy 127.0.0.1:3000 +} +``` + +Allow only SSH, HTTP, and HTTPS through the VPS firewall. If Cloudflare proxies the hostname, enable authenticated origin pulls or restrict ports 80/443 to Cloudflare's published ranges where operationally practical. + +## Backups and updates + +Schedule `npm run db:backup` and copy encrypted backups off the VPS. Test restoration periodically. + +```bash +git pull --ff-only +docker compose up -d --build +docker compose ps +``` + +Database migrations run before the updated application and worker start. + +## Final checks + +- Log in and log out successfully over HTTPS. +- Confirm `/dashboard`, `/logs`, and `/security` redirect when logged out. +- Create a site and confirm its exact domain is configured. +- Install the snippet and check a pageview appears. +- Confirm query strings containing test values do not appear in the event stream. +- Confirm PostgreSQL and port 3000 are unreachable from the public internet. +- Confirm backups exist off-host and can be restored. diff --git a/Dockerfile b/Dockerfile index 662a6c2..4174aca 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,6 +19,16 @@ COPY package*.json ./ COPY prisma ./prisma CMD ["npm", "run", "prisma:deploy"] +FROM node:22-alpine AS worker +WORKDIR /app +ENV NODE_ENV=production +COPY --from=deps /app/node_modules ./node_modules +COPY package*.json ./ +COPY prisma ./prisma +COPY scripts/background-worker.mjs ./scripts/background-worker.mjs +RUN npx prisma generate +CMD ["node", "scripts/background-worker.mjs"] + FROM node:22-alpine AS runner WORKDIR /app ENV NODE_ENV=production diff --git a/README.md b/README.md index fe0d4a8..132c77a 100644 --- a/README.md +++ b/README.md @@ -6,11 +6,13 @@ BufferDash is a self-hosted, first-party web analytics dashboard with traffic-qu - Multi-site tracking with public site keys - Tiny public `/tracker.js` script -- Page views, sessions, unique visitors, referrers, browsers, OS, devices, and live visitors +- Page views, sessions, bounce rate, unique visitors, referrers, browsers, OS, devices, locations, and live visitors +- Selectable 24-hour, 7-day, 30-day, and 90-day analytics ranges - Secure IP handling with optional anonymization and hashed IPs -- Bot and unusual-path signals visible to the browser tracker +- Bot, unknown-path, failed-login, and rate-limit security signals +- Optional structured SSH, Fail2Ban, and reverse-proxy event ingestion - Protected admin dashboard with signed HTTP-only sessions and CSRF checks for UI mutations -- Optional metrics visible to the BufferDash process, clearly identified as container/runtime data when applicable +- Background retention cleanup and optional one-minute runtime metric collection - Docker Compose setup with PostgreSQL ## Quick Start @@ -38,7 +40,8 @@ http://localhost:3000 ```bash cp .env.example .env -docker compose up -d +# Replace every production placeholder in .env first. +docker compose up -d --build ``` The Compose stack starts PostgreSQL, waits for it to become healthy, runs Prisma migrations with the `migrate` service, and then starts the app. PostgreSQL is persisted in the `postgres_data` Docker volume and is not published on a host port. BufferDash binds only to `127.0.0.1:3000` by default. @@ -69,13 +72,19 @@ ADMIN_PASSWORD_HASH=replace_with_a_bcrypt_hash TRUST_PROXY=true ANONYMIZE_IP=true ENABLE_SERVER_METRICS=false +IPINFO_TOKEN= +IPINFO_TIER=lite +ENABLE_LOG_INGESTION=false +INGESTION_SECRET= ``` -Generate secrets and the admin password hash: +Generate secrets and the admin password hash. Generate the hash on a trusted machine after `npm ci`; the interactive form keeps the password out of shell history: ```bash openssl rand -base64 48 -node -e "const bcrypt=require('bcryptjs'); bcrypt.hash(process.argv[1], 12).then(console.log)" 'your-password' +read -s ADMIN_PASSWORD; export ADMIN_PASSWORD +node -e 'require("bcryptjs").hash(process.env.ADMIN_PASSWORD, 12).then(console.log)' +unset ADMIN_PASSWORD ``` Start or update the app: @@ -118,10 +127,12 @@ ADMIN_PASSWORD_HASH= SESSION_SECRET=replace_with_a_long_random_secret ``` -Generate a bcrypt password hash: +Generate a bcrypt password hash without putting the password in shell history: ```bash -node -e "const bcrypt=require('bcryptjs'); bcrypt.hash(process.argv[1], 12).then(console.log)" 'your-password' +read -s ADMIN_PASSWORD; export ADMIN_PASSWORD +node -e 'require("bcryptjs").hash(process.env.ADMIN_PASSWORD, 12).then(console.log)' +unset ADMIN_PASSWORD ``` For local development only, `ADMIN_PASSWORD` can be used as a fallback. @@ -142,7 +153,26 @@ window.bufferdash.track("tool_used", { }); ``` -The tracker does not collect form inputs, cookies, localStorage contents, passwords, or URL fragments. +The tracker does not collect form inputs, cookies, localStorage contents, passwords, URL fragments, or query strings by default. Add `data-include-query` only when you have audited every tracked URL and intentionally want query-string analytics. + +## GeoIP + +When `TRUST_PROXY=true`, BufferDash uses trusted Cloudflare or Vercel location headers when present. On a normal VPS, set `IPINFO_TOKEN` to enable server-side enrichment. `IPINFO_TIER=lite` provides country and ASN data; `core` provides city and region as well. IP lookups are cached for six hours and private network addresses are never sent to the provider. + +GeoIP sends a visitor IP to the configured provider. Leave `IPINFO_TOKEN` empty if that does not fit your privacy policy. + +## Optional Host Security Events + +Set `ENABLE_LOG_INGESTION=true` and generate a distinct `INGESTION_SECRET` of at least 32 characters. Trusted host tooling can then send a single structured event or a batch of up to 50 events: + +```bash +curl -fsS -X POST https://dash.example.com/api/security/ingest \ + -H "Authorization: Bearer $INGESTION_SECRET" \ + -H "Content-Type: application/json" \ + --data '{"source":"fail2ban","type":"ban","message":"Banned repeated SSH failures","ip":"203.0.113.10"}' +``` + +Keep ingestion disabled unless you actively use it. The endpoint returns `404` when disabled or unauthorized. ## Environment Variables @@ -159,6 +189,9 @@ See `.env.example` for the full set. The most important production values are: - `ANONYMIZE_IP` - `TRUST_PROXY` - `ENFORCE_TRACKING_ORIGIN` +- `IPINFO_TOKEN` and `IPINFO_TIER` (optional) +- `ENABLE_LOG_INGESTION` and `INGESTION_SECRET` (optional) +- `METRICS_INTERVAL_SECONDS` and `CLEANUP_INTERVAL_HOURS` Settings are environment-driven in v1 so secrets and operational toggles are not exposed through a browser editor. @@ -168,6 +201,8 @@ BufferDash can log IP addresses and user agents. If you deploy it, disclose anal Data retention cleanup is available from `/settings`. It removes old events, sessions, orphaned visitor identifiers, traffic flags, and runtime metrics. The default retention window is controlled by `DATA_RETENTION_DAYS`. +The background worker performs this cleanup automatically and has its own Docker health check. The manual settings action remains available for immediate cleanup. + ## Security Notes - `.env` is ignored by Git. @@ -176,6 +211,7 @@ Data retention cleanup is available from `/settings`. It removes old events, ses - Production rejects placeholder secrets, non-HTTPS `APP_URL` values, and missing bcrypt admin hashes. - `/api/track` validates payloads with Zod and rate limits by IP. - Tracking requests are restricted to each site's configured domain by default. This limits accidental or casual key reuse, though browser origin headers are not a substitute for a private credential. +- Query strings and fragments are excluded from tracked URLs by default. - Public APIs never return analytics data. - Client-submitted IP, country, city, browser, OS, and device values are not trusted. - v1 intentionally does not include a browser terminal, arbitrary file browser, or `.env` editor. @@ -200,13 +236,14 @@ server { ## Roadmap -- GeoIP enrichment with IPinfo or MaxMind -- Optional reverse-proxy, Fail2Ban, and SSH log ingestion with a dedicated least-privilege agent +- Offline MaxMind GeoIP database support +- Packaged least-privilege host agents for common SSH and reverse-proxy formats - User roles and TOTP - Read-only dashboards - Scheduled uptime, latency, HTTP status, and TLS-expiry monitoring -- Background runtime metric and retention workers -- Public screenshots and deployment guides +- Public screenshots + +See [DEPLOYMENT.md](DEPLOYMENT.md) for the production checklist, [SECURITY.md](SECURITY.md) for reporting and operational boundaries, and [CONTRIBUTING.md](CONTRIBUTING.md) for development guidance. ## License diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..4ba5870 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,17 @@ +# Security Policy + +## Reporting + +Please report suspected vulnerabilities privately to the repository owner rather than opening a public issue. Include affected versions, reproduction steps, impact, and any suggested mitigation. Do not include real visitor data or credentials. + +## Operational boundaries + +- The public site key is an ingestion identifier, not a credential. Origin checks reduce casual misuse but cannot stop forged server-side requests. +- Keep the dashboard behind HTTPS and use a long, unique admin password. An additional access layer such as a VPN or identity-aware proxy is recommended for high-risk deployments. +- PostgreSQL must remain on the internal Docker network. Port 3000 must remain bound to localhost. +- `TRUST_PROXY=true` is safe only when the reverse proxy overwrites forwarding headers and the app is not directly reachable. +- GeoIP is optional because it sends visitor IP addresses to the configured provider. +- Host log ingestion is disabled by default and requires a separate random bearer secret. +- Runtime metrics describe what the container can observe and are not a substitute for full VPS monitoring. + +If any credential is committed or exposed, remove it from use and rotate it immediately. Rewriting Git history or making the repository private is not sufficient. diff --git a/app/(dashboard)/dashboard/page.tsx b/app/(dashboard)/dashboard/page.tsx index a010b7c..c3d2d30 100644 --- a/app/(dashboard)/dashboard/page.tsx +++ b/app/(dashboard)/dashboard/page.tsx @@ -1,13 +1,16 @@ import { TimelineChart, TopBarChart } from "@/components/Charts"; import { MetricCard } from "@/components/MetricCard"; import { PageHeader } from "@/components/PageHeader"; +import { RangeSelector } from "@/components/RangeSelector"; import { TopList } from "@/components/TopList"; import { getDashboardData, getRecentEvents } from "@/lib/data"; import { compactDuration, numberFormat, shortDate } from "@/lib/format"; import { maskIp } from "@/lib/ip"; +import { parseRange, rangeLabel } from "@/lib/range"; -export default async function DashboardPage() { - const [data, recentEvents] = await Promise.all([getDashboardData(), getRecentEvents(undefined, 10)]); +export default async function DashboardPage({ searchParams }: { searchParams: Promise<{ range?: string }> }) { + const range = parseRange((await searchParams).range); + const [data, recentEvents] = await Promise.all([getDashboardData(undefined, range), getRecentEvents(undefined, 10)]); const { overview } = data; return ( @@ -17,17 +20,19 @@ export default async function DashboardPage() { title="Traffic, health, and security at a glance" description="A live command center for buffer.lol and any other site you add." /> +
- - + + - + +

Requests over time

- Last 24 hours + {rangeLabel(range)}
@@ -41,13 +46,17 @@ export default async function DashboardPage() { - + + + + +

Recent events

- {numberFormat(overview.securityEvents)} traffic flags today + {numberFormat(overview.securityEvents)} traffic flags · {rangeLabel(range)}
diff --git a/app/(dashboard)/live/page.tsx b/app/(dashboard)/live/page.tsx index c474bf5..89b6fd8 100644 --- a/app/(dashboard)/live/page.tsx +++ b/app/(dashboard)/live/page.tsx @@ -1,4 +1,5 @@ import { PageHeader } from "@/components/PageHeader"; +import { AutoRefresh } from "@/components/AutoRefresh"; import { getLiveVisitors } from "@/lib/data"; import { shortDate } from "@/lib/format"; import { maskIp } from "@/lib/ip"; @@ -9,15 +10,17 @@ export default async function LivePage() { return ( <> +
- + {visitors.map((event) => ( + @@ -26,7 +29,7 @@ export default async function LivePage() { ))} - {visitors.length === 0 && } + {visitors.length === 0 && }
TimeIPSitePageReferrerBrowserOSDevice
TimeIPLocationSitePageReferrerBrowserOSDevice
{shortDate(event.createdAt)} {maskIp(event.ipAddress)}{[event.city, event.country].filter(Boolean).join(", ") || "Unknown"} {event.site.name} {event.path || event.type} {event.referrerDomain || "Direct"}{event.device || "Unknown"}
No active visitors right now.
No active visitors right now.
diff --git a/app/(dashboard)/security/page.tsx b/app/(dashboard)/security/page.tsx index 5cadf86..c6064e4 100644 --- a/app/(dashboard)/security/page.tsx +++ b/app/(dashboard)/security/page.tsx @@ -1,29 +1,32 @@ import { PageHeader } from "@/components/PageHeader"; import { TopList } from "@/components/TopList"; -import { getSecurityEvents, getSuspiciousIps } from "@/lib/data"; +import { getSecurityEventCounts, getSecurityEvents, getSuspiciousIps } from "@/lib/data"; import { shortDate } from "@/lib/format"; import { maskIp } from "@/lib/ip"; export default async function SecurityPage() { - const [events, suspiciousIps] = await Promise.all([getSecurityEvents(), getSuspiciousIps()]); + const [events, suspiciousIps, signalCounts] = await Promise.all([getSecurityEvents(), getSuspiciousIps(), getSecurityEventCounts()]); return ( <> - +
+
-

Tracker coverage

+

Signal coverage

Known bots and crawlers

Unusual paths reached by JavaScript-capable clients

Empty or abnormal user agents

Tracking endpoint rate-limit enforcement

+

Failed and rate-limited admin logins

+

Optional SSH, reverse-proxy, and Fail2Ban events

-

Traffic flags

+

Security events

diff --git a/app/(dashboard)/server/page.tsx b/app/(dashboard)/server/page.tsx index 6398af5..f5b91f1 100644 --- a/app/(dashboard)/server/page.tsx +++ b/app/(dashboard)/server/page.tsx @@ -2,6 +2,7 @@ import { ServerChart } from "@/components/Charts"; import { MetricCard } from "@/components/MetricCard"; import { PageHeader } from "@/components/PageHeader"; import { getServerMetrics } from "@/lib/server-metrics"; +import { bytes } from "@/lib/format"; export default async function ServerPage() { const { latest, history } = await getServerMetrics(); @@ -16,9 +17,11 @@ export default async function ServerPage() { + +
-

Resource history

Sampled when this page or its API is read
+

Resource history

Sampled every minute by the background worker
diff --git a/app/(dashboard)/settings/page.tsx b/app/(dashboard)/settings/page.tsx index d071c0a..8c9a8e7 100644 --- a/app/(dashboard)/settings/page.tsx +++ b/app/(dashboard)/settings/page.tsx @@ -20,6 +20,8 @@ export default async function SettingsPage() {

Bot filtering{env.filterBots ? "On" : "Off"}

Retention default{env.dataRetentionDays} days

Runtime metrics{env.enableServerMetrics ? "On" : "Off"}

+

GeoIP{env.ipinfoToken ? `IPinfo ${env.ipinfoTier}` : "Proxy headers only"}

+

Host log ingestion{env.enableLogIngestion ? "On" : "Off"}

diff --git a/app/(dashboard)/sites/[siteId]/page.tsx b/app/(dashboard)/sites/[siteId]/page.tsx index ab16784..bf7e21f 100644 --- a/app/(dashboard)/sites/[siteId]/page.tsx +++ b/app/(dashboard)/sites/[siteId]/page.tsx @@ -3,41 +3,51 @@ import { TimelineChart, TopBarChart } from "@/components/Charts"; import { CopySnippet } from "@/components/CopySnippet"; import { MetricCard } from "@/components/MetricCard"; import { PageHeader } from "@/components/PageHeader"; +import { RangeSelector } from "@/components/RangeSelector"; import { TopList } from "@/components/TopList"; import { getDashboardData, getRecentEvents, getSite } from "@/lib/data"; import { compactDuration, numberFormat, shortDate } from "@/lib/format"; import { maskIp } from "@/lib/ip"; import { trackingSnippet } from "@/lib/snippet"; +import { parseRange, rangeLabel } from "@/lib/range"; -export default async function SiteDetailPage({ params }: { params: Promise<{ siteId: string }> }) { +export default async function SiteDetailPage({ params, searchParams }: { params: Promise<{ siteId: string }>; searchParams: Promise<{ range?: string }> }) { const { siteId } = await params; + const range = parseRange((await searchParams).range); const site = await getSite(siteId); if (!site) notFound(); - const [data, recentEvents] = await Promise.all([getDashboardData(site.id), getRecentEvents(site.id, 30)]); + const [data, recentEvents] = await Promise.all([getDashboardData(site.id, range), getRecentEvents(site.id, 30)]); const { overview } = data; return ( <> +

Tracking snippet

- - + + + +
-

Last 24 hours

+

{rangeLabel(range)}

- + + + + +

Devices

@@ -47,19 +57,20 @@ export default async function SiteDetailPage({ params }: { params: Promise<{ sit

Visitor log

TimeTypeIPSourceMessage
- + {recentEvents.map((event) => ( + ))} - {recentEvents.length === 0 && } + {recentEvents.length === 0 && }
TimePathVisitorReferrerBrowserOS
TimePathVisitorLocationReferrerBrowserOS
{shortDate(event.createdAt)} {event.path || event.type} {maskIp(event.ipAddress)}{[event.city, event.country].filter(Boolean).join(", ") || "Unknown"} {event.referrerDomain || "Direct"} {event.browser || "Unknown"} {event.os || "Unknown"}
No events for this site yet.
No events for this site yet.
diff --git a/app/[...path]/page.tsx b/app/[...path]/page.tsx new file mode 100644 index 0000000..c7d2619 --- /dev/null +++ b/app/[...path]/page.tsx @@ -0,0 +1,23 @@ +import { headers } from "next/headers"; +import { notFound } from "next/navigation"; +import { isSuspiciousPath } from "@/lib/bot"; +import { env } from "@/lib/env"; +import { getClientIpFromHeaders } from "@/lib/ip"; +import { rateLimit } from "@/lib/rate-limit"; +import { recordSecurityEvent } from "@/lib/security-events"; + +export default async function UnknownPathPage({ params }: { params: Promise<{ path: string[] }> }) { + const path = `/${(await params).path.join("/")}`.slice(0, 500); + const headerStore = await headers(); + const ip = getClientIpFromHeaders(headerStore) || "0.0.0.0"; + if (rateLimit(`not-found:${ip}`, Math.min(env.trackingRateLimit, 30)).allowed) { + await recordSecurityEvent({ + source: "http", + type: isSuspiciousPath(path) ? "suspicious_path" : "not_found", + ip, + message: `Unknown path requested: ${path}`, + metadata: { path, userAgent: (headerStore.get("user-agent") || "").slice(0, 300) } + }); + } + notFound(); +} diff --git a/app/actions.ts b/app/actions.ts index 078f4f8..c5ebc1a 100644 --- a/app/actions.ts +++ b/app/actions.ts @@ -20,6 +20,7 @@ import { env } from "@/lib/env"; import { getClientIpFromHeaders } from "@/lib/ip"; import { prisma } from "@/lib/prisma"; import { rateLimit } from "@/lib/rate-limit"; +import { recordSecurityEvent } from "@/lib/security-events"; export type ActionState = { error?: string; @@ -39,11 +40,13 @@ export async function loginAction(_state: ActionState, formData: FormData): Prom const limit = rateLimit(`login:${ip}:${email}`, env.adminRateLimit); if (!limit.allowed) { + await recordSecurityEvent({ source: "auth", type: "login_rate_limited", ip, message: "Admin login rate limit exceeded", metadata: { email: email.slice(0, 120) } }); return { error: "Too many login attempts. Wait a minute and try again." }; } const isValid = await verifyAdminCredentials(email, password); if (!isValid) { + await recordSecurityEvent({ source: "auth", type: "login_failed", ip, message: "Invalid admin login attempt", metadata: { email: email.slice(0, 120) } }); return { error: "Invalid email or password." }; } diff --git a/app/api/security/ingest/route.ts b/app/api/security/ingest/route.ts new file mode 100644 index 0000000..2f537f3 --- /dev/null +++ b/app/api/security/ingest/route.ts @@ -0,0 +1,45 @@ +import crypto from "node:crypto"; +import { NextRequest, NextResponse } from "next/server"; +import { z } from "zod"; +import { env } from "@/lib/env"; +import { getClientIp } from "@/lib/ip"; +import { rateLimit } from "@/lib/rate-limit"; +import { recordSecurityEvent } from "@/lib/security-events"; + +export const ingestEventSchema = z.object({ + source: z.string().min(1).max(80), + type: z.string().min(1).max(80), + message: z.string().min(1).max(500), + ip: z.string().ip().optional().nullable(), + metadata: z.record(z.union([z.string().max(500), z.number(), z.boolean(), z.null()])) + .refine((value) => Object.keys(value).length <= 20, "Too many metadata fields") + .optional() +}); +const bodySchema = z.union([ingestEventSchema, z.array(ingestEventSchema).min(1).max(50)]); + +export async function POST(request: NextRequest) { + if (!env.enableLogIngestion || !authorized(request.headers.get("authorization"))) { + return NextResponse.json({ error: "not_found" }, { status: 404 }); + } + const callerIp = getClientIp(request); + if (!rateLimit(`ingest:${callerIp}`, 120).allowed) { + return NextResponse.json({ error: "rate_limited" }, { status: 429 }); + } + const raw = await request.text(); + if (raw.length > 65_536) return NextResponse.json({ error: "payload_too_large" }, { status: 413 }); + let body: unknown; + try { body = JSON.parse(raw); } catch { body = null; } + const parsed = bodySchema.safeParse(body); + if (!parsed.success) return NextResponse.json({ error: "invalid_payload" }, { status: 400 }); + const events = Array.isArray(parsed.data) ? parsed.data : [parsed.data]; + await Promise.all(events.map((event) => recordSecurityEvent({ ...event, ip: event.ip || callerIp }))); + return new NextResponse(null, { status: 204 }); +} + +function authorized(header: string | null) { + const supplied = header?.startsWith("Bearer ") ? header.slice(7) : ""; + const expected = env.ingestionSecret; + const left = Buffer.from(supplied); + const right = Buffer.from(expected); + return Boolean(supplied && expected && left.length === right.length && crypto.timingSafeEqual(left, right)); +} diff --git a/app/api/track/route.ts b/app/api/track/route.ts index 4178b2a..712322e 100644 --- a/app/api/track/route.ts +++ b/app/api/track/route.ts @@ -3,6 +3,7 @@ import { env } from "@/lib/env"; import { getClientIp } from "@/lib/ip"; import { rateLimit } from "@/lib/rate-limit"; import { recordTrackingEvent, trackSchema } from "@/lib/tracking"; +import { recordSecurityEvent } from "@/lib/security-events"; export const dynamic = "force-dynamic"; @@ -20,12 +21,17 @@ export async function POST(request: NextRequest) { const ip = getClientIp(request); const limit = rateLimit(`track:${ip}`, env.trackingRateLimit); if (!limit.allowed) { + await recordSecurityEvent({ source: "tracking", type: "rate_limited", ip, message: "Tracking rate limit exceeded" }); return NextResponse.json({ error: "rate_limited" }, { status: 429, headers: corsHeaders }); } let json: unknown; try { - json = await request.json(); + const body = await request.text(); + if (body.length > 32_768) { + return NextResponse.json({ error: "payload_too_large" }, { status: 413, headers: corsHeaders }); + } + json = JSON.parse(body); } catch { return NextResponse.json({ error: "invalid_json" }, { status: 400, headers: corsHeaders }); } @@ -39,7 +45,8 @@ export async function POST(request: NextRequest) { parsed.data, ip, request.headers.get("user-agent"), - request.headers.get("origin") + request.headers.get("origin"), + request.headers ); if (!result.ok) { const error = result.status === 403 ? "origin_not_allowed" : "unknown_site"; diff --git a/app/globals.css b/app/globals.css index 34d5cc4..ade17d4 100644 --- a/app/globals.css +++ b/app/globals.css @@ -41,6 +41,46 @@ button, input { font: inherit; } button { color: inherit; } h1, h2, p { margin: 0; } +.range-selector { + grid-column: 1 / -1; + display: flex; + flex-wrap: wrap; + gap: 0.45rem; + margin: -0.25rem 0 1.25rem; +} + +.range-selector a { + padding: 0.45rem 0.75rem; + border: 1px solid var(--line); + border-radius: 999px; + color: var(--muted); + font-size: 0.85rem; +} + +.range-selector a:hover, .range-selector a.active { + border-color: rgba(167, 139, 250, 0.55); + background: rgba(167, 139, 250, 0.12); + color: var(--text); +} + +.live-indicator { + grid-column: 1 / -1; + display: inline-flex; + align-items: center; + gap: 0.5rem; + margin: -0.25rem 0 1.25rem; + color: var(--muted); + font-size: 0.85rem; +} + +.live-indicator i { + width: 0.55rem; + height: 0.55rem; + border-radius: 50%; + background: var(--green); + box-shadow: 0 0 0.7rem rgba(52, 211, 153, 0.75); +} + .dash-shell { display: grid; grid-template-columns: 260px minmax(0, 1fr); @@ -225,6 +265,11 @@ label span { gap: 1rem; } +.dashboard-grid > .panel, +.site-list > .panel { + grid-column: auto; +} + .chart-frame { width: 100%; min-height: 260px; diff --git a/app/tracker.js/route.ts b/app/tracker.js/route.ts index 03cda0d..0901434 100644 --- a/app/tracker.js/route.ts +++ b/app/tracker.js/route.ts @@ -51,12 +51,23 @@ export const tracker = String.raw` function payload(type, metadata) { var url = new URL(window.location.href); url.hash = ""; + var includeQuery = script.hasAttribute && script.hasAttribute("data-include-query"); + if (!includeQuery) url.search = ""; + var referrer = null; + if (document.referrer) { + try { + var referrerUrl = new URL(document.referrer); + referrerUrl.hash = ""; + if (!includeQuery) referrerUrl.search = ""; + referrer = referrerUrl.toString(); + } catch (error) {} + } return { siteId: siteId, type: type || "pageview", path: window.location.pathname, url: url.toString(), - referrer: document.referrer || null, + referrer: referrer, title: document.title || null, screenWidth: window.screen && window.screen.width, screenHeight: window.screen && window.screen.height, @@ -101,6 +112,12 @@ export const tracker = String.raw` if (!target) return; var href = target.href; if (href && target.hostname !== window.location.hostname) { + try { + var outboundUrl = new URL(href); + outboundUrl.hash = ""; + if (!(script.hasAttribute && script.hasAttribute("data-include-query"))) outboundUrl.search = ""; + href = outboundUrl.toString(); + } catch (error) {} send(payload("outbound_click", { href: href }), true); } }); diff --git a/components/AutoRefresh.tsx b/components/AutoRefresh.tsx new file mode 100644 index 0000000..b8c042f --- /dev/null +++ b/components/AutoRefresh.tsx @@ -0,0 +1,13 @@ +"use client"; + +import { useEffect } from "react"; +import { useRouter } from "next/navigation"; + +export function AutoRefresh({ seconds = 10 }: { seconds?: number }) { + const router = useRouter(); + useEffect(() => { + const timer = window.setInterval(() => router.refresh(), seconds * 1000); + return () => window.clearInterval(timer); + }, [router, seconds]); + return Refreshing every {seconds}s; +} diff --git a/components/RangeSelector.tsx b/components/RangeSelector.tsx new file mode 100644 index 0000000..81feb72 --- /dev/null +++ b/components/RangeSelector.tsx @@ -0,0 +1,19 @@ +import Link from "next/link"; +import { rangeOptions, type RangeKey } from "@/lib/range"; + +export function RangeSelector({ selected, basePath }: { selected: RangeKey; basePath: string }) { + return ( + + ); +} diff --git a/components/Shell.tsx b/components/Shell.tsx index b680242..1ef0173 100644 --- a/components/Shell.tsx +++ b/components/Shell.tsx @@ -6,7 +6,7 @@ const navItems = [ ["Overview", "/dashboard"], ["Sites", "/sites"], ["Live", "/live"], - ["Traffic quality", "/security"], + ["Security", "/security"], ["Runtime", "/server"], ["Event stream", "/logs"], ["Settings", "/settings"] diff --git a/docker-compose.yml b/docker-compose.yml index f49d97a..359f673 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,9 +3,9 @@ services: build: . restart: unless-stopped env_file: - - .env + - ${ENV_FILE:-.env} ports: - - "127.0.0.1:3000:3000" + - "${BIND_ADDRESS:-127.0.0.1}:${APP_PORT:-3000}:3000" depends_on: postgres: condition: service_healthy @@ -24,11 +24,30 @@ services: target: migrate restart: "no" env_file: - - .env + - ${ENV_FILE:-.env} depends_on: postgres: condition: service_healthy + worker: + build: + context: . + target: worker + restart: unless-stopped + env_file: + - ${ENV_FILE:-.env} + depends_on: + postgres: + condition: service_healthy + migrate: + condition: service_completed_successfully + healthcheck: + test: ["CMD-SHELL", "node -e \"require('fs').stat('/tmp/bufferdash-worker-heartbeat',(e,s)=>process.exit(!e&&Date.now()-s.mtimeMs<120000?0:1))\""] + interval: 30s + timeout: 5s + retries: 3 + start_period: 20s + postgres: image: postgres:16 restart: unless-stopped diff --git a/lib/data.ts b/lib/data.ts index 4e0f67b..9d51244 100644 --- a/lib/data.ts +++ b/lib/data.ts @@ -2,17 +2,9 @@ import "server-only"; import { Prisma, type Event, type SecurityEvent, type Site } from "@prisma/client"; import { prisma } from "@/lib/prisma"; +import { rangeHours, rangeStart, type RangeKey } from "@/lib/range"; -export type TopRow = { - label: string; - value: number; -}; - -export function startOfToday(now = new Date()) { - const date = new Date(now); - date.setHours(0, 0, 0, 0); - return date; -} +export type TopRow = { label: string; value: number }; export function sinceHours(hours: number, now = new Date()) { return new Date(now.getTime() - hours * 60 * 60 * 1000); @@ -21,50 +13,47 @@ export function sinceHours(hours: number, now = new Date()) { export async function getSites() { return prisma.site.findMany({ orderBy: { createdAt: "desc" }, - include: { - _count: { select: { events: true, sessions: true } } - } + include: { _count: { select: { events: true, sessions: true } } } }); } export async function getSite(siteId: string) { return prisma.site.findUnique({ where: { id: siteId }, - include: { - _count: { select: { events: true, sessions: true } } - } + include: { _count: { select: { events: true, sessions: true } } } }); } -export async function getOverview(siteId?: string) { +export async function getOverview(siteId: string | undefined, range: RangeKey) { const now = new Date(); - const today = startOfToday(now); + const start = rangeStart(range, now); const fiveMinutesAgo = new Date(now.getTime() - 5 * 60 * 1000); - const timelineStart = startOfHour(sinceHours(23, now)); const where = siteId ? { siteId } : {}; - const todayWhere = { ...where, createdAt: { gte: today } }; + const periodWhere = { ...where, createdAt: { gte: start, lte: now } }; - const [pageViewsToday, uniqueVisitorsToday, sessionsToday, liveVisitors, sessionDuration, timeline, sites, securityEvents] = + const [pageViews, uniqueVisitors, sessions, liveVisitors, sessionDuration, bounceRate, timeline, sites, securityEvents] = await Promise.all([ - prisma.event.count({ where: { ...todayWhere, type: "pageview" } }), - countDistinctVisitorsSince(today, siteId), - prisma.session.count({ where: { ...where, startedAt: { gte: today } } }), - countDistinctVisitorsSince(fiveMinutesAgo, siteId), + prisma.event.count({ where: { ...periodWhere, type: "pageview" } }), + countDistinctVisitors(start, now, siteId), + prisma.session.count({ where: { ...where, startedAt: { gte: start, lte: now } } }), + countDistinctVisitors(fiveMinutesAgo, now, siteId), prisma.session.aggregate({ - where: { ...where, startedAt: { gte: today }, durationMs: { not: null } }, + where: { ...where, startedAt: { gte: start, lte: now }, durationMs: { not: null } }, _avg: { durationMs: true } }), - getHourlyTimeline(timelineStart, now, siteId), + getBounceRate(start, now, siteId), + getTimeline(start, now, siteId, range), prisma.site.count(), - prisma.securityEvent.count({ where: { createdAt: { gte: today } } }) + prisma.securityEvent.count({ where: { createdAt: { gte: start, lte: now } } }) ]); return { - pageViewsToday, - uniqueVisitorsToday, - sessionsToday, + pageViews, + uniqueVisitors, + sessions, liveVisitors, averageSessionDuration: sessionDuration._avg.durationMs || 0, + bounceRate, sites, securityEvents, timeline @@ -72,14 +61,16 @@ export async function getOverview(siteId?: string) { } export async function topByField( - field: "path" | "referrerDomain" | "country" | "browser" | "os" | "device", - siteId?: string, + field: "path" | "referrerDomain" | "country" | "city" | "browser" | "os" | "device", + siteId: string | undefined, + start: Date, limit = 6 ): Promise { const rows = await prisma.event.groupBy({ by: [field], where: { ...(siteId ? { siteId } : {}), + createdAt: { gte: start }, type: "pageview", [field]: { not: null } }, @@ -88,39 +79,38 @@ export async function topByField( take: limit }); - return rows.map((row) => ({ - label: String(row[field] || "Unknown"), - value: row._count._all - })); + return rows.map((row) => ({ label: String(row[field] || "Unknown"), value: row._count._all })); } -export async function getTopTools(siteId?: string, limit = 6): Promise { +export async function getTopTools(siteId: string | undefined, start: Date, limit = 6): Promise { const rows = await prisma.$queryRaw>(Prisma.sql` SELECT "metadata"->>'tool' AS "label", COUNT(*) AS "count" FROM "Event" WHERE "type" = 'tool_used' - AND "createdAt" >= ${sinceHours(24)} + AND "createdAt" >= ${start} AND "metadata"->>'tool' IS NOT NULL ${siteSqlFilter(siteId)} GROUP BY 1 ORDER BY 2 DESC LIMIT ${limit} `); - return rows.map((row) => ({ label: row.label, value: Number(row.count) })); } -export async function getDashboardData(siteId?: string) { - const [overview, topPages, referrers, browsers, devices, topTools] = await Promise.all([ - getOverview(siteId), - topByField("path", siteId), - topByField("referrerDomain", siteId), - topByField("browser", siteId), - topByField("device", siteId), - getTopTools(siteId) +export async function getDashboardData(siteId: string | undefined, range: RangeKey) { + const start = rangeStart(range); + const [overview, topPages, referrers, countries, cities, browsers, operatingSystems, devices, topTools] = await Promise.all([ + getOverview(siteId, range), + topByField("path", siteId, start), + topByField("referrerDomain", siteId, start), + topByField("country", siteId, start), + topByField("city", siteId, start), + topByField("browser", siteId, start), + topByField("os", siteId, start), + topByField("device", siteId, start), + getTopTools(siteId, start) ]); - - return { overview, topPages, referrers, browsers, devices, topTools }; + return { overview, topPages, referrers, countries, cities, browsers, operatingSystems, devices, topTools }; } export async function getRecentEvents(siteId?: string, limit = 40) { @@ -133,17 +123,12 @@ export async function getRecentEvents(siteId?: string, limit = 40) { } export async function getLiveVisitors(siteId?: string) { - const now = new Date(); const events = await prisma.event.findMany({ - where: { - ...(siteId ? { siteId } : {}), - createdAt: { gte: new Date(now.getTime() - 5 * 60 * 1000) } - }, + where: { ...(siteId ? { siteId } : {}), createdAt: { gte: sinceHours(5 / 60) } }, orderBy: { createdAt: "desc" }, include: { site: true }, - take: 100 + take: 200 }); - const seen = new Set(); return events.filter((event) => { const key = event.visitorId || event.ipHash || event.id; @@ -153,11 +138,19 @@ export async function getLiveVisitors(siteId?: string) { }); } -export async function getSecurityEvents(limit = 80) { - return prisma.securityEvent.findMany({ - orderBy: { createdAt: "desc" }, - take: limit +export async function getSecurityEvents(limit = 100) { + return prisma.securityEvent.findMany({ orderBy: { createdAt: "desc" }, take: limit }); +} + +export async function getSecurityEventCounts() { + const rows = await prisma.securityEvent.groupBy({ + by: ["type"], + where: { createdAt: { gte: sinceHours(24) } }, + _count: { _all: true }, + orderBy: { _count: { type: "desc" } }, + take: 8 }); + return rows.map((row) => ({ label: row.type.replaceAll("_", " "), value: row._count._all })); } export async function getSuspiciousIps() { @@ -168,26 +161,17 @@ export async function getSuspiciousIps() { orderBy: { _count: { ipHash: "desc" } }, take: 10 }); - - return rows.map((row) => ({ - label: row.ipHash?.slice(0, 12) || "unknown", - value: row._count._all - })); + return rows.map((row) => ({ label: row.ipHash?.slice(0, 12) || "unknown", value: row._count._all })); } -export async function getAppLogs() { +export async function getAppLogs(limit = 100) { const [security, tracking] = await Promise.all([ - prisma.securityEvent.findMany({ orderBy: { createdAt: "desc" }, take: 40 }), - prisma.event.findMany({ orderBy: { createdAt: "desc" }, include: { site: true }, take: 40 }) + prisma.securityEvent.findMany({ orderBy: { createdAt: "desc" }, take: limit }), + prisma.event.findMany({ orderBy: { createdAt: "desc" }, include: { site: true }, take: limit }) ]); - return [ ...security.map((event: SecurityEvent) => ({ - id: event.id, - createdAt: event.createdAt, - source: event.source, - type: event.type, - message: event.message + id: event.id, createdAt: event.createdAt, source: event.source, type: event.type, message: event.message })), ...tracking.map((event: Event & { site: Site }) => ({ id: event.id, @@ -196,73 +180,77 @@ export async function getAppLogs() { type: event.type, message: `${event.path || event.url || "event"} from ${event.browser || "unknown browser"}` })) - ].sort((a, b) => b.createdAt.getTime() - a.createdAt.getTime()).slice(0, 80); + ].sort((a, b) => b.createdAt.getTime() - a.createdAt.getTime()).slice(0, limit); } -type HourlyTimelineRow = { - hour: Date; - pageviews: bigint | number; - events: bigint | number; -}; +type TimelineRow = { bucket: Date; pageviews: bigint | number; events: bigint | number }; function siteSqlFilter(siteId?: string) { return siteId ? Prisma.sql`AND "siteId" = ${siteId}` : Prisma.empty; } -async function countDistinctVisitorsSince(since: Date, siteId?: string) { +async function countDistinctVisitors(start: Date, end: Date, siteId?: string) { const rows = await prisma.$queryRaw>(Prisma.sql` - SELECT COUNT(DISTINCT "visitorId") AS "count" - FROM "Event" - WHERE "createdAt" >= ${since} - AND "visitorId" IS NOT NULL - ${siteSqlFilter(siteId)} + SELECT COUNT(DISTINCT "visitorId") AS "count" FROM "Event" + WHERE "createdAt" BETWEEN ${start} AND ${end} AND "visitorId" IS NOT NULL ${siteSqlFilter(siteId)} `); - return Number(rows[0]?.count || 0); } -async function getHourlyTimeline(start: Date, now: Date, siteId?: string) { - const rows = await prisma.$queryRaw(Prisma.sql` - SELECT - date_trunc('hour', "createdAt") AS "hour", - COUNT(*) FILTER (WHERE "type" = 'pageview') AS "pageviews", - COUNT(*) AS "events" - FROM "Event" - WHERE "createdAt" >= ${start} - ${siteSqlFilter(siteId)} - GROUP BY 1 - ORDER BY 1 ASC +async function getBounceRate(start: Date, end: Date, siteId?: string) { + const rows = await prisma.$queryRaw>(Prisma.sql` + WITH session_views AS ( + SELECT "sessionId", COUNT(*) FILTER (WHERE "type" = 'pageview') AS views + FROM "Event" + WHERE "createdAt" BETWEEN ${start} AND ${end} AND "sessionId" IS NOT NULL ${siteSqlFilter(siteId)} + GROUP BY "sessionId" + ) + SELECT COUNT(*) AS sessions, COUNT(*) FILTER (WHERE views <= 1) AS bounced FROM session_views `); + const sessions = Number(rows[0]?.sessions || 0); + return sessions ? Math.round((Number(rows[0]?.bounced || 0) / sessions) * 100) : 0; +} - return buildHourlyTimeline(rows, now); +async function getTimeline(start: Date, now: Date, siteId: string | undefined, range: RangeKey) { + const isHourly = rangeHours(range) <= 48; + const bucketExpression = isHourly ? Prisma.sql`date_trunc('hour', "createdAt")` : Prisma.sql`date_trunc('day', "createdAt")`; + const rows = await prisma.$queryRaw(Prisma.sql` + SELECT ${bucketExpression} AS bucket, + COUNT(*) FILTER (WHERE "type" = 'pageview') AS pageviews, COUNT(*) AS events + FROM "Event" WHERE "createdAt" BETWEEN ${start} AND ${now} ${siteSqlFilter(siteId)} + GROUP BY 1 ORDER BY 1 ASC + `); + return buildTimeline(rows, start, now, isHourly); } -function startOfHour(date: Date) { - const hour = new Date(date); - hour.setMinutes(0, 0, 0); - return hour; +function floorDate(date: Date, hourly: boolean) { + const value = new Date(date); + if (hourly) value.setMinutes(0, 0, 0); + else value.setHours(0, 0, 0, 0); + return value; } -function buildHourlyTimeline(rows: HourlyTimelineRow[], now: Date) { - const buckets = Array.from({ length: 24 }, (_, index) => { - const date = startOfHour(new Date(now.getTime() - (23 - index) * 60 * 60 * 1000)); - return { +function buildTimeline(rows: TimelineRow[], start: Date, now: Date, hourly: boolean) { + const step = hourly ? 60 * 60 * 1000 : 24 * 60 * 60 * 1000; + const first = floorDate(start, hourly); + const last = floorDate(now, hourly); + const buckets = []; + for (let time = first.getTime(); time <= last.getTime(); time += step) { + const date = new Date(time); + buckets.push({ key: date.toISOString(), - time: date.toLocaleTimeString("en-US", { hour: "numeric" }), + time: hourly ? date.toLocaleTimeString("en-US", { hour: "numeric" }) : date.toLocaleDateString("en-US", { month: "short", day: "numeric" }), pageviews: 0, events: 0 - }; - }); - - const indexByKey = new Map(buckets.map((bucket, index) => [bucket.key, index])); - + }); + } + const byKey = new Map(buckets.map((bucket) => [bucket.key, bucket])); for (const row of rows) { - const date = startOfHour(new Date(row.hour)); - const index = indexByKey.get(date.toISOString()); - if (index === undefined) continue; - buckets[index].events = Number(row.events || 0); - buckets[index].pageviews = Number(row.pageviews || 0); + const bucket = byKey.get(floorDate(new Date(row.bucket), hourly).toISOString()); + if (bucket) { + bucket.events = Number(row.events || 0); + bucket.pageviews = Number(row.pageviews || 0); + } } - return buckets; } diff --git a/lib/env.ts b/lib/env.ts index 3df109a..342eb44 100644 --- a/lib/env.ts +++ b/lib/env.ts @@ -12,7 +12,11 @@ export const env = { adminRateLimit: Number(process.env.RATE_LIMIT_ADMIN_PER_MINUTE || 60), enableServerMetrics: process.env.ENABLE_SERVER_METRICS === "true", dataRetentionDays: Number(process.env.DATA_RETENTION_DAYS || 90), - filterBots: process.env.FILTER_BOTS === "true" + filterBots: process.env.FILTER_BOTS === "true", + ipinfoToken: process.env.IPINFO_TOKEN || "", + ipinfoTier: process.env.IPINFO_TIER === "core" ? "core" as const : "lite" as const, + enableLogIngestion: process.env.ENABLE_LOG_INGESTION === "true", + ingestionSecret: process.env.INGESTION_SECRET || "" }; export function isProduction() { @@ -57,6 +61,9 @@ export function assertProductionEnv() { if (!Number.isFinite(env.adminRateLimit) || env.adminRateLimit < 1) { errors.push("RATE_LIMIT_ADMIN_PER_MINUTE must be a positive number"); } + if (env.enableLogIngestion && (env.ingestionSecret.length < 32 || looksLikePlaceholder(env.ingestionSecret))) { + errors.push("INGESTION_SECRET must be a random value of at least 32 characters when log ingestion is enabled"); + } if (errors.length > 0) { throw new Error(`Invalid production configuration:\n- ${errors.join("\n- ")}`); diff --git a/lib/format.ts b/lib/format.ts index 78d643d..2f1b507 100644 --- a/lib/format.ts +++ b/lib/format.ts @@ -23,3 +23,13 @@ export function shortDate(date: Date) { minute: "2-digit" }).format(date); } + +export function bytes(value: number | bigint | null | undefined) { + const amount = Number(value || 0); + if (amount < 1024) return `${amount} B`; + const units = ["KB", "MB", "GB", "TB"]; + let result = amount / 1024; + let index = 0; + while (result >= 1024 && index < units.length - 1) { result /= 1024; index += 1; } + return `${result.toFixed(result >= 10 ? 0 : 1)} ${units[index]}`; +} diff --git a/lib/geo.ts b/lib/geo.ts new file mode 100644 index 0000000..ca13739 --- /dev/null +++ b/lib/geo.ts @@ -0,0 +1,77 @@ +import net from "node:net"; +import { env } from "./env"; + +export type GeoData = { country: string | null; region: string | null; city: string | null; asn: string | null; isp: string | null }; +const emptyGeo: GeoData = { country: null, region: null, city: null, asn: null, isp: null }; +const cache = new Map(); + +function headerValue(headers: Headers, names: string[]) { + for (const name of names) { + const value = headers.get(name)?.trim(); + if (value) { + try { return decodeURIComponent(value); } catch { return value; } + } + } + return null; +} + +export function geoFromTrustedHeaders(headers: Headers): GeoData { + if (!env.trustProxy) return emptyGeo; + const country = headerValue(headers, ["cf-ipcountry", "x-vercel-ip-country", "x-geo-country"]); + return { + country: country && country !== "XX" ? country.slice(0, 120) : null, + region: headerValue(headers, ["x-vercel-ip-country-region", "x-geo-region"])?.slice(0, 120) || null, + city: headerValue(headers, ["x-vercel-ip-city", "x-geo-city"])?.slice(0, 120) || null, + asn: headerValue(headers, ["cf-asn", "x-geo-asn"])?.slice(0, 80) || null, + isp: headerValue(headers, ["x-geo-isp"])?.slice(0, 180) || null + }; +} + +export async function resolveGeo(ip: string, headers: Headers): Promise { + const fromHeaders = geoFromTrustedHeaders(headers); + if (fromHeaders.country || fromHeaders.city || !env.ipinfoToken || !isPublicIp(ip)) return fromHeaders; + const cached = cache.get(ip); + if (cached && cached.expiresAt > Date.now()) return cached.value; + if (cache.size >= 10_000) cache.clear(); + + try { + const endpoint = env.ipinfoTier === "core" ? "lookup" : "lite"; + const response = await fetch(`https://api.ipinfo.io/${endpoint}/${encodeURIComponent(ip)}`, { + headers: { authorization: `Bearer ${env.ipinfoToken}`, accept: "application/json" }, + signal: AbortSignal.timeout(2_500) + }); + if (!response.ok) return fromHeaders; + const body = await response.json() as Record; + const value = parseIpinfo(body); + cache.set(ip, { value, expiresAt: Date.now() + 6 * 60 * 60 * 1000 }); + return value; + } catch { + return fromHeaders; + } +} + +function parseIpinfo(body: Record): GeoData { + const geo = typeof body.geo === "object" && body.geo ? body.geo as Record : {}; + const as = typeof body.as === "object" && body.as ? body.as as Record : {}; + return { + country: stringValue(geo.country || geo.country_code || body.country || body.country_code, 120), + region: stringValue(geo.region || body.region, 120), + city: stringValue(geo.city || body.city, 120), + asn: stringValue(as.asn || body.asn, 80), + isp: stringValue(as.name || body.as_name || body.isp, 180) + }; +} + +function stringValue(value: unknown, max: number) { + return typeof value === "string" && value.trim() ? value.trim().slice(0, max) : null; +} + +export function isPublicIp(ip: string) { + if (!net.isIP(ip) || ip === "0.0.0.0" || ip === "::1") return false; + if (net.isIPv4(ip)) { + const [a, b] = ip.split(".").map(Number); + return !(a === 10 || a === 127 || a === 0 || (a === 169 && b === 254) || (a === 172 && b >= 16 && b <= 31) || (a === 192 && b === 168)); + } + const value = ip.toLowerCase(); + return !(value.startsWith("fc") || value.startsWith("fd") || /^fe[89ab]/.test(value)); +} diff --git a/lib/range.ts b/lib/range.ts new file mode 100644 index 0000000..107e36e --- /dev/null +++ b/lib/range.ts @@ -0,0 +1,30 @@ +export const rangeOptions = [ + { key: "24h", label: "24 hours", hours: 24 }, + { key: "7d", label: "7 days", hours: 24 * 7 }, + { key: "30d", label: "30 days", hours: 24 * 30 }, + { key: "90d", label: "90 days", hours: 24 * 90 } +] as const; + +export type RangeKey = (typeof rangeOptions)[number]["key"]; + +export function parseRange(value: string | string[] | undefined): RangeKey { + const candidate = Array.isArray(value) ? value[0] : value; + return rangeOptions.some((option) => option.key === candidate) ? (candidate as RangeKey) : "24h"; +} + +export function rangeHours(range: RangeKey) { + return rangeOptions.find((option) => option.key === range)?.hours || 24; +} + +export function rangeLabel(range: RangeKey) { + return rangeOptions.find((option) => option.key === range)?.label || "24 hours"; +} + +export function rangeStart(range: RangeKey, now = new Date()) { + if (range === "24h") return new Date(now.getTime() - 24 * 60 * 60 * 1000); + const days = range === "7d" ? 7 : range === "30d" ? 30 : 90; + const start = new Date(now); + start.setHours(0, 0, 0, 0); + start.setDate(start.getDate() - (days - 1)); + return start; +} diff --git a/lib/security-events.ts b/lib/security-events.ts new file mode 100644 index 0000000..1aa472d --- /dev/null +++ b/lib/security-events.ts @@ -0,0 +1,18 @@ +import "server-only"; +import { hashIp, storedIp } from "@/lib/ip"; +import { prisma } from "@/lib/prisma"; + +export async function recordSecurityEvent(input: { source: string; type: string; ip?: string | null; message: string; metadata?: Record }) { + try { + const ip = input.ip || null; + await prisma.securityEvent.create({ + data: { + source: input.source.slice(0, 80), type: input.type.slice(0, 80), + ipAddress: ip ? storedIp(ip) : null, ipHash: ip ? hashIp(ip) : null, + message: input.message.slice(0, 500), metadata: input.metadata + } + }); + } catch { + // Telemetry must never make login or public error handling unavailable. + } +} diff --git a/lib/server-metrics.ts b/lib/server-metrics.ts index 17b8ac4..72c8c0e 100644 --- a/lib/server-metrics.ts +++ b/lib/server-metrics.ts @@ -2,6 +2,7 @@ import "server-only"; import os from "os"; import si from "systeminformation"; +import type { ServerMetric } from "@prisma/client"; import { env } from "@/lib/env"; import { prisma } from "@/lib/prisma"; @@ -51,11 +52,15 @@ export async function collectServerMetric() { } export async function getServerMetrics() { - const latest = await collectServerMetric(); - const history = await prisma.serverMetric.findMany({ + let history = await prisma.serverMetric.findMany({ orderBy: { createdAt: "desc" }, take: 48 }); + let latest: ServerMetric | null = history[0] || null; + if (env.enableServerMetrics && (!latest || Date.now() - latest.createdAt.getTime() > 2 * 60 * 1000)) { + latest = await collectServerMetric(); + history = await prisma.serverMetric.findMany({ orderBy: { createdAt: "desc" }, take: 48 }); + } return { latest, diff --git a/lib/tracking.ts b/lib/tracking.ts index e5babee..c337c82 100644 --- a/lib/tracking.ts +++ b/lib/tracking.ts @@ -7,6 +7,7 @@ import { detectBot, isSuspiciousPath } from "@/lib/bot"; import { assertProductionEnv, env } from "@/lib/env"; import { hashIp, storedIp } from "@/lib/ip"; import { isAllowedTrackingOrigin } from "@/lib/origin"; +import { resolveGeo } from "@/lib/geo"; import { prisma } from "@/lib/prisma"; export const trackSchema = z.object({ @@ -44,22 +45,23 @@ export function trimMetadata(metadata: Record | null | undefine ) as Prisma.InputJsonObject; } -function jsonValue(value: unknown): Prisma.InputJsonValue { +function jsonValue(value: unknown, depth = 0): Prisma.InputJsonValue { + if (depth >= 4) return "[truncated]"; if (value === null) return ""; if (typeof value === "number" || typeof value === "boolean") return value; if (typeof value === "string") return value.slice(0, 500); - if (Array.isArray(value)) return value.slice(0, 20).map(jsonValue); + if (Array.isArray(value)) return value.slice(0, 20).map((child) => jsonValue(child, depth + 1)); if (typeof value === "object") { return Object.fromEntries( Object.entries(value as Record) .slice(0, 20) - .map(([key, child]) => [key.slice(0, 80), jsonValue(child)]) + .map(([key, child]) => [key.slice(0, 80), jsonValue(child, depth + 1)]) ) as Prisma.InputJsonObject; } return String(value).slice(0, 120); } -export async function recordTrackingEvent(input: z.infer, ip: string, userAgent: string | null, origin: string | null) { +export async function recordTrackingEvent(input: z.infer, ip: string, userAgent: string | null, origin: string | null, headers = new Headers()) { assertProductionEnv(); const site = await prisma.site.findUnique({ where: { publicKey: input.siteId } }); if (!site) return { ok: false as const, status: 404 }; @@ -72,6 +74,7 @@ export async function recordTrackingEvent(input: z.infer, ip const bot = detectBot(userAgent); const ipHash = hashIp(ip); const persistedIp = storedIp(ip); + const geo = await resolveGeo(ip, headers); if (env.filterBots && bot.isBot) { return { ok: true as const, status: 204 }; @@ -116,6 +119,11 @@ export async function recordTrackingEvent(input: z.infer, ip referrerDomain: referrerDomain(input.referrer), ipAddress: persistedIp, ipHash, + country: geo.country, + region: geo.region, + city: geo.city, + asn: geo.asn, + isp: geo.isp, userAgent: userAgent || null, browser: ua.browser.name || null, os: ua.os.name || null, diff --git a/package.json b/package.json index ee2d53b..b6f9247 100644 --- a/package.json +++ b/package.json @@ -9,6 +9,7 @@ "lint": "eslint app components lib tests --max-warnings=0", "test": "vitest run", "typecheck": "tsc --noEmit", + "validate": "npm run lint && npm run typecheck && npm test && npm run build", "db:backup": "scripts/backup-postgres.sh", "db:restore": "scripts/restore-postgres.sh", "prisma:generate": "prisma generate", diff --git a/scripts/background-worker.mjs b/scripts/background-worker.mjs new file mode 100644 index 0000000..de2eff0 --- /dev/null +++ b/scripts/background-worker.mjs @@ -0,0 +1,75 @@ +import os from "node:os"; +import { writeFile } from "node:fs/promises"; +import { PrismaClient } from "@prisma/client"; +import si from "systeminformation"; + +const prisma = new PrismaClient(); +const metricsEnabled = process.env.ENABLE_SERVER_METRICS === "true"; +const retentionDays = boundedNumber(process.env.DATA_RETENTION_DAYS, 90, 1, 3650); +const metricsSeconds = boundedNumber(process.env.METRICS_INTERVAL_SECONDS, 60, 15, 3600); +const cleanupHours = boundedNumber(process.env.CLEANUP_INTERVAL_HOURS, 24, 1, 168); + +function boundedNumber(value, fallback, min, max) { + const number = Number(value || fallback); + return Number.isFinite(number) ? Math.min(max, Math.max(min, number)) : fallback; +} + +async function collectMetric() { + if (!metricsEnabled) return; + try { + const [load, mem, disks, networks] = await Promise.all([si.currentLoad(), si.mem(), si.fsSize(), si.networkStats()]); + const disk = disks.find((item) => item.mount === "/") || disks[0]; + const network = networks.reduce((total, item) => ({ rx: total.rx + item.rx_bytes, tx: total.tx + item.tx_bytes }), { rx: 0, tx: 0 }); + await prisma.serverMetric.create({ data: { + cpuPercent: load.currentLoad, + memoryUsedMb: (mem.total - mem.available) / 1024 / 1024, memoryTotalMb: mem.total / 1024 / 1024, + diskUsedGb: disk ? disk.used / 1024 / 1024 / 1024 : null, diskTotalGb: disk ? disk.size / 1024 / 1024 / 1024 : null, + load1: os.loadavg()[0], load5: os.loadavg()[1], load15: os.loadavg()[2], uptimeSeconds: Math.round(os.uptime()), + networkRxBytes: BigInt(Math.max(0, Math.round(network.rx))), networkTxBytes: BigInt(Math.max(0, Math.round(network.tx))) + } }); + } catch (error) { + console.error("metric collection failed", error instanceof Error ? error.message : error); + } +} + +async function cleanup() { + const cutoff = new Date(Date.now() - retentionDays * 24 * 60 * 60 * 1000); + try { + await prisma.$transaction(async (tx) => { + await tx.event.deleteMany({ where: { createdAt: { lt: cutoff } } }); + await tx.securityEvent.deleteMany({ where: { createdAt: { lt: cutoff } } }); + await tx.serverMetric.deleteMany({ where: { createdAt: { lt: cutoff } } }); + await tx.session.deleteMany({ where: { endedAt: { lt: cutoff } } }); + await tx.visitor.deleteMany({ where: { events: { none: {} }, sessions: { none: {} } } }); + }); + } catch (error) { + console.error("retention cleanup failed", error instanceof Error ? error.message : error); + } +} + +async function heartbeat() { + try { + await prisma.$queryRaw`SELECT 1`; + await writeFile("/tmp/bufferdash-worker-heartbeat", String(Date.now()), "utf8"); + } catch (error) { + console.error("worker heartbeat failed", error instanceof Error ? error.message : error); + } +} + +await prisma.$connect(); +await cleanup(); +await collectMetric(); +await heartbeat(); +const metricTimer = metricsEnabled ? setInterval(collectMetric, metricsSeconds * 1000) : null; +const cleanupTimer = setInterval(cleanup, cleanupHours * 60 * 60 * 1000); +const heartbeatTimer = setInterval(heartbeat, 30 * 1000); + +async function shutdown() { + if (metricTimer) clearInterval(metricTimer); + clearInterval(cleanupTimer); + clearInterval(heartbeatTimer); + await prisma.$disconnect(); + process.exit(0); +} +process.on("SIGTERM", shutdown); +process.on("SIGINT", shutdown); diff --git a/tests/data.integration.test.ts b/tests/data.integration.test.ts new file mode 100644 index 0000000..8aab431 --- /dev/null +++ b/tests/data.integration.test.ts @@ -0,0 +1,46 @@ +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; + +vi.mock("server-only", () => ({})); + +const describeWithDatabase = process.env.TEST_DATABASE_URL ? describe : describe.skip; + +describeWithDatabase("analytics queries", () => { + let prisma: typeof import("../lib/prisma").prisma; + let siteId = ""; + let userId = ""; + let visitorId = ""; + + beforeAll(async () => { + prisma = (await import("../lib/prisma")).prisma; + const suffix = Date.now().toString(36); + const user = await prisma.user.create({ data: { email: `integration-${suffix}@example.test`, passwordHash: "test" } }); + userId = user.id; + const site = await prisma.site.create({ data: { name: "Integration", domain: `integration-${suffix}.test`, publicKey: `integration-${suffix}`, ownerId: user.id } }); + siteId = site.id; + const visitor = await prisma.visitor.create({ data: { visitorKey: `integration-visitor-${suffix}` } }); + visitorId = visitor.id; + const session = await prisma.session.create({ data: { sessionKey: `integration-session-${suffix}`, siteId, visitorId, endedAt: new Date(), durationMs: 30_000 } }); + await prisma.event.create({ data: { + siteId, visitorId, sessionId: session.id, type: "pageview", path: "/test", browser: "Test Browser", + os: "Test OS", device: "desktop", country: "US", city: "New York", createdAt: new Date() + } }); + }); + + afterAll(async () => { + if (userId) await prisma.user.delete({ where: { id: userId } }); + if (visitorId) await prisma.visitor.delete({ where: { id: visitorId } }); + await prisma.$disconnect(); + }); + + it("returns ranged metrics, breakdowns, bounce rate, and timeline", async () => { + const { getDashboardData } = await import("../lib/data"); + const data = await getDashboardData(siteId, "24h"); + expect(data.overview.pageViews).toBe(1); + expect(data.overview.uniqueVisitors).toBe(1); + expect(data.overview.sessions).toBe(1); + expect(data.overview.bounceRate).toBe(100); + expect(data.countries).toContainEqual({ label: "US", value: 1 }); + expect(data.cities).toContainEqual({ label: "New York", value: 1 }); + expect(data.overview.timeline.reduce((sum, bucket) => sum + bucket.pageviews, 0)).toBe(1); + }); +}); diff --git a/tests/geo.test.ts b/tests/geo.test.ts new file mode 100644 index 0000000..1594593 --- /dev/null +++ b/tests/geo.test.ts @@ -0,0 +1,17 @@ +import { describe, expect, it, vi } from "vitest"; + +describe("GeoIP helpers", () => { + it("uses trusted proxy geo headers", async () => { + vi.resetModules(); + vi.stubEnv("TRUST_PROXY", "true"); + const { geoFromTrustedHeaders } = await import("../lib/geo"); + const headers = new Headers({ "cf-ipcountry": "US", "x-vercel-ip-city": "New%20York", "x-vercel-ip-country-region": "NY" }); + expect(geoFromTrustedHeaders(headers)).toMatchObject({ country: "US", city: "New York", region: "NY" }); + }); + + it("does not send private addresses to a provider", async () => { + const { isPublicIp } = await import("../lib/geo"); + expect(isPublicIp("192.168.1.2")).toBe(false); + expect(isPublicIp("8.8.8.8")).toBe(true); + }); +}); diff --git a/tests/range.test.ts b/tests/range.test.ts new file mode 100644 index 0000000..b7a3fe1 --- /dev/null +++ b/tests/range.test.ts @@ -0,0 +1,15 @@ +import { describe, expect, it } from "vitest"; +import { parseRange, rangeStart } from "../lib/range"; + +describe("analytics ranges", () => { + it("rejects unknown range values", () => { + expect(parseRange("all-time")).toBe("24h"); + }); + + it("uses seven calendar days including today", () => { + const now = new Date(2026, 6, 10, 13, 30); + const start = rangeStart("7d", now); + expect(start.getDate()).toBe(4); + expect(start.getHours()).toBe(0); + }); +}); diff --git a/tests/tracker.test.ts b/tests/tracker.test.ts index 9a9d9b8..95c4fea 100644 --- a/tests/tracker.test.ts +++ b/tests/tracker.test.ts @@ -17,7 +17,7 @@ class MemoryStorage { function runTracker(localStorage: MemoryStorage, sessionStorage: MemoryStorage) { const requests: Array> = []; const window = { - location: new URL("https://buffer.lol/tools/dns-lookup"), + location: new URL("https://buffer.lol/tools/dns-lookup?token=private#private-fragment"), screen: { width: 1440, height: 900 }, addEventListener() {}, bufferdash: undefined as undefined | { track: (type: string, metadata?: Record) => void } @@ -63,7 +63,7 @@ describe("browser tracker", () => { expect(second.sessionId).toBe(first.sessionId); }); - it("does not include URL fragments", () => { + it("does not include URL fragments or query strings by default", () => { const request = runTracker(new MemoryStorage(), new MemoryStorage())[0]; expect(request.url).toBe("https://buffer.lol/tools/dns-lookup"); }); diff --git a/vitest.config.ts b/vitest.config.ts new file mode 100644 index 0000000..bb61788 --- /dev/null +++ b/vitest.config.ts @@ -0,0 +1,6 @@ +import { fileURLToPath } from "node:url"; +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + resolve: { alias: { "@": fileURLToPath(new URL(".", import.meta.url)) } } +});