diff --git a/apps/cli/package.json b/apps/cli/package.json index e32639f7..4af19de9 100644 --- a/apps/cli/package.json +++ b/apps/cli/package.json @@ -1,6 +1,6 @@ { "name": "@profullstack/threatcrush", - "version": "0.13.21", + "version": "0.13.22", "description": "All-in-one security agent daemon — monitor, detect, scan, and protect servers in real-time", "bin": { "threatcrush": "./dist/index.js" diff --git a/apps/cli/src/commands/monitor.ts b/apps/cli/src/commands/monitor.ts index 1a352b5c..2d25ac60 100644 --- a/apps/cli/src/commands/monitor.ts +++ b/apps/cli/src/commands/monitor.ts @@ -1,6 +1,6 @@ -import { existsSync, createReadStream, statSync, accessSync, constants } from 'node:fs'; -import { createInterface } from 'node:readline'; +import { existsSync, accessSync, constants } from 'node:fs'; import { watch } from 'node:fs'; +import { LogTail } from '../core/log-tail.js'; import chalk from 'chalk'; import { formatEvent, logger, banner } from '../core/logger.js'; import { assessNginxRequest, attackSeverity, autoDetectParser, configureAttackDetection, parseAuthLog, parseNginxLog } from '../core/log-parser.js'; @@ -110,37 +110,15 @@ export async function monitorCommand(options: MonitorOptions): Promise { function tailLog(source: LogWatcher): void { const { path, name, category } = source; - // Start reading from end of file - const stat = statSync(path); - let position = stat.size; - + // From the end of the file; each new line exactly once (see LogTail). + const tail = new LogTail(path); + let warned = false; const checkForNewData = () => { try { - const currentStat = statSync(path); - if (currentStat.size <= position) { - if (currentStat.size < position) position = 0; // file rotated - return; - } - - const stream = createReadStream(path, { start: position, encoding: 'utf-8' }); - stream.on('error', (err) => { - // Permission or IO error mid-read; log once and stop polling this source. - logger.warn(`stopped tailing ${path}: ${(err as NodeJS.ErrnoException).code || err.message}`); - position = currentStat.size; - }); - const rl = createInterface({ input: stream }); - rl.on('error', () => { /* surfaced via stream 'error' above */ }); - - rl.on('line', (line) => { - if (!line.trim()) return; - processLine(line, name, category); - }); - - rl.on('close', () => { - position = currentStat.size; - }); - } catch { - // file may not be readable + tail.poll((line) => processLine(line, name, category)); + } catch (err) { + if (!warned) logger.warn(`cannot read ${path}: ${(err as NodeJS.ErrnoException).code || (err as Error).message}`); + warned = true; } }; diff --git a/apps/cli/src/core/__tests__/log-tail.test.ts b/apps/cli/src/core/__tests__/log-tail.test.ts new file mode 100644 index 00000000..e45884e0 --- /dev/null +++ b/apps/cli/src/core/__tests__/log-tail.test.ts @@ -0,0 +1,70 @@ +import { describe, it, expect } from 'vitest'; +import { appendFileSync, mkdtempSync, renameSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { LogTail } from '../log-tail.js'; + +const fresh = () => { + const file = join(mkdtempSync(join(tmpdir(), 'tc-tail-')), 'access.log'); + writeFileSync(file, 'old line that predates the tail\n'); + return file; +}; + +const collect = (tail: LogTail, maxBytes?: number) => { + const lines: string[] = []; + tail.poll((l) => lines.push(l), maxBytes); + return lines; +}; + +describe('LogTail', () => { + it('starts at the end and hands over each new line exactly once', () => { + const file = fresh(); + const tail = new LogTail(file); + expect(collect(tail)).toEqual([]); + appendFileSync(file, 'a\nb\n'); + expect(collect(tail)).toEqual(['a', 'b']); + // Polling again with nothing new must not replay anything: the old reader + // re-read from a stale offset whenever a read outlived the poll interval. + expect(collect(tail)).toEqual([]); + expect(collect(tail)).toEqual([]); + }); + + it('holds a line until its newline arrives', () => { + const file = fresh(); + const tail = new LogTail(file); + appendFileSync(file, 'half a li'); + expect(collect(tail)).toEqual([]); + appendFileSync(file, 'ne\nnext\n'); + expect(collect(tail)).toEqual(['half a line', 'next']); + }); + + it('follows rotation: a new file (new inode) or a truncated one starts from the top', () => { + const file = fresh(); + const tail = new LogTail(file); + appendFileSync(file, 'before\n'); + expect(collect(tail)).toEqual(['before']); + renameSync(file, `${file}.1`); + writeFileSync(file, 'first in the new file\n'); + expect(collect(tail)).toEqual(['first in the new file']); + writeFileSync(file, 'x\n'); // truncated in place + expect(collect(tail)).toEqual(['x']); + }); + + it('spreads a burst over polls instead of reading it all at once', () => { + const file = fresh(); + const tail = new LogTail(file); + appendFileSync(file, Array.from({ length: 100 }, (_, i) => `line ${String(i).padStart(3, '0')}`).join('\n') + '\n'); + const first = collect(tail, 300); // ~33 lines' worth + expect(first.length).toBeGreaterThan(0); + expect(first.length).toBeLessThan(100); + const all = [...first]; + for (let i = 0; i < 10; i++) all.push(...collect(tail, 300)); + expect(all).toHaveLength(100); + expect(new Set(all).size).toBe(100); + }); + + it('a missing file is not an error', () => { + const tail = new LogTail('/nonexistent/threatcrush/test.log'); + expect(collect(tail)).toEqual([]); + }); +}); diff --git a/apps/cli/src/core/__tests__/state-retention.test.ts b/apps/cli/src/core/__tests__/state-retention.test.ts new file mode 100644 index 00000000..b89be581 --- /dev/null +++ b/apps/cli/src/core/__tests__/state-retention.test.ts @@ -0,0 +1,56 @@ +import { describe, it, expect, beforeAll } from 'vitest'; +import { + initStateDB, + insertEvent, + pruneEvents, + getEventCount, + getThreatCount, + getTopSources, +} from '../state.js'; + +const DAY = 86_400_000; +const NOW = Date.parse('2026-10-06T12:00:00Z'); + +const ev = (ageMs: number, ip: string, severity: 'info' | 'high' = 'info') => + insertEvent({ + timestamp: new Date(NOW - ageMs), + module: 'log-watcher', + category: 'web', + severity, + message: 'GET /', + source_ip: ip, + } as any); + +describe('events retention (the 85 GB state.db that starved the daemon)', () => { + beforeAll(() => { + initStateDB(':memory:'); + for (let i = 0; i < 30; i++) ev(20 * DAY, '10.0.0.1'); // past retention + for (let i = 0; i < 20; i++) ev(1 * DAY, '10.0.0.2', i < 5 ? 'high' : 'info'); + for (let i = 0; i < 10; i++) ev(60_000, '10.0.0.3'); + }); + + it('counts every event ever recorded without scanning the table', () => { + expect(getEventCount()).toBe(60); + }); + + it('deletes events past the retention, oldest first, in batches', () => { + const deleted = pruneEvents({ retentionSeconds: 14 * 86_400, now: NOW, batch: 7, maxBatches: 100 }); + expect(deleted).toBe(30); + expect(getTopSources(10).map((s) => s.ip)).not.toContain('10.0.0.1'); + // "ever recorded" keeps counting what was pruned + expect(getEventCount()).toBe(60); + }); + + it('caps the number of rows, dropping the oldest', () => { + const deleted = pruneEvents({ retentionSeconds: 365 * 86_400, maxEvents: 15, now: NOW, batch: 4, maxBatches: 100 }); + expect(deleted).toBe(15); + const top = getTopSources(10); + expect(top.find((s) => s.ip === '10.0.0.3')?.count).toBe(10); + expect(top.find((s) => s.ip === '10.0.0.2')?.count).toBe(5); + }); + + it('threat and recent counts look at recent events only', () => { + expect(getThreatCount()).toBe(0); // the 5 high events were the oldest 15 dropped + expect(getEventCount(new Date(NOW - 3_600_000))).toBe(10); + }); +}); diff --git a/apps/cli/src/core/config.ts b/apps/cli/src/core/config.ts index 96d31377..913bb38d 100644 --- a/apps/cli/src/core/config.ts +++ b/apps/cli/src/core/config.ts @@ -45,6 +45,7 @@ export function loadConfig(configPath?: string): ThreatCrushConfig { detection: parsed.detection, cloud: parsed.cloud, watchdog: parsed.watchdog, + storage: parsed.storage, license: parsed.license, }; } catch { diff --git a/apps/cli/src/core/log-tail.ts b/apps/cli/src/core/log-tail.ts new file mode 100644 index 00000000..0a4ce4ff --- /dev/null +++ b/apps/cli/src/core/log-tail.ts @@ -0,0 +1,82 @@ +import { closeSync, fstatSync, openSync, readSync, statSync } from 'node:fs'; + +/** + * Reads what was appended to a log since the last poll, exactly once. + * + * The watchers used to start a new read stream on every poll and record the + * new position only when that stream closed. When the daemon was busy (a slow + * state database), a read outlived the poll interval and the next poll started + * another from the same offset: the same lines were processed again and again, + * each copy stored as a new event, which made the database slower still, and + * every overlapping stream held a file descriptor (dev2: 3,259 on syslog, + * 1,549 on rotated-away logs). Here a poll reads synchronously, bounded to the + * bytes that are there now, in fixed chunks, and never overlaps itself. + */ +export class LogTail { + private pos: number; + private partial = ''; + private inode: number | undefined; + + /** Starts at the end of the file: history is not replayed. */ + constructor(readonly path: string, fromStart = false) { + try { + const st = statSync(path); + this.pos = fromStart ? 0 : st.size; + this.inode = st.ino; + } catch { + this.pos = 0; + } + } + + get position(): number { + return this.pos; + } + + /** + * Hands each complete new line to onLine. A line still being written (no + * newline yet) waits for the next poll. Rotation (a new inode, or a file + * shorter than our position) restarts at the top of the new file. + * At most maxBytes are read per poll, so a burst is spread over polls + * instead of blocking the daemon. + */ + poll(onLine: (line: string) => void, maxBytes = 4 << 20): number { + let fd: number; + try { + fd = openSync(this.path, 'r'); + } catch { + return 0; + } + try { + const st = fstatSync(fd); + if ((this.inode !== undefined && st.ino !== this.inode) || st.size < this.pos) { + this.pos = 0; + this.partial = ''; + } + this.inode = st.ino; + const end = Math.min(st.size, this.pos + maxBytes); + if (end <= this.pos) return 0; + const buf = Buffer.allocUnsafe(Math.min(end - this.pos, 1 << 20)); + let lines = 0; + while (this.pos < end) { + const want = Math.min(buf.length, end - this.pos); + const n = readSync(fd, buf, 0, want, this.pos); + if (n <= 0) break; + this.pos += n; + const text = this.partial + buf.toString('utf-8', 0, n); + const parts = text.split('\n'); + this.partial = parts.pop() ?? ''; + for (const line of parts) { + if (line.trim()) { + lines++; + onLine(line); + } + } + } + // A runaway "line" (binary junk, no newline) must not grow forever. + if (this.partial.length > 1 << 20) this.partial = ''; + return lines; + } finally { + closeSync(fd); + } + } +} diff --git a/apps/cli/src/core/state.ts b/apps/cli/src/core/state.ts index 33c9153f..2346b2b6 100644 --- a/apps/cli/src/core/state.ts +++ b/apps/cli/src/core/state.ts @@ -102,38 +102,92 @@ export function getRecentEvents(limit: number = 50): ThreatEvent[] { return rows.map(rowToEvent); } +// Every query the daemon answers over IPC must cost the same on a 50 MB +// database as on an 85 GB one: better-sqlite3 is synchronous, so a scan of the +// whole events table freezes the daemon, and a ban sent from the dashboard +// times out behind it. Counts and top sources therefore look at the newest +// RECENT_WINDOW events only (newest by id, which is the primary key). +export const RECENT_WINDOW = 200_000; + +/** Events ever recorded (the AUTOINCREMENT sequence): O(1), never a scan. */ +function eventsEverRecorded(database: Database.Database): number { + const row = database.prepare(`SELECT seq FROM sqlite_sequence WHERE name = 'events'`).get() as { seq?: number } | undefined; + return row?.seq ?? 0; +} + export function getEventCount(since?: Date): number { const database = tryDb(); if (!database) return 0; if (since) { - return (database.prepare(`SELECT COUNT(*) as count FROM events WHERE timestamp >= ?`) - .get(since.toISOString()) as any).count; + return (database.prepare( + `SELECT COUNT(*) as count FROM (SELECT timestamp FROM events ORDER BY id DESC LIMIT ?) WHERE timestamp >= ?`, + ).get(RECENT_WINDOW, since.toISOString()) as any).count; } - return (database.prepare(`SELECT COUNT(*) as count FROM events`).get() as any).count; + return eventsEverRecorded(database); } export function getThreatCount(since?: Date): number { const database = tryDb(); if (!database) return 0; const severities = "('medium','high','critical')"; - if (since) { - return (database.prepare( - `SELECT COUNT(*) as count FROM events WHERE severity IN ${severities} AND timestamp >= ?` - ).get(since.toISOString()) as any).count; - } + const cutoff = since ? since.toISOString() : ''; return (database.prepare( - `SELECT COUNT(*) as count FROM events WHERE severity IN ${severities}` - ).get() as any).count; + `SELECT COUNT(*) as count FROM (SELECT severity, timestamp FROM events ORDER BY id DESC LIMIT ?) + WHERE severity IN ${severities} AND timestamp >= ?`, + ).get(RECENT_WINDOW, cutoff) as any).count; } export function getTopSources(limit: number = 10): Array<{ ip: string; count: number }> { const database = tryDb(); if (!database) return []; return database.prepare(` - SELECT source_ip as ip, COUNT(*) as count FROM events + SELECT source_ip as ip, COUNT(*) as count + FROM (SELECT source_ip FROM events ORDER BY id DESC LIMIT ?) WHERE source_ip IS NOT NULL GROUP BY source_ip ORDER BY count DESC LIMIT ? - `).all(limit) as any[]; + `).all(RECENT_WINDOW, limit) as any[]; +} + +export const DEFAULT_EVENT_RETENTION_SECONDS = 14 * 86_400; +export const DEFAULT_MAX_EVENTS = 2_000_000; + +/** + * Deletes events older than the retention, and beyond the row cap, in small + * batches so no single call holds the daemon for long. Returns how many went. + * The file does not shrink (SQLite reuses the space), which is what we want: + * it stops growing. + */ +export function pruneEvents( + opts: { retentionSeconds?: number; maxEvents?: number; batch?: number; maxBatches?: number; now?: number } = {}, +): number { + const database = tryDb(); + if (!database) return 0; + const retention = opts.retentionSeconds ?? DEFAULT_EVENT_RETENTION_SECONDS; + const maxEvents = opts.maxEvents ?? DEFAULT_MAX_EVENTS; + const batch = opts.batch ?? 5_000; + const maxBatches = opts.maxBatches ?? 20; + const cutoff = new Date((opts.now ?? Date.now()) - retention * 1000).toISOString(); + const byAge = database.prepare( + `DELETE FROM events WHERE id IN (SELECT id FROM events WHERE timestamp < ? ORDER BY id LIMIT ?)`, + ); + const lowest = database.prepare(`SELECT MIN(id) as lo, MAX(id) as hi FROM events`); + const byCount = database.prepare(`DELETE FROM events WHERE id < ?`); + let deleted = 0; + for (let i = 0; i < maxBatches; i++) { + const n = byAge.run(cutoff, batch).changes; + deleted += n; + if (n < batch) break; + } + // Ids are dense apart from what was deleted, so hi - cap is where to cut; + // one batch at a time from the bottom. + for (let i = 0; i < maxBatches; i++) { + const { lo, hi } = lowest.get() as { lo: number | null; hi: number | null }; + if (lo === null || hi === null || hi - lo + 1 <= maxEvents) break; + const n = byCount.run(Math.min(lo + batch, hi - maxEvents + 1)).changes; + deleted += n; + if (n === 0) break; + } + return deleted; } export function getModuleState(module: string, key: string): unknown { diff --git a/apps/cli/src/daemon/index.ts b/apps/cli/src/daemon/index.ts index 1b601b73..e9359849 100644 --- a/apps/cli/src/daemon/index.ts +++ b/apps/cli/src/daemon/index.ts @@ -20,7 +20,8 @@ import { remediationSettings } from './firewall/settings.js'; import { formatDuration } from './firewall/backoff.js'; import { bus } from './event-bus.js'; import { startWatchdog } from './watchdog-service.js'; -import { initStateDB, closeDB } from '../core/state.js'; +import { initStateDB, closeDB, pruneEvents, DEFAULT_EVENT_RETENTION_SECONDS, DEFAULT_MAX_EVENTS } from '../core/state.js'; +import { parseDuration } from './firewall/backoff.js'; import { loadConfig } from '../core/config.js'; import { configureAttackDetection } from '../core/log-parser.js'; import { captureException, flushTelemetry, initTelemetry } from '../core/telemetry.js'; @@ -112,6 +113,22 @@ export async function runDaemon(): Promise { const config = loadConfig(existsSync(PATHS.configFile) ? PATHS.configFile : undefined); configureAttackDetection(config.detection); + // Bounded history: without this the events table grew without limit (85 GB + // on dev2) and the daemon stopped answering. A few batches every minute + // catches up on a backlog without ever holding the daemon for long. + const retentionSeconds = parseDuration(config.storage?.event_retention) ?? DEFAULT_EVENT_RETENTION_SECONDS; + const maxEvents = config.storage?.max_events ?? DEFAULT_MAX_EVENTS; + const prune = () => { + try { + const n = pruneEvents({ retentionSeconds, maxEvents }); + if (n) logLine(`[daemon] pruned ${n} events (keep ${config.storage?.event_retention ?? '14d'}, at most ${maxEvents})`); + } catch (err) { + logLine(`[daemon] event pruning failed: ${(err as Error).message}`); + } + }; + setTimeout(prune, 5_000); + setInterval(prune, 60_000); + bus.on('event', (event: ThreatEvent) => { logLine(`[event] ${event.severity} ${event.module} ${event.message}`); }); diff --git a/apps/cli/src/daemon/watchers/log-watcher.ts b/apps/cli/src/daemon/watchers/log-watcher.ts index 34ff6bb9..6c229d9e 100644 --- a/apps/cli/src/daemon/watchers/log-watcher.ts +++ b/apps/cli/src/daemon/watchers/log-watcher.ts @@ -1,5 +1,5 @@ -import { existsSync, statSync, createReadStream, accessSync, constants } from 'node:fs'; -import { createInterface } from 'node:readline'; +import { existsSync, accessSync, constants } from 'node:fs'; +import { LogTail } from '../../core/log-tail.js'; import type { EventBus } from '../event-bus.js'; import { assessNginxRequest, attackSeverity, autoDetectParser, classifySshAuth, parseAuthLog, parseNginxLog } from '../../core/log-parser.js'; import { insertEvent } from '../../core/state.js'; @@ -20,7 +20,7 @@ export const DEFAULT_SOURCES: LogSource[] = [ export class LogWatcher { private timers = new Map(); - private positions = new Map(); + private tails = new Map(); private active = new Set(); constructor(private bus: EventBus, private sources: LogSource[] = DEFAULT_SOURCES) {} @@ -40,7 +40,7 @@ export class LogWatcher { stop(): void { for (const t of this.timers.values()) clearInterval(t); this.timers.clear(); - this.positions.clear(); + this.tails.clear(); this.active.clear(); } @@ -58,37 +58,24 @@ export class LogWatcher { } private tail(src: LogSource): void { - try { - this.positions.set(src.path, statSync(src.path).size); - } catch { - this.positions.set(src.path, 0); - } - + const tail = new LogTail(src.path); + this.tails.set(src.path, tail); const timer = setInterval(() => this.poll(src), 1000); this.timers.set(src.path, timer); this.active.add(src.module); } - private poll(src: LogSource): void { - let stat; - try { stat = statSync(src.path); } catch { return; } - const prev = this.positions.get(src.path) ?? 0; - - if (stat.size < prev) { - this.positions.set(src.path, 0); // rotated - return; - } - if (stat.size === prev) return; - - const stream = createReadStream(src.path, { start: prev, encoding: 'utf-8' }); - stream.on('error', () => this.positions.set(src.path, stat.size)); - const rl = createInterface({ input: stream }); - rl.on('error', () => {}); - rl.on('line', (line) => { - if (!line.trim()) return; - this.process(line, src); + /** Each new line exactly once; see LogTail for why this is synchronous. */ + poll(src: LogSource): number { + const tail = this.tails.get(src.path); + if (!tail) return 0; + return tail.poll((line) => { + try { + this.process(line, src); + } catch { + // one bad line must not stop the rest + } }); - rl.on('close', () => this.positions.set(src.path, stat.size)); } private process(line: string, src: LogSource): void { diff --git a/apps/cli/src/index.ts b/apps/cli/src/index.ts index 13eeee5f..931933cc 100644 --- a/apps/cli/src/index.ts +++ b/apps/cli/src/index.ts @@ -1037,8 +1037,10 @@ const serversCmd = program .alias("server") .alias("srv") .description("Manage servers") - .action(async (opts) => { - await serversCommand({ action: opts }); + // No subcommand lists them. Commander hands this action its options object, + // which used to be passed as the action name: "Unknown action: [object Object]". + .action(async () => { + await serversCommand({ action: "list" }); }); serversCmd diff --git a/apps/cli/src/modules/dns-monitor/index.ts b/apps/cli/src/modules/dns-monitor/index.ts index e5be0bc8..0f96ddef 100644 --- a/apps/cli/src/modules/dns-monitor/index.ts +++ b/apps/cli/src/modules/dns-monitor/index.ts @@ -5,9 +5,10 @@ * Sources: resolver logs, systemd-resolved, dnsmasq logs, passive :53 observation. */ -import { existsSync, statSync, createReadStream, accessSync, constants } from 'node:fs'; +import { existsSync, accessSync, constants } from 'node:fs'; import { spawn, spawnSync, type ChildProcess } from 'node:child_process'; import { createInterface } from 'node:readline'; +import { LogTail } from '../../core/log-tail.js'; import type { EventBus } from '../../daemon/event-bus.js'; import { insertEvent } from '../../core/state.js'; import type { ThreatEvent, EventSeverity } from '../../types/events.js'; @@ -54,7 +55,7 @@ export function qtypeName(qtype: number): string { export class DnsMonitor { private active = false; private timers = new Map(); - private positions = new Map(); + private tails = new Map(); // Tracking windows private txtQueryCounts = new Map(); @@ -177,31 +178,12 @@ export class DnsMonitor { isActive(): boolean { return this.active; } private tailLog(path: string): void { - try { - this.positions.set(path, statSync(path).size); - } catch { - this.positions.set(path, 0); - } - - const timer = setInterval(() => this.pollLog(path), 2000); + const tail = new LogTail(path); + this.tails.set(path, tail); + const timer = setInterval(() => tail.poll((line) => this.parseDnsLine(line)), 2000); this.timers.set(path, timer); } - private pollLog(path: string): void { - let stat; - try { stat = statSync(path); } catch { return; } - const prev = this.positions.get(path) ?? 0; - - if (stat.size < prev) { this.positions.set(path, 0); return; } - if (stat.size === prev) return; - - const stream = createReadStream(path, { start: prev, encoding: 'utf-8' }); - stream.on('error', () => this.positions.set(path, stat.size)); - const rl = createInterface({ input: stream }); - rl.on('line', (line) => this.parseDnsLine(line)); - rl.on('close', () => this.positions.set(path, stat.size)); - } - private parseDnsLine(line: string): void { // Our own resolver (moshpit-dns), which logs every query it answers: // [dns] udp 127.0.0.1 app.moshcode.sh 28 forwarded 11ms diff --git a/apps/cli/src/tui/__tests__/copy-error.test.ts b/apps/cli/src/tui/__tests__/copy-error.test.ts new file mode 100644 index 00000000..11166f71 --- /dev/null +++ b/apps/cli/src/tui/__tests__/copy-error.test.ts @@ -0,0 +1,45 @@ +import { describe, expect, it } from 'vitest'; +import { renderToScreen, renderToText } from '@profullstack/hqtui'; +import { renderDashboard } from '../view.js'; +import { threatcrushTheme } from '../theme.js'; +import { ERROR_NOTICE_MS, NOTICE_MS, initialState, reducer } from '../state.js'; + +const SIZE = { width: 160, height: 44, theme: threatcrushTheme }; +const NOW = 1_700_000_000_000; + +describe('error messages can be copied', () => { + it('an error stays 30s (an ok notice 6s) and is remembered for `y`', () => { + const err = reducer(initialState(), { type: 'notice', text: 'ban failed: threatcrushd did not answer', tone: 'error', now: NOW }); + expect(err.notice?.until).toBe(NOW + ERROR_NOTICE_MS); + expect(err.lastError).toBe('ban failed: threatcrushd did not answer'); + const ok = reducer(err, { type: 'notice', text: 'banned 1.2.3.4', tone: 'ok', now: NOW }); + expect(ok.notice?.until).toBe(NOW + NOTICE_MS); + expect(ok.lastError).toBe('ban failed: threatcrushd did not answer'); + }); + + it('a lost daemon is an error to copy; demo mode is not', () => { + const lost = reducer(initialState(), { type: 'connection_lost', label: 'threatcrushd socket not found at /var/run/x.sock' }); + expect(lost.lastError).toBe('threatcrushd socket not found at /var/run/x.sock'); + const demo = reducer(initialState(), { type: 'connection_lost', label: 'demo mode — canned events, no daemon' }); + expect(demo.lastError).toBeNull(); + }); + + it('the status bar marks the error as copyable and offers y', () => { + const state = reducer(initialState(), { type: 'notice', text: 'ban failed: boom', tone: 'error', now: Date.now() }); + const text = renderToText(({ ui, theme }) => renderDashboard(ui, state, theme, { onCopy: () => {} }), SIZE); + expect(text).toContain('ban failed: boom ⧉'); + expect(text).toContain('copy error'); + }); + + it('clicking the error copies its text', () => { + const state = reducer(initialState(), { type: 'notice', text: 'ban failed: boom', tone: 'error', now: Date.now() }); + const copied: string[] = []; + const view = ({ ui, theme }: any) => renderDashboard(ui, state, theme, { onCopy: (t) => copied.push(t) }); + const screen = renderToScreen(view, SIZE); + const lines = renderToText(view, SIZE).split('\n'); + const y = lines.findIndex((l) => l.includes('ban failed: boom')); + const x = lines[y].indexOf('ban failed: boom'); + screen.click(x + 2, y); + expect(copied).toEqual(['ban failed: boom']); + }); +}); diff --git a/apps/cli/src/tui/dashboard.ts b/apps/cli/src/tui/dashboard.ts index 375a79f9..e80e0373 100644 --- a/apps/cli/src/tui/dashboard.ts +++ b/apps/cli/src/tui/dashboard.ts @@ -1,4 +1,4 @@ -import { createApp } from '@profullstack/hqtui'; +import { clipboardSequence, createApp } from '@profullstack/hqtui'; import { IpcClient } from '../core/ipc-client.js'; import { PATHS } from '../daemon/paths.js'; import type { ThreatEvent } from '../types/events.js'; @@ -32,6 +32,19 @@ export async function startDashboard(options: DashboardOptions = {}): Promise { + if (!text) return; + try { + process.stdout.write(clipboardSequence(text, !!process.env.TMUX)); + dispatch({ type: 'notice', text: `copied to clipboard: ${text.length > 60 ? `${text.slice(0, 57)}...` : text}`, tone: 'ok' }); + } catch (err) { + dispatch({ type: 'notice', text: `copy failed: ${(err as Error).message}`, tone: 'error' }); + } + }; + let client: IpcClient | null = null; let stopped = false; const timers: NodeJS.Timeout[] = []; @@ -248,6 +261,9 @@ export async function startDashboard(options: DashboardOptions = {}): Promise { renderDashboard(ui, state, theme, { demo: options.demo, + onCopy: copyText, onFeedScroll: (delta) => dispatch({ type: 'scroll', delta: -delta }), // One click focuses the panel, picks the row, and filters the feed to that // source — "what has this one IP done" in a single click. Clicking the diff --git a/apps/cli/src/tui/state.ts b/apps/cli/src/tui/state.ts index 9bf8c252..81553b06 100644 --- a/apps/cli/src/tui/state.ts +++ b/apps/cli/src/tui/state.ts @@ -46,6 +46,8 @@ export interface FirewallInfo { } export const NOTICE_MS = 6000; +/** Errors stay long enough to read, and to click to copy. */ +export const ERROR_NOTICE_MS = 30_000; export interface State { connection: Connection; @@ -76,6 +78,8 @@ export interface State { threatIndex: number; banIndex: number; notice: Notice | null; + /** The last error shown (a failed ban, a lost daemon), for `y` to copy after it fades. */ + lastError: string | null; /** Set while a ban/unban is in flight, so the key does not fire twice. */ busy: boolean; /** Selected row in MODULES. */ @@ -190,6 +194,7 @@ export function initialState(now: number = Date.now()): State { threatIndex: 0, banIndex: 0, notice: null, + lastError: null, busy: false, moduleIndex: 0, moduleFilter: null, @@ -299,7 +304,14 @@ export function reducer(state: State, action: Action): State { }; case 'connection_lost': - return { ...state, connection: 'lost', connectionLabel: action.label, daemon: null, watchdog: OFF_WATCHDOG }; + return { + ...state, + connection: 'lost', + connectionLabel: action.label, + daemon: null, + watchdog: OFF_WATCHDOG, + lastError: action.label.startsWith('demo mode') ? state.lastError : action.label, + }; case 'event': { // Pausing parks events instead of dropping them. The old build discarded @@ -407,8 +419,9 @@ export function reducer(state: State, action: Action): State { notice: { text: action.text, tone: action.tone, - until: (action.now ?? Date.now()) + NOTICE_MS, + until: (action.now ?? Date.now()) + (action.tone === 'error' ? ERROR_NOTICE_MS : NOTICE_MS), }, + lastError: action.tone === 'error' ? action.text : state.lastError, }; case 'busy': diff --git a/apps/cli/src/tui/view.ts b/apps/cli/src/tui/view.ts index f5b58f77..c9000558 100644 --- a/apps/cli/src/tui/view.ts +++ b/apps/cli/src/tui/view.ts @@ -24,6 +24,8 @@ export interface ViewOptions { onBanSelect?: (visibleRow: number) => void; /** Click a module row to show only its events. */ onModuleSelect?: (visibleRow: number) => void; + /** Click a status-bar message (an error, a ban result) to copy its text. */ + onCopy?: (text: string) => void; demo?: boolean; } @@ -498,8 +500,10 @@ function throughputPanel(parent: Container, state: State, theme: Theme): void { ); } -function footer(ui: Container, state: State, theme: Theme): void { +function footer(ui: Container, state: State, theme: Theme, options: ViewOptions = {}): void { const onFeed = state.focus === 'feed'; + const notice = state.notice; + const copyNotice = notice && options.onCopy ? () => options.onCopy?.(notice.text) : undefined; ui.statusBar({ size: 1, items: [ @@ -519,11 +523,17 @@ function footer(ui: Container, state: State, theme: Theme): void { ...(state.muteCron || state.events.some(isCronLine) ? [{ key: 'c', label: state.muteCron ? 'show cron' : 'mute cron' }] : []), + ...(state.lastError ? [{ key: 'y', label: 'copy error' }] : []), ], right: [ // The notice is the answer to "did my keypress do anything", so it takes // the slot for as long as it lives. - { label: state.notice?.text ?? '', color: state.notice?.tone === 'error' ? severityColors.high : theme.success }, + // Click it to copy the text: an error is something to paste somewhere. + { + label: notice ? (notice.tone === 'error' && copyNotice ? `${notice.text} ⧉` : notice.text) : '', + color: notice?.tone === 'error' ? severityColors.high : theme.success, + onPress: copyNotice, + }, { label: state.paused && state.parked.length > 0 ? `${state.parked.length} held` : '', color: severityColors.medium, @@ -573,6 +583,6 @@ export function renderDashboard( }); }); - footer(col, state, theme); + footer(col, state, theme, options); }); } diff --git a/apps/cli/src/types/config.ts b/apps/cli/src/types/config.ts index c82d0172..51ec13df 100644 --- a/apps/cli/src/types/config.ts +++ b/apps/cli/src/types/config.ts @@ -102,6 +102,13 @@ export interface ThreatCrushConfig { detection?: DetectionSection; cloud?: CloudSection; watchdog?: WatchdogSection; + /** How long the events table keeps history; see core/state.ts pruneEvents. */ + storage?: { + /** e.g. "14d" (default). Older events are deleted. */ + event_retention?: string; + /** Hard cap on stored events (default 2,000,000); the oldest go first. */ + max_events?: number; + }; license?: { key_file?: string; key?: string; diff --git a/apps/desktop/package.json b/apps/desktop/package.json index bda997f7..0369578e 100644 --- a/apps/desktop/package.json +++ b/apps/desktop/package.json @@ -1,6 +1,6 @@ { "name": "@profullstack/threatcrush-desktop", - "version": "0.7.21", + "version": "0.7.22", "description": "ThreatCrush Desktop - Security Monitoring Dashboard", "main": "./out/main/index.js", "homepage": "https://threatcrush.com", diff --git a/apps/extension/package.json b/apps/extension/package.json index a62cbf3f..b96d51c7 100644 --- a/apps/extension/package.json +++ b/apps/extension/package.json @@ -1,6 +1,6 @@ { "name": "@profullstack/threatcrush-extension", - "version": "0.7.21", + "version": "0.7.22", "private": true, "type": "module", "scripts": { diff --git a/apps/mobile/package.json b/apps/mobile/package.json index 5fb4a8c0..944e5ec6 100644 --- a/apps/mobile/package.json +++ b/apps/mobile/package.json @@ -1,6 +1,6 @@ { "name": "@profullstack/threatcrush-mobile", - "version": "0.7.21", + "version": "0.7.22", "private": true, "main": "expo-router/entry", "scripts": { diff --git a/apps/sdk/package.json b/apps/sdk/package.json index 1312d4f3..0acf70b9 100644 --- a/apps/sdk/package.json +++ b/apps/sdk/package.json @@ -1,6 +1,6 @@ { "name": "@threatcrush/sdk", - "version": "0.7.21", + "version": "0.7.22", "description": "ThreatCrush Module SDK — build security modules for the ThreatCrush platform", "main": "dist/index.js", "types": "dist/index.d.ts", diff --git a/apps/web/package.json b/apps/web/package.json index 045636aa..0077ad5d 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -1,6 +1,6 @@ { "name": "@profullstack/threatcrush-web", - "version": "0.7.21", + "version": "0.7.22", "private": true, "scripts": { "dev": "next dev", diff --git a/package.json b/package.json index 31cb1d30..ab9d9340 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "threatcrush", "packageManager": "pnpm@10.33.0", - "version": "0.7.21", + "version": "0.7.22", "private": true, "scripts": { "dev": "pnpm --filter @profullstack/threatcrush-web dev", diff --git a/packages/scan/package.json b/packages/scan/package.json index bb02996f..bd0239a9 100644 --- a/packages/scan/package.json +++ b/packages/scan/package.json @@ -1,6 +1,6 @@ { "name": "@threatcrush/scan", - "version": "0.13.21", + "version": "0.13.22", "description": "ThreatCrush scan rules and engine, shared by the CLI, web, desktop and extension.", "license": "MIT", "type": "module",