fix(cli): stop recording expected outcomes as errors, enable checkpoints by default

apps/cli/src/bin.ts

@@ -1,12 +1,13 @@
 #!/usr/bin/env bun
 import { BunRuntime } from "@effect/platform-bun"
 import * as BunServices from "@effect/platform-bun/BunServices"
-import { Effect, Layer, Metric } from "effect"
+import { Effect, Layer, Metric, Runtime } from "effect"
 import * as Command from "effect/unstable/cli/Command"
 import { FetchHttpClient } from "effect/unstable/http"
 import { cli } from "./cli"
 import { MapleConfig } from "./core/config"
 import { Mode } from "./core/mode"
+import { annotateOutcome, recoverExpected } from "./core/outcomes"
 import { TelemetryLayer } from "./core/telemetry"
 import { maybeNotifyUpdate } from "./core/update"
 import { WarehouseExecutorFromMode } from "./core/warehouse"
@@ -75,6 +76,46 @@ if (checkpointProbeDataDir !== undefined) {
 				process.exitCode = 1
 			}),
 		),
+		// Expected outcomes, recovered inside the span for the same reason as the
+		// archive error above. These are the CLI behaving correctly — a refused
+		// precondition, an unresolvable backend, `--help` — and letting them reach
+		// `withSpan` closed the root span `Error`. They dominated the CLI's error
+		// stream (~24k events for the already-running guard alone) and buried real
+		// failures under outcomes nobody needs to act on.
+		//
+		// The outcome is annotated rather than dropped, so `maple.cli.outcome` still
+		// answers "how often do people hit this?" without the span being an error.
+		// Same rule `apps/ingest` applies to expected 4xx rejections.
+		//
+		// Genuine failures stay uncaught on purpose: `@maple/cli/ServerError` (bind
+		// failure, dirty store, incompatible store) and every other tag still close
+		// the span `Error` and are reported by `runMain`.
+		Effect.catchTags({
+			"@maple/cli/ServerStateError": recoverExpected,
+			// Mode resolution ("No Maple backend found", "Cannot use --remote and
+			// --local together") reaches here as a `WarehouseConfigError`, remapped by
+			// `WarehouseExecutorFromMode` in core/warehouse.ts. `pipeName` is the
+			// discriminator: only "mode" is an expected outcome — every other
+			// `WarehouseConfigError` is a real query failure (unknown table, bad
+			// column) and is re-raised so it still closes the span `Error`.
+			"@maple/http/errors/WarehouseConfigError": (error) =>
+				error.pipeName === "mode" ? recoverExpected(error) : Effect.fail(error),
+			// `Command.runWith` renders the help text and then re-fails with the same
+			// error, so `maple --help` recorded as an error span. The text is already
+			// on stdout by now; only the exit code is left to honour — 0 for a plain
+			// `--help`, 1 when help was shown because parsing failed.
+			//
+			// The tag is "ShowHelp"; `~effect/cli/CliError/ShowHelp` (what shows up in
+			// telemetry) is the schema *identifier*, not the tag.
+			ShowHelp: (error) =>
+				annotateOutcome(error._tag).pipe(
+					Effect.andThen(
+						Effect.sync(() => {
+							process.exitCode = Runtime.getErrorExitCode(error)
+						}),
+					),
+				),
+		}),
 		Effect.withSpan("maple", { attributes: { "cli.argv": process.argv.slice(2).join(" ") } }),
 		Effect.provide(MainLayer),
 		Effect.provide(TelemetryLayer),

apps/cli/src/commands/server.ts

@@ -16,6 +16,7 @@ import {
 	reconcileCheckpointRecovery,
 	resetLiveStorePreservingCheckpoints,
 	restoreCheckpoint,
+	writeBackupConfig,
 } from "../server/checkpoints"
 import { resolveUiAssets } from "../server/ui-assets"
 import { amber, bold, cyan, dim, green, MARK_LINES, MARK_WIDTH, underline } from "../lib/style"
@@ -40,6 +41,18 @@ class ServerError extends Schema.TaggedError<ServerError>()("@maple/cli/ServerEr
 	message: Schema.String,
 }) {}
 
+/**
+ * A refused command whose precondition simply wasn't met — the server is already
+ * running, or isn't running at all. The message and the non-zero exit are
+ * identical to `ServerError`; the separate tag exists so `bin.ts` can close the
+ * root span `Ok` for these without also swallowing genuine start failures
+ * ("failed to bind …", "did not come up within 10s"), which stay on
+ * `ServerError`. Same rule the ingest gateway follows for expected 4xx.
+ */
+export class ServerStateError extends Schema.TaggedError<ServerStateError>()("@maple/cli/ServerStateError", {
+	message: Schema.String,
+}) {}
+
 const defaultDataDir = (): string => join(homedir(), ".maple", "data")
 
 /** Collapse the home directory to `~` for tidy paths. */
@@ -178,7 +191,9 @@ const dataDirFlag = Flag.optional(
 
 const chdbConfigFileFlag = Flag.optional(
 	Flag.string("chdb-config-file").pipe(
-		Flag.withDescription("Optional ClickHouse config file passed to embedded chDB"),
+		Flag.withDescription(
+			"ClickHouse config file for embedded chDB (default: a generated backups-enabled config beside the data dir)",
+		),
 	),
 )
 
@@ -230,13 +245,57 @@ const offlineFlag = Flag.boolean("offline").pipe(
 // Log file for `--background` runs, beside the PID file (e.g. ~/.maple/maple.log).
 const logFilePath = (dataDir: string): string => join(dirname(dataDir), "maple.log")
 
+// Generated chDB config, beside the PID and log files (e.g. ~/.maple/chdb-config.xml).
+export const chdbConfigPath = (dataDir: string): string => join(dirname(dataDir), "chdb-config.xml")
+
+/**
+ * Resolve the chDB config file, generating a backups-enabled default when the
+ * user did not supply one.
+ *
+ * `BACKUP DATABASE default TO Disk('default', …)` — how every checkpoint is
+ * taken — needs `<backups><allowed_disk>` in the config of the *running* chDB
+ * connection. chDB allows one connection per process, acquired once at start and
+ * held for the process lifetime, and `maple checkpoint` is a separate process
+ * talking over HTTP: it cannot inject config into a live connection. So a server
+ * started without a backups config can never checkpoint, and `maple checkpoint`
+ * could only ever report that after the fact.
+ *
+ * The effect was that checkpoints were unusable out of the box and, because the
+ * dirty-store recovery path tells users to run `maple restore --yes`, that advice
+ * pointed at a checkpoint which could not exist. Generating the default here
+ * fixes both. A user-supplied `--chdb-config-file` is honoured untouched.
+ */
+export const resolveChdbConfigFile = (dataDir: string, supplied: string | undefined) =>
+	Effect.gen(function* () {
+		if (supplied !== undefined) return supplied
+		const fs = yield* FileSystem
+		const path = chdbConfigPath(dataDir)
+		// Regenerated every start: idempotent, and it self-heals a truncated or
+		// hand-edited file. Failing to write is not fatal — the server still starts,
+		// checkpoints just stay unavailable, which is the old behaviour.
+		yield* fs.makeDirectory(dirname(path), { recursive: true }).pipe(Effect.ignore)
+		return yield* Effect.try(() => {
+			writeBackupConfig(path)
+			return path
+		}).pipe(Effect.orElseSucceed(() => undefined))
+	})
+
 /** Non-fatal `/health` probe used while waiting for a detached server to bind.
- *  A transport error or a >300ms timeout collapses to `false` (not yet up). */
+ *  A transport error or a >300ms timeout collapses to `false` (not yet up).
+ *
+ *  Untraced: the loop below polls until the child binds, so ECONNREFUSED is the
+ *  expected answer for the first ~10 attempts. Each one used to close an
+ *  `http.client GET` span as `Error` inside an otherwise-`Ok` root span — 9k
+ *  events of pure noise. `orElseSucceed` cannot help: it sits outside the client
+ *  call, which has already ended the span by then. `TracerDisabledWhen` is the
+ *  hook that skips span creation entirely, and it is scoped to this request
+ *  rather than provided layer-wide so real `/health` calls stay traced. */
 const probeHealth = (addr: string) =>
 	HttpClient.get(`${addr}/health`).pipe(
 		Effect.map((res) => res.status >= 200 && res.status < 300),
 		Effect.timeout("300 millis"),
 		Effect.orElseSucceed(() => false),
+		Effect.provideService(HttpClient.TracerDisabledWhen, () => true),
 	)
 
 /**
@@ -351,7 +410,7 @@ export const start = Command.make("start", {
 			// Already-running guard.
 			const existingPid = yield* readPid(fs, pidPath)
 			if (Option.isSome(existingPid) && isProcessAlive(existingPid.value)) {
-				return yield* new ServerError({
+				return yield* new ServerStateError({
 					message: `maple is already running (PID ${existingPid.value}) — stop it with \`maple stop\``,
 				})
 			}
@@ -474,6 +533,10 @@ export const start = Command.make("start", {
 			}
 
 			const requestedRetentionDays = Option.getOrUndefined(a.minimumRawTelemetryRetentionDays)
+			const chdbConfigFile = yield* resolveChdbConfigFile(
+				dataDir,
+				Option.getOrUndefined(a.chdbConfigFile),
+			)
 
 			// Detached: spawn the same command without --background and exit.
 			if (a.background)
@@ -483,7 +546,7 @@ export const start = Command.make("start", {
 					a.port,
 					dataDir,
 					a.offline,
-					Option.getOrUndefined(a.chdbConfigFile),
+					chdbConfigFile,
 					a.onDirtyStore,
 					requestedRetentionDays,
 				)
@@ -525,7 +588,7 @@ export const start = Command.make("start", {
 						corsOrigin: hostedUiOrigin(hostedUiUrl),
 						port: a.port,
 						dataDir,
-						configFile: Option.getOrUndefined(a.chdbConfigFile),
+						configFile: chdbConfigFile,
 						minimumRawTelemetryRetentionDays: requestedRetentionDays,
 						assets,
 					}).pipe(
@@ -578,12 +641,12 @@ export const stop = Command.make("stop", { dataDir: dataDirFlag }).pipe(
 			const pidOpt = yield* readPid(fs, pidPath)
 
 			if (Option.isNone(pidOpt)) {
-				return yield* new ServerError({ message: "maple is not running (no PID file found)" })
+				return yield* new ServerStateError({ message: "maple is not running (no PID file found)" })
 			}
 			const pid = pidOpt.value
 			if (!isProcessAlive(pid)) {
 				yield* fs.remove(pidPath, { force: true }).pipe(Effect.ignore)
-				return yield* new ServerError({
+				return yield* new ServerStateError({
 					message: "maple is not running (stale PID file, cleaned up)",
 				})
 			}
@@ -630,7 +693,7 @@ export const reset = Command.make("reset", { dataDir: dataDirFlag, yes: yesFlag
 			// Refuse while a server still owns the store.
 			const pidOpt = yield* readPid(fs, pidFilePath(dataDir))
 			if (Option.isSome(pidOpt) && isProcessAlive(pidOpt.value)) {
-				return yield* new ServerError({
+				return yield* new ServerStateError({
 					message: `maple is running (PID ${pidOpt.value}) — stop it first with \`maple stop\``,
 				})
 			}
@@ -707,7 +770,7 @@ export const restore = Command.make("restore", {
 
 			const pidOpt = yield* readPid(fs, pidFilePath(dataDir))
 			if (Option.isSome(pidOpt) && isProcessAlive(pidOpt.value)) {
-				return yield* new ServerError({
+				return yield* new ServerStateError({
 					message: `maple is running (PID ${pidOpt.value}) — stop it first with \`maple stop\``,
 				})
 			}

apps/cli/src/core/mode.ts

@@ -28,13 +28,20 @@ type ResolvedMode =
 const hasFlag = (name: string): boolean =>
 	typeof process !== "undefined" && Array.isArray(process.argv) && process.argv.includes(name)
 
-/** Fast, non-fatal liveness probe of the local binary's `/health` route. */
+/** Fast, non-fatal liveness probe of the local binary's `/health` route.
+ *
+ *  Untraced, for the same reason as `probeHealth` in commands/server.ts: "no
+ *  local server running" is the normal answer for anyone on remote mode, and
+ *  recording it as an `Error` span buried real failures. `TracerDisabledWhen`
+ *  skips span creation rather than producing an `Ok` span, and is scoped to this
+ *  request so other `/health` calls stay traced. */
 const probeLocal = (client: HttpClient.HttpClient, baseUrl: string): Effect.Effect<boolean> => {
 	const request = HttpClientRequest.get(`${baseUrl.replace(/\/$/, "")}/health`)
 	return client.execute(request).pipe(
 		Effect.map((response) => response.status >= 200 && response.status < 300),
 		Effect.timeoutOrElse({ duration: Duration.millis(400), orElse: () => Effect.succeed(false) }),
 		Effect.orElseSucceed(() => false),
+		Effect.provideService(HttpClient.TracerDisabledWhen, () => true),
 	)
 }
 

apps/cli/src/core/outcomes.ts

@@ -0,0 +1,44 @@
+import { Effect } from "effect"
+
+/**
+ * Expected-outcome handling for the root `maple` span.
+ *
+ * A CLI failure is not automatically a *problem*. Refusing to start because the
+ * server is already running, reporting that no backend is configured, printing
+ * `--help` — these are the CLI working, but they travel through Effect's error
+ * channel, so `Effect.withSpan("maple", …)` in bin.ts closed the root span
+ * `Error` for every one of them. They were the CLI's top error sources by a wide
+ * margin (~24k events for the already-running guard alone) and buried the
+ * failures worth acting on.
+ *
+ * These helpers recover such outcomes *inside* the span — the same placement
+ * bin.ts already uses for `ArchiveError`, and for the same reason: applied
+ * outside, the span has already closed by the time recovery runs.
+ *
+ * They live in their own module because bin.ts executes the CLI at import time
+ * and cannot be imported by a test.
+ */
+
+/** Record which expected outcome ended the run, on the root `maple` span. */
+export const annotateOutcome = (tag: string): Effect.Effect<void> =>
+	Effect.annotateCurrentSpan({ "maple.cli.outcome": tag })
+
+/**
+ * Recover an expected, user-facing outcome: annotate it, print its message, and
+ * exit non-zero while leaving the root span `Ok`.
+ *
+ * `runMain` would otherwise render the failure itself, so the message has to be
+ * written here — the exit status and stderr output the user sees are unchanged.
+ */
+export const recoverExpected = (error: {
+	readonly _tag: string
+	readonly message: string
+}): Effect.Effect<void> =>
+	annotateOutcome(error._tag).pipe(
+		Effect.andThen(
+			Effect.sync(() => {
+				process.stderr.write(`${error.message}\n`)
+				process.exitCode = 1
+			}),
+		),
+	)

apps/cli/src/core/telemetry.ts

@@ -8,6 +8,24 @@ import { MAPLE_VERSION } from "../version"
 // new CLI release. An explicit `MAPLE_INGEST_KEY` still wins (see below).
 const DEFAULT_INGEST_KEY = "maple_pk_bwGJomBwDO4B15sopcuinQVqNFCDjhE2"
 
+/**
+ * Where this invocation is running.
+ *
+ * Without this the SDK falls back to `Config.withDefault("development")` — none
+ * of `MAPLE_ENVIRONMENT` / `RAILWAY_ENVIRONMENT_NAME` / `DEPLOYMENT_ENV` exist
+ * on a laptop — so *every* CLI install reported `development`, indistinguishable
+ * from a Maple worker running locally and, worse, from our own CI. Triaging the
+ * CLI's error stream meant reading paths out of error strings to guess whether a
+ * failure came from a user or a GitHub Actions runner.
+ *
+ * `CI` is the de-facto standard variable, set by GitHub Actions, GitLab, CircleCI
+ * and Buildkite alike. An explicit `MAPLE_ENVIRONMENT` still wins.
+ */
+const resolveEnvironment = (): string => {
+	if (process.env.MAPLE_ENVIRONMENT) return process.env.MAPLE_ENVIRONMENT
+	return process.env.CI ? "ci" : "cli"
+}
+
 /**
  * OpenTelemetry layer for the CLI — traces + logs about the CLI itself
  * (commands, warehouse queries) and, when running `maple start`, the server's
@@ -27,13 +45,14 @@ export const TelemetryLayer = Maple.layer({
 	serviceName: "maple-cli",
 	serviceNamespace: "backend",
 	serviceVersion: MAPLE_VERSION,
+	environment: resolveEnvironment(),
 	repositoryUrl: "https://github.com/Makisuo/maple",
 	ingestKey: process.env.MAPLE_INGEST_KEY ?? DEFAULT_INGEST_KEY,
 	shutdownTimeout: "3 seconds",
-	// NOTE: expected user errors (bad flag, "maple is already running") still
-	// record as Error spans here. `anticipatedErrorIdentifiers` — which the API
-	// and alerting workers use to map 4xx-ish outcomes to Ok — is implemented in
-	// Maple's flushable tracer, and this server layer wires Effect's stock
-	// `Otlp.layerJson` instead. Supporting it on the CLI means giving the server
-	// SDK the flushable tracer, which is an SDK change, not a CLI one.
+	// NOTE: expected user outcomes ("maple is already running", `--help`) no
+	// longer record as Error spans — `bin.ts` recovers them inside the root span
+	// and annotates `maple.cli.outcome` instead. That is a CLI-side fix, not the
+	// SDK's `anticipatedErrorIdentifiers`, which lives in Maple's flushable tracer
+	// while this server layer wires Effect's stock `Otlp.layerJson`. Anything
+	// still arriving as an Error span here is a genuine failure.
 })

apps/cli/src/server/checkpoints.ts

@@ -1492,8 +1492,15 @@ export const createCheckpoint = Effect.fn("CheckpointService.create")(function*
 					createError(
 						isMissingBackupConfigurationError(error)
 							? new Error(
-									"checkpoints require the local server to be started with `--chdb-config-file` " +
-										"pointing at a ClickHouse backups config",
+									// `maple start` generates a backups-enabled config when
+									// `--chdb-config-file` is absent, so reaching this means the
+									// server was started with a custom config carrying no
+									// `<backups>` stanza — or with a build predating that default.
+									"the running server's chDB config has no `<backups>` stanza, so it " +
+										"cannot take checkpoints. Restart `maple start` without " +
+										"`--chdb-config-file` to use the generated default, or add " +
+										"`<backups><allowed_disk>default</allowed_disk>" +
+										"<allowed_path>backups</allowed_path></backups>` to your config.",
 									{ cause: error },
 								)
 							: error,