diff --git a/packages/mongodb-runner/README.md b/packages/mongodb-runner/README.md index 6740f4e9..bc31a208 100644 --- a/packages/mongodb-runner/README.md +++ b/packages/mongodb-runner/README.md @@ -50,6 +50,8 @@ Options: --docker Docker image name to run server instances under [string] --id ID to save the cluster metadata under [string] --all for `stop`: stop all clusters [boolean] + --json for `start` and `ls`: print machine-readable JSON + on stdout instead of plain text [boolean] --debug Enable debug output [boolean] --help Show help [boolean] ``` @@ -109,6 +111,38 @@ You can specify a config file for the CLI using `--config`: $ npx @mongodb-js/mongodb-runner start --config ``` +## Machine-readable output + +By default `start` prints diagnostics to stderr and the bare connection string +to stdout, and `ls` prints one `id: uri` line per running instance. Pass +`--json` to get structured output on stdout instead (diagnostics still go to +stderr, and are not part of the JSON). The JSON is printed on a single line so +it can be captured and parsed without whitespace splitting: + +```sh +$ npx mongodb-runner start -t replset --json +{"id":"2b56a4d3-...","connectionString":"mongodb://localhost:27017,localhost:27018/?replicaSet=repl0"} +``` + +For OIDC clusters the result also contains `oidcIssuer` and +`connectionStringWithOidc` (the connection string with +`authMechanism=MONGODB-OIDC` appended). For DSC clusters it contains an `sls` +object with the allocated `ports` and `services`. Capture it to a file rather +than scraping stdout: + +```sh +$ npx mongodb-runner start -t replset --json > cluster.json +``` + +`ls --json` prints a JSON array of the running instances, with the serialized +cluster data for each one. Credentials (keyfile contents, user passwords, and +TLS certificate passwords) are redacted: + +```sh +$ npx mongodb-runner ls --json +[{"id":"2b56a4d3-...","connectionString":"mongodb://localhost:27017/","filepath":"/home/user/.mongodb/runner2/m-2b56a4d3-....json","topology":"standalone","servers":[{"_id":"...","port":27017}],...}] +``` + ## DSC clusters mongodb-runner can launch clusters that use DSC, backed by an SLS storage diff --git a/packages/mongodb-runner/docs/disaggregated-storage.md b/packages/mongodb-runner/docs/disaggregated-storage.md index 4107ec9d..79414caf 100644 --- a/packages/mongodb-runner/docs/disaggregated-storage.md +++ b/packages/mongodb-runner/docs/disaggregated-storage.md @@ -75,10 +75,22 @@ aws ecr get-login-password --region us-east-1 | \ # --downloadUrl=https://.../dsc-mongod.tgz ``` -This prints the connection string once the cluster is up. The first run pulls -all SLS images, which can take several minutes (`--debug` shows the -progress). `mongodb-runner stop --id=...` (printed by `start`) tears down the -mongod processes and the compose project together. +This prints the connection string once the cluster is up. To get the allocated +SLS service ports/URIs as structured output, add `--json` and capture stdout to +a file instead of scraping it: + +```bash +mongodb-runner start -t replset \ + --slsCompose=$MONGO_REPO/buildscripts/modules/atlas/sls-multicell-docker-compose.yml \ + --binDir=/path/to/dsc-mongod/bin \ + --json > cluster.json +``` + +The JSON object contains `id` and `connectionString` plus an `sls` field with +the `ports` and `services` (the same info printed to stderr without `--json`). +The first run pulls all SLS images, which can take several minutes (`--debug` +shows the progress). `mongodb-runner stop --id=...` (printed by `start`) tears +down the mongod processes and the compose project together. The same works for `-t standalone` and `-t sharded`. A DSC-capable `mongod` is required (see Prerequisites) — with a stock binary, startup fails diff --git a/packages/mongodb-runner/src/cli.spec.ts b/packages/mongodb-runner/src/cli.spec.ts index be2b3ff1..de31415d 100644 --- a/packages/mongodb-runner/src/cli.spec.ts +++ b/packages/mongodb-runner/src/cli.spec.ts @@ -140,4 +140,77 @@ describe('cli', function () { await runCli(['stop', '--all']); }); + + it('emits structured output for start with --json', async function () { + const stdout = await runCli([ + 'start', + '--topology', + 'standalone', + '--json', + ]); + + const parsed = JSON.parse(stdout) as { + id: string; + connectionString: string; + }; + + expect(stdout.trim(), 'JSON output should be a single line').to.not.contain( + '\n', + ); + expect(parsed.id, 'result should carry the cluster id').to.be.a('string'); + expect( + parsed.connectionString, + 'result should carry the connection string', + ).to.match(/^mongodb:\/\//); + + const client = new MongoClient(parsed.connectionString); + const result = await client.db('admin').command({ ping: 1 }); + await client.close(); + expect(result.ok, 'reported connection string should be usable').to.eq(1); + + await runCli(['stop', '--all']); + await runCli(['prune']); + }); + + it('emits structured output for ls with --json', async function () { + const startStdout = await runCli([ + 'start', + '--topology', + 'standalone', + '--json', + ]); + const started = JSON.parse(startStdout); + + const lsStdout = await runCli(['ls', '--json']); + expect( + lsStdout.trim(), + 'ls --json output should be a single line', + ).to.not.contain('\n'); + let parsed: any; + expect(() => { + parsed = JSON.parse(lsStdout); + }, 'ls --json stdout must be parseable as JSON').to.not.throw(); + + expect(parsed, 'ls --json should yield an array').to.be.an('array'); + const entry = parsed.find((e: any) => e.id === started.id); + expect(entry, 'the started cluster should be listed').to.not.equal( + undefined, + ); + expect( + entry.connectionString, + 'listed connection string should match the one start reported', + ).to.equal(started.connectionString); + + await runCli(['stop', '--all']); + await runCli(['prune']); + }); + + it('emits an empty JSON array for ls --json with no instances', async function () { + const emptyDir = path.join(tmpDir, 'empty-runner'); + await fs.mkdir(emptyDir, { recursive: true }); + + const stdout = await runCli(['ls', '--json', '--runnerDir', emptyDir]); + + expect(JSON.parse(stdout)).to.deep.equal([]); + }); }); diff --git a/packages/mongodb-runner/src/cli.ts b/packages/mongodb-runner/src/cli.ts index 834692ba..6f217386 100644 --- a/packages/mongodb-runner/src/cli.ts +++ b/packages/mongodb-runner/src/cli.ts @@ -5,6 +5,7 @@ import os from 'os'; import path from 'path'; import createDebug from 'debug'; import * as utilities from './index'; +import { redactSerializedCluster } from './mongocluster'; import { ConnectionString } from 'mongodb-connection-string-url'; import type { MongoClientOptions } from 'mongodb'; @@ -95,6 +96,12 @@ import type { MongoClientOptions } from 'mongodb'; }) .option('debug', { type: 'boolean', describe: 'Enable debug output' }) .option('verbose', { type: 'boolean', describe: 'Enable verbose output' }) + .option('json', { + type: 'boolean', + default: false, + describe: + 'For `start` and `ls`: print machine-readable JSON on stdout instead of plain text', + }) .command('start', 'Start a MongoDB instance') .command('stop', 'Stop a MongoDB instance') .command('prune', 'Clean up metadata for any dead MongoDB instances') @@ -170,7 +177,29 @@ import type { MongoClientOptions } from 'mongodb'; ? `--runnerDir=${argv.runnerDir}` : ''), ); - console.log(cs.toString()); + if (argv.json) { + const result = { + id, + connectionString: cluster.connectionString, + ...(cluster.oidcIssuer + ? { + oidcIssuer: cluster.oidcIssuer, + connectionStringWithOidc: cs.toString(), + } + : {}), + ...(disaggregatedStorage && 'sls' in disaggregatedStorage + ? { + sls: { + ports: disaggregatedStorage.sls.ports, + services: disaggregatedStorage.sls.services, + }, + } + : {}), + }; + console.log(JSON.stringify(result)); + } else { + console.log(cs.toString()); + } cluster.unref(); } @@ -182,8 +211,16 @@ import type { MongoClientOptions } from 'mongodb'; } async function ls() { - for await (const { id, connectionString } of utilities.instances(argv)) { - console.log(`${id}: ${connectionString}`); + if (argv.json) { + const entries: unknown[] = []; + for await (const { serialized, ...rest } of utilities.instances(argv)) { + entries.push({ ...rest, ...redactSerializedCluster(serialized) }); + } + console.log(JSON.stringify(entries)); + } else { + for await (const { id, connectionString } of utilities.instances(argv)) { + console.log(`${id}: ${connectionString}`); + } } } diff --git a/packages/mongodb-runner/src/index.ts b/packages/mongodb-runner/src/index.ts index ba79a336..870df0f3 100644 --- a/packages/mongodb-runner/src/index.ts +++ b/packages/mongodb-runner/src/index.ts @@ -2,6 +2,7 @@ export { MongoServer, type MongoServerEvents, MongoServerOptions, + SerializedServerProperties as MongoServerSerializedProperties, } from './mongoserver'; export { MongoCluster, @@ -14,6 +15,7 @@ export { ShardedOptions as MongoClusterShardedOptions, type ShardDescriptor, type DisaggregatedStorageOptions, + SerializedClusterProperties as MongoClusterSerializedProperties, } from './mongocluster'; export { createSLSMultiCellEnvironment, diff --git a/packages/mongodb-runner/src/mongocluster.spec.ts b/packages/mongodb-runner/src/mongocluster.spec.ts index 58091c02..9bac4f79 100644 --- a/packages/mongodb-runner/src/mongocluster.spec.ts +++ b/packages/mongodb-runner/src/mongocluster.spec.ts @@ -1,5 +1,9 @@ import { expect } from 'chai'; -import { MongoCluster } from './mongocluster'; +import { + MongoCluster, + redactSerializedCluster, + type SerializedClusterProperties, +} from './mongocluster'; import { promises as fs } from 'fs'; import path from 'path'; import os from 'os'; @@ -918,4 +922,96 @@ describe('MongoCluster', function () { expect(client.serverApi?.version).to.eq('1'); }); }); + + describe('serialization', function () { + it('can serialize metadata written before the options field existed', async function () { + // Older metadata has no `options` key; serializing it again must not throw. + const cluster = await MongoCluster.deserialize({ + topology: 'standalone', + servers: [], + shards: [], + defaultConnectionOptions: {}, + users: [], + }); + expect(() => cluster.serialize()).to.not.throw(); + }); + }); + + describe('redactSerializedCluster', function () { + it('removes credentials recursively without mutating the input', function () { + const serialized = { + topology: 'replset', + replSetName: 'repl0', + servers: [ + { + _id: 'a', + keyFileContents: 'server-keyfile-secret', + commandline: ['mongod', '--auth'], + }, + { + _id: 'b', + commandline: [ + 'mongod', + '--tlsCertificateKeyFilePassword', + 'arg-secret', + ], + }, + { + _id: 'c', + commandline: [ + 'mongod', + '--tlsCertificateKeyFilePassword=eq-secret', + ], + }, + ], + shards: [ + { + topology: 'replset', + servers: [{ _id: 'd', keyFileContents: 'shard-keyfile-secret' }], + shards: [], + defaultConnectionOptions: { auth: { password: 'nested-password' } }, + users: [ + { username: 'nested', password: 'nested-password', roles: [] }, + ], + }, + ], + defaultConnectionOptions: { + tlsCertificateKeyFilePassword: 'tls-secret', + }, + users: [{ username: 'top', password: 'top-password', roles: [] }], + options: { + topology: 'replset', + args: ['--auth'], + users: [{ username: 'opt', password: 'option-password', roles: [] }], + }, + } as unknown as SerializedClusterProperties; + + const redacted = redactSerializedCluster(serialized); + + const json = JSON.stringify(redacted); + for (const secret of [ + 'server-keyfile-secret', + 'shard-keyfile-secret', + 'arg-secret', + 'eq-secret', + 'nested-password', + 'top-password', + 'option-password', + 'tls-secret', + ]) { + expect(json, `should not contain ${secret}`).to.not.contain(secret); + } + + // Non-secret data is preserved. + expect(redacted.servers[0]._id).to.eq('a'); + expect(redacted.users[0].username).to.eq('top'); + expect(redacted.options?.args).to.deep.eq(['--auth']); + + // The stored metadata is left untouched. + expect(serialized.servers[0].keyFileContents).to.eq( + 'server-keyfile-secret', + ); + expect(serialized.users[0].password).to.eq('top-password'); + }); + }); }); diff --git a/packages/mongodb-runner/src/mongocluster.ts b/packages/mongodb-runner/src/mongocluster.ts index 1e0919e7..e88a3e94 100644 --- a/packages/mongodb-runner/src/mongocluster.ts +++ b/packages/mongodb-runner/src/mongocluster.ts @@ -1,4 +1,8 @@ -import type { MongoServerEvents, MongoServerOptions } from './mongoserver'; +import type { + MongoServerEvents, + MongoServerOptions, + SerializedServerProperties, +} from './mongoserver'; import { MongoServer } from './mongoserver'; import { ConnectionString } from 'mongodb-connection-string-url'; import type { DownloadOptions } from '@mongodb-js/mongodb-downloader'; @@ -251,6 +255,77 @@ export type MongoClusterEvents = { removeListener: [keyof MongoClusterEvents]; }; +export interface SerializedClusterProperties { + topology: MongoClusterOptions['topology']; + replSetName?: string; + servers: SerializedServerProperties[]; + shards: SerializedClusterProperties[]; + oidcMockProviderProcess?: ReturnType; + dockerComposeProject?: ReturnType; + defaultConnectionOptions: Partial; + users: MongoDBUserDoc[]; + options?: MongoClusterOptions; +} + +// Object keys whose values are credentials and must never be shown in the +// CLI's JSON output. `keyFileContents` is the keyfile used as the `__system` +// password, and `tlsCertificateKeyFilePassword` unlocks a client certificate. +const redactedKeys = new Set([ + 'password', + 'pwd', + 'keyFileContents', + 'tlsCertificateKeyFilePassword', +]); + +// Secrets can also appear as command-line argument values rather than object +// properties (e.g. `--tlsCertificateKeyFilePassword=...`). +const redactedArgFlags = ['--tlsCertificateKeyFilePassword']; + +function redactSecrets(value: T): T { + if (Array.isArray(value)) { + const result: unknown[] = []; + for (let i = 0; i < value.length; i++) { + const item = value[i]; + if (typeof item === 'string') { + const flag = redactedArgFlags.find((f) => item === f); + if (flag !== undefined) { + result.push(`${flag}=`); + i++; // Skip the flag's value. + continue; + } + const prefix = redactedArgFlags.find((f) => item.startsWith(`${f}=`)); + if (prefix !== undefined) { + result.push(`${prefix}=`); + continue; + } + } + result.push(redactSecrets(item)); + } + return result as T; + } + if (value !== null && typeof value === 'object') { + const result: Record = Object.create(null); + for (const [key, val] of Object.entries(value)) { + if (redactedKeys.has(key)) continue; + result[key] = redactSecrets(val); + } + return result as T; + } + return value; +} + +/** + * Produce a credential-free view of a serialized cluster for display. Unlike + * `serialize()`, whose output is persisted and must round-trip, this drops + * secrets (keyfile contents, user passwords, TLS certificate passwords) + * recursively, including copies nested in shards and options. + */ +export function redactSerializedCluster( + serialized: SerializedClusterProperties, +): SerializedClusterProperties { + return redactSecrets(serialized); +} + function removePortArg([...args]: string[]): string[] { let portArgIndex = -1; if ((portArgIndex = args.indexOf('--port')) !== -1) { @@ -391,6 +466,7 @@ export class MongoCluster extends EventEmitter { private dockerComposeProject?: DockerComposeProject; private defaultConnectionOptions: Partial = {}; private users: MongoDBUserDoc[] = []; + private originalOptions?: MongoClusterOptions; private constructor() { super(); @@ -424,7 +500,7 @@ export class MongoCluster extends EventEmitter { }); } - serialize(): unknown /* JSON-serializable */ { + serialize(): SerializedClusterProperties { return { topology: this.topology, replSetName: this.replSetName, @@ -434,6 +510,9 @@ export class MongoCluster extends EventEmitter { dockerComposeProject: this.dockerComposeProject?.serialize(), defaultConnectionOptions: jsonClone(this.defaultConnectionOptions ?? {}), users: jsonClone(this.users), + options: this.originalOptions + ? jsonClone(this.originalOptions) + : undefined, }; } @@ -444,17 +523,19 @@ export class MongoCluster extends EventEmitter { return true; } - static async deserialize(serialized: any): Promise { + static async deserialize( + serialized: SerializedClusterProperties, + ): Promise { const cluster = new MongoCluster(); cluster.topology = serialized.topology; cluster.replSetName = serialized.replSetName; cluster.defaultConnectionOptions = serialized.defaultConnectionOptions; cluster.users = serialized.users; cluster.servers = await safePromiseAll( - serialized.servers.map((srv: any) => MongoServer.deserialize(srv)), + serialized.servers.map((srv) => MongoServer.deserialize(srv)), ); cluster.shards = await safePromiseAll( - serialized.shards.map((shard: any) => MongoCluster.deserialize(shard)), + serialized.shards.map((shard) => MongoCluster.deserialize(shard)), ); cluster.oidcMockProviderProcess = serialized.oidcMockProviderProcess ? OIDCMockProviderProcess.deserialize(serialized.oidcMockProviderProcess) @@ -462,6 +543,7 @@ export class MongoCluster extends EventEmitter { cluster.dockerComposeProject = serialized.dockerComposeProject ? DockerComposeProject.deserialize(serialized.dockerComposeProject) : undefined; + cluster.originalOptions = serialized.options; return cluster; } @@ -522,6 +604,7 @@ export class MongoCluster extends EventEmitter { try { options = { ...options, ...(await handleTLSClientKeyOptions(options)) }; + cluster.originalOptions = options; cluster.topology = options.topology; cluster.users = options.users ?? []; cluster.defaultConnectionOptions = { ...options.internalClientOptions }; diff --git a/packages/mongodb-runner/src/mongoserver.ts b/packages/mongodb-runner/src/mongoserver.ts index ae2e2a92..10cc1377 100644 --- a/packages/mongodb-runner/src/mongoserver.ts +++ b/packages/mongodb-runner/src/mongoserver.ts @@ -69,7 +69,7 @@ export interface MongoServerOptions { detached?: boolean; } -interface SerializedServerProperties { +export interface SerializedServerProperties { _id: string; pid?: number; port?: number; @@ -84,6 +84,7 @@ interface SerializedServerProperties { isDSC?: boolean; isAuth?: boolean; keyFileContents?: string; + commandline?: string[]; } export interface MongoServerEvents { @@ -129,6 +130,7 @@ export class MongoServer extends EventEmitter { public isAuth = false; private keyFileContents?: string; private defaultConnectionOptions?: Partial; + private commandline: string[] = []; get id(): string { return this.uuid; @@ -155,6 +157,7 @@ export class MongoServer extends EventEmitter { isDSC: this.isDSC, isAuth: this.isAuth, keyFileContents: this.keyFileContents, + commandline: this.commandline, }; } @@ -177,6 +180,7 @@ export class MongoServer extends EventEmitter { srv.isConfigSvr = !!serialized.isConfigSvr; srv.isAuth = !!serialized.isAuth; srv.keyFileContents = serialized.keyFileContents; + srv.commandline = serialized.commandline ?? []; if (!srv.closing) { srv.pid = serialized.pid; srv.dbPath = serialized.dbPath; @@ -302,6 +306,7 @@ export class MongoServer extends EventEmitter { debug('starting server', commandline); const [executable, ...args] = commandline; + srv.commandline = commandline; const proc = spawn(executable, args, { stdio: ['inherit', 'pipe', 'pipe'], cwd: options.tmpDir, diff --git a/packages/mongodb-runner/src/runner-helpers.ts b/packages/mongodb-runner/src/runner-helpers.ts index 367aebb3..11285253 100644 --- a/packages/mongodb-runner/src/runner-helpers.ts +++ b/packages/mongodb-runner/src/runner-helpers.ts @@ -1,6 +1,9 @@ import { BSON } from 'mongodb'; import path from 'path'; -import type { MongoClusterOptions } from './mongocluster'; +import type { + MongoClusterOptions, + SerializedClusterProperties, +} from './mongocluster'; import { MongoCluster } from './mongocluster'; import { parallelForEach } from './util'; import * as fs from 'fs/promises'; @@ -10,7 +13,7 @@ import { once } from 'events'; interface StoredInstance { id: string; filepath: string; - serialized: string; + serialized: SerializedClusterProperties; connectionString: string; } @@ -31,7 +34,7 @@ export async function start( ...argv, args, }); - const serialized = await cluster.serialize(); + const serialized = cluster.serialize(); const { connectionString } = cluster; await fs.writeFile(