diff --git a/README.md b/README.md index f25af677768..5a186532d0c 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@

Ask DeepWiki - Set Up with Cursor + Set Up with Cursor

diff --git a/apps/docs/content/docs/platform/self-hosting/docker.mdx b/apps/docs/content/docs/platform/self-hosting/docker.mdx index 9fa470a7bb1..56cfe3a5027 100644 --- a/apps/docs/content/docs/platform/self-hosting/docker.mdx +++ b/apps/docs/content/docs/platform/self-hosting/docker.mdx @@ -43,13 +43,15 @@ EOF Do not set `DATABASE_URL` or `BETTER_AUTH_URL` in `.env` — `docker-compose.prod.yml` composes both on the service definition, and a value set here is ignored. Change `POSTGRES_*` and `NEXT_PUBLIC_APP_URL` instead. + + Because `DATABASE_URL` is composed from them, keep `POSTGRES_USER`, `POSTGRES_PASSWORD`, and `POSTGRES_DB` URL-safe — letters, digits, and `.` `_` `~` `-`. They are inserted into the connection string as written, so a value containing `@`, `/`, `?`, `#`, `%`, or a space can initialize the database while leaving the app and migrations unable to connect. `openssl rand -hex` output is always safe. Save `ENCRYPTION_KEY` and `API_ENCRYPTION_KEY` somewhere outside this server. `ENCRYPTION_KEY` encrypts workspace and personal environment variables, stored provider API keys, MCP OAuth credentials, and deployment/chat secrets; `API_ENCRYPTION_KEY` encrypts user-generated Sim API keys. Neither can be regenerated — a database restore paired with a different key leaves the data it protected permanently unreadable. -The compose file refuses to start if `BETTER_AUTH_SECRET`, `ENCRYPTION_KEY`, or `INTERNAL_API_SECRET` is missing, rather than booting with empty values. `CRON_SECRET` is treated more gently: without it the `cron` service prints what to set and exits, leaving the rest of the stack running — so upgrading from a compose file that predates the scheduler still works. +The compose file refuses to start if `BETTER_AUTH_SECRET`, `ENCRYPTION_KEY`, `INTERNAL_API_SECRET`, or `POSTGRES_PASSWORD` is missing, rather than booting with empty or well-known values. Postgres applies `POSTGRES_PASSWORD` only when it first creates the database volume — see [Postgres on Compose](/platform/self-hosting/security#postgres-on-compose) before changing it on an existing install. `CRON_SECRET` is treated more gently: without it the `cron` service prints what to set and exits, leaving the rest of the stack running — so upgrading from a compose file that predates the scheduler still works. Images track `latest` unless you pin them. For production, see [Upgrades](/platform/self-hosting/upgrades). @@ -65,7 +67,7 @@ Six services start: |---|---|---| | `simstudio` | 3000 | Main application (8 GB memory limit) | | `realtime` | 3002 | WebSocket server (1 GB memory limit) | -| `db` | 5432 | PostgreSQL 17 with pgvector | +| `db` | internal | PostgreSQL 17 with pgvector — not published to the host | | `redis` | internal | Pub/sub and shared cache — not published to the host | | `cron` | — | Runs the [background jobs](/platform/self-hosting/background-jobs) on a schedule | | `migrations` | — | Applies schema migrations once, then exits | @@ -206,5 +208,5 @@ npx sim-setup update { question: "Do scheduled workflows work on Docker Compose?", answer: "Yes. The cron service runs the same jobs the Helm chart schedules as Kubernetes CronJobs, using the schedules in docker/crontab. It needs CRON_SECRET — without it the service prints what to set and exits, and the rest of the stack keeps running."}, { question: "Why is there a Redis container?", answer: "Redis backs pub/sub for live Chat task status and table events, plus shared caches. Pub/sub has no fallback that works across processes, so live status would not stream without it. The port is deliberately not published so it cannot collide with a local Redis."}, { question: "How do I back up and restore the database?", answer: "Back up with: docker compose -f docker-compose.prod.yml exec -T db pg_dump -U postgres simstudio > backup.sql. The -T matters — without it exec allocates a TTY and corrupts the redirected dump. Restore with: docker compose -f docker-compose.prod.yml exec -T db psql -U postgres simstudio < backup.sql. The database data is persisted in a Docker volume named postgres_data."}, - { question: "Can I customize the PostgreSQL credentials?", answer: "Yes. The docker-compose.prod.yml uses environment variable defaults: POSTGRES_USER (default: postgres), POSTGRES_PASSWORD (default: postgres), POSTGRES_DB (default: simstudio), and POSTGRES_PORT (default: 5432). Set these in your .env file to override them." }, + { question: "Can I customize the PostgreSQL credentials?", answer: "Yes. Set POSTGRES_USER (default: postgres) and POSTGRES_DB (default: simstudio) in .env before the first start. POSTGRES_PASSWORD has no default — the compose file will not start without it. Postgres applies all three only when it creates the database volume, so changing them later does not change the existing database; to rotate the password, see Postgres on Compose in the security guide." }, ]} /> diff --git a/apps/docs/content/docs/platform/self-hosting/security.mdx b/apps/docs/content/docs/platform/self-hosting/security.mdx index 1bd8b0fb7a4..a60da61710a 100644 --- a/apps/docs/content/docs/platform/self-hosting/security.mdx +++ b/apps/docs/content/docs/platform/self-hosting/security.mdx @@ -271,19 +271,29 @@ networkPolicy: The service bundles ~2.2 GB of spaCy models, so first start takes around three minutes and it needs at least 4 GB of memory. -## The shipped Compose file publishes Postgres +## Postgres on Compose - - `docker-compose.prod.yml` maps the database to the host: `${POSTGRES_PORT:-5432}:5432`, with `POSTGRES_USER` and `POSTGRES_PASSWORD` both defaulting to `postgres`. A plain `docker compose up -d` against that file, on a machine with a public interface, therefore exposes an open Postgres on 5432 with credentials anyone can guess. The local and Ollama stacks map the database the same way, so apply the fix to whichever file started your install. +The Compose files do not publish the `db` service to the host: `simstudio`, `realtime`, and `migrations` reach it over the Compose network as `db:5432`, and nothing outside the stack can. `docker-compose.prod.yml` also refuses to start without `POSTGRES_PASSWORD`, the same way it refuses to start without `BETTER_AUTH_SECRET`. - The [Docker guide](/platform/self-hosting/docker#1-configure-environment) tells you to generate `POSTGRES_PASSWORD` before the first start — do that, and additionally close the port: + + Installs created from an earlier Compose file published the database on every interface of the host (`5432:5432`), and a `POSTGRES_PASSWORD` left unset fell back to `postgres`. A Docker `ports:` mapping writes its own iptables rules, so a host firewall that looks like it blocks 5432 usually does not. Update the Compose file, then check what your database was created with: - - **Do not need host access.** Delete the `ports:` block from the `db` service. Every other service reaches it over the Compose network by name. - - **Need host access.** Bind it to loopback only — `127.0.0.1:${POSTGRES_PORT:-5432}:5432` — and reach it over an SSH tunnel. + - **You set `POSTGRES_PASSWORD` before the first start.** Nothing else to do — updating the file closes the port. + - **You never set it.** Postgres applies `POSTGRES_PASSWORD` only when it creates the data volume, so the database still uses `postgres`. Set `POSTGRES_PASSWORD=postgres` in `.env` so the stack starts, then rotate it: run `docker compose -f docker-compose.prod.yml exec db psql -U postgres -c "ALTER ROLE CURRENT_USER PASSWORD ''"` (with your `POSTGRES_USER` in place of `postgres` if you set one), set `POSTGRES_PASSWORD` to the same value, and run `docker compose -f docker-compose.prod.yml up -d`. Setting a new value in `.env` alone does not change the database's password and locks the app out. - A Docker `ports:` mapping writes its own iptables rules, so a host firewall that looks like it blocks 5432 usually does not. + `npx sim-setup start` and `npx sim-setup update` write the right value for you and print the rotation steps. +To reach the database from the host — `psql`, a desktop client, a backup job — use `docker compose -f docker-compose.prod.yml exec db psql -U postgres simstudio`, or pass a second Compose file that publishes it on loopback only and connect over an SSH tunnel: + +```yaml +# db-port.yml — docker compose -f docker-compose.prod.yml -f db-port.yml up -d +services: + db: + ports: + - '127.0.0.1:5432:5432' +``` + ## Pre-launch checklist - All five secrets generated fresh, stored in a secret manager, and **`ENCRYPTION_KEY` backed up separately** @@ -297,7 +307,7 @@ The service bundles ~2.2 GB of spaCy models, so first start takes around three m - NetworkPolicy enabled and `ingressFrom` scoped to the ingress controller - Namespace labelled `pod-security.kubernetes.io/enforce=restricted` - Object storage buckets private, with CORS limited to your Sim origin -- Database reachable only from the deployment — on Compose, the `db` service's host `ports:` mapping removed or bound to `127.0.0.1` — with a generated `POSTGRES_PASSWORD` +- Database reachable only from the deployment — on Compose, no host `ports:` mapping on the `db` service, or one bound to `127.0.0.1` — with a generated `POSTGRES_PASSWORD` - TLS enforced (`sslMode: require`) on an externally managed database, or on the bundled one once you have configured it for TLS — the shipped Compose database does not enable it - Backups configured **and a restore rehearsed** - Sandbox strategy decided for user code diff --git a/apps/docs/content/docs/platform/self-hosting/upgrades.mdx b/apps/docs/content/docs/platform/self-hosting/upgrades.mdx index 23a59933898..1006325ae40 100644 --- a/apps/docs/content/docs/platform/self-hosting/upgrades.mdx +++ b/apps/docs/content/docs/platform/self-hosting/upgrades.mdx @@ -193,11 +193,13 @@ install uses: | Install | What it runs | |---|---| -| `docker-compose.prod.yml` | Refreshes its managed copy of the Compose file, then `docker compose pull` — the versions configured by `SIM_VERSION`, or `latest` when unset | +| `docker-compose.prod.yml` | Refreshes its managed copy of the Compose file, writes `POSTGRES_PASSWORD` to `.env` if it is missing, then `docker compose pull` — the versions configured by `SIM_VERSION`, or `latest` when unset | | `docker-compose.local.yml` | `docker compose build --pull` — rebuilds from source against refreshed base images, no pull of published images | Inspect the result with `npx sim-setup logs`, which targets whichever Compose file the install uses. +`docker-compose.prod.yml` requires `POSTGRES_PASSWORD`. If you manage the file yourself and Compose stops with `required variable POSTGRES_PASSWORD is missing a value`, your database was created with the password `postgres` — set exactly that in `.env`, not a new value, then see [Postgres on Compose](/platform/self-hosting/security#postgres-on-compose) to rotate it. + The CLI detects only those two files. An install started from `docker-compose.ollama.yml` is invisible to it: `update`, `logs`, and `status` report no install, or — in a source checkout that also carries per-application env files — report that checkout's dev install instead. Upgrade that stack directly: ```bash diff --git a/docker-compose.local.yml b/docker-compose.local.yml index 04b3a520c1a..48daa577a06 100644 --- a/docker-compose.local.yml +++ b/docker-compose.local.yml @@ -111,11 +111,13 @@ services: timeout: 5s retries: 5 + # Not published to the host: the other services reach it over the Compose + # network as db:5432. A host `ports:` mapping binds every interface, and the + # password below defaults to a well-known value. For host access use + # `docker compose exec db psql -U postgres simstudio`. db: image: pgvector/pgvector:pg17 restart: always - ports: - - '${POSTGRES_PORT:-5432}:5432' environment: - POSTGRES_USER=${POSTGRES_USER:-postgres} - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-postgres} diff --git a/docker-compose.ollama.yml b/docker-compose.ollama.yml index 39104560346..4311732add4 100644 --- a/docker-compose.ollama.yml +++ b/docker-compose.ollama.yml @@ -87,12 +87,14 @@ services: command: ['bun', 'run', 'db:migrate'] restart: 'no' - # PostgreSQL Database with Vector Extension + # PostgreSQL Database with Vector Extension. + # Not published to the host: the other services reach it over the Compose + # network as db:5432. A host `ports:` mapping binds every interface, and the + # password below defaults to a well-known value. For host access use + # `docker compose exec db psql -U postgres simstudio`. db: image: pgvector/pgvector:pg17 restart: always - ports: - - '${POSTGRES_PORT:-5432}:5432' environment: - POSTGRES_USER=${POSTGRES_USER:-postgres} - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-postgres} diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 2cea451fee6..0c85846041b 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -15,7 +15,7 @@ services: memory: 8G environment: - NODE_ENV=production - - DATABASE_URL=postgresql://${POSTGRES_USER:-postgres}:${POSTGRES_PASSWORD:-postgres}@db:5432/${POSTGRES_DB:-simstudio} + - 'DATABASE_URL=postgresql://${POSTGRES_USER:-postgres}:${POSTGRES_PASSWORD:?is required. For a new install set it in .env to the output of: openssl rand -hex 24. An install created before it was required uses the password postgres - set POSTGRES_PASSWORD=postgres to keep its data.}@db:5432/${POSTGRES_DB:-simstudio}' - BETTER_AUTH_URL=${NEXT_PUBLIC_APP_URL:-http://localhost:3000} - NEXT_PUBLIC_APP_URL=${NEXT_PUBLIC_APP_URL:-http://localhost:3000} # TRUSTED_ORIGINS: comma-separated public origins to trust for auth in @@ -85,7 +85,7 @@ services: memory: 1G environment: - NODE_ENV=production - - DATABASE_URL=postgresql://${POSTGRES_USER:-postgres}:${POSTGRES_PASSWORD:-postgres}@db:5432/${POSTGRES_DB:-simstudio} + - 'DATABASE_URL=postgresql://${POSTGRES_USER:-postgres}:${POSTGRES_PASSWORD:?is required. For a new install set it in .env to the output of: openssl rand -hex 24. An install created before it was required uses the password postgres - set POSTGRES_PASSWORD=postgres to keep its data.}@db:5432/${POSTGRES_DB:-simstudio}' - NEXT_PUBLIC_APP_URL=${NEXT_PUBLIC_APP_URL:-http://localhost:3000} # Derived from NEXT_PUBLIC_APP_URL, matching the simstudio service — a # single public-origin variable keeps the two from disagreeing. @@ -109,7 +109,7 @@ services: image: ghcr.io/simstudioai/migrations:${SIM_VERSION:-latest} working_dir: /app/packages/db environment: - - DATABASE_URL=postgresql://${POSTGRES_USER:-postgres}:${POSTGRES_PASSWORD:-postgres}@db:5432/${POSTGRES_DB:-simstudio} + - 'DATABASE_URL=postgresql://${POSTGRES_USER:-postgres}:${POSTGRES_PASSWORD:?is required. For a new install set it in .env to the output of: openssl rand -hex 24. An install created before it was required uses the password postgres - set POSTGRES_PASSWORD=postgres to keep its data.}@db:5432/${POSTGRES_DB:-simstudio}' depends_on: db: condition: service_healthy @@ -151,14 +151,21 @@ services: simstudio: condition: service_healthy + # Not published to the host: app, realtime and migrations reach it over the + # Compose network as db:5432. A host `ports:` mapping binds every interface and + # Docker's own iptables rules bypass host firewalls, so it would put the + # database on the network. For host access (psql, a desktop client), use + # `docker compose exec db psql -U postgres simstudio`, or pass a second -f + # file that publishes '127.0.0.1:5432:5432' (-f disables override files). db: image: pgvector/pgvector:pg17 restart: unless-stopped - ports: - - '${POSTGRES_PORT:-5432}:5432' environment: - POSTGRES_USER=${POSTGRES_USER:-postgres} - - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-postgres} + # Postgres applies this only when it creates the data volume; later + # changes are ignored. An install created before this was required got + # the password `postgres` — set exactly that to keep it working. + - 'POSTGRES_PASSWORD=${POSTGRES_PASSWORD:?is required. For a new install set it in .env to the output of: openssl rand -hex 24. An install created before it was required uses the password postgres - set POSTGRES_PASSWORD=postgres to keep its data.}' - POSTGRES_DB=${POSTGRES_DB:-simstudio} volumes: - postgres_data:/var/lib/postgresql/data diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index c862c419e99..83b606afee8 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -280,8 +280,6 @@ async function main() { 'POSTGRES_DB=simstudio', '-v', `${dataDir}/postgres:/var/lib/postgresql/data`, - '-p', - '5432:5432', 'pgvector/pgvector:pg17', ]) diff --git a/packages/sim-setup/src/compose-database.test.ts b/packages/sim-setup/src/compose-database.test.ts new file mode 100644 index 00000000000..07cdba3c390 --- /dev/null +++ b/packages/sim-setup/src/compose-database.test.ts @@ -0,0 +1,124 @@ +import { readFileSync } from 'node:fs' +import path from 'node:path' +import { fileURLToPath } from 'node:url' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { + choosePostgresPassword, + composeFileRequiresPostgresPassword, + LEGACY_POSTGRES_PASSWORD, + postgresUser, +} from './compose-database' + +const REPO_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../../..') +const COMPOSE_FILES = [ + 'docker-compose.prod.yml', + 'docker-compose.local.yml', + 'docker-compose.ollama.yml', +] as const + +function composePath(file: string): string { + return path.join(REPO_ROOT, file) +} + +/** The lines of one top-level service, up to the next service or top-level key. */ +function serviceBlock(file: string, service: string): string { + const lines = readFileSync(composePath(file), 'utf8').split('\n') + const start = lines.findIndex((line) => line === ` ${service}:`) + if (start === -1) throw new Error(`${file} has no ${service} service`) + const end = lines.findIndex((line, index) => index > start && /^ {0,2}[A-Za-z0-9_-]+:/.test(line)) + return lines.slice(start, end === -1 ? undefined : end).join('\n') +} + +afterEach(() => { + vi.unstubAllEnvs() +}) + +describe('choosePostgresPassword', () => { + const noShell = {} + + it('leaves a password already in .env alone without looking for a volume', () => { + const hasDatabaseVolume = vi.fn(() => true) + expect( + choosePostgresPassword('in-env-file', 'sim-abc', { shell: noShell, hasDatabaseVolume }) + ).toBeNull() + expect(hasDatabaseVolume).not.toHaveBeenCalled() + }) + + it('generates a password for a project with no database volume yet', () => { + const hasDatabaseVolume = vi.fn(() => false) + const choice = choosePostgresPassword(undefined, 'sim-abc', { + shell: noShell, + hasDatabaseVolume, + }) + expect(hasDatabaseVolume).toHaveBeenCalledWith('sim-abc') + expect(choice?.source).toBe('generated') + expect(choice?.value).toMatch(/^[0-9a-f]{64}$/) + }) + + it('keeps the legacy password for a volume created before it was required', () => { + expect( + choosePostgresPassword('', 'sim-abc', { shell: noShell, hasDatabaseVolume: () => true }) + ).toEqual({ value: LEGACY_POSTGRES_PASSWORD, source: 'legacy' }) + }) + + it('leaves a shell-exported password to the operator, since Compose uses it', () => { + const hasDatabaseVolume = vi.fn(() => true) + expect( + choosePostgresPassword(undefined, 'sim-abc', { + shell: { POSTGRES_PASSWORD: 'in-shell' }, + hasDatabaseVolume, + }) + ).toBeNull() + expect(hasDatabaseVolume).not.toHaveBeenCalled() + }) + + it('refuses an empty shell export, which Compose would use over .env', () => { + for (const envFileValue of [undefined, 'in-env-file']) { + expect(() => + choosePostgresPassword(envFileValue, 'sim-abc', { shell: { POSTGRES_PASSWORD: '' } }) + ).toThrow(/exported but empty/) + } + }) + + it('reads the process environment by default', () => { + vi.stubEnv('POSTGRES_PASSWORD', 'from-process') + expect( + choosePostgresPassword(undefined, 'sim-abc', { hasDatabaseVolume: () => true }) + ).toBeNull() + }) +}) + +describe('postgresUser', () => { + it('follows Compose: a shell export wins even when empty, then .env, then the default', () => { + vi.stubEnv('POSTGRES_USER', 'from-shell') + expect(postgresUser('from-env-file')).toBe('from-shell') + vi.stubEnv('POSTGRES_USER', '') + expect(postgresUser('from-env-file')).toBe('postgres') + vi.stubEnv('POSTGRES_USER', undefined) + expect(postgresUser('from-env-file')).toBe('from-env-file') + expect(postgresUser(undefined)).toBe('postgres') + }) +}) + +describe('shipped Compose files', () => { + it('only the production file requires POSTGRES_PASSWORD', () => { + expect(composeFileRequiresPostgresPassword(composePath('docker-compose.prod.yml'))).toBe(true) + expect(composeFileRequiresPostgresPassword(composePath('docker-compose.local.yml'))).toBe(false) + expect(composeFileRequiresPostgresPassword(composePath('docker-compose.ollama.yml'))).toBe( + false + ) + }) + + it('never falls back to a default password in the production file', () => { + const contents = readFileSync(composePath('docker-compose.prod.yml'), 'utf8') + const references = contents.match(/\$\{POSTGRES_PASSWORD[^}]*\}/g) ?? [] + expect(references).toHaveLength(4) + for (const reference of references) { + expect(reference.startsWith('${POSTGRES_PASSWORD:?')).toBe(true) + } + }) + + it.each(COMPOSE_FILES)('%s does not publish the database to the host', (file) => { + expect(serviceBlock(file, 'db')).not.toMatch(/^\s+ports:/m) + }) +}) diff --git a/packages/sim-setup/src/compose-database.ts b/packages/sim-setup/src/compose-database.ts new file mode 100644 index 00000000000..8af13f0686a --- /dev/null +++ b/packages/sim-setup/src/compose-database.ts @@ -0,0 +1,156 @@ +import { spawnSync } from 'node:child_process' +import { readFileSync } from 'node:fs' +import { generateSecret } from './env-files' +import { SetupError } from './errors' +import * as p from './prompter' + +/** + * The password Postgres was initialized with on a Compose install created while + * `docker-compose.prod.yml` still defaulted it. Postgres applies + * `POSTGRES_PASSWORD` only when it creates the data volume, so such a volume + * keeps this password until it is rotated inside the database. + */ +export const LEGACY_POSTGRES_PASSWORD = 'postgres' + +/** The volume key the Sim Compose files declare for Postgres data. */ +const POSTGRES_VOLUME = 'postgres_data' + +/** + * Whether a Compose file refuses to start without `POSTGRES_PASSWORD`. Only the + * production file does, and the version that requires it is also the one that + * stops publishing the database, so older files and dev stacks are left alone. + */ +export function composeFileRequiresPostgresPassword(composeFile: string): boolean { + return readFileSync(composeFile, 'utf8').includes('${POSTGRES_PASSWORD:?') +} + +/** Where a chosen `POSTGRES_PASSWORD` came from. */ +export type PostgresPasswordSource = 'generated' | 'legacy' + +export interface PostgresPasswordChoice { + value: string + source: PostgresPasswordSource +} + +interface ChooseOptions { + /** The shell environment, whose `POSTGRES_PASSWORD` Compose interpolates over `.env`. */ + shell?: NodeJS.ProcessEnv + hasDatabaseVolume?: (project: string) => boolean +} + +/** + * Picks the `POSTGRES_PASSWORD` to write to a Compose install's `.env`, or null + * when nothing should be written. The production Compose file requires the + * variable, and the value must match what the data volume was created with — + * Postgres ignores `POSTGRES_PASSWORD` on an existing data directory, so a + * wrong value locks the app out of its own database: + * + * - A value exported in the shell is the operator's to manage: Compose + * interpolates it over `.env`, so nothing is written. An empty export is + * refused, because Compose would use the empty value and refuse to start + * however good the one in `.env` is. + * - Otherwise a value in `.env` stands, an existing volume was created with the + * legacy password the file used to default to, and a project with no volume + * yet gets a generated one. + */ +export function choosePostgresPassword( + envFileValue: string | undefined, + project: string, + { shell = process.env, hasDatabaseVolume = composeDatabaseVolumeExists }: ChooseOptions = {} +): PostgresPasswordChoice | null { + const shellValue = shell.POSTGRES_PASSWORD + if (shellValue === '') { + throw new SetupError( + 'POSTGRES_PASSWORD is exported but empty, and Compose uses it over .env.', + ['unset it (unset POSTGRES_PASSWORD) so the value in .env applies'] + ) + } + if (shellValue !== undefined || envFileValue) return null + return hasDatabaseVolume(project) + ? { value: LEGACY_POSTGRES_PASSWORD, source: 'legacy' } + : { value: generateSecret(), source: 'generated' } +} + +/** Whether a Compose project already has a Postgres data volume, found by Compose's own labels. */ +export function composeDatabaseVolumeExists(project: string): boolean { + const result = spawnSync( + 'docker', + [ + 'volume', + 'ls', + '-q', + '--filter', + `label=com.docker.compose.project=${project}`, + '--filter', + `label=com.docker.compose.volume=${POSTGRES_VOLUME}`, + ], + { encoding: 'utf8' } + ) + if (result.status !== 0) { + throw new SetupError( + `could not check for an existing database volume: ${result.stderr.trim() || result.stdout.trim()}` + ) + } + return result.stdout.trim().length > 0 +} + +/** + * The project name Compose resolves for a file run from `cwd` — from `-p`, + * `COMPOSE_PROJECT_NAME`, or the directory. Read without interpolation, so it + * works before the required variables exist. + */ +export function composeProjectName(composeFile: string, cwd: string): string { + const result = spawnSync( + 'docker', + ['compose', '-f', composeFile, 'config', '--no-interpolate', '--format', 'json'], + { cwd, encoding: 'utf8' } + ) + const name = result.status === 0 ? parseProjectName(result.stdout) : null + if (name) return name + throw new SetupError( + `could not resolve the Compose project name for ${composeFile}: ${result.stderr.trim() || 'no name in docker compose config'}` + ) +} + +function parseProjectName(stdout: string): string | null { + try { + const { name } = JSON.parse(stdout) as { name?: unknown } + return typeof name === 'string' && name ? name : null + } catch { + return null + } +} + +/** + * Reports what was written. `compose` is the pinned `docker compose -p … -f …` + * prefix for the install, and `user` the effective `POSTGRES_USER`, both used + * in the legacy rotation steps. + */ +export function reportPostgresPasswordChoice( + choice: PostgresPasswordChoice, + { compose, user, envPath }: { compose: string; user: string; envPath: string } +): void { + if (choice.source === 'generated') { + p.log.step(`Generated POSTGRES_PASSWORD in ${envPath}`) + return + } + p.note( + [ + `This database was created with the password "${LEGACY_POSTGRES_PASSWORD}", so ${envPath}`, + `now sets POSTGRES_PASSWORD=${LEGACY_POSTGRES_PASSWORD} to keep it working. The database is not`, + 'published to the host, so only the containers in this stack can reach it. To rotate it:', + ` ${compose} exec db psql -U ${user} -c "ALTER ROLE CURRENT_USER PASSWORD ''"`, + ' then set POSTGRES_PASSWORD= in .env and run: npx sim-setup start', + ].join('\n'), + 'Database password' + ) +} + +/** + * The Postgres role Compose initializes from `${POSTGRES_USER:-postgres}`. A shell + * export overrides `.env` even when empty, and an empty value takes the default. + */ +export function postgresUser(envFileValue: string | undefined): string { + const shellUser = process.env.POSTGRES_USER + return shellUser === undefined ? envFileValue || 'postgres' : shellUser || 'postgres' +} diff --git a/packages/sim-setup/src/lifecycle.ts b/packages/sim-setup/src/lifecycle.ts index e7c2128c0c4..ddf3047b7c1 100644 --- a/packages/sim-setup/src/lifecycle.ts +++ b/packages/sim-setup/src/lifecycle.ts @@ -2,10 +2,16 @@ import { spawnSync } from 'node:child_process' import { existsSync, readFileSync } from 'node:fs' import path from 'node:path' import { ensureProductionComposeFile } from './compose-asset' +import { + choosePostgresPassword, + composeFileRequiresPostgresPassword, + postgresUser, + reportPostgresPasswordChoice, +} from './compose-database' import { legacyComposeProjectName } from './compose-project' import { directoryOverride, resolveSetupContextAtRoot, SETUP_CONTEXT } from './context' import { DB_CONTAINER, type Detection, REDIS_CONTAINER, runDetection } from './detect' -import { archiveEnvFile, archiveFile, parseEnv, ROOT } from './env-files' +import { archiveEnvFile, archiveFile, parseEnv, ROOT, upsertEnv, writeEnvFile } from './env-files' import { SetupError } from './errors' import { forwardCommands, isLocalKubeContext } from './modes/k8s' import { httpHealth } from './probes' @@ -196,6 +202,25 @@ function composeArgs(install: ComposeInstall, ...verb: string[]): string[] { return ['compose', '-p', install.project, '-f', install.file, ...verb] } +/** + * Gives a Compose install the `POSTGRES_PASSWORD` its file requires before the + * stack is brought up, matching whatever its data volume was created with. + */ +function ensureComposePostgresPassword(install: ComposeInstall): void { + if (!composeFileRequiresPostgresPassword(install.file)) return + const envPath = path.join(install.dir, '.env') + const content = existsSync(envPath) ? readFileSync(envPath, 'utf8') : '' + const vars = parseEnv(content) + const choice = choosePostgresPassword(vars.get('POSTGRES_PASSWORD'), install.project) + if (!choice) return + writeEnvFile(envPath, upsertEnv(content, 'POSTGRES_PASSWORD', choice.value)) + reportPostgresPasswordChoice(choice, { + compose: `docker compose -p ${install.project} -f ${install.file}`, + user: postgresUser(vars.get('POSTGRES_USER')), + envPath, + }) +} + /** Dev mode owns the split env files and, usually, the managed Postgres/Redis. */ function devInstall(detection: Detection): DevInstall | null { const postgres = detection.dbContainer?.managed ?? false @@ -294,6 +319,7 @@ function k8sReachHints(context: string): string { function start(install: Install): void { if (install.kind === 'compose') { + ensureComposePostgresPassword(install) const spin = p.spinner() spin.start('Starting containers…') dockerRun(composeArgs(install, 'up', '-d'), 'docker compose up failed', install.dir) @@ -358,6 +384,7 @@ function stop(install: Install): void { function restart(install: Install): void { if (install.kind === 'compose') { + ensureComposePostgresPassword(install) const spin = p.spinner() spin.start('Restarting containers…') dockerRun(composeArgs(install, 'restart'), 'docker compose restart failed', install.dir) @@ -404,9 +431,10 @@ function update(install: Install): void { } const mode = getComposeUpdateMode(install.file) + if (mode === 'pull') install.file = refreshComposeFileForUpdate(install.file, install.dir) + ensureComposePostgresPassword(install) const spin = p.spinner() if (mode === 'pull') { - install.file = refreshComposeFileForUpdate(install.file, install.dir) spin.start('Pulling configured Sim images…') dockerRun(composeArgs(install, 'pull'), 'docker compose pull failed', install.dir) } else { diff --git a/packages/sim-setup/src/modes/compose.ts b/packages/sim-setup/src/modes/compose.ts index 4f05cc37a6e..0b9792c6204 100644 --- a/packages/sim-setup/src/modes/compose.ts +++ b/packages/sim-setup/src/modes/compose.ts @@ -3,6 +3,13 @@ import path from 'node:path' import { EMAIL_SETUP, STORAGE_SETUP } from '../capability-config' import { promptCapabilitySetup, stageCapabilitySetupTransition } from '../capability-setup' import { ensureProductionComposeFile } from '../compose-asset' +import { + choosePostgresPassword, + composeFileRequiresPostgresPassword, + composeProjectName, + postgresUser, + reportPostgresPasswordChoice, +} from '../compose-database' import { legacyComposeProjectName, standaloneComposeProjectName } from '../compose-project' import { SETUP_CONTEXT } from '../context' import type { Detection } from '../detect' @@ -179,6 +186,13 @@ export async function runComposeMode(detection: Detection, quick: boolean): Prom if (composeProject && !configuredComposeProject) { values.COMPOSE_PROJECT_NAME = composeProject } + const postgresPassword = composeFileRequiresPostgresPassword(composeFile) + ? choosePostgresPassword( + root.vars.get('POSTGRES_PASSWORD'), + composeProject ?? composeProjectName(composeFile, ROOT) + ) + : null + if (postgresPassword) values.POSTGRES_PASSWORD = postgresPassword.value // Before the key is minted: a half-set override mints against one environment // and validates against the other, and warning afterwards is too late — the // bad key is already stored, and the next run offers to keep it. @@ -218,6 +232,13 @@ export async function runComposeMode(detection: Detection, quick: boolean): Prom for (const key of Object.keys(values)) remove.delete(key) reconcileEnvValues('root', [...remove], values) p.log.step('Wrote .env (compose reads it for variable substitution)') + if (postgresPassword) { + reportPostgresPasswordChoice(postgresPassword, { + compose: composeCommand(composeFile, composeProject), + user: postgresUser(root.vars.get('POSTGRES_USER')), + envPath: root.path, + }) + } const validation = spawnSync('docker', composeArgs(composeFile, composeProject, 'config'), { cwd: ROOT,