From 811a52de9b9e56a7ca3be726c047683d5ac4dbed Mon Sep 17 00:00:00 2001 From: Luke Melia Date: Sun, 18 Oct 2026 23:20:34 -0400 Subject: [PATCH 01/13] Add ADR 003: reach proprietary cloud services only through adapters Runtime code reaches proprietary cloud services (KMS today) only through capability-named interfaces with a provider adapter and a local adapter, selected by configuration. Open protocols, including the S3 API, are the boundary instead, provided code uses only what the protocol defines and S3 clients take their endpoint from configuration. Runtime code also stops detecting its hosting platform from provider-injected variables and headers. The ADR lists where the runtime does not yet comply. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01ABJLVRj5d5YngGbpKqi4MP --- ...3-reach-cloud-services-through-adapters.md | 168 ++++++++++++++++++ docs/adr/README.md | 1 + 2 files changed, 169 insertions(+) create mode 100644 docs/adr/003-reach-cloud-services-through-adapters.md diff --git a/docs/adr/003-reach-cloud-services-through-adapters.md b/docs/adr/003-reach-cloud-services-through-adapters.md new file mode 100644 index 00000000000..3680fe5416f --- /dev/null +++ b/docs/adr/003-reach-cloud-services-through-adapters.md @@ -0,0 +1,168 @@ +--- +status: proposed +date: 2026-10-18 +scope: + - packages/realm-server/** + - packages/runtime-common/** + - packages/postgres/** + - packages/billing/** + - packages/ai-bot/** + - packages/bot-runner/** + - packages/host/app/** +--- + +# 003. Reach proprietary cloud services only through adapters + +Boxel is deployed on AWS. This is a deployment choice, so the runtime code must +not depend on it. Each proprietary cloud service the runtime uses sits behind an +interface. The interface is named for the capability that Boxel needs. An +adapter implements the interface for one provider, and a local adapter +implements it for development and tests. Services that speak an open protocol +(Postgres, a POSIX filesystem, HTTP, the S3 API) need no interface, but the +code must use only what that protocol defines. The runtime must not detect +which platform hosts it. To move to a different provider, a team writes +adapters and changes configuration. It does not rewrite the code that calls the +adapters. + +## 1. Decision + +The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", +"SHOULD NOT", "RECOMMENDED", "NOT RECOMMENDED", "MAY", and "OPTIONAL" in this +document are to be interpreted as described in BCP 14 (RFC 2119, RFC 8174) when, +and only when, they appear in all capitals, as shown here. + +### 1.1. Scope + +This ADR governs runtime code: code that runs in a deployed service or in the +browser. Infrastructure-as-code, deploy configuration such as +`packages/host/config/deploy.js`, CI workflows, and operational runbooks and +skills are provider-specific by nature, and this ADR does not govern them. + +### 1.2. Proprietary services sit behind an adapter + +Runtime code MUST reach a proprietary cloud service, such as KMS, only through +an interface. The interface MUST be named and shaped for the capability that +Boxel needs, for example `SecretsKeyProvider` or `MediaCacheAdapter`. It MUST +NOT be named or shaped for the service. An interface MUST NOT expose provider +types, request or command objects, resource identifiers such as ARNs, or +provider error codes. An adapter MUST translate provider errors into the +interface's own results or errors. + +A provider SDK (`@aws-sdk/*`, and the SDK of any other cloud provider) MUST be +imported only in an adapter module. A lint rule MUST enforce this. The rule's +allowlist names each adapter module. A change to the allowlist is a reviewed +change to the boundary. + +### 1.3. Open protocols are the boundary + +Runtime code MAY reach a service that speaks an open, widely implemented +protocol directly, without an interface. These protocols include Postgres, a +POSIX filesystem, HTTP, and the S3 API. Code that uses such a service MUST use +only what the protocol defines. It MUST NOT depend on extensions that only one +provider offers, such as engine-specific features of a managed Postgres or +S3 features that S3-compatible stores do not implement. + +An S3 client MUST take its endpoint, region, and path-style addressing from +configuration, so that it can reach any S3-compatible store. It MUST get +credentials from configuration or the SDK's default chain, and it MUST NOT +assume a particular source for them. The runtime SHOULD construct S3 clients +through one shared helper. + +### 1.4. Every capability runs locally + +Every interface from §1.2 MUST have a local adapter that runs without cloud +credentials. Development and CI MUST use it. For an S3-API store, +development and CI MAY instead use a local S3-compatible server or a +local-disk adapter. + +### 1.5. Configuration selects the adapter + +One factory per capability MUST choose the adapter from configuration at +startup. Code outside the factory MUST depend on the interface, and it MUST +NOT know which adapter is in use. Partial or conflicting configuration MUST +fail startup. It MUST NOT fall back silently. Settings that only one adapter +reads MAY be named for its provider, such as `SECRETS_KMS_KEY_ARN`. Only that +adapter's factory MAY read them. + +### 1.6. No assumptions about the hosting platform + +Runtime code MUST NOT detect its platform from environment variables that a +provider injects, such as `ECS_CONTAINER_METADATA_URI`. It MUST NOT change +behavior based on provider-specific request headers or user agents, such as +`ELB-HealthChecker`. A fact about the deployment, such as an instance id or +whether the server is deployed, MUST come from a provider-neutral setting that +the deployment configuration sets. Health and readiness MUST be served on +paths that any load balancer or orchestrator can probe. + +### 1.7. New dependencies + +A new dependency on a hosted service SHOULD use an open protocol rather than a +proprietary API. A new proprietary dependency MUST meet §1.2–§1.5 when it is +added. This ADR does not require an adapter for another provider until a +deployment needs one. + +## 2. Rationale + +Boxel will not necessarily stay on its current provider. Cost, a customer's +data-residency or self-hosting requirement, or a service that another provider +offers better can each make a move worthwhile. Each piece of code that is +coupled to one provider makes a move more expensive. Coupling also spreads, +because new code copies the code around it. Today only a few runtime modules +touch AWS. This makes the rule cheap to adopt now, and the cost of adopting it +will only increase. + +The two existing seams show the shape of the rule. `MediaCacheAdapter` and +`SecretsKeyProvider` each have a cloud adapter and a local one. A factory +chooses between them from the environment. The local adapter makes the rule +cheap to keep. Development and CI already run without AWS, so the interface +is used by two implementations from the start, and it cannot become a thin +wrapper over one provider's API. + +The S3 API counts as an open protocol because many object stores implement +it, including Cloudflare R2, MinIO, Backblaze B2, and Google Cloud Storage +through its XML API. An interface in front of it would add a layer with no +gain in portability. What it does need is a configurable endpoint (§1.3). + +The import rule is a lint rule because the boundary can break quietly. An +SDK import in a large diff is easy to miss in review, and agents write much of +this code. A lint failure states the rule at the point where the code breaks +it. + +## 3. Considered options + +- **Adapters for proprietary services, open protocols as the boundary + (chosen).** Portability work goes only where it is needed, and code stays + direct where a standard already makes it portable. +- **Call the provider directly, and port when needed.** This costs nothing + now, but coupling spreads. Assumptions about the platform, such as which + environment variables exist, stay invisible until the code runs somewhere + else. +- **A generic multi-cloud library.** Its lowest-common-denominator API is + wider than what Boxel needs. It does not model capabilities such as + envelope-key unwrapping, and it adds a dependency that must track every + provider. Small interfaces of our own are cheaper. +- **A Boxel interface in front of the S3 API as well.** Rejected; see §2. +- **Interfaces for everything external, including Postgres and the + filesystem.** These are already portable standards. An interface would cost + effort without making the code more portable. +- **Self-hosted open-source services only, such as MinIO or Vault, in + production.** This avoids proprietary APIs, but at an operational cost that + adapters make unnecessary. + +## 4. Consequences + +The runtime does not yet meet this ADR in these places. Each is follow-up +work: + +- No lint rule restricts provider SDK imports (§1.2). +- `S3MediaCacheAdapter` (`packages/realm-server/media-cache/s3-adapter.ts`) + and the prerender artifact sink + (`packages/realm-server/prerender/artifact-sink.ts`) construct `S3Client` + with a region only, and no endpoint can be configured (§1.3). +- `worker.ts` and `worker-manager.ts` derive worker ids from + `ECS_CONTAINER_METADATA_URI`. `worker-manager.ts` enables the local indexing + dashboard when that variable is absent (§1.6). +- `healthCheck` in `packages/realm-server/middleware/index.ts` answers only + requests whose user agent is `ELB-HealthChecker` (§1.6). +- The `ecsMetadata` middleware sends an `X-ECS-Container-Metadata-URI-v4` + response header (§1.6). diff --git a/docs/adr/README.md b/docs/adr/README.md index e8fd91ac2cb..5dd2c7928f1 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -15,4 +15,5 @@ or backfill one. | ADR | Title | Status | | ---------------------------------------------------------------------------- | --------------------------------------------------------------------------------- | -------- | | [001](001-record-architecture-decisions.md) | Record architecture decisions as short, normative ADRs | proposed | +| [003](003-reach-cloud-services-through-adapters.md) | Reach proprietary cloud services only through adapters | proposed | | [005](005-authorize-key-changes-only-with-a-session-card-code-cant-reach.md) | Authorize changes to a user's keys only with a session that card code can't reach | proposed | From 8cbbd81871efad5cc67a2b21ed40fce3612c57d1 Mon Sep 17 00:00:00 2001 From: Luke Melia Date: Sun, 18 Oct 2026 23:39:23 -0400 Subject: [PATCH 02/13] Bring the runtime into line with ADR 003 - A lint rule confines cloud provider SDK imports to the modules that adapt them, listed in eslint/cloud-provider-sdk-imports.cjs. - Every S3 client comes from lib/s3-client.ts, which takes an endpoint and path-style addressing from S3_ENDPOINT and S3_FORCE_PATH_STYLE, so the media cache and the prerender artifact sink can reach any S3-compatible store. - Worker ids and a new X-Boxel-Instance-Id response header name the instance by BOXEL_INSTANCE_ID, or by hostname plus a per-boot suffix, instead of the ECS container metadata URI. The load harness reads the new header. - The worker manager serves its indexing dashboard when no realm it serves maps to a deployed origin, instead of when ECS metadata is absent. - The health check answers the user agents named in HEALTH_CHECK_USER_AGENTS, which the deployed start scripts set to ELB-HealthChecker, so the load balancer's probe is unchanged. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01ABJLVRj5d5YngGbpKqi4MP --- .eslintrc.js | 21 +++++++ ...3-reach-cloud-services-through-adapters.md | 57 +++++++---------- docs/realm-server-health-signals.md | 7 ++- eslint/cloud-provider-sdk-imports.cjs | 42 +++++++++++++ packages/ai-bot/.eslintrc.cjs | 8 ++- packages/host/.eslintrc.js | 11 ++++ packages/observability/README.md | 14 ++--- packages/realm-server/lib/instance-id.ts | 27 ++++++++ packages/realm-server/lib/s3-client.ts | 31 +++++++++ .../realm-server/media-cache/s3-adapter.ts | 16 ++--- packages/realm-server/middleware/index.ts | 25 +++++--- .../realm-server/prerender/artifact-sink.ts | 21 +++---- .../realm-server/prerender/prerender-app.ts | 4 +- .../scripts/load-harness/README.md | 6 +- .../scripts/load-harness/lib/deploy-pin.ts | 30 ++++----- .../realm-server/scripts/start-production.sh | 1 + .../realm-server/scripts/start-staging.sh | 1 + packages/realm-server/server.ts | 4 +- .../realm-server/tests/health-check-test.ts | 63 +++++++++++++++++++ .../realm-server/tests/instance-id-test.ts | 59 +++++++++++++++++ .../realm-server/tests/load-harness-test.ts | 14 ++--- packages/realm-server/tests/s3-client-test.ts | 53 ++++++++++++++++ packages/realm-server/worker-manager.ts | 23 +++---- packages/realm-server/worker.ts | 10 +-- 24 files changed, 414 insertions(+), 134 deletions(-) create mode 100644 eslint/cloud-provider-sdk-imports.cjs create mode 100644 packages/realm-server/lib/instance-id.ts create mode 100644 packages/realm-server/lib/s3-client.ts create mode 100644 packages/realm-server/tests/health-check-test.ts create mode 100644 packages/realm-server/tests/instance-id-test.ts create mode 100644 packages/realm-server/tests/s3-client-test.ts diff --git a/.eslintrc.js b/.eslintrc.js index b715464fbba..2a277b41bfe 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -5,6 +5,11 @@ const { CJS_GLOBALS_IN_ESM, } = require('./eslint/erasable-syntax-selectors.cjs'); const { DATA_TEST_SELECTORS } = require('./eslint/data-test-selectors.cjs'); +const { + CLOUD_PROVIDER_SDK_IMPORTS, + RUNTIME_PACKAGE_FILES, + CLOUD_PROVIDER_ADAPTER_MODULES, +} = require('./eslint/cloud-provider-sdk-imports.cjs'); module.exports = { root: true, @@ -100,5 +105,21 @@ module.exports = { 'no-restricted-globals': ['error', ...CJS_GLOBALS_IN_ESM], }, }, + { + // Cloud provider SDKs only in adapter modules (ADR 003 §1.2). The + // runtime packages with their own `root: true` config, `ai-bot` and + // `host`, re-declare it. + files: RUNTIME_PACKAGE_FILES, + excludedFiles: ['**/tests/**'], + rules: { + 'no-restricted-imports': ['error', CLOUD_PROVIDER_SDK_IMPORTS], + }, + }, + { + files: CLOUD_PROVIDER_ADAPTER_MODULES, + rules: { + 'no-restricted-imports': 'off', + }, + }, ], }; diff --git a/docs/adr/003-reach-cloud-services-through-adapters.md b/docs/adr/003-reach-cloud-services-through-adapters.md index 3680fe5416f..6ddefef044a 100644 --- a/docs/adr/003-reach-cloud-services-through-adapters.md +++ b/docs/adr/003-reach-cloud-services-through-adapters.md @@ -35,8 +35,9 @@ and only when, they appear in all capitals, as shown here. This ADR governs runtime code: code that runs in a deployed service or in the browser. Infrastructure-as-code, deploy configuration such as -`packages/host/config/deploy.js`, CI workflows, and operational runbooks and -skills are provider-specific by nature, and this ADR does not govern them. +`packages/host/config/deploy.js` and the realm-server's deployed +`scripts/start-*.sh`, CI workflows, and operational runbooks and skills are +provider-specific by nature, and this ADR does not govern them. ### 1.2. Proprietary services sit behind an adapter @@ -49,9 +50,10 @@ provider error codes. An adapter MUST translate provider errors into the interface's own results or errors. A provider SDK (`@aws-sdk/*`, and the SDK of any other cloud provider) MUST be -imported only in an adapter module. A lint rule MUST enforce this. The rule's -allowlist names each adapter module. A change to the allowlist is a reviewed -change to the boundary. +imported only in an adapter module, or in a module that speaks the S3 API +(§1.3). A lint rule MUST enforce this. The rule's allowlist, +`eslint/cloud-provider-sdk-imports.cjs`, names each such module. A change to +the allowlist is a reviewed change to the boundary. ### 1.3. Open protocols are the boundary @@ -66,7 +68,7 @@ An S3 client MUST take its endpoint, region, and path-style addressing from configuration, so that it can reach any S3-compatible store. It MUST get credentials from configuration or the SDK's default chain, and it MUST NOT assume a particular source for them. The runtime SHOULD construct S3 clients -through one shared helper. +through `packages/realm-server/lib/s3-client.ts`. ### 1.4. Every capability runs locally @@ -79,20 +81,20 @@ local-disk adapter. One factory per capability MUST choose the adapter from configuration at startup. Code outside the factory MUST depend on the interface, and it MUST -NOT know which adapter is in use. Partial or conflicting configuration MUST -fail startup. It MUST NOT fall back silently. Settings that only one adapter -reads MAY be named for its provider, such as `SECRETS_KMS_KEY_ARN`. Only that -adapter's factory MAY read them. +NOT know which adapter is in use. Incomplete configuration for an adapter MUST +fail startup. It MUST NOT fall back silently to another adapter. Settings that +only one adapter reads MAY be named for its provider, such as +`SECRETS_KMS_KEY_ARN`. Only that adapter's factory MAY read them. ### 1.6. No assumptions about the hosting platform Runtime code MUST NOT detect its platform from environment variables that a -provider injects, such as `ECS_CONTAINER_METADATA_URI`. It MUST NOT change -behavior based on provider-specific request headers or user agents, such as -`ELB-HealthChecker`. A fact about the deployment, such as an instance id or -whether the server is deployed, MUST come from a provider-neutral setting that -the deployment configuration sets. Health and readiness MUST be served on -paths that any load balancer or orchestrator can probe. +provider injects, such as `ECS_CONTAINER_METADATA_URI`. Code MUST NOT name a +provider-specific request header or user agent, such as `ELB-HealthChecker`. +When behavior depends on one, the deployment configuration MUST name it. A +fact about the deployment, such as an instance id or whether the server is +deployed, MUST come from a provider-neutral setting or derivation, such as the +server's own URL. ### 1.7. New dependencies @@ -148,21 +150,8 @@ it. - **Self-hosted open-source services only, such as MinIO or Vault, in production.** This avoids proprietary APIs, but at an operational cost that adapters make unnecessary. - -## 4. Consequences - -The runtime does not yet meet this ADR in these places. Each is follow-up -work: - -- No lint rule restricts provider SDK imports (§1.2). -- `S3MediaCacheAdapter` (`packages/realm-server/media-cache/s3-adapter.ts`) - and the prerender artifact sink - (`packages/realm-server/prerender/artifact-sink.ts`) construct `S3Client` - with a region only, and no endpoint can be configured (§1.3). -- `worker.ts` and `worker-manager.ts` derive worker ids from - `ECS_CONTAINER_METADATA_URI`. `worker-manager.ts` enables the local indexing - dashboard when that variable is absent (§1.6). -- `healthCheck` in `packages/realm-server/middleware/index.ts` answers only - requests whose user agent is `ELB-HealthChecker` (§1.6). -- The `ecsMetadata` middleware sends an `X-ECS-Container-Metadata-URI-v4` - response header (§1.6). +- **A dedicated health-check path instead of a configured user agent.** This + is the more common pattern. But it moves the load balancer's probe, so the + application and the infrastructure must change in a set order. A + configured user agent keeps the probe where it is, and a different load + balancer needs only a new setting. diff --git a/docs/realm-server-health-signals.md b/docs/realm-server-health-signals.md index e709976feb7..1cac26555fd 100644 --- a/docs/realm-server-health-signals.md +++ b/docs/realm-server-health-signals.md @@ -15,9 +15,10 @@ signal answers which question, and what to do with each. ### `GET /` — the load balancer's check `healthCheck` in `packages/realm-server/middleware/index.ts` answers `OK` from -memory to any request whose user-agent starts with `ELB-HealthChecker`, ahead of -the host-app and realm-serving middleware. It touches no database, no filesystem, -and no realm state. +memory to any request whose user-agent starts with a prefix listed in +`HEALTH_CHECK_USER_AGENTS`, ahead of the host-app and realm-serving middleware. +The deployed start scripts set it to `ELB-HealthChecker`. It touches no +database, no filesystem, and no realm state. It is served **by the event loop**, so it answers only when the loop has capacity to answer. That is what makes it the right signal for routing: a target that diff --git a/eslint/cloud-provider-sdk-imports.cjs b/eslint/cloud-provider-sdk-imports.cjs new file mode 100644 index 00000000000..5af0fbf328c --- /dev/null +++ b/eslint/cloud-provider-sdk-imports.cjs @@ -0,0 +1,42 @@ +'use strict'; + +// Keeps cloud provider SDKs inside the modules that adapt them (ADR 003 +// §1.2). Runtime code reaches a proprietary cloud service only through an +// interface named for the capability Boxel needs, so the import of a +// provider's SDK belongs only in that interface's adapter, or in a module +// that speaks the S3 API (ADR 003 §1.3). +// +// Adding a module to CLOUD_PROVIDER_ADAPTER_MODULES moves the boundary, so it +// is a reviewed change, not a way to silence the rule. + +const CLOUD_PROVIDER_SDK_IMPORTS = { + patterns: [ + { + group: ['@aws-sdk/*', 'aws-sdk', '@google-cloud/*', '@azure/*'], + message: + 'Import a cloud provider SDK only in an adapter module (ADR 003 §1.2). Put the call behind an interface named for the capability, or add an adapter module to CLOUD_PROVIDER_ADAPTER_MODULES in eslint/cloud-provider-sdk-imports.cjs.', + }, + ], +}; + +// The runtime packages ADR 003 governs, relative to the repository root. +const RUNTIME_PACKAGE_FILES = [ + 'packages/realm-server/**/*', + 'packages/runtime-common/**/*', + 'packages/postgres/**/*', + 'packages/billing/**/*', + 'packages/bot-runner/**/*', +]; + +const CLOUD_PROVIDER_ADAPTER_MODULES = [ + 'packages/realm-server/lib/s3-client.ts', + 'packages/realm-server/lib/secrets-at-rest/providers.ts', + 'packages/realm-server/media-cache/s3-adapter.ts', + 'packages/realm-server/prerender/artifact-sink.ts', +]; + +module.exports = { + CLOUD_PROVIDER_SDK_IMPORTS, + RUNTIME_PACKAGE_FILES, + CLOUD_PROVIDER_ADAPTER_MODULES, +}; diff --git a/packages/ai-bot/.eslintrc.cjs b/packages/ai-bot/.eslintrc.cjs index 5b111560323..522e9b0bb17 100644 --- a/packages/ai-bot/.eslintrc.cjs +++ b/packages/ai-bot/.eslintrc.cjs @@ -1,11 +1,15 @@ 'use strict'; // `root: true` here, so the root config does not apply and the -// erasable-syntax rule must be re-declared from the shared list. +// erasable-syntax and cloud-provider-SDK rules must be re-declared from the +// shared lists. const { NO_COMPILATION_REQUIRED_TS_SELECTORS, CJS_GLOBALS_IN_ESM, } = require('../../eslint/erasable-syntax-selectors.cjs'); +const { + CLOUD_PROVIDER_SDK_IMPORTS, +} = require('../../eslint/cloud-provider-sdk-imports.cjs'); module.exports = { root: true, @@ -60,6 +64,8 @@ module.exports = { { prefer: 'class-property' }, ], 'no-restricted-syntax': ['error', ...NO_COMPILATION_REQUIRED_TS_SELECTORS], + // Re-declared from the root config (ADR 003 §1.2). + 'no-restricted-imports': ['error', CLOUD_PROVIDER_SDK_IMPORTS], }, overrides: [ { diff --git a/packages/host/.eslintrc.js b/packages/host/.eslintrc.js index 7bd718f9cd8..89f2c0b8af9 100644 --- a/packages/host/.eslintrc.js +++ b/packages/host/.eslintrc.js @@ -2,6 +2,9 @@ const MISSING_INVOKABLES_CONFIG = require('../runtime-common/etc/eslint/missing-invokables-config'); const { DATA_TEST_SELECTORS } = require('../../eslint/data-test-selectors.cjs'); +const { + CLOUD_PROVIDER_SDK_IMPORTS, +} = require('../../eslint/cloud-provider-sdk-imports.cjs'); // Applies to all of JS, TS, GJS, and GTS in the browser context. const sharedBrowserConfig = { @@ -76,6 +79,14 @@ module.exports = { browser: true, }, overrides: [ + { + // Re-declared from the root config, which this `root: true` config does + // not inherit (ADR 003 §1.2). + files: ['app/**/*'], + rules: { + 'no-restricted-imports': ['error', CLOUD_PROVIDER_SDK_IMPORTS], + }, + }, { // Disallow `data-test-*` CSS/DOM selectors in app code. ember-test-selectors // strips these attributes in production, so selectors like diff --git a/packages/observability/README.md b/packages/observability/README.md index f5df7f36580..8c5759708f6 100644 --- a/packages/observability/README.md +++ b/packages/observability/README.md @@ -268,12 +268,12 @@ you change it, change it everywhere — `alloy/config.alloy` (local) and `cardstack/infra:modules/aws/ecs/firelens/templates/extra.conf.tftpl` (staging + production). -| Label | Local source | Staging / production source | When set | -| ----------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------- | -| `env` | constant `local` — Alloy relabel rule (Docker source) and inline target attribute (file source) | constant `staging` / `production` (Fluent Bit static `Labels`) | always | -| `service` | Docker container name, leading `/` stripped (Docker source) — or `realm-server` / `worker` / `prerender` / `prerender-manager` (file source, mise dev tasks) | ECS task family — `realm-server`, `worker`, `prerender`, `prerender-manager`, `synapse` | always | -| `realm` | opt-in via Docker label `boxel.realm=` | task env when the task pins a single realm (omitted on multi-realm workloads) | when meaningful | -| `worker_id` | not set locally | per-process worker id (`-pid-`), parsed from `[worker priority N]:` log line prefixes by a Fluent Bit Lua filter. Matches `job_reservations.worker_id`. | worker tasks only, lines with the prefix | +| Label | Local source | Staging / production source | When set | +| ----------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ---------------------------------------- | +| `env` | constant `local` — Alloy relabel rule (Docker source) and inline target attribute (file source) | constant `staging` / `production` (Fluent Bit static `Labels`) | always | +| `service` | Docker container name, leading `/` stripped (Docker source) — or `realm-server` / `worker` / `prerender` / `prerender-manager` (file source, mise dev tasks) | ECS task family — `realm-server`, `worker`, `prerender`, `prerender-manager`, `synapse` | always | +| `realm` | opt-in via Docker label `boxel.realm=` | task env when the task pins a single realm (omitted on multi-realm workloads) | when meaningful | +| `worker_id` | not set locally | per-process worker id (`-pid-`), parsed from `[worker priority N]:` log line prefixes by a Fluent Bit Lua filter. Matches `job_reservations.worker_id`. | worker tasks only, lines with the prefix | The local Alloy scraper drops the observability stack's own Compose services (`grafana`, `loki`, `alloy`) so `{env="local"}` queries don't @@ -356,7 +356,7 @@ window, not the query window. # Per-worker tail (worker_id matches job_reservations.worker_id — # look it up there, e.g. for the worker that ran a specific job). -{env="", service="worker", worker_id="abc123-3236013547-pid-42"} +{env="", service="worker", worker_id="ip-10-0-1-23.ec2.internal-9f3c2a1b-pid-42"} # Logs around a specific job id (LogQL line-filter, not regex) {env="", service="worker"} |= "job_id=42" diff --git a/packages/realm-server/lib/instance-id.ts b/packages/realm-server/lib/instance-id.ts new file mode 100644 index 00000000000..90f7aa932e8 --- /dev/null +++ b/packages/realm-server/lib/instance-id.ts @@ -0,0 +1,27 @@ +import { randomBytes } from 'node:crypto'; +import { hostname } from 'node:os'; + +// Names the running instance of a service. Worker ids begin with it, and so +// do `job_reservations.worker_id` and the `worker_id` log label. The +// `X-Boxel-Instance-Id` response header carries it, which tells apart the +// replicas behind a load balancer. +// +// A deployment may set BOXEL_INSTANCE_ID when its platform has a better name. +// Otherwise the id is the hostname, which is unique among running containers +// on every common platform, plus a per-boot suffix, so that an instance on a +// reused hostname never takes an earlier instance's id. Nothing here reads a +// platform-specific variable (ADR 003 §1.6). +// +// The worker manager passes its id to its workers in BOXEL_INSTANCE_ID, so +// every worker id begins with the id of the instance that runs it. +export const INSTANCE_ID_ENV = 'BOXEL_INSTANCE_ID'; +export const INSTANCE_ID_HEADER = 'X-Boxel-Instance-Id'; + +let instanceId: string | undefined; + +export function getInstanceId(): string { + instanceId ??= + process.env[INSTANCE_ID_ENV]?.trim() || + `${hostname()}-${randomBytes(4).toString('hex')}`; + return instanceId; +} diff --git a/packages/realm-server/lib/s3-client.ts b/packages/realm-server/lib/s3-client.ts new file mode 100644 index 00000000000..84e23dd065f --- /dev/null +++ b/packages/realm-server/lib/s3-client.ts @@ -0,0 +1,31 @@ +import { S3Client, type S3ClientConfig } from '@aws-sdk/client-s3'; + +// The one place the runtime constructs S3 clients. Any store that speaks the +// S3 API can stand behind them (ADR 003 §1.3): AWS when S3_ENDPOINT is unset, +// or another S3-compatible store, such as Cloudflare R2, MinIO, or Google +// Cloud Storage's XML API, at the URL it names. Credentials come from the +// SDK's default chain: the task role on ECS, or AWS_ACCESS_KEY_ID and +// AWS_SECRET_ACCESS_KEY for a store that issues keys. +// +// S3_ENDPOINT the store's URL +// S3_FORCE_PATH_STYLE `true` to address buckets by path rather than by +// subdomain, as MinIO and most self-hosted stores need +// AWS_REGION the region, when the caller names none + +const DEFAULT_REGION = 'us-east-1'; + +export function s3ClientConfig( + { region }: { region?: string } = {}, + env: NodeJS.ProcessEnv = process.env, +): S3ClientConfig { + let endpoint = env.S3_ENDPOINT?.trim(); + return { + region: region?.trim() || env.AWS_REGION?.trim() || DEFAULT_REGION, + ...(endpoint ? { endpoint } : {}), + forcePathStyle: env.S3_FORCE_PATH_STYLE?.trim() === 'true', + }; +} + +export function createS3Client(opts: { region?: string } = {}): S3Client { + return new S3Client(s3ClientConfig(opts)); +} diff --git a/packages/realm-server/media-cache/s3-adapter.ts b/packages/realm-server/media-cache/s3-adapter.ts index f290c3ca178..aa3d1843731 100644 --- a/packages/realm-server/media-cache/s3-adapter.ts +++ b/packages/realm-server/media-cache/s3-adapter.ts @@ -1,16 +1,15 @@ // S3-backed MediaCache object store. The bucket holds only content-addressed // derived media (captures); the `media_cache_ledger` table is the sole // catalog of what's in here, so nothing in this adapter ever lists the -// bucket. Region/credential resolution mirrors `prerender/artifact-sink.ts`: -// in ECS the write grant rides on the task role, which the SDK resolves from -// the container credentials endpoint — no keys are configured here. +// bucket. The client comes from `lib/s3-client.ts`, which resolves the +// endpoint, region and credentials for every S3 client in the runtime. import { DeleteObjectCommand, GetObjectCommand, HeadObjectCommand, PutObjectCommand, - S3Client, + type S3Client, } from '@aws-sdk/client-s3'; import type { MediaCacheAdapter, @@ -18,8 +17,7 @@ import type { MediaPutResult, } from '@cardstack/runtime-common'; import type { Readable } from 'stream'; - -const DEFAULT_REGION = 'us-east-1'; +import { createS3Client } from '../lib/s3-client.ts'; export class S3MediaCacheAdapter implements MediaCacheAdapter { #client: S3Client; @@ -42,11 +40,7 @@ export class S3MediaCacheAdapter implements MediaCacheAdapter { }) { this.#bucket = bucket; this.#keyPrefix = keyPrefix; - this.#client = - client ?? - new S3Client({ - region: region ?? process.env.AWS_REGION?.trim() ?? DEFAULT_REGION, - }); + this.#client = client ?? createS3Client({ region }); } private objectKey(key: string): string { diff --git a/packages/realm-server/middleware/index.ts b/packages/realm-server/middleware/index.ts index 4de676cb537..8392409c7e2 100644 --- a/packages/realm-server/middleware/index.ts +++ b/packages/realm-server/middleware/index.ts @@ -19,6 +19,7 @@ import { import { nodeStreamToText, nodeStreamToBuffer } from '../stream.ts'; import { guardDeclaredLength } from '../lib/declared-length-guard.ts'; import { retrieveUserSessionClaim } from '../utils/jwt.ts'; +import { getInstanceId, INSTANCE_ID_HEADER } from '../lib/instance-id.ts'; import { verifyOperatorToken, type OperatorScope, @@ -190,9 +191,18 @@ export function livenessCheck(ctxt: Koa.Context, _next: Koa.Next) { ctxt.set('server', '@cardstack/host'); } -// Respond to AWS ELB health check +// Answers the load balancer's health check from memory, ahead of the host-app +// and realm-serving middleware. The deployment names its health checker's +// user-agent prefixes in HEALTH_CHECK_USER_AGENTS (comma-separated), so the +// code carries no provider's name (ADR 003 §1.6). Read per request, so tests +// and operators can change it without a stale snapshot. export function healthCheck(ctxt: Koa.Context, next: Koa.Next) { - if (ctxt.req.headers['user-agent']?.startsWith('ELB-HealthChecker')) { + let userAgent = ctxt.req.headers['user-agent']; + let prefixes = (process.env.HEALTH_CHECK_USER_AGENTS ?? '') + .split(',') + .map((prefix) => prefix.trim()) + .filter(Boolean); + if (userAgent && prefixes.some((prefix) => userAgent.startsWith(prefix))) { ctxt.body = 'OK'; return; } @@ -489,13 +499,10 @@ async function shedSearch(ctxt: Koa.Context, waitedMs: number) { ); } -export function ecsMetadata(ctxt: Koa.Context, next: Koa.Next) { - if (process.env['ECS_CONTAINER_METADATA_URI_V4']) { - ctxt.set( - 'X-ECS-Container-Metadata-URI-v4', - process.env['ECS_CONTAINER_METADATA_URI_V4'], - ); - } +// Names the replica that answered, so a client behind a load balancer can tell +// replicas apart and see when one is replaced. +export function instanceIdHeader(ctxt: Koa.Context, next: Koa.Next) { + ctxt.set(INSTANCE_ID_HEADER, getInstanceId()); return next(); } diff --git a/packages/realm-server/prerender/artifact-sink.ts b/packages/realm-server/prerender/artifact-sink.ts index bfaa4bfa919..5b12fe9f323 100644 --- a/packages/realm-server/prerender/artifact-sink.ts +++ b/packages/realm-server/prerender/artifact-sink.ts @@ -27,14 +27,14 @@ // budget is a soft ceiling, never a source of truncated/!invalid // blobs). // -// In ECS the bucket write grant rides on the task role, which the AWS SDK -// resolves automatically from the container credentials endpoint — there -// are no access keys or profiles to configure here. +// The client comes from `lib/s3-client.ts`, which resolves the endpoint, +// region and credentials for every S3 client in the runtime. import type { Readable } from 'stream'; -import { S3Client } from '@aws-sdk/client-s3'; +import type { S3Client } from '@aws-sdk/client-s3'; import { Upload } from '@aws-sdk/lib-storage'; import { logger } from '@cardstack/runtime-common'; +import { createS3Client } from '../lib/s3-client.ts'; const log = logger('prerenderer'); @@ -44,9 +44,6 @@ const log = logger('prerenderer'); // the bucket's lifecycle expiry; an operator targeting a long session can // raise it. const DEFAULT_MAX_SESSION_BYTES = 5 * 1024 * 1024 * 1024; // 5 GiB -// Fallback S3 region. Both deployed environments live in us-east-1; an -// operator can override per-process without a code change. -const DEFAULT_REGION = 'us-east-1'; // The artifact formats this sink carries. Each maps to a file suffix the // usual tools recognise: `.cpuprofile` (Chrome DevTools / speedscope), @@ -121,12 +118,8 @@ function artifactEnv(): string { return env.length > 0 ? env : 'unknown'; } -function artifactRegion(): string { - return ( - process.env.PRERENDER_ARTIFACTS_REGION?.trim() || - process.env.AWS_REGION?.trim() || - DEFAULT_REGION - ); +function artifactRegion(): string | undefined { + return process.env.PRERENDER_ARTIFACTS_REGION?.trim() || undefined; } // The configured per-session byte budget, or the built-in default when the @@ -255,7 +248,7 @@ function sanitizeSegment(value: string | undefined): string { let client: S3Client | undefined; function getClient(): S3Client { if (!client) { - client = new S3Client({ region: artifactRegion() }); + client = createS3Client({ region: artifactRegion() }); } return client; } diff --git a/packages/realm-server/prerender/prerender-app.ts b/packages/realm-server/prerender/prerender-app.ts index b91c6f6fb48..6650cf7ffe2 100644 --- a/packages/realm-server/prerender/prerender-app.ts +++ b/packages/realm-server/prerender/prerender-app.ts @@ -24,7 +24,7 @@ import { type CapturePrerenderResponse, } from '@cardstack/runtime-common'; import { - ecsMetadata, + instanceIdHeader, fullRequestURL, livenessCheck, fetchRequestFromContext, @@ -1771,7 +1771,7 @@ export function buildPrerenderApp(options: { }); return next(); }) - .use(ecsMetadata) + .use(instanceIdHeader) .use(router.routes()); app.on('error', (err: any) => { diff --git a/packages/realm-server/scripts/load-harness/README.md b/packages/realm-server/scripts/load-harness/README.md index 08d8078215b..24c7c9f8b4c 100644 --- a/packages/realm-server/scripts/load-harness/README.md +++ b/packages/realm-server/scripts/load-harness/README.md @@ -486,9 +486,9 @@ document at `/_standby`, over plain HTTP, with no AWS session: (`assets/main-.js`) and the host's own build version from its config meta (`0.0.0+`). Either moving means the client half of a measurement changed underneath it. -- **Which replica answered** — the container id in - `X-ECS-Container-Metadata-URI-v4`. A fleet that turned over is visible even - when the task definition did not change, and a replaced task is a cold one. +- **Which replica answered** — the instance id in `X-Boxel-Instance-Id`. A + fleet that turned over is visible even when the build did not change, and a + replaced replica is a cold one. A run opens with the reading printed: diff --git a/packages/realm-server/scripts/load-harness/lib/deploy-pin.ts b/packages/realm-server/scripts/load-harness/lib/deploy-pin.ts index ff766a4d3e9..2a411623248 100644 --- a/packages/realm-server/scripts/load-harness/lib/deploy-pin.ts +++ b/packages/realm-server/scripts/load-harness/lib/deploy-pin.ts @@ -18,10 +18,11 @@ // its config meta (`0.0.0+`). Either one moving means the client half // of any measurement changed underneath it. // -// - WHICH REPLICAS ANSWERED. `X-ECS-Container-Metadata-URI-v4` carries the -// answering container's id. A fleet that turned over mid-run is then -// visible even when the task definition did not change, and a replaced -// task is a cold one, which invalidates the window on its own. +// - WHICH REPLICAS ANSWERED. `X-Boxel-Instance-Id` names the answering +// process, and a restarted or replaced process answers with a new id. A +// fleet that turned over mid-run is then visible even when the build did +// not change, and a replaced replica is a cold one, which invalidates the +// window on its own. // // The realm server fetches the boot document once per process and caches it // for that process's life, so what a replica serves is pinned to when that @@ -51,7 +52,7 @@ export interface ServedBuild { } export interface FleetReading { - // Which replicas answered at all, by container id — including the ones whose + // Which replicas answered at all, by instance id — including the ones whose // answer was unusable, because a replica that says "502" has still said it is // there, and dropping it would report it as departed. A replica that answered // no probe is absent, so this is what was SEEN and never the whole fleet. @@ -66,9 +67,10 @@ export interface FleetReading { responses: number; } -// The key a response with no container id is filed under. A deployment that -// identifies no replicas — a local stack, anything not on ECS — collapses into -// this single entry, so its readings compare builds and never turnover. +// The key a response with no instance id is filed under. A target that +// identifies no replicas — a proxy that strips the header, a server that +// predates it — collapses into this single entry, so its readings compare +// builds and never turnover. export const UNIDENTIFIED_REPLICA = 'unidentified'; // `_standby` is the cheapest document that names the bundle: it serves the @@ -136,15 +138,9 @@ function hostVersion(html: string): string | undefined { } export function replicaIdFromHeaders(headers: Headers): string | undefined { - // The header carries the task metadata endpoint, `…/v4/`, - // which is reachable only from inside the container. The id is the useful - // part: it changes when the task is replaced, which a deploy always does and - // a recycle does too. - let uri = headers.get('x-ecs-container-metadata-uri-v4'); - if (!uri) { - return undefined; - } - return uri.split('/').filter(Boolean).pop() || undefined; + // The id changes when the process restarts or is replaced, which a deploy + // always does and a recycle does too. + return headers.get('x-boxel-instance-id')?.trim() || undefined; } export interface ReadFleetOptions { diff --git a/packages/realm-server/scripts/start-production.sh b/packages/realm-server/scripts/start-production.sh index f1afc0dea8b..100d1e76c25 100755 --- a/packages/realm-server/scripts/start-production.sh +++ b/packages/realm-server/scripts/start-production.sh @@ -33,6 +33,7 @@ BOXEL_HOMEPAGE_REALM_URL="${RESOLVED_BOXEL_HOMEPAGE_REALM_URL:-$DEFAULT_BOXEL_HO echo "[start-production] pid=$$ ppid=$PPID about to exec node at $(date -Iseconds)" >&2 NODE_NO_WARNINGS=1 \ + HEALTH_CHECK_USER_AGENTS='ELB-HealthChecker' \ LOW_CREDIT_THRESHOLD=2000 \ MATRIX_URL=https://matrix.boxel.ai \ MATRIX_SERVER_NAME=boxel.ai \ diff --git a/packages/realm-server/scripts/start-staging.sh b/packages/realm-server/scripts/start-staging.sh index ff4e636e344..f78890405a3 100755 --- a/packages/realm-server/scripts/start-staging.sh +++ b/packages/realm-server/scripts/start-staging.sh @@ -33,6 +33,7 @@ BOXEL_HOMEPAGE_REALM_URL="${RESOLVED_BOXEL_HOMEPAGE_REALM_URL:-$DEFAULT_BOXEL_HO echo "[start-staging] pid=$$ ppid=$PPID about to exec node at $(date -Iseconds)" >&2 NODE_NO_WARNINGS=1 \ + HEALTH_CHECK_USER_AGENTS='ELB-HealthChecker' \ LOW_CREDIT_THRESHOLD=2000 \ MATRIX_URL=https://matrix-staging.stack.cards \ MATRIX_SERVER_NAME=stack.cards \ diff --git a/packages/realm-server/server.ts b/packages/realm-server/server.ts index f88ab7006ef..ecd79917fa1 100644 --- a/packages/realm-server/server.ts +++ b/packages/realm-server/server.ts @@ -25,7 +25,7 @@ import fsExtra from 'fs-extra'; const { ensureDirSync } = fsExtra; import { httpLogging, - ecsMetadata, + instanceIdHeader, searchAdmission, orderConnectionsByRealm, methodOverrideSupport, @@ -931,7 +931,7 @@ export class RealmServer { // headers it sets. .use(clientAddress(this.clientAddress)) .use(httpLogging) - .use(ecsMetadata) + .use(instanceIdHeader) .use( cors({ origin: '*', diff --git a/packages/realm-server/tests/health-check-test.ts b/packages/realm-server/tests/health-check-test.ts new file mode 100644 index 00000000000..f7f64626ced --- /dev/null +++ b/packages/realm-server/tests/health-check-test.ts @@ -0,0 +1,63 @@ +import QUnit from 'qunit'; +const { module, test } = QUnit; +import { basename } from 'path'; +import type Koa from 'koa'; +import { healthCheck } from '../middleware/index.ts'; + +function probe(userAgent: string | undefined) { + let ctxt = { + req: { headers: userAgent ? { 'user-agent': userAgent } : {} }, + body: undefined as unknown, + } as unknown as Koa.Context; + let passedOn = false; + healthCheck(ctxt, async () => { + passedOn = true; + }); + return { answered: ctxt.body, passedOn }; +} + +module(basename(import.meta.filename), function (hooks) { + let original: string | undefined; + hooks.beforeEach(function () { + original = process.env.HEALTH_CHECK_USER_AGENTS; + }); + hooks.afterEach(function () { + if (original === undefined) { + delete process.env.HEALTH_CHECK_USER_AGENTS; + } else { + process.env.HEALTH_CHECK_USER_AGENTS = original; + } + }); + + test('answers a configured health checker from memory', function (assert) { + process.env.HEALTH_CHECK_USER_AGENTS = 'ELB-HealthChecker, kube-probe/'; + assert.deepEqual(probe('ELB-HealthChecker/2.0'), { + answered: 'OK', + passedOn: false, + }); + assert.deepEqual(probe('kube-probe/1.31'), { + answered: 'OK', + passedOn: false, + }); + }); + + test('passes any other request on', function (assert) { + process.env.HEALTH_CHECK_USER_AGENTS = 'ELB-HealthChecker'; + assert.deepEqual(probe('Mozilla/5.0'), { + answered: undefined, + passedOn: true, + }); + assert.deepEqual(probe(undefined), { + answered: undefined, + passedOn: true, + }); + }); + + test('with no health checker configured, recognizes none', function (assert) { + delete process.env.HEALTH_CHECK_USER_AGENTS; + assert.deepEqual(probe('ELB-HealthChecker/2.0'), { + answered: undefined, + passedOn: true, + }); + }); +}); diff --git a/packages/realm-server/tests/instance-id-test.ts b/packages/realm-server/tests/instance-id-test.ts new file mode 100644 index 00000000000..99be9d2a41b --- /dev/null +++ b/packages/realm-server/tests/instance-id-test.ts @@ -0,0 +1,59 @@ +import QUnit from 'qunit'; +const { module, test } = QUnit; +import { basename, join } from 'path'; +import { hostname } from 'os'; +import { execFileSync } from 'child_process'; +import type Koa from 'koa'; +import { instanceIdHeader } from '../middleware/index.ts'; +import { getInstanceId, INSTANCE_ID_HEADER } from '../lib/instance-id.ts'; + +// The id is fixed once per process, so each case reads it in a fresh one. +function instanceIdOfFreshProcess(env: NodeJS.ProcessEnv): string { + let modulePath = join(import.meta.dirname, '..', 'lib', 'instance-id.ts'); + return execFileSync( + process.execPath, + [ + '--input-type=module', + '--eval', + `import { getInstanceId } from ${JSON.stringify(modulePath)}; + process.stdout.write(getInstanceId());`, + ], + { env, encoding: 'utf8' }, + ); +} + +module(basename(import.meta.filename), function () { + test('is the hostname plus a per-boot suffix by default', function (assert) { + let env = { ...process.env }; + delete env.BOXEL_INSTANCE_ID; + let first = instanceIdOfFreshProcess(env); + let second = instanceIdOfFreshProcess(env); + assert.ok( + first.startsWith(`${hostname()}-`), + `${first} begins with the hostname`, + ); + assert.ok(/-[0-9a-f]{8}$/.test(first), `${first} ends with the suffix`); + assert.notStrictEqual(first, second, 'each boot gets its own id'); + }); + + test('is BOXEL_INSTANCE_ID when the deployment sets it', function (assert) { + assert.strictEqual( + instanceIdOfFreshProcess({ + ...process.env, + BOXEL_INSTANCE_ID: 'replica-7', + }), + 'replica-7', + ); + }); + + test('every response names the instance that sent it', async function (assert) { + let headers: Record = {}; + let ctxt = { + set(name: string, value: string) { + headers[name] = value; + }, + } as unknown as Koa.Context; + await instanceIdHeader(ctxt, async () => {}); + assert.deepEqual(headers, { [INSTANCE_ID_HEADER]: getInstanceId() }); + }); +}); diff --git a/packages/realm-server/tests/load-harness-test.ts b/packages/realm-server/tests/load-harness-test.ts index 4a5c0ae43ec..5cd9b37d6b0 100644 --- a/packages/realm-server/tests/load-harness-test.ts +++ b/packages/realm-server/tests/load-harness-test.ts @@ -1274,10 +1274,7 @@ module(basename(import.meta.filename), function () { } = {}): Response { let headers = new Headers({ 'content-type': 'text/html' }); if (replicaId) { - headers.set( - 'x-ecs-container-metadata-uri-v4', - `http://169.254.170.2/v4/${replicaId}`, - ); + headers.set('x-boxel-instance-id', replicaId); } return new Response(bootDocument(build), { status, headers }); } @@ -1372,15 +1369,14 @@ module(basename(import.meta.filename), function () { assert.true(pinIsReadable(reading({ 'task-a': served }))); }); - test('the replica id comes off the container metadata header', function (assert) { + test('the replica id comes off the instance id header', function (assert) { assert.strictEqual( replicaIdFromHeaders( new Headers({ - 'x-ecs-container-metadata-uri-v4': - 'http://169.254.170.2/v4/6654a7425d164a77803fe13a88ae21ab-3236013547', + 'x-boxel-instance-id': 'ip-10-0-1-23.ec2.internal-9f3c2a1b', }), ), - '6654a7425d164a77803fe13a88ae21ab-3236013547', + 'ip-10-0-1-23.ec2.internal-9f3c2a1b', ); assert.strictEqual( replicaIdFromHeaders(new Headers()), @@ -1566,7 +1562,7 @@ module(basename(import.meta.filename), function () { status: 200, headers: new Headers({ 'content-type': 'text/html', - 'x-ecs-container-metadata-uri-v4': 'http://169.254.170.2/v4/task-b', + 'x-boxel-instance-id': 'task-b', }), }, ); diff --git a/packages/realm-server/tests/s3-client-test.ts b/packages/realm-server/tests/s3-client-test.ts new file mode 100644 index 00000000000..183561cf91f --- /dev/null +++ b/packages/realm-server/tests/s3-client-test.ts @@ -0,0 +1,53 @@ +import QUnit from 'qunit'; +const { module, test } = QUnit; +import { basename } from 'path'; +import { s3ClientConfig } from '../lib/s3-client.ts'; + +module(basename(import.meta.filename), function () { + test('with no settings, reaches AWS in the default region', function (assert) { + assert.deepEqual(s3ClientConfig({}, {}), { + region: 'us-east-1', + forcePathStyle: false, + }); + }); + + test('takes the region from AWS_REGION when the caller names none', function (assert) { + assert.strictEqual( + s3ClientConfig({}, { AWS_REGION: 'eu-west-1' }).region, + 'eu-west-1', + ); + assert.strictEqual( + s3ClientConfig({ region: 'ap-south-1' }, { AWS_REGION: 'eu-west-1' }) + .region, + 'ap-south-1', + "the caller's region wins", + ); + }); + + test('reaches an S3-compatible store at S3_ENDPOINT', function (assert) { + assert.deepEqual( + s3ClientConfig( + { region: 'auto' }, + { + S3_ENDPOINT: 'http://localhost:9000', + S3_FORCE_PATH_STYLE: 'true', + }, + ), + { + region: 'auto', + endpoint: 'http://localhost:9000', + forcePathStyle: true, + }, + ); + }); + + test('a blank setting counts as unset', function (assert) { + assert.deepEqual( + s3ClientConfig( + { region: ' ' }, + { S3_ENDPOINT: ' ', AWS_REGION: '', S3_FORCE_PATH_STYLE: ' ' }, + ), + { region: 'us-east-1', forcePathStyle: false }, + ); + }); +}); diff --git a/packages/realm-server/worker-manager.ts b/packages/realm-server/worker-manager.ts index 790c9e4a8f9..d0684e02785 100644 --- a/packages/realm-server/worker-manager.ts +++ b/packages/realm-server/worker-manager.ts @@ -60,7 +60,7 @@ import pluralize from 'pluralize'; import Koa from 'koa'; import Router from '@koa/router'; import { - ecsMetadata, + instanceIdHeader, fullRequestURL, livenessCheck, } from './middleware/index.ts'; @@ -91,6 +91,7 @@ import { } from './lib/worker-request-forwarder.ts'; import { isDeployedServerURL } from './lib/outbound/client.ts'; import { withCredentialHeadersMasked } from './lib/request-telemetry.ts'; +import { getInstanceId, INSTANCE_ID_ENV } from './lib/instance-id.ts'; import { encodeSecretsKeyringIpc, SECRETS_KEYRING_REQUEST, @@ -120,12 +121,6 @@ const WORKER_START_TIMEOUT_MS = (() => { return Number.isFinite(parsed) && parsed > 0 ? parsed : 30_000; })(); -// This is an ENV var we get from ECS that looks like: -// http://169.254.170.2/v3/a1de500d004f49bea02ace30cefb0f01-3236013547 where the -// last segment is the "container runtime ID", where the value on the left of -// the '-' is the task ID. -const ECS_CONTAINER_METADATA_URI = process.env.ECS_CONTAINER_METADATA_URI; - const REALM_SECRET_SEED = process.env.REALM_SECRET_SEED; if (!REALM_SECRET_SEED) { log.error( @@ -210,8 +205,11 @@ let { let isReady = false; let isExiting = false; let workers: ChildProcess[] = []; +// The indexing dashboard is for development. A worker manager is deployed when +// any realm it serves is mapped to a deployed origin, the same test that the +// secrets-at-rest startup check uses. function isIndexingDashboardEnabled(): boolean { - return !ECS_CONTAINER_METADATA_URI; + return !toUrls.some((to) => isDeployedServerURL(String(to))); } // Always create the sink — its in-memory state is small, and the @@ -323,7 +321,7 @@ if (port != null) { }); return next(); }) - .use(ecsMetadata); + .use(instanceIdHeader); webServer.on('error', (err: any) => { log.error(`worker manager HTTP server error: ${err.message}`); @@ -858,13 +856,10 @@ async function startWorker( ], { stdio: ['pipe', 'pipe', 'pipe', 'ipc'], + env: { ...process.env, [INSTANCE_ID_ENV]: getInstanceId() }, }, ); - let name = String( - (ECS_CONTAINER_METADATA_URI - ? `${ECS_CONTAINER_METADATA_URI.split('/').pop()!}-pid-${worker.pid}` - : worker.pid)!, - ); + let name = `${getInstanceId()}-pid-${worker.pid}`; let watchdog: NodeJS.Timeout; let workerId: string | undefined; diff --git a/packages/realm-server/worker.ts b/packages/realm-server/worker.ts index d2193ecc356..fe7b8da0c12 100644 --- a/packages/realm-server/worker.ts +++ b/packages/realm-server/worker.ts @@ -51,6 +51,7 @@ import { buildCreatePrerenderAuth } from './prerender/auth.ts'; import { createMediaCacheAdapterFromEnv } from './media-cache/index.ts'; import { finalizeChildReservationAsFailure } from './lib/finalize-child-fatal-failure.ts'; import { receiveSecretsKeyring } from './lib/secrets-at-rest/index.ts'; +import { getInstanceId } from './lib/instance-id.ts'; let log = logger('worker'); @@ -69,14 +70,7 @@ if (!REALM_SERVER_MATRIX_USERNAME) { process.exit(-1); } -// This is an ENV var we get from ECS that looks like: -// http://169.254.170.2/v3/a1de500d004f49bea02ace30cefb0f01-3236013547 where the -// last segment is the "container runtime ID", where the value on the left of -// the '-' is the task ID. -const ECS_CONTAINER_METADATA_URI = process.env.ECS_CONTAINER_METADATA_URI; -let workerId = ECS_CONTAINER_METADATA_URI - ? `${ECS_CONTAINER_METADATA_URI.split('/').pop()!}-pid-${process.pid}` - : `worker-pid-${process.pid}`; +let workerId = `${getInstanceId()}-pid-${process.pid}`; let { matrixURL, From bf716fd732e68c1370aa54fbf5bc7a00359622a4 Mon Sep 17 00:00:00 2001 From: Luke Melia Date: Mon, 19 Oct 2026 12:51:14 -0400 Subject: [PATCH 03/13] Ground ADR 003's rationale in realm server federation Realm servers that people outside our organization host may not run on AWS, and they run the same code we do. That is the main reason the runtime must not depend on one provider. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01ABJLVRj5d5YngGbpKqi4MP --- ...3-reach-cloud-services-through-adapters.md | 39 ++++++++++++------- 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/docs/adr/003-reach-cloud-services-through-adapters.md b/docs/adr/003-reach-cloud-services-through-adapters.md index 6ddefef044a..76340771d7a 100644 --- a/docs/adr/003-reach-cloud-services-through-adapters.md +++ b/docs/adr/003-reach-cloud-services-through-adapters.md @@ -13,16 +13,19 @@ scope: # 003. Reach proprietary cloud services only through adapters -Boxel is deployed on AWS. This is a deployment choice, so the runtime code must -not depend on it. Each proprietary cloud service the runtime uses sits behind an +Our organization deploys Boxel on AWS, but Boxel is designed as a federation +of realm servers that other people host +([ADR 004](004-federate-realm-servers-each-paired-with-its-own-homeserver.md)), +and they may not use AWS. So the runtime code must not depend on any one +provider. Each proprietary cloud service the runtime uses sits behind an interface. The interface is named for the capability that Boxel needs. An adapter implements the interface for one provider, and a local adapter implements it for development and tests. Services that speak an open protocol (Postgres, a POSIX filesystem, HTTP, the S3 API) need no interface, but the code must use only what that protocol defines. The runtime must not detect -which platform hosts it. To move to a different provider, a team writes -adapters and changes configuration. It does not rewrite the code that calls the -adapters. +which platform hosts it. To run a realm server on a different provider, a host +writes adapters and changes configuration. It does not rewrite the code that +calls the adapters. ## 1. Decision @@ -105,13 +108,23 @@ deployment needs one. ## 2. Rationale -Boxel will not necessarily stay on its current provider. Cost, a customer's -data-residency or self-hosting requirement, or a service that another provider -offers better can each make a move worthwhile. Each piece of code that is -coupled to one provider makes a move more expensive. Coupling also spreads, -because new code copies the code around it. Today only a few runtime modules -touch AWS. This makes the rule cheap to adopt now, and the cost of adopting it -will only increase. +Boxel is designed as a federation of realm servers (ADR 004). People and +organizations other than ours will host their own realm servers, which work +with ours and with the rest of Boxel. They will pick their own hosting: another +cloud provider, their own data center, or a single machine. The realm server +they run is the same code we run, so every AWS assumption in it becomes a +barrier to joining the federation. A host that has to fork the code to replace +one service falls behind our releases, and out of step with the servers it +federates with. + +The same rule keeps our own options open. Cost, a customer's data-residency +requirement, or a service that another provider offers better can each make a +move worthwhile for us too. + +Each piece of code that is coupled to one provider raises the cost of hosting +anywhere else. Coupling also spreads, because new code copies the code around +it. Today only a few runtime modules touch AWS. This makes the rule cheap to +adopt now, and the cost of adopting it will only increase. The two existing seams show the shape of the rule. `MediaCacheAdapter` and `SecretsKeyProvider` each have a cloud adapter and a local one. A factory @@ -136,7 +149,7 @@ it. (chosen).** Portability work goes only where it is needed, and code stays direct where a standard already makes it portable. - **Call the provider directly, and port when needed.** This costs nothing - now, but coupling spreads. Assumptions about the platform, such as which + now, but the port would fall to each outside host, and coupling spreads. Assumptions about the platform, such as which environment variables exist, stay invisible until the code runs somewhere else. - **A generic multi-cloud library.** Its lowest-common-denominator API is From 56954f4fe503e2b3efe0a906700dfa8f865a5b3c Mon Sep 17 00:00:00 2001 From: Luke Melia Date: Mon, 19 Oct 2026 12:55:37 -0400 Subject: [PATCH 04/13] Describe --worker-id by the new worker id format Worker ids are now -pid- rather than an ECS task id. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01ABJLVRj5d5YngGbpKqi4MP --- .claude/skills/tail-logs/SKILL.md | 38 +++++++++++++++---------------- packages/observability/README.md | 2 +- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/.claude/skills/tail-logs/SKILL.md b/.claude/skills/tail-logs/SKILL.md index 8974cf5c9bb..b9b8ee2c1f5 100644 --- a/.claude/skills/tail-logs/SKILL.md +++ b/.claude/skills/tail-logs/SKILL.md @@ -15,7 +15,7 @@ Tail Loki logs from a laptop. The script handles auth (SSM bearer token), URL re - Correlating a request id, job id, or realm name across services - Cross-checking that something just shipped a log line you expect -If the user is asking what *changed* (deploy diffs, code history) rather than what *happened*, this is the wrong tool — go to git/PR APIs instead. +If the user is asking what _changed_ (deploy diffs, code history) rather than what _happened_, this is the wrong tool — go to git/PR APIs instead. ## How to invoke @@ -39,19 +39,19 @@ Common flag patterns: ./scripts/tail-logs.sh --env production --service synapse --since 30m --no-follow --confirm ``` -| Flag | Default | Notes | -|------|---------|-------| -| `--env` | (required) | `local`, `staging`, `production` | -| `--service` | (required) | `realm-server`, `worker`, `prerender`, `prerender-manager`, `synapse` (or any local container name) | -| `--realm` | unset | Filter to a single realm. Only set on tasks where realm is meaningful. | -| `--worker-id` | unset | Per-Fargate-task id; only available on workers. | -| `--filter` | unset | LogQL line-filter (`\|=` literal substring). Mutually exclusive with `--regex`. | -| `--regex` | unset | LogQL line-regex (`\|~`). | -| `--since` | `15m` | `30s`, `15m`, `1h`, `2d` — anything matching `^\d+[smhd]$`. | -| `--limit` | `200` | Max lines per batch. | -| `--no-follow` | follow on | One-shot mode for diagnostics. **Always use `--no-follow` from a skill** — follow mode is interactive. | -| `--json` | text | Raw Loki response per batch. Use when you need to parse beyond the formatted line. | -| `--confirm` | n/a | Required for `--env production`. | +| Flag | Default | Notes | +| ------------- | ---------- | ------------------------------------------------------------------------------------------------------ | +| `--env` | (required) | `local`, `staging`, `production` | +| `--service` | (required) | `realm-server`, `worker`, `prerender`, `prerender-manager`, `synapse` (or any local container name) | +| `--realm` | unset | Filter to a single realm. Only set on tasks where realm is meaningful. | +| `--worker-id` | unset | One worker process, `-pid-`; only available on workers. | +| `--filter` | unset | LogQL line-filter (`\|=` literal substring). Mutually exclusive with `--regex`. | +| `--regex` | unset | LogQL line-regex (`\|~`). | +| `--since` | `15m` | `30s`, `15m`, `1h`, `2d` — anything matching `^\d+[smhd]$`. | +| `--limit` | `200` | Max lines per batch. | +| `--no-follow` | follow on | One-shot mode for diagnostics. **Always use `--no-follow` from a skill** — follow mode is interactive. | +| `--json` | text | Raw Loki response per batch. Use when you need to parse beyond the formatted line. | +| `--confirm` | n/a | Required for `--env production`. | ## Auth requirements @@ -69,12 +69,12 @@ Labels are sorted alphabetically inside the brackets. With `--json`, you get the ## Patterns by task family -| You want to... | Suggested invocation | -|---|---| +| You want to... | Suggested invocation | +| ----------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | Find the last 100 errors in any staging service | `--env staging --service --regex '(?i)error\|exception\|fatal' --since 1h --limit 100 --no-follow` (run per service; LogQL doesn't take an OR over labels in one query) | -| Watch a slow indexer batch | `--env staging --service realm-server --regex 'duration_ms=[0-9]{4,}'` | -| See what one Fargate worker is doing | `--env staging --service worker --worker-id --since 30m --no-follow` | -| Tail synapse | `--env staging --service synapse --since 15m --no-follow` | +| Watch a slow indexer batch | `--env staging --service realm-server --regex 'duration_ms=[0-9]{4,}'` | +| See what one Fargate worker is doing | `--env staging --service worker --worker-id --since 30m --no-follow` | +| Tail synapse | `--env staging --service synapse --since 15m --no-follow` | ## Limits and caveats diff --git a/packages/observability/README.md b/packages/observability/README.md index 8c5759708f6..06439873440 100644 --- a/packages/observability/README.md +++ b/packages/observability/README.md @@ -401,7 +401,7 @@ URL from SSM. | `--env` | (required) | `local`, `staging`, `production` | | `--service` | (required) | `realm-server`, `worker`, `prerender`, `prerender-manager`, `synapse` (or any local container name) | | `--realm` | unset | Restrict to a single realm. | -| `--worker-id` | unset | Per-Fargate-task id; workers only. | +| `--worker-id` | unset | One worker process, `-pid-`; workers only. | | `--filter` | unset | LogQL line-filter (`\|=`); literal substring. | | `--regex` | unset | LogQL line-regex (`\|~`). Mutually exclusive with `--filter`. | | `--since` | `15m` | `30s`, `15m`, `1h`, `2d` — pattern `^\d+[smhd]$`. | From 8bcccceed7b5e73685cb5f4c614bda853bd9c029 Mon Sep 17 00:00:00 2001 From: Luke Melia Date: Mon, 19 Oct 2026 14:31:40 -0400 Subject: [PATCH 05/13] Rewrap a line in ADR 003's considered options Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01ABJLVRj5d5YngGbpKqi4MP --- docs/adr/003-reach-cloud-services-through-adapters.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/adr/003-reach-cloud-services-through-adapters.md b/docs/adr/003-reach-cloud-services-through-adapters.md index 76340771d7a..444a2e0ed32 100644 --- a/docs/adr/003-reach-cloud-services-through-adapters.md +++ b/docs/adr/003-reach-cloud-services-through-adapters.md @@ -149,9 +149,9 @@ it. (chosen).** Portability work goes only where it is needed, and code stays direct where a standard already makes it portable. - **Call the provider directly, and port when needed.** This costs nothing - now, but the port would fall to each outside host, and coupling spreads. Assumptions about the platform, such as which - environment variables exist, stay invisible until the code runs somewhere - else. + now, but the port would fall to each outside host, and coupling spreads. + Assumptions about the platform, such as which environment variables exist, + stay invisible until the code runs somewhere else. - **A generic multi-cloud library.** Its lowest-common-denominator API is wider than what Boxel needs. It does not model capabilities such as envelope-key unwrapping, and it adds a dependency that must track every From 73b4cb20ccb9e2ba241b581f17750090c7f6725a Mon Sep 17 00:00:00 2001 From: Luke Melia Date: Mon, 19 Oct 2026 15:35:48 -0400 Subject: [PATCH 06/13] Label workers in Grafana by their per-boot suffix and pid Worker ids now begin with the hostname, which on Fargate is the task's private-IP DNS name. The first six characters, which the job panels used as the label, are then the same for every worker. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01ABJLVRj5d5YngGbpKqi4MP --- .../resources/dashboards/boxel-status/indexing.json | 8 ++++---- .../resources/dashboards/boxel-status/job-queue.json | 4 ++-- .../resources/dashboards/boxel-status/prerender-html.json | 8 ++++---- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/packages/observability/grafanactl/resources/dashboards/boxel-status/indexing.json b/packages/observability/grafanactl/resources/dashboards/boxel-status/indexing.json index afffd7cf20d..c846949c503 100644 --- a/packages/observability/grafanactl/resources/dashboards/boxel-status/indexing.json +++ b/packages/observability/grafanactl/resources/dashboards/boxel-status/indexing.json @@ -840,7 +840,7 @@ "value": [ { "options": { - "pattern": "^(.{6}).*$", + "pattern": "^.*-([0-9a-f]{8}-pid-\\d+)$", "result": { "index": 0, "text": "View logs ($1)" @@ -1287,7 +1287,7 @@ "value": [ { "options": { - "pattern": "^(.{6}).*$", + "pattern": "^.*-([0-9a-f]{8}-pid-\\d+)$", "result": { "index": 0, "text": "View logs ($1)" @@ -1412,7 +1412,7 @@ "value": [ { "options": { - "pattern": "^(.{6}).*$", + "pattern": "^.*-([0-9a-f]{8}-pid-\\d+)$", "result": { "index": 0, "text": "View logs ($1)" @@ -1549,7 +1549,7 @@ "value": [ { "options": { - "pattern": "^(.{6}).*$", + "pattern": "^.*-([0-9a-f]{8}-pid-\\d+)$", "result": { "index": 0, "text": "View logs ($1)" diff --git a/packages/observability/grafanactl/resources/dashboards/boxel-status/job-queue.json b/packages/observability/grafanactl/resources/dashboards/boxel-status/job-queue.json index 043a8761375..ca6817358a5 100644 --- a/packages/observability/grafanactl/resources/dashboards/boxel-status/job-queue.json +++ b/packages/observability/grafanactl/resources/dashboards/boxel-status/job-queue.json @@ -850,7 +850,7 @@ "value": [ { "options": { - "pattern": "^(.{6}).*$", + "pattern": "^.*-([0-9a-f]{8}-pid-\\d+)$", "result": { "index": 0, "text": "View logs ($1)" @@ -1035,7 +1035,7 @@ "value": [ { "options": { - "pattern": "^(.{6}).*$", + "pattern": "^.*-([0-9a-f]{8}-pid-\\d+)$", "result": { "index": 0, "text": "View logs ($1)" diff --git a/packages/observability/grafanactl/resources/dashboards/boxel-status/prerender-html.json b/packages/observability/grafanactl/resources/dashboards/boxel-status/prerender-html.json index bdeb9f43103..734fcffa676 100644 --- a/packages/observability/grafanactl/resources/dashboards/boxel-status/prerender-html.json +++ b/packages/observability/grafanactl/resources/dashboards/boxel-status/prerender-html.json @@ -947,7 +947,7 @@ "value": [ { "options": { - "pattern": "^(.{6}).*$", + "pattern": "^.*-([0-9a-f]{8}-pid-\\d+)$", "result": { "index": 0, "text": "View logs ($1)" @@ -1407,7 +1407,7 @@ "value": [ { "options": { - "pattern": "^(.{6}).*$", + "pattern": "^.*-([0-9a-f]{8}-pid-\\d+)$", "result": { "index": 0, "text": "View logs ($1)" @@ -1532,7 +1532,7 @@ "value": [ { "options": { - "pattern": "^(.{6}).*$", + "pattern": "^.*-([0-9a-f]{8}-pid-\\d+)$", "result": { "index": 0, "text": "View logs ($1)" @@ -1669,7 +1669,7 @@ "value": [ { "options": { - "pattern": "^(.{6}).*$", + "pattern": "^.*-([0-9a-f]{8}-pid-\\d+)$", "result": { "index": 0, "text": "View logs ($1)" From b5381150281212689c31c34509728384a4b8029d Mon Sep 17 00:00:00 2001 From: Luke Melia Date: Mon, 19 Oct 2026 15:35:49 -0400 Subject: [PATCH 07/13] Run ESLint on bot-runner in CI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ADR 003 §1.2 needs the cloud provider SDK rule enforced in every runtime package, and bot-runner's lint ran only the typechecker. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01ABJLVRj5d5YngGbpKqi4MP --- packages/bot-runner/package.json | 7 ++++++- packages/bot-runner/tests/bot-runner-test.ts | 2 +- pnpm-lock.yaml | 3 +++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/packages/bot-runner/package.json b/packages/bot-runner/package.json index b087a057583..1942a543675 100644 --- a/packages/bot-runner/package.json +++ b/packages/bot-runner/package.json @@ -13,11 +13,16 @@ "@glint/ember-tsc": "catalog:", "@types/node": "catalog:", "@types/qunit": "catalog:", + "concurrently": "catalog:", "qunit": "catalog:" }, "scripts": { "start": "NODE_NO_WARNINGS=1 node main.ts", "test": "NODE_NO_WARNINGS=1 node tests/index.ts", - "lint": "ember-tsc --noEmit" + "lint": "concurrently \"pnpm:lint:*(!fix)\" --names \"lint:\"", + "lint:fix": "concurrently \"pnpm:lint:*:fix\" --names \"fix:\"", + "lint:js": "eslint . --report-unused-disable-directives --cache", + "lint:js:fix": "eslint . --report-unused-disable-directives --fix", + "lint:types": "ember-tsc --noEmit" } } diff --git a/packages/bot-runner/tests/bot-runner-test.ts b/packages/bot-runner/tests/bot-runner-test.ts index 3b57d098709..0a95f860c54 100644 --- a/packages/bot-runner/tests/bot-runner-test.ts +++ b/packages/bot-runner/tests/bot-runner-test.ts @@ -274,7 +274,7 @@ module('timeline handler', () => { allFileContents: [ { filename: 'experiments/MyListing/listing.json', - contents: '{\"data\":{\"type\":\"card\"}}', + contents: '{"data":{"type":"card"}}', }, { filename: 'experiments/MyListing/readme.md', diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a0752a0c836..1bfc2e5db77 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1139,6 +1139,9 @@ importers: '@types/qunit': specifier: 'catalog:' version: 2.19.14 + concurrently: + specifier: 'catalog:' + version: 8.2.2 qunit: specifier: 'catalog:' version: 2.26.0 From 1b195c3797549d58ef2e8c5e70f63823d865ed84 Mon Sep 17 00:00:00 2001 From: Luke Melia Date: Tue, 20 Oct 2026 14:10:00 -0400 Subject: [PATCH 08/13] Add an empty Remediation section to ADR 003 A proposed ADR ends with a Remediation section listing the code in its scope that doesn't conform yet. A survey of ADR 003's scope found none. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01ABJLVRj5d5YngGbpKqi4MP --- docs/adr/003-reach-cloud-services-through-adapters.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/adr/003-reach-cloud-services-through-adapters.md b/docs/adr/003-reach-cloud-services-through-adapters.md index 444a2e0ed32..10eaa86ca95 100644 --- a/docs/adr/003-reach-cloud-services-through-adapters.md +++ b/docs/adr/003-reach-cloud-services-through-adapters.md @@ -168,3 +168,7 @@ it. application and the infrastructure must change in a set order. A configured user agent keeps the probe where it is, and a different load balancer needs only a new setting. + +## 4. Remediation + +No known gaps. From a7e910d7c35ca1cce431b72889b6ffeaf1fe5784 Mon Sep 17 00:00:00 2001 From: Luke Melia Date: Tue, 20 Oct 2026 15:14:32 -0400 Subject: [PATCH 09/13] Keep instance ids unique under BOXEL_INSTANCE_ID, and send a digest A deployment sets BOXEL_INSTANCE_ID once for every replica, and it survives restarts, so using it verbatim gave replicas the same worker ids. The orphan cleanup then closed another replica's live reservations. The setting now replaces only the hostname, and the per-boot suffix is always added. Workers take their manager's whole id from BOXEL_INHERITED_INSTANCE_ID, so the suffix isn't added twice. X-Boxel-Instance-Id carries a 12-hex digest of the instance id rather than the id itself, so the hostname, which on Fargate spells out the task's private IP, stays off public responses. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01ABJLVRj5d5YngGbpKqi4MP --- packages/realm-server/lib/instance-id.ts | 45 ++++++++++++------ packages/realm-server/middleware/index.ts | 8 ++-- .../scripts/load-harness/README.md | 4 +- .../realm-server/tests/instance-id-test.ts | 46 ++++++++++++++----- packages/realm-server/worker-manager.ts | 4 +- 5 files changed, 73 insertions(+), 34 deletions(-) diff --git a/packages/realm-server/lib/instance-id.ts b/packages/realm-server/lib/instance-id.ts index 90f7aa932e8..567a7d362d1 100644 --- a/packages/realm-server/lib/instance-id.ts +++ b/packages/realm-server/lib/instance-id.ts @@ -1,27 +1,44 @@ -import { randomBytes } from 'node:crypto'; +import { createHash, randomBytes } from 'node:crypto'; import { hostname } from 'node:os'; // Names the running instance of a service. Worker ids begin with it, and so -// do `job_reservations.worker_id` and the `worker_id` log label. The -// `X-Boxel-Instance-Id` response header carries it, which tells apart the -// replicas behind a load balancer. +// do `job_reservations.worker_id` and the `worker_id` log label, so it must +// be unique among running instances and across restarts: the orphan cleanup +// closes every open reservation under a worker id. // -// A deployment may set BOXEL_INSTANCE_ID when its platform has a better name. -// Otherwise the id is the hostname, which is unique among running containers -// on every common platform, plus a per-boot suffix, so that an instance on a -// reused hostname never takes an earlier instance's id. Nothing here reads a -// platform-specific variable (ADR 003 §1.6). +// The id is a base name plus a per-boot suffix. The base name is +// BOXEL_INSTANCE_ID when the deployment sets it, as a more readable name than +// the hostname, and the hostname otherwise. The suffix keeps the id unique +// when replicas share the setting, or when an instance reuses a hostname. +// Nothing here reads a platform-specific variable (ADR 003 §1.6). // -// The worker manager passes its id to its workers in BOXEL_INSTANCE_ID, so -// every worker id begins with the id of the instance that runs it. +// The worker manager passes its whole id to its workers in +// BOXEL_INHERITED_INSTANCE_ID, so every worker id begins with the id of the +// instance that runs it. export const INSTANCE_ID_ENV = 'BOXEL_INSTANCE_ID'; +export const INHERITED_INSTANCE_ID_ENV = 'BOXEL_INHERITED_INSTANCE_ID'; export const INSTANCE_ID_HEADER = 'X-Boxel-Instance-Id'; let instanceId: string | undefined; +let publicInstanceId: string | undefined; export function getInstanceId(): string { - instanceId ??= - process.env[INSTANCE_ID_ENV]?.trim() || - `${hostname()}-${randomBytes(4).toString('hex')}`; + if (instanceId === undefined) { + let inherited = process.env[INHERITED_INSTANCE_ID_ENV]?.trim(); + let base = process.env[INSTANCE_ID_ENV]?.trim() || hostname(); + instanceId = inherited || `${base}-${randomBytes(4).toString('hex')}`; + } return instanceId; } + +// What the `X-Boxel-Instance-Id` response header carries: a digest of the +// instance id. It tells replicas apart and changes when one restarts, without +// giving every client the hostname, which on most platforms names the +// instance's private address. +export function getPublicInstanceId(): string { + publicInstanceId ??= createHash('sha256') + .update(getInstanceId()) + .digest('hex') + .slice(0, 12); + return publicInstanceId; +} diff --git a/packages/realm-server/middleware/index.ts b/packages/realm-server/middleware/index.ts index 8392409c7e2..1865d2a6f10 100644 --- a/packages/realm-server/middleware/index.ts +++ b/packages/realm-server/middleware/index.ts @@ -19,7 +19,7 @@ import { import { nodeStreamToText, nodeStreamToBuffer } from '../stream.ts'; import { guardDeclaredLength } from '../lib/declared-length-guard.ts'; import { retrieveUserSessionClaim } from '../utils/jwt.ts'; -import { getInstanceId, INSTANCE_ID_HEADER } from '../lib/instance-id.ts'; +import { getPublicInstanceId, INSTANCE_ID_HEADER } from '../lib/instance-id.ts'; import { verifyOperatorToken, type OperatorScope, @@ -499,10 +499,10 @@ async function shedSearch(ctxt: Koa.Context, waitedMs: number) { ); } -// Names the replica that answered, so a client behind a load balancer can tell -// replicas apart and see when one is replaced. +// Names the replica that answered, by a digest of its instance id, so a client +// behind a load balancer can tell replicas apart and see when one is replaced. export function instanceIdHeader(ctxt: Koa.Context, next: Koa.Next) { - ctxt.set(INSTANCE_ID_HEADER, getInstanceId()); + ctxt.set(INSTANCE_ID_HEADER, getPublicInstanceId()); return next(); } diff --git a/packages/realm-server/scripts/load-harness/README.md b/packages/realm-server/scripts/load-harness/README.md index 24c7c9f8b4c..7bfd5d93d70 100644 --- a/packages/realm-server/scripts/load-harness/README.md +++ b/packages/realm-server/scripts/load-harness/README.md @@ -486,8 +486,8 @@ document at `/_standby`, over plain HTTP, with no AWS session: (`assets/main-.js`) and the host's own build version from its config meta (`0.0.0+`). Either moving means the client half of a measurement changed underneath it. -- **Which replica answered** — the instance id in `X-Boxel-Instance-Id`. A - fleet that turned over is visible even when the build did not change, and a +- **Which replica answered** — the digest of its instance id in + `X-Boxel-Instance-Id`. A fleet that turned over is visible even when the build did not change, and a replaced replica is a cold one. A run opens with the reading printed: diff --git a/packages/realm-server/tests/instance-id-test.ts b/packages/realm-server/tests/instance-id-test.ts index 99be9d2a41b..649e05f5900 100644 --- a/packages/realm-server/tests/instance-id-test.ts +++ b/packages/realm-server/tests/instance-id-test.ts @@ -5,10 +5,17 @@ import { hostname } from 'os'; import { execFileSync } from 'child_process'; import type Koa from 'koa'; import { instanceIdHeader } from '../middleware/index.ts'; -import { getInstanceId, INSTANCE_ID_HEADER } from '../lib/instance-id.ts'; +import { + getInstanceId, + getPublicInstanceId, + INSTANCE_ID_HEADER, +} from '../lib/instance-id.ts'; // The id is fixed once per process, so each case reads it in a fresh one. -function instanceIdOfFreshProcess(env: NodeJS.ProcessEnv): string { +function instanceIdOfFreshProcess(settings: NodeJS.ProcessEnv = {}): string { + let env = { ...process.env }; + delete env.BOXEL_INSTANCE_ID; + delete env.BOXEL_INHERITED_INSTANCE_ID; let modulePath = join(import.meta.dirname, '..', 'lib', 'instance-id.ts'); return execFileSync( process.execPath, @@ -18,16 +25,14 @@ function instanceIdOfFreshProcess(env: NodeJS.ProcessEnv): string { `import { getInstanceId } from ${JSON.stringify(modulePath)}; process.stdout.write(getInstanceId());`, ], - { env, encoding: 'utf8' }, + { env: { ...env, ...settings }, encoding: 'utf8' }, ); } module(basename(import.meta.filename), function () { test('is the hostname plus a per-boot suffix by default', function (assert) { - let env = { ...process.env }; - delete env.BOXEL_INSTANCE_ID; - let first = instanceIdOfFreshProcess(env); - let second = instanceIdOfFreshProcess(env); + let first = instanceIdOfFreshProcess(); + let second = instanceIdOfFreshProcess(); assert.ok( first.startsWith(`${hostname()}-`), `${first} begins with the hostname`, @@ -36,17 +41,28 @@ module(basename(import.meta.filename), function () { assert.notStrictEqual(first, second, 'each boot gets its own id'); }); - test('is BOXEL_INSTANCE_ID when the deployment sets it', function (assert) { + test('BOXEL_INSTANCE_ID replaces the hostname, not the per-boot suffix', function (assert) { + let first = instanceIdOfFreshProcess({ BOXEL_INSTANCE_ID: 'replica-7' }); + let second = instanceIdOfFreshProcess({ BOXEL_INSTANCE_ID: 'replica-7' }); + assert.ok(/^replica-7-[0-9a-f]{8}$/.test(first), `${first} is suffixed`); + assert.notStrictEqual( + first, + second, + 'replicas that share the setting still get their own ids', + ); + }); + + test("a worker takes its manager's id as it is", function (assert) { assert.strictEqual( instanceIdOfFreshProcess({ - ...process.env, BOXEL_INSTANCE_ID: 'replica-7', + BOXEL_INHERITED_INSTANCE_ID: 'replica-7-9f3c2a1b', }), - 'replica-7', + 'replica-7-9f3c2a1b', ); }); - test('every response names the instance that sent it', async function (assert) { + test('every response names the instance that sent it, by digest', async function (assert) { let headers: Record = {}; let ctxt = { set(name: string, value: string) { @@ -54,6 +70,12 @@ module(basename(import.meta.filename), function () { }, } as unknown as Koa.Context; await instanceIdHeader(ctxt, async () => {}); - assert.deepEqual(headers, { [INSTANCE_ID_HEADER]: getInstanceId() }); + assert.deepEqual(headers, { [INSTANCE_ID_HEADER]: getPublicInstanceId() }); + assert.ok(/^[0-9a-f]{12}$/.test(headers[INSTANCE_ID_HEADER])); + assert.false( + headers[INSTANCE_ID_HEADER].includes(hostname()), + 'the hostname stays private', + ); + assert.notStrictEqual(headers[INSTANCE_ID_HEADER], getInstanceId()); }); }); diff --git a/packages/realm-server/worker-manager.ts b/packages/realm-server/worker-manager.ts index d0684e02785..9eb30663e74 100644 --- a/packages/realm-server/worker-manager.ts +++ b/packages/realm-server/worker-manager.ts @@ -91,7 +91,7 @@ import { } from './lib/worker-request-forwarder.ts'; import { isDeployedServerURL } from './lib/outbound/client.ts'; import { withCredentialHeadersMasked } from './lib/request-telemetry.ts'; -import { getInstanceId, INSTANCE_ID_ENV } from './lib/instance-id.ts'; +import { getInstanceId, INHERITED_INSTANCE_ID_ENV } from './lib/instance-id.ts'; import { encodeSecretsKeyringIpc, SECRETS_KEYRING_REQUEST, @@ -856,7 +856,7 @@ async function startWorker( ], { stdio: ['pipe', 'pipe', 'pipe', 'ipc'], - env: { ...process.env, [INSTANCE_ID_ENV]: getInstanceId() }, + env: { ...process.env, [INHERITED_INSTANCE_ID_ENV]: getInstanceId() }, }, ); let name = `${getInstanceId()}-pid-${worker.pid}`; From 0d1d70fdc57a1c6e830d7de30d9d896eb052a69c Mon Sep 17 00:00:00 2001 From: Luke Melia Date: Tue, 20 Oct 2026 15:15:13 -0400 Subject: [PATCH 10/13] Warn when no health checker is configured, and parse its setting once A deployed server that doesn't start through our scripts never sets HEALTH_CHECK_USER_AGENTS, so its load balancer's probe on `/` quietly falls through to the host app. It now logs a warning at boot. The setting is parsed once and kept until its raw value changes, rather than split and trimmed on every request to `/` and `/_standby`. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01ABJLVRj5d5YngGbpKqi4MP --- docs/realm-server-health-signals.md | 3 +- packages/realm-server/main.ts | 11 ++++++ packages/realm-server/middleware/index.ts | 36 ++++++++++++++----- .../realm-server/tests/health-check-test.ts | 13 ++++++- 4 files changed, 52 insertions(+), 11 deletions(-) diff --git a/docs/realm-server-health-signals.md b/docs/realm-server-health-signals.md index 1cac26555fd..99dcb28767a 100644 --- a/docs/realm-server-health-signals.md +++ b/docs/realm-server-health-signals.md @@ -17,7 +17,8 @@ signal answers which question, and what to do with each. `healthCheck` in `packages/realm-server/middleware/index.ts` answers `OK` from memory to any request whose user-agent starts with a prefix listed in `HEALTH_CHECK_USER_AGENTS`, ahead of the host-app and realm-serving middleware. -The deployed start scripts set it to `ELB-HealthChecker`. It touches no +The deployed start scripts set it to `ELB-HealthChecker`, and a deployed server +with it unset logs a warning at boot. It touches no database, no filesystem, and no realm state. It is served **by the event loop**, so it answers only when the loop has capacity diff --git a/packages/realm-server/main.ts b/packages/realm-server/main.ts index 379492435e6..abf722eb4a9 100644 --- a/packages/realm-server/main.ts +++ b/packages/realm-server/main.ts @@ -53,6 +53,7 @@ import { ModuleCacheCoordinator } from './lib/module-cache-coordination.ts'; import { JobsFinishedListener } from './lib/jobs-finished-listener.ts'; import { JobScopedSearchCache } from './job-scoped-search-cache.ts'; import { startHealthSampler } from './health-sampler.ts'; +import { healthCheckUserAgents } from './middleware/index.ts'; import { isDeployedServerURL, startOutboundGuard, @@ -605,6 +606,16 @@ const reportHostShellToManager = async (dbAdapter: PgAdapter) => { console.error(`Secrets-at-rest self-check failed: ${e.message}`); process.exit(-1); } + // Without it, the load balancer's probe on `/` falls through to the host app, + // which is slower and can fail the probe under load or during boot. + if ( + isDeployedServerURL(String(serverURL)) && + healthCheckUserAgents().length === 0 + ) { + console.warn( + `HEALTH_CHECK_USER_AGENTS is not set, so no health check is answered from memory. Set it to your load balancer's health-check user-agent prefix (see docs/realm-server-health-signals.md).`, + ); + } try { await smokeTestHostApp(); } catch (e: any) { diff --git a/packages/realm-server/middleware/index.ts b/packages/realm-server/middleware/index.ts index 1865d2a6f10..9ddf2622b3c 100644 --- a/packages/realm-server/middleware/index.ts +++ b/packages/realm-server/middleware/index.ts @@ -191,18 +191,36 @@ export function livenessCheck(ctxt: Koa.Context, _next: Koa.Next) { ctxt.set('server', '@cardstack/host'); } +let parsedHealthCheckUserAgents: + | { raw: string; prefixes: string[] } + | undefined; + +// The user-agent prefixes the deployment names in HEALTH_CHECK_USER_AGENTS +// (comma-separated), so the code carries no provider's name (ADR 003 §1.6). +// The parse is kept until the setting changes, so tests and operators can +// change it without a restart. +export function healthCheckUserAgents(): string[] { + let raw = process.env.HEALTH_CHECK_USER_AGENTS ?? ''; + if (parsedHealthCheckUserAgents?.raw !== raw) { + parsedHealthCheckUserAgents = { + raw, + prefixes: raw + .split(',') + .map((prefix) => prefix.trim()) + .filter(Boolean), + }; + } + return parsedHealthCheckUserAgents.prefixes; +} + // Answers the load balancer's health check from memory, ahead of the host-app -// and realm-serving middleware. The deployment names its health checker's -// user-agent prefixes in HEALTH_CHECK_USER_AGENTS (comma-separated), so the -// code carries no provider's name (ADR 003 §1.6). Read per request, so tests -// and operators can change it without a stale snapshot. +// and realm-serving middleware. export function healthCheck(ctxt: Koa.Context, next: Koa.Next) { let userAgent = ctxt.req.headers['user-agent']; - let prefixes = (process.env.HEALTH_CHECK_USER_AGENTS ?? '') - .split(',') - .map((prefix) => prefix.trim()) - .filter(Boolean); - if (userAgent && prefixes.some((prefix) => userAgent.startsWith(prefix))) { + if ( + userAgent && + healthCheckUserAgents().some((prefix) => userAgent.startsWith(prefix)) + ) { ctxt.body = 'OK'; return; } diff --git a/packages/realm-server/tests/health-check-test.ts b/packages/realm-server/tests/health-check-test.ts index f7f64626ced..bcccbff70ff 100644 --- a/packages/realm-server/tests/health-check-test.ts +++ b/packages/realm-server/tests/health-check-test.ts @@ -2,7 +2,7 @@ import QUnit from 'qunit'; const { module, test } = QUnit; import { basename } from 'path'; import type Koa from 'koa'; -import { healthCheck } from '../middleware/index.ts'; +import { healthCheck, healthCheckUserAgents } from '../middleware/index.ts'; function probe(userAgent: string | undefined) { let ctxt = { @@ -53,6 +53,17 @@ module(basename(import.meta.filename), function (hooks) { }); }); + test('follows a change to the setting without a restart', function (assert) { + process.env.HEALTH_CHECK_USER_AGENTS = ' ELB-HealthChecker , ,'; + assert.deepEqual(healthCheckUserAgents(), ['ELB-HealthChecker']); + process.env.HEALTH_CHECK_USER_AGENTS = 'kube-probe/'; + assert.deepEqual(healthCheckUserAgents(), ['kube-probe/']); + assert.deepEqual(probe('ELB-HealthChecker/2.0'), { + answered: undefined, + passedOn: true, + }); + }); + test('with no health checker configured, recognizes none', function (assert) { delete process.env.HEALTH_CHECK_USER_AGENTS; assert.deepEqual(probe('ELB-HealthChecker/2.0'), { From 978de53b4481c0cf50da674208aed43f1b6b67f8 Mon Sep 17 00:00:00 2001 From: Luke Melia Date: Tue, 20 Oct 2026 15:15:30 -0400 Subject: [PATCH 11/13] Keep the View logs label on worker ids of any shape The ten worker_id cells label the per-boot suffix and pid when the id has them. A second mapping now catches every other id, such as rows written before the current format, and labels it in full, so no cell loses its View logs label. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01ABJLVRj5d5YngGbpKqi4MP --- .../dashboards/boxel-status/indexing.json | 40 +++++++++++++++++++ .../dashboards/boxel-status/job-queue.json | 20 ++++++++++ .../boxel-status/prerender-html.json | 40 +++++++++++++++++++ 3 files changed, 100 insertions(+) diff --git a/packages/observability/grafanactl/resources/dashboards/boxel-status/indexing.json b/packages/observability/grafanactl/resources/dashboards/boxel-status/indexing.json index c846949c503..ab42230b111 100644 --- a/packages/observability/grafanactl/resources/dashboards/boxel-status/indexing.json +++ b/packages/observability/grafanactl/resources/dashboards/boxel-status/indexing.json @@ -847,6 +847,16 @@ } }, "type": "regex" + }, + { + "options": { + "pattern": "^(.+)$", + "result": { + "index": 1, + "text": "View logs ($1)" + } + }, + "type": "regex" } ] } @@ -1294,6 +1304,16 @@ } }, "type": "regex" + }, + { + "options": { + "pattern": "^(.+)$", + "result": { + "index": 1, + "text": "View logs ($1)" + } + }, + "type": "regex" } ] } @@ -1419,6 +1439,16 @@ } }, "type": "regex" + }, + { + "options": { + "pattern": "^(.+)$", + "result": { + "index": 1, + "text": "View logs ($1)" + } + }, + "type": "regex" } ] } @@ -1556,6 +1586,16 @@ } }, "type": "regex" + }, + { + "options": { + "pattern": "^(.+)$", + "result": { + "index": 1, + "text": "View logs ($1)" + } + }, + "type": "regex" } ] } diff --git a/packages/observability/grafanactl/resources/dashboards/boxel-status/job-queue.json b/packages/observability/grafanactl/resources/dashboards/boxel-status/job-queue.json index ca6817358a5..878c0677772 100644 --- a/packages/observability/grafanactl/resources/dashboards/boxel-status/job-queue.json +++ b/packages/observability/grafanactl/resources/dashboards/boxel-status/job-queue.json @@ -857,6 +857,16 @@ } }, "type": "regex" + }, + { + "options": { + "pattern": "^(.+)$", + "result": { + "index": 1, + "text": "View logs ($1)" + } + }, + "type": "regex" } ] } @@ -1042,6 +1052,16 @@ } }, "type": "regex" + }, + { + "options": { + "pattern": "^(.+)$", + "result": { + "index": 1, + "text": "View logs ($1)" + } + }, + "type": "regex" } ] } diff --git a/packages/observability/grafanactl/resources/dashboards/boxel-status/prerender-html.json b/packages/observability/grafanactl/resources/dashboards/boxel-status/prerender-html.json index 734fcffa676..73390a4c571 100644 --- a/packages/observability/grafanactl/resources/dashboards/boxel-status/prerender-html.json +++ b/packages/observability/grafanactl/resources/dashboards/boxel-status/prerender-html.json @@ -954,6 +954,16 @@ } }, "type": "regex" + }, + { + "options": { + "pattern": "^(.+)$", + "result": { + "index": 1, + "text": "View logs ($1)" + } + }, + "type": "regex" } ] } @@ -1414,6 +1424,16 @@ } }, "type": "regex" + }, + { + "options": { + "pattern": "^(.+)$", + "result": { + "index": 1, + "text": "View logs ($1)" + } + }, + "type": "regex" } ] } @@ -1539,6 +1559,16 @@ } }, "type": "regex" + }, + { + "options": { + "pattern": "^(.+)$", + "result": { + "index": 1, + "text": "View logs ($1)" + } + }, + "type": "regex" } ] } @@ -1676,6 +1706,16 @@ } }, "type": "regex" + }, + { + "options": { + "pattern": "^(.+)$", + "result": { + "index": 1, + "text": "View logs ($1)" + } + }, + "type": "regex" } ] } From 87fe3039b5ed95c543478d123b34167fa85c4573 Mon Sep 17 00:00:00 2001 From: Luke Melia Date: Tue, 20 Oct 2026 15:16:12 -0400 Subject: [PATCH 12/13] Serve the indexing dashboard only when the start command asks Inferring development from the realm URLs counted a self-hosted deployment whose realms map to private addresses as local, and served /_indexing-dashboard, which shows pending jobs and their args. The worker manager now serves it only under --indexingDashboard, which the development worker task passes. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01ABJLVRj5d5YngGbpKqi4MP --- mise-tasks/services/worker | 1 + packages/realm-server/worker-manager.ts | 13 +++++++++---- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/mise-tasks/services/worker b/mise-tasks/services/worker index 73db0df85dd..38abe280064 100755 --- a/mise-tasks/services/worker +++ b/mise-tasks/services/worker @@ -51,6 +51,7 @@ NODE_ENV=development \ --port="${WORKER_PORT}" \ --matrixURL="${MATRIX_URL_VAL}" \ --prerendererUrl="${PRERENDER_MGR_URL}" \ + --indexingDashboard \ \ --fromUrl='https://cardstack.com/base/' \ --toUrl="${REALM_BASE_URL}/base/" \ diff --git a/packages/realm-server/worker-manager.ts b/packages/realm-server/worker-manager.ts index 9eb30663e74..3e7f85f193b 100644 --- a/packages/realm-server/worker-manager.ts +++ b/packages/realm-server/worker-manager.ts @@ -138,6 +138,7 @@ let { fromUrl: fromUrls, toUrl: toUrls, migrateDB, + indexingDashboard, prerendererUrl, serviceName = 'worker', skipPrerenderHtmlRealm: skipPrerenderHtmlRealms = [], @@ -169,6 +170,11 @@ let { 'When this flag is set the database will automatically migrate when server is started', type: 'boolean', }, + indexingDashboard: { + description: + 'Serve the indexing dashboard at /_indexing-dashboard. It shows pending jobs, their realms and their args, so only the development worker manager passes this flag.', + type: 'boolean', + }, fromUrl: { description: 'the source of the realm URL proxy', demandOption: true, @@ -205,11 +211,10 @@ let { let isReady = false; let isExiting = false; let workers: ChildProcess[] = []; -// The indexing dashboard is for development. A worker manager is deployed when -// any realm it serves is mapped to a deployed origin, the same test that the -// secrets-at-rest startup check uses. +// The indexing dashboard is for development, so it is off unless the start +// command asks for it. function isIndexingDashboardEnabled(): boolean { - return !toUrls.some((to) => isDeployedServerURL(String(to))); + return indexingDashboard === true; } // Always create the sink — its in-memory state is small, and the From 0bc2d976328c459eaea56e965092d2d8874ec49d Mon Sep 17 00:00:00 2001 From: Luke Melia Date: Tue, 20 Oct 2026 15:20:55 -0400 Subject: [PATCH 13/13] Catch every way of loading a cloud provider SDK, in every runtime package no-restricted-imports sees only static imports, so a dynamic import() or require() of a provider SDK linted clean. The new @cardstack/boxel/no-cloud-provider-sdk rule reports static imports, re-exports, dynamic import(), require() and require.resolve() whatever require is bound to, and TypeScript's import = require(). The rule covers base, bxl and boxel-ui too, which run in the browser or in the realm server, and ADR 003's scope names them. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01ABJLVRj5d5YngGbpKqi4MP --- .eslintrc.js | 11 +- ...3-reach-cloud-services-through-adapters.md | 3 + eslint/cloud-provider-sdk-imports.cjs | 29 ++--- package.json | 1 + packages/ai-bot/.eslintrc.cjs | 6 +- packages/ai-bot/package.json | 1 + packages/boxel-ui/eslint.config.mjs | 14 +++ packages/eslint-plugin-boxel/README.md | 1 + .../docs/rules/no-cloud-provider-sdk.md | 13 +++ .../lib/rules/no-cloud-provider-sdk.js | 103 ++++++++++++++++++ .../lib/rules/no-cloud-provider-sdk-test.js | 79 ++++++++++++++ packages/host/.eslintrc.js | 4 +- pnpm-lock.yaml | 6 + 13 files changed, 247 insertions(+), 24 deletions(-) create mode 100644 packages/eslint-plugin-boxel/docs/rules/no-cloud-provider-sdk.md create mode 100644 packages/eslint-plugin-boxel/lib/rules/no-cloud-provider-sdk.js create mode 100644 packages/eslint-plugin-boxel/tests/lib/rules/no-cloud-provider-sdk-test.js diff --git a/.eslintrc.js b/.eslintrc.js index 2a277b41bfe..a50bb0e67ba 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -6,7 +6,7 @@ const { } = require('./eslint/erasable-syntax-selectors.cjs'); const { DATA_TEST_SELECTORS } = require('./eslint/data-test-selectors.cjs'); const { - CLOUD_PROVIDER_SDK_IMPORTS, + CLOUD_PROVIDER_SDK_RULE, RUNTIME_PACKAGE_FILES, CLOUD_PROVIDER_ADAPTER_MODULES, } = require('./eslint/cloud-provider-sdk-imports.cjs'); @@ -107,18 +107,19 @@ module.exports = { }, { // Cloud provider SDKs only in adapter modules (ADR 003 §1.2). The - // runtime packages with their own `root: true` config, `ai-bot` and - // `host`, re-declare it. + // runtime packages with their own config, `ai-bot`, `host` and + // `boxel-ui`, re-declare it. files: RUNTIME_PACKAGE_FILES, excludedFiles: ['**/tests/**'], + plugins: ['@cardstack/boxel'], rules: { - 'no-restricted-imports': ['error', CLOUD_PROVIDER_SDK_IMPORTS], + '@cardstack/boxel/no-cloud-provider-sdk': CLOUD_PROVIDER_SDK_RULE, }, }, { files: CLOUD_PROVIDER_ADAPTER_MODULES, rules: { - 'no-restricted-imports': 'off', + '@cardstack/boxel/no-cloud-provider-sdk': 'off', }, }, ], diff --git a/docs/adr/003-reach-cloud-services-through-adapters.md b/docs/adr/003-reach-cloud-services-through-adapters.md index 10eaa86ca95..4f4611ad241 100644 --- a/docs/adr/003-reach-cloud-services-through-adapters.md +++ b/docs/adr/003-reach-cloud-services-through-adapters.md @@ -9,6 +9,9 @@ scope: - packages/ai-bot/** - packages/bot-runner/** - packages/host/app/** + - packages/base/** + - packages/bxl/** + - packages/boxel-ui/** --- # 003. Reach proprietary cloud services only through adapters diff --git a/eslint/cloud-provider-sdk-imports.cjs b/eslint/cloud-provider-sdk-imports.cjs index 5af0fbf328c..fe766081d23 100644 --- a/eslint/cloud-provider-sdk-imports.cjs +++ b/eslint/cloud-provider-sdk-imports.cjs @@ -2,30 +2,31 @@ // Keeps cloud provider SDKs inside the modules that adapt them (ADR 003 // §1.2). Runtime code reaches a proprietary cloud service only through an -// interface named for the capability Boxel needs, so the import of a -// provider's SDK belongs only in that interface's adapter, or in a module -// that speaks the S3 API (ADR 003 §1.3). +// interface named for the capability Boxel needs, so loading a provider's SDK +// belongs only in that interface's adapter, or in a module that speaks the S3 +// API (ADR 003 §1.3). `@cardstack/boxel/no-cloud-provider-sdk` reports every +// way of loading one: static import, re-export, dynamic `import()`, and +// `require()`. // // Adding a module to CLOUD_PROVIDER_ADAPTER_MODULES moves the boundary, so it // is a reviewed change, not a way to silence the rule. -const CLOUD_PROVIDER_SDK_IMPORTS = { - patterns: [ - { - group: ['@aws-sdk/*', 'aws-sdk', '@google-cloud/*', '@azure/*'], - message: - 'Import a cloud provider SDK only in an adapter module (ADR 003 §1.2). Put the call behind an interface named for the capability, or add an adapter module to CLOUD_PROVIDER_ADAPTER_MODULES in eslint/cloud-provider-sdk-imports.cjs.', - }, - ], -}; +const CLOUD_PROVIDER_SDK_RULE = [ + 'error', + { patterns: ['@aws-sdk/*', 'aws-sdk', '@google-cloud/*', '@azure/*'] }, +]; -// The runtime packages ADR 003 governs, relative to the repository root. +// The runtime packages ADR 003 governs that lint through an eslintrc config, +// relative to the repository root. `host` and `ai-bot` are `root: true` and +// `boxel-ui` uses a flat config, so each of those declares the rule itself. const RUNTIME_PACKAGE_FILES = [ 'packages/realm-server/**/*', 'packages/runtime-common/**/*', 'packages/postgres/**/*', 'packages/billing/**/*', 'packages/bot-runner/**/*', + 'packages/base/**/*', + 'packages/bxl/**/*', ]; const CLOUD_PROVIDER_ADAPTER_MODULES = [ @@ -36,7 +37,7 @@ const CLOUD_PROVIDER_ADAPTER_MODULES = [ ]; module.exports = { - CLOUD_PROVIDER_SDK_IMPORTS, + CLOUD_PROVIDER_SDK_RULE, RUNTIME_PACKAGE_FILES, CLOUD_PROVIDER_ADAPTER_MODULES, }; diff --git a/package.json b/package.json index 437ac9b32f5..054cb169660 100644 --- a/package.json +++ b/package.json @@ -26,6 +26,7 @@ "@babel/core": "catalog:", "@babel/parser": "catalog:", "@babel/traverse": "catalog:", + "@cardstack/eslint-plugin-boxel": "workspace:*", "@glimmer/component": "catalog:", "@glimmer/syntax": "^0.95.0", "@glimmer/tracking": "^1.1.2", diff --git a/packages/ai-bot/.eslintrc.cjs b/packages/ai-bot/.eslintrc.cjs index 522e9b0bb17..99b9e5fc07e 100644 --- a/packages/ai-bot/.eslintrc.cjs +++ b/packages/ai-bot/.eslintrc.cjs @@ -8,7 +8,7 @@ const { CJS_GLOBALS_IN_ESM, } = require('../../eslint/erasable-syntax-selectors.cjs'); const { - CLOUD_PROVIDER_SDK_IMPORTS, + CLOUD_PROVIDER_SDK_RULE, } = require('../../eslint/cloud-provider-sdk-imports.cjs'); module.exports = { @@ -24,7 +24,7 @@ module.exports = { legacyDecorators: true, }, }, - plugins: ['ember'], + plugins: ['ember', '@cardstack/boxel'], extends: [ 'eslint:recommended', 'plugin:@typescript-eslint/recommended', @@ -65,7 +65,7 @@ module.exports = { ], 'no-restricted-syntax': ['error', ...NO_COMPILATION_REQUIRED_TS_SELECTORS], // Re-declared from the root config (ADR 003 §1.2). - 'no-restricted-imports': ['error', CLOUD_PROVIDER_SDK_IMPORTS], + '@cardstack/boxel/no-cloud-provider-sdk': CLOUD_PROVIDER_SDK_RULE, }, overrides: [ { diff --git a/packages/ai-bot/package.json b/packages/ai-bot/package.json index bef9947773d..71e0226d96c 100644 --- a/packages/ai-bot/package.json +++ b/packages/ai-bot/package.json @@ -22,6 +22,7 @@ "undici": "catalog:" }, "devDependencies": { + "@cardstack/eslint-plugin-boxel": "workspace:*", "@cardstack/local-types": "workspace:*", "@glint/ember-tsc": "catalog:", "@sinonjs/fake-timers": "catalog:", diff --git a/packages/boxel-ui/eslint.config.mjs b/packages/boxel-ui/eslint.config.mjs index 3df70119b60..bb63c62579d 100644 --- a/packages/boxel-ui/eslint.config.mjs +++ b/packages/boxel-ui/eslint.config.mjs @@ -31,6 +31,9 @@ const require = createRequire(import.meta.url); const boxel = require('@cardstack/eslint-plugin-boxel'); const MISSING_INVOKABLES_CONFIG = require('../runtime-common/etc/eslint/missing-invokables-config'); const { DATA_TEST_SELECTORS } = require('../../eslint/data-test-selectors.cjs'); +const { + CLOUD_PROVIDER_SDK_RULE, +} = require('../../eslint/cloud-provider-sdk-imports.cjs'); const esmParserOptions = { ecmaFeatures: { modules: true }, @@ -149,6 +152,17 @@ export default defineConfig([ 'import/extensions': ['error', 'always', { ignorePackages: true }], }, }, + { + // Cloud provider SDKs only in adapter modules, and boxel-ui has none + // (ADR 003 §1.2). + files: ['src/**/*'], + plugins: { + '@cardstack/boxel': boxel, + }, + rules: { + '@cardstack/boxel/no-cloud-provider-sdk': CLOUD_PROVIDER_SDK_RULE, + }, + }, { files: ['**/*.{gts,gjs}'], plugins: { diff --git a/packages/eslint-plugin-boxel/README.md b/packages/eslint-plugin-boxel/README.md index ace59f72105..05a46809a5e 100644 --- a/packages/eslint-plugin-boxel/README.md +++ b/packages/eslint-plugin-boxel/README.md @@ -37,6 +37,7 @@ Then configure the rules you want to use under the rules section: | Name                                | Description | 💼 | 🔧 | | :--------------------------------------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | :-- | :-- | | [missing-card-api-import](docs/rules/missing-card-api-import.md) | disallow usage of card-api with missing imports with auto-fix | ✅ | 🔧 | +| [no-cloud-provider-sdk](docs/rules/no-cloud-provider-sdk.md) | Disallow loading a cloud provider SDK outside the modules that adapt it, by static import, re-export, dynamic `import()`, or `require()` | | | | [no-css-position-fixed](docs/rules/no-css-position-fixed.md) | disallow `position: fixed` in card CSS because cards should not break out of their bounding box | | | | [no-duplicate-imports](docs/rules/no-duplicate-imports.md) | Prevent duplicate imports from the same module | ✅ | 🔧 | | [no-forbidden-head-tags](docs/rules/no-forbidden-head-tags.md) | disallow forbidden HTML elements in `static head` templates — only ``, `<meta>`, and `<link>` are permitted | ✅ | | diff --git a/packages/eslint-plugin-boxel/docs/rules/no-cloud-provider-sdk.md b/packages/eslint-plugin-boxel/docs/rules/no-cloud-provider-sdk.md new file mode 100644 index 00000000000..d8de01ad9cc --- /dev/null +++ b/packages/eslint-plugin-boxel/docs/rules/no-cloud-provider-sdk.md @@ -0,0 +1,13 @@ +# Disallow loading a cloud provider SDK outside the modules that adapt it, by static import, re-export, dynamic `import()`, or `require()` (`@cardstack/boxel/no-cloud-provider-sdk`) + +<!-- end auto-generated rule header --> + +Runtime code reaches a proprietary cloud service only through an interface named for the capability Boxel needs, so a provider's SDK belongs only in that interface's adapter (ADR 003 §1.2). This rule reports every way a module can load one: a static import or re-export, a dynamic `import()`, `require()` or `require.resolve()` (whatever `require` is bound to, so a `createRequire` result counts), and TypeScript's `import x = require()`. + +A source the rule can't read statically, such as `import(name)`, is not reported. + +## Options + +- `patterns` — the restricted modules. A pattern ending in `/*` names every module under that scope or path, such as `@aws-sdk/*`. Any other pattern names one module exactly, such as `aws-sdk`. + +The repository configures the patterns, the packages the rule covers, and the adapter modules exempt from it in `eslint/cloud-provider-sdk-imports.cjs`. diff --git a/packages/eslint-plugin-boxel/lib/rules/no-cloud-provider-sdk.js b/packages/eslint-plugin-boxel/lib/rules/no-cloud-provider-sdk.js new file mode 100644 index 00000000000..5e23f5138db --- /dev/null +++ b/packages/eslint-plugin-boxel/lib/rules/no-cloud-provider-sdk.js @@ -0,0 +1,103 @@ +'use strict'; + +// A pattern ending in `/*` names every module under that scope or path, and +// any other pattern names one module exactly. +function isRestricted(source, patterns) { + return patterns.some((pattern) => + pattern.endsWith('/*') + ? source.startsWith(pattern.slice(0, -1)) + : source === pattern, + ); +} + +// The module a source node names, when it is written as a string. +function staticSource(node) { + if (!node) { + return undefined; + } + if (node.type === 'Literal' && typeof node.value === 'string') { + return node.value; + } + if (node.type === 'TemplateLiteral' && node.expressions.length === 0) { + return node.quasis[0].value.cooked; + } + return undefined; +} + +/** @type {import('eslint').Rule.RuleModule} */ +module.exports = { + meta: { + type: 'problem', + docs: { + description: + 'Disallow loading a cloud provider SDK outside the modules that adapt it, by static import, re-export, dynamic `import()`, or `require()`', + category: 'Architecture', + recommended: false, + }, + schema: [ + { + type: 'object', + properties: { + patterns: { + type: 'array', + items: { type: 'string' }, + }, + }, + required: ['patterns'], + additionalProperties: false, + }, + ], + messages: { + noCloudProviderSdk: + "Load the cloud provider SDK '{{source}}' only in an adapter module (ADR 003 §1.2). Put the call behind an interface named for the capability, or add an adapter module to CLOUD_PROVIDER_ADAPTER_MODULES in eslint/cloud-provider-sdk-imports.cjs.", + }, + }, + + create(context) { + let { patterns } = context.options[0] ?? { patterns: [] }; + + function check(node, sourceNode) { + let source = staticSource(sourceNode); + if (source !== undefined && isRestricted(source, patterns)) { + context.report({ + node, + messageId: 'noCloudProviderSdk', + data: { source }, + }); + } + } + + return { + ImportDeclaration(node) { + check(node, node.source); + }, + ExportAllDeclaration(node) { + check(node, node.source); + }, + ExportNamedDeclaration(node) { + check(node, node.source); + }, + ImportExpression(node) { + check(node, node.source); + }, + // `require('x')` and `require.resolve('x')`, whatever `require` is bound + // to, so a `createRequire` result is caught too. + CallExpression(node) { + let { callee } = node; + let isRequire = + (callee.type === 'Identifier' && callee.name === 'require') || + (callee.type === 'MemberExpression' && + callee.object.type === 'Identifier' && + callee.object.name === 'require' && + callee.property.type === 'Identifier' && + callee.property.name === 'resolve'); + if (isRequire) { + check(node, node.arguments[0]); + } + }, + TSExternalModuleReference(node) { + check(node, node.expression); + }, + }; + }, +}; diff --git a/packages/eslint-plugin-boxel/tests/lib/rules/no-cloud-provider-sdk-test.js b/packages/eslint-plugin-boxel/tests/lib/rules/no-cloud-provider-sdk-test.js new file mode 100644 index 00000000000..99e00f74c97 --- /dev/null +++ b/packages/eslint-plugin-boxel/tests/lib/rules/no-cloud-provider-sdk-test.js @@ -0,0 +1,79 @@ +'use strict'; + +const rule = require('../../../lib/rules/no-cloud-provider-sdk'); +const RuleTester = require('eslint').RuleTester; + +const ruleTester = new RuleTester({ + parser: require.resolve('@typescript-eslint/parser'), + parserOptions: { + ecmaVersion: 2022, + sourceType: 'module', + }, +}); + +const options = [{ patterns: ['@aws-sdk/*', 'aws-sdk', '@azure/*'] }]; +const error = (source) => ({ + messageId: 'noCloudProviderSdk', + data: { source }, +}); + +ruleTester.run('no-cloud-provider-sdk', rule, { + valid: [ + { code: `import { Pool } from 'pg';`, options }, + { code: `import x from 'aws-sdk-mock';`, options }, + { code: `import x from '@aws-sdkx/client';`, options }, + { code: `const m = await import('./local.js');`, options }, + { code: `const m = await import(moduleName);`, options }, + { code: `const m = require('pg');`, options }, + { code: `export { x } from './x';`, options }, + { code: `export const y = 1;`, options }, + ], + + invalid: [ + { + code: `import { S3Client } from '@aws-sdk/client-s3';`, + options, + errors: [error('@aws-sdk/client-s3')], + }, + { + code: `import type { S3Client } from '@aws-sdk/client-s3';`, + options, + errors: [error('@aws-sdk/client-s3')], + }, + { + code: `export * from '@azure/storage-blob';`, + options, + errors: [error('@azure/storage-blob')], + }, + { + code: `export { BlobClient } from '@azure/storage-blob';`, + options, + errors: [error('@azure/storage-blob')], + }, + { + code: `const m = await import('@aws-sdk/client-sqs');`, + options, + errors: [error('@aws-sdk/client-sqs')], + }, + { + code: 'const m = await import(`@aws-sdk/client-sqs`);', + options, + errors: [error('@aws-sdk/client-sqs')], + }, + { + code: `const AWS = require('aws-sdk');`, + options, + errors: [error('aws-sdk')], + }, + { + code: `const path = require.resolve('@aws-sdk/client-kms');`, + options, + errors: [error('@aws-sdk/client-kms')], + }, + { + code: `import AWS = require('aws-sdk');`, + options, + errors: [error('aws-sdk')], + }, + ], +}); diff --git a/packages/host/.eslintrc.js b/packages/host/.eslintrc.js index 89f2c0b8af9..f6db37da8cd 100644 --- a/packages/host/.eslintrc.js +++ b/packages/host/.eslintrc.js @@ -3,7 +3,7 @@ const MISSING_INVOKABLES_CONFIG = require('../runtime-common/etc/eslint/missing-invokables-config'); const { DATA_TEST_SELECTORS } = require('../../eslint/data-test-selectors.cjs'); const { - CLOUD_PROVIDER_SDK_IMPORTS, + CLOUD_PROVIDER_SDK_RULE, } = require('../../eslint/cloud-provider-sdk-imports.cjs'); // Applies to all of JS, TS, GJS, and GTS in the browser context. @@ -84,7 +84,7 @@ module.exports = { // not inherit (ADR 003 §1.2). files: ['app/**/*'], rules: { - 'no-restricted-imports': ['error', CLOUD_PROVIDER_SDK_IMPORTS], + '@cardstack/boxel/no-cloud-provider-sdk': CLOUD_PROVIDER_SDK_RULE, }, }, { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1bfc2e5db77..f3573650c4e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -775,6 +775,9 @@ importers: '@babel/traverse': specifier: 'catalog:' version: 7.27.0(supports-color@8.1.1) + '@cardstack/eslint-plugin-boxel': + specifier: workspace:* + version: link:packages/eslint-plugin-boxel '@glimmer/component': specifier: ^2.0.0 version: 2.1.1(supports-color@8.1.1) @@ -923,6 +926,9 @@ importers: specifier: 'catalog:' version: 7.30.0 devDependencies: + '@cardstack/eslint-plugin-boxel': + specifier: workspace:* + version: link:../eslint-plugin-boxel '@cardstack/local-types': specifier: workspace:* version: link:../local-types