From 5f0e1d62cf97c9e603f1964c8e50fde5574b69de Mon Sep 17 00:00:00 2001 From: Kit <344842+roodboi@users.noreply.github.com> Date: Fri, 13 Nov 2026 15:35:40 +0000 Subject: [PATCH 01/15] docs(native): design private Compose env-file support (HACK-2062) --- docs/native/environment-files.md | 69 ++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 docs/native/environment-files.md diff --git a/docs/native/environment-files.md b/docs/native/environment-files.md new file mode 100644 index 000000000..cbe5acd04 --- /dev/null +++ b/docs/native/environment-files.md @@ -0,0 +1,69 @@ +# Native Compose environment files + +Design for HACK-2062. The Compose backend keeps its existing behavior. + +Native accepts service `env_file` as a path string, a list of path strings or +long-form entries (`path`, optional `required`, optional `format`). `required` +defaults to true; false skips an absent file, but never an unreadable or unsafe +file. The default format uses Compose dotenv quoting, comments and interpolation; +`format: raw` preserves value text, including quotes and dollar signs. Unknown +options and formats refuse before hooks, image acquisition or VM work. + +Files are resolved relative to the Compose file, within the current project root. +Absolute paths inside that root are accepted; escaping the root is refused. Every +path component inside the root is checked without following symlinks. Files must +be regular, single-link, owned by the current user, owner-readable and grant no +group or other permissions. Directories must be owned by the current user and not +group/other-writable. Reads are bounded and recheck file and directory identities, +size and timestamps. Native does not change permissions or migrate files. + +Container precedence, highest first: + +1. Hack-managed environment, matching Hack's generated Compose environment override. +2. Explicit service `environment`, including empty values. +3. `env_file`, read in list order; the last declaration for a name wins. +4. Image environment defaults for names not otherwise declared. + +Interpolation uses the caller's allowed shell variables before the Compose +project directory's `.env`. Native's Compose project directory is the directory +of its selected Compose file. It does not search a primary worktree or load a +second root `.env`, and `.env` alone does not inject variables into containers. +Within an env file, earlier entries are available to later entries; service env +files may also refer to values established by earlier files. Managed values win +the final service destination; service-scoped managed values are not an ambient +interpolation source. Hack key material and adapter-owned credential names cannot +be copied from the caller's shell or files around the managed delivery boundary. + +The first unit resolves references only in service `environment` and env-file +values. It supports `$NAME`, `${NAME}`, defaults (`-`, `:-`), alternatives (`+`, +`:+`), required references (`?`, `:?`), nesting and `$$`. Required-reference errors +never echo the authored error word. `.env` references in public fields (images, +commands, ports, labels, mounts and builds) remain unsupported: materializing such +values would put private data in plans, argv or receipts. Paths must be literal. +Unresolved bare declarations that require removal of an image ENV entry refuse +rather than silently becoming an empty value. Explicit empty assignments work. + +Start and `doctor --native-preflight` call the same file parser, permission checks +and environment normalizer. Preflight reads these inputs only for validation and +names; it returns no values. Missing required files, malformed input and unsafe +reads produce closed diagnostics with the service, field and source path only. +Optional missing files produce no blocker. Parser excerpts, interpolated error +words and operating-system error text are never included. + +All file-derived service values become bare name declarations in the normalized +Compose document. Values stay in memory outside the managed-env fingerprint and +travel through the existing bounded private stdin envelope, then the expiring +guest environment allocation and launcher. They never enter CLI argv, public JSON, +plans, plan identities, receipts, logs or generated files. Changing file values +without changing declarations must leave normalized Compose and its plan identity +unchanged. Preflight never writes the values to its temporary review document. +Existing private-delivery service/key/byte limits apply before effects. + +Out of scope: Compose override/include/extends support, CLI `--env-file`, dotenv +selection of Compose files or profiles, command substitution, shell execution, +interpolation in public fields, arbitrary external files, changes to Compose +backend semantics, and automatic permission repair. Tests compare precedence with +[Compose's environment-file rules](https://compose-spec.github.io/compose-spec/05-services.html#env_file) +and [interpolation rules](https://docs.docker.com/compose/how-tos/environment-variables/variable-interpolation/), +with sentinel values checked against public output, diagnostics, normalized +plans and identity inputs. Actual guest delivery is qualified separately on a Mac. From 1877d9b05f59d75bb48ab3fe9ae3de44b6edfa2e Mon Sep 17 00:00:00 2001 From: Kit <344842+roodboi@users.noreply.github.com> Date: Fri, 13 Nov 2026 19:06:09 +0000 Subject: [PATCH 02/15] feat(native): load Compose environment files privately (HACK-2062) --- docs/guides/native-candidate.md | 156 ++-- docs/native/environment-files.md | 172 +++-- docs/native/environment-files.mmd | 15 + docs/native/environment-files.svg | 66 ++ .../runtime-core/src/project/compose/mod.rs | 33 + .../src/project/compose/service.rs | 52 +- packages/runtime-core/src/project/inputs.rs | 18 + .../runtime-core/src/project/live_source.rs | 67 ++ packages/runtime-core/src/project/mod.rs | 3 + packages/runtime-core/src/project/source.rs | 39 +- .../src/provider/graph/image_environment.rs | 41 +- .../tests/normalized_cli_contract.rs | 110 +++ .../runtime-core/tests/project_contract.rs | 77 ++ src/backends/native-aws-environment.ts | 9 + src/backends/native-compose-env-syntax.ts | 350 +++++++++ src/backends/native-compose-environment.ts | 726 ++++++++++++++++++ src/backends/native-env-file-source.ts | 238 ++++++ src/backends/native-environment-unsets.ts | 37 + src/backends/native-exec-environment.ts | 10 +- src/backends/native-exec-stale-env.ts | 1 + src/backends/native-gcloud-environment.ts | 9 + src/backends/native-preflight.ts | 220 +++--- src/backends/native-project-builds.ts | 4 +- src/backends/native-project-input.ts | 189 +++-- .../native-project-restart-preflight.ts | 1 + src/backends/native-project-review.ts | 11 + .../native-project-start-preflight.ts | 12 +- src/backends/native-project-start.ts | 26 +- src/backends/native-shell-environment.ts | 151 +--- ...e-compose-adoption-file-generation.test.ts | 2 +- ...ative-compose-backend-features-cli.test.ts | 10 +- tests/native-compose-env-syntax.test.ts | 167 ++++ tests/native-compose-environment.test.ts | 511 ++++++++++++ tests/native-env-file-source.test.ts | 151 ++++ tests/native-exec-environment.test.ts | 28 + tests/native-exec-stale-env.test.ts | 42 + tests/native-preflight-cli.test.ts | 152 +++- tests/native-project-builds.test.ts | 2 +- tests/native-project-input.test.ts | 165 ++-- tests/native-project-start.test.ts | 7 +- tests/native-refusal-cli.test.ts | 11 +- tests/native-shell-environment-cli.test.ts | 86 ++- 42 files changed, 3478 insertions(+), 699 deletions(-) create mode 100644 docs/native/environment-files.mmd create mode 100644 docs/native/environment-files.svg create mode 100644 src/backends/native-compose-env-syntax.ts create mode 100644 src/backends/native-compose-environment.ts create mode 100644 src/backends/native-env-file-source.ts create mode 100644 src/backends/native-environment-unsets.ts create mode 100644 tests/native-compose-env-syntax.test.ts create mode 100644 tests/native-compose-environment.test.ts create mode 100644 tests/native-env-file-source.test.ts diff --git a/docs/guides/native-candidate.md b/docs/guides/native-candidate.md index 67d63f56b..080afdd93 100644 --- a/docs/guides/native-candidate.md +++ b/docs/guides/native-candidate.md @@ -219,8 +219,8 @@ symbolic link. Unreadable, changing or oversized input is reported as It changes nothing in the project and starts nothing: no VM, no image pull, no lifecycle hook. It accepts only `--path`, `--profile` and `--json`. The report never prints authored values. A finding names a closed code, a service and a -field; environment values, label keys and values, paths and image names are -left out. +field; environment values, label keys and values and image names are left out. +Environment-file findings may name their source path. The runtime review half needs a native runtime, so run the preflight from the `hack-next` launcher for the full list. Without native selection only the CLI's @@ -236,10 +236,11 @@ Preflight and start use one file-local admission collector. Preflight lists its refusals; start returns its first refusal before acquiring runtime inputs. This can change which reason start reports first when a file has several problems. Review assumes routing is enrolled and cache initializers are admitted. The -readiness defaults are checked against the CLI over shared fixtures. Managed env -metadata and eligible shell references contribute service names only, so restart -policies incompatible with private delivery are blockers even when no decryption -key is available. Shell values and credentials are never read for this check. +readiness defaults are checked against the CLI over shared fixtures. Preflight +uses start's environment preparation, including allowed shell values, guarded +env-file reads and managed-env decryption. A missing decryption key is a blocker. +Only names and paths reach the report. Restart policies incompatible with private +delivery are blockers. Credential adapters, image defaults, occupied ports and other inputs only known at start remain runtime boundaries. AWS/gcloud credentials are added after static review; a restart policy on a @@ -258,11 +259,11 @@ With `--json` the report is the `data` of the standard result envelope: { "severity": "blocker", "gate": "cli", - "code": "env_file", + "code": "environment_file_input", "service": "web", "field": "env_file", - "message": "The service uses env_file, which native does not read.", - "fix": "Move its variables into Hack managed env with hack env set --service, or run hack doctor --migrate-env-config." + "message": "A required environment file is missing or unsafe; values omitted.", + "fix": "Check the named source; use an owned regular file that is not writable by others." } ], "warnings": [] @@ -296,10 +297,10 @@ With `--json` the report is the `data` of the standard result envelope: `internal_hosts_without_dependencies`, `external_network`, `external_volume`, `reserved_label`, `unsupported_build`, `foreign_platform`, `invalid_service_name`, `invalid_service`, `build_requires_opt_in`, - `unsupported_build_form`, `emulated_build`, `env_file`, `missing_image`, + `unsupported_build_form`, `emulated_build`, `environment_file_input`, `missing_image`, `extra_hosts_without_dependencies`, `unsupported_logging`, `invalid_restart`, `unsupported_label`, `invalid_http_probe`, `managed_env_unreadable`, - `dotenv_without_managed_env`, `shared_source_grant_required`, + `environment_delivery`, `shared_source_grant_required`, `route_probe_required` and `too_many_bridges`. Runtime findings use the review's own diagnostic codes, for example `privileged_service` or `invalid_mount`, and `review_incompatible` when the review does not admit the @@ -326,12 +327,10 @@ The standalone preflight does not cover: - `--env`, and a retained run's environment. It uses the project's default environment to select the per-env adaptation and dependency files; `hack up` uses `--env` when given, or the environment of a run it resumes. -- Checks that need secret values or a resolved managed environment: a build - argument that carries a stored secret, a managed variable whose name is not - a valid environment name, and managed env that fails to resolve. `hack up` - also replaces the `environment` entries that managed env supplies before its - review, so a malformed authored value under such a key is a blocker here and - not there. +- Credential-adapter results and build arguments that only those later results + can reveal as private. File and managed-environment admission now uses the same + preparation as start, including decryption and permitted shell lookups; a + missing project key is a closed preflight blocker. - Image pinning. `hack up` reviews images by content ID after it has admitted and pulled them; the preflight reviews the references as written. - What only a start can find: an image that cannot be pulled or admitted, a @@ -1533,67 +1532,66 @@ by an older bundle, so clean up such a graph with the bundle that started it. - A reference to a Hack-managed value of the same service, for example `${TOKEN:-fallback}` when Hack delivers `TOKEN` to that service, refuses with `execution_environment_owner`. Managed values are delivered privately and are - never interpolated into commands, users, health checks or other values, and a + never interpolated into commands, users or health checks, and a default is never used in their place. Declare the key bare (`TOKEN:` or `- TOKEN`) - to receive the managed value. - -### Shell variables in service environment - -Compose reads a variable reference from the shell that runs it. The native runtime -reads no ambient environment, so two forms of a service `environment` entry could -never start on native. `hack up` and `hack restart` resolve those forms from the -caller's environment. Only source variables named by the project's Compose file -are read; the ambient environment is never forwarded as a whole. - -Every native start prints one line to stderr naming the forwarded source -variables, or `(none)`, including detached and JSON starts. Names are deduplicated -and sorted; values are never printed. Foreground starts also report unset -references separately. There is no consent prompt in this feature. - -| Entry | Shell has the variable | Shell does not | -| --- | --- | --- | -| `API_KEY: ${API_KEY}` or `API_KEY: $API_KEY` | The service gets the shell's value. | The service gets an empty value, as with Compose. | -| `- TOKEN` (a bare name, or `TOKEN:` with no value) | The service gets the shell's value. | The service gets an empty value. Compose would leave it unset instead. | - -An entire `environment: null` declaration, including an `environment:` key with -only comments beneath it, is empty. Preflight and start both accept it; managed -keys are still added as private references when configured for the service. - -A value read from the shell is never written to the reviewed Compose document, a -plan, a receipt, an env fingerprint or a log. The entry stays a reference and the -value travels in the same private delivery as Hack managed env. Only the variables -these entries name are read; nothing else of your environment is. The reviewed -document is the same whether or not the variable is set, so a start from another -shell resumes the same plan. - -What stays as it was: - -- **A reference with a default or another operator** (`${PORT:-3000}`, `${X:+alt}`, - `${X-d}`, `${X:?err}`). The runtime resolves it without your shell, so - `${PORT:-3000}` is `3000` even when your shell sets `PORT`. Compose would use the - shell's value. Write `PORT: ${PORT}` to take the value from the shell. -- **A reference inside a longer string** (`http://${HOST}:3000`), and a reference in - any other field (`image`, `ports`, `command`, `volumes`, labels). A reference with - no default there still fails the start with `execution_environment_missing`. -- **A variable Hack manages for the service.** It comes from managed env, never from - the shell, and an entry that references a managed variable is still refused by the - runtime (`execution_environment_owner`). -- **`COMPOSE_PROJECT_NAME`.** The runtime supplies it; it is not read from the shell. -- **Hack's own variables** (`HACK_*`), as the entry's name or as the variable it - references. They include the key that decrypts a project's managed secrets, so a - Compose file can never copy one into a container. -- **Cloud names** (`AWS_*`, `GOOGLE_*`, `CLOUDSDK_*`). Long-lived shell credentials - are not copied into containers; use the scoped AWS and gcloud delivery described - below. The rule goes by prefix, so it also covers non-secret names such as - `AWS_REGION` and `GOOGLE_CLOUD_PROJECT`. - - An entry under either rule is left as written. It does not pass through as it - would with Compose: the start still fails with `execution_environment_missing`, - as it did before. -- **A project `.env` file.** Hack does not parse it for shell references. A launch - runtime such as Bun may preload dotenv into its process environment. Only - explicitly referenced, non-excluded names are read from that environment; - a missing reference is reported as unset. + to receive the managed value. Global managed values can interpolate privately + inside service environment and env-file values. + +### Environment files and interpolation + +Native `up` and `restart` accept Compose `env_file` strings, lists and +long-form `path`/`required`/`format` entries. Optional missing files are skipped; +unreadable or unsafe files still refuse. `format: raw` preserves quotes and dollar +signs. Files are read relative to the selected Compose file and must stay inside +this project root. Reads do not follow symlinks, refuse other-owner or multi-link +files, and require owner-read access with no group/other write access. Ordinary +0644 files work; Hack never changes permissions. + +The automatic interpolation file is `.env` beside the Compose file (normally +`.hack/.env`). A sibling `.env.state.json` causes a refusal: the file may be +obsolete output of `hack env materialize`. Use current Hack env layers, then +remove obsolete materialized output deliberately. A root `.env` outside the +Compose directory is not read, and dotenv alone does not inject values. + +| Source | Precedence and delivery | +| --- | --- | +| Hack-managed values | Win the final container destination, as Hack's Compose override does. Global values also win interpolation; service-scoped values are not ambient sources. | +| Service `environment` | Overrides env-file values, including explicit empty values and unresolved bare keys. | +| `env_file` | Processed in order. Later resolved declarations win. An unresolved bare file key keeps the previous value. | +| Image defaults | Apply unless replaced or explicitly unset by service environment. | + +Within service environment, managed globals win over allowed caller-shell values, +then dotenv. Within env files those sources win over resolved service environment, +then earlier file assignments. Native supports Compose interpolation operators, +embedded references, single-quoted file literals and double-quoted file escapes. +A missing plain reference produces an empty value with a names-only notice. A +bare service key with no source unsets the key, including an image default; it no +longer substitutes empty. The selected runtime must confirm these unset names. +Changing source presence can change reviewed names and plan identity; changing +only values cannot. + +File and shell values stay in memory outside env fingerprints and travel only +through private stdin and the expiring guest allocation. The normalized document +contains bare names, never those values. Each start reports forwarded shell names +on stderr, including JSON starts, without values. Only referenced, allowed shell +variables are read. Hack key material is refused; `HACK_*` copies from files or +shell references are refused. Authored public runtime-metadata literals retain +their existing behavior. Shell +credential names under `AWS_*`, `GOOGLE_*` and `CLOUDSDK_*` remain excluded. +Literal cloud entries in an explicit env file are allowed unless a selected +credential adapter owns that destination. + +Interpolation of private sources in public fields (images, commands, ports, +labels, builds, mounts, users and healthchecks) refuses before effects, including +when the expression contains a default. It never silently selects a different +fallback. Runtime-bound `COMPOSE_PROJECT_NAME` remains public; private expressions +cannot combine it with other sources. Pure literal/default expressions that read +no private source retain their existing runtime behavior. File-derived private +values inherit the launcher's restart-policy limits; an edit to file values needs +an explicit `hack restart` to reach an existing run. + +See [the full environment-file contract](../native/environment-files.md) for file +bounds, materialization, source exclusions and intentionally unsupported forms. Three consequences of the private delivery: @@ -1983,10 +1981,10 @@ under native `hack up -d` above. `hack restart --detach` still refuses with `E_USAGE`: run `hack restart` in the foreground in its own terminal and use `hack ps` and `hack logs` from another. Routed services and host dependencies require the explicit native selections described below. Unsupported Compose or environment input -(for example a service `env_file`, or a project `.env` without managed env) and +(for example an unsafe environment file or private interpolation in a public field) and unreviewed route labels refuse with `E_NATIVE_PROJECT_UNSUPPORTED`. The input refusal names its cause and, where known, the service and Compose field (for -example `service api field env_file`), never values or file contents. Secret +example `service api field env_file` and its source path), never values or file contents. Secret managed env without a project env key refuses with `E_CONFIG_INVALID` and names the recovery actions; other unreadable managed env refuses with `E_CONFIG_INVALID` without layer excerpts. Invalid branch diff --git a/docs/native/environment-files.md b/docs/native/environment-files.md index cbe5acd04..3141616dd 100644 --- a/docs/native/environment-files.md +++ b/docs/native/environment-files.md @@ -2,68 +2,126 @@ Design for HACK-2062. The Compose backend keeps its existing behavior. +![Public admission and private delivery paths](environment-files.svg) + +[Diagram source](environment-files.mmd). + Native accepts service `env_file` as a path string, a list of path strings or long-form entries (`path`, optional `required`, optional `format`). `required` -defaults to true; false skips an absent file, but never an unreadable or unsafe -file. The default format uses Compose dotenv quoting, comments and interpolation; -`format: raw` preserves value text, including quotes and dollar signs. Unknown -options and formats refuse before hooks, image acquisition or VM work. - -Files are resolved relative to the Compose file, within the current project root. -Absolute paths inside that root are accepted; escaping the root is refused. Every -path component inside the root is checked without following symlinks. Files must -be regular, single-link, owned by the current user, owner-readable and grant no -group or other permissions. Directories must be owned by the current user and not -group/other-writable. Reads are bounded and recheck file and directory identities, -size and timestamps. Native does not change permissions or migrate files. +defaults to true; false skips an absent file, never an unreadable or unsafe file. +Omitted or empty `format` follows Compose dotenv syntax; the literal `default` +is not a Compose format. `format: raw` does not unquote +or interpolate values. Unknown options and formats refuse before hooks, image +acquisition or VM work. Repeating a file is allowed and preserves list order. +Files of inactive-profile services are not acquired; their paths remain excluded +from snapshots and live sharing, including missing required paths. +The reviewed runtime active set must agree before private delivery or effects; +an active service with unprepared file inputs refuses. + +Paths are literal and relative to the Compose file, within the current canonical +project root. Absolute paths inside that root are accepted. Escaping the root or +following a symlink inside it is refused. Files must be regular, single-link, +owned by the current user, owner-readable and not writable by group or others; +ordinary 0644 checkouts work. Directories inside the root must be owned by the +current user and not writable by group or others. Reads are bounded and recheck +file and directory identities, size, permissions and timestamps. Native changes +no file or permission. Bounds: 256 KiB and 4,096 lines per file; 64 file references +and 1 MiB total source text per preparation. Existing private delivery limits +(32 services, 256 keys and 32 KiB per service, 256 KiB complete stdin) also apply. + +The automatic interpolation file is `.env` beside the selected Compose file: +normally `.hack/.env`, matching Hack's Compose project directory. An adjacent +`.env.state.json` marks possible Hack-materialized output; native refuses to use +that `.env`, whether the state is current, stale or unreadable, rather than copy +another selection's materialized values. The operator can use current Hack env +layers and remove obsolete materialized output deliberately. A root `.env` outside +the Compose directory is not read or refused solely for existing. Native does not +search another worktree or overlay dotenv. Every `.env` uses the same guards. Container precedence, highest first: 1. Hack-managed environment, matching Hack's generated Compose environment override. -2. Explicit service `environment`, including empty values. -3. `env_file`, read in list order; the last declaration for a name wins. +2. Explicit service `environment`, including empty values and unresolved bare keys. +3. `env_file`, in list order; the last resolved declaration for a name wins. 4. Image environment defaults for names not otherwise declared. -Interpolation uses the caller's allowed shell variables before the Compose -project directory's `.env`. Native's Compose project directory is the directory -of its selected Compose file. It does not search a primary worktree or load a -second root `.env`, and `.env` alone does not inject variables into containers. -Within an env file, earlier entries are available to later entries; service env -files may also refer to values established by earlier files. Managed values win -the final service destination; service-scoped managed values are not an ambient -interpolation source. Hack key material and adapter-owned credential names cannot -be copied from the caller's shell or files around the managed delivery boundary. - -The first unit resolves references only in service `environment` and env-file -values. It supports `$NAME`, `${NAME}`, defaults (`-`, `:-`), alternatives (`+`, -`:+`), required references (`?`, `:?`), nesting and `$$`. Required-reference errors -never echo the authored error word. `.env` references in public fields (images, -commands, ports, labels, mounts and builds) remain unsupported: materializing such -values would put private data in plans, argv or receipts. Paths must be literal. -Unresolved bare declarations that require removal of an image ENV entry refuse -rather than silently becoming an empty value. Explicit empty assignments work. - -Start and `doctor --native-preflight` call the same file parser, permission checks -and environment normalizer. Preflight reads these inputs only for validation and -names; it returns no values. Missing required files, malformed input and unsafe -reads produce closed diagnostics with the service, field and source path only. -Optional missing files produce no blocker. Parser excerpts, interpolated error -words and operating-system error text are never included. - -All file-derived service values become bare name declarations in the normalized -Compose document. Values stay in memory outside the managed-env fingerprint and -travel through the existing bounded private stdin envelope, then the expiring -guest environment allocation and launcher. They never enter CLI argv, public JSON, -plans, plan identities, receipts, logs or generated files. Changing file values -without changing declarations must leave normalized Compose and its plan identity -unchanged. Preflight never writes the values to its temporary review document. -Existing private-delivery service/key/byte limits apply before effects. - -Out of scope: Compose override/include/extends support, CLI `--env-file`, dotenv -selection of Compose files or profiles, command substitution, shell execution, -interpolation in public fields, arbitrary external files, changes to Compose -backend semantics, and automatic permission repair. Tests compare precedence with -[Compose's environment-file rules](https://compose-spec.github.io/compose-spec/05-services.html#env_file) -and [interpolation rules](https://docs.docker.com/compose/how-tos/environment-variables/variable-interpolation/), -with sentinel values checked against public output, diagnostics, normalized -plans and identity inputs. Actual guest delivery is qualified separately on a Mac. +A bare env-file name inherits from the interpolation context if present; otherwise +it leaves earlier file assignments intact. A bare service environment key resolves +from the interpolation context; when unresolved it removes that key, including an +image default. Native removes the key from normalized `environment`, then carries its name in +explicit normalized metadata and public engine unset entries, separate from private +value delivery. The runtime must echo the exact unset names during review; +ordinary missing private bindings must continue to refuse. + +Interpolation lookup order is Hack-managed global values, the caller's allowed +shell variables, then `.env`. Service-scoped managed values win the final service +destination but are not ambient interpolation sources. For env-file expansion, +that project context wins over resolved service `environment`, then current and +earlier file assignments. A current file assignment replaces the previous file's +assignment before the next line is read. `.env` alone injects no container variables. + +Support includes export prefixes, `=` and `:` separators in default format, +comments, single-quoted literals, double-quoted escapes, multiline quoted values, +`$NAME`, `${NAME}`, defaults (`-`, `:-`), alternatives (`+`, `:+`), required references +(`?`, `:?`), nesting to four levels, and `$$`. An unset `${NAME}` yields empty with a +names-only notice; required-reference errors omit the authored error word. Raw +format follows the separately qualified Compose raw parser. Hack key material and +`HACK_*` cannot enter containers via files or shell references. Authored public +runtime-metadata literals keep their existing behavior. Cloud credential +names are not read from the shell; explicit literal file entries remain available +unless the selected credential adapter owns that service's name. Compose control +names (`COMPOSE_FILE`, `COMPOSE_PROFILES`, `COMPOSE_PROJECT_NAME`, `COMPOSE_ENV_FILES`, +`COMPOSE_DISABLE_ENV_FILE`) +in `.env` refuse with names only rather than silently change the selected project. + +The approved first unit resolves file values only in service `environment` and +env-file values. If a public field references a name defined by `.env`, managed +global env or allowed shell input, it refuses before effects, even if the reference +has a default. It must never silently select a different fallback. Public fields +include images, commands, ports, labels, mounts, users, healthchecks and builds. +The lead approved this privacy boundary: materializing such values would otherwise +put them in plans, argv or receipts. + +Start and `doctor --native-preflight` use the same parser, lookup, permissions and +normalizer, including the same allow-listed shell environment and managed values. +Preflight may therefore need the same project decryption key as start. It returns +only names and paths. Missing required files, malformed input, unsafe reads and +unsupported public interpolation produce closed diagnostics with service, field +and source path only. Optional absent files produce no blocker. Parser excerpts, +interpolated error words and raw OS diagnostics never leave the boundary. + +The normalized document removes `env_file` and replaces file-derived environment +values with bare name declarations. Values remain in memory outside managed-env +fingerprints, then travel through the bounded private stdin envelope and the +expiring guest allocation/launcher. They never enter CLI argv, public JSON, plans, +plan identities, receipts, logs or generated files. Original env-file declarations +remain source exclusions in Rust even when normalization removes them, including +absent optional targets, inactive-service files, raw inputs +and duplicate paths. `.env*` remain excluded by the existing source policy. Tests +must cover snapshot and live-share exclusion when an optional file appears later. + +Private delivery retains its existing limitations: it cannot use automatic +restart policies, and fresh values need a new incarnation. A file-value-only edit +has no public fingerprint; the operator must run `hack restart` to deliver it to +an existing run. Literal/default-only expressions that consult no private source +stay on the existing public path and retain their restart behavior and identity. +Fresh `hack exec` resolves the selected service's file values again using the +same allowed caller context and delivers them privately to that exec process. +These fresh values may differ from the running container's startup values. +Managed-env stale checks remain advisory and do not read ephemeral file inputs; +file-value-only changes are not detected by their managed fingerprints. + +Unused Compose `x-*` extensions are ignored; Hack-consumed `x-hack-*` fields +still receive admission checks. Exact AWS/ADC mounts recognized by the credential +adapters keep their private adapter path and are never expanded into public plans. +Unreadable reserved bare names remain runtime/adapter-owned, rather than becoming +explicit image-default removals. + +Out of scope: override/include/extends support, CLI `--env-file`, dotenv selection +of Compose files or profiles, shell execution/command substitution, public-field +interpolation without an explicit future public-data contract, external files, +automatic permission repair, Compose-backend changes and automatic secret-value +change detection. Tests use sentinel values and recorded Docker Compose v5.6.1 / +compose-go v2.16.1 results, alongside the documented [environment-file rules](https://compose-spec.github.io/compose-spec/05-services.html#env_file) +and [interpolation rules](https://docs.docker.com/compose/how-tos/environment-variables/variable-interpolation/). +Actual guest delivery is qualified separately on a Mac. diff --git a/docs/native/environment-files.mmd b/docs/native/environment-files.mmd new file mode 100644 index 000000000..b172724b8 --- /dev/null +++ b/docs/native/environment-files.mmd @@ -0,0 +1,15 @@ +--- +title: Native environment input boundaries +--- +flowchart TD + Sources["Files and allowed sources"] --> Guard["Guarded source reads"] + Guard --> Normalize["Shared environment preparation"] + Normalize -->|"names and public declarations"| Review["Read-only runtime admission"] + Normalize -->|"values in memory"| Private["Bounded private delivery"] + Normalize -->|"private reference in public field"| Refuse["Closed refusal"] + Review -->|"admitted active services"| Private + Private --> Stdin["Private stdin envelope"] + Stdin --> Guest["Expiring guest allocation"] + Guest --> Launch["Service launcher"] + class Refuse fail + class Review,Launch ok diff --git a/docs/native/environment-files.svg b/docs/native/environment-files.svg new file mode 100644 index 000000000..d145d7610 --- /dev/null +++ b/docs/native/environment-files.svg @@ -0,0 +1,66 @@ +Native environment input boundariesnames and public declarationsvalues in memoryprivate reference in public fieldadmitted active servicesFiles and allowed sourcesGuarded source readsShared environmentpreparationRead-only runtimeadmissionBounded private deliveryClosed refusalPrivate stdin envelopeExpiring guest allocationService launcher \ No newline at end of file diff --git a/packages/runtime-core/src/project/compose/mod.rs b/packages/runtime-core/src/project/compose/mod.rs index f4584a478..a1fa8dbe8 100644 --- a/packages/runtime-core/src/project/compose/mod.rs +++ b/packages/runtime-core/src/project/compose/mod.rs @@ -13,6 +13,38 @@ mod service; use fields::*; pub(crate) use project_name::VARIABLE as COMPOSE_PROJECT_NAME; +/// Explicit names-only image-variable removals, disjoint from every environment +/// declaration. A missing private binding must never be reinterpreted as unset. +pub(super) fn environment_unsets( + value: &Map, +) -> Result, CandidateError> { + let mut result = std::collections::BTreeSet::new(); + let Some(raw) = value.get("x-hack-unset-environment") else { + return Ok(result); + }; + let fail = || { + problem( + "invalid_environment_unset", + "Environment removals must be at most 256 distinct variable names, separate from declared environment bindings; values omitted.", + ) + }; + let names = raw + .as_array() + .filter(|names| names.len() <= 256) + .ok_or_else(fail)?; + let declared = redacted_map(value.get("environment"), true)?; + for name in names { + let name = name + .as_str() + .filter(|name| variable_name(name)) + .ok_or_else(fail)?; + if declared.contains_key(name) || !result.insert(name.to_owned()) { + return Err(fail()); + } + } + Ok(result) +} + pub(super) fn identifier(value: &str) -> Result<(), CandidateError> { fields::identifier(value) } @@ -37,6 +69,7 @@ pub(super) fn original_environment_files( object(service)?.get("env_file"), "original.env_file", &mut Vec::new(), + true, )?); } Ok(result) diff --git a/packages/runtime-core/src/project/compose/service.rs b/packages/runtime-core/src/project/compose/service.rs index 778c4342e..7527f229d 100644 --- a/packages/runtime-core/src/project/compose/service.rs +++ b/packages/runtime-core/src/project/compose/service.rs @@ -454,6 +454,7 @@ pub(super) fn environment_files( value: Option<&Value>, field: &str, diagnostics: &mut Vec, + ownership_only: bool, ) -> Result, CandidateError> { let values: Vec<&Value> = match value { None => vec![], @@ -462,13 +463,17 @@ pub(super) fn environment_files( }; let mut result = Vec::new(); for value in values { + let required = !ownership_only + && match value { + Value::Object(m) => boolean(m.get("required"), true)?, + _ => true, + }; let path = if let Value::Object(m) = value { keys(m, &["path", "required", "format"], field, diagnostics); - if !boolean(m.get("required"), true)? - || m.get("format") - .is_some_and(|v| v.as_str() != Some("default")) + if m.get("format") + .is_some_and(|v| !matches!(v.as_str(), Some("" | "raw"))) { - diagnostics.push(Diagnostic::error("unsupported_env_file_mode",field,"Only required default-format env_file references are modeled. Their contents are never read during planning.")); + diagnostics.push(Diagnostic::error("unsupported_env_file_mode",field,"Only default and raw env_file formats are modeled. Their contents are never read during planning.")); } string( m.get("path") @@ -477,7 +482,11 @@ pub(super) fn environment_files( } else { string(value)? }; - let resolved = source::resolve(project, base, path, true)?; + let resolved = if required { + source::resolve(project, base, path, true)? + } else { + source::resolve_optional_env_file(project, base, path)? + }; if resolved .file_name() .is_some_and(|name| name == ".gitignore" || name == ".ignore") @@ -487,22 +496,23 @@ pub(super) fn environment_files( "An environment input cannot also own source ignore rules.", )); } - let metadata = fs::symlink_metadata(&resolved) - .map_err(|_| problem("invalid_env_file", "Cannot inspect the env_file reference."))?; - if !metadata.is_file() || metadata.nlink() != 1 { - return Err(problem( - "invalid_env_file", - "env_file must refer to a regular file; its contents are not loaded.", - )); + match fs::symlink_metadata(&resolved) { + Ok(metadata) if metadata.is_file() && metadata.nlink() == 1 => {} + Err(error) if !required && error.kind() == std::io::ErrorKind::NotFound => {} + _ => { + return Err(problem( + "invalid_env_file", + "env_file must refer to a regular file; its contents are not loaded.", + )); + } } let relative = relative(project, &resolved); - if result.contains(&relative) { - return Err(problem( - "duplicate_env_file", - "Duplicate environment file ownership is not accepted.", - )); + // This is an ownership inventory, not a parser execution order. Repeated + // Compose inputs are read in order by the private normalizer, while one + // exclusion per path protects snapshot and future live-share contents. + if !result.contains(&relative) { + result.push(relative); } - result.push(relative); } Ok(result) } @@ -728,6 +738,7 @@ pub(super) fn compile( identifier(name)?; let field = format!("services.{name}"); let m = object(value)?; + let environment_unsets = super::environment_unsets(m)?.into_iter().collect(); keys( m, &[ @@ -739,6 +750,7 @@ pub(super) fn compile( "volumes", "depends_on", "environment", + "x-hack-unset-environment", "extra_hosts", "env_file", "healthcheck", @@ -1108,8 +1120,8 @@ pub(super) fn compile( profiles:declared_profiles,active, dependencies:dependencies(m.get("depends_on"),&format!("{field}.depends_on"),diagnostics)?, mounts:declared_mounts, - environment:redacted_map(m.get("environment"),true)?,extra_hosts, - environment_files:environment_files(project,base,m.get("env_file"),&format!("{field}.env_file"),diagnostics)?, + environment:redacted_map(m.get("environment"),true)?,environment_unsets,extra_hosts, + environment_files:environment_files(project,base,m.get("env_file"),&format!("{field}.env_file"),diagnostics,false)?, environment_precedence:"Compose environment keys override explicitly listed env_file inputs; bare keys and references remain unresolved; no ambient/.env values are loaded".into(), healthcheck:health(m.get("healthcheck"),&format!("{field}.healthcheck"),diagnostics)?, ports:ports(m.get("ports"),&format!("{field}.ports"),diagnostics)?,exposed_ports,networks,network_aliases,network_mode:if ["project-networks", "none"].contains(&network_mode) {network_mode.into()} else {"[unsupported]".into()}, diff --git a/packages/runtime-core/src/project/inputs.rs b/packages/runtime-core/src/project/inputs.rs index 24ab98c26..9b1db427e 100644 --- a/packages/runtime-core/src/project/inputs.rs +++ b/packages/runtime-core/src/project/inputs.rs @@ -697,6 +697,24 @@ fn compile_inner( private, )?; let (environment, scoped) = resolver.environment(environment.as_ref(), private)?; + let unsets = &service.environment_unsets; + if unsets.iter().any(|key| { + private.is_some_and(|values| values.contains_key(key)) + || review.plan.registry.as_ref().is_some_and(|registry| { + service.dependency_cache.is_some() + && registry.required_environment.contains(key) + }) + }) { + return Err(error( + "execution_environment_owner", + "Environment removals cannot overlap private or registry-required bindings; values omitted.", + )); + } + let mut environment = environment; + // Docker Engine Config.Env accepts bare names as explicit removals from + // image defaults. These entries contain no value and need no launcher. + environment.extend(unsets.iter().cloned()); + environment.sort(); if !scoped.is_empty() { managed_environment.insert(name.clone(), scoped); } diff --git a/packages/runtime-core/src/project/live_source.rs b/packages/runtime-core/src/project/live_source.rs index 4a8657341..43482df55 100644 --- a/packages/runtime-core/src/project/live_source.rs +++ b/packages/runtime-core/src/project/live_source.rs @@ -460,6 +460,73 @@ mod tests { } } + #[test] + fn optional_env_file_created_after_review_stays_out_of_live_source() { + let (_fixture, candidate, source) = fixture(); + let path = source.join("compose.yaml"); + let mut original = super::super::yaml::parse(&fs::read(&path).unwrap()).unwrap(); + original["services"]["deps"]["env_file"] = serde_json::json!([ + {"path":"secrets/private.txt", "required":false, "format":"raw"} + ]); + let original_bytes = serde_json::to_vec(&original).unwrap(); + fs::write(&path, &original_bytes).unwrap(); + original["services"]["deps"] + .as_object_mut() + .unwrap() + .remove("env_file"); + let normalized = serde_json::to_vec(&original).unwrap(); + let namespace = candidate.plan(&source).unwrap().namespace; + let original_hash = format!("{:x}", Sha256::digest(&original_bytes)); + let capture = || { + let plan = project::plan_normalized( + &candidate, + project::NormalizedComposeOptions { + branch: None, + project: &source, + compose_file: Path::new("compose.yaml"), + profiles: &[], + expected_namespace: &namespace, + expected_compose_sha256: &original_hash, + compose_bytes: &normalized, + }, + ) + .unwrap(); + let snapshot = snapshot::capture_plan(&plan.plan).unwrap(); + assert!( + snapshot + .receipt() + .entries + .iter() + .all(|entry| entry.path != "secrets/private.txt" && entry.path != ".env") + ); + (plan, snapshot) + }; + let (initial, snapshot) = capture(); + let contract = Contract::from_plan(&initial.plan, snapshot.receipt()).unwrap(); + fs::create_dir(source.join("secrets")).unwrap(); + fs::write(source.join("secrets/private.txt"), "PRIVATE_FUTURE_CANARY").unwrap(); + fs::write(source.join(".env"), "PRIVATE_DOTENV_CANARY").unwrap(); + let (with_file, snapshot) = capture(); + contract + .verify(&with_file.plan, snapshot.receipt()) + .unwrap(); + fs::write( + source.join("secrets/private.txt"), + "PRIVATE_CHANGED_CANARY_LONGER", + ) + .unwrap(); + let (changed_value, snapshot) = capture(); + assert_eq!(with_file.plan_id, changed_value.plan_id); + contract + .verify(&changed_value.plan, snapshot.receipt()) + .unwrap(); + assert!( + !serde_json::to_string(&changed_value) + .unwrap() + .contains("CANARY") + ); + } + #[test] fn content_add_remove_and_excluded_secrets_do_not_freeze_plan_identity() { let (_fixture, candidate, source) = fixture(); diff --git a/packages/runtime-core/src/project/mod.rs b/packages/runtime-core/src/project/mod.rs index 0deb479ae..2fcae573a 100644 --- a/packages/runtime-core/src/project/mod.rs +++ b/packages/runtime-core/src/project/mod.rs @@ -291,6 +291,9 @@ pub struct ServicePlan { pub dependencies: BTreeMap, pub mounts: Vec, pub environment: BTreeMap, + /// Explicit names removed from image defaults, separate from private bindings. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub environment_unsets: Vec, #[serde(default, skip_serializing_if = "Vec::is_empty")] pub extra_hosts: Vec, pub environment_files: Vec, diff --git a/packages/runtime-core/src/project/source.rs b/packages/runtime-core/src/project/source.rs index f928acc31..9e2329b6c 100644 --- a/packages/runtime-core/src/project/source.rs +++ b/packages/runtime-core/src/project/source.rs @@ -184,6 +184,27 @@ pub fn resolve( base: &Path, value: &str, allow_credentials: bool, +) -> Result { + resolve_inner(project, base, value, allow_credentials, false) +} + +/// Resolve ownership of an optional env-file path, including a currently absent +/// tail. This grants no read or mount authority: callers must retain the path as +/// a source exclusion even if the file or its parent appears after review. +pub(super) fn resolve_optional_env_file( + project: &Path, + base: &Path, + value: &str, +) -> Result { + resolve_inner(project, base, value, true, true) +} + +fn resolve_inner( + project: &Path, + base: &Path, + value: &str, + allow_credentials: bool, + optional: bool, ) -> Result { if value.is_empty() || value.contains('$') || value.contains('\0') || value.contains('~') { return Err(problem( @@ -224,12 +245,18 @@ pub fn resolve( let mut current = project.to_owned(); for part in relative.components() { current.push(part); - let metadata = fs::symlink_metadata(¤t).map_err(|_| { - problem( - "missing_source", - "Declared source does not exist or cannot be inspected.", - ) - })?; + let metadata = match fs::symlink_metadata(¤t) { + Ok(metadata) => metadata, + Err(error) if optional && error.kind() == std::io::ErrorKind::NotFound => { + return Ok(resolved); + } + Err(_) => { + return Err(problem( + "missing_source", + "Declared source does not exist or cannot be inspected.", + )); + } + }; if metadata.file_type().is_symlink() { return Err(problem( "source_symlink", diff --git a/packages/runtime-core/src/provider/graph/image_environment.rs b/packages/runtime-core/src/provider/graph/image_environment.rs index 0336a7ec2..0ed5b41aa 100644 --- a/packages/runtime-core/src/provider/graph/image_environment.rs +++ b/packages/runtime-core/src/provider/graph/image_environment.rs @@ -33,7 +33,27 @@ pub(super) fn compose( public: &Value, ) -> Result, CandidateError> { let mut expected = entries(image)?; - expected.extend(entries(public)?); + if !public.is_null() { + let mut seen = std::collections::BTreeSet::new(); + for entry in public.as_array().ok_or_else(refused)? { + let entry = entry.as_str().ok_or_else(refused)?; + let (key, value) = entry + .split_once('=') + .map_or((entry, None), |(key, value)| (key, Some(value))); + if key.is_empty() + || key.contains('\0') + || value.is_some_and(|value| value.contains('\0')) + || !seen.insert(key) + { + return Err(refused()); + } + if let Some(value) = value { + expected.insert(key.to_owned(), value.to_owned()); + } else { + expected.remove(key); + } + } + } Ok(expected) } pub(super) fn verify( @@ -83,7 +103,6 @@ mod tests { fn malformed_or_duplicate_fields_refuse_without_value_diagnostics() { for bad in [ json!(["KEY=canary", "KEY=second"]), - json!(["MISSING"]), json!(["=canary"]), json!(["KEY\u{0000}=canary"]), json!(["KEY=canary\u{0000}"]), @@ -96,5 +115,23 @@ mod tests { } assert!(verify(&BTreeMap::new(), &bad).is_err()); } + assert!(compose(&json!(["MISSING"]), &Value::Null).is_err()); + assert!(verify(&BTreeMap::new(), &json!(["MISSING"])).is_err()); + } + #[test] + fn explicit_bare_names_remove_image_defaults_but_empty_values_remain() { + let expected = compose( + &json!(["PATH=/bin", "REMOVE=image-default", "EMPTY=image-default"]), + &json!(["REMOVE", "EMPTY=", "MISSING"]), + ) + .unwrap(); + verify(&expected, &json!(["EMPTY=", "PATH=/bin"])).unwrap(); + for actual in [ + json!(["EMPTY=", "PATH=/bin", "REMOVE=image-default"]), + json!(["EMPTY=", "PATH=/bin", "REMOVE="]), + ] { + assert!(verify(&expected, &actual).is_err()); + } + assert!(compose(&Value::Null, &json!(["REMOVE", "REMOVE=second"])).is_err()); } } diff --git a/packages/runtime-core/tests/normalized_cli_contract.rs b/packages/runtime-core/tests/normalized_cli_contract.rs index ee9eda9f0..72c787a74 100644 --- a/packages/runtime-core/tests/normalized_cli_contract.rs +++ b/packages/runtime-core/tests/normalized_cli_contract.rs @@ -280,3 +280,113 @@ fn normalized_capture_keeps_removed_original_credential_files_excluded() { .contains("PRIVATE_CLI_CANARY") ); } + +#[test] +fn optional_raw_and_repeated_env_files_keep_future_secret_paths_excluded() { + let mut fixture = Fixture::new(); + let original = ORIGINAL.replace(" image:", " env_file:\n - {path: future/private.txt, required: false, format: raw}\n - {path: future/private.txt, required: false}\n image:"); + fs::write(fixture.project.join("compose.yml"), &original).unwrap(); + fixture.hash = format!("{:x}", Sha256::digest(original.as_bytes())); + for exists in [false, true] { + if exists { + fs::create_dir(fixture.project.join("future")).unwrap(); + fs::write( + fixture.project.join("future/private.txt"), + "TOKEN=OPTIONAL_INPUT_CANARY", + ) + .unwrap(); + fs::write(fixture.project.join(".env"), "TOKEN=DOTENV_INPUT_CANARY").unwrap(); + } + let output = fixture.invoke(&["project", "plan"], &["--json"]); + assert!(output.status.success(), "{output:?}"); + let plan: Value = serde_json::from_slice(&output.stdout).unwrap(); + assert_eq!( + plan["plan"]["original_environment_files"], + serde_json::json!(["future/private.txt"]) + ); + let output = fixture.invoke( + &["project", "capture"], + &["--expect-plan", plan["plan_id"].as_str().unwrap(), "--json"], + ); + assert!(output.status.success(), "{output:?}"); + let capture: Value = serde_json::from_slice(&output.stdout).unwrap(); + assert!( + capture["entries"] + .as_array() + .unwrap() + .iter() + .all(|entry| entry["path"] != "future/private.txt" && entry["path"] != ".env") + ); + assert!( + !String::from_utf8(output.stdout) + .unwrap() + .contains("INPUT_CANARY") + ); + } +} + +#[test] +fn removed_inactive_required_env_file_retains_ownership_while_absent() { + let mut fixture = Fixture::new(); + fs::create_dir(fixture.project.join("future")).unwrap(); + let original = format!( + "{ORIGINAL} inactive:\n image: alpine:3.20\n profiles: [extra]\n env_file: future/inactive.txt\n" + ); + fs::write(fixture.project.join("compose.yml"), &original).unwrap(); + fs::write( + &fixture.normalized, + original.replace(" env_file: future/inactive.txt\n", ""), + ) + .unwrap(); + fixture.hash = format!("{:x}", Sha256::digest(original.as_bytes())); + let output = fixture.invoke(&["project", "plan"], &["--json"]); + assert!(output.status.success(), "{output:?}"); + let plan: Value = serde_json::from_slice(&output.stdout).unwrap(); + assert_eq!( + plan["plan"]["original_environment_files"], + serde_json::json!(["future/inactive.txt"]) + ); + assert_eq!(plan["plan"]["services"]["inactive"]["active"], false); + fs::write( + fixture.project.join("future/inactive.txt"), + "TOKEN=INACTIVE_FILE_CANARY", + ) + .unwrap(); + // File presence changes source metadata; an old review still must refuse. + assert_eq!( + error(fixture.invoke( + &["project", "capture"], + &["--expect-plan", plan["plan_id"].as_str().unwrap(), "--json"] + )), + "stale_plan" + ); + let current = fixture.invoke(&["project", "plan"], &["--json"]); + assert!(current.status.success(), "{current:?}"); + let current: Value = serde_json::from_slice(¤t.stdout).unwrap(); + assert_eq!( + current["plan"]["original_environment_files"], + plan["plan"]["original_environment_files"] + ); + let output = fixture.invoke( + &["project", "capture"], + &[ + "--expect-plan", + current["plan_id"].as_str().unwrap(), + "--json", + ], + ); + assert!(output.status.success(), "{output:?}"); + let capture: Value = serde_json::from_slice(&output.stdout).unwrap(); + assert!( + !capture["entries"] + .as_array() + .unwrap() + .iter() + .any(|entry| entry["path"] == "future/inactive.txt") + ); + assert!( + !String::from_utf8(output.stdout) + .unwrap() + .contains("INACTIVE_FILE_CANARY") + ); +} diff --git a/packages/runtime-core/tests/project_contract.rs b/packages/runtime-core/tests/project_contract.rs index 88b11938e..40564dee3 100644 --- a/packages/runtime-core/tests/project_contract.rs +++ b/packages/runtime-core/tests/project_contract.rs @@ -2179,6 +2179,83 @@ fn scoped_environment_refuses_cross_service_fallback_and_conflicting_owners() { assert!(!fixture.candidate.state_root.exists()); } +#[test] +fn explicit_environment_unsets_are_public_names_and_cannot_mask_missing_private_values() { + use project::inputs; + use std::collections::BTreeMap; + let fixture = Fixture::new( + "services:\n web:\n image: alpine:3.21\n environment: {PUBLIC: literal, PRIVATE: null}\n x-hack-unset-environment: [REMOVE, UNKNOWN]\n", + ); + let review = fixture.plan(); + let private = BTreeMap::from([( + "web".into(), + BTreeMap::from([("PRIVATE".into(), "UNSET_PRIVATE_CANARY".into())]), + )]); + let compiled = inputs::compile_scoped( + &fixture.candidate, + fixture.options(), + &review.plan_id, + &BTreeMap::new(), + &private, + ) + .unwrap(); + assert_eq!( + compiled.executable.services["web"].environment, + ["PUBLIC=literal", "REMOVE", "UNKNOWN"] + ); + assert!( + compiled + .executable + .service_requires_managed_environment("web") + ); + assert!( + !serde_json::to_string(&review) + .unwrap() + .contains("UNSET_PRIVATE_CANARY") + ); + let failure = inputs::compile_scoped( + &fixture.candidate, + fixture.options(), + &review.plan_id, + &BTreeMap::new(), + &BTreeMap::new(), + ) + .err() + .unwrap(); + assert_eq!(failure.code, "execution_environment_missing"); + let only_unsets = Fixture::new( + "services:\n web:\n image: alpine:3.21\n restart: unless-stopped\n x-hack-unset-environment: [REMOVE]\n", + ); + let review = only_unsets.plan(); + let compiled = inputs::compile_scoped( + &only_unsets.candidate, + only_unsets.options(), + &review.plan_id, + &BTreeMap::new(), + &BTreeMap::new(), + ) + .unwrap(); + assert_eq!(compiled.executable.services["web"].environment, ["REMOVE"]); + assert!(!compiled.executable.requires_managed_environment()); + for metadata in [ + "[PRIVATE]", + "[PUBLIC]", + "[REMOVE, REMOVE]", + "[bad-key]", + "false", + ] { + let conflict = Fixture::new(&format!( + "services:\n web:\n image: alpine:3.21\n environment: {{PUBLIC: value, PRIVATE: null}}\n x-hack-unset-environment: {metadata}\n" + )); + assert_eq!( + project::plan(&conflict.candidate, conflict.options()) + .unwrap_err() + .code, + "invalid_environment_unset" + ); + } +} + #[test] fn scoped_environment_cannot_interpolate_into_any_executable_field() { use project::inputs; diff --git a/src/backends/native-aws-environment.ts b/src/backends/native-aws-environment.ts index 619332ac3..adcb6cae6 100644 --- a/src/backends/native-aws-environment.ts +++ b/src/backends/native-aws-environment.ts @@ -128,6 +128,15 @@ function awsMount(value: unknown, home: string): boolean { return true; } +/** Exact authored mount forms consumed privately by AWS adaptation. No file is read. */ +export function nativeAwsCredentialMount(value: unknown): boolean { + try { + return awsMount(value, ""); + } catch { + return false; + } +} + function serviceEnvironment(value: unknown): Record { if (value === undefined) { return {}; diff --git a/src/backends/native-compose-env-syntax.ts b/src/backends/native-compose-env-syntax.ts new file mode 100644 index 000000000..578a01f77 --- /dev/null +++ b/src/backends/native-compose-env-syntax.ts @@ -0,0 +1,350 @@ +/** Compose dotenv syntax and interpolation. All inputs/results may be private. */ +const NAME = /^[A-Za-z_][A-Za-z0-9_]*/; +const ENTRY = + /^(?:export[ \t]+)?([A-Za-z_][A-Za-z0-9_]*)[ \t]*(?:([=:])[ \t]*(.*))?$/; +const COMMENT = /[ \t]+#.*$/; +const OPERATOR = /^(?::)?[-+?]/; +const BOM = /^\uFEFF/; +const ESCAPE_SEQUENCE = /\\(?:[abcfnrtv$"\\]|0\d{0,3})/g; +const OCTAL = /^[0-7]{3}$/; +const MAX_BYTES = 256 * 1024; +const MAX_LINES = 4096; +const ESCAPES: Readonly> = { + a: "\x07", + b: "\b", + f: "\f", + n: "\n", + r: "\r", + t: "\t", + v: "\v", + "\\": "\\", + $: "$$", +}; + +export type ComposeEnvLookup = (name: string) => string | undefined; + +/** Closed failures never include the input, an authored error word or values. */ +export class NativeEnvSyntaxError extends Error { + constructor() { + super("Invalid or unresolved Compose environment input; values omitted."); + } +} +function invalid(): never { + throw new NativeEnvSyntaxError(); +} + +function braced( + text: string, + begin: number, + depth: number +): { readonly body: string; readonly next: number } { + let nesting = 1; + let cursor = begin; + while (cursor < text.length) { + if (text.startsWith("$$", cursor)) { + cursor += 2; + continue; + } + if (text.startsWith("${", cursor)) { + nesting += 1; + if (depth + nesting > 4) { + return invalid(); + } + cursor += 2; + continue; + } + if (text[cursor] === "}") { + nesting -= 1; + if (nesting === 0) { + return { body: text.slice(begin, cursor), next: cursor + 1 }; + } + } + cursor += 1; + } + return invalid(); +} + +function expression(opts: { + readonly body: string; + readonly lookup: ComposeEnvLookup; + readonly expand: (word: string) => string; + readonly onMissing?: (name: string) => void; +}): string { + const name = NAME.exec(opts.body)?.[0]; + if (name === undefined) { + return invalid(); + } + const suffix = opts.body.slice(name.length); + const value = opts.lookup(name); + if (suffix === "") { + if (value === undefined) { + opts.onMissing?.(name); + } + return value ?? ""; + } + const operator = OPERATOR.exec(suffix)?.[0]; + if (operator === undefined) { + return invalid(); + } + const present = + value !== undefined && (!operator.startsWith(":") || value !== ""); + const kind = operator.at(-1); + if (kind === "?" && !present) { + return invalid(); + } + if ((kind === "-" && !present) || (kind === "+" && present)) { + return opts.expand(suffix.slice(operator.length)); + } + return kind === "+" ? "" : (value ?? ""); +} + +function unbraced(opts: { + readonly text: string; + readonly cursor: number; + readonly lookup: ComposeEnvLookup; + readonly onMissing?: (name: string) => void; +}): { readonly value: string; readonly next: number } { + const name = NAME.exec(opts.text.slice(opts.cursor))?.[0]; + if (name === undefined) { + return { value: "$", next: opts.cursor }; + } + const value = opts.lookup(name); + if (value === undefined) { + opts.onMissing?.(name); + } + return { value: value ?? "", next: opts.cursor + name.length }; +} + +/** + * Compose operators, including unset versus empty, nested words and $$ escaping. + * The caller owns the lookup boundary; no ambient environment is consulted here. + */ +export function interpolateComposeEnvironment(opts: { + readonly text: string; + readonly lookup: ComposeEnvLookup; + readonly onMissing?: (name: string) => void; +}): string { + let remaining = MAX_BYTES; + function expand(text: string, depth: number): string { + if (depth > 4 || Buffer.byteLength(text) > MAX_BYTES) { + return invalid(); + } + let result = ""; + let cursor = 0; + function append(value: string): void { + remaining -= Buffer.byteLength(value); + if (remaining < 0 || value.includes("\0")) { + invalid(); + } + result += value; + } + while (cursor < text.length) { + const dollar = text.indexOf("$", cursor); + if (dollar < 0) { + append(text.slice(cursor)); + break; + } + append(text.slice(cursor, dollar)); + cursor = dollar + 1; + if (text[cursor] === "$") { + append("$"); + cursor += 1; + continue; + } + if (text[cursor] === "{") { + if (depth >= 4) { + return invalid(); + } + const found = braced(text, cursor + 1, depth); + append( + expression({ + body: found.body, + lookup: opts.lookup, + onMissing: opts.onMissing, + expand: (word) => expand(word, depth + 1), + }) + ); + cursor = found.next; + continue; + } + const found = unbraced({ + text, + cursor, + lookup: opts.lookup, + onMissing: opts.onMissing, + }); + append(found.value); + cursor = found.next; + } + return result; + } + return expand(opts.text, 0); +} + +/** Match compose-go's escape pass after escaped quote delimiters are removed. */ +function doubleQuotedEscapes(value: string): string { + return value.replace(ESCAPE_SEQUENCE, (match) => { + if (match.startsWith("\\0")) { + const digits = match.slice(2); + return OCTAL.test(digits) + ? String.fromCharCode(Number.parseInt(digits, 8)) + : `\\${digits}`; + } + const key = match[1] ?? ""; + return Object.hasOwn(ESCAPES, key) ? (ESCAPES[key] ?? match) : match; + }); +} + +/** Preserve escape pairs until the Compose double-quote escape pass. */ +function quotedCharacter(opts: { + readonly char: string; + readonly quote: string | undefined; + readonly escaped: boolean; +}): { readonly text: string; readonly escaped: boolean } { + if (opts.char === "\\" && !opts.escaped) { + return { text: "", escaped: true }; + } + return { + text: + opts.escaped && opts.char !== opts.quote ? `\\${opts.char}` : opts.char, + escaped: false, + }; +} + +/** Read a possibly multiline quoted dotenv value, with Compose quote escaping. */ +function quoted(opts: { + readonly lines: readonly string[]; + readonly start: number; + readonly value: string; +}): { readonly value: string; readonly end: number } { + const quote = opts.value[0]; + let decoded = ""; + let rest = opts.value.slice(1); + let position = 0; + let index = opts.start; + let escaped = false; + for (;;) { + const ch = rest[position]; + if (ch === undefined) { + index += 1; + if (index >= opts.lines.length) { + return invalid(); + } + decoded += escaped ? "\\\n" : "\n"; + escaped = false; + rest = opts.lines[index] ?? ""; + position = 0; + continue; + } + if (ch === quote && !escaped) { + const tail = rest.slice(position + 1).trim(); + if (tail !== "" && !tail.startsWith("#")) { + return invalid(); + } + return { + value: quote === '"' ? doubleQuotedEscapes(decoded) : decoded, + end: index, + }; + } + const next = quotedCharacter({ char: ch, quote, escaped }); + decoded += next.text; + escaped = next.escaped; + position += 1; + } +} + +function dotenvValue(opts: { + readonly raw: boolean; + readonly line: string; + readonly value: string; + readonly lines: readonly string[]; + readonly start: number; + readonly lookup: ComposeEnvLookup; + readonly onMissing?: (name: string) => void; +}): { readonly value: string; readonly end: number } { + if (opts.raw) { + const equals = opts.line.indexOf("="); + if (equals < 0) { + return invalid(); + } + return { value: opts.line.slice(equals + 1), end: opts.start }; + } + if (opts.value[0] !== "'" && opts.value[0] !== '"') { + return { + value: interpolateComposeEnvironment({ + text: opts.value.replace(COMMENT, "").trimEnd(), + lookup: opts.lookup, + onMissing: opts.onMissing, + }), + end: opts.start, + }; + } + const decoded = quoted(opts); + return { + value: + opts.value[0] === "'" + ? decoded.value + : interpolateComposeEnvironment({ + text: decoded.value, + lookup: opts.lookup, + onMissing: opts.onMissing, + }), + end: decoded.end, + }; +} + +/** + * Sequential Compose assignments, seeded by earlier files. A bare name inherits + * only from project/service lookup; when unresolved it leaves prior entries + * alone, as compose-go does. Raw format does not unquote or interpolate values. + */ +export function parseComposeEnvironmentFile(opts: { + readonly text: string; + readonly raw?: boolean; + readonly lookup: ComposeEnvLookup; + readonly initial?: ReadonlyMap; + readonly onMissing?: (name: string) => void; +}): ReadonlyMap { + if (Buffer.byteLength(opts.text) > MAX_BYTES || opts.text.includes("\0")) { + return invalid(); + } + const result = new Map(opts.initial); + const lines = opts.text.replace(BOM, "").replaceAll("\r\n", "\n").split("\n"); + if (lines.length > MAX_LINES) { + return invalid(); + } + const lookup = (name: string) => opts.lookup(name) ?? result.get(name); + for (let index = 0; index < lines.length; index += 1) { + const line = (lines[index] ?? "").trimStart(); + if (line.trim() === "" || line.startsWith("#")) { + continue; + } + const entry = ENTRY.exec(line); + const key = entry?.[1]; + if (key === undefined) { + return invalid(); + } + if (opts.raw && entry?.[2] !== "=" && entry?.[2] !== undefined) { + return invalid(); + } + if (entry?.[2] === undefined) { + const inherited = opts.lookup(key); + if (inherited !== undefined) { + result.set(key, inherited); + } + continue; + } + const decoded = dotenvValue({ + raw: opts.raw ?? false, + line, + value: entry?.[3] ?? "", + lines, + start: index, + lookup, + onMissing: opts.onMissing, + }); + index = decoded.end; + result.set(key, decoded.value); + } + return result; +} diff --git a/src/backends/native-compose-environment.ts b/src/backends/native-compose-environment.ts new file mode 100644 index 000000000..6e85fb641 --- /dev/null +++ b/src/backends/native-compose-environment.ts @@ -0,0 +1,726 @@ +import { lstat, realpath } from "node:fs/promises"; +import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path"; +import { HACK_KEY_MATERIAL_VARIABLES } from "../lib/child-environment.ts"; +import { isRecord } from "../lib/guards.ts"; +import { nativeAwsCredentialMount } from "./native-aws-environment.ts"; +import { + interpolateComposeEnvironment, + parseComposeEnvironmentFile, +} from "./native-compose-env-syntax.ts"; +import { + NativeEnvFileRefusal, + nativeEnvFileSourceLocation, + readNativeEnvFileSource, +} from "./native-env-file-source.ts"; +import { nativeGcloudCredentialMount } from "./native-gcloud-environment.ts"; +import { nativeServiceActive } from "./native-project-builds.ts"; +import { shellVariableReadable } from "./native-shell-environment.ts"; + +const NAME = /^[A-Za-z_][A-Za-z0-9_]*$/; +const CONTROLS = new Set([ + "COMPOSE_FILE", + "COMPOSE_PROFILES", + "COMPOSE_PROJECT_NAME", + "COMPOSE_ENV_FILES", + "COMPOSE_DISABLE_ENV_FILE", +]); +const UNSETS = "x-hack-unset-environment"; +type Values = Readonly>; +export type NativeComposeEnvironmentService = { + readonly document: Record; + readonly environment: Record; +}; +type FileSpec = { + readonly path: string; + readonly required: boolean; + readonly raw: boolean; +}; +type Source = { readonly path: string; readonly text: string } | null; +type Options = { + readonly projectRoot: string; + readonly composeFile: string; + readonly compose: Record; + readonly services: Readonly>; + readonly managed: Readonly>; + readonly global: Values; + readonly env?: Readonly>; + readonly profiles?: readonly string[]; + readonly selectedServices?: readonly string[]; +}; +type Sources = { + readonly lookup: (name: string) => string | undefined; + readonly read: (spec: FileSpec, service?: string) => Promise; + readonly dotenv: ReadonlyMap; + readonly shellNames: Set; +}; +type Notice = { readonly service: string; readonly line: string }; + +function refused( + source: string, + reason: string, + service?: string +): NativeEnvFileRefusal { + return new NativeEnvFileRefusal({ source, reason, service }); +} +function protectedName(name: string): boolean { + return name.startsWith("HACK_") || HACK_KEY_MATERIAL_VARIABLES.has(name); +} +function ownValue(values: Values, name: string): string | undefined { + return Object.hasOwn(values, name) ? values[name] : undefined; +} +async function present(path: string): Promise { + try { + await lstat(path); + return true; + } catch (error: unknown) { + if (isRecord(error) && error.code === "ENOENT") { + return false; + } + throw refused(".env", "the interpolation source cannot be inspected"); + } +} + +/** Identity comparison also catches case aliases on case-insensitive filesystems. */ +async function sameSource(a: string, b: string): Promise { + if (a === b) { + return true; + } + try { + const [left, right] = await Promise.all([lstat(a), lstat(b)]); + return left.dev === right.dev && left.ino === right.ino; + } catch (error: unknown) { + if (isRecord(error) && error.code === "ENOENT") { + return false; + } + throw refused(".env", "the interpolation source cannot be inspected"); + } +} +function fileSpecs(value: unknown, service: string): FileSpec[] { + if (value === undefined) { + return []; + } + const values = Array.isArray(value) ? value : [value]; + return values.map((entry) => { + if (typeof entry === "string") { + return { path: entry, required: true, raw: false }; + } + if ( + !(isRecord(entry) && typeof entry.path === "string") || + Object.keys(entry).some( + (key) => !["path", "required", "format"].includes(key) + ) || + (entry.required !== undefined && typeof entry.required !== "boolean") || + (entry.format !== undefined && + entry.format !== "" && + entry.format !== "raw") + ) { + throw refused( + "env_file", + "use a path or path/required/format entry with omitted, empty or raw format", + service + ); + } + return { + path: entry.path, + required: entry.required !== false, + raw: entry.format === "raw", + }; + }); +} + +/** Reject a public expression if resolving it would consult a private source. */ +function checkPublicFields(opts: { + readonly value: unknown; + readonly lookup: Sources["lookup"]; + readonly service?: string; + readonly field?: string; +}): void { + if (typeof opts.value === "string") { + try { + interpolateComposeEnvironment({ + text: opts.value, + lookup: (name) => { + if (opts.lookup(name) !== undefined) { + throw new NativeEnvFileRefusal({ + source: "Compose public field", + service: opts.service, + field: opts.field, + reason: `variable ${name} may only interpolate inside service environment or env-file values`, + }); + } + return undefined; + }, + }); + } catch (error: unknown) { + if (error instanceof NativeEnvFileRefusal) { + throw error; + } + throw new NativeEnvFileRefusal({ + source: "Compose public field", + service: opts.service, + field: opts.field, + reason: "invalid or over-budget interpolation", + }); + } + } else if (Array.isArray(opts.value)) { + for (const value of opts.value) { + if ( + opts.field === "volumes" && + (nativeAwsCredentialMount(value) || nativeGcloudCredentialMount(value)) + ) { + continue; + } + checkPublicFields({ ...opts, value }); + } + } else if (isRecord(opts.value)) { + for (const value of Object.values(opts.value)) { + checkPublicFields({ ...opts, value }); + } + } +} + +function ignoredExtension(field: string): boolean { + return field.startsWith("x-") && !field.startsWith("x-hack-"); +} + +/** The project selector may be a root alias; no links inside that root gain authority. */ +async function canonicalSourceOptions(opts: Options): Promise { + const member = relative(resolve(opts.projectRoot), resolve(opts.composeFile)); + if (member === ".." || member.startsWith(`..${sep}`) || isAbsolute(member)) { + throw refused( + "env_file", + "the Compose file must stay inside the project root" + ); + } + const root = await realpath(opts.projectRoot); + return { ...opts, projectRoot: root, composeFile: resolve(root, member) }; +} + +async function readAutomaticDotenv(opts: { + readonly automatic: string; + readonly read: Sources["read"]; + readonly lookup: Sources["lookup"]; +}): Promise> { + // An absent automatic input grants no file access and adds no new directory + // admission to projects that use no env files. Explicit optional paths still + // go through the complete source guard, including their existing parents. + if (!(await present(opts.automatic))) { + return new Map(); + } + const source = await opts.read({ path: ".env", required: true, raw: false }); + if (source === null) { + return new Map(); + } + let dotenv: ReadonlyMap; + try { + dotenv = parseComposeEnvironmentFile({ + text: source.text, + lookup: opts.lookup, + }); + } catch (error: unknown) { + if (error instanceof NativeEnvFileRefusal) { + throw error; + } + throw refused( + ".env", + "invalid dotenv syntax, unresolved required reference or source budget" + ); + } + for (const key of dotenv.keys()) { + if (CONTROLS.has(key) || protectedName(key)) { + throw refused( + ".env", + `control variable ${key} is not supported as a dotenv input` + ); + } + } + return dotenv; +} + +async function environmentSources(opts: Options): Promise { + const shellNames = new Set(); + const cached = new Map(); + const automatic = join(dirname(opts.composeFile), ".env"); + const env = opts.env ?? {}; + let bytes = 0; + let references = 0; + const shell = (name: string): string | undefined => { + if (!(shellVariableReadable(name) && Object.hasOwn(env, name))) { + return undefined; + } + const value = env[name]; + if (typeof value === "string") { + shellNames.add(name); + return value; + } + return undefined; + }; + const read: Sources["read"] = async (spec, service) => { + references += 1; + if (references > 64) { + throw refused( + spec.path, + "more than 64 environment source references", + service + ); + } + const { path } = nativeEnvFileSourceLocation({ + ...opts, + path: spec.path, + service, + }); + if ( + (await present(`${automatic}.state.json`)) && + (await sameSource(path, automatic)) + ) { + throw refused( + spec.path, + "possible Hack materialized output: use current Hack env layers and deliberately remove obsolete .env and .env.state.json before retrying", + service + ); + } + // Cache one verified snapshot per literal path so repeated references and + // different services cannot consume different versions during preparation. + if (cached.has(path)) { + const prior = cached.get(path) ?? null; + if (prior === null && spec.required) { + throw refused(spec.path, "the required source is missing", service); + } + return prior; + } + const source = await readNativeEnvFileSource({ + ...opts, + path: spec.path, + required: spec.required, + service, + }); + if (source !== null) { + bytes += Buffer.byteLength(source.text); + } + if (bytes > 1024 * 1024) { + throw refused( + spec.path, + "environment source text exceeds 1 MiB", + service + ); + } + cached.set(path, source); + return source; + }; + const baseLookup = (name: string): string | undefined => { + if (protectedName(name)) { + throw refused( + "environment", + `protected variable ${name} cannot be interpolated` + ); + } + return ownValue(opts.global, name) ?? shell(name); + }; + const dotenv = await readAutomaticDotenv({ + automatic, + read, + lookup: baseLookup, + }); + return { + shellNames, + dotenv, + read, + lookup: (name) => baseLookup(name) ?? dotenv.get(name), + }; +} + +type Resolution = { + readonly environment: Record; + readonly resolved: Map; + readonly delivered: Record; + readonly unset: string[]; + readonly unbound: Set; + readonly changed: boolean; +}; +type ServiceContext = { + readonly name: string; + readonly managed: Values; + readonly global: Values; + readonly lookup: Sources["lookup"]; + readonly notices: Notice[]; +}; +function missingNotice(opts: { + readonly context: ServiceContext; + readonly name: string; + readonly key: string; + readonly unset: boolean; +}): void { + opts.context.notices.push({ + service: opts.context.name, + line: `Service ${opts.context.name}: ${opts.key} is ${opts.unset ? "unset" : "empty"} because ${opts.name} is not set in the interpolation context.`, + }); +} +function expandServiceValue(opts: { + readonly text: string; + readonly key: string; + readonly context: ServiceContext; +}): { + readonly value: string; + readonly private: boolean; + readonly unbound: boolean; +} { + const { context } = opts; + let privateSource = false; + let unbound = false; + try { + const value = interpolateComposeEnvironment({ + text: opts.text, + onMissing: (name) => { + if (!shellVariableReadable(name)) { + unbound = true; + return; + } + privateSource = true; + missingNotice({ context, name, key: opts.key, unset: false }); + }, + lookup: (name) => { + if (name === "COMPOSE_PROJECT_NAME") { + unbound = true; + return undefined; + } + if ( + Object.hasOwn(context.managed, name) && + !Object.hasOwn(context.global, name) + ) { + throw refused( + "environment", + `service-scoped variable ${name} is not an interpolation source`, + context.name + ); + } + const found = context.lookup(name); + privateSource ||= found !== undefined; + return found; + }, + }); + if (privateSource && unbound) { + throw refused( + "environment", + "private interpolation cannot combine with unreadable or runtime-bound sources", + context.name + ); + } + if (privateSource && !shellVariableReadable(opts.key)) { + throw refused( + "environment", + "use an explicit literal env-file entry or the credential adapter for a reserved destination", + context.name + ); + } + return { value, private: privateSource, unbound }; + } catch (error: unknown) { + if (error instanceof NativeEnvFileRefusal) { + throw error; + } + throw refused( + "environment", + "invalid interpolation, required reference or NUL value", + context.name + ); + } +} +function scalar(value: unknown): value is number | boolean { + return typeof value === "number" || typeof value === "boolean"; +} + +function resolveBareService(opts: { + readonly context: ServiceContext; + readonly key: string; + readonly environment: Record; + readonly delivered: Record; + readonly resolved: Map; + readonly unset: string[]; +}): void { + const { key, context } = opts; + const found = context.lookup(key); + if (found !== undefined) { + opts.environment[key] = null; + opts.delivered[key] = found; + opts.resolved.set(key, found); + return; + } + if (!shellVariableReadable(key)) { + opts.environment[key] = null; + return; + } + opts.unset.push(key); + missingNotice({ context, name: key, key, unset: true }); +} +function resolveService(opts: { + readonly context: ServiceContext; + readonly environment: Record; +}): Resolution { + const environment: Record = Object.create(null); + const delivered: Record = Object.create(null); + const resolved = new Map(); + const unset: string[] = []; + const unbound = new Set(); + const { context } = opts; + let changed = false; + for (const [key, value] of Object.entries(opts.environment)) { + if (!NAME.test(key)) { + throw refused( + "environment", + "variable names must be valid identifiers", + context.name + ); + } + const managed = ownValue(context.managed, key); + if (managed !== undefined) { + environment[key] = null; + resolved.set(key, managed); + continue; + } + if (HACK_KEY_MATERIAL_VARIABLES.has(key)) { + throw refused( + "environment", + "Hack key material cannot be copied into services", + context.name + ); + } + if (value === null) { + changed = true; + resolveBareService({ + key, + context, + environment, + delivered, + resolved, + unset, + }); + continue; + } + if (typeof value !== "string") { + if (!scalar(value)) { + throw refused( + "environment", + "environment values must be scalar", + context.name + ); + } + environment[key] = value; + resolved.set(key, String(value)); + continue; + } + const expanded = expandServiceValue({ text: value, key, context }); + environment[key] = expanded.private ? null : value; + if (expanded.unbound) { + unbound.add(key); + } else { + resolved.set(key, expanded.value); + } + if (expanded.private) { + delivered[key] = expanded.value; + changed = true; + } + } + return { environment, resolved, delivered, unset, unbound, changed }; +} + +async function serviceFiles(opts: { + readonly context: ServiceContext; + readonly sources: Sources; + readonly specs: readonly FileSpec[]; + readonly resolution: Resolution; +}): Promise> { + let files: ReadonlyMap = new Map(); + for (const spec of opts.specs) { + const source = await opts.sources.read(spec, opts.context.name); + if (source === null) { + continue; + } + try { + files = parseComposeEnvironmentFile({ + text: source.text, + raw: spec.raw, + initial: files, + onMissing: (key) => + missingNotice({ + context: opts.context, + name: key, + key, + unset: false, + }), + lookup: (key) => { + const project = opts.context.lookup(key); + if (project !== undefined) { + return project; + } + if ( + key === "COMPOSE_PROJECT_NAME" || + opts.resolution.unbound.has(key) + ) { + throw refused( + spec.path, + `runtime-dependent variable ${key} cannot interpolate private file values`, + opts.context.name + ); + } + if ( + !( + opts.resolution.resolved.has(key) || + shellVariableReadable(key) || + Object.hasOwn(opts.context.global, key) || + opts.sources.dotenv.has(key) + ) + ) { + throw refused( + spec.path, + `variable ${key} cannot be read from the caller's shell; use an explicit file value or credential adapter`, + opts.context.name + ); + } + return opts.resolution.resolved.get(key); + }, + }); + } catch (error: unknown) { + if (error instanceof NativeEnvFileRefusal) { + throw error; + } + throw refused( + spec.path, + "invalid dotenv syntax, unresolved required reference or source budget", + opts.context.name + ); + } + } + return files; +} + +async function normalizeService(opts: { + readonly context: ServiceContext; + readonly sources: Sources; + readonly service: NativeComposeEnvironmentService; + readonly active: boolean; +}): Promise> { + const { document, environment: authored } = opts.service; + if (Object.hasOwn(document, UNSETS)) { + throw refused( + "environment", + "the native unset metadata field is reserved", + opts.context.name + ); + } + for (const [field, value] of Object.entries(document)) { + if ( + field !== "environment" && + field !== "env_file" && + !ignoredExtension(field) + ) { + checkPublicFields({ + value, + lookup: opts.context.lookup, + service: opts.context.name, + field, + }); + } + } + const resolution = resolveService({ + context: opts.context, + environment: authored, + }); + const specs = fileSpecs(document.env_file, opts.context.name); + const files = opts.active + ? await serviceFiles({ ...opts, specs, resolution }) + : new Map(); + const environment: Record = Object.assign( + Object.create(null), + resolution.environment + ); + for (const [key, value] of files) { + if (protectedName(key)) { + throw refused( + "env_file", + `protected variable ${key} cannot be delivered`, + opts.context.name + ); + } + if (Object.hasOwn(authored, key)) { + continue; + } + environment[key] = null; + resolution.delivered[key] = value; + } + if (specs.length > 0 || resolution.changed) { + document.environment = environment; + } + if (resolution.unset.length > 0) { + // Names are ASCII identifiers: default code-unit order equals Rust BTreeSet order. + document[UNSETS] = resolution.unset.sort(); + } + Reflect.deleteProperty(document, "env_file"); + return resolution.delivered; +} + +/** + * Native-only normalization. Public declarations contain names, never file or + * shell values. Returned maps are private ephemeral inputs, not managed-env + * fingerprints. Start and preflight call this same preparation path. + */ +export async function normalizeNativeComposeEnvironment( + opts: Options +): Promise<{ + readonly shellEnvironment: Record>; + readonly notices: Notice[]; + readonly forwardedNames: Record; + readonly privateValues: readonly string[]; + readonly fileServices: Readonly>; +}> { + const sources = await environmentSources(await canonicalSourceOptions(opts)); + const privateValues = new Set(); + for (const [field, value] of Object.entries(opts.compose)) { + if (field !== "services" && !ignoredExtension(field)) { + checkPublicFields({ value, lookup: sources.lookup, field }); + } + } + const shellEnvironment: Record< + string, + Record + > = Object.create(null); + const notices: Notice[] = []; + const forwardedNames: Record = Object.create(null); + const fileServices: Record = Object.create(null); + for (const [name, service] of Object.entries(opts.services)) { + sources.shellNames.clear(); + const active = + opts.selectedServices === undefined + ? nativeServiceActive(service.document, opts.profiles ?? []) + : opts.selectedServices.includes(name); + if (Object.hasOwn(service.document, "env_file")) { + fileServices[name] = active; + } + const delivered = await normalizeService({ + sources, + service, + active, + context: { + name, + global: opts.global, + managed: Object.hasOwn(opts.managed, name) + ? (opts.managed[name] ?? {}) + : {}, + lookup: sources.lookup, + notices, + }, + }); + if (Object.keys(delivered).length > 0) { + shellEnvironment[name] = delivered; + } + for (const value of Object.values(delivered)) { + privateValues.add(value); + } + forwardedNames[name] = [...sources.shellNames].sort(); + } + return { + shellEnvironment, + notices, + forwardedNames, + privateValues: [...privateValues].filter((value) => value !== ""), + fileServices, + }; +} diff --git a/src/backends/native-env-file-source.ts b/src/backends/native-env-file-source.ts new file mode 100644 index 000000000..045ae72ee --- /dev/null +++ b/src/backends/native-env-file-source.ts @@ -0,0 +1,238 @@ +import { constants, type Stats } from "node:fs"; +import { lstat, open } from "node:fs/promises"; +import { + basename, + dirname, + isAbsolute, + relative, + resolve, + sep, +} from "node:path"; +import { HackCliError } from "../lib/cli-result.ts"; +import { isRecord } from "../lib/guards.ts"; + +const MAX_FILE_BYTES = 256 * 1024; +const SERVICE = /^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$/; +const VARIABLE_PATH = /[$\0\r\n]/; +const FIELD = /^[A-Za-z0-9_.-]{1,128}$/; +const IGNORE_SOURCES = new Set([".gitignore", ".ignore"]); + +/** A closed native-only refusal; source paths and names are allowed, values are not. */ +export class NativeEnvFileRefusal extends HackCliError { + readonly service?: string; + readonly source: string; + readonly field: string; + + constructor(opts: { + readonly source: string; + readonly reason: string; + readonly service?: string; + readonly field?: string; + }) { + const service = + opts.service !== undefined && SERVICE.test(opts.service) + ? opts.service + : undefined; + const fallbackField = + opts.source === "environment" ? "environment" : "env_file"; + const field = + opts.field !== undefined && FIELD.test(opts.field) + ? opts.field + : fallbackField; + super({ + code: "E_NATIVE_PROJECT_UNSUPPORTED", + message: `Native environment input${service === undefined ? "" : ` for service ${service} field ${field}`} source ${JSON.stringify(opts.source)} refused: ${opts.reason}; values omitted.`, + }); + this.source = opts.source; + this.service = service; + this.field = field; + } +} + +function missing(error: unknown): boolean { + return isRecord(error) && error.code === "ENOENT"; +} + +function sameVersion(a: Stats, b: Stats): boolean { + return ( + a.dev === b.dev && + a.ino === b.ino && + a.size === b.size && + a.mtimeMs === b.mtimeMs && + a.ctimeMs === b.ctimeMs && + a.mode === b.mode && + a.uid === b.uid && + a.nlink === b.nlink + ); +} + +function safeFile(info: Stats): boolean { + return ( + info.isFile() && + info.uid === process.getuid?.() && + info.nlink === 1 && + (info.mode & 0o022) === 0 && + (info.mode & 0o400) !== 0 && + info.size <= MAX_FILE_BYTES + ); +} + +async function readStableBytes(opts: { + readonly path: string; + readonly observed: Stats; + readonly refusal: (reason: string) => Error; +}): Promise { + const file = await open( + opts.path, + constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK + ); + try { + const before = await file.stat(); + if (!sameVersion(opts.observed, before)) { + throw opts.refusal("the source changed before reading"); + } + const buffer = Buffer.alloc(before.size + 1); + let size = 0; + while (size < buffer.length) { + const chunk = await file.read(buffer, size, buffer.length - size, size); + if (chunk.bytesRead === 0) { + break; + } + size += chunk.bytesRead; + } + if ( + size !== before.size || + !sameVersion(before, await file.stat()) || + !sameVersion(before, await lstat(opts.path)) + ) { + throw opts.refusal("the source changed while reading"); + } + return buffer.subarray(0, size); + } finally { + await file.close(); + } +} + +/** Literal source admission also runs before reusing a cached read. */ +export function nativeEnvFileSourceLocation(opts: { + readonly projectRoot: string; + readonly composeFile: string; + readonly path: string; + readonly service?: string; +}): { readonly root: string; readonly path: string; readonly member: string } { + const refusal = (reason: string) => + new NativeEnvFileRefusal({ + source: opts.path, + service: opts.service, + reason, + }); + if ( + opts.path.length === 0 || + opts.path.length > 4096 || + VARIABLE_PATH.test(opts.path) || + opts.path.includes("~") + ) { + throw refusal("use a literal project-local path"); + } + const root = resolve(opts.projectRoot); + const path = resolve(dirname(opts.composeFile), opts.path); + if (IGNORE_SOURCES.has(basename(path))) { + throw refusal("an environment source cannot also own source ignore rules"); + } + const member = relative(root, path); + if ( + member === "" || + member === ".." || + member.startsWith(`..${sep}`) || + isAbsolute(member) + ) { + throw refusal("the source must stay inside this project root"); + } + return { root, path, member }; +} + +/** + * Read one project-owned env source without mutations. Every ancestor within + * the canonical project root is a stable, owned, non-writable-by-others real + * directory. The final entry is opened O_NOFOLLOW|O_NONBLOCK and must be a + * stable single-link regular file that other users cannot write. Optional + * ignores initial ENOENT only, never a disappearance during a read. + */ +export async function readNativeEnvFileSource(opts: { + readonly projectRoot: string; + readonly composeFile: string; + readonly path: string; + readonly required: boolean; + readonly service?: string; +}): Promise<{ readonly path: string; readonly text: string } | null> { + const { root, path, member } = nativeEnvFileSourceLocation(opts); + const refusal = (reason: string) => + new NativeEnvFileRefusal({ + source: opts.path, + service: opts.service, + reason, + }); + const directories: { readonly path: string; readonly info: Stats }[] = []; + const paths = [root]; + let parent = root; + for (const component of member.split(sep).slice(0, -1)) { + parent = resolve(parent, component); + paths.push(parent); + } + const recheckDirectories = async () => { + for (const directory of directories) { + const now = await lstat(directory.path); + if ( + !now.isDirectory() || + now.dev !== directory.info.dev || + now.ino !== directory.info.ino || + now.uid !== directory.info.uid || + (now.mode & 0o022) !== 0 + ) { + throw refusal("a source directory changed or is unsafe"); + } + } + }; + let observedSource = false; + try { + for (const directory of paths) { + const info = await lstat(directory); + if ( + !info.isDirectory() || + info.uid !== process.getuid?.() || + (info.mode & 0o022) !== 0 + ) { + throw refusal( + "source directories must be real, owned by this user and not writable by others" + ); + } + directories.push({ path: directory, info }); + } + const observed = await lstat(path); + observedSource = true; + if (!safeFile(observed)) { + throw refusal( + "use a single-link regular file owned by this user, owner-readable and not writable by others, at most 256 KiB" + ); + } + const buffer = await readStableBytes({ path, observed, refusal }); + await recheckDirectories(); + return { + path, + text: new TextDecoder("utf-8", { fatal: true }).decode(buffer), + }; + } catch (error: unknown) { + if (missing(error) && !observedSource && directories.length > 0) { + await recheckDirectories().catch(() => { + throw refusal("a source directory changed"); + }); + if (!opts.required) { + return null; + } + throw refusal("the required source is missing"); + } + throw error instanceof NativeEnvFileRefusal + ? error + : refusal("the source is unreadable, unsafe or not valid UTF-8"); + } +} diff --git a/src/backends/native-environment-unsets.ts b/src/backends/native-environment-unsets.ts new file mode 100644 index 000000000..873a1b514 --- /dev/null +++ b/src/backends/native-environment-unsets.ts @@ -0,0 +1,37 @@ +import { isRecord } from "../lib/guards.ts"; + +/** + * Old runtimes ignore extension metadata. Require the reviewed plan to echo + * every names-only removal before any effect, so a new CLI cannot silently + * retain an image default when paired with an older runtime binary. + */ +export function nativeEnvironmentUnsetsReviewed(opts: { + readonly compose: unknown; + readonly plan: unknown; +}): boolean { + if (!(isRecord(opts.compose) && isRecord(opts.compose.services))) { + return false; + } + for (const [name, service] of Object.entries(opts.compose.services)) { + if ( + !( + isRecord(service) && Array.isArray(service["x-hack-unset-environment"]) + ) || + service["x-hack-unset-environment"].length === 0 + ) { + continue; + } + if (!(isRecord(opts.plan) && isRecord(opts.plan.services))) { + return false; + } + const reviewed = opts.plan.services[name]; + if ( + !isRecord(reviewed) || + JSON.stringify(reviewed.environment_unsets) !== + JSON.stringify(service["x-hack-unset-environment"]) + ) { + return false; + } + } + return true; +} diff --git a/src/backends/native-exec-environment.ts b/src/backends/native-exec-environment.ts index 7c54b101b..763cf4397 100644 --- a/src/backends/native-exec-environment.ts +++ b/src/backends/native-exec-environment.ts @@ -1,7 +1,10 @@ import { HackCliError } from "../lib/cli-result.ts"; import { adaptNativeAwsEnvironment } from "./native-aws-environment.ts"; import { adaptNativeGcloudEnvironment } from "./native-gcloud-environment.ts"; -import { prepareNativeProjectInput } from "./native-project-input.ts"; +import { + mergeNativeShellEnvironment, + prepareNativeProjectInput, +} from "./native-project-input.ts"; import { describeNativeRunEnvironmentChange, type NativeProjectRun, @@ -36,6 +39,7 @@ export async function prepareNativeExecEnvironment(opts: { readonly run: NativeProjectRun; readonly composeFile: string; readonly service: string; + readonly env?: Readonly>; readonly prepare?: typeof prepareNativeProjectInput; readonly adaptAws?: typeof adaptNativeAwsEnvironment; readonly adaptGcloud?: typeof adaptNativeGcloudEnvironment; @@ -58,6 +62,9 @@ export async function prepareNativeExecEnvironment(opts: { projectDir: opts.scope.projectDir, composeFile: opts.composeFile, envName: opts.run.effectiveEnvName, + env: opts.env ?? process.env, + profiles: opts.run.profiles ?? [], + selectedServices: [opts.service], }); if ( !input.serviceNames.includes(opts.service) || @@ -84,6 +91,7 @@ export async function prepareNativeExecEnvironment(opts: { ) { throw new SelectionChanged(); } + input = mergeNativeShellEnvironment(input); if (opts.run.aws) { input = ( await (opts.adaptAws ?? adaptNativeAwsEnvironment)({ diff --git a/src/backends/native-exec-stale-env.ts b/src/backends/native-exec-stale-env.ts index 6e05dfa3e..84d440e46 100644 --- a/src/backends/native-exec-stale-env.ts +++ b/src/backends/native-exec-stale-env.ts @@ -62,6 +62,7 @@ export async function resolveNativeImageExecStaleEnvWarning(opts: { projectDir: opts.scope.projectDir, composeFile: opts.composeFile, envName: mapping.effectiveEnvName, + managedOnly: true, }); if (input.effectiveEnvName !== mapping.effectiveEnvName) { return null; diff --git a/src/backends/native-gcloud-environment.ts b/src/backends/native-gcloud-environment.ts index 340678234..ea746348a 100644 --- a/src/backends/native-gcloud-environment.ts +++ b/src/backends/native-gcloud-environment.ts @@ -148,6 +148,15 @@ function adcMount(value: unknown, home: string): string | undefined { return longAdcMount(value, sources); } +/** Exact authored mount forms consumed privately by ADC adaptation. No file is read. */ +export function nativeGcloudCredentialMount(value: unknown): boolean { + try { + return adcMount(value, "") !== undefined; + } catch { + return false; + } +} + function mountTarget(mount: unknown): string { if (typeof mount === "string") { const parts = mount.split(":"); diff --git a/src/backends/native-preflight.ts b/src/backends/native-preflight.ts index 62a813041..565acd171 100644 --- a/src/backends/native-preflight.ts +++ b/src/backends/native-preflight.ts @@ -1,5 +1,5 @@ import { createHash } from "node:crypto"; -import { access, chmod, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { chmod, mkdtemp, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join, relative } from "node:path"; import { isRecord } from "../lib/guards.ts"; @@ -8,17 +8,19 @@ import { classifyComposeBackendReading, RESERVED_OWNERSHIP_LABEL_PREFIXES, } from "../lib/native-compose-backend-features.ts"; -import { - projectEnvConfigExists, - resolveProjectEnvMetadata, -} from "../lib/project-env-config.ts"; +import { NativeEnvFileRefusal } from "./native-env-file-source.ts"; +import { nativeEnvironmentUnsetsReviewed } from "./native-environment-unsets.ts"; import { admitNativeHttpProbe } from "./native-http-probe-admission.ts"; import { prepareNativeProjectAdaptation } from "./native-project-adaptation.ts"; import { admitNativeProjectBuild } from "./native-project-builds.ts"; import type { NativeProjectInput } from "./native-project-input.ts"; import { - nativeServiceEnvironment, + mergeNativeShellEnvironment, + nativeProjectInputRefusalMessage, + prepareNativeProjectInput, readNativeComposeBytes, + requireNativeShellDeliveryFits, + selectActiveNativeProjectInput, } from "./native-project-input.ts"; import { applyNativeInternalHostAliases, @@ -42,7 +44,6 @@ import { NativeRuntimeRequestError, type NativeRuntimeSelection, } from "./native-runtime-client.ts"; -import { shellReference } from "./native-shell-environment.ts"; import { NATIVE_START_REFUSAL_ENTRIES } from "./native-start-refusal-entries.ts"; import { readNativeStartRefusals } from "./native-start-refusals.ts"; @@ -369,7 +370,7 @@ const RUNTIME_FINDINGS: Readonly> = { }, unsupported_env_file_mode: { message: "env_file uses an option native does not accept.", - fix: "Move the variables into Hack managed env with hack env set --service.", + fix: "Use an omitted or empty format for Compose dotenv syntax, or raw.", }, invalid_compose_yaml: { message: "The Compose file is not valid YAML.", @@ -489,6 +490,7 @@ export const REVIEW_STOP_CODES: ReadonlySet = new Set([ "invalid_duration", "invalid_env_file", "invalid_environment_key", + "invalid_environment_unset", "invalid_extra_hosts", "invalid_healthcheck", "invalid_identifier", @@ -723,17 +725,6 @@ function cliServiceFindings(opts: { builds.add(name); } } - if (isRecord(service) && Object.hasOwn(service, "env_file")) { - found.push( - finding( - { ...base, code: "env_file", field: "env_file" }, - { - message: "The service uses env_file, which native does not read.", - fix: "Move its variables into Hack managed env with hack env set --service, or run hack doctor --migrate-env-config.", - } - ) - ); - } for (const problem of nativeServicePreparationProblems({ name, service, @@ -778,13 +769,6 @@ function cliServiceFindings(opts: { return found; } -async function exists(path: string): Promise { - return await access(path).then( - () => true, - () => false - ); -} - /** * Reads the Compose file with the bounds native start applies * (`readCompose` in `native-project-input.ts`): a regular file opened without @@ -823,51 +807,37 @@ type AdaptedProject = { /** False when the adaptation was refused and the document is unadapted. */ readonly adaptationValid: boolean; readonly managedServices: readonly string[]; + /** In-memory check only. Captured private values never enter report JSON. */ + readonly checkDelivery: (plan: unknown) => void; }; }; -/** - * Normalize the same managed keys and shell references as start, without a - * decryption key or access to process.env. Only service names leave this helper. - * An unset shell reference is still private delivery (of an empty value) at start. - */ -async function preflightEnvironment(opts: { - readonly projectRoot: string; - readonly projectDir: string; - readonly effectiveEnvName: string | null; - readonly services: Record; -}): Promise { - const metadata = await resolveProjectEnvMetadata({ - projectRoot: opts.projectRoot, - projectDir: opts.projectDir, - envName: opts.effectiveEnvName, - serviceNames: Object.keys(opts.services), - }); - const delivered: string[] = []; - for (const [name, service] of Object.entries(opts.services)) { - if (!isRecord(service)) { - continue; - } - const managed = new Set( - Object.keys(metadata?.effectiveMetadata[name] ?? {}) - ); - const environment = nativeServiceEnvironment(service.environment, name); - let hasDelivery = managed.size > 0; - for (const key of managed) { - environment[key] = null; - } - for (const [key, value] of Object.entries(environment)) { - if (shellReference({ key, value, managed }) !== undefined) { - environment[key] = null; - hasDelivery = true; - } - } - if (hasDelivery) { - service.environment = environment; - delivered.push(name); +function environmentInputFinding( + error: unknown, + delivery = false +): NativePreflightFinding { + const source = error instanceof NativeEnvFileRefusal ? error : undefined; + const inputCode = + source === undefined ? "managed_env_unreadable" : "environment_file_input"; + return finding( + { + severity: "blocker", + gate: "cli", + code: delivery ? "environment_delivery" : inputCode, + service: source?.service, + field: source?.field, + }, + { + message: + source?.message ?? + nativeProjectInputRefusalMessage(error) ?? + "The project's environment declarations or Hack managed env could not be read or decrypted.", + fix: + source === undefined + ? "Check the environment declarations and run hack env list to see the problem." + : "Check the named source and environment declaration; use owned regular files that are not writable by others.", } - } - return delivered.sort(byCodePoint); + ); } /** @@ -886,6 +856,7 @@ async function adaptForPreflight(opts: { readonly internalHostSources: Readonly>; readonly hasDefaultDependencyFile: boolean; readonly effectiveEnvName: string | null; + readonly profiles: readonly string[]; }): Promise { const blocker = { severity: "blocker", gate: "cli" } as const; // One bounded read: the parsed document and the digest come from the same bytes. @@ -911,26 +882,6 @@ async function adaptForPreflight(opts: { ...structuredClone(compose), services: structuredClone(compose.services), }; - let managedServices: readonly string[] = []; - try { - managedServices = await preflightEnvironment({ - ...opts, - services: normalized.services, - }); - } catch { - findings.push( - finding( - { ...blocker, code: "managed_env_unreadable" }, - { - message: - "The project's service environment declarations or Hack managed env metadata could not be read.", - fix: "Check the service environment declarations and run hack env list to see the problem.", - } - ) - ); - } - // Only the public document is adapted; no private value is involved. Every - // field is named so a helper that starts reading one fails the type check. let input: NativeProjectInput = { originalSha256: createHash("sha256").update(bytes).digest("hex"), normalizedComposeJson: JSON.stringify(normalized), @@ -941,6 +892,25 @@ async function adaptForPreflight(opts: { environmentFiles: [], serviceNames: Object.keys(compose.services), }; + try { + const prepared = await prepareNativeProjectInput({ + ...opts, + envName: opts.effectiveEnvName, + env: process.env, + }); + if (prepared.originalSha256 !== input.originalSha256) { + throw new Error("Compose input changed during preflight"); + } + input = prepared; + } catch (error: unknown) { + findings.push(environmentInputFinding(error)); + } + const managedServices = [ + ...new Set([ + ...Object.keys(input.managedEnvironment), + ...Object.keys(input.shellEnvironment ?? {}), + ]), + ].sort(byCodePoint); let adaptationValid = true; try { input = await prepareNativeProjectAdaptation({ @@ -998,6 +968,13 @@ async function adaptForPreflight(opts: { originalSha256: input.originalSha256, originalCompose: compose, managedServices, + checkDelivery: (plan) => + requireNativeShellDeliveryFits( + selectActiveNativeProjectInput({ + input: mergeNativeShellEnvironment(input), + reviewedPlan: plan, + }) + ), normalizedComposeJson: input.normalizedComposeJson, services: isRecord(adapted) && isRecord(adapted.services) ? adapted.services : {}, @@ -1063,35 +1040,6 @@ async function collectNativeCliFindings(opts: { }) ); } - const managedEnv = await projectEnvConfigExists({ - projectDir: opts.projectDir, - }).catch(() => null); - if (managedEnv === null) { - found.push( - finding( - { ...blocker, code: "managed_env_unreadable" }, - { - message: "The project's Hack managed env files could not be read.", - fix: "Run hack env list to see the problem.", - } - ) - ); - } else if ( - !managedEnv && - ((await exists(join(opts.projectDir, ".env"))) || - (await exists(join(opts.projectRoot, ".env")))) - ) { - found.push( - finding( - { ...blocker, code: "dotenv_without_managed_env" }, - { - message: - "The project has a .env file and no Hack managed env. Native does not read .env files.", - fix: "Migrate it with hack doctor --migrate-env-config, or remove the file.", - } - ) - ); - } if (!opts.sharedSource) { found.push( finding( @@ -1291,6 +1239,33 @@ async function collectNativeRuntimeFindings(opts: { if (!(isRecord(reply) && isRecord(reply.plan))) { return { review: { status: "unavailable" }, findings: [] }; } + if ( + !nativeEnvironmentUnsetsReviewed({ + compose: { services: opts.adapted.services }, + plan: reply.plan, + }) + ) { + return { + review: { + status: "unavailable", + code: "native_environment_unset_unavailable", + }, + findings: [ + finding( + { + severity: "blocker", + gate: "runtime", + code: "native_environment_unset_unavailable", + }, + { + message: + "The selected native runtime cannot confirm explicit environment removals.", + fix: "Update the selected native runtime and retry; no VM was started.", + } + ), + ], + }; + } const services = Object.keys(opts.adapted.services).filter((name) => SERVICE.test(name) ); @@ -1460,6 +1435,18 @@ export type NativePreflightOptions = { readonly invoke?: typeof invokeNativeRuntime; }; +function deliveryFindings( + document: NonNullable, + plan: unknown +): NativePreflightFinding[] { + try { + document.checkDelivery(plan); + return []; + } catch (error: unknown) { + return [environmentInputFinding(error, true)]; + } +} + /** Runs both gates over one project and merges their findings. */ export async function collectNativePreflight( opts: NativePreflightOptions @@ -1498,6 +1485,7 @@ export async function collectNativePreflight( invoke: opts.invoke, }); if (runtime.plan) { + cli.push(...deliveryFindings(adapted.document, runtime.plan)); cli.push( ...routeFindings({ plan: runtime.plan, diff --git a/src/backends/native-project-builds.ts b/src/backends/native-project-builds.ts index e10516b5f..251dc85fa 100644 --- a/src/backends/native-project-builds.ts +++ b/src/backends/native-project-builds.ts @@ -499,7 +499,7 @@ export function refuseEmulatedBuilds(opts: { /** * Refuse, before any hook, VM or build effect, a build argument whose value equals a - * secret value Hack manages for this project and environment (see below). + * managed secret or value actually delivered privately to this project's services. */ export function refuseManagedSecretBuildArguments(opts: { readonly builds: ReadonlyMap; @@ -514,7 +514,7 @@ export function refuseManagedSecretBuildArguments(opts: { if (secrets.has(value)) { throw new HackCliError({ code: "E_NATIVE_PROJECT_UNSUPPORTED", - message: `Native foreground up cannot build service ${service}: build arg ${name} equals a hack env secret value. Build args are not secrets (images record them); pass secrets with BuildKit build secrets instead, which native builds do not support yet. Values were omitted; no hook, VM or build was started.`, + message: `Native foreground up cannot build service ${service}: build arg ${name} equals a privately delivered environment value. Build args are not secrets (images record them); pass secrets with BuildKit build secrets instead, which native builds do not support yet. Values were omitted; no hook, VM or build was started.`, }); } } diff --git a/src/backends/native-project-input.ts b/src/backends/native-project-input.ts index a47e44279..0074d356e 100644 --- a/src/backends/native-project-input.ts +++ b/src/backends/native-project-input.ts @@ -1,7 +1,6 @@ import { createHash } from "node:crypto"; import { constants } from "node:fs"; -import { lstat, open } from "node:fs/promises"; -import { join } from "node:path"; +import { open } from "node:fs/promises"; import { HackCliError } from "../lib/cli-result.ts"; import { isRecord } from "../lib/guards.ts"; import { @@ -11,15 +10,17 @@ import { resolveProjectEnvSelection, selectProjectEnvValuesForExecutionTarget, } from "../lib/project-env-config.ts"; +import { + type NativeComposeEnvironmentService, + normalizeNativeComposeEnvironment, +} from "./native-compose-environment.ts"; +import { NativeEnvFileRefusal } from "./native-env-file-source.ts"; import { MAX_NATIVE_PRIVATE_INPUT_BYTES, nativePrivateEnvironmentPayload, } from "./native-private-environment-payload.ts"; import type { NativeEmulationPolicy } from "./native-project-platform.ts"; -import { - resolveShellEnvironment, - shellDeliveryProblem, -} from "./native-shell-environment.ts"; +import { shellDeliveryProblem } from "./native-shell-environment.ts"; const LIMIT = 256 * 1024; const KEY = /^[A-Za-z_][A-Za-z0-9_]*$/; @@ -52,8 +53,8 @@ export type NativeProjectInput = { Record >; /** - * Private in-memory only: the values Hack stores as secrets for this project and - * environment, so build arguments can refuse to carry one. Never log or write it. + * Private in-memory disclosure guard: managed secrets plus private file/shell + * inputs, so build arguments cannot carry them. Not a stored-env fingerprint. */ readonly managedSecretValues?: readonly string[]; /** Private lifecycle environment; not container environment. */ @@ -65,16 +66,18 @@ export type NativeProjectInput = { readonly effectiveEnvName: string | null; readonly environmentFiles: readonly string[]; readonly serviceNames: readonly string[]; + /** Names-only activation expectation, checked against the runtime before effects. */ + readonly environmentFileServices?: Readonly>; /** - * Private in-memory transport only: values read from the caller's shell for - * service `environment` references, by service. Kept apart from + * Private in-memory transport only: file and caller-shell values for service + * environment inputs. Kept apart from * `managedEnvironment` so they are never fingerprinted or treated as values * Hack stores; see {@link mergeNativeShellEnvironment}. Never log this map. */ readonly shellEnvironment?: Readonly< Record>> >; - /** Services whose private delivery includes a value from the shell, once merged. */ + /** Services receiving ephemeral file/shell values, once merged. */ readonly shellServices?: readonly string[]; /** Forwarded source names by service, for value-free operator notices only. */ readonly forwardedShellVariables?: Readonly< @@ -101,6 +104,13 @@ const SERVICE_NAME = /^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$/; */ class NativeProjectInputRefusal extends HackCliError {} +/** Only this boundary's closed messages may reach a preflight finding. */ +export function nativeProjectInputRefusalMessage( + error: unknown +): string | undefined { + return error instanceof NativeProjectInputRefusal ? error.message : undefined; +} + function refused( cause = "unsupported or invalid Compose/environment configuration", code: @@ -189,7 +199,7 @@ export function nativeServiceEnvironment( if (!Array.isArray(value)) { throw invalid(); } - const result: Record = {}; + const result: Record = Object.create(null); for (const entry of value) { if (typeof entry !== "string") { throw invalid(); @@ -204,23 +214,14 @@ export function nativeServiceEnvironment( return result; } -async function exists(path: string): Promise { - try { - await lstat(path); - return true; - } catch (error) { - if (isRecord(error) && error.code === "ENOENT") { - return false; - } - throw error; - } -} - function normalizeManagedEnvironment( services: Record, resolved: Awaited> ): Record> { - const managedEnvironment: Record> = {}; + const managedEnvironment: Record< + string, + Record + > = Object.create(null); for (const name of Object.keys(services)) { const service = services[name]; if (!isRecord(service)) { @@ -253,57 +254,9 @@ function normalizeManagedEnvironment( } /** - * Resolves the shell references in each service's `environment` (see - * `native-shell-environment.ts`). The public document keeps only bare - * references; the values are returned separately and are not part of - * `managedEnvironment` until {@link mergeNativeShellEnvironment}. - */ -function resolveServiceShellEnvironment(opts: { - readonly services: Record; - readonly managedEnvironment: Record>; - readonly env?: Readonly>; -}): { - readonly shellEnvironment: Record>; - readonly notices: NativeEnvironmentNotice[]; - readonly forwardedNames: Readonly>; -} { - const shellEnvironment: Record> = {}; - const notices: NativeEnvironmentNotice[] = []; - const forwardedNames: Record = {}; - for (const [name, service] of Object.entries(opts.services)) { - // An `environment:` key with every entry commented out is YAML null. - if ( - !isRecord(service) || - service.environment === undefined || - service.environment === null - ) { - continue; - } - const resolution = resolveShellEnvironment({ - service: name, - environment: nativeServiceEnvironment(service.environment, name), - env: opts.env, - managed: new Set(Object.keys(opts.managedEnvironment[name] ?? {})), - }); - if (resolution === null) { - continue; - } - service.environment = resolution.environment; - shellEnvironment[name] = resolution.delivered; - forwardedNames[name] = resolution.forwardedNames; - if (SERVICE_NAME.test(name)) { - notices.push( - ...resolution.notices.map((line) => ({ service: name, line })) - ); - } - } - return { shellEnvironment, notices, forwardedNames }; -} - -/** - * Adds the values read from the caller's shell to the private delivery. Native + * Adds ephemeral file/shell values to the private delivery. Native * start and restart call this once, after the env fingerprints are taken, so a - * shell value is delivered to its service and never digested to disk. Commands + * value is delivered to its service and never digested to disk. Commands * that prepare input without calling it (exec, stale-env checks) deliver no * shell value. Calling it again changes nothing. */ @@ -440,8 +393,9 @@ export function nativeReviewedActiveServices( * Restrict private delivery to the reviewed, profile-active service set. * * Normalization declares managed keys on every authored service, including - * services outside the selected profiles, so public Compose and the reviewed - * plan stay identical across profile selections. The runtime refuses managed + * services outside the selected profiles. File declarations follow the selected + * profiles; the runtime's active set must not require any unprepared file inputs. + * The runtime refuses managed * values for a service it will not schedule, so an optional disabled service * must not receive them and block the selected graph. */ @@ -450,6 +404,13 @@ export function selectActiveNativeProjectInput(opts: { readonly reviewedPlan: unknown; }): NativeProjectInput { const active = nativeReviewedActiveServices(opts.reviewedPlan); + for (const name of active) { + if (opts.input.environmentFileServices?.[name] === false) { + throw refused( + `${serviceLabel(name)} env_file activation differs from the reviewed runtime; retry with a matching CLI/runtime` + ); + } + } return { ...opts.input, managedEnvironment: Object.fromEntries( @@ -479,11 +440,16 @@ export async function prepareNativeProjectInput(opts: { readonly projectDir: string; readonly composeFile: string; readonly envName?: string | null; + readonly profiles?: readonly string[]; + /** Fresh exec restricts file acquisition to the selected running service. */ + readonly selectedServices?: readonly string[]; + /** Advisory managed fingerprints do not acquire ephemeral file/shell inputs. */ + readonly managedOnly?: boolean; /** * The caller's environment, read only for the variables that service * `environment` entries reference. Omitted, nothing is read and a shell - * reference resolves as unset: only native start and restart pass the - * process environment. + * reference resolves as unset. Start, preflight and fresh exec pass the + * same allowed caller context. */ readonly env?: Readonly>; }): Promise { @@ -501,16 +467,13 @@ export async function prepareNativeProjectInput(opts: { throw refused("the Compose file has no services mapping"); } const serviceNames = Object.keys(compose.services); - // Refuse before resolving/decrypting any values. Never read env_file contents. + const services: Record = + Object.create(null); for (const [name, service] of Object.entries(compose.services)) { if (!isRecord(service)) { throw refused(`${serviceLabel(name)} must be a mapping`); } - if (Object.hasOwn(service, "env_file")) { - throw refused( - `${serviceLabel(name)} field env_file is not supported by native up; move its variables into Hack managed env with hack env set --service` - ); - } + services[name] = { document: service, environment: {} }; } const resolved = await resolveProjectEnvConfig({ ...opts, @@ -518,24 +481,41 @@ export async function prepareNativeProjectInput(opts: { }).catch((error: unknown) => { throw environmentRefused(error); }); - if ( - !resolved && - ((await exists(join(opts.projectDir, ".env"))) || - (await exists(join(opts.projectRoot, ".env")))) - ) { - throw refused( - "a project .env file is present without Hack managed env; native up does not read .env files. Migrate it with hack doctor --migrate-env-config or remove it" - ); - } const managedEnvironment = normalizeManagedEnvironment( compose.services, resolved ); - const shell = resolveServiceShellEnvironment({ - services: compose.services, - managedEnvironment, - env: opts.env, - }); + for (const [name, service] of Object.entries(services)) { + services[name] = { + document: service.document, + environment: nativeServiceEnvironment( + service.document.environment, + name + ), + }; + } + const shell = opts.managedOnly + ? { + shellEnvironment: {}, + notices: [], + forwardedNames: {}, + privateValues: [], + fileServices: {}, + } + : await normalizeNativeComposeEnvironment({ + ...opts, + compose, + services, + managed: managedEnvironment, + global: resolved + ? selectProjectEnvValuesForExecutionTarget({ + resolved, + scopeName: "global", + target: "compose", + }) + : {}, + env: opts.env, + }); const normalizedComposeJson = JSON.stringify(compose); if (Buffer.byteLength(normalizedComposeJson) > LIMIT) { throw refused("the normalized Compose project exceeds 256 KiB"); @@ -553,7 +533,10 @@ export async function prepareNativeProjectInput(opts: { originalSha256: createHash("sha256").update(bytes).digest("hex"), normalizedComposeJson, managedEnvironment, - managedSecretValues: managedSecretValues(resolved), + managedSecretValues: [ + ...managedSecretValues(resolved), + ...shell.privateValues, + ], lifecycleHostEnvironment: resolved ? selectProjectEnvValuesForExecutionTarget({ resolved, @@ -564,12 +547,16 @@ export async function prepareNativeProjectInput(opts: { effectiveEnvName: selection.effectiveEnv, environmentFiles: resolved?.files ?? [], serviceNames, + environmentFileServices: shell.fileServices, shellEnvironment: shell.shellEnvironment, forwardedShellVariables: shell.forwardedNames, environmentNotices: shell.notices, }; } catch (error: unknown) { // Only this module's closed refusals carry a cause; anything else stays generic. - throw error instanceof NativeProjectInputRefusal ? error : refused(); + throw error instanceof NativeProjectInputRefusal || + error instanceof NativeEnvFileRefusal + ? error + : refused(); } } diff --git a/src/backends/native-project-restart-preflight.ts b/src/backends/native-project-restart-preflight.ts index 3d941ef16..07e371fb4 100644 --- a/src/backends/native-project-restart-preflight.ts +++ b/src/backends/native-project-restart-preflight.ts @@ -451,6 +451,7 @@ export async function preflightNativeRestart(opts: { ...opts.scope, composeFile: opts.composeFile, envName: selected.envName, + profiles: selected.profiles, // As native start: `environment` references read the caller's shell. env: process.env, }) diff --git a/src/backends/native-project-review.ts b/src/backends/native-project-review.ts index 732d29476..61286b0de 100644 --- a/src/backends/native-project-review.ts +++ b/src/backends/native-project-review.ts @@ -3,6 +3,7 @@ import { tmpdir } from "node:os"; import { join, relative } from "node:path"; import { ClosedCliError, HackCliError } from "../lib/cli-result.ts"; import { isRecord } from "../lib/guards.ts"; +import { nativeEnvironmentUnsetsReviewed } from "./native-environment-unsets.ts"; import { nativeProjectBranchArgs, prepareNativeProjectBranch, @@ -339,6 +340,16 @@ export async function withNativeProjectReview(opts: { "Native project returned an invalid reviewed identity." ); } + if ( + !nativeEnvironmentUnsetsReviewed({ + compose: JSON.parse(opts.input.normalizedComposeJson), + plan: report.plan, + }) + ) { + throw new ClosedCliError( + "The selected native runtime cannot confirm explicit environment removals. Update it and retry; no VM was started." + ); + } return await opts.run({ planId: report.plan_id, namespace, diff --git a/src/backends/native-project-start-preflight.ts b/src/backends/native-project-start-preflight.ts index d18dd8d38..248ea3c62 100644 --- a/src/backends/native-project-start-preflight.ts +++ b/src/backends/native-project-start-preflight.ts @@ -1,6 +1,10 @@ import { ClosedCliError } from "../lib/cli-result.ts"; import { isRecord } from "../lib/guards.ts"; -import type { NativeProjectInput } from "./native-project-input.ts"; +import { + type NativeProjectInput, + requireNativeShellDeliveryFits, + selectActiveNativeProjectInput, +} from "./native-project-input.ts"; import { withNativeProjectReview } from "./native-project-review.ts"; import { invokeNativeRuntime, @@ -52,6 +56,12 @@ export async function requireNativeStaticStartReview(opts: { if (!(isRecord(plan) && isRecord(plan.services))) { throw new ClosedCliError("Native static start review is invalid."); } + requireNativeShellDeliveryFits( + selectActiveNativeProjectInput({ + input: opts.input, + reviewedPlan: plan, + }) + ); const services = Object.keys(plan.services); const refusals = readNativeStartRefusals(report.start_refusals, services); if (refusals === null) { diff --git a/src/backends/native-project-start.ts b/src/backends/native-project-start.ts index fb8f6998d..7c8e47a03 100644 --- a/src/backends/native-project-start.ts +++ b/src/backends/native-project-start.ts @@ -1927,26 +1927,23 @@ async function startNativeProjectAttempt( }); requireActiveStartup(opts.signal); } - // A down hook of an earlier generation with unknown completion blocks this - // one before any hook runs; any other leftover record is superseded. - await deps.admitDownHooks(opts.scope); const profiles = selection.profiles; progress({ phase: "input" }); - await preflightNativeProjectSource({ - runtime: opts.runtime, - projectRoot: opts.scope.projectRoot, - signal: opts.signal, - invoke: deps.invoke, - }); - await deps.prepareStorage(opts.scope); + // Keep ps aware of this frontend while input is read. This advisory marker + // carries no values and is retired even if admission refuses the input. await observeStartup(); + // Read and admit private project inputs before any pool-share or storage + // mutation. An unsafe env file or public interpolation must have no effects. let input = await deps.prepare({ ...opts.scope, composeFile: opts.composeFile, envName: selection.envName, - // Service `environment` references are resolved from the caller's shell. + profiles, env: process.env, }); + // A down hook of an earlier generation with unknown completion blocks this + // one before any hook runs; any other leftover record is superseded. + await deps.admitDownHooks(opts.scope); // Digest the selection now: later steps adapt the delivered values. const selectedEnvironment = opts.onEnvironmentReady ? withComposeServiceEnvFingerprints({ @@ -2020,6 +2017,13 @@ async function startNativeProjectAttempt( invoke: deps.invoke, signal: opts.signal, }); + await preflightNativeProjectSource({ + runtime: opts.runtime, + projectRoot: opts.scope.projectRoot, + signal: opts.signal, + invoke: deps.invoke, + }); + await deps.prepareStorage(opts.scope); // A graph that captures source (bind mounts or a dependency cache) measures it // before hooks, image work or runtime start, so an oversized project refuses with // its size and largest paths. An image-only graph never reads source content. diff --git a/src/backends/native-shell-environment.ts b/src/backends/native-shell-environment.ts index 7c09985d6..d1d7374f4 100644 --- a/src/backends/native-shell-environment.ts +++ b/src/backends/native-shell-environment.ts @@ -1,50 +1,9 @@ import { HACK_KEY_MATERIAL_VARIABLES } from "../lib/child-environment.ts"; import { COMPOSE_PROJECT_NAME_VARIABLE } from "./native-compose-project-name.ts"; -/** - * Shell references in a service's `environment`, resolved the way Compose - * resolves them, for the native backend. - * - * Compose reads a variable reference and a bare `NAME` entry from the shell - * that runs it. The native runtime deliberately consults no ambient - * environment, so such an entry used to fail the start. This module resolves - * the two forms that could never start, in the CLI, and hands each value to - * the private delivery that Hack managed env already uses. - * - * Invariants: - * - A value read from the shell is never written to the normalized Compose - * document, a plan, a receipt, an env fingerprint or a log. The entry - * becomes a bare reference (`null`) and the value travels only in the - * private delivery. - * - The reviewed document does not depend on the shell: a resolved entry is - * always a bare reference, whether or not the variable is set. So a start - * from another shell reviews the same plan. - * - Only variables an entry references are read, and only own string - * properties of the environment. - * - Nothing that starts today changes. A reference with a default, an - * alternative or an error operator is left to the runtime, as before. - * - A key Hack manages for the service is never taken from the shell, and a - * reference to a managed variable is left for the runtime, which refuses it. - * - A name the runtime supplies itself, Hack's own `HACK_*` variables and the - * cloud credential names the scoped adapters own are never read from the - * shell. - * - * Scope: an entry whose whole value is one plain reference, or a bare name. - * A reference inside a longer string, or in any field other than - * `environment`, is not resolved here. - */ - -/** A valid environment variable name. */ -const NAME = /^[A-Za-z_][A-Za-z0-9_]*$/; -/** The whole value is one braced reference with no operator. */ -const BRACED = /^\$\{([A-Za-z_][A-Za-z0-9_]*)\}$/; -/** The whole value is one unbraced reference. */ -const UNBRACED = /^\$([A-Za-z_][A-Za-z0-9_]*)$/; - -/** - * Names the native runtime binds itself during review. Taking one from the - * shell would give a service another instance's value and remove the - * reference the runtime looks for. +/** Shell source-name policy and private delivery limits. Resolution, including + * dotenv and unset semantics, is owned by native-compose-environment.ts. + * Values never enter this module except to measure a private delivery. */ const RUNTIME_SUPPLIED_NAMES: ReadonlySet = new Set([ COMPOSE_PROJECT_NAME_VARIABLE, @@ -79,110 +38,6 @@ export function shellVariableReadable(name: string): boolean { ); } -/** The shell's own string value for `name`, never an inherited property. */ -function shellValue( - env: Readonly>, - name: string -): string | undefined { - if (!Object.hasOwn(env, name)) { - return; - } - const value = env[name]; - return typeof value === "string" ? value : undefined; -} - -/** The variable an entry reads from the shell, or nothing when it reads none. */ -export function shellReference(opts: { - readonly key: string; - readonly value: unknown; - /** Names Hack delivers for this service. */ - readonly managed: ReadonlySet; -}): string | undefined { - if (opts.managed.has(opts.key) || !NAME.test(opts.key)) { - return; - } - let name: string | undefined; - if (opts.value === null) { - // A bare `NAME`: pass the shell's value through. - name = opts.key; - } else if (typeof opts.value === "string") { - name = BRACED.exec(opts.value)?.[1] ?? UNBRACED.exec(opts.value)?.[1]; - } - if (name === undefined || opts.managed.has(name)) { - return; - } - // The entry's own key is checked too: it is the name that gets delivered. - return shellVariableReadable(name) && shellVariableReadable(opts.key) - ? name - : undefined; -} - -export type ShellEnvironmentResolution = { - /** The service's environment as a mapping, with resolved entries set to `null`. */ - readonly environment: Record; - /** Values to deliver privately, by variable name. Never log this map. */ - readonly delivered: Record; - /** Value-free notes for the user: names only. */ - readonly notices: string[]; - /** Source names actually present in the caller's environment; never values. */ - readonly forwardedNames: readonly string[]; -}; - -/** - * Resolves every shell reference of one service. Returns `null` when the - * service has none, so the caller leaves its `environment` untouched. - * - * A referenced variable the shell does not set is delivered as an empty - * value, with a notice that names it. Compose substitutes an empty string for - * an unset reference; for a bare name Compose leaves the variable unset in the - * container, and native delivers it empty instead so the reviewed document - * stays the same from one shell to the next. - */ -export function resolveShellEnvironment(opts: { - readonly service: string; - /** The service's environment as a mapping (`NAME` → value or `null`). */ - readonly environment: Readonly>; - readonly env?: Readonly>; - readonly managed: ReadonlySet; -}): ShellEnvironmentResolution | null { - // Null-prototype maps: `__proto__` is a valid variable name. - const environment: Record = Object.create(null); - const delivered: Record = Object.create(null); - const notices: string[] = []; - const forwardedNames = new Set(); - let changed = false; - for (const [key, value] of Object.entries(opts.environment)) { - const name = shellReference({ key, value, managed: opts.managed }); - if (name === undefined) { - environment[key] = value; - continue; - } - changed = true; - const found = - opts.env === undefined ? undefined : shellValue(opts.env, name); - environment[key] = null; - delivered[key] = found ?? ""; - if (found !== undefined) { - forwardedNames.add(name); - } - if (found === undefined) { - notices.push( - name === key - ? `Service ${opts.service}: ${key} is empty because it is not set in your shell.` - : `Service ${opts.service}: ${key} is empty because ${name} is not set in your shell.` - ); - } - } - return changed - ? { - environment, - delivered, - notices, - forwardedNames: [...forwardedNames].sort(), - } - : null; -} - /** The runtime's limits on one service's private delivery (`provider/environment.rs`). */ const MAX_SERVICE_PAYLOAD_BYTES = 32 * 1024; const MAX_SERVICE_KEYS = 256; diff --git a/tests/native-compose-adoption-file-generation.test.ts b/tests/native-compose-adoption-file-generation.test.ts index 6b48c1756..5f5db3b98 100644 --- a/tests/native-compose-adoption-file-generation.test.ts +++ b/tests/native-compose-adoption-file-generation.test.ts @@ -828,4 +828,4 @@ test("HACK-1359: file8 rejects missing and non-finite mutation deadlines without } finally { await store.close(); } -}); +}, 30_000); diff --git a/tests/native-compose-backend-features-cli.test.ts b/tests/native-compose-backend-features-cli.test.ts index c83ee430c..541f30635 100644 --- a/tests/native-compose-backend-features-cli.test.ts +++ b/tests/native-compose-backend-features-cli.test.ts @@ -37,9 +37,9 @@ const STACK_FRAME = /^\s+at /m; const SPAWN_TIMEOUT_MS = 60_000; const IMAGE = `sha256:${"e".repeat(64)}`; const ADMITTED_START_REQUESTS = [ - "runtime check-project-share", "project plan", "project plan", + "runtime check-project-share", "runtime up", ]; const SELECTION = @@ -416,8 +416,8 @@ test( : refusal; expectOneClosedLine(result, `E_NATIVE_PROJECT_UNSUPPORTED: ${expected}`); expect(result.output).not.toContain("HACK_RUNTIME_BACKEND=compose"); - // The check passed: native up went on to its own source preflight. - expect(await f.calls()).toEqual(["runtime check-project-share"]); + // Static input admission refuses before the provider share check. + expect(await f.calls()).toEqual([]); } }, SPAWN_TIMEOUT_MS @@ -474,12 +474,12 @@ test( const result = await cli(f, ["up"], BUILDS_ENV); expectOneClosedLine( result, - "E_NATIVE_PROJECT_UNSUPPORTED: Native foreground up cannot build service web: build arg GREETING equals a hack env secret value. Build args are not secrets (images record them); pass secrets with BuildKit build secrets instead, which native builds do not support yet. Values were omitted; no hook, VM or build was started." + "E_NATIVE_PROJECT_UNSUPPORTED: Native foreground up cannot build service web: build arg GREETING equals a privately delivered environment value. Build args are not secrets (images record them); pass secrets with BuildKit build secrets instead, which native builds do not support yet. Values were omitted; no hook, VM or build was started." ); expect(result.output).not.toContain(secret); // The Compose backend would record the value in the image history too. expect(result.output).not.toContain("HACK_RUNTIME_BACKEND=compose"); - expect(await f.calls()).toEqual(["runtime check-project-share"]); + expect(await f.calls()).toEqual([]); }, SPAWN_TIMEOUT_MS ); diff --git a/tests/native-compose-env-syntax.test.ts b/tests/native-compose-env-syntax.test.ts new file mode 100644 index 000000000..e4d9fe45e --- /dev/null +++ b/tests/native-compose-env-syntax.test.ts @@ -0,0 +1,167 @@ +import { expect, test } from "bun:test"; +import { + interpolateComposeEnvironment, + parseComposeEnvironmentFile, +} from "../src/backends/native-compose-env-syntax.ts"; + +const SENTINEL = "synthetic-private-environment-canary"; +const variables: Readonly> = { + SET: SENTINEL, + EMPTY: "", + OTHER: "second", +}; +const lookup = (name: string) => + Object.hasOwn(variables, name) ? variables[name] : undefined; + +test.each([ + ["$SET/${SET}", `${SENTINEL}/${SENTINEL}`], + ["${SET:-fallback}", SENTINEL], + ["${EMPTY:-fallback}", "fallback"], + ["${EMPTY-fallback}", ""], + ["${MISSING-fallback}", "fallback"], + ["${SET:+yes}${EMPTY:+no}${EMPTY+set}", "yesset"], + ["${MISSING:-${EMPTY:-${OTHER}}}", "second"], + ["$$SET/$5/end$", "$SET/$5/end$"], + ["${EMPTY?not used}", ""], + ["${SET:?not used}", SENTINEL], +])("Compose interpolation %s", (text, expected) => { + expect(interpolateComposeEnvironment({ text, lookup })).toBe(expected); +}); + +test.each([ + `\u0000${SENTINEL}`, + `\${MISSING:?${SENTINEL}}`, + `\${EMPTY:?${SENTINEL}}`, + `\${MISSING?${SENTINEL}}`, + `\${SET/old/${SENTINEL}}`, + "${SET", + "${A:-${A:-${A:-${A:-${A:-${A}}}}}}", + "${SET:-${A:-${A:-${A:-${A}}}}}", +])("refusals omit authored error words and private values", (text) => { + try { + interpolateComposeEnvironment({ text, lookup }); + throw new Error("unexpected success"); + } catch (error) { + expect(String(error)).toContain("values omitted"); + expect(String(error)).not.toContain(SENTINEL); + } +}); + +test("four nested expressions are admitted at the declared depth bound", () => { + expect( + interpolateComposeEnvironment({ + text: "${A:-${A:-${A:-${OTHER}}}}", + lookup, + }) + ).toBe("second"); +}); + +test("Compose quote delimiters and octal escapes follow the pinned parser", () => { + const result = parseComposeEnvironmentFile({ + text: String.raw`OCTAL="\0123/\x41/\u0041" +SINGLE='a\\' +DOUBLE="a\\" +`, + lookup, + }); + expect(Object.fromEntries(result)).toEqual({ + OCTAL: "S/\\x41/\\u0041", + SINGLE: "a\\\\", + DOUBLE: "a\\", + }); +}); + +test("Compose dotenv quoting, comments, multiline values and sequential lookups", () => { + const result = parseComposeEnvironmentFile({ + text: [ + "# comment", + "export FIRST=one # comment", + "SECOND=${FIRST}", + "FIRST=two", + "THIRD=${FIRST}", + "EXTERNAL=${SET}", + "COLON: value", + "HASH=a#literal", + "QUOTE='${SET} # literal'", + 'MULTI="line1', + 'line2\\tvalue" # comment', + 'ESCAPED="\\$SET/$$SET"', + 'LITERAL="{\\"key\\": \\"value\\"}"', + "BARE", + "EMPTY=", + "__proto__=own", + "", + ].join("\n"), + lookup, + }); + expect(Object.fromEntries(result)).toEqual({ + FIRST: "two", + SECOND: "one", + THIRD: "two", + EXTERNAL: SENTINEL, + COLON: "value", + HASH: "a#literal", + QUOTE: "${SET} # literal", + MULTI: "line1\nline2\tvalue", + ESCAPED: "$SET/$SET", + LITERAL: '{"key": "value"}', + EMPTY: "", + ["__proto__"]: "own", + }); + expect(result.has("BARE")).toBe(false); + expect(result.get("__proto__")).toBe("own"); +}); + +test("lookup takes precedence, current file overrides earlier files, unresolved bare does not erase", () => { + const result = parseComposeEnvironmentFile({ + text: "FIRST=new\nCOPY=${FIRST}\nKEPT\nSET\nFROM_PROJECT=${SET}\n", + initial: new Map([ + ["FIRST", "earlier"], + ["KEPT", "earlier"], + ]), + lookup, + }); + expect(Object.fromEntries(result)).toEqual({ + FIRST: "new", + COPY: "new", + KEPT: "earlier", + SET: SENTINEL, + FROM_PROJECT: SENTINEL, + }); +}); + +test("raw leaves dollar references, quote characters and comments literal", () => { + expect( + Object.fromEntries( + parseComposeEnvironmentFile({ + text: 'A="${SET}" # text\nB= $SET \n', + raw: true, + lookup, + }) + ) + ).toEqual({ A: '"${SET}" # text', B: " $SET " }); +}); + +test("pinned Compose removes the escaped single-quote delimiter backslash", () => { + expect( + parseComposeEnvironmentFile({ text: "A='it\\'s'\n", lookup }).get("A") + ).toBe("it's"); +}); + +test("raw does not accept a colon separator before an equals in the value", () => { + expect(() => + parseComposeEnvironmentFile({ text: "KEY: a=b\n", raw: true, lookup }) + ).toThrow("values omitted"); +}); + +test.each([ + `BAD KEY=${SENTINEL}`, + `BROKEN='${SENTINEL}`, + `A="${SENTINEL}"garbage`, + "\n".repeat(4097), + `A=${"a".repeat(256 * 1024)}`, +])("malformed or over-budget dotenv never echoes contents", (text) => { + expect(() => parseComposeEnvironmentFile({ text, lookup })).toThrow( + "values omitted" + ); +}); diff --git a/tests/native-compose-environment.test.ts b/tests/native-compose-environment.test.ts new file mode 100644 index 000000000..2412d9da3 --- /dev/null +++ b/tests/native-compose-environment.test.ts @@ -0,0 +1,511 @@ +import { afterEach, expect, test } from "bun:test"; +import { + chmod, + link, + mkdir, + mkdtemp, + readFile, + realpath, + rm, + symlink, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { adaptNativeAwsEnvironment } from "../src/backends/native-aws-environment.ts"; +import { adaptNativeGcloudEnvironment } from "../src/backends/native-gcloud-environment.ts"; +import { nativePrivateEnvironmentPayload } from "../src/backends/native-private-environment-payload.ts"; +import { + mergeNativeShellEnvironment, + prepareNativeProjectInput, + requireNativeShellDeliveryFits, + selectActiveNativeProjectInput, +} from "../src/backends/native-project-input.ts"; + +const FILE = "synthetic-file-value-canary"; +const DOTENV = "synthetic-dotenv-value-canary"; +const SHELL = "synthetic-shell-value-canary"; +const MANAGED = "synthetic-managed-value-canary"; +const roots: string[] = []; +afterEach(async () => { + for (const root of roots.splice(0)) { + await rm(root, { recursive: true, force: true }); + } +}); +async function fixture( + service: Record, + extra: Record = {} +) { + const projectRoot = await realpath( + await mkdtemp(join(tmpdir(), "native-compose-env-")) + ); + roots.push(projectRoot); + const projectDir = join(projectRoot, ".hack"); + await mkdir(projectDir, { mode: 0o700 }); + const composeFile = join(projectDir, "docker-compose.yml"); + await writeFile( + composeFile, + JSON.stringify({ + services: { app: { image: "alpine:3.21", ...service } }, + ...extra, + }) + ); + const source = async (path: string, text: string) => + await writeFile(join(projectDir, path), text, { mode: 0o600 }); + return { projectRoot, projectDir, composeFile, source }; +} +async function managed(projectDir: string, values: unknown) { + await writeFile( + join(projectDir, "hack.env.default.yaml"), + JSON.stringify({ + version: 1, + environment: "default", + secretsprovider: "project_key", + values, + }) + ); +} +function publicService( + input: Awaited> +) { + return JSON.parse(input.normalizedComposeJson).services.app; +} + +test("service environment beats ordered files; interpolation is private and dotenv does not inject", async () => { + const f = await fixture({ + env_file: "service.env", + environment: { + OVERRIDE: "explicit", + EMPTY: "", + PASSED: null, + ABSENT: null, + INTERPOLATED: "prefix-${SOURCE}", + }, + }); + await f.source(".env", `SOURCE=${DOTENV}\nNOT_INJECTED=${DOTENV}\n`); + await f.source( + "service.env", + `FILE=${FILE}\nOVERRIDE=${FILE}\nEMPTY=${FILE}\nABSENT=${FILE}\nCOPY=\${SOURCE}\n` + ); + const original = await readFile(f.composeFile); + const input = await prepareNativeProjectInput({ + ...f, + env: { SOURCE: SHELL, PASSED: SHELL }, + }); + expect(input.managedEnvironment).toEqual({}); + expect(input.shellEnvironment).toEqual({ + app: { FILE, COPY: SHELL, PASSED: SHELL, INTERPOLATED: `prefix-${SHELL}` }, + }); + expect(publicService(input)).toEqual({ + image: "alpine:3.21", + environment: { + FILE: null, + COPY: null, + OVERRIDE: "explicit", + EMPTY: "", + PASSED: null, + INTERPOLATED: null, + }, + "x-hack-unset-environment": ["ABSENT"], + }); + expect(publicService(input).environment).not.toHaveProperty("NOT_INJECTED"); + expect(await readFile(f.composeFile)).toEqual(original); + for (const canary of [FILE, DOTENV, SHELL]) { + expect(input.normalizedComposeJson).not.toContain(canary); + expect(input.originalSha256).not.toContain(canary); + expect(JSON.stringify(input.environmentNotices)).not.toContain(canary); + } + const delivered = mergeNativeShellEnvironment(input); + expect(() => requireNativeShellDeliveryFits(delivered)).not.toThrow(); + const wire = nativePrivateEnvironmentPayload({ + services: delivered.managedEnvironment, + planId: "a".repeat(64), + run: "b".repeat(32), + }); + expect(JSON.parse(wire.toString()).services.app.FILE).toBe(FILE); +}); + +test("short/list/long optional/raw and duplicate entries preserve Compose order", async () => { + const f = await fixture({ + env_file: [ + { path: "one.env", format: "" }, + { path: "missing.env", required: false }, + { path: "two.env", format: "raw" }, + "one.env", + ], + }); + await f.source("one.env", `VALUE=${FILE}\nBARE\n`); + await f.source( + "two.env", + 'VALUE="${SOURCE}" # literal\nRAW="${SOURCE}" # literal\n' + ); + const input = await prepareNativeProjectInput(f); + expect(input.shellEnvironment).toEqual({ + app: { VALUE: FILE, RAW: '"${SOURCE}" # literal' }, + }); + expect(publicService(input)).not.toHaveProperty("env_file"); +}); + +test("Compose rejects the literal default format name", async () => { + const f = await fixture({ + env_file: { path: "service.env", format: "default" }, + }); + await f.source("service.env", `TOKEN=${FILE}\n`); + await expect(prepareNativeProjectInput(f)).rejects.toThrow("format"); +}); + +test("explicit cloud values may seed env-file expansion without reading cloud shell variables", async () => { + const f = await fixture({ + env_file: "service.env", + environment: { AWS_REGION: "explicit-region" }, + }); + await f.source("service.env", "REGION=${AWS_REGION}\n"); + const input = await prepareNativeProjectInput({ + ...f, + env: { AWS_REGION: SHELL }, + }); + expect(input.shellEnvironment).toEqual({ + app: { REGION: "explicit-region" }, + }); + expect(input.normalizedComposeJson).not.toContain(SHELL); +}); + +test("NUL private inputs refuse with closed diagnostics", async () => { + const f = await fixture({ env_file: "service.env" }); + await f.source("service.env", `TOKEN=${FILE}\0\n`); + const error: unknown = await prepareNativeProjectInput(f).catch( + (caught: unknown) => caught + ); + expect(String(error)).toContain("values omitted"); + expect(String(error)).not.toContain(FILE); +}); + +test("documented Compose precedence includes managed globals, service values, files and interpolation sources", async () => { + // Compose v5.6.1 / compose-go v2.16.1: project lookup > service environment + // > current/previous env-file assignments; service environment wins finally. + const f = await fixture({ + env_file: ["one.env", "two.env"], + environment: { + K: "service", + BARE: null, + EMPTY: "", + FROM_CONTEXT: "${CONTEXT}", + }, + }); + await f.source(".env", `CONTEXT=${DOTENV}\n`); + await f.source( + "one.env", + `K=${FILE}\nEARLIER=${FILE}\nBARE=${FILE}\nEMPTY=${FILE}\n` + ); + await f.source("two.env", "R=${K}\nS=${EARLIER}\nEARLIER\nT=${CONTEXT}\n"); + await managed(f.projectDir, { + global: { CONTEXT: MANAGED }, + app: { K: MANAGED }, + }); + const input = await prepareNativeProjectInput({ + ...f, + env: { CONTEXT: SHELL }, + }); + const final = mergeNativeShellEnvironment(input); + expect(final.managedEnvironment.app).toEqual({ + CONTEXT: MANAGED, + K: MANAGED, + FROM_CONTEXT: MANAGED, + EARLIER: FILE, + R: MANAGED, + S: FILE, + T: MANAGED, + }); + expect(publicService(input).environment.EMPTY).toBe(""); + expect(publicService(input)["x-hack-unset-environment"]).toEqual(["BARE"]); + expect(final.forwardedShellVariables?.app).not.toContain("CONTEXT"); +}); + +test("file value-only edits do not enter normalized identity inputs or managed fingerprints", async () => { + const f = await fixture({ + env_file: "service.env", + environment: { COPY: "${SOURCE}" }, + }); + const prepare = async (value: string) => { + await f.source(".env", `SOURCE=${value}\n`); + await f.source("service.env", `TOKEN=${value}\n`); + return await prepareNativeProjectInput(f); + }; + const a = await prepare(FILE); + const b = await prepare(DOTENV); + expect(a.normalizedComposeJson).toBe(b.normalizedComposeJson); + expect(a.originalSha256).toBe(b.originalSha256); + expect(a.managedEnvironment).toEqual(b.managedEnvironment); + expect(a.shellEnvironment).not.toEqual(b.shellEnvironment); +}); + +test("unreferenced dotenv does not create private delivery or rewrite pure defaults", async () => { + const f = await fixture({ + restart: "unless-stopped", + environment: { MODE: "${NOT_SET:-safe}", LITERAL: "public" }, + }); + await f.source(".env", `UNUSED=${DOTENV}\n`); + const input = await prepareNativeProjectInput(f); + expect(input.shellEnvironment).toEqual({}); + expect(publicService(input).environment).toEqual({ + MODE: "${NOT_SET:-safe}", + LITERAL: "public", + }); +}); + +test.each([ + { image: "example:${SOURCE:-fallback}" }, + { command: ["echo", "${SOURCE:-fallback}"] }, + { entrypoint: ["${SOURCE}"] }, + { ports: ["${SOURCE:-3000}:3000"] }, + { labels: { note: "${SOURCE}" } }, + { labels: { "x-note": "${SOURCE}" } }, + { user: "${SOURCE:-1000}" }, + { healthcheck: { test: ["CMD", "${SOURCE}"] } }, + { build: { args: { SOURCE: "${SOURCE}" } } }, +])("public-field interpolation refuses before values can leave private preparation", async (service) => { + const f = await fixture(service); + await f.source(".env", `SOURCE=${DOTENV}\n`); + const error: unknown = await prepareNativeProjectInput(f).catch( + (caught: unknown) => caught + ); + expect(error).toBeInstanceOf(Error); + expect(String(error)).toContain("SOURCE"); + expect(String(error)).not.toContain(DOTENV); +}); + +test("missing required references never echo the authored error message", async () => { + const f = await fixture({ env_file: "service.env" }); + await f.source("service.env", `TOKEN=\${MISSING:?${FILE}}\n`); + const error: unknown = await prepareNativeProjectInput(f).catch( + (caught: unknown) => caught + ); + expect(error).toBeInstanceOf(Error); + expect(String(error)).toContain("service.env"); + expect(String(error)).not.toContain(FILE); +}); + +test("possible materialized dotenv refuses; root dotenv is not an interpolation source", async () => { + const f = await fixture({ environment: { ROOT_ONLY: "${ROOT_ONLY}" } }); + await writeFile(join(f.projectRoot, ".env"), `ROOT_ONLY=${DOTENV}\n`, { + mode: 0o600, + }); + const input = await prepareNativeProjectInput(f); + expect(input.shellEnvironment?.app?.ROOT_ONLY).toBe(""); + await f.source(".env", `ROOT_ONLY=${DOTENV}\n`); + await f.source(".env.state.json", "not trusted even if readable"); + await expect(prepareNativeProjectInput(f)).rejects.toThrow("materialized"); +}); + +test.each([ + "COMPOSE_FILE", + "COMPOSE_PROFILES", + "COMPOSE_PROJECT_NAME", + "COMPOSE_ENV_FILES", + "HACK_ENV_SECRET_KEY", +])("dotenv cannot change project controls or forward protected %s", async (key) => { + const f = await fixture({}); + await f.source(".env", `${key}=${DOTENV}\n`); + const error: unknown = await prepareNativeProjectInput(f).catch( + (caught: unknown) => caught + ); + expect(error).toBeInstanceOf(Error); + expect(String(error)).toContain(key); + expect(String(error)).not.toContain(DOTENV); +}); + +test("file delivery is counted by the private service budget", async () => { + const f = await fixture({ env_file: "service.env" }); + await f.source("service.env", `LARGE=${"x".repeat(33 * 1024)}\n`); + const input = mergeNativeShellEnvironment(await prepareNativeProjectInput(f)); + expect(() => requireNativeShellDeliveryFits(input)).toThrow("32 KiB"); + await chmod(join(f.projectDir, "service.env"), 0o666); + await expect(prepareNativeProjectInput(f)).rejects.toThrow( + "not writable by others" + ); +}); + +test("unused extension nodes do not block aliased private environment", async () => { + const f = await fixture( + { environment: { TOKEN: "${SOURCE}" }, "x-env": { TOKEN: "${SOURCE}" } }, + { "x-env": { TOKEN: "${SOURCE}" } } + ); + await f.source(".env", `SOURCE=${DOTENV}\n`); + expect((await prepareNativeProjectInput(f)).shellEnvironment).toEqual({ + app: { TOKEN: DOTENV }, + }); +}); + +test("a project-root alias is canonicalized without permitting source symlinks", async () => { + const f = await fixture({ env_file: "service.env" }); + await f.source("service.env", `TOKEN=${FILE}\n`); + const alias = join(f.projectRoot, "root-alias"); + await symlink(f.projectRoot, alias); + const input = await prepareNativeProjectInput({ + projectRoot: alias, + projectDir: join(alias, ".hack"), + composeFile: join(alias, ".hack/docker-compose.yml"), + }); + expect(input.shellEnvironment).toEqual({ app: { TOKEN: FILE } }); +}); + +test.each([ + "AWS_REGION", + "GOOGLE_APPLICATION_CREDENTIALS", + "CLOUDSDK_CORE_PROJECT", + "COMPOSE_PROJECT_NAME", +])("unreadable bare %s remains runtime-owned rather than unset", async (key) => { + const f = await fixture({ environment: { [key]: null } }); + const input = await prepareNativeProjectInput({ + ...f, + env: { [key]: SHELL }, + }); + expect(publicService(input).environment).toEqual({ [key]: null }); + expect(publicService(input)).not.toHaveProperty("x-hack-unset-environment"); + expect(input.shellEnvironment).toEqual({}); +}); + +test("only delivered private values join the build disclosure guard", async () => { + const f = await fixture({ + env_file: "service.env", + environment: { MODE: "explicit" }, + }); + await f.source(".env", "UNUSED=production\n"); + await f.source("service.env", `MODE=production\nTOKEN=${FILE}\n`); + const input = await prepareNativeProjectInput(f); + expect(input.managedSecretValues).toEqual([FILE]); +}); + +test("authored public runtime metadata literals survive while file/shell copies stay refused", async () => { + const f = await fixture({ + environment: { HACK_SERVICE_URL: "author-owned" }, + }); + const input = await prepareNativeProjectInput(f); + expect(publicService(input).environment.HACK_SERVICE_URL).toBe( + "author-owned" + ); + expect(input.shellEnvironment).toEqual({}); + await writeFile( + f.composeFile, + JSON.stringify({ + services: { + app: { + image: "alpine", + environment: { HACK_SERVICE_URL: "${SOURCE}" }, + }, + }, + }) + ); + await expect( + prepareNativeProjectInput({ ...f, env: { SOURCE: SHELL } }) + ).rejects.toThrow("reserved destination"); + await writeFile( + f.composeFile, + JSON.stringify({ + services: { + app: { image: "alpine", environment: { HACK_ENV_SECRET_KEY: FILE } }, + }, + }) + ); + const error: unknown = await prepareNativeProjectInput(f).catch( + (caught: unknown) => caught + ); + expect(String(error)).toContain("key material"); + expect(String(error)).not.toContain(FILE); +}); + +test("inactive service files are not read, but selected profiles require them", async () => { + const f = await fixture( + {}, + { + services: { + app: { image: "alpine" }, + worker: { + image: "alpine", + profiles: ["worker"], + env_file: "missing.env", + }, + }, + } + ); + const inactive = await prepareNativeProjectInput(f); + expect(inactive.shellEnvironment).toEqual({}); + expect(() => + selectActiveNativeProjectInput({ + input: inactive, + reviewedPlan: { + services: { app: { active: true }, worker: { active: true } }, + }, + }) + ).toThrow("env_file activation differs"); + expect(() => + selectActiveNativeProjectInput({ + input: inactive, + reviewedPlan: { + services: { app: { active: true }, worker: { active: false } }, + }, + }) + ).not.toThrow(); + await expect( + prepareNativeProjectInput({ ...f, profiles: ["worker"] }) + ).rejects.toThrow("required source is missing"); + await f.source("missing.env", `TOKEN=${FILE}\n`); + await chmod(join(f.projectDir, "missing.env"), 0o666); + expect((await prepareNativeProjectInput(f)).shellEnvironment).toEqual({}); +}); + +test("materialized dotenv identity aliases are refused before source reading", async () => { + const f = await fixture({ env_file: "alias.env" }); + await f.source(".env", `TOKEN=${DOTENV}\n`); + await f.source(".env.state.json", "{}\n"); + await link(join(f.projectDir, ".env"), join(f.projectDir, "alias.env")); + await expect(prepareNativeProjectInput(f)).rejects.toThrow("materialized"); +}); + +test("exact credential-adapter mounts remain eligible with HOME in the shell", async () => { + const aws = await fixture({ + volumes: ["${HOME}/.aws:/root/.aws:ro"], + environment: { AWS_REGION: null }, + }); + const awsInput = mergeNativeShellEnvironment( + await prepareNativeProjectInput({ ...aws, env: { HOME: SHELL } }) + ); + const adapted = await adaptNativeAwsEnvironment({ + input: awsInput, + profile: "fixture", + region: "us-east-1", + homeDirectory: aws.projectRoot, + exportSource: async () => ({ + credentials: { + Version: 1, + AccessKeyId: "synthetic-access", + SecretAccessKey: "synthetic-secret", + SessionToken: "synthetic-session", + Expiration: new Date(Date.now() + 3_600_000).toISOString(), + }, + region: "us-east-1", + }), + }); + expect( + JSON.parse(adapted.input.normalizedComposeJson).services.app + ).not.toHaveProperty("x-hack-unset-environment"); + const gcloud = await fixture({ + volumes: [ + "${HOME}/.config/gcloud/application_default_credentials.json:/tmp/adc.json:ro", + ], + environment: { GOOGLE_APPLICATION_CREDENTIALS: "/tmp/adc.json" }, + }); + const gcloudInput = await prepareNativeProjectInput({ + ...gcloud, + env: { HOME: SHELL }, + }); + const google = await adaptNativeGcloudEnvironment({ + input: gcloudInput, + mode: "file", + deliver: false, + homeDirectory: gcloud.projectRoot, + }); + expect( + JSON.parse(google.input.normalizedComposeJson).services.app.volumes ?? [] + ).toEqual([]); +}); diff --git a/tests/native-env-file-source.test.ts b/tests/native-env-file-source.test.ts new file mode 100644 index 000000000..149729992 --- /dev/null +++ b/tests/native-env-file-source.test.ts @@ -0,0 +1,151 @@ +import { afterEach, expect, test } from "bun:test"; +import { + chmod, + link, + mkdir, + mkdtemp, + readFile, + realpath, + rm, + symlink, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { readNativeEnvFileSource } from "../src/backends/native-env-file-source.ts"; + +const roots: string[] = []; +const SENTINEL = "synthetic-private-file-source-canary"; +afterEach(async () => { + for (const root of roots.splice(0)) { + await rm(root, { recursive: true, force: true }); + } +}); +async function fixture() { + const projectRoot = await realpath( + await mkdtemp(join(tmpdir(), "native-env-source-")) + ); + roots.push(projectRoot); + await mkdir(join(projectRoot, ".hack"), { mode: 0o700 }); + return { + projectRoot, + composeFile: join(projectRoot, ".hack/docker-compose.yml"), + path: "service.env", + required: true, + service: "app", + }; +} +test("reads exact text privately, including empty files, without changing modes or bytes", async () => { + const opts = await fixture(); + const path = join(opts.projectRoot, ".hack/service.env"); + for (const mode of [0o644, 0o600, 0o400]) { + await writeFile(path, ` KEY=${SENTINEL} \n`); + await chmod(path, mode); + const result = await readNativeEnvFileSource(opts); + expect(result?.text).toBe(` KEY=${SENTINEL} \n`); + expect(await readFile(path, "utf8")).toBe(result?.text ?? ""); + } + await chmod(path, 0o600); + await writeFile(path, ""); + expect((await readNativeEnvFileSource(opts))?.text).toBe(""); +}); +test("only absent optional sources are ignored, including a missing parent", async () => { + const opts = await fixture(); + expect( + await readNativeEnvFileSource({ ...opts, required: false }) + ).toBeNull(); + expect( + await readNativeEnvFileSource({ + ...opts, + path: "missing/optional.env", + required: false, + }) + ).toBeNull(); + await expect(readNativeEnvFileSource(opts)).rejects.toThrow( + "required source is missing" + ); +}); +test.each([ + "symlink", + "dangling", + "hardlink", + "directory", + "unreadable", + "writable", + "oversize", + "invalid-utf8", +])("optional %s source refuses without reading or echoing its contents", async (kind) => { + const opts = await fixture(); + const path = join(opts.projectRoot, ".hack/service.env"); + const target = join(opts.projectRoot, "target"); + await writeFile(target, SENTINEL, { mode: 0o600 }); + switch (kind) { + case "symlink": + await symlink(target, path); + break; + case "dangling": + await symlink(join(opts.projectRoot, "absent"), path); + break; + case "hardlink": + await link(target, path); + break; + case "directory": + await mkdir(path); + break; + default: { + const content = + kind === "oversize" + ? Buffer.alloc(256 * 1024 + 1) + : Buffer.from(SENTINEL); + await writeFile( + path, + kind === "invalid-utf8" ? new Uint8Array([255]) : content, + { mode: 0o600 } + ); + if (kind === "unreadable") { + await chmod(path, 0); + } + if (kind === "writable") { + await chmod(path, 0o666); + } + } + } + const error: unknown = await readNativeEnvFileSource({ + ...opts, + required: false, + }).catch((failure: unknown) => failure); + expect(error).toBeInstanceOf(Error); + expect(String(error)).toContain("values omitted"); + expect(String(error)).toContain("service.env"); + expect(String(error)).not.toContain(SENTINEL); + expect(await readFile(target, "utf8")).toBe(SENTINEL); +}); +test("rejects a symlinked parent and a writable directory, even for an absent optional source", async () => { + const opts = await fixture(); + await mkdir(join(opts.projectRoot, "actual")); + await symlink( + join(opts.projectRoot, "actual"), + join(opts.projectRoot, ".hack/alias") + ); + await expect( + readNativeEnvFileSource({ ...opts, path: "alias/file", required: false }) + ).rejects.toThrow("source directories must be real"); + await chmod(join(opts.projectRoot, ".hack"), 0o777); + try { + await expect( + readNativeEnvFileSource({ ...opts, required: false }) + ).rejects.toThrow("source directories must be real"); + } finally { + await chmod(join(opts.projectRoot, ".hack"), 0o700); + } +}); +test.each([ + "../../external", + "${SOURCE}", + "~/file", + "bad\0path", +])("refuses unsupported paths before reading", async (path) => { + await expect( + readNativeEnvFileSource({ ...(await fixture()), path }) + ).rejects.toThrow("refused"); +}); diff --git a/tests/native-exec-environment.test.ts b/tests/native-exec-environment.test.ts index 17859d765..8a16a7170 100644 --- a/tests/native-exec-environment.test.ts +++ b/tests/native-exec-environment.test.ts @@ -70,6 +70,34 @@ test("fresh exec reads selected service overlay and ignores a changed default", }) ).toEqual({ VALUE: "base" }); }); + +test("fresh exec privately resolves selected-service env files with the caller context", async () => { + const opts = await fixture(); + const sentinel = "synthetic-fresh-exec-file-canary"; + await writeFile( + opts.composeFile, + JSON.stringify({ + services: { + web: { image: "fixture", env_file: "web.env" }, + other: { image: "fixture", env_file: "missing-other.env" }, + }, + }) + ); + await writeFile( + join(opts.scope.projectDir, "web.env"), + "TOKEN=${SOURCE:?}\nVALUE=overridden-file-value\n", + { mode: 0o600 } + ); + expect( + await prepareNativeExecEnvironment({ ...opts, env: { SOURCE: sentinel } }) + ).toEqual({ VALUE: "selected", TOKEN: sentinel }); + const error: unknown = await prepareNativeExecEnvironment({ + ...opts, + env: {}, + }).catch((caught: unknown) => caught); + expect(String(error)).toContain("Values omitted"); + expect(String(error)).not.toContain(sentinel); +}); test("fresh exec refuses missing selectors or deleted overlays before AWS export", async () => { const opts = await fixture(); let calls = 0; diff --git a/tests/native-exec-stale-env.test.ts b/tests/native-exec-stale-env.test.ts index 070fe60f1..a3ed4b4fb 100644 --- a/tests/native-exec-stale-env.test.ts +++ b/tests/native-exec-stale-env.test.ts @@ -318,6 +318,48 @@ function preparedInput( } describe("native image exec stale env check", () => { + test("managed stale checks do not acquire or evaluate ephemeral env files", async () => { + const f = await project(); + const composeFile = join(f.projectDir, "docker-compose.yml"); + await writeFile( + composeFile, + JSON.stringify({ + services: { + web: { + image: "fixture", + env_file: "missing.env", + environment: { FROM_SHELL: "${REQUIRES_SHELL:?}" }, + }, + }, + }) + ); + await writeFile( + join(f.projectDir, "hack.env.default.yaml"), + JSON.stringify({ + version: 1, + environment: "default", + secretsprovider: "project_key", + values: { web: { [KEY]: SECRET_AFTER } }, + }) + ); + await recordImage({ + projectDir: f.projectDir, + env: { web: { [KEY]: SECRET_BEFORE } }, + }); + const warning = await resolveNativeImageExecStaleEnvWarning({ + scope: { + projectRoot: f.root, + projectDir: f.projectDir, + nativeHome: f.root, + branch: null, + }, + service: "web", + composeFile, + loadRun: async () => imageMapping(), + }); + expect(warning).toContain("hack restart"); + expectValueFree(warning ?? ""); + }); async function resolve(opts: { readonly projectDir: string; readonly root: string; diff --git a/tests/native-preflight-cli.test.ts b/tests/native-preflight-cli.test.ts index 2b971917e..8f469c508 100644 --- a/tests/native-preflight-cli.test.ts +++ b/tests/native-preflight-cli.test.ts @@ -49,7 +49,7 @@ async function fixture(compose: Record) { const projectDir = join(project, ".hack"); const nativeHome = join(root, "native-home"); const stub = join(root, "stub"); - await mkdir(projectDir, { recursive: true }); + await mkdir(projectDir, { recursive: true, mode: 0o700 }); await mkdir(nativeHome, { mode: 0o700 }); await mkdir(stub); await writeFile( @@ -215,6 +215,140 @@ function codes(report: NativePreflightReport, gate: "cli" | "runtime") { const GRANT = { HACK_NATIVE_SHARED_SOURCE: "1" }; +test("preflight and start share env-file admission and never publish file values", async () => { + const f = await fixture({ + services: { + web: { + image: IMAGE, + env_file: "service.env", + environment: { COPY: "${SOURCE}" }, + }, + }, + }); + await chmod(f.project, 0o700); + await writeFile(join(f.projectDir, "service.env"), `TOKEN=${PRIVATE}\n`, { + mode: 0o600, + }); + await writeFile(join(f.projectDir, ".env"), `SOURCE=${PRIVATE}\n`, { + mode: 0o600, + }); + await f.review([]); + const env = { ...f.native, ...GRANT }; + const observed = await preflight(f, env); + expect(observed.report.ready).toBe(true); + const normalized = await readFile( + join(f.root, "stub/normalized.json"), + "utf8" + ); + expect(JSON.parse(normalized).services.web.environment).toEqual({ + TOKEN: null, + COPY: null, + }); + expect(normalized).not.toContain(PRIVATE); + expect(observed.output).not.toContain(PRIVATE); + expect((await f.argv()).join("\n")).not.toContain(PRIVATE); + for (const unsafe of ["missing", "writable", "public-reference"]) { + if (unsafe === "missing") { + await rm(join(f.projectDir, "service.env")); + } + if (unsafe === "writable") { + await writeFile(join(f.projectDir, "service.env"), `TOKEN=${PRIVATE}\n`, { + mode: 0o666, + }); + await chmod(join(f.projectDir, "service.env"), 0o666); + } + if (unsafe === "public-reference") { + await chmod(join(f.projectDir, "service.env"), 0o600); + await writeFile( + join(f.projectDir, "docker-compose.yml"), + JSON.stringify({ + services: { + web: { image: IMAGE, command: ["echo", "${SOURCE:-fallback}"] }, + }, + }) + ); + } + const report = await preflight(f, env); + expect(codes(report.report, "cli")).toContain("environment_file_input:web"); + const before = await f.calls(); + const up = await cli(f, ["up", "--json"], env); + expect(up.code).toBe(1); + expect(up.output).not.toContain(PRIVATE); + expect(report.output).not.toContain(PRIVATE); + expect((await f.calls()).slice(before.length)).toEqual([]); + } +}, 30_000); + +test("preflight and static start refuse oversized file delivery before pool effects", async () => { + const f = await fixture({ + services: { web: { image: IMAGE, env_file: "service.env" } }, + }); + await chmod(f.project, 0o700); + await writeFile( + join(f.projectDir, "service.env"), + `TOKEN=${"private-quota-canary".repeat(2500)}\n`, + { mode: 0o600 } + ); + await f.review([], { services: { web: { active: true } } }); + const env = { ...f.native, ...GRANT }; + const report = await preflight(f, env); + expect(codes(report.report, "cli")).toContain("environment_delivery"); + expect(report.output).not.toContain("private-quota-canary"); + const before = await f.calls(); + const up = await cli(f, ["up"], env); + expect(up.code).toBe(1); + expect(up.output).toContain("32 KiB"); + expect(up.output).not.toContain("private-quota-canary"); + expect( + (await f.calls()) + .slice(before.length) + .every((call) => call === "project plan") + ).toBe(true); +}); + +test("preflight requires the same decryption key as start and closes key errors", async () => { + const f = await fixture({ services: { web: { image: IMAGE } } }); + await writeFile( + join(f.projectDir, "hack.env.default.yaml"), + JSON.stringify({ + version: 1, + environment: "default", + secretsprovider: "project_key", + values: { global: { TOKEN: { secure: PRIVATE } } }, + }) + ); + await f.review([]); + const report = await preflight(f, { ...f.native, ...GRANT }); + expect(codes(report.report, "cli")).toEqual(["managed_env_unreadable"]); + expect(report.output).not.toContain(PRIVATE); + const before = await f.calls(); + const up = await cli(f, ["up"], { ...f.native, ...GRANT }); + expect(up.code).toBe(1); + expect(up.output).toContain("E_CONFIG_INVALID"); + expect(up.output).not.toContain(PRIVATE); + expect((await f.calls()).slice(before.length)).toEqual([]); +}); + +test("old runtimes cannot silently ignore explicit environment unsets", async () => { + const f = await fixture({ + services: { + web: { image: IMAGE, environment: { ABSENT_FIXTURE_NAME: null } }, + }, + }); + await f.review([]); + const report = await preflight(f, { ...f.native, ...GRANT }); + expect(report.report.runtime_review.code).toBe( + "native_environment_unset_unavailable" + ); + await f.review([], { + services: { + web: { active: true, environment_unsets: ["ABSENT_FIXTURE_NAME"] }, + }, + }); + const supported = await preflight(f, { ...f.native, ...GRANT }); + expect(supported.report.ready).toBe(true); +}); + /** Replies that carry native `up` past its source checks to service admission. */ async function scriptSourceShare(f: Fixture): Promise { await f.reply("runtime-check-project-share.out", { @@ -285,9 +419,7 @@ test( expect(report.runtime_review).toEqual({ status: "completed" }); expect(codes(report, "cli")).toEqual([ "build_requires_opt_in:web", - "dotenv_without_managed_env", - "env_file:api", - "env_file:jobs", + "environment_file_input:api", "external_network", "extra_hosts_without_dependencies:web", "invalid_restart:jobs", @@ -388,7 +520,7 @@ test( expect(code).toBe(1); expect(report.runtime_review).toEqual({ status: "skipped" }); expect(report.ready).toBe(false); - expect(codes(report, "cli")).toEqual(["env_file:web"]); + expect(codes(report, "cli")).toEqual(["environment_file_input:web"]); expect(await f.calls()).toEqual([]); // No blocker is still not "ready" when half the checks did not run. const clean = await fixture({ services: { web: { image: IMAGE } } }); @@ -486,7 +618,7 @@ test( "preflight agrees with native up: each CLI blocker is a refusal, and a clean project passes the gate", async () => { for (const [service, extra, expected] of [ - [{ image: IMAGE, env_file: ".env" }, {}, "env_file:web"], + [{ image: IMAGE, env_file: ".env" }, {}, "environment_file_input:web"], [ { image: IMAGE, logging: { driver: "syslog" } }, {}, @@ -556,7 +688,7 @@ test( expect(up.output).toContain( "Run `hack doctor --native-preflight` to list every blocker with its fix." ); - expect(await f.calls()).toEqual(["runtime check-project-share"]); + expect(await f.calls()).toEqual([]); }, SPAWN_TIMEOUT_MS ); @@ -1021,7 +1153,7 @@ test("preflight cannot report ready when a runtime omits or corrupts start-admis } }); -test("preflight supplies names-only private-delivery context without decrypting managed env or reading shell values", async () => { +test("preflight uses the same managed resolution and shell context but sends only names to review", async () => { const f = await fixture({ services: { managed: { image: IMAGE, restart: "always" }, @@ -1033,7 +1165,7 @@ test("preflight supplies names-only private-delivery context without decrypting literal: { image: IMAGE, environment: { TOKEN: PRIVATE } }, excluded: { image: IMAGE, - environment: ["AWS_SECRET_ACCESS_KEY", "COMPOSE_PROJECT_NAME"], + environment: { AWS_REGION: "explicit-literal" }, }, }, }); @@ -1043,7 +1175,7 @@ test("preflight supplies names-only private-delivery context without decrypting version: 1, environment: "default", secretsprovider: "project_key", - values: { managed: { TOKEN: { secure: "invalid-ciphertext-sentinel" } } }, + values: { managed: { TOKEN: PRIVATE } }, }) ); await f.review( diff --git a/tests/native-project-builds.test.ts b/tests/native-project-builds.test.ts index 5e35771b8..d4ed064e0 100644 --- a/tests/native-project-builds.test.ts +++ b/tests/native-project-builds.test.ts @@ -402,7 +402,7 @@ test("a build arg equal to a managed secret value refuses without showing it", ( // HACK-1925: the refusal names the service and the argument, never the value, // and does not point at the Compose backend, which would bake it into history. expect(error.message).toContain( - "cannot build service web: build arg TOKEN equals a hack env secret value" + "cannot build service web: build arg TOKEN equals a privately delivered environment value" ); expect(error.message).not.toContain(secret); expect(error.message).not.toContain("HACK_RUNTIME_BACKEND"); diff --git a/tests/native-project-input.test.ts b/tests/native-project-input.test.ts index 52647b701..eaf7d5b7c 100644 --- a/tests/native-project-input.test.ts +++ b/tests/native-project-input.test.ts @@ -37,7 +37,7 @@ async function fixture(compose: unknown) { ); roots.push(projectRoot); const projectDir = join(projectRoot, ".hack"); - await mkdir(projectDir); + await mkdir(projectDir, { mode: 0o700 }); const composeFile = join(projectDir, "docker-compose.yml"); await writeFile(composeFile, JSON.stringify(compose)); return { projectRoot, projectDir, composeFile }; @@ -163,12 +163,12 @@ test("fourteen services and 110 managed variables each preserve overlay selectio expect(result.normalizedComposeJson).not.toContain("synthetic-"); expect(result.normalizedComposeJson).not.toContain("overlay-value"); }); -test("env_file and malformed modern input refuse without diagnostic contents", async () => { +test("missing env_file and malformed modern input refuse without diagnostic contents", async () => { const opts = await fixture({ services: { api: { env_file: "private-source", image: "bun:1" } }, }); await env(opts.projectDir, "default", { - global: { TOKEN: { secure: "invalid-secret-canary" } }, + global: { TOKEN: "managed-synthetic" }, }); await expect(prepareNativeProjectInput(opts)).rejects.toMatchObject({ code: "E_NATIVE_PROJECT_UNSUPPORTED", @@ -187,7 +187,7 @@ test("env_file and malformed modern input refuse without diagnostic contents", a expect(String(error)).toContain("values omitted"); } }); -test("legacy dotenv refuses and unmanaged public projects preserve original fields", async () => { +test("unreferenced dotenv does not inject values and public projects preserve original fields", async () => { const original = { services: { api: { image: "bun:1", environment: { STATIC: "tracked" } } }, }; @@ -195,11 +195,12 @@ test("legacy dotenv refuses and unmanaged public projects preserve original fiel expect( JSON.parse((await prepareNativeProjectInput(opts)).normalizedComposeJson) ).toEqual(original); - await writeFile(join(opts.projectDir, ".env"), "PRIVATE=synthetic\n"); - await expect(prepareNativeProjectInput(opts)).rejects.toMatchObject({ - code: "E_NATIVE_PROJECT_UNSUPPORTED", - message: expect.stringContaining("values omitted"), + await writeFile(join(opts.projectDir, ".env"), "PRIVATE=synthetic\n", { + mode: 0o644, }); + const withDotenv = await prepareNativeProjectInput(opts); + expect(JSON.parse(withDotenv.normalizedComposeJson)).toEqual(original); + expect(withDotenv.shellEnvironment).toEqual({}); }); test("the effective env is the requested or default env even without managed env files", async () => { @@ -448,7 +449,7 @@ function environments(result: { readonly normalizedComposeJson: string }) { * writing the value into the public document: the entry stays a reference and * the value is kept apart, for the private delivery. */ -test("plain shell references and bare names are resolved privately; everything else is left as written", async () => { +test("shell expressions are private, unresolved bare names unset, and public literals stay authored", async () => { const PRIVATE = "synthetic-shell-value"; const opts = await fixture({ services: { @@ -460,22 +461,11 @@ test("plain shell references and bare names are resolved privately; everything e UNSET: reference("SHELL_UNSET"), PASSED: null, NOT_PASSED: null, - // Forms that start today are not touched, whatever the shell holds. WITH_DEFAULT: reference("SHELL_SET", "fallback"), LITERAL: "plain", PARTIAL: `http://${reference("SHELL_SET")}:80`, ESCAPED: `$${reference("SHELL_SET")}`, - // Hack's own variables: the env decryption key must never be copyable. - LEAKED_KEY: reference("HACK_ENV_SECRET_KEY"), - HACK_ENV_SECRET_KEY: null, - HACK_GATEWAY_TOKEN: reference("HACK_GATEWAY_TOKEN"), - // Names the runtime supplies or the credential adapters own. PROJECT: reference("COMPOSE_PROJECT_NAME"), - COMPOSE_PROJECT_NAME: reference("SHELL_SET"), - AWS_SECRET_ACCESS_KEY: reference("AWS_SECRET_ACCESS_KEY"), - COPIED_KEY: reference("AWS_ACCESS_KEY_ID"), - GOOGLE_APPLICATION_CREDENTIALS: null, - // Valid names that are also Object.prototype properties. constructor: reference("constructor"), INHERITED: reference("toString"), }, @@ -491,7 +481,6 @@ test("plain shell references and bare names are resolved privately; everything e }, plain: { image: "bun:1" }, literal: { image: "bun:1", environment: ["A=1", "B=two"] }, - // Every entry commented out is YAML null; it must stay accepted. empty: { image: "bun:1", environment: null }, }, }); @@ -501,105 +490,80 @@ test("plain shell references and bare names are resolved privately; everything e SHELL_SET: PRIVATE, PASSED: `${PRIVATE}-passed`, COMPOSE_PROJECT_NAME: "other-instance", - AWS_SECRET_ACCESS_KEY: "long-lived-credential", - AWS_ACCESS_KEY_ID: "long-lived-id", - GOOGLE_APPLICATION_CREDENTIALS: "credential-path", - HACK_ENV_SECRET_KEY: "project-env-decryption-key", - HACK_GATEWAY_TOKEN: "gateway-token", - // Never read: no entry references it. UNRELATED: "unrelated-private", }, }); - // Shell values are kept apart from what Hack manages, and never fingerprinted. expect(result.managedEnvironment).toEqual({}); - expect(JSON.parse(JSON.stringify(result.shellEnvironment))).toEqual({ + expect(result.shellEnvironment).toEqual({ api: { WHOLE: PRIVATE, UNBRACED: PRIVATE, - // Compose substitutes an empty string for an unset variable. UNSET: "", PASSED: `${PRIVATE}-passed`, - // Delivered empty rather than left out, so the reviewed document is the - // same from a shell that sets it and one that does not. - NOT_PASSED: "", - // Unset in this shell, and not taken from Object.prototype. + WITH_DEFAULT: PRIVATE, + PARTIAL: `http://${PRIVATE}:80`, constructor: "", INHERITED: "", }, - worker: { PASSED: `${PRIVATE}-passed`, NOT_PASSED: "", WHOLE: PRIVATE }, - }); - expect(result.normalizedComposeJson).not.toContain(PRIVATE); - // Not read at all: neither public nor private. - for (const hidden of [ - "unrelated-private", - "other-instance", - "long-lived-credential", - "long-lived-id", - "credential-path", - "project-env-decryption-key", - "gateway-token", - ]) { - expect(result.normalizedComposeJson).not.toContain(hidden); - expect(JSON.stringify(result.shellEnvironment)).not.toContain(hidden); - } + worker: { PASSED: `${PRIVATE}-passed`, WHOLE: PRIVATE }, + }); const services = environments(result); - expect(JSON.parse(JSON.stringify(services.api?.environment))).toEqual({ + expect(services.api?.environment).toEqual({ WHOLE: null, UNBRACED: null, UNSET: null, PASSED: null, - NOT_PASSED: null, - constructor: null, - INHERITED: null, - WITH_DEFAULT: reference("SHELL_SET", "fallback"), + WITH_DEFAULT: null, LITERAL: "plain", - PARTIAL: `http://${reference("SHELL_SET")}:80`, - ESCAPED: `$${reference("SHELL_SET")}`, + PARTIAL: null, + ESCAPED: "$${SHELL_SET}", PROJECT: reference("COMPOSE_PROJECT_NAME"), - COMPOSE_PROJECT_NAME: reference("SHELL_SET"), - AWS_SECRET_ACCESS_KEY: reference("AWS_SECRET_ACCESS_KEY"), - COPIED_KEY: reference("AWS_ACCESS_KEY_ID"), - GOOGLE_APPLICATION_CREDENTIALS: null, - LEAKED_KEY: reference("HACK_ENV_SECRET_KEY"), - HACK_ENV_SECRET_KEY: null, - HACK_GATEWAY_TOKEN: reference("HACK_GATEWAY_TOKEN"), + constructor: null, + INHERITED: null, }); + expect( + JSON.parse(result.normalizedComposeJson).services.api[ + "x-hack-unset-environment" + ] + ).toEqual(["NOT_PASSED"]); expect(services.worker?.environment).toEqual({ PASSED: null, - NOT_PASSED: null, WHOLE: null, KEEP: "1", }); expect(services.plain?.environment).toBeUndefined(); - // No reference, no rewrite: the list form is exactly what was authored. expect(services.literal?.environment).toEqual(["A=1", "B=two"]); expect(services.empty?.environment).toBeNull(); - // Names only, never values. - expect(result.environmentNotices).toEqual([ - { - service: "api", - line: "Service api: UNSET is empty because SHELL_UNSET is not set in your shell.", - }, - { - service: "api", - line: "Service api: NOT_PASSED is empty because it is not set in your shell.", - }, - { - service: "api", - line: "Service api: constructor is empty because it is not set in your shell.", - }, - { - service: "api", - line: "Service api: INHERITED is empty because toString is not set in your shell.", - }, - { - service: "worker", - line: "Service worker: NOT_PASSED is empty because it is not set in your shell.", - }, - ]); + expect(result.environmentNotices).toHaveLength(5); + expect(JSON.stringify(result.environmentNotices)).toContain( + "NOT_PASSED is unset" + ); + for (const hidden of [PRIVATE, "other-instance", "unrelated-private"]) { + expect(result.normalizedComposeJson).not.toContain(hidden); + expect(JSON.stringify(result.environmentNotices)).not.toContain(hidden); + } }); -test("the reviewed document does not depend on the shell, and nothing is read unless the caller passes an environment", async () => { +test("Hack key material is refused without reading the caller's value", async () => { + for (const key of ["HACK_ENV_SECRET_KEY", "HACK_GATEWAY_TOKEN"]) { + const opts = await fixture({ + services: { + api: { image: "bun:1", environment: { COPY: reference(key) } }, + }, + }); + const caller = Object.defineProperty({}, key, { + enumerable: true, + get() { + throw new Error("private getter must never be invoked"); + }, + }); + await expect( + prepareNativeProjectInput({ ...opts, env: caller }) + ).rejects.toThrow("protected variable"); + } +}); + +test("unset names affect only names in the document; ambient values require an explicit caller environment", async () => { const compose = { services: { api: { @@ -616,8 +580,13 @@ test("the reviewed document does not depend on the shell, and nothing is read un ...(await fixture(compose)), env: {}, }); - // Same public document from a shell that sets the variables and one that does not. - expect(set.normalizedComposeJson).toBe(unset.normalizedComposeJson); + // Bare unset is distinct from an empty string and only the name enters identity. + expect(set.normalizedComposeJson).not.toBe(unset.normalizedComposeJson); + expect( + JSON.parse(unset.normalizedComposeJson).services.api[ + "x-hack-unset-environment" + ] + ).toEqual(["PASSED"]); // Without `env`, as exec and the stale-env check call it, the process // environment is not consulted. const previous = process.env.SHELL_TOKEN; @@ -625,9 +594,9 @@ test("the reviewed document does not depend on the shell, and nothing is read un try { const ambient = await prepareNativeProjectInput(await fixture(compose)); expect(ambient.shellEnvironment).toEqual({ - api: { TOKEN: "", PASSED: "" }, + api: { TOKEN: "" }, }); - expect(ambient.normalizedComposeJson).toBe(set.normalizedComposeJson); + expect(ambient.normalizedComposeJson).toBe(unset.normalizedComposeJson); } finally { restoreEnv("SHELL_TOKEN", previous); } @@ -662,12 +631,11 @@ test("a key Hack manages is delivered from managed env, never from the shell", a TOKEN: "managed-private", FROM_MANAGED: "managed-too", }); - // Nothing of this service is taken from the shell. - expect(result.shellEnvironment).toEqual({}); - // A reference to a managed variable is left for the runtime to refuse. + // A global managed value participates in interpolation above shell and dotenv. + expect(result.shellEnvironment).toEqual({ api: { COPY: "managed-private" } }); expect( (environments(result).api?.environment as Record).COPY - ).toBe(reference("TOKEN")); + ).toBeNull(); for (const hidden of ["shell-private", "shell-loses"]) { expect(result.normalizedComposeJson).not.toContain(hidden); } @@ -713,7 +681,6 @@ test("merged shell values join the private delivery, and a delivery the runtime // JSON escaping counts: the runtime receives the escaped form. [1, '"'.repeat(17 * 1024), "are over 32 KiB together"], [257, "#", "more than 256 variables"], - [1, "bad\0value", "contains a NUL byte"], ] as const) { const input = await prepared(count, value); const delivery = mergeNativeShellEnvironment(input); diff --git a/tests/native-project-start.test.ts b/tests/native-project-start.test.ts index c2e0d539e..716dc74e0 100644 --- a/tests/native-project-start.test.ts +++ b/tests/native-project-start.test.ts @@ -3967,21 +3967,22 @@ test("pending retained restart refuses fresh startup before effects", async () = expect(events).toEqual([]); }); -test("startup prepares mapping storage before input and plan review", async () => { +test("startup admits private input before storage and prepares storage before plan effects", async () => { const { opts, events } = await fixture(false); const prepare = opts.dependencies.prepare!; const review = opts.dependencies.review!; opts.dependencies.prepareStorage = async (scope) => { expect(scope).toEqual(opts.scope); + expect(events).toContain("input"); events.push("storage"); }; opts.dependencies.prepare = async (request) => { - expect(events[0]).toBe("storage"); + expect(events).not.toContain("storage"); events.push("input"); return await prepare(request); }; opts.dependencies.review = async (request) => { - expect(events.indexOf("storage")).toBeLessThan(events.indexOf("input")); + expect(events.indexOf("input")).toBeLessThan(events.indexOf("storage")); return await review(request); }; expect(await startNativeProject(opts)).toBe(0); diff --git a/tests/native-refusal-cli.test.ts b/tests/native-refusal-cli.test.ts index 1cf160beb..bb5f2190f 100644 --- a/tests/native-refusal-cli.test.ts +++ b/tests/native-refusal-cli.test.ts @@ -501,9 +501,9 @@ test( "E_NATIVE_PROJECT_UNSUPPORTED: Selected (non-ignored) source is 140 MiB in 3 files; the native source limit is 128 MiB (default). Native up refuses source over the limit; no runtime, image or build work was started. Largest directories: assets (140 MiB), assets/video (140 MiB). Largest files: assets/video/intro.bin (140 MiB). Exclude paths the containers do not need in the project's .gitignore or .ignore file (for example a line such as `assets/video/`) or raise native.sourceLimitMiB in .hack/hack.config.json (maximum 512)." ); expect(await f.calls()).toEqual([ - "runtime check-project-share", "project plan", "project plan", + "runtime check-project-share", "project source-size", ]); }, @@ -527,9 +527,9 @@ test( "E_CONFIG_INVALID: Native host dependencies refused: the selection must be {version: 1, dependencies: [1..128 entries]} with no other fields; listener values omitted." ); expect(await f.calls()).toEqual([ - "runtime check-project-share", "project plan", "project plan", + "runtime check-project-share", ]); }, SPAWN_TIMEOUT_MS @@ -546,10 +546,7 @@ test( result, "Native project input changed before review; prepare it again." ); - expect(await f.calls()).toEqual([ - "runtime check-project-share", - "project plan", - ]); + expect(await f.calls()).toEqual(["project plan"]); }, SPAWN_TIMEOUT_MS ); @@ -905,7 +902,7 @@ test( result, "E_CONFIG_INVALID: Native workspace cache layout conflicts with declared mounts or is invalid; values omitted." ); - expect(await f.calls()).toEqual(["runtime check-project-share"]); + expect(await f.calls()).toEqual([]); }, SPAWN_TIMEOUT_MS ); diff --git a/tests/native-shell-environment-cli.test.ts b/tests/native-shell-environment-cli.test.ts index a838e9bdd..347baba40 100644 --- a/tests/native-shell-environment-cli.test.ts +++ b/tests/native-shell-environment-cli.test.ts @@ -16,7 +16,6 @@ import { prepareNativeProjectInput, } from "../src/backends/native-project-input.ts"; import { nativeEnvironmentDelivery } from "../src/backends/native-project-start.ts"; -import { resolveShellEnvironment } from "../src/backends/native-shell-environment.ts"; import { nativeSourceSizeReceipt } from "./helpers/native-source-size.ts"; import { privateCliEnv } from "./helpers/private-cli-env.ts"; @@ -190,7 +189,7 @@ for (const mode of ["foreground", "detached", "json"] as const) { if (!detached) { expect( output.split( - "Service web: REGION is empty because FIXTURE_REGION is not set in your shell.\n" + "Service web: REGION is empty because FIXTURE_REGION is not set in the interpolation context.\n" ) ).toHaveLength(2); } @@ -254,28 +253,43 @@ test("shell values travel only in the private stdin payload, never in the public }); }); -test("a variable named like an Object.prototype property is an ordinary variable", () => { - const resolution = resolveShellEnvironment({ - service: "web", - environment: Object.fromEntries([ - ["__proto__", reference("FIXTURE_SET")], - ["constructor", null], - ["PLAIN", reference("toString")], - ]), +test("a variable named like an Object.prototype property is an ordinary variable", async () => { + const f = await fixture(); + await writeFile( + f.composeFile, + JSON.stringify({ + services: { + web: { + image: IMAGE, + environment: Object.fromEntries([ + ["__proto__", reference("FIXTURE_SET")], + ["constructor", null], + ["PLAIN", reference("toString")], + ]), + }, + }, + }) + ); + const resolution = await prepareNativeProjectInput({ + ...f, + projectRoot: f.project, env: { FIXTURE_SET: PRIVATE }, - managed: new Set(), }); - expect(Object.entries(resolution?.delivered ?? {})).toEqual([ + expect(Object.entries(resolution.shellEnvironment?.web ?? {})).toEqual([ ["__proto__", PRIVATE], // Not set in this shell; never the inherited function. - ["constructor", ""], ["PLAIN", ""], ]); - expect(Object.keys(resolution?.environment ?? {})).toEqual([ - "__proto__", - "constructor", - "PLAIN", - ]); + expect( + Object.keys( + JSON.parse(resolution.normalizedComposeJson).services.web.environment + ) + ).toEqual(["__proto__", "PLAIN"]); + expect( + JSON.parse(resolution.normalizedComposeJson).services.web[ + "x-hack-unset-environment" + ] + ).toEqual(["constructor"]); }); async function regularFiles(directory: string): Promise { @@ -305,7 +319,7 @@ async function regularFiles(directory: string): Promise { return result; } -test("only source names referenced by Compose are read and reported, never the ambient environment", () => { +test("only source names referenced by Compose are read and reported, never the ambient environment", async () => { const caller = Object.defineProperties( { REFERENCED: PRIVATE }, { @@ -317,16 +331,30 @@ test("only source names referenced by Compose are read and reported, never the a }, } ); - const resolved = resolveShellEnvironment({ - service: "web", - environment: { - API_KEY: reference("REFERENCED"), - COPY: reference("REFERENCED"), - }, + const f = await fixture(); + await writeFile( + f.composeFile, + JSON.stringify({ + services: { + web: { + image: IMAGE, + environment: { + API_KEY: reference("REFERENCED"), + COPY: reference("REFERENCED"), + }, + }, + }, + }) + ); + const resolved = await prepareNativeProjectInput({ + ...f, + projectRoot: f.project, env: caller, - managed: new Set(), }); - expect(resolved?.delivered).toEqual({ API_KEY: PRIVATE, COPY: PRIVATE }); - expect(resolved?.forwardedNames).toEqual(["REFERENCED"]); - expect(JSON.stringify(resolved?.environment)).not.toContain(PRIVATE); + expect(resolved.shellEnvironment?.web).toEqual({ + API_KEY: PRIVATE, + COPY: PRIVATE, + }); + expect(resolved.forwardedShellVariables?.web).toEqual(["REFERENCED"]); + expect(resolved.normalizedComposeJson).not.toContain(PRIVATE); }); From bf5f281fcbe33b51a3a9d07f784cd78a06dafc50 Mon Sep 17 00:00:00 2001 From: Kit <344842+roodboi@users.noreply.github.com> Date: Fri, 13 Nov 2026 21:26:01 +0000 Subject: [PATCH 03/15] fix(native): preserve env-file interpolation context Resolve authored service fallbacks before adding file-backed bindings, including unselected branches and escaped dollar signs. Allow file-local cloud names before refusing unresolved restricted shell lookups. --- docs/native/environment-files.md | 5 + src/backends/native-compose-env-syntax.ts | 56 ++++++++++- src/backends/native-compose-environment.ts | 87 ++++++++++++----- tests/native-compose-env-syntax.test.ts | 9 ++ tests/native-compose-environment.test.ts | 104 +++++++++++++++++++++ 5 files changed, 235 insertions(+), 26 deletions(-) diff --git a/docs/native/environment-files.md b/docs/native/environment-files.md index 3141616dd..434f59069 100644 --- a/docs/native/environment-files.md +++ b/docs/native/environment-files.md @@ -59,6 +59,11 @@ destination but are not ambient interpolation sources. For env-file expansion, that project context wins over resolved service `environment`, then current and earlier file assignments. A current file assignment replaces the previous file's assignment before the next line is read. `.env` alone injects no container variables. +Service-environment defaults resolve only against the project context: an env-file +`HOST` does not replace `${HOST:-localhost}` with its file value. Defaults that +reference newly file-backed names are frozen before runtime private ownership. +File-local cloud names can be used by later assignments without reading those +names from the shell. Support includes export prefixes, `=` and `:` separators in default format, comments, single-quoted literals, double-quoted escapes, multiline quoted values, diff --git a/src/backends/native-compose-env-syntax.ts b/src/backends/native-compose-env-syntax.ts index 578a01f77..dafec71fe 100644 --- a/src/backends/native-compose-env-syntax.ts +++ b/src/backends/native-compose-env-syntax.ts @@ -23,6 +23,28 @@ const ESCAPES: Readonly> = { export type ComposeEnvLookup = (name: string) => string | undefined; +/** All syntactic source names, including unselected words, without reading values. */ +export function composeEnvironmentReferences( + text: string +): ReadonlySet { + const result = new Set(); + for (let cursor = 0; cursor < text.length; cursor += 1) { + if (text[cursor] !== "$") { + continue; + } + if (text[cursor + 1] === "$") { + cursor += 1; + continue; + } + const begin = cursor + (text[cursor + 1] === "{" ? 2 : 1); + const name = NAME.exec(text.slice(begin))?.[0]; + if (name !== undefined) { + result.add(name); + } + } + return result; +} + /** Closed failures never include the input, an authored error word or values. */ export class NativeEnvSyntaxError extends Error { constructor() { @@ -293,6 +315,20 @@ function dotenvValue(opts: { }; } +function inheritBareFileName(opts: { + readonly key: string; + readonly lookup: ComposeEnvLookup; + readonly values: Map; + readonly onUnresolved?: (name: string) => void; +}): void { + const inherited = opts.lookup(opts.key); + if (inherited !== undefined) { + opts.values.set(opts.key, inherited); + } else if (!opts.values.has(opts.key)) { + opts.onUnresolved?.(opts.key); + } +} + /** * Sequential Compose assignments, seeded by earlier files. A bare name inherits * only from project/service lookup; when unresolved it leaves prior entries @@ -304,6 +340,8 @@ export function parseComposeEnvironmentFile(opts: { readonly lookup: ComposeEnvLookup; readonly initial?: ReadonlyMap; readonly onMissing?: (name: string) => void; + /** Source admission after both project lookup and parsed-file assignments miss. */ + readonly onUnresolved?: (name: string) => void; }): ReadonlyMap { if (Buffer.byteLength(opts.text) > MAX_BYTES || opts.text.includes("\0")) { return invalid(); @@ -313,7 +351,13 @@ export function parseComposeEnvironmentFile(opts: { if (lines.length > MAX_LINES) { return invalid(); } - const lookup = (name: string) => opts.lookup(name) ?? result.get(name); + const lookup = (name: string) => { + const found = opts.lookup(name) ?? result.get(name); + if (found === undefined) { + opts.onUnresolved?.(name); + } + return found; + }; for (let index = 0; index < lines.length; index += 1) { const line = (lines[index] ?? "").trimStart(); if (line.trim() === "" || line.startsWith("#")) { @@ -328,10 +372,12 @@ export function parseComposeEnvironmentFile(opts: { return invalid(); } if (entry?.[2] === undefined) { - const inherited = opts.lookup(key); - if (inherited !== undefined) { - result.set(key, inherited); - } + inheritBareFileName({ + key, + lookup: opts.lookup, + values: result, + onUnresolved: opts.onUnresolved, + }); continue; } const decoded = dotenvValue({ diff --git a/src/backends/native-compose-environment.ts b/src/backends/native-compose-environment.ts index 6e85fb641..4132f4d82 100644 --- a/src/backends/native-compose-environment.ts +++ b/src/backends/native-compose-environment.ts @@ -4,6 +4,7 @@ import { HACK_KEY_MATERIAL_VARIABLES } from "../lib/child-environment.ts"; import { isRecord } from "../lib/guards.ts"; import { nativeAwsCredentialMount } from "./native-aws-environment.ts"; import { + composeEnvironmentReferences, interpolateComposeEnvironment, parseComposeEnvironmentFile, } from "./native-compose-env-syntax.ts"; @@ -216,6 +217,14 @@ async function readAutomaticDotenv(opts: { dotenv = parseComposeEnvironmentFile({ text: source.text, lookup: opts.lookup, + onUnresolved: (name) => { + if (!shellVariableReadable(name)) { + throw refused( + ".env", + `variable ${name} cannot be read from the caller's shell; use an explicit file value` + ); + } + }, }); } catch (error: unknown) { if (error instanceof NativeEnvFileRefusal) { @@ -335,6 +344,7 @@ type Resolution = { readonly delivered: Record; readonly unset: string[]; readonly unbound: Set; + readonly references: Map>; readonly changed: boolean; }; type ServiceContext = { @@ -424,8 +434,18 @@ function expandServiceValue(opts: { ); } } -function scalar(value: unknown): value is number | boolean { - return typeof value === "number" || typeof value === "boolean"; +function serviceScalar(opts: { + readonly value: unknown; + readonly service: string; +}): number | boolean { + if (typeof opts.value === "number" || typeof opts.value === "boolean") { + return opts.value; + } + throw refused( + "environment", + "environment values must be scalar", + opts.service + ); } function resolveBareService(opts: { @@ -460,6 +480,7 @@ function resolveService(opts: { const resolved = new Map(); const unset: string[] = []; const unbound = new Set(); + const references = new Map>(); const { context } = opts; let changed = false; for (const [key, value] of Object.entries(opts.environment)) { @@ -496,15 +517,9 @@ function resolveService(opts: { continue; } if (typeof value !== "string") { - if (!scalar(value)) { - throw refused( - "environment", - "environment values must be scalar", - context.name - ); - } - environment[key] = value; - resolved.set(key, String(value)); + const literal = serviceScalar({ value, service: context.name }); + environment[key] = literal; + resolved.set(key, String(literal)); continue; } const expanded = expandServiceValue({ text: value, key, context }); @@ -513,13 +528,24 @@ function resolveService(opts: { unbound.add(key); } else { resolved.set(key, expanded.value); + if (!expanded.private) { + references.set(key, composeEnvironmentReferences(value)); + } } if (expanded.private) { delivered[key] = expanded.value; changed = true; } } - return { environment, resolved, delivered, unset, unbound, changed }; + return { + environment, + resolved, + delivered, + unset, + unbound, + references, + changed, + }; } async function serviceFiles(opts: { @@ -561,21 +587,16 @@ async function serviceFiles(opts: { opts.context.name ); } - if ( - !( - opts.resolution.resolved.has(key) || - shellVariableReadable(key) || - Object.hasOwn(opts.context.global, key) || - opts.sources.dotenv.has(key) - ) - ) { + return opts.resolution.resolved.get(key); + }, + onUnresolved: (key) => { + if (!shellVariableReadable(key)) { throw refused( spec.path, `variable ${key} cannot be read from the caller's shell; use an explicit file value or credential adapter`, opts.context.name ); } - return opts.resolution.resolved.get(key); }, }); } catch (error: unknown) { @@ -632,6 +653,7 @@ async function normalizeService(opts: { Object.create(null), resolution.environment ); + const fileBindings = new Set(); for (const [key, value] of files) { if (protectedName(key)) { throw refused( @@ -645,7 +667,9 @@ async function normalizeService(opts: { } environment[key] = null; resolution.delivered[key] = value; + fileBindings.add(key); } + resolveFileBindingDefaults({ environment, resolution, fileBindings }); if (specs.length > 0 || resolution.changed) { document.environment = environment; } @@ -657,6 +681,27 @@ async function normalizeService(opts: { return resolution.delivered; } +/** + * YAML environment interpolation uses the project context, never env-file values. + * Freeze a public default before a newly private file name becomes runtime-owned. + * Re-escape dollars because the runtime still evaluates the public declarations. + */ +function resolveFileBindingDefaults(opts: { + readonly environment: Record; + readonly resolution: Resolution; + readonly fileBindings: ReadonlySet; +}): void { + for (const [key, references] of opts.resolution.references) { + const value = opts.resolution.resolved.get(key); + if ( + value !== undefined && + [...references].some((name) => opts.fileBindings.has(name)) + ) { + opts.environment[key] = value.replaceAll("$", () => "$$"); + } + } +} + /** * Native-only normalization. Public declarations contain names, never file or * shell values. Returned maps are private ephemeral inputs, not managed-env diff --git a/tests/native-compose-env-syntax.test.ts b/tests/native-compose-env-syntax.test.ts index e4d9fe45e..fd119ab69 100644 --- a/tests/native-compose-env-syntax.test.ts +++ b/tests/native-compose-env-syntax.test.ts @@ -1,5 +1,6 @@ import { expect, test } from "bun:test"; import { + composeEnvironmentReferences, interpolateComposeEnvironment, parseComposeEnvironmentFile, } from "../src/backends/native-compose-env-syntax.ts"; @@ -13,6 +14,14 @@ const variables: Readonly> = { const lookup = (name: string) => Object.hasOwn(variables, name) ? variables[name] : undefined; +test("source names include unselected branches while escaped dollars stay literal", () => { + expect([ + ...composeEnvironmentReferences( + "${OTHER:+${HOST}} $$ESCAPED $${BRACED} $$$REAL ${MISSING:-$$LITERAL}" + ), + ]).toEqual(["OTHER", "HOST", "REAL", "MISSING"]); +}); + test.each([ ["$SET/${SET}", `${SENTINEL}/${SENTINEL}`], ["${SET:-fallback}", SENTINEL], diff --git a/tests/native-compose-environment.test.ts b/tests/native-compose-environment.test.ts index 2412d9da3..517973047 100644 --- a/tests/native-compose-environment.test.ts +++ b/tests/native-compose-environment.test.ts @@ -170,6 +170,110 @@ test("explicit cloud values may seed env-file expansion without reading cloud sh expect(input.normalizedComposeJson).not.toContain(SHELL); }); +test("file-backed names do not capture service defaults from the project interpolation context", async () => { + const f = await fixture({ + env_file: "service.env", + environment: { + URL: "${HOST:-localhost}", + ALT: "${HOST:+present}", + DOLLAR: "${HOST:-$$LITERAL}", + UNSELECTED: "${OTHER:+${HOST}}", + UNAFFECTED: "${OTHER:-public}", + }, + }); + await f.source("service.env", `HOST=${FILE}\n`); + const first = await prepareNativeProjectInput(f); + expect(first.shellEnvironment).toEqual({ app: { HOST: FILE } }); + expect(publicService(first).environment).toEqual({ + HOST: null, + URL: "localhost", + ALT: "", + DOLLAR: "$$LITERAL", + UNSELECTED: "", + UNAFFECTED: "${OTHER:-public}", + }); + expect(first.normalizedComposeJson).not.toContain(FILE); + await f.source("service.env", `HOST=${DOTENV}\n`); + const second = await prepareNativeProjectInput(f); + expect(second.normalizedComposeJson).toBe(first.normalizedComposeJson); + expect(second.shellEnvironment).toEqual({ app: { HOST: DOTENV } }); +}); + +test("a default combined with a private source stays private after file bindings are introduced", async () => { + const f = await fixture({ + env_file: "service.env", + environment: { URL: "${HOST:-localhost}/${SOURCE}" }, + }); + await f.source("service.env", `HOST=${FILE}\n`); + await f.source(".env", `SOURCE=${DOTENV}\n`); + const input = await prepareNativeProjectInput(f); + expect(publicService(input).environment).toEqual({ HOST: null, URL: null }); + expect(input.shellEnvironment).toEqual({ + app: { HOST: FILE, URL: `localhost/${DOTENV}` }, + }); + expect(input.normalizedComposeJson).not.toContain(FILE); + expect(input.normalizedComposeJson).not.toContain(DOTENV); +}); + +test("env-file-local cloud values resolve without reading cloud shell values", async () => { + const f = await fixture({ env_file: ["one.env", "two.env"] }); + await f.source( + "one.env", + `AWS_REGION=${FILE}\nSAME=\${AWS_REGION}\nGOOGLE_PROJECT=${DOTENV}\nCLOUDSDK_CORE_PROJECT=${FILE}\n` + ); + await f.source( + "two.env", + "REGION=${AWS_REGION}\nPROJECT=${GOOGLE_PROJECT}\nCLOUD=${CLOUDSDK_CORE_PROJECT}\nAWS_REGION\n" + ); + const env: Record = {}; + for (const key of ["AWS_REGION", "GOOGLE_PROJECT", "CLOUDSDK_CORE_PROJECT"]) { + Object.defineProperty(env, key, { + enumerable: true, + get() { + throw new Error("cloud shell value must never be read"); + }, + }); + } + const input = await prepareNativeProjectInput({ ...f, env }); + expect(input.shellEnvironment).toEqual({ + app: { + AWS_REGION: FILE, + SAME: FILE, + GOOGLE_PROJECT: DOTENV, + CLOUDSDK_CORE_PROJECT: FILE, + REGION: FILE, + PROJECT: DOTENV, + CLOUD: FILE, + }, + }); + expect(input.forwardedShellVariables?.app).toEqual([]); + expect(input.normalizedComposeJson).not.toContain(FILE); + expect(input.normalizedComposeJson).not.toContain(DOTENV); + await f.source("one.env", "REGION=${AWS_REGION}\n"); + await expect(prepareNativeProjectInput({ ...f, env })).rejects.toThrow( + "cannot be read from the caller's shell" + ); +}); + +test("dotenv cloud references use earlier local values and refuse an unavailable source", async () => { + const f = await fixture({ environment: { REGION: "${REGION}" } }); + await f.source(".env", `AWS_REGION=${DOTENV}\nREGION=\${AWS_REGION}\n`); + const input = await prepareNativeProjectInput({ + ...f, + env: { AWS_REGION: SHELL }, + }); + expect(input.shellEnvironment).toEqual({ app: { REGION: DOTENV } }); + expect(input.normalizedComposeJson).not.toContain(DOTENV); + expect(input.normalizedComposeJson).not.toContain(SHELL); + await f.source(".env", "REGION=${AWS_REGION}\n"); + const error: unknown = await prepareNativeProjectInput({ + ...f, + env: { AWS_REGION: SHELL }, + }).catch((caught: unknown) => caught); + expect(String(error)).toContain("AWS_REGION cannot be read"); + expect(String(error)).not.toContain(SHELL); +}); + test("NUL private inputs refuse with closed diagnostics", async () => { const f = await fixture({ env_file: "service.env" }); await f.source("service.env", `TOKEN=${FILE}\0\n`); From 04ae59c91613010f6db84d04dfcda2d9f0b82bcd Mon Sep 17 00:00:00 2001 From: Kit <344842+roodboi@users.noreply.github.com> Date: Fri, 13 Nov 2026 21:40:51 +0000 Subject: [PATCH 04/15] test: avoid random metadata substring failures --- tests/background-command-journal.test.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/background-command-journal.test.ts b/tests/background-command-journal.test.ts index 659beb680..8cafa42b3 100644 --- a/tests/background-command-journal.test.ts +++ b/tests/background-command-journal.test.ts @@ -132,7 +132,10 @@ async function expectCode(promise: Promise, code: string) { describe("journal events and cursors", () => { test("accepted is persisted first; writers keep seq dense and strictly increasing", async () => { - const { paths, meta } = await acceptCommand({ input: input(), root }); + const { paths, meta } = await acceptCommand({ + input: input({ caller: "test:env" }), + root, + }); expect((await stat(paths.dir)).mode & 0o777).toBe(0o700); expect((await stat(paths.meta)).mode & 0o777).toBe(0o600); expect((await stat(paths.events)).mode & 0o777).toBe(0o600); @@ -154,7 +157,10 @@ describe("journal events and cursors", () => { seq: 4, }) ); - expect(JSON.stringify(meta)).not.toContain("env"); + const persisted: unknown = JSON.parse(await readFile(paths.meta, "utf8")); + expect(persisted).toEqual(meta); + expect(persisted).not.toHaveProperty("env"); + expect(persisted).not.toHaveProperty("environment"); }); test("an unterminated final line is a write in progress, not an event", () => { From 39c57c708c4f2c4c931100e5e650af414f4752d7 Mon Sep 17 00:00:00 2001 From: Kit <344842+roodboi@users.noreply.github.com> Date: Fri, 13 Nov 2026 21:49:49 +0000 Subject: [PATCH 05/15] fix(native): exclude environment file path aliases from source --- docs/native/environment-files.md | 5 ++- .../src/project/compose/service.rs | 5 ++- .../runtime-core/src/project/live_source.rs | 27 ++++++++++--- packages/runtime-core/src/project/source.rs | 39 ++++++++++++++++++- .../tests/normalized_cli_contract.rs | 22 +++++++++++ 5 files changed, 89 insertions(+), 9 deletions(-) diff --git a/docs/native/environment-files.md b/docs/native/environment-files.md index 434f59069..b880bbf68 100644 --- a/docs/native/environment-files.md +++ b/docs/native/environment-files.md @@ -102,7 +102,10 @@ expiring guest allocation/launcher. They never enter CLI argv, public JSON, plan plan identities, receipts, logs or generated files. Original env-file declarations remain source exclusions in Rust even when normalization removes them, including absent optional targets, inactive-service files, raw inputs -and duplicate paths. `.env*` remain excluded by the existing source policy. Tests +and duplicate paths. Exclusions compare paths with Unicode canonical normalization +and case folding, so host path aliases cannot enter snapshot archives or live source updates. +Ambiguous matching directory entries refuse, and environment inputs cannot own +source ignore-rule files through an alias. `.env*` remain excluded by the existing source policy. Tests must cover snapshot and live-share exclusion when an optional file appears later. Private delivery retains its existing limitations: it cannot use automatic diff --git a/packages/runtime-core/src/project/compose/service.rs b/packages/runtime-core/src/project/compose/service.rs index 7527f229d..680ac20fd 100644 --- a/packages/runtime-core/src/project/compose/service.rs +++ b/packages/runtime-core/src/project/compose/service.rs @@ -489,7 +489,10 @@ pub(super) fn environment_files( }; if resolved .file_name() - .is_some_and(|name| name == ".gitignore" || name == ".ignore") + .and_then(|name| name.to_str()) + .is_some_and(|name| { + matches!(source::folded_name(name).as_str(), ".gitignore" | ".ignore") + }) { return Err(problem( "conflicting_environment_owner", diff --git a/packages/runtime-core/src/project/live_source.rs b/packages/runtime-core/src/project/live_source.rs index 43482df55..8a2cd727f 100644 --- a/packages/runtime-core/src/project/live_source.rs +++ b/packages/runtime-core/src/project/live_source.rs @@ -461,12 +461,14 @@ mod tests { } #[test] - fn optional_env_file_created_after_review_stays_out_of_live_source() { + fn optional_env_file_aliases_created_after_review_stay_out_of_source_and_live_updates() { let (_fixture, candidate, source) = fixture(); let path = source.join("compose.yaml"); let mut original = super::super::yaml::parse(&fs::read(&path).unwrap()).unwrap(); original["services"]["deps"]["env_file"] = serde_json::json!([ - {"path":"secrets/private.txt", "required":false, "format":"raw"} + {"path":"config/Private.txt", "required":false, "format":"raw"}, + {"path":"config/café.vars", "required":false}, + {"path":"Config/Straße.vars", "required":false} ]); let original_bytes = serde_json::to_vec(&original).unwrap(); fs::write(&path, &original_bytes).unwrap(); @@ -497,21 +499,34 @@ mod tests { .receipt() .entries .iter() - .all(|entry| entry.path != "secrets/private.txt" && entry.path != ".env") + .all(|entry| !entry.path.starts_with("config/") && entry.path != ".env") + ); + assert!( + !snapshot + .archive() + .unwrap() + .windows(b"PRIVATE_".len()) + .any(|bytes| bytes == b"PRIVATE_") ); (plan, snapshot) }; let (initial, snapshot) = capture(); let contract = Contract::from_plan(&initial.plan, snapshot.receipt()).unwrap(); - fs::create_dir(source.join("secrets")).unwrap(); - fs::write(source.join("secrets/private.txt"), "PRIVATE_FUTURE_CANARY").unwrap(); + fs::create_dir(source.join("config")).unwrap(); + for path in [ + "config/private.txt", + "config/cafe\u{301}.vars", + "config/STRASSE.vars", + ] { + fs::write(source.join(path), "PRIVATE_FUTURE_CANARY").unwrap(); + } fs::write(source.join(".env"), "PRIVATE_DOTENV_CANARY").unwrap(); let (with_file, snapshot) = capture(); contract .verify(&with_file.plan, snapshot.receipt()) .unwrap(); fs::write( - source.join("secrets/private.txt"), + source.join("config/private.txt"), "PRIVATE_CHANGED_CANARY_LONGER", ) .unwrap(); diff --git a/packages/runtime-core/src/project/source.rs b/packages/runtime-core/src/project/source.rs index 9e2329b6c..69b38659a 100644 --- a/packages/runtime-core/src/project/source.rs +++ b/packages/runtime-core/src/project/source.rs @@ -377,6 +377,11 @@ pub fn inventory( )]; let mut entries = Vec::new(); let mut excluded_paths = Vec::new(); + // Ownership declarations may use an alias accepted by a case-insensitive or + // Unicode-normalizing host. Match directory entries by the same collision + // key used for source admission, including currently absent optional files. + let environment_files: BTreeSet<_> = environment_files.iter().map(|p| folded_name(p)).collect(); + let mut excluded_environment_names = BTreeSet::new(); let mut folded = BTreeSet::new(); let mut ignore_files = BTreeMap::new(); let mut visited = 0; @@ -411,7 +416,14 @@ pub fn inventory( "Source names must be valid UTF-8.", ) })?; - if excluded(path) || environment_files.contains(text) { + let environment_file = environment_files.contains(&folded_name(text)); + if environment_file && !excluded_environment_names.insert(folded_name(text)) { + return Err(problem( + "source_case_collision", + "Environment input paths collide under Unicode canonical normalization and case folding.", + )); + } + if excluded(path) || environment_file { excluded_paths.push(text.to_owned()); continue; } @@ -562,6 +574,31 @@ mod tests { assert_ne!(folded_name("src/資料"), folded_name("src/other")); } + #[test] + fn ambiguous_environment_file_aliases_refuse_inventory() { + let root = std::env::temp_dir().join(format!( + "hack-env-alias-{}-{}", + std::process::id(), + crate::next_test_scratch() + )); + fs::create_dir(&root).unwrap(); + fs::write(root.join("Values.txt"), "PRIVATE_UPPER_CANARY").unwrap(); + fs::write(root.join("values.txt"), "PRIVATE_LOWER_CANARY").unwrap(); + let distinct = fs::metadata(root.join("Values.txt")).unwrap().ino() + != fs::metadata(root.join("values.txt")).unwrap().ino(); + let result = inventory(&root, &BTreeSet::from(["VALUES.txt".into()]), &mut vec![]); + fs::remove_dir_all(&root).unwrap(); + if distinct { + let error = result.unwrap_err(); + assert_eq!(error.code, "source_case_collision"); + assert!(!error.message.contains("PRIVATE_")); + } else { + let selection = result.unwrap(); + assert!(selection.entries.is_empty()); + assert_eq!(selection.excluded_paths.len(), 1); + } + } + #[test] fn directory_alias_graph_rejects_ancestry_and_mutual_cycles() { assert!(!acyclic_directories( diff --git a/packages/runtime-core/tests/normalized_cli_contract.rs b/packages/runtime-core/tests/normalized_cli_contract.rs index 72c787a74..f41a5943f 100644 --- a/packages/runtime-core/tests/normalized_cli_contract.rs +++ b/packages/runtime-core/tests/normalized_cli_contract.rs @@ -325,6 +325,28 @@ fn optional_raw_and_repeated_env_files_keep_future_secret_paths_excluded() { } } +#[test] +fn environment_file_ignore_rule_aliases_refuse_without_reading_values() { + let mut fixture = Fixture::new(); + fs::write( + fixture.project.join(".ignore"), + "IGNORE_RULE_PRIVATE_CANARY", + ) + .unwrap(); + for path in [".IGNORE", ".GitIgnore"] { + let original = ORIGINAL.replace( + " image:", + &format!(" env_file: [{{path: {path}, required: false}}]\n image:"), + ); + fs::write(fixture.project.join("compose.yml"), &original).unwrap(); + fixture.hash = format!("{:x}", Sha256::digest(original.as_bytes())); + let output = fixture.invoke(&["project", "plan"], &["--json"]); + assert!(!String::from_utf8_lossy(&output.stdout).contains("PRIVATE_CANARY")); + assert!(!String::from_utf8_lossy(&output.stderr).contains("PRIVATE_CANARY")); + assert_eq!(error(output), "conflicting_environment_owner"); + } +} + #[test] fn removed_inactive_required_env_file_retains_ownership_while_absent() { let mut fixture = Fixture::new(); From abc17763df2161dc430b95aadcb174c9f005d675 Mon Sep 17 00:00:00 2001 From: Kit <344842+roodboi@users.noreply.github.com> Date: Fri, 13 Nov 2026 22:10:30 +0000 Subject: [PATCH 06/15] fix(native): preserve public defaults beside env files --- docs/native/environment-files.md | 4 + src/backends/native-compose-environment.ts | 81 ++++++++++++++-- tests/native-compose-environment.test.ts | 104 +++++++++++++++++++++ 3 files changed, 179 insertions(+), 10 deletions(-) diff --git a/docs/native/environment-files.md b/docs/native/environment-files.md index b880bbf68..21e850013 100644 --- a/docs/native/environment-files.md +++ b/docs/native/environment-files.md @@ -62,6 +62,10 @@ assignment before the next line is read. `.env` alone injects no container varia Service-environment defaults resolve only against the project context: an env-file `HOST` does not replace `${HOST:-localhost}` with its file value. Defaults that reference newly file-backed names are frozen before runtime private ownership. +The same rule applies to public command, entrypoint, user, healthcheck test and +extra-host target defaults: only authored fallback text is frozen, never file +values. Other runtime-bound sources in the same expression refuse rather than +freeze a different value. Public expressions unrelated to file bindings stay intact. File-local cloud names can be used by later assignments without reading those names from the shell. diff --git a/src/backends/native-compose-environment.ts b/src/backends/native-compose-environment.ts index 4132f4d82..63f1b778a 100644 --- a/src/backends/native-compose-environment.ts +++ b/src/backends/native-compose-environment.ts @@ -129,16 +129,18 @@ function fileSpecs(value: unknown, service: string): FileSpec[] { }); } -/** Reject a public expression if resolving it would consult a private source. */ +/** Reject private interpolation; optionally freeze defaults before file names gain ownership. */ function checkPublicFields(opts: { readonly value: unknown; readonly lookup: Sources["lookup"]; readonly service?: string; readonly field?: string; -}): void { + readonly fileBindings?: ReadonlySet; + readonly managedNames?: ReadonlySet; +}): unknown { if (typeof opts.value === "string") { try { - interpolateComposeEnvironment({ + const expanded = interpolateComposeEnvironment({ text: opts.value, lookup: (name) => { if (opts.lookup(name) !== undefined) { @@ -152,6 +154,21 @@ function checkPublicFields(opts: { return undefined; }, }); + const references = composeEnvironmentReferences(opts.value); + if ([...references].some((name) => opts.fileBindings?.has(name))) { + const runtimeName = [...references].find( + (name) => !shellVariableReadable(name) || opts.managedNames?.has(name) + ); + if (runtimeName !== undefined) { + throw new NativeEnvFileRefusal({ + source: "Compose public field", + service: opts.service, + field: opts.field, + reason: `variable ${runtimeName} cannot combine runtime-bound interpolation with a file-backed default`, + }); + } + return expanded.replaceAll("$", () => "$$"); + } } catch (error: unknown) { if (error instanceof NativeEnvFileRefusal) { throw error; @@ -164,20 +181,24 @@ function checkPublicFields(opts: { }); } } else if (Array.isArray(opts.value)) { - for (const value of opts.value) { + return opts.value.map((value) => { if ( opts.field === "volumes" && (nativeAwsCredentialMount(value) || nativeGcloudCredentialMount(value)) ) { - continue; + return value; } - checkPublicFields({ ...opts, value }); - } + return checkPublicFields({ ...opts, value }); + }); } else if (isRecord(opts.value)) { - for (const value of Object.values(opts.value)) { - checkPublicFields({ ...opts, value }); - } + return Object.fromEntries( + Object.entries(opts.value).map(([key, value]) => [ + key, + checkPublicFields({ ...opts, value }), + ]) + ); } + return opts.value; } function ignoredExtension(field: string): boolean { @@ -670,6 +691,11 @@ async function normalizeService(opts: { fileBindings.add(key); } resolveFileBindingDefaults({ environment, resolution, fileBindings }); + resolvePublicFileBindingDefaults({ + document, + context: opts.context, + fileBindings, + }); if (specs.length > 0 || resolution.changed) { document.environment = environment; } @@ -681,6 +707,41 @@ async function normalizeService(opts: { return resolution.delivered; } +/** Only executable fields consumed by the runtime get a frozen public default. */ +function resolvePublicFileBindingDefaults(opts: { + readonly document: Record; + readonly context: ServiceContext; + readonly fileBindings: ReadonlySet; +}): void { + if (opts.fileBindings.size === 0) { + return; + } + const managedNames = new Set(Object.keys(opts.context.managed)); + const prepare = (value: unknown, field: string) => + checkPublicFields({ + value, + field, + service: opts.context.name, + lookup: opts.context.lookup, + fileBindings: opts.fileBindings, + managedNames, + }); + for (const field of ["command", "entrypoint", "user", "extra_hosts"]) { + if (Object.hasOwn(opts.document, field)) { + opts.document[field] = prepare(opts.document[field], field); + } + } + if ( + isRecord(opts.document.healthcheck) && + Object.hasOwn(opts.document.healthcheck, "test") + ) { + opts.document.healthcheck.test = prepare( + opts.document.healthcheck.test, + "healthcheck.test" + ); + } +} + /** * YAML environment interpolation uses the project context, never env-file values. * Freeze a public default before a newly private file name becomes runtime-owned. diff --git a/tests/native-compose-environment.test.ts b/tests/native-compose-environment.test.ts index 517973047..ec56d4c99 100644 --- a/tests/native-compose-environment.test.ts +++ b/tests/native-compose-environment.test.ts @@ -215,6 +215,110 @@ test("a default combined with a private source stays private after file bindings expect(input.normalizedComposeJson).not.toContain(DOTENV); }); +test.each([ + { + field: "command", + authored: [ + "echo", + "${HOST:-localhost}", + "${HOST:+present}", + "${OTHER:+${HOST}}", + "${HOST:-$$LITERAL}", + "${COMPOSE_PROJECT_NAME}", + "${OTHER:-untouched}", + "$$HOST", + ], + expected: [ + "echo", + "localhost", + "", + "", + "$$LITERAL", + "${COMPOSE_PROJECT_NAME}", + "${OTHER:-untouched}", + "$$HOST", + ], + }, + { + field: "command", + authored: "echo ${HOST:-localhost}", + expected: "echo localhost", + }, + { field: "entrypoint", authored: ["${HOST:-echo}"], expected: ["echo"] }, + { field: "entrypoint", authored: "${HOST:-echo}", expected: "echo" }, + { field: "user", authored: "${HOST:-1000}", expected: "1000" }, + { + field: "healthcheck", + authored: { test: ["CMD", "echo", "${HOST:-healthy}"] }, + expected: { test: ["CMD", "echo", "healthy"] }, + }, + { + field: "healthcheck", + authored: { test: "echo ${HOST:-healthy}" }, + expected: { test: "echo healthy" }, + }, + { + field: "extra_hosts", + authored: ["relay.hack:${HOST:-host-gateway}"], + expected: ["relay.hack:host-gateway"], + }, + { + field: "extra_hosts", + authored: { "relay.hack": "${HOST:-host-gateway}" }, + expected: { "relay.hack": "host-gateway" }, + }, +])("public executable defaults use project context before file bindings (%s)", async ({ + field, + authored, + expected, +}) => { + const f = await fixture({ env_file: "service.env", [field]: authored }); + await f.source("service.env", `HOST=${FILE}\n`); + const first = await prepareNativeProjectInput(f); + expect(publicService(first)[field]).toEqual(expected); + expect(first.shellEnvironment).toEqual({ app: { HOST: FILE } }); + expect(first.normalizedComposeJson).not.toContain(FILE); + await f.source("service.env", `HOST=${DOTENV}\n`); + const second = await prepareNativeProjectInput(f); + expect(second.normalizedComposeJson).toBe(first.normalizedComposeJson); + expect(second.shellEnvironment).toEqual({ app: { HOST: DOTENV } }); +}); + +test.each([ + "COMPOSE_PROJECT_NAME", + "AWS_REGION", + "MANAGED", +])("public defaults cannot freeze runtime-bound %s into a different value", async (name) => { + const f = await fixture({ + env_file: "service.env", + command: ["echo", `\${HOST:-localhost}/\${${name}:-fallback}`], + }); + await f.source("service.env", `HOST=${FILE}\n`); + await managed(f.projectDir, { app: { MANAGED } }); + const error: unknown = await prepareNativeProjectInput(f).catch( + (caught: unknown) => caught + ); + expect(String(error)).toContain(name); + expect(String(error)).toContain("runtime-bound"); + expect(String(error)).not.toContain(FILE); + expect(String(error)).not.toContain(MANAGED); +}); + +test("public executable defaults still refuse a defined project source beside a file binding", async () => { + const f = await fixture({ + env_file: "service.env", + command: ["echo", "${HOST:-localhost}/${SOURCE}"], + }); + await f.source("service.env", `HOST=${FILE}\n`); + await f.source(".env", `SOURCE=${DOTENV}\n`); + const error: unknown = await prepareNativeProjectInput(f).catch( + (caught: unknown) => caught + ); + expect(String(error)).toContain("SOURCE"); + expect(String(error)).not.toContain(FILE); + expect(String(error)).not.toContain(DOTENV); +}); + test("env-file-local cloud values resolve without reading cloud shell values", async () => { const f = await fixture({ env_file: ["one.env", "two.env"] }); await f.source( From 434dc0cd9f0ae71a1babeceac302ad371dd5337d Mon Sep 17 00:00:00 2001 From: Kit <344842+roodboi@users.noreply.github.com> Date: Fri, 13 Nov 2026 22:13:26 +0000 Subject: [PATCH 07/15] fix(native): classify source path collisions as preflight blockers --- src/backends/native-preflight.ts | 6 ++++++ tests/native-preflight-cli.test.ts | 28 ++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/src/backends/native-preflight.ts b/src/backends/native-preflight.ts index 565acd171..a29c8f828 100644 --- a/src/backends/native-preflight.ts +++ b/src/backends/native-preflight.ts @@ -286,6 +286,11 @@ const RUNTIME_FINDINGS: Readonly> = { message: "A mount source is outside the project.", fix: "Mount only paths inside the project root.", }, + source_case_collision: { + message: + "Project source paths collide under Unicode normalization and case folding.", + fix: "Rename the conflicting files or directories so their normalized, case-folded paths are distinct, then correct any env_file references.", + }, unresolved_source_path: { message: "A mount source cannot be resolved inside the project.", fix: "Use a plain path relative to the project.", @@ -514,6 +519,7 @@ export const REVIEW_STOP_CODES: ReadonlySet = new Set([ "registry_template_unsafe", "service_budget", "source_budget", + "source_case_collision", "source_escape", "source_generated", "source_limit_config", diff --git a/tests/native-preflight-cli.test.ts b/tests/native-preflight-cli.test.ts index 8f469c508..dcd1d7283 100644 --- a/tests/native-preflight-cli.test.ts +++ b/tests/native-preflight-cli.test.ts @@ -550,6 +550,34 @@ test( SPAWN_TIMEOUT_MS ); +test( + "ambiguous env-file aliases are a preflight blocker in JSON and text", + async () => { + const f = await fixture({ services: { web: { image: IMAGE } } }); + await f.reviewStops("source_case_collision"); + const env = { ...f.native, ...GRANT }; + const result = await preflight(f, env); + expect(result.code).toBe(1); + expect(result.report.ready).toBe(false); + expect(result.report.runtime_review).toEqual({ + status: "stopped", + code: "source_case_collision", + }); + expect(result.report.blockers).toHaveLength(1); + expect(result.report.blockers[0]).toMatchObject({ + gate: "runtime", + code: "source_case_collision", + }); + expect(result.report.blockers[0]?.fix).toContain("Rename"); + const text = await cli(f, ["doctor", "--native-preflight"], env); + expect(text.code).toBe(1); + expect(text.output).toContain("Unicode normalization and case folding"); + expect(text.output).toContain("Rename"); + expect(text.output).not.toContain(PRIVATE); + }, + SPAWN_TIMEOUT_MS +); + test( "a review that stops early is reported as a blocker, and one that fails is reported as unavailable", async () => { From 065cec44487ae2a8bb0158b65c5fafdbba740b1a Mon Sep 17 00:00:00 2001 From: Kit <344842+roodboi@users.noreply.github.com> Date: Fri, 13 Nov 2026 22:22:59 +0000 Subject: [PATCH 08/15] fix(native): preserve extra hosts hostname defaults --- docs/native/environment-files.md | 5 +- src/backends/native-compose-environment.ts | 33 ++++++++-- tests/native-compose-environment.test.ts | 74 ++++++++++++++++++++++ 3 files changed, 105 insertions(+), 7 deletions(-) diff --git a/docs/native/environment-files.md b/docs/native/environment-files.md index 21e850013..7285e7269 100644 --- a/docs/native/environment-files.md +++ b/docs/native/environment-files.md @@ -63,9 +63,12 @@ Service-environment defaults resolve only against the project context: an env-fi `HOST` does not replace `${HOST:-localhost}` with its file value. Defaults that reference newly file-backed names are frozen before runtime private ownership. The same rule applies to public command, entrypoint, user, healthcheck test and -extra-host target defaults: only authored fallback text is frozen, never file +extra-host hostname/target defaults: only authored fallback text is frozen, never file values. Other runtime-bound sources in the same expression refuse rather than freeze a different value. Public expressions unrelated to file bindings stay intact. +Native already evaluates `extra_hosts` hostnames in both list and map forms; +map keys receive the same private-source admission, and duplicate frozen keys refuse. +Other mapping keys retain Compose's ordinary key semantics. File-local cloud names can be used by later assignments without reading those names from the shell. diff --git a/src/backends/native-compose-environment.ts b/src/backends/native-compose-environment.ts index 63f1b778a..ff7cfa333 100644 --- a/src/backends/native-compose-environment.ts +++ b/src/backends/native-compose-environment.ts @@ -191,12 +191,33 @@ function checkPublicFields(opts: { return checkPublicFields({ ...opts, value }); }); } else if (isRecord(opts.value)) { - return Object.fromEntries( - Object.entries(opts.value).map(([key, value]) => [ - key, - checkPublicFields({ ...opts, value }), - ]) - ); + const result: Record = Object.create(null); + for (const [key, value] of Object.entries(opts.value)) { + // Native executes extra_hosts hostnames as well as targets. Other + // mapping keys keep their ordinary Compose key semantics. + const preparedKey = + opts.field === "extra_hosts" + ? checkPublicFields({ + ...opts, + value: key, + field: "extra_hosts.hostname", + }) + : key; + if ( + typeof preparedKey !== "string" || + Object.hasOwn(result, preparedKey) + ) { + throw new NativeEnvFileRefusal({ + source: "Compose public field", + service: opts.service, + field: opts.field, + reason: + "extra_hosts hostnames must remain distinct after interpolation; values omitted", + }); + } + result[preparedKey] = checkPublicFields({ ...opts, value }); + } + return result; } return opts.value; } diff --git a/tests/native-compose-environment.test.ts b/tests/native-compose-environment.test.ts index ec56d4c99..f0383571f 100644 --- a/tests/native-compose-environment.test.ts +++ b/tests/native-compose-environment.test.ts @@ -267,6 +267,16 @@ test.each([ authored: { "relay.hack": "${HOST:-host-gateway}" }, expected: { "relay.hack": "host-gateway" }, }, + { + field: "extra_hosts", + authored: { "${HOST:-relay.hack}": "host-gateway" }, + expected: { "relay.hack": "host-gateway" }, + }, + { + field: "extra_hosts", + authored: { "${OTHER:+${HOST}}relay.hack": "host-gateway" }, + expected: { "relay.hack": "host-gateway" }, + }, ])("public executable defaults use project context before file bindings (%s)", async ({ field, authored, @@ -284,6 +294,70 @@ test.each([ expect(second.shellEnvironment).toEqual({ app: { HOST: DOTENV } }); }); +test("extra_hosts keys that freeze to one hostname refuse without echoing file values", async () => { + const f = await fixture({ + env_file: "service.env", + extra_hosts: { + "${HOST:-relay.hack}": "host-gateway", + "relay.hack": "host-gateway", + }, + }); + await f.source("service.env", `HOST=${FILE}\n`); + const error: unknown = await prepareNativeProjectInput(f).catch( + (caught: unknown) => caught + ); + expect(String(error)).toContain("hostnames must remain distinct"); + expect(String(error)).not.toContain(FILE); +}); + +test("extra_hosts hostname interpolation refuses project values and runtime-bound compounds", async () => { + const f = await fixture({ + env_file: "service.env", + extra_hosts: { "${HOST:-relay.hack}": "host-gateway" }, + }); + await f.source("service.env", `HOST=${FILE}\n`); + const projectError: unknown = await prepareNativeProjectInput({ + ...f, + env: { HOST: SHELL }, + }).catch((caught: unknown) => caught); + expect(String(projectError)).toContain("variable HOST"); + expect(String(projectError)).not.toContain(SHELL); + expect(String(projectError)).not.toContain(FILE); + await writeFile( + f.composeFile, + JSON.stringify({ + services: { + app: { + image: "alpine", + env_file: "service.env", + extra_hosts: { + "${HOST:-relay}.${COMPOSE_PROJECT_NAME}.hack": "host-gateway", + }, + }, + }, + }) + ); + const runtimeError: unknown = await prepareNativeProjectInput(f).catch( + (caught: unknown) => caught + ); + expect(String(runtimeError)).toContain("COMPOSE_PROJECT_NAME"); + expect(String(runtimeError)).toContain("runtime-bound"); + expect(String(runtimeError)).not.toContain(FILE); +}); + +test("other public mapping keys keep their ordinary Compose semantics", async () => { + const f = await fixture({ + env_file: "service.env", + labels: { "${HOST:-literal-label}": "public" }, + }); + await f.source("service.env", `HOST=${FILE}\n`); + const input = await prepareNativeProjectInput(f); + expect(publicService(input).labels).toEqual({ + "${HOST:-literal-label}": "public", + }); + expect(input.normalizedComposeJson).not.toContain(FILE); +}); + test.each([ "COMPOSE_PROJECT_NAME", "AWS_REGION", From 8c631c44ce5a8c0e4a2bb94b57d3af12ea03d01f Mon Sep 17 00:00:00 2001 From: Kit <344842+roodboi@users.noreply.github.com> Date: Fri, 13 Nov 2026 22:50:52 +0000 Subject: [PATCH 09/15] fix(native): keep environment sources out of build inputs --- docs/native/environment-files.md | 10 ++ src/backends/native-compose-environment.ts | 41 +++++++- src/backends/native-env-build-context.ts | 98 ++++++++++++++++++ tests/native-compose-environment.test.ts | 115 +++++++++++++++++++++ tests/native-preflight-cli.test.ts | 40 ++++++- 5 files changed, 302 insertions(+), 2 deletions(-) create mode 100644 src/backends/native-env-build-context.ts diff --git a/docs/native/environment-files.md b/docs/native/environment-files.md index 7285e7269..9db2a27e4 100644 --- a/docs/native/environment-files.md +++ b/docs/native/environment-files.md @@ -123,6 +123,16 @@ stay on the existing public path and retain their restart behavior and identity. Fresh `hack exec` resolves the selected service's file values again using the same allowed caller context and delivers them privately to that exec process. These fresh values may differ from the running container's startup values. + +Native builds read raw host contexts. Start and preflight therefore refuse a +native build whose context, Dockerfile or Dockerfile-specific ignore input +overlaps any declared env-file path or the selected automatic `.env` path. +Optional absent and inactive-service declarations remain protected; authored +ignore rules do not lift this refusal. ASCII paths compare without case, and +non-ASCII components are treated as potentially aliased whenever disjointness +cannot be proved. Use separate build inputs, a prebuilt image, or Compose for +overlapping projects. Disjoint contexts and ordinary builds with no env-file or +dotenv input retain their behavior. Fresh selected-service exec does not build. Managed-env stale checks remain advisory and do not read ephemeral file inputs; file-value-only changes are not detected by their managed fingerprints. diff --git a/src/backends/native-compose-environment.ts b/src/backends/native-compose-environment.ts index ff7cfa333..c498eb73e 100644 --- a/src/backends/native-compose-environment.ts +++ b/src/backends/native-compose-environment.ts @@ -8,6 +8,7 @@ import { interpolateComposeEnvironment, parseComposeEnvironmentFile, } from "./native-compose-env-syntax.ts"; +import { requireNativeEnvBuildSeparation } from "./native-env-build-context.ts"; import { NativeEnvFileRefusal, nativeEnvFileSourceLocation, @@ -53,6 +54,7 @@ type Sources = { readonly read: (spec: FileSpec, service?: string) => Promise; readonly dotenv: ReadonlyMap; readonly shellNames: Set; + readonly automaticDotenv: string | undefined; }; type Notice = { readonly service: string; readonly line: string }; @@ -375,6 +377,7 @@ async function environmentSources(opts: Options): Promise { return { shellNames, dotenv, + automaticDotenv: cached.has(automatic) ? automatic : undefined, read, lookup: (name) => baseLookup(name) ?? dotenv.get(name), }; @@ -784,6 +787,40 @@ function resolveFileBindingDefaults(opts: { } } +/** Fresh exec acquires selected-service values but cannot initiate a build. */ +function requireBuildSeparation( + opts: Options & { readonly sources: Sources } +): void { + if (opts.selectedServices === undefined) { + const privatePaths = Object.entries(opts.services).flatMap( + ([service, value]) => + fileSpecs(value.document.env_file, service).map( + (spec) => + nativeEnvFileSourceLocation({ + ...opts, + path: spec.path, + service, + }).path + ) + ); + if (privatePaths.length > 0 || opts.sources.automaticDotenv !== undefined) { + // Retain absent optional declarations and the automatic dotenv location. + // Nothing is read by this guard; ignore patterns do not grant exclusion. + privatePaths.push(join(dirname(opts.composeFile), ".env")); + requireNativeEnvBuildSeparation({ + ...opts, + services: Object.fromEntries( + Object.entries(opts.services).map(([name, value]) => [ + name, + value.document, + ]) + ), + privatePaths, + }); + } + } +} + /** * Native-only normalization. Public declarations contain names, never file or * shell values. Returned maps are private ephemeral inputs, not managed-env @@ -798,7 +835,9 @@ export async function normalizeNativeComposeEnvironment( readonly privateValues: readonly string[]; readonly fileServices: Readonly>; }> { - const sources = await environmentSources(await canonicalSourceOptions(opts)); + const canonical = await canonicalSourceOptions(opts); + const sources = await environmentSources(canonical); + requireBuildSeparation({ ...canonical, sources }); const privateValues = new Set(); for (const [field, value] of Object.entries(opts.compose)) { if (field !== "services" && !ignoredExtension(field)) { diff --git a/src/backends/native-env-build-context.ts b/src/backends/native-env-build-context.ts new file mode 100644 index 000000000..7866ff89b --- /dev/null +++ b/src/backends/native-env-build-context.ts @@ -0,0 +1,98 @@ +import { dirname, relative, resolve, sep } from "node:path"; +import { HackCliError } from "../lib/cli-result.ts"; +import { NativeEnvFileRefusal } from "./native-env-file-source.ts"; +import { + admitNativeProjectBuild, + type NativeBuildRequest, +} from "./native-project-builds.ts"; + +const ASCII = /^[\x00-\x7f]*$/; + +/** + * Prove disjointness without consulting a mutable host directory. ASCII names + * compare case-insensitively. A non-ASCII component is conservatively ambiguous, + * covering every host Unicode alias without implementing another case-fold table. + */ +function maySharePrefix( + parent: readonly string[], + child: readonly string[] +): boolean { + return ( + parent.length <= child.length && + parent.every((part, index) => { + const other = child[index] ?? ""; + return ( + !(ASCII.test(part) && ASCII.test(other)) || + part.toLowerCase() === other.toLowerCase() + ); + }) + ); +} + +/** The project-relative components of a literal, already bounded source path. */ +function parts(root: string, path: string): string[] { + return relative(root, path) + .split(sep) + .filter((part) => part !== "" && part !== "."); +} + +/** + * Buildx reads raw host contexts and may use a Dockerfile-specific ignore file. + * Until builds can carry enforced private exclusions, reject overlapping inputs + * before any hook/provider/build effect, regardless of authored ignore patterns. + */ +export function requireNativeEnvBuildSeparation(opts: { + readonly projectRoot: string; + readonly composeFile: string; + readonly services: Readonly< + Record>> + >; + readonly privatePaths: readonly string[]; +}): void { + const root = resolve(opts.projectRoot); + const privatePaths = opts.privatePaths.map((path) => ({ + path, + parts: parts(root, path), + })); + for (const [name, service] of Object.entries(opts.services)) { + let request: NativeBuildRequest | undefined; + try { + request = admitNativeProjectBuild({ ...opts, name, service }); + } catch (error: unknown) { + // The dedicated build admission already refuses unsupported declarations. + if ( + error instanceof HackCliError && + error.code === "E_NATIVE_PROJECT_UNSUPPORTED" + ) { + continue; + } + throw error; + } + if (request === undefined) { + continue; + } + const context = parts(root, resolve(root, request.context)); + const dockerfile = resolve(root, request.context, request.dockerfile); + const definitions = [dockerfile, `${dockerfile}.dockerignore`].map((path) => + parts(root, path) + ); + const overlap = privatePaths.find( + (source) => + maySharePrefix(context, source.parts) || + definitions.some( + (definition) => + definition.length === source.parts.length && + maySharePrefix(definition, source.parts) + ) + ); + if (overlap !== undefined) { + throw new NativeEnvFileRefusal({ + service: name, + field: "build", + source: relative(dirname(opts.composeFile), overlap.path), + reason: + "native build inputs overlap an environment source; use disjoint build inputs, a prebuilt image, or the Compose backend", + }); + } + } +} diff --git a/tests/native-compose-environment.test.ts b/tests/native-compose-environment.test.ts index f0383571f..77dcfba71 100644 --- a/tests/native-compose-environment.test.ts +++ b/tests/native-compose-environment.test.ts @@ -658,6 +658,121 @@ test("only delivered private values join the build disclosure guard", async () = expect(input.managedSecretValues).toEqual([FILE]); }); +test.each([ + { context: "..", path: "service.env" }, + { context: ".", path: "service.env" }, + { context: "../APP", path: "../app/secret.vars" }, + { context: "../STRASSE", path: "../Straße/secret.vars" }, + { context: "../café", path: "../cafe\u0301/secret.vars" }, + { context: "../app", path: "../app/nested/secret.vars" }, +])("build contexts cannot overlap an optional env-file path (%s)", async ({ + context, + path, +}) => { + const f = await fixture( + {}, + { + services: { + app: { + image: "alpine", + env_file: { path, required: false }, + profiles: ["inactive"], + }, + builder: { build: { context } }, + }, + } + ); + const error: unknown = await prepareNativeProjectInput(f).catch( + (caught: unknown) => caught + ); + expect(String(error)).toContain( + "native build inputs overlap an environment source" + ); + expect(String(error)).toContain("service builder field build"); + expect(String(error)).not.toContain(FILE); +}); + +test.each([ + "../.hack/service.env", + "../.hack/service.env.dockerignore", +])("build definition inputs outside the context cannot read env-file %s", async (path) => { + const f = await fixture( + {}, + { + services: { + app: { + image: "alpine", + env_file: { + path: path.endsWith(".dockerignore") + ? "service.env.dockerignore" + : "service.env", + required: false, + }, + }, + builder: { + build: { context: "../app", dockerfile: "../.hack/service.env" }, + }, + }, + } + ); + const error: unknown = await prepareNativeProjectInput(f).catch( + (caught: unknown) => caught + ); + expect(String(error)).toContain( + "native build inputs overlap an environment source" + ); +}); + +test("a selected dotenv cannot enter the raw build context even with no delivered key", async () => { + const f = await fixture({}, { services: { builder: { build: ".." } } }); + await f.source(".env", `# ${DOTENV}\n`); + const error: unknown = await prepareNativeProjectInput(f).catch( + (caught: unknown) => caught + ); + expect(String(error)).toContain("overlap an environment source"); + expect(String(error)).not.toContain(DOTENV); +}); + +test("env-file sources stay private with disjoint native build inputs", async () => { + const f = await fixture( + {}, + { + services: { + app: { image: "alpine", env_file: "service.env" }, + builder: { build: { context: "../app", dockerfile: "Dockerfile" } }, + }, + } + ); + await f.source("service.env", `TOKEN=${FILE}\n`); + await f.source(".env", `UNUSED=${DOTENV}\n`); + const input = await prepareNativeProjectInput(f); + expect(input.shellEnvironment).toEqual({ app: { TOKEN: FILE } }); + expect(publicService(input).environment).toEqual({ TOKEN: null }); + expect(input.normalizedComposeJson).not.toContain(FILE); + expect(input.normalizedComposeJson).not.toContain(DOTENV); +}); + +test("source-free builds and selected-service exec retain their existing behavior", async () => { + const plain = await fixture({}, { services: { builder: { build: ".." } } }); + expect((await prepareNativeProjectInput(plain)).shellEnvironment).toEqual({}); + const f = await fixture( + {}, + { + services: { + app: { image: "alpine", env_file: "service.env" }, + builder: { build: ".." }, + }, + } + ); + await f.source("service.env", `TOKEN=${FILE}\n`); + const input = await prepareNativeProjectInput({ + ...f, + selectedServices: ["app"], + }); + expect(input.shellEnvironment).toEqual({ app: { TOKEN: FILE } }); + expect(input.normalizedComposeJson).not.toContain(FILE); +}); + test("authored public runtime metadata literals survive while file/shell copies stay refused", async () => { const f = await fixture({ environment: { HACK_SERVICE_URL: "author-owned" }, diff --git a/tests/native-preflight-cli.test.ts b/tests/native-preflight-cli.test.ts index dcd1d7283..67114a70e 100644 --- a/tests/native-preflight-cli.test.ts +++ b/tests/native-preflight-cli.test.ts @@ -306,6 +306,44 @@ test("preflight and static start refuse oversized file delivery before pool effe ).toBe(true); }); +test( + "preflight and start refuse private sources in native build contexts before effects", + async () => { + const f = await fixture({ + services: { + web: { image: IMAGE, env_file: "service.env" }, + builder: { build: ".." }, + }, + }); + await chmod(f.project, 0o700); + await writeFile(join(f.projectDir, "service.env"), `TOKEN=${PRIVATE}\n`, { + mode: 0o600, + }); + await f.review([]); + const env = { ...f.native, ...GRANT, HACK_NATIVE_BUILDS: "1" }; + const report = await preflight(f, env); + expect(report.code).toBe(1); + expect(report.report.ready).toBe(false); + expect(codes(report.report, "cli")).toContain( + "environment_file_input:builder" + ); + expect(report.output).toContain("disjoint build inputs"); + const before = await f.calls(); + const up = await cli(f, ["up"], env); + expect(up.code).toBe(1); + expect(up.output).toContain("overlap an environment source"); + expect((await f.calls()).slice(before.length)).toEqual([]); + expect(report.output).not.toContain(PRIVATE); + expect(up.output).not.toContain(PRIVATE); + expect((await f.argv()).join("\n")).not.toContain(PRIVATE); + const detached = await cli(f, ["up", "--json"], env); + expect(detached.code).toBe(1); + expect(detached.output).not.toContain(PRIVATE); + expect((await f.calls()).slice(before.length)).toEqual([]); + }, + SPAWN_TIMEOUT_MS +); + test("preflight requires the same decryption key as start and closes key errors", async () => { const f = await fixture({ services: { web: { image: IMAGE } } }); await writeFile( @@ -419,7 +457,7 @@ test( expect(report.runtime_review).toEqual({ status: "completed" }); expect(codes(report, "cli")).toEqual([ "build_requires_opt_in:web", - "environment_file_input:api", + "environment_file_input:web", "external_network", "extra_hosts_without_dependencies:web", "invalid_restart:jobs", From 20cf8d4be9ed7e115ff5e05b39e4557bccd7e11a Mon Sep 17 00:00:00 2001 From: Kit <344842+roodboi@users.noreply.github.com> Date: Sat, 14 Nov 2026 03:41:29 +0000 Subject: [PATCH 10/15] docs(native): pin links to upcoming environment guidance --- docs/versions/v5/native/preflight.md | 4 ++-- docs/versions/v5/native/start.md | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/versions/v5/native/preflight.md b/docs/versions/v5/native/preflight.md index ebeec82fd..7b7c6c6b0 100644 --- a/docs/versions/v5/native/preflight.md +++ b/docs/versions/v5/native/preflight.md @@ -23,8 +23,8 @@ decrypts managed env and reads `.env` beside Compose (normally `.hack/.env`), using the same file checks as native start. A missing decryption key is a structured blocker with recovery guidance; the command still produces a report. A project-root `.env` outside the Compose directory is ignored with one notice. -See the [environment-file rules](../../../native/environment-files.md) and -[migration notes](../../../guides/v5-release-notes.md#next17-what-changed-and-what-to-do) +See the [environment-file rules](https://github.com/hack-dance/hack/blob/e1fa21b7d16d1b2ba2c5aee4ae52ea44f564c087/docs/native/environment-files.md) and +[migration notes](https://github.com/hack-dance/hack/blob/e1fa21b7d16d1b2ba2c5aee4ae52ea44f564c087/docs/guides/v5-release-notes.md#next17-what-changed-and-what-to-do) before upgrading an earlier prerelease. ## Read the result diff --git a/docs/versions/v5/native/start.md b/docs/versions/v5/native/start.md index a479cda05..ff6524e14 100644 --- a/docs/versions/v5/native/start.md +++ b/docs/versions/v5/native/start.md @@ -70,8 +70,8 @@ to force a start. Keep the project on Compose if it needs unsupported behavior. See [Troubleshooting](../troubleshooting.md) and [the preflight reference](preflight.md). Env-file support intended for next.17 changes input checks and shell-default -selection. Before upgrading, read the [environment-file rules](../../../native/environment-files.md) -and [migration notes](../../../guides/v5-release-notes.md#next17-what-changed-and-what-to-do). +selection. Before upgrading, read the [environment-file rules](https://github.com/hack-dance/hack/blob/e1fa21b7d16d1b2ba2c5aee4ae52ea44f564c087/docs/native/environment-files.md) +and [migration notes](https://github.com/hack-dance/hack/blob/e1fa21b7d16d1b2ba2c5aee4ae52ea44f564c087/docs/guides/v5-release-notes.md#next17-what-changed-and-what-to-do). They explain which files are read, which declarations refuse and how to fix them. ## Start, check and stop From 9cd2053affe4f3307703e97d5bb4cd7fc3049830 Mon Sep 17 00:00:00 2001 From: Kit <344842+roodboi@users.noreply.github.com> Date: Sat, 14 Nov 2026 04:01:11 +0000 Subject: [PATCH 11/15] fix(native): keep ignored root dotenv out of build inputs --- docs/guides/native-candidate.md | 3 ++ docs/guides/v5-release-notes.md | 2 +- docs/native/environment-files.md | 9 ++++- src/backends/native-compose-environment.ts | 19 +++++++--- tests/native-compose-environment.test.ts | 43 ++++++++++++++++++++++ tests/native-preflight-cli.test.ts | 29 +++++++++++++++ 6 files changed, 98 insertions(+), 7 deletions(-) diff --git a/docs/guides/native-candidate.md b/docs/guides/native-candidate.md index 6f7140d6f..648dbc57c 100644 --- a/docs/guides/native-candidate.md +++ b/docs/guides/native-candidate.md @@ -1555,6 +1555,9 @@ every native start and preflight, even with managed env. Symlinks, other-owner o group/other-writable files and project ancestors refuse, as do `COMPOSE_*` and `HACK_*` entries and dotted/dashed identifiers. Use an owned regular file, deliberately correct permissions and rename unsupported keys, or use managed env. +An ignored root `.env` still may not overlap a native build context or +Dockerfile input: Buildx reads raw host files. Such a build now refuses even +without `env_file` or adjacent `.env`; use disjoint inputs or a prebuilt image. | Source | Precedence and delivery | | --- | --- | diff --git a/docs/guides/v5-release-notes.md b/docs/guides/v5-release-notes.md index ceeebde0d..04f02ac2a 100644 --- a/docs/guides/v5-release-notes.md +++ b/docs/guides/v5-release-notes.md @@ -26,7 +26,7 @@ its original behavior. | **Native service-environment defaults take allowed shell values.** Shell presence can change normalized declarations and plan identity, without storing values in that identity. | Unset the exported name to retain the authored fallback. In projects with `env_file` or adjacent `.env`, public-field references to ambient values now refuse even with a default: unset the source or write a public literal. Without those file sources, public defaults keep their previous behavior. | | **Native removes `dotenv_without_managed_env`.** A project-root `.env` outside the Compose directory is now ignored with one names-only notice. | Move required inputs beside Compose (normally `.hack/.env`) or into managed env. Dotenv alone does not inject service variables. | | **Native preflight reads the caller's allowed environment and decrypts managed env.** A missing key becomes a structured blocker; the report remains usable. | Run from the intended shell and recover the existing project key when asked. Do not generate a replacement key for existing ciphertext. | -| **Native env-file paths must stay separate from build inputs.** Contexts, Dockerfiles and Dockerfile-specific ignore inputs may not overlap declared env files or adjacent `.env`, including optional and inactive paths. | Use disjoint build inputs or prebuilt images. The build-argument value guard stays limited to nonempty managed `secret: true` values, with no minimum length; plain file and shell values do not widen it. See [the full rules](../native/environment-files.md). | +| **Native env-file paths must stay separate from build inputs.** Contexts, Dockerfiles and Dockerfile-specific ignore inputs may not overlap declared env files, adjacent `.env` or a present project-root `.env`, including optional and inactive paths. A root `.env` triggers this refusal even when there is no `env_file` or adjacent `.env`; ignored for interpolation does not exclude it from `COPY .`. | Use disjoint build inputs or prebuilt images. The build-argument value guard stays limited to nonempty managed `secret: true` values, with no minimum length; plain file and shell values do not widen it. See [the full rules](../native/environment-files.md). | | **Native plain `logs SERVICE --follow` output changes.** It now prints application-text lines, matching a finite snapshot, instead of adding a `SERVICE \| ` prefix. Both service streams go to stdout; Hack notices go to stderr. | Stop parsing the old plain prefix. Use `--json` and the log event's `entry.service`, `entry.stream` and `entry.message` when a script needs those fields. Use `--timestamps` to request Engine timestamps. | | **Env listings hide values by default, including migrated plaintext.** Ordinary `env` / `env list` text and JSON output are metadata. JSON omits `value` and reports `has_value`. | Use `has_value` for availability checks. For a value a script actually needs, call `hack env get KEY` with the same `--env` and `--service`; that command prints plaintext. Bulk `--show-values` requires interactive stdin and stdout; `--show-secrets` remains an alias. | | **Migration cleanup preserves live keys (HACK-2172).** A legacy store's `keyPath` can no longer make the current project/shared key a cleanup target. Current config and YAML inputs are protected too, including filesystem aliases. | Keep key and env backups before migration. On next.16 or an earlier prerelease, do not run `doctor --migrate-env-config` without a private readable copy of `.hack.secret.key`. The fix does not recover a key already removed. | diff --git a/docs/native/environment-files.md b/docs/native/environment-files.md index bce11bd23..75002e930 100644 --- a/docs/native/environment-files.md +++ b/docs/native/environment-files.md @@ -142,7 +142,12 @@ These fresh values may differ from the running container's startup values. Native builds read raw host contexts. Start and preflight therefore refuse a native build whose context, Dockerfile or Dockerfile-specific ignore input -overlaps any declared env-file path or the selected automatic `.env` path. +overlaps any declared env-file path, the selected automatic `.env` path or a +present project-root `.env`. Ignoring root dotenv for interpolation does not +exclude it from Buildx's raw context: even with no `env_file` or adjacent `.env`, +a root-context build refuses when root `.env` exists. This is an explicit +compatibility exception to prevent `COPY .` from embedding ignored dotenv. +Root dotenv is detected by presence only, without reading its contents. Optional absent and inactive-service declarations remain protected; authored ignore rules do not lift this refusal. ASCII paths compare without case, and non-ASCII components are treated as potentially aliased whenever disjointness @@ -188,6 +193,8 @@ Actual guest delivery is qualified separately on a Mac. - The former `dotenv_without_managed_env` refusal is removed. A project-root `.env` outside the Compose directory is ignored with one notice. Move needed variables beside Compose (normally `.hack/.env`) or into managed env. + Builds overlapping this ignored root file now refuse; use disjoint build + inputs or a prebuilt image, even when the project has no other env-file input. - Preflight now reads the caller's allowed environment and decrypts managed env. Recover the existing project key when reported missing; do not generate a replacement for existing ciphertext. diff --git a/src/backends/native-compose-environment.ts b/src/backends/native-compose-environment.ts index 5387566b9..338fea1c7 100644 --- a/src/backends/native-compose-environment.ts +++ b/src/backends/native-compose-environment.ts @@ -868,7 +868,10 @@ function resolveFileBindingDefaults(opts: { /** Fresh exec acquires selected-service values but cannot initiate a build. */ function requireBuildSeparation( - opts: Options & { readonly sources: Sources } + opts: Options & { + readonly sources: Sources; + readonly ignoredRootDotenv: boolean; + } ): void { if (opts.selectedServices === undefined) { const privatePaths = Object.entries(opts.services).flatMap( @@ -886,6 +889,12 @@ function requireBuildSeparation( // Retain absent optional declarations and the automatic dotenv location. // Nothing is read by this guard; ignore patterns do not grant exclusion. privatePaths.push(join(dirname(opts.composeFile), ".env")); + } + if (opts.ignoredRootDotenv) { + // Ignored for interpolation does not make the raw Buildx context safe. + privatePaths.push(join(opts.projectRoot, ".env")); + } + if (privatePaths.length > 0) { requireNativeEnvBuildSeparation({ ...opts, services: Object.fromEntries( @@ -917,15 +926,15 @@ export async function normalizeNativeComposeEnvironment( const canonical = await canonicalSourceOptions(opts); const notices: Notice[] = []; const sources = await environmentSources(canonical, notices); - requireBuildSeparation({ ...canonical, sources }); + const ignoredRootDotenv = + dirname(canonical.composeFile) !== canonical.projectRoot && + (await present(join(canonical.projectRoot, ".env"))); + requireBuildSeparation({ ...canonical, sources, ignoredRootDotenv }); const filePrivacy = sources.automaticDotenv !== undefined || Object.values(opts.services).some((service) => Object.hasOwn(service.document, "env_file") ); - const ignoredRootDotenv = - dirname(canonical.composeFile) !== canonical.projectRoot && - (await present(join(canonical.projectRoot, ".env"))); if (ignoredRootDotenv) { notices.push({ line: NATIVE_ROOT_DOTENV_IGNORED_NOTICE }); } diff --git a/tests/native-compose-environment.test.ts b/tests/native-compose-environment.test.ts index 12a4efda9..2dff99693 100644 --- a/tests/native-compose-environment.test.ts +++ b/tests/native-compose-environment.test.ts @@ -991,6 +991,49 @@ test("env-file sources stay private with disjoint native build inputs", async () expect(input.normalizedComposeJson).not.toContain(DOTENV); }); +test.each([ + { context: ".." }, + { context: "../app", dockerfile: "../.env" }, +])("ignored root dotenv stays out of native build inputs %s", async (build) => { + const f = await fixture({}, { services: { builder: { build } } }); + // No env_file or adjacent .env: this is the explicitly approved root-only exception. + await writeFile(join(f.projectRoot, ".env"), `TOKEN=${FILE}\n`, { + mode: 0o000, + }); + const error: unknown = await prepareNativeProjectInput(f).catch( + (caught: unknown) => caught + ); + expect(error).toBeInstanceOf(Error); + expect(String(error)).toContain('source "../.env"'); + expect(String(error)).toContain("service builder field build"); + expect(String(error)).toContain("overlap an environment source"); + expect(String(error)).not.toContain(FILE); +}); + +test("ignored root dotenv preserves disjoint builds and selected-service exec without reading its values", async () => { + const f = await fixture( + {}, + { services: { builder: { build: { context: "../app" } } } } + ); + await writeFile(join(f.projectRoot, ".env"), `TOKEN=${FILE}\n`, { + mode: 0o000, + }); + const input = await prepareNativeProjectInput(f); + expect(input.ignoredRootDotenv).toBe(true); + expect(input.shellEnvironment).toEqual({}); + expect(JSON.stringify(input.environmentNotices)).not.toContain(FILE); + await writeFile( + f.composeFile, + JSON.stringify({ services: { builder: { build: ".." } } }) + ); + const exec = await prepareNativeProjectInput({ + ...f, + selectedServices: ["builder"], + }); + expect(exec.ignoredRootDotenv).toBe(true); + expect(exec.shellEnvironment).toEqual({}); +}); + test("source-free builds and selected-service exec retain their existing behavior", async () => { const plain = await fixture({}, { services: { builder: { build: ".." } } }); expect((await prepareNativeProjectInput(plain)).shellEnvironment).toEqual({}); diff --git a/tests/native-preflight-cli.test.ts b/tests/native-preflight-cli.test.ts index 1dec3e123..c717931ad 100644 --- a/tests/native-preflight-cli.test.ts +++ b/tests/native-preflight-cli.test.ts @@ -344,6 +344,35 @@ test( SPAWN_TIMEOUT_MS ); +test("preflight and start refuse ignored root dotenv in a native build before effects", async () => { + const f = await fixture({ services: { builder: { build: ".." } } }); + await writeFile(join(f.project, ".env"), `TOKEN=${PRIVATE}\n`, { + mode: 0o000, + }); + await f.review([]); + const env = { ...f.native, ...GRANT, HACK_NATIVE_BUILDS: "1" }; + const report = await preflight(f, env); + expect(report.code).toBe(1); + expect(report.report.ready).toBe(false); + expect(codes(report.report, "cli")).toContain( + "environment_file_input:builder" + ); + expect( + report.report.blockers.find( + (entry) => entry.code === "environment_file_input" + )?.message + ).toContain('source "../.env"'); + expect(report.output).not.toContain(PRIVATE); + const before = await f.calls(); + for (const args of [["up"], ["up", "--json"]]) { + const started = await cli(f, args, env); + expect(started.code).toBe(1); + expect(started.output).toContain("overlap an environment source"); + expect(started.output).not.toContain(PRIVATE); + expect((await f.calls()).slice(before.length)).toEqual([]); + } +}); + test("preflight requires the same decryption key as start and closes key errors", async () => { const f = await fixture({ services: { web: { image: IMAGE } } }); await writeFile( From 08c031a242d1828575c47d674aa490c3c30b851d Mon Sep 17 00:00:00 2001 From: Kit <344842+roodboi@users.noreply.github.com> Date: Sat, 14 Nov 2026 04:03:27 +0000 Subject: [PATCH 12/15] test(native): verify detached dotenv refusal and refresh migration links --- docs/versions/v5/native/preflight.md | 4 ++-- docs/versions/v5/native/start.md | 4 ++-- tests/native-preflight-cli.test.ts | 9 ++++++++- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/docs/versions/v5/native/preflight.md b/docs/versions/v5/native/preflight.md index 7b7c6c6b0..5ec6111ff 100644 --- a/docs/versions/v5/native/preflight.md +++ b/docs/versions/v5/native/preflight.md @@ -23,8 +23,8 @@ decrypts managed env and reads `.env` beside Compose (normally `.hack/.env`), using the same file checks as native start. A missing decryption key is a structured blocker with recovery guidance; the command still produces a report. A project-root `.env` outside the Compose directory is ignored with one notice. -See the [environment-file rules](https://github.com/hack-dance/hack/blob/e1fa21b7d16d1b2ba2c5aee4ae52ea44f564c087/docs/native/environment-files.md) and -[migration notes](https://github.com/hack-dance/hack/blob/e1fa21b7d16d1b2ba2c5aee4ae52ea44f564c087/docs/guides/v5-release-notes.md#next17-what-changed-and-what-to-do) +See the [environment-file rules](https://github.com/hack-dance/hack/blob/9cd2053affe4f3307703e97d5bb4cd7fc3049830/docs/native/environment-files.md) and +[migration notes](https://github.com/hack-dance/hack/blob/9cd2053affe4f3307703e97d5bb4cd7fc3049830/docs/guides/v5-release-notes.md#next17-what-changed-and-what-to-do) before upgrading an earlier prerelease. ## Read the result diff --git a/docs/versions/v5/native/start.md b/docs/versions/v5/native/start.md index ff6524e14..f6683426a 100644 --- a/docs/versions/v5/native/start.md +++ b/docs/versions/v5/native/start.md @@ -70,8 +70,8 @@ to force a start. Keep the project on Compose if it needs unsupported behavior. See [Troubleshooting](../troubleshooting.md) and [the preflight reference](preflight.md). Env-file support intended for next.17 changes input checks and shell-default -selection. Before upgrading, read the [environment-file rules](https://github.com/hack-dance/hack/blob/e1fa21b7d16d1b2ba2c5aee4ae52ea44f564c087/docs/native/environment-files.md) -and [migration notes](https://github.com/hack-dance/hack/blob/e1fa21b7d16d1b2ba2c5aee4ae52ea44f564c087/docs/guides/v5-release-notes.md#next17-what-changed-and-what-to-do). +selection. Before upgrading, read the [environment-file rules](https://github.com/hack-dance/hack/blob/9cd2053affe4f3307703e97d5bb4cd7fc3049830/docs/native/environment-files.md) +and [migration notes](https://github.com/hack-dance/hack/blob/9cd2053affe4f3307703e97d5bb4cd7fc3049830/docs/guides/v5-release-notes.md#next17-what-changed-and-what-to-do). They explain which files are read, which declarations refuse and how to fix them. ## Start, check and stop diff --git a/tests/native-preflight-cli.test.ts b/tests/native-preflight-cli.test.ts index c717931ad..c10751415 100644 --- a/tests/native-preflight-cli.test.ts +++ b/tests/native-preflight-cli.test.ts @@ -367,7 +367,14 @@ test("preflight and start refuse ignored root dotenv in a native build before ef for (const args of [["up"], ["up", "--json"]]) { const started = await cli(f, args, env); expect(started.code).toBe(1); - expect(started.output).toContain("overlap an environment source"); + if (args.includes("--json")) { + expect(JSON.parse(started.stdout)).toMatchObject({ + ok: false, + error: { code: "E_STARTUP_INCOMPLETE" }, + }); + } else { + expect(started.output).toContain("overlap an environment source"); + } expect(started.output).not.toContain(PRIVATE); expect((await f.calls()).slice(before.length)).toEqual([]); } From 262e9dca9a22a5fea87bd5e631061f4fa20954d2 Mon Sep 17 00:00:00 2001 From: Kit <344842+roodboi@users.noreply.github.com> Date: Sat, 14 Nov 2026 04:07:22 +0000 Subject: [PATCH 13/15] test(native): match managed-secret build refusal wording --- tests/native-compose-backend-features-cli.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/native-compose-backend-features-cli.test.ts b/tests/native-compose-backend-features-cli.test.ts index 541f30635..45e727f2f 100644 --- a/tests/native-compose-backend-features-cli.test.ts +++ b/tests/native-compose-backend-features-cli.test.ts @@ -474,7 +474,7 @@ test( const result = await cli(f, ["up"], BUILDS_ENV); expectOneClosedLine( result, - "E_NATIVE_PROJECT_UNSUPPORTED: Native foreground up cannot build service web: build arg GREETING equals a privately delivered environment value. Build args are not secrets (images record them); pass secrets with BuildKit build secrets instead, which native builds do not support yet. Values were omitted; no hook, VM or build was started." + "E_NATIVE_PROJECT_UNSUPPORTED: Native foreground up cannot build service web: build arg GREETING equals a hack env secret value. Build args are not secrets (images record them); pass secrets with BuildKit build secrets instead, which native builds do not support yet. Values were omitted; no hook, VM or build was started." ); expect(result.output).not.toContain(secret); // The Compose backend would record the value in the image history too. From 9f173de7632e51c243932f137547e67458e70c69 Mon Sep 17 00:00:00 2001 From: Kit <344842+roodboi@users.noreply.github.com> Date: Sat, 14 Nov 2026 04:53:40 +0000 Subject: [PATCH 14/15] docs(native): clarify unset and exec environment changes --- docs/guides/v5-release-notes.md | 14 ++++++++++++ docs/native/environment-files.md | 32 +++++++++++++++++++++++----- docs/versions/v5/native/preflight.md | 2 ++ 3 files changed, 43 insertions(+), 5 deletions(-) diff --git a/docs/guides/v5-release-notes.md b/docs/guides/v5-release-notes.md index 04f02ac2a..e0c736453 100644 --- a/docs/guides/v5-release-notes.md +++ b/docs/guides/v5-release-notes.md @@ -24,6 +24,8 @@ its original behavior. | --- | --- | | **Native reads `.hack/.env` on every start and preflight, even with managed env.** Symlinks, other-owner or group/other-writable files and project ancestors refuse, as do `COMPOSE_*`/`HACK_*` keys and dotted/dashed identifiers. | Use an owned regular file, deliberately correct permissions, remove control entries and rename unsupported identifiers, or move values into managed env. A sibling `.env.state.json` also refuses possible materialized output; deliberately retire that output after moving to managed env. | | **Native service-environment defaults take allowed shell values.** Shell presence can change normalized declarations and plan identity, without storing values in that identity. | Unset the exported name to retain the authored fallback. In projects with `env_file` or adjacent `.env`, public-field references to ambient values now refuse even with a default: unset the source or write a public literal. Without those file sources, public defaults keep their previous behavior. | +| **An unresolved bare service-environment key is now unset instead of empty, including in projects without env files.** It removes an image default; an older runtime that cannot confirm the unset names refuses the review. Reserved runtime/adapter-owned names keep their existing behavior. | Use a matching current CLI/runtime bundle. Write `KEY: ""` when you intend an explicitly empty value instead of removing the key. | +| **Fresh native `hack exec` now delivers allowed caller-shell and selected-service file values privately.** Shell delivery also changes projects without env files; exec values may differ from those used at service startup. | Run exec from the intended shell and environment selection. Restart the service when changed inputs should also replace its startup environment; exec alone does not do that. | | **Native removes `dotenv_without_managed_env`.** A project-root `.env` outside the Compose directory is now ignored with one names-only notice. | Move required inputs beside Compose (normally `.hack/.env`) or into managed env. Dotenv alone does not inject service variables. | | **Native preflight reads the caller's allowed environment and decrypts managed env.** A missing key becomes a structured blocker; the report remains usable. | Run from the intended shell and recover the existing project key when asked. Do not generate a replacement key for existing ciphertext. | | **Native env-file paths must stay separate from build inputs.** Contexts, Dockerfiles and Dockerfile-specific ignore inputs may not overlap declared env files, adjacent `.env` or a present project-root `.env`, including optional and inactive paths. A root `.env` triggers this refusal even when there is no `env_file` or adjacent `.env`; ignored for interpolation does not exclude it from `COPY .`. | Use disjoint build inputs or prebuilt images. The build-argument value guard stays limited to nonempty managed `secret: true` values, with no minimum length; plain file and shell values do not widen it. See [the full rules](../native/environment-files.md). | @@ -37,6 +39,18 @@ its original behavior. | **Native route readiness can be inferred.** A complete standard Caddy route with no authored healthcheck gets a reviewed HTTP probe and 240 seconds of initial grace. A complete HTTP 200–499 response establishes endpoint readiness, not application correctness. | Review the route port and startup logs. Explicit healthchecks are not replaced, and inferred readiness does not satisfy `depends_on: condition: service_healthy`. The grace does not extend the whole graph's deadline. | | **Native start waits for bridge sockets after a stopped pool resumes.** An immediate start no longer fails merely because provider readiness preceded the bridge listener. | Use normal native `up`; startup still requires a successful bridge connection within its deadline. Do not replace readiness checks with a fixed sleep. | +Detached native starts do not show per-key "unresolved input" notices because +the owner's stdio is ignored. Use a foreground start or native preflight to +inspect those notices; only forwarded shell names and the fixed ignored-root +dotenv notice reach the detached caller. + +Native `.env` admission refuses **every** `COMPOSE_*` key, broader than Compose's +recognized control-variable set. Native reserves the whole prefix so new Compose +controls cannot silently become ordinary interpolation inputs: project, file and +profile selection stay explicit. Rename application-specific variables without +that prefix. This rule applies to the adjacent interpolation `.env`, not every +literal service `env_file` entry. + ### Env scripts and migrated plaintext Replace noninteractive bulk reveal with explicit single-key reads; do not log or diff --git a/docs/native/environment-files.md b/docs/native/environment-files.md index 75002e930..2a3c7cae0 100644 --- a/docs/native/environment-files.md +++ b/docs/native/environment-files.md @@ -51,8 +51,12 @@ it leaves earlier file assignments intact. A bare service environment key resolv from the interpolation context; when unresolved it removes that key, including an image default. Native removes the key from normalized `environment`, then carries its name in explicit normalized metadata and public engine unset entries, separate from private -value delivery. The runtime must echo the exact unset names during review; -ordinary missing private bindings must continue to refuse. +value delivery. This changes projects without env files too: earlier native +starts delivered an empty value for an unresolved bare service key. Write +`KEY: ""` when that empty value is intentional. The runtime must echo the exact +unset names during review; older runtimes that cannot do so refuse. Use a +matching current CLI/runtime bundle. Ordinary missing private bindings must +continue to refuse. Reserved runtime/adapter-owned names retain their behavior. Interpolation lookup order is Hack-managed global values, the caller's allowed shell variables, then `.env`. Service-scoped managed values win the final service @@ -94,6 +98,12 @@ names are not read from the shell; explicit literal file entries remain availabl unless the selected credential adapter owns that service's name. Compose control names (`COMPOSE_*`) and protected `HACK_*` keys in `.env` refuse with names only rather than silently change the selected project. +This deliberately reserves every `COMPOSE_*` key, broader than Compose's known +control-variable set, so future controls cannot silently become ordinary +interpolation inputs. Native keeps project, file and profile selection explicit. +Rename application-specific variables without that prefix. This prefix-wide +rule is for the adjacent interpolation `.env`; literal service `env_file` entries +are not all subject to it. The approved first unit resolves file values only in service `environment` and env-file values. In a project that declares `env_file` or has an adjacent `.env`, @@ -139,6 +149,10 @@ stay on the existing public path and retain their restart behavior and identity. Fresh `hack exec` resolves the selected service's file values again using the same allowed caller context and delivers them privately to that exec process. These fresh values may differ from the running container's startup values. +Fresh exec now also delivers allowed caller-shell values rather than managed +values alone, including in projects without env files. Run it from the intended +shell and environment selection. Use restart when changed inputs should replace +the running service's startup environment; exec alone does not update it. Native builds read raw host contexts. Start and preflight therefore refuse a native build whose context, Dockerfile or Dockerfile-specific ignore input @@ -190,6 +204,12 @@ Actual guest delivery is qualified separately on a Mac. - Service defaults take allowed shell values as described above. Public-field interpolation gains the strict boundary only for file-enabled projects; use literal public fields or remove the ambient source. +- Unresolved bare service keys now remove the key instead of delivering empty, + including without env files. Use `KEY: ""` for explicit empty and a matching + runtime that can confirm unset names; an older runtime refuses that review. +- Fresh `hack exec` now delivers current allowed shell and selected-service file + values privately. Shell delivery also affects projects without env files; run + from the intended shell and restart if startup values must change too. - The former `dotenv_without_managed_env` refusal is removed. A project-root `.env` outside the Compose directory is ignored with one notice. Move needed variables beside Compose (normally `.hack/.env`) or into managed env. @@ -200,6 +220,8 @@ Actual guest delivery is qualified separately on a Mac. replacement for existing ciphertext. Foreground notices go to stderr. Detached owners discard stderr rather than -write it to a log file; only validated forwarded shell names and the fixed -root-dotenv notice cross IPC to the caller. Tokens referenced only inside file -values are excluded from the forwarded-names notice as well. +write it to a log file, so detached starts do not show per-key "unresolved input" +notices. Use a foreground start or preflight to inspect them. Only validated +forwarded shell names and the fixed root-dotenv notice cross IPC to the caller. +Tokens referenced only inside file values are excluded from the forwarded-names +notice as well. diff --git a/docs/versions/v5/native/preflight.md b/docs/versions/v5/native/preflight.md index 5ec6111ff..d614465c3 100644 --- a/docs/versions/v5/native/preflight.md +++ b/docs/versions/v5/native/preflight.md @@ -23,6 +23,8 @@ decrypts managed env and reads `.env` beside Compose (normally `.hack/.env`), using the same file checks as native start. A missing decryption key is a structured blocker with recovery guidance; the command still produces a report. A project-root `.env` outside the Compose directory is ignored with one notice. +Detached starts omit per-key "unresolved input" notices because owner stdio is +ignored; use this preflight or a foreground start to inspect missing inputs. See the [environment-file rules](https://github.com/hack-dance/hack/blob/9cd2053affe4f3307703e97d5bb4cd7fc3049830/docs/native/environment-files.md) and [migration notes](https://github.com/hack-dance/hack/blob/9cd2053affe4f3307703e97d5bb4cd7fc3049830/docs/guides/v5-release-notes.md#next17-what-changed-and-what-to-do) before upgrading an earlier prerelease. From aab610e2eea7e9b8562fb472a99d2d412480b9bc Mon Sep 17 00:00:00 2001 From: Kit <344842+roodboi@users.noreply.github.com> Date: Sat, 14 Nov 2026 06:16:48 +0000 Subject: [PATCH 15/15] fix(native): verify requested bare environment removals after create --- .gitleaks.toml | 7 ++ .../src/provider/graph/cache_provenance.rs | 8 +- .../provider/graph/cold_environment_tests.rs | 75 +++++++++++++ .../src/provider/graph/image_environment.rs | 106 ++++++++++++++---- .../src/provider/graph/initializer_cache.rs | 8 +- .../runtime-core/src/provider/graph/mod.rs | 47 +++++--- .../src/provider/graph/native/runtime.rs | 6 +- .../native/runtime/cold_environment_tests.rs | 63 +++++++++++ .../provider/graph/native/runtime/tests.rs | 7 ++ .../runtime-core/src/provider/graph/probes.rs | 8 +- .../runtime-core/src/provider/graph/tests.rs | 2 + .../fixtures/native-bare-env-engine.json | 61 ++++++++++ 12 files changed, 358 insertions(+), 40 deletions(-) create mode 100644 packages/runtime-core/src/provider/graph/cold_environment_tests.rs create mode 100644 packages/runtime-core/src/provider/graph/native/runtime/cold_environment_tests.rs create mode 100644 packages/runtime-core/tests/fixtures/native-bare-env-engine.json diff --git a/.gitleaks.toml b/.gitleaks.toml index 20bba3f16..2aa6c1658 100644 --- a/.gitleaks.toml +++ b/.gitleaks.toml @@ -36,3 +36,10 @@ condition = "AND" paths = ['''^tests/project-task-catalog\.test\.ts$'''] regexTarget = "line" regexes = ['''(?m)^\s*const SECRET_CANARY = "secret-canary-91c4d0";\s*$'''] + +[[rules.allowlists]] +description = "Real-engine fixture records the public Python image signing-key fingerprint, not a credential" +condition = "AND" +paths = ['''^packages/runtime-core/tests/fixtures/native-bare-env-engine\.json$'''] +regexTarget = "line" +regexes = ['''^\s*"GPG_KEY=7169605F62C751356D054A26A821E680E5FA6305",?\s*$'''] diff --git a/packages/runtime-core/src/provider/graph/cache_provenance.rs b/packages/runtime-core/src/provider/graph/cache_provenance.rs index 8d324e0eb..9fce7c71f 100644 --- a/packages/runtime-core/src/provider/graph/cache_provenance.rs +++ b/packages/runtime-core/src/provider/graph/cache_provenance.rs @@ -247,7 +247,13 @@ impl Session<'_, '_> { let resource = self.receipt.resources[&format!("container:{service}")].clone(); let value = inspect_resource(&self.engine, &self.receipt, &resource)?.ok_or_else(refused)?; - self.verify_config(service, &value)?; + Self::verify_config( + &self.receipt, + service, + &self.configs[service], + &self.expected_environment[service], + &value, + )?; for key in keys { let volume = &self.receipt.resources[&key]; verify(&self.engine, &self.receipt, volume)?; diff --git a/packages/runtime-core/src/provider/graph/cold_environment_tests.rs b/packages/runtime-core/src/provider/graph/cold_environment_tests.rs new file mode 100644 index 000000000..ad5e843f7 --- /dev/null +++ b/packages/runtime-core/src/provider/graph/cold_environment_tests.rs @@ -0,0 +1,75 @@ +use super::*; + +#[test] +fn cold_create_inspection_accepts_only_compiled_removal_markers_from_real_engine() { + for capture in image_environment::engine_removal_captures() { + let fixture = Fixture::new(); + let home = Fixture::new(); + let candidate = Candidate::discover(&home.0).unwrap(); + state::write( + &fixture.0.join("compose.yaml"), + &json!({"services":{"app":{ + "image":capture["imageId"],"network_mode":"none", + "environment":{"HQ_PLAIN":"plain"}, + "x-hack-unset-environment":[capture["removedKey"]] + }}}), + ) + .unwrap(); + let review = project::plan(&candidate, fixture.options()).unwrap(); + let inputs = project::inputs::compile( + &candidate, + fixture.options(), + &review.plan_id, + &BTreeMap::new(), + ) + .unwrap(); + let readiness = BTreeMap::from([("app".to_owned(), Condition::Started)]); + let prepared = + config::prepare(inputs, &readiness, &"a".repeat(32), &"b".repeat(32), None).unwrap(); + let config = &prepared.configs["app"]; + let requested = config["Env"].as_array().unwrap(); + assert_eq!(requested.len(), 2); + for entry in capture["requestedEnvironment"].as_array().unwrap() { + assert!(requested.contains(entry)); + } + let receipt: Receipt = serde_json::from_value(json!({ + "version":1,"run":"a".repeat(32),"owner":"b".repeat(32), + "namespace":review.plan.namespace,"plan_id":review.plan_id, + "phase":"preparing","readiness":readiness,"resources":prepared.resources + })) + .unwrap(); + let expected = + image_environment::compose(&capture["imageEnvironment"], &config["Env"]).unwrap(); + let mut inspected = json!({"Config":config,"HostConfig":config["HostConfig"],"State":{"Running":false},"NetworkSettings":{"Networks":{}}}); + inspected["Config"]["Env"] = capture["inspectedEnvironment"].clone(); + // This is the same gate cold start calls between create and start. + Session::verify_config(&receipt, "app", config, &expected, &inspected).unwrap(); + for extra in [ + json!("UNREQUESTED"), + capture["removedKey"].clone(), + json!("HQ_PLAIN=other"), + ] { + let mut wrong = inspected.clone(); + wrong["Config"]["Env"].as_array_mut().unwrap().push(extra); + assert_eq!( + Session::verify_config(&receipt, "app", config, &expected, &wrong) + .unwrap_err() + .code, + "graph_config_mismatch" + ); + } + let mut restored = inspected.clone(); + let removed = capture["removedKey"].as_str().unwrap(); + for entry in restored["Config"]["Env"].as_array_mut().unwrap() { + if entry.as_str() == Some(removed) { + *entry = json!(format!("{removed}=restored")); + } + } + assert!(Session::verify_config(&receipt, "app", config, &expected, &restored).is_err()); + let mut changed_default = inspected.clone(); + changed_default["Config"]["Env"][2] = json!("PATH=/different"); + assert!( + Session::verify_config(&receipt, "app", config, &expected, &changed_default).is_err() + ); + } +} diff --git a/packages/runtime-core/src/provider/graph/image_environment.rs b/packages/runtime-core/src/provider/graph/image_environment.rs index 0ed5b41aa..88aad6c79 100644 --- a/packages/runtime-core/src/provider/graph/image_environment.rs +++ b/packages/runtime-core/src/provider/graph/image_environment.rs @@ -7,32 +7,52 @@ fn refused() -> CandidateError { "Pinned image or container environment is malformed or differs; values omitted.", ) } -fn entries(value: &Value) -> Result, CandidateError> { +/// In-memory expectation bound to the pinned image and compiled public overrides. +/// Removal names must survive composition: an arbitrary bare inspected name is +/// not evidence that the reviewed configuration requested its removal. +#[derive(Default)] +pub(super) struct Expected { + values: BTreeMap, + removed: BTreeSet, +} + +fn entries( + value: &Value, + removed: &BTreeSet, +) -> Result, CandidateError> { if value.is_null() { return Ok(BTreeMap::new()); } let array = value.as_array().ok_or_else(refused)?; let mut result = BTreeMap::new(); + let mut seen = BTreeSet::new(); for entry in array { + let entry = entry.as_str().ok_or_else(refused)?; let (key, value) = entry - .as_str() - .and_then(|v| v.split_once('=')) - .ok_or_else(refused)?; + .split_once('=') + .map_or((entry, None), |(key, value)| (key, Some(value))); if key.is_empty() || key.contains('\0') - || value.contains('\0') - || result.insert(key.into(), value.into()).is_some() + || value.is_some_and(|value| value.contains('\0')) + || !seen.insert(key) { return Err(refused()); } + if let Some(value) = value { + result.insert(key.into(), value.into()); + } else if !removed.contains(key) { + return Err(refused()); + } } Ok(result) } -pub(super) fn compose( - image: &Value, - public: &Value, -) -> Result, CandidateError> { - let mut expected = entries(image)?; +pub(super) fn compose(image: &Value, public: &Value) -> Result { + // Image defaults must be assignments; only compiled public overrides can + // authorize bare removal markers in the engine's inspected configuration. + let mut expected = Expected { + values: entries(image, &BTreeSet::new())?, + removed: BTreeSet::new(), + }; if !public.is_null() { let mut seen = std::collections::BTreeSet::new(); for entry in public.as_array().ok_or_else(refused)? { @@ -48,27 +68,63 @@ pub(super) fn compose( return Err(refused()); } if let Some(value) = value { - expected.insert(key.to_owned(), value.to_owned()); + expected.values.insert(key.to_owned(), value.to_owned()); } else { - expected.remove(key); + expected.values.remove(key); + expected.removed.insert(key.to_owned()); } } } Ok(expected) } -pub(super) fn verify( - expected: &BTreeMap, - actual: &Value, -) -> Result<(), CandidateError> { - if &entries(actual)? != expected { +pub(super) fn verify(expected: &Expected, actual: &Value) -> Result<(), CandidateError> { + if entries(actual, &expected.removed)? != expected.values { return Err(refused()); } Ok(()) } + +#[cfg(test)] +pub(super) fn engine_removal_captures() -> Vec { + // Docker Engine 29.9.2/API 1.56: public python image defaults and synthetic + // inputs only. No host paths, owner labels or private process environment. + let fixture: Value = serde_json::from_str(include_str!( + "../../../tests/fixtures/native-bare-env-engine.json" + )) + .unwrap(); + assert_eq!(fixture["engineVersion"], "29.9.2"); + assert_eq!(fixture["apiVersion"], "1.56"); + fixture["captures"].as_array().unwrap().clone() +} #[cfg(test)] mod tests { use super::*; #[test] + fn real_engine_inspection_preserves_requested_bare_removals_and_other_defaults() { + for capture in engine_removal_captures() { + let expected = compose( + &capture["imageEnvironment"], + &capture["requestedEnvironment"], + ) + .unwrap(); + assert_eq!( + expected.removed, + BTreeSet::from([capture["removedKey"].as_str().unwrap().to_owned()]) + ); + verify(&expected, &capture["inspectedEnvironment"]).unwrap(); + assert_eq!( + capture["processObservation"], + json!({"unset_absent":true,"plain_present":true,"other_defaults_present":true}) + ); + for entry in capture["imageEnvironment"].as_array().unwrap() { + let (key, value) = entry.as_str().unwrap().split_once('=').unwrap(); + if key != capture["removedKey"].as_str().unwrap() { + assert_eq!(expected.values.get(key).map(String::as_str), Some(value)); + } + } + } + } + #[test] fn pinned_defaults_and_public_overrides_match_exactly_independent_of_order() { let expected = compose( &json!(["PATH=/bin", "MODE=image", "EMPTY=image"]), @@ -110,13 +166,13 @@ mod tests { json!({"KEY":"canary"}), ] { for result in [compose(&bad, &Value::Null), compose(&Value::Null, &bad)] { - let failure = result.unwrap_err(); + let failure = result.err().unwrap(); assert!(!failure.message.contains("canary")); } - assert!(verify(&BTreeMap::new(), &bad).is_err()); + assert!(verify(&Expected::default(), &bad).is_err()); } assert!(compose(&json!(["MISSING"]), &Value::Null).is_err()); - assert!(verify(&BTreeMap::new(), &json!(["MISSING"])).is_err()); + assert!(verify(&Expected::default(), &json!(["MISSING"])).is_err()); } #[test] fn explicit_bare_names_remove_image_defaults_but_empty_values_remain() { @@ -126,9 +182,17 @@ mod tests { ) .unwrap(); verify(&expected, &json!(["EMPTY=", "PATH=/bin"])).unwrap(); + verify( + &expected, + &json!(["EMPTY=", "PATH=/bin", "REMOVE", "MISSING"]), + ) + .unwrap(); for actual in [ json!(["EMPTY=", "PATH=/bin", "REMOVE=image-default"]), json!(["EMPTY=", "PATH=/bin", "REMOVE="]), + json!(["EMPTY=", "PATH=/bin", "UNREQUESTED"]), + json!(["EMPTY=", "PATH=/bin", "REMOVE", "REMOVE"]), + json!(["EMPTY=", "PATH=/bin", "REMOVE", "REMOVE=value"]), ] { assert!(verify(&expected, &actual).is_err()); } diff --git a/packages/runtime-core/src/provider/graph/initializer_cache.rs b/packages/runtime-core/src/provider/graph/initializer_cache.rs index 471882c69..68559f60c 100644 --- a/packages/runtime-core/src/provider/graph/initializer_cache.rs +++ b/packages/runtime-core/src/provider/graph/initializer_cache.rs @@ -316,7 +316,13 @@ impl Release for Session<'_, '_> { }; let observed = inspect_resource(&self.engine, &self.receipt, resource)?.ok_or_else(refused)?; - self.verify_config(service, &observed)?; + Self::verify_config( + &self.receipt, + service, + &self.configs[service], + &self.expected_environment[service], + &observed, + )?; if observed["State"]["Status"] != "exited" || observed["State"]["Running"] != false || observed["State"]["Paused"] != false diff --git a/packages/runtime-core/src/provider/graph/mod.rs b/packages/runtime-core/src/provider/graph/mod.rs index 95a555d5e..a8a553563 100644 --- a/packages/runtime-core/src/provider/graph/mod.rs +++ b/packages/runtime-core/src/provider/graph/mod.rs @@ -764,7 +764,7 @@ struct Session<'a, 's> { root: PathBuf, receipt: Receipt, configs: BTreeMap, - expected_environment: BTreeMap>, + expected_environment: BTreeMap, restarting: bool, fresh_cache_completion: bool, cache_initializers: BTreeMap, @@ -802,12 +802,16 @@ impl Session<'_, '_> { fn fault_pause(&self, point: &str) -> Result<(), CandidateError> { fault_pause(&self.root, &self.receipt.run, point) } - fn verify_config(&self, service: &str, inspected: &Value) -> Result<(), CandidateError> { - image_environment::verify( - &self.expected_environment[service], - &inspected["Config"]["Env"], - )?; - let expected = &self.configs[service]; + // Pure create/inspect gate: explicit compiled inputs keep it testable without + // constructing a provider lease or a live guest engine. + fn verify_config( + receipt: &Receipt, + service: &str, + expected: &Value, + environment: &image_environment::Expected, + inspected: &Value, + ) -> Result<(), CandidateError> { + image_environment::verify(environment, &inspected["Config"]["Env"])?; for (key, value) in expected.as_object().expect("compiled config") { if key == "Env" { continue; @@ -842,8 +846,7 @@ impl Session<'_, '_> { )); } for network in networks.keys() { - let resource = self - .receipt + let resource = receipt .resources .values() .find(|resource| resource.kind == Kind::Network && resource.name == *network) @@ -1074,7 +1077,13 @@ impl Driver for Session<'_, '_> { self.prepare_probe(service)?; let inspected = inspect_resource(&self.engine, &self.receipt, &resource)? .ok_or_else(|| error("graph_resource_missing", "Restart resource is missing."))?; - self.verify_config(service, &inspected)?; + Self::verify_config( + &self.receipt, + service, + &self.configs[service], + &self.expected_environment[service], + &inspected, + )?; let id = resource.id.as_deref().ok_or_else(|| { error( "graph_replay_refused", @@ -1160,7 +1169,13 @@ impl Driver for Session<'_, '_> { let inspected = inspect_resource(&self.engine, &self.receipt, &self.receipt.resources[&key])? .ok_or_else(|| error("graph_resource_missing", "Created container is missing."))?; - self.verify_config(service, &inspected)?; + Self::verify_config( + &self.receipt, + service, + &self.configs[service], + &self.expected_environment[service], + &inspected, + )?; self.reserve(&key, "start-intent")?; if let Some(lease) = self.leases.get(service) { lease.verified_path_with_guest(self.engine.guest(), service)?; @@ -1192,7 +1207,13 @@ impl Driver for Session<'_, '_> { let resource = &self.receipt.resources[&format!("container:{service}")]; let value = inspect_resource(&self.engine, &self.receipt, resource)? .ok_or_else(|| error("graph_resource_missing", "Owned service disappeared."))?; - self.verify_config(service, &value)?; + Self::verify_config( + &self.receipt, + service, + &self.configs[service], + &self.expected_environment[service], + &value, + )?; probes::observe(&self.engine, &self.receipt, service, &value) } } @@ -2650,7 +2671,7 @@ fn verify_images( configs: &mut BTreeMap, private_services: &BTreeSet, file_owners: &BTreeSet, -) -> Result>, CandidateError> { +) -> Result, CandidateError> { let mut expected = BTreeMap::new(); let mut refusals = process_refusals::Collector::default(); for resource in resources.values().filter(|r| r.kind == Kind::Container) { diff --git a/packages/runtime-core/src/provider/graph/native/runtime.rs b/packages/runtime-core/src/provider/graph/native/runtime.rs index f3d0ceb87..35b4237a8 100644 --- a/packages/runtime-core/src/provider/graph/native/runtime.rs +++ b/packages/runtime-core/src/provider/graph/native/runtime.rs @@ -530,7 +530,7 @@ fn verify_config( receipt: &Receipt, service: &str, expected: &Value, - environment: &BTreeMap, + environment: &image_environment::Expected, actual: &Value, ) -> Result<(), CandidateError> { image_environment::verify(environment, &actual["Config"]["Env"])?; @@ -768,7 +768,7 @@ struct Session<'a, B> { root: PathBuf, receipt: Receipt, configs: BTreeMap, - expected_environment: BTreeMap>, + expected_environment: BTreeMap, environments: BTreeMap, } impl Session<'_, B> { @@ -1784,7 +1784,7 @@ fn verify_native_images( resources: &BTreeMap, configs: &mut BTreeMap, private: &BTreeSet, -) -> Result>, CandidateError> { +) -> Result, CandidateError> { let mut expected = BTreeMap::new(); for resource in resources .values() diff --git a/packages/runtime-core/src/provider/graph/native/runtime/cold_environment_tests.rs b/packages/runtime-core/src/provider/graph/native/runtime/cold_environment_tests.rs new file mode 100644 index 000000000..4c0f9b8a2 --- /dev/null +++ b/packages/runtime-core/src/provider/graph/native/runtime/cold_environment_tests.rs @@ -0,0 +1,63 @@ +use super::*; + +#[test] +fn cold_start_uses_real_engine_removal_markers_and_refuses_unrequested_or_changed_env() { + for capture in image_environment::engine_removal_captures() { + let mut invalid = Vec::new(); + for extra in [ + json!("UNREQUESTED"), + capture["removedKey"].clone(), + json!("HQ_PLAIN=other"), + json!(17), + ] { + let mut actual = capture["inspectedEnvironment"].clone(); + actual.as_array_mut().unwrap().push(extra); + invalid.push(actual); + } + let mut restored = capture["inspectedEnvironment"].clone(); + let removed = capture["removedKey"].as_str().unwrap(); + for entry in restored.as_array_mut().unwrap() { + if entry.as_str() == Some(removed) { + *entry = json!(format!("{removed}=restored")); + } + } + invalid.push(restored); + let mut changed_default = capture["inspectedEnvironment"].clone(); + changed_default[2] = json!("PATH=/different"); + invalid.push(changed_default); + for (actual, accepted) in std::iter::once((capture["inspectedEnvironment"].clone(), true)) + .chain(invalid.into_iter().map(|value| (value, false))) + { + let mut project = basic(); + project["services"]["web"]["image"] = capture["imageId"].clone(); + let fixture = Fixture::new(project); + let (graph, mut session) = + fixture.session(fixture.prepared(json!({"web":{}}), &BTreeMap::new())); + // Supply the reviewed public removal intent and a pinned-image + // expectation; the fake engine returns the separately captured Env. + session.configs.get_mut("web").unwrap()["Env"] = + capture["requestedEnvironment"].clone(); + session.expected_environment.insert( + "web".into(), + image_environment::compose( + &capture["imageEnvironment"], + &session.configs["web"]["Env"], + ) + .unwrap(), + ); + session.backend.state.borrow_mut().inspected_environment = Some(actual); + let result = execution::run(&graph, &mut session, RUN_BUDGET); + let state = session.backend.state.borrow(); + assert!(state.effects.iter().any(|effect| effect == "create:web")); + if accepted { + result.unwrap(); + assert_eq!(session.receipt.phase, Phase::ReadyObserved); + assert!(state.effects.iter().any(|effect| effect == "start:web")); + } else { + assert_eq!(result.unwrap_err().code, "graph_config_mismatch"); + assert!(!state.effects.iter().any(|effect| effect == "start:web")); + assert_ne!(session.receipt.phase, Phase::ReadyObserved); + } + } + } +} diff --git a/packages/runtime-core/src/provider/graph/native/runtime/tests.rs b/packages/runtime-core/src/provider/graph/native/runtime/tests.rs index 72c687762..1c50328ff 100644 --- a/packages/runtime-core/src/provider/graph/native/runtime/tests.rs +++ b/packages/runtime-core/src/provider/graph/native/runtime/tests.rs @@ -3,6 +3,8 @@ use super::*; mod authored_exec; #[path = "run_tests.rs"] mod authored_run; +#[path = "cold_environment_tests.rs"] +mod cold_environment; use hack_config_compiler::environment::EnvMetadata; use std::{ cell::{Cell, RefCell}, @@ -671,6 +673,8 @@ fn foreground_cancellation_after_network_creation_prevents_container_effects_and #[derive(Default)] struct FakeState { + /// Real Engine Config.Env capture used at the create/inspect boundary. + inspected_environment: Option, log_reads: Vec<(String, u16)>, log_source_deadlines: Vec, log_data_deadlines: Vec, @@ -883,6 +887,9 @@ impl Backend for Fake { let id = format!("{:064x}", state.containers.len() + 1); let mut process = config.clone(); process.as_object_mut().unwrap().remove("HostConfig"); + if let Some(environment) = &state.inspected_environment { + process["Env"] = environment.clone(); + } let selected = resource.networks.as_ref().unwrap(); let mut attached = serde_json::Map::new(); for logical in selected { diff --git a/packages/runtime-core/src/provider/graph/probes.rs b/packages/runtime-core/src/provider/graph/probes.rs index 46e349f8d..bc52249b4 100644 --- a/packages/runtime-core/src/provider/graph/probes.rs +++ b/packages/runtime-core/src/provider/graph/probes.rs @@ -578,7 +578,13 @@ impl Session<'_, '_> { let resource = &self.receipt.resources[&format!("container:{name}")]; let value = inspect_resource(&self.engine, &self.receipt, resource)? .ok_or_else(|| error("graph_probe_identity", "Restart container disappeared."))?; - self.verify_config(name, &value)?; + Self::verify_config( + &self.receipt, + name, + &self.configs[name], + &self.expected_environment[name], + &value, + )?; if !matches!(observation(&value)?, Observation::Exited { .. }) { return Err(error( "graph_probe_running", diff --git a/packages/runtime-core/src/provider/graph/tests.rs b/packages/runtime-core/src/provider/graph/tests.rs index 2b536d119..dadbb12b8 100644 --- a/packages/runtime-core/src/provider/graph/tests.rs +++ b/packages/runtime-core/src/provider/graph/tests.rs @@ -1,4 +1,6 @@ use super::*; +#[path = "cold_environment_tests.rs"] +mod cold_environment; use std::{io::Read, path::Path, process::Command}; pub(super) struct Fixture(pub(super) PathBuf); impl Fixture { diff --git a/packages/runtime-core/tests/fixtures/native-bare-env-engine.json b/packages/runtime-core/tests/fixtures/native-bare-env-engine.json new file mode 100644 index 000000000..f302d6133 --- /dev/null +++ b/packages/runtime-core/tests/fixtures/native-bare-env-engine.json @@ -0,0 +1,61 @@ +{ + "engineVersion": "29.9.2", + "apiVersion": "1.56", + "captures": [ + { + "case": "image-default", + "removedKey": "PYTHON_VERSION", + "imageReference": "python:3.12-alpine", + "imageId": "sha256:c5ba3deef00c7758561a0f03c26f44f10589263d40e732b8748e0d1f38f6dd89", + "imageEnvironment": [ + "PATH=/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + "LANG=C.UTF-8", + "GPG_KEY=7169605F62C751356D054A26A821E680E5FA6305", + "PYTHON_VERSION=3.12.16", + "PYTHON_SHA256=b5796915dc45ff745cfb4e990a7e7c93d6f546328a102ce31448d69ebc37575c" + ], + "requestedEnvironment": ["HQ_PLAIN=plain", "PYTHON_VERSION"], + "inspectedEnvironment": [ + "HQ_PLAIN=plain", + "PYTHON_VERSION", + "PATH=/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + "LANG=C.UTF-8", + "GPG_KEY=7169605F62C751356D054A26A821E680E5FA6305", + "PYTHON_SHA256=b5796915dc45ff745cfb4e990a7e7c93d6f546328a102ce31448d69ebc37575c" + ], + "processObservation": { + "unset_absent": true, + "plain_present": true, + "other_defaults_present": true + } + }, + { + "case": "absent-image-key", + "removedKey": "HQ_NOT_IN_IMAGE", + "imageReference": "python:3.12-alpine", + "imageId": "sha256:c5ba3deef00c7758561a0f03c26f44f10589263d40e732b8748e0d1f38f6dd89", + "imageEnvironment": [ + "PATH=/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + "LANG=C.UTF-8", + "GPG_KEY=7169605F62C751356D054A26A821E680E5FA6305", + "PYTHON_VERSION=3.12.16", + "PYTHON_SHA256=b5796915dc45ff745cfb4e990a7e7c93d6f546328a102ce31448d69ebc37575c" + ], + "requestedEnvironment": ["HQ_PLAIN=plain", "HQ_NOT_IN_IMAGE"], + "inspectedEnvironment": [ + "HQ_PLAIN=plain", + "HQ_NOT_IN_IMAGE", + "PATH=/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + "LANG=C.UTF-8", + "GPG_KEY=7169605F62C751356D054A26A821E680E5FA6305", + "PYTHON_VERSION=3.12.16", + "PYTHON_SHA256=b5796915dc45ff745cfb4e990a7e7c93d6f546328a102ce31448d69ebc37575c" + ], + "processObservation": { + "unset_absent": true, + "plain_present": true, + "other_defaults_present": true + } + } + ] +}