diff --git a/.gitleaks.toml b/.gitleaks.toml index 20bba3f16..2aa6c1658 100644 --- a/.gitleaks.toml +++ b/.gitleaks.toml @@ -36,3 +36,10 @@ condition = "AND" paths = ['''^tests/project-task-catalog\.test\.ts$'''] regexTarget = "line" regexes = ['''(?m)^\s*const SECRET_CANARY = "secret-canary-91c4d0";\s*$'''] + +[[rules.allowlists]] +description = "Real-engine fixture records the public Python image signing-key fingerprint, not a credential" +condition = "AND" +paths = ['''^packages/runtime-core/tests/fixtures/native-bare-env-engine\.json$'''] +regexTarget = "line" +regexes = ['''^\s*"GPG_KEY=7169605F62C751356D054A26A821E680E5FA6305",?\s*$'''] diff --git a/docs/guides/native-candidate.md b/docs/guides/native-candidate.md index a875ff67e..648dbc57c 100644 --- a/docs/guides/native-candidate.md +++ b/docs/guides/native-candidate.md @@ -196,8 +196,8 @@ symbolic link. Unreadable, changing or oversized input is reported as It changes nothing in the project and starts nothing: no VM, no image pull, no lifecycle hook. It accepts only `--path`, `--profile` and `--json`. The report never prints authored values. A finding names a closed code, a service and a -field; environment values, label keys and values, paths and image names are -left out. +field; environment values, label keys and values and image names are left out. +Environment-file findings may name their source path. The runtime review half needs a native runtime, so run the preflight from the `hack-next` launcher for the full list. Without native selection only the CLI's @@ -213,10 +213,11 @@ Preflight and start use one file-local admission collector. Preflight lists its refusals; start returns its first refusal before acquiring runtime inputs. This can change which reason start reports first when a file has several problems. Review assumes routing is enrolled and cache initializers are admitted. The -readiness defaults are checked against the CLI over shared fixtures. Managed env -metadata and eligible shell references contribute service names only, so restart -policies incompatible with private delivery are blockers even when no decryption -key is available. Shell values and credentials are never read for this check. +readiness defaults are checked against the CLI over shared fixtures. Preflight +uses start's environment preparation, including allowed shell values, guarded +env-file reads and managed-env decryption. A missing decryption key is a blocker. +Only names and paths reach the report. Restart policies incompatible with private +delivery are blockers. Credential adapters, image defaults, occupied ports and other inputs only known at start remain runtime boundaries. AWS/gcloud credentials are added after static review; a restart policy on a @@ -235,11 +236,11 @@ With `--json` the report is the `data` of the standard result envelope: { "severity": "blocker", "gate": "cli", - "code": "env_file", + "code": "environment_file_input", "service": "web", "field": "env_file", - "message": "The service uses env_file, which native does not read.", - "fix": "Move its variables into Hack managed env with hack env set --service, or run hack doctor --migrate-env-config." + "message": "A required environment file is missing or unsafe; values omitted.", + "fix": "Check the named source; use an owned regular file that is not writable by others." } ], "warnings": [] @@ -273,10 +274,10 @@ With `--json` the report is the `data` of the standard result envelope: `internal_hosts_without_dependencies`, `external_network`, `external_volume`, `reserved_label`, `unsupported_build`, `foreign_platform`, `invalid_service_name`, `invalid_service`, `build_requires_opt_in`, - `unsupported_build_form`, `emulated_build`, `env_file`, `missing_image`, + `unsupported_build_form`, `emulated_build`, `environment_file_input`, `missing_image`, `extra_hosts_without_dependencies`, `unsupported_logging`, `invalid_restart`, `unsupported_label`, `invalid_http_probe`, `managed_env_unreadable`, - `dotenv_without_managed_env`, `shared_source_grant_required`, + `environment_delivery`, `shared_source_grant_required`, `route_probe_required` and `too_many_bridges`. Runtime findings use the review's own diagnostic codes, for example `privileged_service` or `invalid_mount`, and `review_incompatible` when the review does not admit the @@ -303,12 +304,10 @@ The standalone preflight does not cover: - `--env`, and a retained run's environment. It uses the project's default environment to select the per-env adaptation and dependency files; `hack up` uses `--env` when given, or the environment of a run it resumes. -- Checks that need secret values or a resolved managed environment: a build - argument that carries a stored secret, a managed variable whose name is not - a valid environment name, and managed env that fails to resolve. `hack up` - also replaces the `environment` entries that managed env supplies before its - review, so a malformed authored value under such a key is a blocker here and - not there. +- Credential-adapter results and build arguments that only those later results + can reveal as private. File and managed-environment admission now uses the same + preparation as start, including decryption and permitted shell lookups; a + missing project key is a closed preflight blocker. - Image pinning. `hack up` reviews images by content ID after it has admitted and pulled them; the preflight reviews the references as written. - What only a start can find: an image that cannot be pulled or admitted, a @@ -1531,67 +1530,87 @@ by an older bundle, so clean up such a graph with the bundle that started it. - A reference to a Hack-managed value of the same service, for example `${TOKEN:-fallback}` when Hack delivers `TOKEN` to that service, refuses with `execution_environment_owner`. Managed values are delivered privately and are - never interpolated into commands, users, health checks or other values, and a + never interpolated into commands, users or health checks, and a default is never used in their place. Declare the key bare (`TOKEN:` or `- TOKEN`) - to receive the managed value. - -### Shell variables in service environment - -Compose reads a variable reference from the shell that runs it. The native runtime -reads no ambient environment, so two forms of a service `environment` entry could -never start on native. `hack up` and `hack restart` resolve those forms from the -caller's environment. Only source variables named by the project's Compose file -are read; the ambient environment is never forwarded as a whole. - -Every native start prints one line to stderr naming the forwarded source -variables, or `(none)`, including detached and JSON starts. Names are deduplicated -and sorted; values are never printed. Foreground starts also report unset -references separately. There is no consent prompt in this feature. - -| Entry | Shell has the variable | Shell does not | -| --- | --- | --- | -| `API_KEY: ${API_KEY}` or `API_KEY: $API_KEY` | The service gets the shell's value. | The service gets an empty value, as with Compose. | -| `- TOKEN` (a bare name, or `TOKEN:` with no value) | The service gets the shell's value. | The service gets an empty value. Compose would leave it unset instead. | - -An entire `environment: null` declaration, including an `environment:` key with -only comments beneath it, is empty. Preflight and start both accept it; managed -keys are still added as private references when configured for the service. - -A value read from the shell is never written to the reviewed Compose document, a -plan, a receipt, an env fingerprint or a log. The entry stays a reference and the -value travels in the same private delivery as Hack managed env. Only the variables -these entries name are read; nothing else of your environment is. The reviewed -document is the same whether or not the variable is set, so a start from another -shell resumes the same plan. - -What stays as it was: - -- **A reference with a default or another operator** (`${PORT:-3000}`, `${X:+alt}`, - `${X-d}`, `${X:?err}`). The runtime resolves it without your shell, so - `${PORT:-3000}` is `3000` even when your shell sets `PORT`. Compose would use the - shell's value. Write `PORT: ${PORT}` to take the value from the shell. -- **A reference inside a longer string** (`http://${HOST}:3000`), and a reference in - any other field (`image`, `ports`, `command`, `volumes`, labels). A reference with - no default there still fails the start with `execution_environment_missing`. -- **A variable Hack manages for the service.** It comes from managed env, never from - the shell, and an entry that references a managed variable is still refused by the - runtime (`execution_environment_owner`). -- **`COMPOSE_PROJECT_NAME`.** The runtime supplies it; it is not read from the shell. -- **Hack's own variables** (`HACK_*`), as the entry's name or as the variable it - references. They include the key that decrypts a project's managed secrets, so a - Compose file can never copy one into a container. -- **Cloud names** (`AWS_*`, `GOOGLE_*`, `CLOUDSDK_*`). Long-lived shell credentials - are not copied into containers; use the scoped AWS and gcloud delivery described - below. The rule goes by prefix, so it also covers non-secret names such as - `AWS_REGION` and `GOOGLE_CLOUD_PROJECT`. - - An entry under either rule is left as written. It does not pass through as it - would with Compose: the start still fails with `execution_environment_missing`, - as it did before. -- **A project `.env` file.** Hack does not parse it for shell references. A launch - runtime such as Bun may preload dotenv into its process environment. Only - explicitly referenced, non-excluded names are read from that environment; - a missing reference is reported as unset. + to receive the managed value. Global managed values can interpolate privately + inside service environment and env-file values. + +### Environment files and interpolation + +Native `up` and `restart` accept Compose `env_file` strings, lists and +long-form `path`/`required`/`format` entries. Optional missing files are skipped; +unreadable or unsafe files still refuse. `format: raw` preserves quotes and dollar +signs. Files are read relative to the selected Compose file and must stay inside +this project root. Reads do not follow symlinks, refuse other-owner or multi-link +files, and require owner-read access with no group/other write access. Ordinary +0644 files work; Hack never changes permissions. + +The automatic interpolation file is `.env` beside the Compose file (normally +`.hack/.env`). A sibling `.env.state.json` causes a refusal: the file may be +obsolete output of `hack env materialize`. Use current Hack env layers, then +remove obsolete materialized output deliberately. A root `.env` outside the +Compose directory is not read; start and preflight report once that it is +ignored. Dotenv alone does not inject values. The adjacent `.env` is now read on +every native start and preflight, even with managed env. Symlinks, other-owner or +group/other-writable files and project ancestors refuse, as do `COMPOSE_*` and +`HACK_*` entries and dotted/dashed identifiers. Use an owned regular file, +deliberately correct permissions and rename unsupported keys, or use managed env. +An ignored root `.env` still may not overlap a native build context or +Dockerfile input: Buildx reads raw host files. Such a build now refuses even +without `env_file` or adjacent `.env`; use disjoint inputs or a prebuilt image. + +| Source | Precedence and delivery | +| --- | --- | +| Hack-managed values | Win the final container destination, as Hack's Compose override does. Global values also win interpolation; service-scoped values are not ambient sources. | +| Service `environment` | Overrides env-file values, including explicit empty values and unresolved bare keys. | +| `env_file` | Processed in order. Later resolved declarations win. An unresolved bare file key keeps the previous value. | +| Image defaults | Apply unless replaced or explicitly unset by service environment. | + +Within service environment, managed globals win over allowed caller-shell values, +then dotenv. Within env files those sources win over resolved service environment, +then earlier file assignments. Native supports Compose interpolation operators, +embedded references, single-quoted file literals and double-quoted file escapes. +A missing plain reference produces an empty value with a names-only notice. A +bare service key with no source unsets the key, including an image default; it no +longer substitutes empty. The selected runtime must confirm these unset names. +Changing source presence can change reviewed names and plan identity; changing +only values cannot. +Service defaults such as `${PORT:-3000}` now take an allowed exported shell +value. Shell presence can therefore change the normalized declaration and plan +identity; the value stays private. Unset the variable to retain the fallback. + +File and shell values stay in memory outside env fingerprints and travel only +through private stdin and the expiring guest allocation. The normalized document +contains bare names, never those values. Each start reports forwarded shell names +on stderr, including JSON starts, without values. Only referenced, allowed shell +variables are read. Hack key material is refused; `HACK_*` copies from files or +shell references are refused. Authored public runtime-metadata literals retain +their existing behavior. Shell +credential names under `AWS_*`, `GOOGLE_*` and `CLOUDSDK_*` remain excluded. +Literal cloud entries in an explicit env file are allowed unless a selected +credential adapter owns that destination. + +In projects declaring `env_file` or containing an adjacent `.env`, interpolation +of private sources in public fields (images, commands, ports, +labels, builds, mounts, users and healthchecks) refuses before effects, including +when the expression contains a default. It never silently selects a different +fallback. Unset the ambient source or write the public value literally. Projects +with neither file source retain their previous public-field defaults, including +`${PORT:-3000}` when the caller exports `PORT`. +Runtime-bound `COMPOSE_PROJECT_NAME` remains public; private expressions +cannot combine it with other sources. Pure literal/default expressions that read +no private source retain their existing runtime behavior. File-derived private +values inherit the launcher's restart-policy limits; an edit to file values needs +an explicit `hack restart` to reach an existing run. + +See [the full environment-file contract](../native/environment-files.md) for file +bounds, materialization, source exclusions and intentionally unsupported forms. +Syntax errors name the file and line; notices name the destination key rather +than tokens from values. The build-argument guard compares only nonempty managed +values explicitly marked `secret: true`, with no minimum length; ordinary file +and shell values do not join it. Preflight now reads allowed caller variables +and decrypts managed env; a missing key is a reported blocker with recovery +guidance, not a failure to produce the report. Three consequences of the private delivery: @@ -1983,10 +2002,10 @@ under native `hack up -d` above. `hack restart --detach` still refuses with `E_USAGE`: run `hack restart` in the foreground in its own terminal and use `hack ps` and `hack logs` from another. Routed services and host dependencies require the explicit native selections described below. Unsupported Compose or environment input -(for example a service `env_file`, or a project `.env` without managed env) and +(for example an unsafe environment file or private interpolation in a public field) and unreviewed route labels refuse with `E_NATIVE_PROJECT_UNSUPPORTED`. The input refusal names its cause and, where known, the service and Compose field (for -example `service api field env_file`), never values or file contents. Secret +example `service api field env_file` and its source path), never values or file contents. Secret managed env without a project env key refuses with `E_CONFIG_INVALID` and names the recovery actions; other unreadable managed env refuses with `E_CONFIG_INVALID` without layer excerpts. Invalid branch diff --git a/docs/guides/v5-release-notes.md b/docs/guides/v5-release-notes.md index 1a68266b9..e0c736453 100644 --- a/docs/guides/v5-release-notes.md +++ b/docs/guides/v5-release-notes.md @@ -22,6 +22,13 @@ its original behavior. | Change | What to do | | --- | --- | +| **Native reads `.hack/.env` on every start and preflight, even with managed env.** Symlinks, other-owner or group/other-writable files and project ancestors refuse, as do `COMPOSE_*`/`HACK_*` keys and dotted/dashed identifiers. | Use an owned regular file, deliberately correct permissions, remove control entries and rename unsupported identifiers, or move values into managed env. A sibling `.env.state.json` also refuses possible materialized output; deliberately retire that output after moving to managed env. | +| **Native service-environment defaults take allowed shell values.** Shell presence can change normalized declarations and plan identity, without storing values in that identity. | Unset the exported name to retain the authored fallback. In projects with `env_file` or adjacent `.env`, public-field references to ambient values now refuse even with a default: unset the source or write a public literal. Without those file sources, public defaults keep their previous behavior. | +| **An unresolved bare service-environment key is now unset instead of empty, including in projects without env files.** It removes an image default; an older runtime that cannot confirm the unset names refuses the review. Reserved runtime/adapter-owned names keep their existing behavior. | Use a matching current CLI/runtime bundle. Write `KEY: ""` when you intend an explicitly empty value instead of removing the key. | +| **Fresh native `hack exec` now delivers allowed caller-shell and selected-service file values privately.** Shell delivery also changes projects without env files; exec values may differ from those used at service startup. | Run exec from the intended shell and environment selection. Restart the service when changed inputs should also replace its startup environment; exec alone does not do that. | +| **Native removes `dotenv_without_managed_env`.** A project-root `.env` outside the Compose directory is now ignored with one names-only notice. | Move required inputs beside Compose (normally `.hack/.env`) or into managed env. Dotenv alone does not inject service variables. | +| **Native preflight reads the caller's allowed environment and decrypts managed env.** A missing key becomes a structured blocker; the report remains usable. | Run from the intended shell and recover the existing project key when asked. Do not generate a replacement key for existing ciphertext. | +| **Native env-file paths must stay separate from build inputs.** Contexts, Dockerfiles and Dockerfile-specific ignore inputs may not overlap declared env files, adjacent `.env` or a present project-root `.env`, including optional and inactive paths. A root `.env` triggers this refusal even when there is no `env_file` or adjacent `.env`; ignored for interpolation does not exclude it from `COPY .`. | Use disjoint build inputs or prebuilt images. The build-argument value guard stays limited to nonempty managed `secret: true` values, with no minimum length; plain file and shell values do not widen it. See [the full rules](../native/environment-files.md). | | **Native plain `logs SERVICE --follow` output changes.** It now prints application-text lines, matching a finite snapshot, instead of adding a `SERVICE \| ` prefix. Both service streams go to stdout; Hack notices go to stderr. | Stop parsing the old plain prefix. Use `--json` and the log event's `entry.service`, `entry.stream` and `entry.message` when a script needs those fields. Use `--timestamps` to request Engine timestamps. | | **Env listings hide values by default, including migrated plaintext.** Ordinary `env` / `env list` text and JSON output are metadata. JSON omits `value` and reports `has_value`. | Use `has_value` for availability checks. For a value a script actually needs, call `hack env get KEY` with the same `--env` and `--service`; that command prints plaintext. Bulk `--show-values` requires interactive stdin and stdout; `--show-secrets` remains an alias. | | **Migration cleanup preserves live keys (HACK-2172).** A legacy store's `keyPath` can no longer make the current project/shared key a cleanup target. Current config and YAML inputs are protected too, including filesystem aliases. | Keep key and env backups before migration. On next.16 or an earlier prerelease, do not run `doctor --migrate-env-config` without a private readable copy of `.hack.secret.key`. The fix does not recover a key already removed. | @@ -32,6 +39,18 @@ its original behavior. | **Native route readiness can be inferred.** A complete standard Caddy route with no authored healthcheck gets a reviewed HTTP probe and 240 seconds of initial grace. A complete HTTP 200–499 response establishes endpoint readiness, not application correctness. | Review the route port and startup logs. Explicit healthchecks are not replaced, and inferred readiness does not satisfy `depends_on: condition: service_healthy`. The grace does not extend the whole graph's deadline. | | **Native start waits for bridge sockets after a stopped pool resumes.** An immediate start no longer fails merely because provider readiness preceded the bridge listener. | Use normal native `up`; startup still requires a successful bridge connection within its deadline. Do not replace readiness checks with a fixed sleep. | +Detached native starts do not show per-key "unresolved input" notices because +the owner's stdio is ignored. Use a foreground start or native preflight to +inspect those notices; only forwarded shell names and the fixed ignored-root +dotenv notice reach the detached caller. + +Native `.env` admission refuses **every** `COMPOSE_*` key, broader than Compose's +recognized control-variable set. Native reserves the whole prefix so new Compose +controls cannot silently become ordinary interpolation inputs: project, file and +profile selection stay explicit. Rename application-specific variables without +that prefix. This rule applies to the adjacent interpolation `.env`, not every +literal service `env_file` entry. + ### Env scripts and migrated plaintext Replace noninteractive bulk reveal with explicit single-key reads; do not log or diff --git a/docs/native/environment-files.md b/docs/native/environment-files.md new file mode 100644 index 000000000..2a3c7cae0 --- /dev/null +++ b/docs/native/environment-files.md @@ -0,0 +1,227 @@ +# Native Compose environment files + +Design for HACK-2062. The Compose backend keeps its existing behavior. + +![Public admission and private delivery paths](environment-files.svg) + +[Diagram source](environment-files.mmd). + +Native accepts service `env_file` as a path string, a list of path strings or +long-form entries (`path`, optional `required`, optional `format`). `required` +defaults to true; false skips an absent file, never an unreadable or unsafe file. +Omitted or empty `format` follows Compose dotenv syntax; the literal `default` +is not a Compose format. `format: raw` does not unquote +or interpolate values. Unknown options and formats refuse before hooks, image +acquisition or VM work. Repeating a file is allowed and preserves list order. +Files of inactive-profile services are not acquired; their paths remain excluded +from snapshots and live sharing, including missing required paths. +The reviewed runtime active set must agree before private delivery or effects; +an active service with unprepared file inputs refuses. + +Paths are literal and relative to the Compose file, within the current canonical +project root. Absolute paths inside that root are accepted. Escaping the root or +following a symlink inside it is refused. Files must be regular, single-link, +owned by the current user, owner-readable and not writable by group or others; +ordinary 0644 checkouts work. Directories inside the root must be owned by the +current user and not writable by group or others. Reads are bounded and recheck +file and directory identities, size, permissions and timestamps. Native changes +no file or permission. Bounds: 256 KiB and 4,096 lines per file; 64 file references +and 1 MiB total source text per preparation. Existing private delivery limits +(32 services, 256 keys and 32 KiB per service, 256 KiB complete stdin) also apply. + +The automatic interpolation file is `.env` beside the selected Compose file: +normally `.hack/.env`, matching Hack's Compose project directory. An adjacent +`.env.state.json` marks possible Hack-materialized output; native refuses to use +that `.env`, whether the state is current, stale or unreadable, rather than copy +another selection's materialized values. The operator can use current Hack env +layers and remove obsolete materialized output deliberately. A root `.env` outside +the Compose directory is not read or refused solely for existing; start and +preflight report once that it is ignored, without reading its contents. Native does not +search another worktree or overlay dotenv. Every `.env` uses the same guards. + +Container precedence, highest first: + +1. Hack-managed environment, matching Hack's generated Compose environment override. +2. Explicit service `environment`, including empty values and unresolved bare keys. +3. `env_file`, in list order; the last resolved declaration for a name wins. +4. Image environment defaults for names not otherwise declared. + +A bare env-file name inherits from the interpolation context if present; otherwise +it leaves earlier file assignments intact. A bare service environment key resolves +from the interpolation context; when unresolved it removes that key, including an +image default. Native removes the key from normalized `environment`, then carries its name in +explicit normalized metadata and public engine unset entries, separate from private +value delivery. This changes projects without env files too: earlier native +starts delivered an empty value for an unresolved bare service key. Write +`KEY: ""` when that empty value is intentional. The runtime must echo the exact +unset names during review; older runtimes that cannot do so refuse. Use a +matching current CLI/runtime bundle. Ordinary missing private bindings must +continue to refuse. Reserved runtime/adapter-owned names retain their behavior. + +Interpolation lookup order is Hack-managed global values, the caller's allowed +shell variables, then `.env`. Service-scoped managed values win the final service +destination but are not ambient interpolation sources. For env-file expansion, +that project context wins over resolved service `environment`, then current and +earlier file assignments. A current file assignment replaces the previous file's +assignment before the next line is read. `.env` alone injects no container variables. +Service-environment defaults now select an allowed exported shell value before +their authored fallback. Shell presence can change the normalized declaration +from a public default expression to a private bare name, changing plan identity; +the shell value itself never enters that identity. Unset the variable to retain +the fallback. This also applies to projects without env files. +Service-environment defaults resolve only against the project context: an env-file +`HOST` does not replace `${HOST:-localhost}` with its file value. Defaults that +reference newly file-backed names are frozen before runtime private ownership. +The same rule applies to public command, entrypoint, user, healthcheck test and +extra-host hostname/target defaults: only authored fallback text is frozen, never file +values. Other runtime-bound sources in the same expression refuse rather than +freeze a different value. Public expressions unrelated to file bindings stay intact. +Native already evaluates `extra_hosts` hostnames in both list and map forms; +map keys receive the same private-source admission, and duplicate frozen keys refuse. +Other mapping keys retain Compose's ordinary key semantics. +File-local cloud names can be used by later assignments without reading those +names from the shell. + +Support includes export prefixes, `=` and `:` separators in default format, +comments, single-quoted literals, double-quoted escapes, multiline quoted values, +`$NAME`, `${NAME}`, defaults (`-`, `:-`), alternatives (`+`, `:+`), required references +(`?`, `:?`), nesting to four levels, and `$$`. An unset `${NAME}` yields empty with a +notice naming the destination key, file and assignment line; neither notices nor +refusals print reference tokens extracted from a value. Required-reference errors +omit the authored error word. Syntax refusals name the file and physical line. +Identifiers must match `[A-Za-z_][A-Za-z0-9_]*`; dotted, dashed, leading-digit and +Unicode identifiers refuse. Raw +format follows the separately qualified Compose raw parser. Hack key material and +`HACK_*` cannot enter containers via files or shell references. Authored public +runtime-metadata literals keep their existing behavior. Cloud credential +names are not read from the shell; explicit literal file entries remain available +unless the selected credential adapter owns that service's name. Compose control +names (`COMPOSE_*`) and protected `HACK_*` keys +in `.env` refuse with names only rather than silently change the selected project. +This deliberately reserves every `COMPOSE_*` key, broader than Compose's known +control-variable set, so future controls cannot silently become ordinary +interpolation inputs. Native keeps project, file and profile selection explicit. +Rename application-specific variables without that prefix. This prefix-wide +rule is for the adjacent interpolation `.env`; literal service `env_file` entries +are not all subject to it. + +The approved first unit resolves file values only in service `environment` and +env-file values. In a project that declares `env_file` or has an adjacent `.env`, +if a public field references a name defined by `.env`, managed +global env or allowed shell input, it refuses before effects, even if the reference +has a default. Unset the ambient source or write the intended public value +literally. Projects with neither `env_file` nor adjacent `.env` retain their +previous public-field behavior: `${PORT:-3000}` stays authored even when the +caller exports `PORT`; the runtime uses its existing public default rules. +Public fields +include images, commands, ports, labels, mounts, users, healthchecks and builds. +The lead approved this privacy boundary: materializing such values would otherwise +put them in plans, argv or receipts. + +Start and `doctor --native-preflight` use the same parser, lookup, permissions and +normalizer, including the same allow-listed shell environment and managed values. +Preflight may therefore need the same project decryption key as start. A missing +key produces a structured `managed_env_unreadable` blocker and recovery guidance; +the report remains usable and continues other checks. It returns +only names and paths. Missing required files, malformed input, unsafe reads and +unsupported public interpolation produce closed diagnostics with service, field +and source path, destination key and line where available. Optional absent files produce no blocker. Parser excerpts, +interpolated error words and raw OS diagnostics never leave the boundary. + +The normalized document removes `env_file` and replaces file-derived environment +values with bare name declarations. Values remain in memory outside managed-env +fingerprints, then travel through the bounded private stdin envelope and the +expiring guest allocation/launcher. They never enter CLI argv, public JSON, plans, +plan identities, receipts, logs or generated files. Original env-file declarations +remain source exclusions in Rust even when normalization removes them, including +absent optional targets, inactive-service files, raw inputs +and duplicate paths. Exclusions compare paths with Unicode canonical normalization +and case folding, so host path aliases cannot enter snapshot archives or live source updates. +Ambiguous matching directory entries refuse, and environment inputs cannot own +source ignore-rule files through an alias. `.env*` remain excluded by the existing source policy. Tests +must cover snapshot and live-share exclusion when an optional file appears later. + +Private delivery retains its existing limitations: it cannot use automatic +restart policies, and fresh values need a new incarnation. A file-value-only edit +has no public fingerprint; the operator must run `hack restart` to deliver it to +an existing run. Literal/default-only expressions that consult no private source +stay on the existing public path and retain their restart behavior and identity. +Fresh `hack exec` resolves the selected service's file values again using the +same allowed caller context and delivers them privately to that exec process. +These fresh values may differ from the running container's startup values. +Fresh exec now also delivers allowed caller-shell values rather than managed +values alone, including in projects without env files. Run it from the intended +shell and environment selection. Use restart when changed inputs should replace +the running service's startup environment; exec alone does not update it. + +Native builds read raw host contexts. Start and preflight therefore refuse a +native build whose context, Dockerfile or Dockerfile-specific ignore input +overlaps any declared env-file path, the selected automatic `.env` path or a +present project-root `.env`. Ignoring root dotenv for interpolation does not +exclude it from Buildx's raw context: even with no `env_file` or adjacent `.env`, +a root-context build refuses when root `.env` exists. This is an explicit +compatibility exception to prevent `COPY .` from embedding ignored dotenv. +Root dotenv is detected by presence only, without reading its contents. +Optional absent and inactive-service declarations remain protected; authored +ignore rules do not lift this refusal. ASCII paths compare without case, and +non-ASCII components are treated as potentially aliased whenever disjointness +cannot be proved. Use separate build inputs, a prebuilt image, or Compose for +overlapping projects. Disjoint contexts and ordinary builds with no env-file or +dotenv input retain their behavior. Fresh selected-service exec does not build. +The build-argument value guard keeps its existing policy: exact equality with +any nonempty managed value explicitly marked `secret: true` refuses, with no +minimum length. Plain env-file and shell values do not widen that guard because +they have no secret metadata. `NODE_ENV=production` in an env file may coexist +with the literal build argument `production`. Build arguments enter image +history: never put a credential in a build argument merely because it came from +a plain file. File-path separation still applies. +Managed-env stale checks remain advisory and do not read ephemeral file inputs; +file-value-only changes are not detected by their managed fingerprints. + +Unused Compose `x-*` extensions are ignored; Hack-consumed `x-hack-*` fields +still receive admission checks. Exact AWS/ADC mounts recognized by the credential +adapters keep their private adapter path and are never expanded into public plans. +Unreadable reserved bare names remain runtime/adapter-owned, rather than becoming +explicit image-default removals. + +Out of scope: override/include/extends support, CLI `--env-file`, dotenv selection +of Compose files or profiles, shell execution/command substitution, public-field +interpolation without an explicit future public-data contract, external files, +automatic permission repair, Compose-backend changes and automatic secret-value +change detection. Tests use sentinel values and recorded Docker Compose v5.6.1 / +compose-go v2.16.1 results, alongside the documented [environment-file rules](https://compose-spec.github.io/compose-spec/05-services.html#env_file) +and [interpolation rules](https://docs.docker.com/compose/how-tos/environment-variables/variable-interpolation/). +Actual guest delivery is qualified separately on a Mac. + +## Changes for earlier v5 prereleases + +- `.hack/.env` is read on every native start and preflight, including when managed + env already exists. A symlink, other-owner file, group/other-writable file or + ancestor inside the project root refuses. Replace the symlink with an owned + regular file and deliberately correct permissions, or move values to managed + env. Remove `COMPOSE_*`/`HACK_*` entries; rename dotted or dashed identifiers + and their references to the accepted identifier form. +- Service defaults take allowed shell values as described above. Public-field + interpolation gains the strict boundary only for file-enabled projects; use + literal public fields or remove the ambient source. +- Unresolved bare service keys now remove the key instead of delivering empty, + including without env files. Use `KEY: ""` for explicit empty and a matching + runtime that can confirm unset names; an older runtime refuses that review. +- Fresh `hack exec` now delivers current allowed shell and selected-service file + values privately. Shell delivery also affects projects without env files; run + from the intended shell and restart if startup values must change too. +- The former `dotenv_without_managed_env` refusal is removed. A project-root + `.env` outside the Compose directory is ignored with one notice. Move needed + variables beside Compose (normally `.hack/.env`) or into managed env. + Builds overlapping this ignored root file now refuse; use disjoint build + inputs or a prebuilt image, even when the project has no other env-file input. +- Preflight now reads the caller's allowed environment and decrypts managed env. + Recover the existing project key when reported missing; do not generate a + replacement for existing ciphertext. + +Foreground notices go to stderr. Detached owners discard stderr rather than +write it to a log file, so detached starts do not show per-key "unresolved input" +notices. Use a foreground start or preflight to inspect them. Only validated +forwarded shell names and the fixed root-dotenv notice cross IPC to the caller. +Tokens referenced only inside file values are excluded from the forwarded-names +notice as well. diff --git a/docs/native/environment-files.mmd b/docs/native/environment-files.mmd new file mode 100644 index 000000000..b172724b8 --- /dev/null +++ b/docs/native/environment-files.mmd @@ -0,0 +1,15 @@ +--- +title: Native environment input boundaries +--- +flowchart TD + Sources["Files and allowed sources"] --> Guard["Guarded source reads"] + Guard --> Normalize["Shared environment preparation"] + Normalize -->|"names and public declarations"| Review["Read-only runtime admission"] + Normalize -->|"values in memory"| Private["Bounded private delivery"] + Normalize -->|"private reference in public field"| Refuse["Closed refusal"] + Review -->|"admitted active services"| Private + Private --> Stdin["Private stdin envelope"] + Stdin --> Guest["Expiring guest allocation"] + Guest --> Launch["Service launcher"] + class Refuse fail + class Review,Launch ok diff --git a/docs/native/environment-files.svg b/docs/native/environment-files.svg new file mode 100644 index 000000000..d145d7610 --- /dev/null +++ b/docs/native/environment-files.svg @@ -0,0 +1,66 @@ +Native environment input boundariesnames and public declarationsvalues in memoryprivate reference in public fieldadmitted active servicesFiles and allowed sourcesGuarded source readsShared environmentpreparationRead-only runtimeadmissionBounded private deliveryClosed refusalPrivate stdin envelopeExpiring guest allocationService launcher \ No newline at end of file diff --git a/docs/versions/v5/native/preflight.md b/docs/versions/v5/native/preflight.md index d414c4240..d614465c3 100644 --- a/docs/versions/v5/native/preflight.md +++ b/docs/versions/v5/native/preflight.md @@ -18,6 +18,17 @@ environment, can select different adaptation and dependency files. Authored native project files take a separate admission path; use `hack config validate` for them. +Changes intended for next.17: preflight reads allowed caller-shell variables, +decrypts managed env and reads `.env` beside Compose (normally `.hack/.env`), +using the same file checks as native start. A missing decryption key is a +structured blocker with recovery guidance; the command still produces a report. +A project-root `.env` outside the Compose directory is ignored with one notice. +Detached starts omit per-key "unresolved input" notices because owner stdio is +ignored; use this preflight or a foreground start to inspect missing inputs. +See the [environment-file rules](https://github.com/hack-dance/hack/blob/9cd2053affe4f3307703e97d5bb4cd7fc3049830/docs/native/environment-files.md) and +[migration notes](https://github.com/hack-dance/hack/blob/9cd2053affe4f3307703e97d5bb4cd7fc3049830/docs/guides/v5-release-notes.md#next17-what-changed-and-what-to-do) +before upgrading an earlier prerelease. + ## Read the result The CLI checks the selected native adaptation and project-wide host aliases, diff --git a/docs/versions/v5/native/start.md b/docs/versions/v5/native/start.md index 6f2beae66..f6683426a 100644 --- a/docs/versions/v5/native/start.md +++ b/docs/versions/v5/native/start.md @@ -69,6 +69,11 @@ If blocked, fix the named field and rerun. Do not remove a check or delete state to force a start. Keep the project on Compose if it needs unsupported behavior. See [Troubleshooting](../troubleshooting.md) and [the preflight reference](preflight.md). +Env-file support intended for next.17 changes input checks and shell-default +selection. Before upgrading, read the [environment-file rules](https://github.com/hack-dance/hack/blob/9cd2053affe4f3307703e97d5bb4cd7fc3049830/docs/native/environment-files.md) +and [migration notes](https://github.com/hack-dance/hack/blob/9cd2053affe4f3307703e97d5bb4cd7fc3049830/docs/guides/v5-release-notes.md#next17-what-changed-and-what-to-do). +They explain which files are read, which declarations refuse and how to fix them. + ## Start, check and stop ```sh diff --git a/packages/runtime-core/src/project/compose/mod.rs b/packages/runtime-core/src/project/compose/mod.rs index f4584a478..a1fa8dbe8 100644 --- a/packages/runtime-core/src/project/compose/mod.rs +++ b/packages/runtime-core/src/project/compose/mod.rs @@ -13,6 +13,38 @@ mod service; use fields::*; pub(crate) use project_name::VARIABLE as COMPOSE_PROJECT_NAME; +/// Explicit names-only image-variable removals, disjoint from every environment +/// declaration. A missing private binding must never be reinterpreted as unset. +pub(super) fn environment_unsets( + value: &Map, +) -> Result, CandidateError> { + let mut result = std::collections::BTreeSet::new(); + let Some(raw) = value.get("x-hack-unset-environment") else { + return Ok(result); + }; + let fail = || { + problem( + "invalid_environment_unset", + "Environment removals must be at most 256 distinct variable names, separate from declared environment bindings; values omitted.", + ) + }; + let names = raw + .as_array() + .filter(|names| names.len() <= 256) + .ok_or_else(fail)?; + let declared = redacted_map(value.get("environment"), true)?; + for name in names { + let name = name + .as_str() + .filter(|name| variable_name(name)) + .ok_or_else(fail)?; + if declared.contains_key(name) || !result.insert(name.to_owned()) { + return Err(fail()); + } + } + Ok(result) +} + pub(super) fn identifier(value: &str) -> Result<(), CandidateError> { fields::identifier(value) } @@ -37,6 +69,7 @@ pub(super) fn original_environment_files( object(service)?.get("env_file"), "original.env_file", &mut Vec::new(), + true, )?); } Ok(result) diff --git a/packages/runtime-core/src/project/compose/service.rs b/packages/runtime-core/src/project/compose/service.rs index 4aee19da9..23994c7d4 100644 --- a/packages/runtime-core/src/project/compose/service.rs +++ b/packages/runtime-core/src/project/compose/service.rs @@ -460,6 +460,7 @@ pub(super) fn environment_files( value: Option<&Value>, field: &str, diagnostics: &mut Vec, + ownership_only: bool, ) -> Result, CandidateError> { let values: Vec<&Value> = match value { None => vec![], @@ -468,13 +469,17 @@ pub(super) fn environment_files( }; let mut result = Vec::new(); for value in values { + let required = !ownership_only + && match value { + Value::Object(m) => boolean(m.get("required"), true)?, + _ => true, + }; let path = if let Value::Object(m) = value { keys(m, &["path", "required", "format"], field, diagnostics); - if !boolean(m.get("required"), true)? - || m.get("format") - .is_some_and(|v| v.as_str() != Some("default")) + if m.get("format") + .is_some_and(|v| !matches!(v.as_str(), Some("" | "raw"))) { - diagnostics.push(Diagnostic::error("unsupported_env_file_mode",field,"Only required default-format env_file references are modeled. Their contents are never read during planning.")); + diagnostics.push(Diagnostic::error("unsupported_env_file_mode",field,"Only default and raw env_file formats are modeled. Their contents are never read during planning.")); } string( m.get("path") @@ -483,32 +488,40 @@ pub(super) fn environment_files( } else { string(value)? }; - let resolved = source::resolve(project, base, path, true)?; + let resolved = if required { + source::resolve(project, base, path, true)? + } else { + source::resolve_optional_env_file(project, base, path)? + }; if resolved .file_name() - .is_some_and(|name| name == ".gitignore" || name == ".ignore") + .and_then(|name| name.to_str()) + .is_some_and(|name| { + matches!(source::folded_name(name).as_str(), ".gitignore" | ".ignore") + }) { return Err(problem( "conflicting_environment_owner", "An environment input cannot also own source ignore rules.", )); } - let metadata = fs::symlink_metadata(&resolved) - .map_err(|_| problem("invalid_env_file", "Cannot inspect the env_file reference."))?; - if !metadata.is_file() || metadata.nlink() != 1 { - return Err(problem( - "invalid_env_file", - "env_file must refer to a regular file; its contents are not loaded.", - )); + match fs::symlink_metadata(&resolved) { + Ok(metadata) if metadata.is_file() && metadata.nlink() == 1 => {} + Err(error) if !required && error.kind() == std::io::ErrorKind::NotFound => {} + _ => { + return Err(problem( + "invalid_env_file", + "env_file must refer to a regular file; its contents are not loaded.", + )); + } } let relative = relative(project, &resolved); - if result.contains(&relative) { - return Err(problem( - "duplicate_env_file", - "Duplicate environment file ownership is not accepted.", - )); + // This is an ownership inventory, not a parser execution order. Repeated + // Compose inputs are read in order by the private normalizer, while one + // exclusion per path protects snapshot and future live-share contents. + if !result.contains(&relative) { + result.push(relative); } - result.push(relative); } Ok(result) } @@ -734,6 +747,7 @@ pub(super) fn compile( identifier(name)?; let field = format!("services.{name}"); let m = object(value)?; + let environment_unsets = super::environment_unsets(m)?.into_iter().collect(); keys( m, &[ @@ -745,6 +759,7 @@ pub(super) fn compile( "volumes", "depends_on", "environment", + "x-hack-unset-environment", "extra_hosts", "env_file", "healthcheck", @@ -1139,8 +1154,8 @@ pub(super) fn compile( profiles:declared_profiles,active, dependencies:dependencies(m.get("depends_on"),&format!("{field}.depends_on"),diagnostics)?, mounts:declared_mounts, - environment:redacted_map(m.get("environment"),true)?,extra_hosts, - environment_files:environment_files(project,base,m.get("env_file"),&format!("{field}.env_file"),diagnostics)?, + environment:redacted_map(m.get("environment"),true)?,environment_unsets,extra_hosts, + environment_files:environment_files(project,base,m.get("env_file"),&format!("{field}.env_file"),diagnostics,false)?, environment_precedence:"Compose environment keys override explicitly listed env_file inputs; bare keys and references remain unresolved; no ambient/.env values are loaded".into(), healthcheck, ports:ports(m.get("ports"),&format!("{field}.ports"),diagnostics)?,exposed_ports,networks,network_aliases,network_mode:if ["project-networks", "none"].contains(&network_mode) {network_mode.into()} else {"[unsupported]".into()}, diff --git a/packages/runtime-core/src/project/inputs.rs b/packages/runtime-core/src/project/inputs.rs index 24ab98c26..9b1db427e 100644 --- a/packages/runtime-core/src/project/inputs.rs +++ b/packages/runtime-core/src/project/inputs.rs @@ -697,6 +697,24 @@ fn compile_inner( private, )?; let (environment, scoped) = resolver.environment(environment.as_ref(), private)?; + let unsets = &service.environment_unsets; + if unsets.iter().any(|key| { + private.is_some_and(|values| values.contains_key(key)) + || review.plan.registry.as_ref().is_some_and(|registry| { + service.dependency_cache.is_some() + && registry.required_environment.contains(key) + }) + }) { + return Err(error( + "execution_environment_owner", + "Environment removals cannot overlap private or registry-required bindings; values omitted.", + )); + } + let mut environment = environment; + // Docker Engine Config.Env accepts bare names as explicit removals from + // image defaults. These entries contain no value and need no launcher. + environment.extend(unsets.iter().cloned()); + environment.sort(); if !scoped.is_empty() { managed_environment.insert(name.clone(), scoped); } diff --git a/packages/runtime-core/src/project/live_source.rs b/packages/runtime-core/src/project/live_source.rs index 4a8657341..8a2cd727f 100644 --- a/packages/runtime-core/src/project/live_source.rs +++ b/packages/runtime-core/src/project/live_source.rs @@ -460,6 +460,88 @@ mod tests { } } + #[test] + fn optional_env_file_aliases_created_after_review_stay_out_of_source_and_live_updates() { + let (_fixture, candidate, source) = fixture(); + let path = source.join("compose.yaml"); + let mut original = super::super::yaml::parse(&fs::read(&path).unwrap()).unwrap(); + original["services"]["deps"]["env_file"] = serde_json::json!([ + {"path":"config/Private.txt", "required":false, "format":"raw"}, + {"path":"config/café.vars", "required":false}, + {"path":"Config/Straße.vars", "required":false} + ]); + let original_bytes = serde_json::to_vec(&original).unwrap(); + fs::write(&path, &original_bytes).unwrap(); + original["services"]["deps"] + .as_object_mut() + .unwrap() + .remove("env_file"); + let normalized = serde_json::to_vec(&original).unwrap(); + let namespace = candidate.plan(&source).unwrap().namespace; + let original_hash = format!("{:x}", Sha256::digest(&original_bytes)); + let capture = || { + let plan = project::plan_normalized( + &candidate, + project::NormalizedComposeOptions { + branch: None, + project: &source, + compose_file: Path::new("compose.yaml"), + profiles: &[], + expected_namespace: &namespace, + expected_compose_sha256: &original_hash, + compose_bytes: &normalized, + }, + ) + .unwrap(); + let snapshot = snapshot::capture_plan(&plan.plan).unwrap(); + assert!( + snapshot + .receipt() + .entries + .iter() + .all(|entry| !entry.path.starts_with("config/") && entry.path != ".env") + ); + assert!( + !snapshot + .archive() + .unwrap() + .windows(b"PRIVATE_".len()) + .any(|bytes| bytes == b"PRIVATE_") + ); + (plan, snapshot) + }; + let (initial, snapshot) = capture(); + let contract = Contract::from_plan(&initial.plan, snapshot.receipt()).unwrap(); + fs::create_dir(source.join("config")).unwrap(); + for path in [ + "config/private.txt", + "config/cafe\u{301}.vars", + "config/STRASSE.vars", + ] { + fs::write(source.join(path), "PRIVATE_FUTURE_CANARY").unwrap(); + } + fs::write(source.join(".env"), "PRIVATE_DOTENV_CANARY").unwrap(); + let (with_file, snapshot) = capture(); + contract + .verify(&with_file.plan, snapshot.receipt()) + .unwrap(); + fs::write( + source.join("config/private.txt"), + "PRIVATE_CHANGED_CANARY_LONGER", + ) + .unwrap(); + let (changed_value, snapshot) = capture(); + assert_eq!(with_file.plan_id, changed_value.plan_id); + contract + .verify(&changed_value.plan, snapshot.receipt()) + .unwrap(); + assert!( + !serde_json::to_string(&changed_value) + .unwrap() + .contains("CANARY") + ); + } + #[test] fn content_add_remove_and_excluded_secrets_do_not_freeze_plan_identity() { let (_fixture, candidate, source) = fixture(); diff --git a/packages/runtime-core/src/project/mod.rs b/packages/runtime-core/src/project/mod.rs index 0deb479ae..2fcae573a 100644 --- a/packages/runtime-core/src/project/mod.rs +++ b/packages/runtime-core/src/project/mod.rs @@ -291,6 +291,9 @@ pub struct ServicePlan { pub dependencies: BTreeMap, pub mounts: Vec, pub environment: BTreeMap, + /// Explicit names removed from image defaults, separate from private bindings. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub environment_unsets: Vec, #[serde(default, skip_serializing_if = "Vec::is_empty")] pub extra_hosts: Vec, pub environment_files: Vec, diff --git a/packages/runtime-core/src/project/source.rs b/packages/runtime-core/src/project/source.rs index f928acc31..69b38659a 100644 --- a/packages/runtime-core/src/project/source.rs +++ b/packages/runtime-core/src/project/source.rs @@ -184,6 +184,27 @@ pub fn resolve( base: &Path, value: &str, allow_credentials: bool, +) -> Result { + resolve_inner(project, base, value, allow_credentials, false) +} + +/// Resolve ownership of an optional env-file path, including a currently absent +/// tail. This grants no read or mount authority: callers must retain the path as +/// a source exclusion even if the file or its parent appears after review. +pub(super) fn resolve_optional_env_file( + project: &Path, + base: &Path, + value: &str, +) -> Result { + resolve_inner(project, base, value, true, true) +} + +fn resolve_inner( + project: &Path, + base: &Path, + value: &str, + allow_credentials: bool, + optional: bool, ) -> Result { if value.is_empty() || value.contains('$') || value.contains('\0') || value.contains('~') { return Err(problem( @@ -224,12 +245,18 @@ pub fn resolve( let mut current = project.to_owned(); for part in relative.components() { current.push(part); - let metadata = fs::symlink_metadata(¤t).map_err(|_| { - problem( - "missing_source", - "Declared source does not exist or cannot be inspected.", - ) - })?; + let metadata = match fs::symlink_metadata(¤t) { + Ok(metadata) => metadata, + Err(error) if optional && error.kind() == std::io::ErrorKind::NotFound => { + return Ok(resolved); + } + Err(_) => { + return Err(problem( + "missing_source", + "Declared source does not exist or cannot be inspected.", + )); + } + }; if metadata.file_type().is_symlink() { return Err(problem( "source_symlink", @@ -350,6 +377,11 @@ pub fn inventory( )]; let mut entries = Vec::new(); let mut excluded_paths = Vec::new(); + // Ownership declarations may use an alias accepted by a case-insensitive or + // Unicode-normalizing host. Match directory entries by the same collision + // key used for source admission, including currently absent optional files. + let environment_files: BTreeSet<_> = environment_files.iter().map(|p| folded_name(p)).collect(); + let mut excluded_environment_names = BTreeSet::new(); let mut folded = BTreeSet::new(); let mut ignore_files = BTreeMap::new(); let mut visited = 0; @@ -384,7 +416,14 @@ pub fn inventory( "Source names must be valid UTF-8.", ) })?; - if excluded(path) || environment_files.contains(text) { + let environment_file = environment_files.contains(&folded_name(text)); + if environment_file && !excluded_environment_names.insert(folded_name(text)) { + return Err(problem( + "source_case_collision", + "Environment input paths collide under Unicode canonical normalization and case folding.", + )); + } + if excluded(path) || environment_file { excluded_paths.push(text.to_owned()); continue; } @@ -535,6 +574,31 @@ mod tests { assert_ne!(folded_name("src/資料"), folded_name("src/other")); } + #[test] + fn ambiguous_environment_file_aliases_refuse_inventory() { + let root = std::env::temp_dir().join(format!( + "hack-env-alias-{}-{}", + std::process::id(), + crate::next_test_scratch() + )); + fs::create_dir(&root).unwrap(); + fs::write(root.join("Values.txt"), "PRIVATE_UPPER_CANARY").unwrap(); + fs::write(root.join("values.txt"), "PRIVATE_LOWER_CANARY").unwrap(); + let distinct = fs::metadata(root.join("Values.txt")).unwrap().ino() + != fs::metadata(root.join("values.txt")).unwrap().ino(); + let result = inventory(&root, &BTreeSet::from(["VALUES.txt".into()]), &mut vec![]); + fs::remove_dir_all(&root).unwrap(); + if distinct { + let error = result.unwrap_err(); + assert_eq!(error.code, "source_case_collision"); + assert!(!error.message.contains("PRIVATE_")); + } else { + let selection = result.unwrap(); + assert!(selection.entries.is_empty()); + assert_eq!(selection.excluded_paths.len(), 1); + } + } + #[test] fn directory_alias_graph_rejects_ancestry_and_mutual_cycles() { assert!(!acyclic_directories( diff --git a/packages/runtime-core/src/provider/graph/cache_provenance.rs b/packages/runtime-core/src/provider/graph/cache_provenance.rs index 8d324e0eb..9fce7c71f 100644 --- a/packages/runtime-core/src/provider/graph/cache_provenance.rs +++ b/packages/runtime-core/src/provider/graph/cache_provenance.rs @@ -247,7 +247,13 @@ impl Session<'_, '_> { let resource = self.receipt.resources[&format!("container:{service}")].clone(); let value = inspect_resource(&self.engine, &self.receipt, &resource)?.ok_or_else(refused)?; - self.verify_config(service, &value)?; + Self::verify_config( + &self.receipt, + service, + &self.configs[service], + &self.expected_environment[service], + &value, + )?; for key in keys { let volume = &self.receipt.resources[&key]; verify(&self.engine, &self.receipt, volume)?; diff --git a/packages/runtime-core/src/provider/graph/cold_environment_tests.rs b/packages/runtime-core/src/provider/graph/cold_environment_tests.rs new file mode 100644 index 000000000..ad5e843f7 --- /dev/null +++ b/packages/runtime-core/src/provider/graph/cold_environment_tests.rs @@ -0,0 +1,75 @@ +use super::*; + +#[test] +fn cold_create_inspection_accepts_only_compiled_removal_markers_from_real_engine() { + for capture in image_environment::engine_removal_captures() { + let fixture = Fixture::new(); + let home = Fixture::new(); + let candidate = Candidate::discover(&home.0).unwrap(); + state::write( + &fixture.0.join("compose.yaml"), + &json!({"services":{"app":{ + "image":capture["imageId"],"network_mode":"none", + "environment":{"HQ_PLAIN":"plain"}, + "x-hack-unset-environment":[capture["removedKey"]] + }}}), + ) + .unwrap(); + let review = project::plan(&candidate, fixture.options()).unwrap(); + let inputs = project::inputs::compile( + &candidate, + fixture.options(), + &review.plan_id, + &BTreeMap::new(), + ) + .unwrap(); + let readiness = BTreeMap::from([("app".to_owned(), Condition::Started)]); + let prepared = + config::prepare(inputs, &readiness, &"a".repeat(32), &"b".repeat(32), None).unwrap(); + let config = &prepared.configs["app"]; + let requested = config["Env"].as_array().unwrap(); + assert_eq!(requested.len(), 2); + for entry in capture["requestedEnvironment"].as_array().unwrap() { + assert!(requested.contains(entry)); + } + let receipt: Receipt = serde_json::from_value(json!({ + "version":1,"run":"a".repeat(32),"owner":"b".repeat(32), + "namespace":review.plan.namespace,"plan_id":review.plan_id, + "phase":"preparing","readiness":readiness,"resources":prepared.resources + })) + .unwrap(); + let expected = + image_environment::compose(&capture["imageEnvironment"], &config["Env"]).unwrap(); + let mut inspected = json!({"Config":config,"HostConfig":config["HostConfig"],"State":{"Running":false},"NetworkSettings":{"Networks":{}}}); + inspected["Config"]["Env"] = capture["inspectedEnvironment"].clone(); + // This is the same gate cold start calls between create and start. + Session::verify_config(&receipt, "app", config, &expected, &inspected).unwrap(); + for extra in [ + json!("UNREQUESTED"), + capture["removedKey"].clone(), + json!("HQ_PLAIN=other"), + ] { + let mut wrong = inspected.clone(); + wrong["Config"]["Env"].as_array_mut().unwrap().push(extra); + assert_eq!( + Session::verify_config(&receipt, "app", config, &expected, &wrong) + .unwrap_err() + .code, + "graph_config_mismatch" + ); + } + let mut restored = inspected.clone(); + let removed = capture["removedKey"].as_str().unwrap(); + for entry in restored["Config"]["Env"].as_array_mut().unwrap() { + if entry.as_str() == Some(removed) { + *entry = json!(format!("{removed}=restored")); + } + } + assert!(Session::verify_config(&receipt, "app", config, &expected, &restored).is_err()); + let mut changed_default = inspected.clone(); + changed_default["Config"]["Env"][2] = json!("PATH=/different"); + assert!( + Session::verify_config(&receipt, "app", config, &expected, &changed_default).is_err() + ); + } +} diff --git a/packages/runtime-core/src/provider/graph/image_environment.rs b/packages/runtime-core/src/provider/graph/image_environment.rs index 0336a7ec2..88aad6c79 100644 --- a/packages/runtime-core/src/provider/graph/image_environment.rs +++ b/packages/runtime-core/src/provider/graph/image_environment.rs @@ -7,48 +7,124 @@ fn refused() -> CandidateError { "Pinned image or container environment is malformed or differs; values omitted.", ) } -fn entries(value: &Value) -> Result, CandidateError> { +/// In-memory expectation bound to the pinned image and compiled public overrides. +/// Removal names must survive composition: an arbitrary bare inspected name is +/// not evidence that the reviewed configuration requested its removal. +#[derive(Default)] +pub(super) struct Expected { + values: BTreeMap, + removed: BTreeSet, +} + +fn entries( + value: &Value, + removed: &BTreeSet, +) -> Result, CandidateError> { if value.is_null() { return Ok(BTreeMap::new()); } let array = value.as_array().ok_or_else(refused)?; let mut result = BTreeMap::new(); + let mut seen = BTreeSet::new(); for entry in array { + let entry = entry.as_str().ok_or_else(refused)?; let (key, value) = entry - .as_str() - .and_then(|v| v.split_once('=')) - .ok_or_else(refused)?; + .split_once('=') + .map_or((entry, None), |(key, value)| (key, Some(value))); if key.is_empty() || key.contains('\0') - || value.contains('\0') - || result.insert(key.into(), value.into()).is_some() + || value.is_some_and(|value| value.contains('\0')) + || !seen.insert(key) { return Err(refused()); } + if let Some(value) = value { + result.insert(key.into(), value.into()); + } else if !removed.contains(key) { + return Err(refused()); + } } Ok(result) } -pub(super) fn compose( - image: &Value, - public: &Value, -) -> Result, CandidateError> { - let mut expected = entries(image)?; - expected.extend(entries(public)?); +pub(super) fn compose(image: &Value, public: &Value) -> Result { + // Image defaults must be assignments; only compiled public overrides can + // authorize bare removal markers in the engine's inspected configuration. + let mut expected = Expected { + values: entries(image, &BTreeSet::new())?, + removed: BTreeSet::new(), + }; + if !public.is_null() { + let mut seen = std::collections::BTreeSet::new(); + for entry in public.as_array().ok_or_else(refused)? { + let entry = entry.as_str().ok_or_else(refused)?; + let (key, value) = entry + .split_once('=') + .map_or((entry, None), |(key, value)| (key, Some(value))); + if key.is_empty() + || key.contains('\0') + || value.is_some_and(|value| value.contains('\0')) + || !seen.insert(key) + { + return Err(refused()); + } + if let Some(value) = value { + expected.values.insert(key.to_owned(), value.to_owned()); + } else { + expected.values.remove(key); + expected.removed.insert(key.to_owned()); + } + } + } Ok(expected) } -pub(super) fn verify( - expected: &BTreeMap, - actual: &Value, -) -> Result<(), CandidateError> { - if &entries(actual)? != expected { +pub(super) fn verify(expected: &Expected, actual: &Value) -> Result<(), CandidateError> { + if entries(actual, &expected.removed)? != expected.values { return Err(refused()); } Ok(()) } + +#[cfg(test)] +pub(super) fn engine_removal_captures() -> Vec { + // Docker Engine 29.9.2/API 1.56: public python image defaults and synthetic + // inputs only. No host paths, owner labels or private process environment. + let fixture: Value = serde_json::from_str(include_str!( + "../../../tests/fixtures/native-bare-env-engine.json" + )) + .unwrap(); + assert_eq!(fixture["engineVersion"], "29.9.2"); + assert_eq!(fixture["apiVersion"], "1.56"); + fixture["captures"].as_array().unwrap().clone() +} #[cfg(test)] mod tests { use super::*; #[test] + fn real_engine_inspection_preserves_requested_bare_removals_and_other_defaults() { + for capture in engine_removal_captures() { + let expected = compose( + &capture["imageEnvironment"], + &capture["requestedEnvironment"], + ) + .unwrap(); + assert_eq!( + expected.removed, + BTreeSet::from([capture["removedKey"].as_str().unwrap().to_owned()]) + ); + verify(&expected, &capture["inspectedEnvironment"]).unwrap(); + assert_eq!( + capture["processObservation"], + json!({"unset_absent":true,"plain_present":true,"other_defaults_present":true}) + ); + for entry in capture["imageEnvironment"].as_array().unwrap() { + let (key, value) = entry.as_str().unwrap().split_once('=').unwrap(); + if key != capture["removedKey"].as_str().unwrap() { + assert_eq!(expected.values.get(key).map(String::as_str), Some(value)); + } + } + } + } + #[test] fn pinned_defaults_and_public_overrides_match_exactly_independent_of_order() { let expected = compose( &json!(["PATH=/bin", "MODE=image", "EMPTY=image"]), @@ -83,7 +159,6 @@ mod tests { fn malformed_or_duplicate_fields_refuse_without_value_diagnostics() { for bad in [ json!(["KEY=canary", "KEY=second"]), - json!(["MISSING"]), json!(["=canary"]), json!(["KEY\u{0000}=canary"]), json!(["KEY=canary\u{0000}"]), @@ -91,10 +166,36 @@ mod tests { json!({"KEY":"canary"}), ] { for result in [compose(&bad, &Value::Null), compose(&Value::Null, &bad)] { - let failure = result.unwrap_err(); + let failure = result.err().unwrap(); assert!(!failure.message.contains("canary")); } - assert!(verify(&BTreeMap::new(), &bad).is_err()); + assert!(verify(&Expected::default(), &bad).is_err()); + } + assert!(compose(&json!(["MISSING"]), &Value::Null).is_err()); + assert!(verify(&Expected::default(), &json!(["MISSING"])).is_err()); + } + #[test] + fn explicit_bare_names_remove_image_defaults_but_empty_values_remain() { + let expected = compose( + &json!(["PATH=/bin", "REMOVE=image-default", "EMPTY=image-default"]), + &json!(["REMOVE", "EMPTY=", "MISSING"]), + ) + .unwrap(); + verify(&expected, &json!(["EMPTY=", "PATH=/bin"])).unwrap(); + verify( + &expected, + &json!(["EMPTY=", "PATH=/bin", "REMOVE", "MISSING"]), + ) + .unwrap(); + for actual in [ + json!(["EMPTY=", "PATH=/bin", "REMOVE=image-default"]), + json!(["EMPTY=", "PATH=/bin", "REMOVE="]), + json!(["EMPTY=", "PATH=/bin", "UNREQUESTED"]), + json!(["EMPTY=", "PATH=/bin", "REMOVE", "REMOVE"]), + json!(["EMPTY=", "PATH=/bin", "REMOVE", "REMOVE=value"]), + ] { + assert!(verify(&expected, &actual).is_err()); } + assert!(compose(&Value::Null, &json!(["REMOVE", "REMOVE=second"])).is_err()); } } diff --git a/packages/runtime-core/src/provider/graph/initializer_cache.rs b/packages/runtime-core/src/provider/graph/initializer_cache.rs index 471882c69..68559f60c 100644 --- a/packages/runtime-core/src/provider/graph/initializer_cache.rs +++ b/packages/runtime-core/src/provider/graph/initializer_cache.rs @@ -316,7 +316,13 @@ impl Release for Session<'_, '_> { }; let observed = inspect_resource(&self.engine, &self.receipt, resource)?.ok_or_else(refused)?; - self.verify_config(service, &observed)?; + Self::verify_config( + &self.receipt, + service, + &self.configs[service], + &self.expected_environment[service], + &observed, + )?; if observed["State"]["Status"] != "exited" || observed["State"]["Running"] != false || observed["State"]["Paused"] != false diff --git a/packages/runtime-core/src/provider/graph/mod.rs b/packages/runtime-core/src/provider/graph/mod.rs index 95a555d5e..a8a553563 100644 --- a/packages/runtime-core/src/provider/graph/mod.rs +++ b/packages/runtime-core/src/provider/graph/mod.rs @@ -764,7 +764,7 @@ struct Session<'a, 's> { root: PathBuf, receipt: Receipt, configs: BTreeMap, - expected_environment: BTreeMap>, + expected_environment: BTreeMap, restarting: bool, fresh_cache_completion: bool, cache_initializers: BTreeMap, @@ -802,12 +802,16 @@ impl Session<'_, '_> { fn fault_pause(&self, point: &str) -> Result<(), CandidateError> { fault_pause(&self.root, &self.receipt.run, point) } - fn verify_config(&self, service: &str, inspected: &Value) -> Result<(), CandidateError> { - image_environment::verify( - &self.expected_environment[service], - &inspected["Config"]["Env"], - )?; - let expected = &self.configs[service]; + // Pure create/inspect gate: explicit compiled inputs keep it testable without + // constructing a provider lease or a live guest engine. + fn verify_config( + receipt: &Receipt, + service: &str, + expected: &Value, + environment: &image_environment::Expected, + inspected: &Value, + ) -> Result<(), CandidateError> { + image_environment::verify(environment, &inspected["Config"]["Env"])?; for (key, value) in expected.as_object().expect("compiled config") { if key == "Env" { continue; @@ -842,8 +846,7 @@ impl Session<'_, '_> { )); } for network in networks.keys() { - let resource = self - .receipt + let resource = receipt .resources .values() .find(|resource| resource.kind == Kind::Network && resource.name == *network) @@ -1074,7 +1077,13 @@ impl Driver for Session<'_, '_> { self.prepare_probe(service)?; let inspected = inspect_resource(&self.engine, &self.receipt, &resource)? .ok_or_else(|| error("graph_resource_missing", "Restart resource is missing."))?; - self.verify_config(service, &inspected)?; + Self::verify_config( + &self.receipt, + service, + &self.configs[service], + &self.expected_environment[service], + &inspected, + )?; let id = resource.id.as_deref().ok_or_else(|| { error( "graph_replay_refused", @@ -1160,7 +1169,13 @@ impl Driver for Session<'_, '_> { let inspected = inspect_resource(&self.engine, &self.receipt, &self.receipt.resources[&key])? .ok_or_else(|| error("graph_resource_missing", "Created container is missing."))?; - self.verify_config(service, &inspected)?; + Self::verify_config( + &self.receipt, + service, + &self.configs[service], + &self.expected_environment[service], + &inspected, + )?; self.reserve(&key, "start-intent")?; if let Some(lease) = self.leases.get(service) { lease.verified_path_with_guest(self.engine.guest(), service)?; @@ -1192,7 +1207,13 @@ impl Driver for Session<'_, '_> { let resource = &self.receipt.resources[&format!("container:{service}")]; let value = inspect_resource(&self.engine, &self.receipt, resource)? .ok_or_else(|| error("graph_resource_missing", "Owned service disappeared."))?; - self.verify_config(service, &value)?; + Self::verify_config( + &self.receipt, + service, + &self.configs[service], + &self.expected_environment[service], + &value, + )?; probes::observe(&self.engine, &self.receipt, service, &value) } } @@ -2650,7 +2671,7 @@ fn verify_images( configs: &mut BTreeMap, private_services: &BTreeSet, file_owners: &BTreeSet, -) -> Result>, CandidateError> { +) -> Result, CandidateError> { let mut expected = BTreeMap::new(); let mut refusals = process_refusals::Collector::default(); for resource in resources.values().filter(|r| r.kind == Kind::Container) { diff --git a/packages/runtime-core/src/provider/graph/native/runtime.rs b/packages/runtime-core/src/provider/graph/native/runtime.rs index f3d0ceb87..35b4237a8 100644 --- a/packages/runtime-core/src/provider/graph/native/runtime.rs +++ b/packages/runtime-core/src/provider/graph/native/runtime.rs @@ -530,7 +530,7 @@ fn verify_config( receipt: &Receipt, service: &str, expected: &Value, - environment: &BTreeMap, + environment: &image_environment::Expected, actual: &Value, ) -> Result<(), CandidateError> { image_environment::verify(environment, &actual["Config"]["Env"])?; @@ -768,7 +768,7 @@ struct Session<'a, B> { root: PathBuf, receipt: Receipt, configs: BTreeMap, - expected_environment: BTreeMap>, + expected_environment: BTreeMap, environments: BTreeMap, } impl Session<'_, B> { @@ -1784,7 +1784,7 @@ fn verify_native_images( resources: &BTreeMap, configs: &mut BTreeMap, private: &BTreeSet, -) -> Result>, CandidateError> { +) -> Result, CandidateError> { let mut expected = BTreeMap::new(); for resource in resources .values() diff --git a/packages/runtime-core/src/provider/graph/native/runtime/cold_environment_tests.rs b/packages/runtime-core/src/provider/graph/native/runtime/cold_environment_tests.rs new file mode 100644 index 000000000..4c0f9b8a2 --- /dev/null +++ b/packages/runtime-core/src/provider/graph/native/runtime/cold_environment_tests.rs @@ -0,0 +1,63 @@ +use super::*; + +#[test] +fn cold_start_uses_real_engine_removal_markers_and_refuses_unrequested_or_changed_env() { + for capture in image_environment::engine_removal_captures() { + let mut invalid = Vec::new(); + for extra in [ + json!("UNREQUESTED"), + capture["removedKey"].clone(), + json!("HQ_PLAIN=other"), + json!(17), + ] { + let mut actual = capture["inspectedEnvironment"].clone(); + actual.as_array_mut().unwrap().push(extra); + invalid.push(actual); + } + let mut restored = capture["inspectedEnvironment"].clone(); + let removed = capture["removedKey"].as_str().unwrap(); + for entry in restored.as_array_mut().unwrap() { + if entry.as_str() == Some(removed) { + *entry = json!(format!("{removed}=restored")); + } + } + invalid.push(restored); + let mut changed_default = capture["inspectedEnvironment"].clone(); + changed_default[2] = json!("PATH=/different"); + invalid.push(changed_default); + for (actual, accepted) in std::iter::once((capture["inspectedEnvironment"].clone(), true)) + .chain(invalid.into_iter().map(|value| (value, false))) + { + let mut project = basic(); + project["services"]["web"]["image"] = capture["imageId"].clone(); + let fixture = Fixture::new(project); + let (graph, mut session) = + fixture.session(fixture.prepared(json!({"web":{}}), &BTreeMap::new())); + // Supply the reviewed public removal intent and a pinned-image + // expectation; the fake engine returns the separately captured Env. + session.configs.get_mut("web").unwrap()["Env"] = + capture["requestedEnvironment"].clone(); + session.expected_environment.insert( + "web".into(), + image_environment::compose( + &capture["imageEnvironment"], + &session.configs["web"]["Env"], + ) + .unwrap(), + ); + session.backend.state.borrow_mut().inspected_environment = Some(actual); + let result = execution::run(&graph, &mut session, RUN_BUDGET); + let state = session.backend.state.borrow(); + assert!(state.effects.iter().any(|effect| effect == "create:web")); + if accepted { + result.unwrap(); + assert_eq!(session.receipt.phase, Phase::ReadyObserved); + assert!(state.effects.iter().any(|effect| effect == "start:web")); + } else { + assert_eq!(result.unwrap_err().code, "graph_config_mismatch"); + assert!(!state.effects.iter().any(|effect| effect == "start:web")); + assert_ne!(session.receipt.phase, Phase::ReadyObserved); + } + } + } +} diff --git a/packages/runtime-core/src/provider/graph/native/runtime/tests.rs b/packages/runtime-core/src/provider/graph/native/runtime/tests.rs index 72c687762..1c50328ff 100644 --- a/packages/runtime-core/src/provider/graph/native/runtime/tests.rs +++ b/packages/runtime-core/src/provider/graph/native/runtime/tests.rs @@ -3,6 +3,8 @@ use super::*; mod authored_exec; #[path = "run_tests.rs"] mod authored_run; +#[path = "cold_environment_tests.rs"] +mod cold_environment; use hack_config_compiler::environment::EnvMetadata; use std::{ cell::{Cell, RefCell}, @@ -671,6 +673,8 @@ fn foreground_cancellation_after_network_creation_prevents_container_effects_and #[derive(Default)] struct FakeState { + /// Real Engine Config.Env capture used at the create/inspect boundary. + inspected_environment: Option, log_reads: Vec<(String, u16)>, log_source_deadlines: Vec, log_data_deadlines: Vec, @@ -883,6 +887,9 @@ impl Backend for Fake { let id = format!("{:064x}", state.containers.len() + 1); let mut process = config.clone(); process.as_object_mut().unwrap().remove("HostConfig"); + if let Some(environment) = &state.inspected_environment { + process["Env"] = environment.clone(); + } let selected = resource.networks.as_ref().unwrap(); let mut attached = serde_json::Map::new(); for logical in selected { diff --git a/packages/runtime-core/src/provider/graph/probes.rs b/packages/runtime-core/src/provider/graph/probes.rs index 46e349f8d..bc52249b4 100644 --- a/packages/runtime-core/src/provider/graph/probes.rs +++ b/packages/runtime-core/src/provider/graph/probes.rs @@ -578,7 +578,13 @@ impl Session<'_, '_> { let resource = &self.receipt.resources[&format!("container:{name}")]; let value = inspect_resource(&self.engine, &self.receipt, resource)? .ok_or_else(|| error("graph_probe_identity", "Restart container disappeared."))?; - self.verify_config(name, &value)?; + Self::verify_config( + &self.receipt, + name, + &self.configs[name], + &self.expected_environment[name], + &value, + )?; if !matches!(observation(&value)?, Observation::Exited { .. }) { return Err(error( "graph_probe_running", diff --git a/packages/runtime-core/src/provider/graph/tests.rs b/packages/runtime-core/src/provider/graph/tests.rs index 2b536d119..dadbb12b8 100644 --- a/packages/runtime-core/src/provider/graph/tests.rs +++ b/packages/runtime-core/src/provider/graph/tests.rs @@ -1,4 +1,6 @@ use super::*; +#[path = "cold_environment_tests.rs"] +mod cold_environment; use std::{io::Read, path::Path, process::Command}; pub(super) struct Fixture(pub(super) PathBuf); impl Fixture { diff --git a/packages/runtime-core/tests/fixtures/native-bare-env-engine.json b/packages/runtime-core/tests/fixtures/native-bare-env-engine.json new file mode 100644 index 000000000..f302d6133 --- /dev/null +++ b/packages/runtime-core/tests/fixtures/native-bare-env-engine.json @@ -0,0 +1,61 @@ +{ + "engineVersion": "29.9.2", + "apiVersion": "1.56", + "captures": [ + { + "case": "image-default", + "removedKey": "PYTHON_VERSION", + "imageReference": "python:3.12-alpine", + "imageId": "sha256:c5ba3deef00c7758561a0f03c26f44f10589263d40e732b8748e0d1f38f6dd89", + "imageEnvironment": [ + "PATH=/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + "LANG=C.UTF-8", + "GPG_KEY=7169605F62C751356D054A26A821E680E5FA6305", + "PYTHON_VERSION=3.12.16", + "PYTHON_SHA256=b5796915dc45ff745cfb4e990a7e7c93d6f546328a102ce31448d69ebc37575c" + ], + "requestedEnvironment": ["HQ_PLAIN=plain", "PYTHON_VERSION"], + "inspectedEnvironment": [ + "HQ_PLAIN=plain", + "PYTHON_VERSION", + "PATH=/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + "LANG=C.UTF-8", + "GPG_KEY=7169605F62C751356D054A26A821E680E5FA6305", + "PYTHON_SHA256=b5796915dc45ff745cfb4e990a7e7c93d6f546328a102ce31448d69ebc37575c" + ], + "processObservation": { + "unset_absent": true, + "plain_present": true, + "other_defaults_present": true + } + }, + { + "case": "absent-image-key", + "removedKey": "HQ_NOT_IN_IMAGE", + "imageReference": "python:3.12-alpine", + "imageId": "sha256:c5ba3deef00c7758561a0f03c26f44f10589263d40e732b8748e0d1f38f6dd89", + "imageEnvironment": [ + "PATH=/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + "LANG=C.UTF-8", + "GPG_KEY=7169605F62C751356D054A26A821E680E5FA6305", + "PYTHON_VERSION=3.12.16", + "PYTHON_SHA256=b5796915dc45ff745cfb4e990a7e7c93d6f546328a102ce31448d69ebc37575c" + ], + "requestedEnvironment": ["HQ_PLAIN=plain", "HQ_NOT_IN_IMAGE"], + "inspectedEnvironment": [ + "HQ_PLAIN=plain", + "HQ_NOT_IN_IMAGE", + "PATH=/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + "LANG=C.UTF-8", + "GPG_KEY=7169605F62C751356D054A26A821E680E5FA6305", + "PYTHON_VERSION=3.12.16", + "PYTHON_SHA256=b5796915dc45ff745cfb4e990a7e7c93d6f546328a102ce31448d69ebc37575c" + ], + "processObservation": { + "unset_absent": true, + "plain_present": true, + "other_defaults_present": true + } + } + ] +} diff --git a/packages/runtime-core/tests/normalized_cli_contract.rs b/packages/runtime-core/tests/normalized_cli_contract.rs index ee9eda9f0..f41a5943f 100644 --- a/packages/runtime-core/tests/normalized_cli_contract.rs +++ b/packages/runtime-core/tests/normalized_cli_contract.rs @@ -280,3 +280,135 @@ fn normalized_capture_keeps_removed_original_credential_files_excluded() { .contains("PRIVATE_CLI_CANARY") ); } + +#[test] +fn optional_raw_and_repeated_env_files_keep_future_secret_paths_excluded() { + let mut fixture = Fixture::new(); + let original = ORIGINAL.replace(" image:", " env_file:\n - {path: future/private.txt, required: false, format: raw}\n - {path: future/private.txt, required: false}\n image:"); + fs::write(fixture.project.join("compose.yml"), &original).unwrap(); + fixture.hash = format!("{:x}", Sha256::digest(original.as_bytes())); + for exists in [false, true] { + if exists { + fs::create_dir(fixture.project.join("future")).unwrap(); + fs::write( + fixture.project.join("future/private.txt"), + "TOKEN=OPTIONAL_INPUT_CANARY", + ) + .unwrap(); + fs::write(fixture.project.join(".env"), "TOKEN=DOTENV_INPUT_CANARY").unwrap(); + } + let output = fixture.invoke(&["project", "plan"], &["--json"]); + assert!(output.status.success(), "{output:?}"); + let plan: Value = serde_json::from_slice(&output.stdout).unwrap(); + assert_eq!( + plan["plan"]["original_environment_files"], + serde_json::json!(["future/private.txt"]) + ); + let output = fixture.invoke( + &["project", "capture"], + &["--expect-plan", plan["plan_id"].as_str().unwrap(), "--json"], + ); + assert!(output.status.success(), "{output:?}"); + let capture: Value = serde_json::from_slice(&output.stdout).unwrap(); + assert!( + capture["entries"] + .as_array() + .unwrap() + .iter() + .all(|entry| entry["path"] != "future/private.txt" && entry["path"] != ".env") + ); + assert!( + !String::from_utf8(output.stdout) + .unwrap() + .contains("INPUT_CANARY") + ); + } +} + +#[test] +fn environment_file_ignore_rule_aliases_refuse_without_reading_values() { + let mut fixture = Fixture::new(); + fs::write( + fixture.project.join(".ignore"), + "IGNORE_RULE_PRIVATE_CANARY", + ) + .unwrap(); + for path in [".IGNORE", ".GitIgnore"] { + let original = ORIGINAL.replace( + " image:", + &format!(" env_file: [{{path: {path}, required: false}}]\n image:"), + ); + fs::write(fixture.project.join("compose.yml"), &original).unwrap(); + fixture.hash = format!("{:x}", Sha256::digest(original.as_bytes())); + let output = fixture.invoke(&["project", "plan"], &["--json"]); + assert!(!String::from_utf8_lossy(&output.stdout).contains("PRIVATE_CANARY")); + assert!(!String::from_utf8_lossy(&output.stderr).contains("PRIVATE_CANARY")); + assert_eq!(error(output), "conflicting_environment_owner"); + } +} + +#[test] +fn removed_inactive_required_env_file_retains_ownership_while_absent() { + let mut fixture = Fixture::new(); + fs::create_dir(fixture.project.join("future")).unwrap(); + let original = format!( + "{ORIGINAL} inactive:\n image: alpine:3.20\n profiles: [extra]\n env_file: future/inactive.txt\n" + ); + fs::write(fixture.project.join("compose.yml"), &original).unwrap(); + fs::write( + &fixture.normalized, + original.replace(" env_file: future/inactive.txt\n", ""), + ) + .unwrap(); + fixture.hash = format!("{:x}", Sha256::digest(original.as_bytes())); + let output = fixture.invoke(&["project", "plan"], &["--json"]); + assert!(output.status.success(), "{output:?}"); + let plan: Value = serde_json::from_slice(&output.stdout).unwrap(); + assert_eq!( + plan["plan"]["original_environment_files"], + serde_json::json!(["future/inactive.txt"]) + ); + assert_eq!(plan["plan"]["services"]["inactive"]["active"], false); + fs::write( + fixture.project.join("future/inactive.txt"), + "TOKEN=INACTIVE_FILE_CANARY", + ) + .unwrap(); + // File presence changes source metadata; an old review still must refuse. + assert_eq!( + error(fixture.invoke( + &["project", "capture"], + &["--expect-plan", plan["plan_id"].as_str().unwrap(), "--json"] + )), + "stale_plan" + ); + let current = fixture.invoke(&["project", "plan"], &["--json"]); + assert!(current.status.success(), "{current:?}"); + let current: Value = serde_json::from_slice(¤t.stdout).unwrap(); + assert_eq!( + current["plan"]["original_environment_files"], + plan["plan"]["original_environment_files"] + ); + let output = fixture.invoke( + &["project", "capture"], + &[ + "--expect-plan", + current["plan_id"].as_str().unwrap(), + "--json", + ], + ); + assert!(output.status.success(), "{output:?}"); + let capture: Value = serde_json::from_slice(&output.stdout).unwrap(); + assert!( + !capture["entries"] + .as_array() + .unwrap() + .iter() + .any(|entry| entry["path"] == "future/inactive.txt") + ); + assert!( + !String::from_utf8(output.stdout) + .unwrap() + .contains("INACTIVE_FILE_CANARY") + ); +} diff --git a/packages/runtime-core/tests/project_contract.rs b/packages/runtime-core/tests/project_contract.rs index 63ebfbd98..8faacbaf7 100644 --- a/packages/runtime-core/tests/project_contract.rs +++ b/packages/runtime-core/tests/project_contract.rs @@ -2300,6 +2300,83 @@ fn scoped_environment_refuses_cross_service_fallback_and_conflicting_owners() { assert!(!fixture.candidate.state_root.exists()); } +#[test] +fn explicit_environment_unsets_are_public_names_and_cannot_mask_missing_private_values() { + use project::inputs; + use std::collections::BTreeMap; + let fixture = Fixture::new( + "services:\n web:\n image: alpine:3.21\n environment: {PUBLIC: literal, PRIVATE: null}\n x-hack-unset-environment: [REMOVE, UNKNOWN]\n", + ); + let review = fixture.plan(); + let private = BTreeMap::from([( + "web".into(), + BTreeMap::from([("PRIVATE".into(), "UNSET_PRIVATE_CANARY".into())]), + )]); + let compiled = inputs::compile_scoped( + &fixture.candidate, + fixture.options(), + &review.plan_id, + &BTreeMap::new(), + &private, + ) + .unwrap(); + assert_eq!( + compiled.executable.services["web"].environment, + ["PUBLIC=literal", "REMOVE", "UNKNOWN"] + ); + assert!( + compiled + .executable + .service_requires_managed_environment("web") + ); + assert!( + !serde_json::to_string(&review) + .unwrap() + .contains("UNSET_PRIVATE_CANARY") + ); + let failure = inputs::compile_scoped( + &fixture.candidate, + fixture.options(), + &review.plan_id, + &BTreeMap::new(), + &BTreeMap::new(), + ) + .err() + .unwrap(); + assert_eq!(failure.code, "execution_environment_missing"); + let only_unsets = Fixture::new( + "services:\n web:\n image: alpine:3.21\n restart: unless-stopped\n x-hack-unset-environment: [REMOVE]\n", + ); + let review = only_unsets.plan(); + let compiled = inputs::compile_scoped( + &only_unsets.candidate, + only_unsets.options(), + &review.plan_id, + &BTreeMap::new(), + &BTreeMap::new(), + ) + .unwrap(); + assert_eq!(compiled.executable.services["web"].environment, ["REMOVE"]); + assert!(!compiled.executable.requires_managed_environment()); + for metadata in [ + "[PRIVATE]", + "[PUBLIC]", + "[REMOVE, REMOVE]", + "[bad-key]", + "false", + ] { + let conflict = Fixture::new(&format!( + "services:\n web:\n image: alpine:3.21\n environment: {{PUBLIC: value, PRIVATE: null}}\n x-hack-unset-environment: {metadata}\n" + )); + assert_eq!( + project::plan(&conflict.candidate, conflict.options()) + .unwrap_err() + .code, + "invalid_environment_unset" + ); + } +} + #[test] fn scoped_environment_cannot_interpolate_into_any_executable_field() { use project::inputs; diff --git a/src/backends/native-aws-environment.ts b/src/backends/native-aws-environment.ts index 619332ac3..adcb6cae6 100644 --- a/src/backends/native-aws-environment.ts +++ b/src/backends/native-aws-environment.ts @@ -128,6 +128,15 @@ function awsMount(value: unknown, home: string): boolean { return true; } +/** Exact authored mount forms consumed privately by AWS adaptation. No file is read. */ +export function nativeAwsCredentialMount(value: unknown): boolean { + try { + return awsMount(value, ""); + } catch { + return false; + } +} + function serviceEnvironment(value: unknown): Record { if (value === undefined) { return {}; diff --git a/src/backends/native-compose-env-syntax.ts b/src/backends/native-compose-env-syntax.ts new file mode 100644 index 000000000..50fe3a8ba --- /dev/null +++ b/src/backends/native-compose-env-syntax.ts @@ -0,0 +1,423 @@ +/** Compose dotenv syntax and interpolation. All inputs/results may be private. */ +const NAME = /^[A-Za-z_][A-Za-z0-9_]*/; +const ENTRY = + /^(?:export[ \t]+)?([A-Za-z_][A-Za-z0-9_]*)[ \t]*(?:([=:])[ \t]*(.*))?$/; +const COMMENT = /[ \t]+#.*$/; +const OPERATOR = /^(?::)?[-+?]/; +const BOM = /^\uFEFF/; +const ESCAPE_SEQUENCE = /\\(?:[abcfnrtv$"\\]|0\d{0,3})/g; +const OCTAL = /^[0-7]{3}$/; +const MAX_BYTES = 256 * 1024; +const MAX_LINES = 4096; +const ESCAPES: Readonly> = { + a: "\x07", + b: "\b", + f: "\f", + n: "\n", + r: "\r", + t: "\t", + v: "\v", + "\\": "\\", + $: "$$", +}; + +export type ComposeEnvLookup = (name: string) => string | undefined; +/** The destination assignment, never a name extracted from its private value. */ +export type ComposeEnvEntry = { readonly key: string; readonly line: number }; + +/** All syntactic source names, including unselected words, without reading values. */ +export function composeEnvironmentReferences( + text: string +): ReadonlySet { + const result = new Set(); + for (let cursor = 0; cursor < text.length; cursor += 1) { + if (text[cursor] !== "$") { + continue; + } + if (text[cursor + 1] === "$") { + cursor += 1; + continue; + } + const begin = cursor + (text[cursor + 1] === "{" ? 2 : 1); + const name = NAME.exec(text.slice(begin))?.[0]; + if (name !== undefined) { + result.add(name); + } + } + return result; +} + +/** Closed failures never include the input, an authored error word or values. */ +export class NativeEnvSyntaxError extends Error { + readonly line?: number; + constructor(line?: number) { + super( + `Invalid or unresolved Compose environment input${line === undefined ? "" : ` at line ${line}`}; values omitted.` + ); + this.line = line; + } +} +function invalid(): never { + throw new NativeEnvSyntaxError(); +} + +function braced( + text: string, + begin: number, + depth: number +): { readonly body: string; readonly next: number } { + let nesting = 1; + let cursor = begin; + while (cursor < text.length) { + if (text.startsWith("$$", cursor)) { + cursor += 2; + continue; + } + if (text.startsWith("${", cursor)) { + nesting += 1; + if (depth + nesting > 4) { + return invalid(); + } + cursor += 2; + continue; + } + if (text[cursor] === "}") { + nesting -= 1; + if (nesting === 0) { + return { body: text.slice(begin, cursor), next: cursor + 1 }; + } + } + cursor += 1; + } + return invalid(); +} + +function expression(opts: { + readonly body: string; + readonly lookup: ComposeEnvLookup; + readonly expand: (word: string) => string; + readonly onMissing?: (name: string) => void; +}): string { + const name = NAME.exec(opts.body)?.[0]; + if (name === undefined) { + return invalid(); + } + const suffix = opts.body.slice(name.length); + const value = opts.lookup(name); + if (suffix === "") { + if (value === undefined) { + opts.onMissing?.(name); + } + return value ?? ""; + } + const operator = OPERATOR.exec(suffix)?.[0]; + if (operator === undefined) { + return invalid(); + } + const present = + value !== undefined && (!operator.startsWith(":") || value !== ""); + const kind = operator.at(-1); + if (kind === "?" && !present) { + return invalid(); + } + if ((kind === "-" && !present) || (kind === "+" && present)) { + return opts.expand(suffix.slice(operator.length)); + } + return kind === "+" ? "" : (value ?? ""); +} + +function unbraced(opts: { + readonly text: string; + readonly cursor: number; + readonly lookup: ComposeEnvLookup; + readonly onMissing?: (name: string) => void; +}): { readonly value: string; readonly next: number } { + const name = NAME.exec(opts.text.slice(opts.cursor))?.[0]; + if (name === undefined) { + return { value: "$", next: opts.cursor }; + } + const value = opts.lookup(name); + if (value === undefined) { + opts.onMissing?.(name); + } + return { value: value ?? "", next: opts.cursor + name.length }; +} + +/** + * Compose operators, including unset versus empty, nested words and $$ escaping. + * The caller owns the lookup boundary; no ambient environment is consulted here. + */ +export function interpolateComposeEnvironment(opts: { + readonly text: string; + readonly lookup: ComposeEnvLookup; + readonly onMissing?: (name: string) => void; +}): string { + let remaining = MAX_BYTES; + function expand(text: string, depth: number): string { + if (depth > 4 || Buffer.byteLength(text) > MAX_BYTES) { + return invalid(); + } + let result = ""; + let cursor = 0; + function append(value: string): void { + remaining -= Buffer.byteLength(value); + if (remaining < 0 || value.includes("\0")) { + invalid(); + } + result += value; + } + while (cursor < text.length) { + const dollar = text.indexOf("$", cursor); + if (dollar < 0) { + append(text.slice(cursor)); + break; + } + append(text.slice(cursor, dollar)); + cursor = dollar + 1; + if (text[cursor] === "$") { + append("$"); + cursor += 1; + continue; + } + if (text[cursor] === "{") { + if (depth >= 4) { + return invalid(); + } + const found = braced(text, cursor + 1, depth); + append( + expression({ + body: found.body, + lookup: opts.lookup, + onMissing: opts.onMissing, + expand: (word) => expand(word, depth + 1), + }) + ); + cursor = found.next; + continue; + } + const found = unbraced({ + text, + cursor, + lookup: opts.lookup, + onMissing: opts.onMissing, + }); + append(found.value); + cursor = found.next; + } + return result; + } + return expand(opts.text, 0); +} + +/** Match compose-go's escape pass after escaped quote delimiters are removed. */ +function doubleQuotedEscapes(value: string): string { + return value.replace(ESCAPE_SEQUENCE, (match) => { + if (match.startsWith("\\0")) { + const digits = match.slice(2); + return OCTAL.test(digits) + ? String.fromCharCode(Number.parseInt(digits, 8)) + : `\\${digits}`; + } + const key = match[1] ?? ""; + return Object.hasOwn(ESCAPES, key) ? (ESCAPES[key] ?? match) : match; + }); +} + +/** Preserve escape pairs until the Compose double-quote escape pass. */ +function quotedCharacter(opts: { + readonly char: string; + readonly quote: string | undefined; + readonly escaped: boolean; +}): { readonly text: string; readonly escaped: boolean } { + if (opts.char === "\\" && !opts.escaped) { + return { text: "", escaped: true }; + } + return { + text: + opts.escaped && opts.char !== opts.quote ? `\\${opts.char}` : opts.char, + escaped: false, + }; +} + +/** Read a possibly multiline quoted dotenv value, with Compose quote escaping. */ +function quoted(opts: { + readonly lines: readonly string[]; + readonly start: number; + readonly value: string; +}): { readonly value: string; readonly end: number } { + const quote = opts.value[0]; + let decoded = ""; + let rest = opts.value.slice(1); + let position = 0; + let index = opts.start; + let escaped = false; + for (;;) { + const ch = rest[position]; + if (ch === undefined) { + index += 1; + if (index >= opts.lines.length) { + return invalid(); + } + decoded += escaped ? "\\\n" : "\n"; + escaped = false; + rest = opts.lines[index] ?? ""; + position = 0; + continue; + } + if (ch === quote && !escaped) { + const tail = rest.slice(position + 1).trim(); + if (tail !== "" && !tail.startsWith("#")) { + return invalid(); + } + return { + value: quote === '"' ? doubleQuotedEscapes(decoded) : decoded, + end: index, + }; + } + const next = quotedCharacter({ char: ch, quote, escaped }); + decoded += next.text; + escaped = next.escaped; + position += 1; + } +} + +function dotenvValue(opts: { + readonly raw: boolean; + readonly line: string; + readonly value: string; + readonly lines: readonly string[]; + readonly start: number; + readonly lookup: ComposeEnvLookup; + readonly onMissing?: (name: string) => void; +}): { readonly value: string; readonly end: number } { + if (opts.raw) { + const equals = opts.line.indexOf("="); + if (equals < 0) { + return invalid(); + } + return { value: opts.line.slice(equals + 1), end: opts.start }; + } + if (opts.value[0] !== "'" && opts.value[0] !== '"') { + return { + value: interpolateComposeEnvironment({ + text: opts.value.replace(COMMENT, "").trimEnd(), + lookup: opts.lookup, + onMissing: opts.onMissing, + }), + end: opts.start, + }; + } + const decoded = quoted(opts); + return { + value: + opts.value[0] === "'" + ? decoded.value + : interpolateComposeEnvironment({ + text: decoded.value, + lookup: opts.lookup, + onMissing: opts.onMissing, + }), + end: decoded.end, + }; +} + +function inheritBareFileName(opts: { + readonly key: string; + readonly lookup: ComposeEnvLookup; + readonly values: Map; + readonly onUnresolved?: (name: string) => void; +}): void { + const inherited = opts.lookup(opts.key); + if (inherited !== undefined) { + opts.values.set(opts.key, inherited); + } else if (!opts.values.has(opts.key)) { + opts.onUnresolved?.(opts.key); + } +} + +/** + * Sequential Compose assignments, seeded by earlier files. A bare name inherits + * only from project/service lookup; when unresolved it leaves prior entries + * alone, as compose-go does. Raw format does not unquote or interpolate values. + */ +export function parseComposeEnvironmentFile(opts: { + readonly text: string; + readonly raw?: boolean; + readonly lookup: (name: string, entry: ComposeEnvEntry) => string | undefined; + readonly initial?: ReadonlyMap; + readonly onEntry?: (entry: ComposeEnvEntry) => void; + readonly onMissing?: (name: string, entry: ComposeEnvEntry) => void; + /** Source admission after both project lookup and parsed-file assignments miss. */ + readonly onUnresolved?: (name: string, entry: ComposeEnvEntry) => void; +}): ReadonlyMap { + if (Buffer.byteLength(opts.text) > MAX_BYTES) { + throw new NativeEnvSyntaxError(1); + } + const nul = opts.text.indexOf("\0"); + if (nul >= 0) { + throw new NativeEnvSyntaxError(opts.text.slice(0, nul).split("\n").length); + } + const result = new Map(opts.initial); + const lines = opts.text.replace(BOM, "").replaceAll("\r\n", "\n").split("\n"); + // A final newline terminates the last physical line rather than adding one. + if (lines.at(-1) === "") { + lines.pop(); + } + if (lines.length > MAX_LINES) { + throw new NativeEnvSyntaxError(MAX_LINES + 1); + } + for (let index = 0; index < lines.length; index += 1) { + const line = (lines[index] ?? "").trimStart(); + if (line.trim() === "" || line.startsWith("#")) { + continue; + } + const entry = ENTRY.exec(line); + const key = entry?.[1]; + if (key === undefined) { + throw new NativeEnvSyntaxError(index + 1); + } + const location = { key, line: index + 1 }; + opts.onEntry?.(location); + const external = (name: string) => opts.lookup(name, location); + const unresolved = (name: string) => opts.onUnresolved?.(name, location); + const lookup = (name: string) => { + const found = external(name) ?? result.get(name); + if (found === undefined) { + unresolved(name); + } + return found; + }; + if (opts.raw && entry?.[2] !== "=" && entry?.[2] !== undefined) { + throw new NativeEnvSyntaxError(location.line); + } + if (entry?.[2] === undefined) { + inheritBareFileName({ + key, + lookup: external, + values: result, + onUnresolved: unresolved, + }); + continue; + } + let decoded: { readonly value: string; readonly end: number }; + try { + decoded = dotenvValue({ + raw: opts.raw ?? false, + line, + value: entry?.[3] ?? "", + lines, + start: index, + lookup, + onMissing: (name) => opts.onMissing?.(name, location), + }); + } catch (error: unknown) { + if (error instanceof NativeEnvSyntaxError) { + throw new NativeEnvSyntaxError(location.line); + } + throw error; + } + index = decoded.end; + result.set(key, decoded.value); + } + return result; +} diff --git a/src/backends/native-compose-environment.ts b/src/backends/native-compose-environment.ts new file mode 100644 index 000000000..338fea1c7 --- /dev/null +++ b/src/backends/native-compose-environment.ts @@ -0,0 +1,989 @@ +import { lstat, realpath } from "node:fs/promises"; +import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path"; +import { HACK_KEY_MATERIAL_VARIABLES } from "../lib/child-environment.ts"; +import { isRecord } from "../lib/guards.ts"; +import { NATIVE_ROOT_DOTENV_IGNORED_NOTICE } from "../lib/native-environment-notices.ts"; +import { nativeAwsCredentialMount } from "./native-aws-environment.ts"; +import { + type ComposeEnvEntry, + composeEnvironmentReferences, + interpolateComposeEnvironment, + NativeEnvSyntaxError, + parseComposeEnvironmentFile, +} from "./native-compose-env-syntax.ts"; +import { requireNativeEnvBuildSeparation } from "./native-env-build-context.ts"; +import { + NativeEnvFileRefusal, + nativeEnvFileSourceLocation, + readNativeEnvFileSource, +} from "./native-env-file-source.ts"; +import { nativeGcloudCredentialMount } from "./native-gcloud-environment.ts"; +import { nativeServiceActive } from "./native-project-builds.ts"; +import { shellVariableReadable } from "./native-shell-environment.ts"; + +const NAME = /^[A-Za-z_][A-Za-z0-9_]*$/; +const UNSETS = "x-hack-unset-environment"; +type Values = Readonly>; +export type NativeComposeEnvironmentService = { + readonly document: Record; + readonly environment: Record; +}; +type FileSpec = { + readonly path: string; + readonly required: boolean; + readonly raw: boolean; +}; +type Source = { readonly path: string; readonly text: string } | null; +type Options = { + readonly projectRoot: string; + readonly composeFile: string; + readonly compose: Record; + readonly services: Readonly>; + readonly managed: Readonly>; + readonly global: Values; + readonly env?: Readonly>; + readonly profiles?: readonly string[]; + readonly selectedServices?: readonly string[]; +}; +type Sources = { + readonly lookup: (name: string, reportName?: boolean) => string | undefined; + readonly read: (spec: FileSpec, service?: string) => Promise; + readonly dotenv: ReadonlyMap; + readonly shellNames: Set; + readonly automaticDotenv: string | undefined; +}; +type Notice = { readonly service?: string; readonly line: string }; + +function refused( + source: string, + reason: string, + service?: string, + entry?: ComposeEnvEntry +): NativeEnvFileRefusal { + return new NativeEnvFileRefusal({ source, reason, service, ...entry }); +} + +/** Preserve only closed reasons and the destination assignment, never a reference token. */ +function fileLookup(opts: { + readonly source: string; + readonly service?: string; + readonly entry: ComposeEnvEntry; + readonly lookup: () => string | undefined; +}): string | undefined { + try { + return opts.lookup(); + } catch (error: unknown) { + throw refused( + opts.source, + error instanceof NativeEnvFileRefusal + ? error.reason + : "the interpolation source could not be read", + opts.service, + opts.entry + ); + } +} + +function syntaxRefusal( + source: string, + error: unknown, + service?: string +): NativeEnvFileRefusal { + return new NativeEnvFileRefusal({ + source, + service, + line: error instanceof NativeEnvSyntaxError ? (error.line ?? 1) : 1, + reason: + "invalid dotenv syntax, unresolved required reference or source budget", + }); +} +function protectedName(name: string): boolean { + return name.startsWith("HACK_") || HACK_KEY_MATERIAL_VARIABLES.has(name); +} +function ownValue(values: Values, name: string): string | undefined { + return Object.hasOwn(values, name) ? values[name] : undefined; +} +async function present(path: string): Promise { + try { + await lstat(path); + return true; + } catch (error: unknown) { + if (isRecord(error) && error.code === "ENOENT") { + return false; + } + throw refused(".env", "the interpolation source cannot be inspected"); + } +} + +/** Identity comparison also catches case aliases on case-insensitive filesystems. */ +async function sameSource(a: string, b: string): Promise { + if (a === b) { + return true; + } + try { + const [left, right] = await Promise.all([lstat(a), lstat(b)]); + return left.dev === right.dev && left.ino === right.ino; + } catch (error: unknown) { + if (isRecord(error) && error.code === "ENOENT") { + return false; + } + throw refused(".env", "the interpolation source cannot be inspected"); + } +} +function fileSpecs(value: unknown, service: string): FileSpec[] { + if (value === undefined) { + return []; + } + const values = Array.isArray(value) ? value : [value]; + return values.map((entry) => { + if (typeof entry === "string") { + return { path: entry, required: true, raw: false }; + } + if ( + !(isRecord(entry) && typeof entry.path === "string") || + Object.keys(entry).some( + (key) => !["path", "required", "format"].includes(key) + ) || + (entry.required !== undefined && typeof entry.required !== "boolean") || + (entry.format !== undefined && + entry.format !== "" && + entry.format !== "raw") + ) { + throw refused( + "env_file", + "use a path or path/required/format entry with omitted, empty or raw format", + service + ); + } + return { + path: entry.path, + required: entry.required !== false, + raw: entry.format === "raw", + }; + }); +} + +/** Reject private interpolation; optionally freeze defaults before file names gain ownership. */ +function checkPublicFields(opts: { + readonly value: unknown; + readonly lookup: Sources["lookup"]; + readonly service?: string; + readonly field?: string; + readonly fileBindings?: ReadonlySet; + readonly managedNames?: ReadonlySet; +}): unknown { + if (typeof opts.value === "string") { + try { + const expanded = interpolateComposeEnvironment({ + text: opts.value, + lookup: (name) => { + if (opts.lookup(name) !== undefined) { + throw new NativeEnvFileRefusal({ + source: "Compose public field", + service: opts.service, + field: opts.field, + reason: + "private variables may only interpolate inside service environment or env-file values", + }); + } + return undefined; + }, + }); + const references = composeEnvironmentReferences(opts.value); + if ([...references].some((name) => opts.fileBindings?.has(name))) { + const runtimeName = [...references].find( + (name) => !shellVariableReadable(name) || opts.managedNames?.has(name) + ); + if (runtimeName !== undefined) { + throw new NativeEnvFileRefusal({ + source: "Compose public field", + service: opts.service, + field: opts.field, + reason: + "runtime-bound interpolation cannot combine with a file-backed default", + }); + } + return expanded.replaceAll("$", () => "$$"); + } + } catch (error: unknown) { + if (error instanceof NativeEnvFileRefusal) { + throw error; + } + throw new NativeEnvFileRefusal({ + source: "Compose public field", + service: opts.service, + field: opts.field, + reason: "invalid or over-budget interpolation", + }); + } + } else if (Array.isArray(opts.value)) { + return opts.value.map((value) => { + if ( + opts.field === "volumes" && + (nativeAwsCredentialMount(value) || nativeGcloudCredentialMount(value)) + ) { + return value; + } + return checkPublicFields({ ...opts, value }); + }); + } else if (isRecord(opts.value)) { + const result: Record = Object.create(null); + for (const [key, value] of Object.entries(opts.value)) { + // Native executes extra_hosts hostnames as well as targets. Other + // mapping keys keep their ordinary Compose key semantics. + const preparedKey = + opts.field === "extra_hosts" + ? checkPublicFields({ + ...opts, + value: key, + field: "extra_hosts.hostname", + }) + : key; + if ( + typeof preparedKey !== "string" || + Object.hasOwn(result, preparedKey) + ) { + throw new NativeEnvFileRefusal({ + source: "Compose public field", + service: opts.service, + field: opts.field, + reason: + "extra_hosts hostnames must remain distinct after interpolation; values omitted", + }); + } + result[preparedKey] = checkPublicFields({ ...opts, value }); + } + return result; + } + return opts.value; +} + +function ignoredExtension(field: string): boolean { + return field.startsWith("x-") && !field.startsWith("x-hack-"); +} + +/** The project selector may be a root alias; no links inside that root gain authority. */ +async function canonicalSourceOptions(opts: Options): Promise { + const member = relative(resolve(opts.projectRoot), resolve(opts.composeFile)); + if (member === ".." || member.startsWith(`..${sep}`) || isAbsolute(member)) { + throw refused( + "env_file", + "the Compose file must stay inside the project root" + ); + } + const root = await realpath(opts.projectRoot); + return { ...opts, projectRoot: root, composeFile: resolve(root, member) }; +} + +async function readAutomaticDotenv(opts: { + readonly automatic: string; + readonly read: Sources["read"]; + readonly lookup: Sources["lookup"]; + readonly notices: Notice[]; +}): Promise> { + // An absent automatic input grants no file access and adds no new directory + // admission to projects that use no env files. Explicit optional paths still + // go through the complete source guard, including their existing parents. + if (!(await present(opts.automatic))) { + return new Map(); + } + const source = await opts.read({ path: ".env", required: true, raw: false }); + if (source === null) { + return new Map(); + } + let dotenv: ReadonlyMap; + try { + dotenv = parseComposeEnvironmentFile({ + text: source.text, + lookup: (name, entry) => + fileLookup({ + source: ".env", + entry, + lookup: () => opts.lookup(name, false), + }), + onEntry: (entry) => { + if (entry.key.startsWith("COMPOSE_") || protectedName(entry.key)) { + throw refused( + ".env", + "control variables are not supported as dotenv inputs", + undefined, + entry + ); + } + }, + onMissing: (_name, entry) => + opts.notices.push({ + line: `Environment source ".env" line ${entry.line} key ${entry.key}: an unresolved reference is empty.`, + }), + onUnresolved: (name, entry) => { + if (!shellVariableReadable(name)) { + throw refused( + ".env", + "a source cannot be read from the caller's shell; use an explicit file value", + undefined, + entry + ); + } + }, + }); + } catch (error: unknown) { + if (error instanceof NativeEnvFileRefusal) { + throw error; + } + throw syntaxRefusal(".env", error); + } + return dotenv; +} + +async function environmentSources( + opts: Options, + notices: Notice[] +): Promise { + const shellNames = new Set(); + const cached = new Map(); + const automatic = join(dirname(opts.composeFile), ".env"); + const env = opts.env ?? {}; + let bytes = 0; + let references = 0; + const shell = (name: string, reportName: boolean): string | undefined => { + if (!(shellVariableReadable(name) && Object.hasOwn(env, name))) { + return undefined; + } + const value = env[name]; + if (typeof value === "string") { + if (reportName) { + shellNames.add(name); + } + return value; + } + return undefined; + }; + const read: Sources["read"] = async (spec, service) => { + references += 1; + if (references > 64) { + throw refused( + spec.path, + "more than 64 environment source references", + service + ); + } + const { path } = nativeEnvFileSourceLocation({ + ...opts, + path: spec.path, + service, + }); + if ( + (await present(`${automatic}.state.json`)) && + (await sameSource(path, automatic)) + ) { + throw refused( + spec.path, + "possible Hack materialized output: use current Hack env layers and deliberately remove obsolete .env and .env.state.json before retrying", + service + ); + } + // Cache one verified snapshot per literal path so repeated references and + // different services cannot consume different versions during preparation. + if (cached.has(path)) { + const prior = cached.get(path) ?? null; + if (prior === null && spec.required) { + throw refused(spec.path, "the required source is missing", service); + } + return prior; + } + const source = await readNativeEnvFileSource({ + ...opts, + path: spec.path, + required: spec.required, + service, + }); + if (source !== null) { + bytes += Buffer.byteLength(source.text); + } + if (bytes > 1024 * 1024) { + throw refused( + spec.path, + "environment source text exceeds 1 MiB", + service + ); + } + cached.set(path, source); + return source; + }; + const baseLookup = (name: string, reportName = true): string | undefined => { + if (protectedName(name)) { + throw refused( + "environment", + "protected variables cannot be interpolated" + ); + } + return ownValue(opts.global, name) ?? shell(name, reportName); + }; + const dotenv = await readAutomaticDotenv({ + automatic, + read, + lookup: baseLookup, + notices, + }); + return { + shellNames, + dotenv, + automaticDotenv: cached.has(automatic) ? automatic : undefined, + read, + lookup: (name, reportName) => + baseLookup(name, reportName) ?? dotenv.get(name), + }; +} + +type Resolution = { + readonly environment: Record; + readonly resolved: Map; + readonly delivered: Record; + readonly unset: string[]; + readonly unbound: Set; + readonly references: Map>; + readonly changed: boolean; +}; +type ServiceContext = { + readonly name: string; + readonly managed: Values; + readonly global: Values; + readonly lookup: Sources["lookup"]; + readonly notices: Notice[]; + readonly filePrivacy: boolean; + readonly composeFile: string; +}; +function missingNotice(opts: { + readonly context: ServiceContext; + readonly key: string; + readonly unset: boolean; + readonly source?: string; + readonly line?: number; +}): void { + opts.context.notices.push({ + service: opts.context.name, + line: `Service ${opts.context.name} source ${JSON.stringify(opts.source ?? opts.context.composeFile)}${opts.line === undefined ? "" : ` line ${opts.line}`} key ${opts.key}: an unresolved input is ${opts.unset ? "unset" : "empty"}.`, + }); +} +function expandServiceValue(opts: { + readonly text: string; + readonly key: string; + readonly context: ServiceContext; +}): { + readonly value: string; + readonly private: boolean; + readonly unbound: boolean; +} { + const { context } = opts; + let privateSource = false; + let unbound = false; + try { + const value = interpolateComposeEnvironment({ + text: opts.text, + onMissing: (name) => { + if (!shellVariableReadable(name)) { + unbound = true; + return; + } + privateSource = true; + missingNotice({ context, key: opts.key, unset: false }); + }, + lookup: (name) => { + if (name === "COMPOSE_PROJECT_NAME") { + unbound = true; + return undefined; + } + if ( + Object.hasOwn(context.managed, name) && + !Object.hasOwn(context.global, name) + ) { + throw refused( + "environment", + "service-scoped values are not interpolation sources", + context.name + ); + } + const found = context.lookup(name); + privateSource ||= found !== undefined; + return found; + }, + }); + if (privateSource && unbound) { + throw refused( + "environment", + "private interpolation cannot combine with unreadable or runtime-bound sources", + context.name + ); + } + if (privateSource && !shellVariableReadable(opts.key)) { + throw refused( + "environment", + "use an explicit literal env-file entry or the credential adapter for a reserved destination", + context.name + ); + } + return { value, private: privateSource, unbound }; + } catch (error: unknown) { + // With no file inputs, preserve next's admission: let the runtime evaluate + // declarations the new file parser cannot resolve, without reading private files. + if (!context.filePrivacy) { + return { value: opts.text, private: false, unbound: true }; + } + if (error instanceof NativeEnvFileRefusal) { + throw error; + } + throw refused( + "environment", + "invalid interpolation, required reference or NUL value", + context.name + ); + } +} +function serviceScalar(opts: { + readonly value: unknown; + readonly service: string; +}): number | boolean { + if (typeof opts.value === "number" || typeof opts.value === "boolean") { + return opts.value; + } + throw refused( + "environment", + "environment values must be scalar", + opts.service + ); +} + +function resolveBareService(opts: { + readonly context: ServiceContext; + readonly key: string; + readonly environment: Record; + readonly delivered: Record; + readonly resolved: Map; + readonly unset: string[]; +}): void { + const { key, context } = opts; + if (!(context.filePrivacy || shellVariableReadable(key))) { + opts.environment[key] = null; + return; + } + const found = context.lookup(key); + if (found !== undefined) { + opts.environment[key] = null; + opts.delivered[key] = found; + opts.resolved.set(key, found); + return; + } + if (!shellVariableReadable(key)) { + opts.environment[key] = null; + return; + } + opts.unset.push(key); + missingNotice({ context, key, unset: true }); +} +function resolveService(opts: { + readonly context: ServiceContext; + readonly environment: Record; +}): Resolution { + const environment: Record = Object.create(null); + const delivered: Record = Object.create(null); + const resolved = new Map(); + const unset: string[] = []; + const unbound = new Set(); + const references = new Map>(); + const { context } = opts; + let changed = false; + for (const [key, value] of Object.entries(opts.environment)) { + if (!NAME.test(key)) { + throw refused( + "environment", + "variable names must be valid identifiers", + context.name + ); + } + const managed = ownValue(context.managed, key); + if (managed !== undefined) { + environment[key] = null; + resolved.set(key, managed); + continue; + } + if (context.filePrivacy && HACK_KEY_MATERIAL_VARIABLES.has(key)) { + throw refused( + "environment", + "Hack key material cannot be copied into services", + context.name + ); + } + if (value === null) { + changed = true; + resolveBareService({ + key, + context, + environment, + delivered, + resolved, + unset, + }); + continue; + } + if (typeof value !== "string") { + const literal = serviceScalar({ value, service: context.name }); + environment[key] = literal; + resolved.set(key, String(literal)); + continue; + } + const expanded = expandServiceValue({ text: value, key, context }); + environment[key] = expanded.private ? null : value; + if (expanded.unbound) { + unbound.add(key); + } else { + resolved.set(key, expanded.value); + if (!expanded.private) { + references.set(key, composeEnvironmentReferences(value)); + } + } + if (expanded.private) { + delivered[key] = expanded.value; + changed = true; + } + } + return { + environment, + resolved, + delivered, + unset, + unbound, + references, + changed, + }; +} + +async function serviceFiles(opts: { + readonly context: ServiceContext; + readonly sources: Sources; + readonly specs: readonly FileSpec[]; + readonly resolution: Resolution; +}): Promise> { + let files: ReadonlyMap = new Map(); + for (const spec of opts.specs) { + const source = await opts.sources.read(spec, opts.context.name); + if (source === null) { + continue; + } + try { + files = parseComposeEnvironmentFile({ + text: source.text, + raw: spec.raw, + initial: files, + onMissing: (_name, entry) => + missingNotice({ + context: opts.context, + key: entry.key, + source: spec.path, + line: entry.line, + unset: false, + }), + onEntry: (entry) => { + if (protectedName(entry.key)) { + throw refused( + spec.path, + "protected environment keys cannot be delivered", + opts.context.name, + entry + ); + } + }, + lookup: (key, entry) => + fileLookup({ + source: spec.path, + service: opts.context.name, + entry, + lookup: () => { + const project = opts.context.lookup(key, false); + if (project !== undefined) { + return project; + } + if ( + key === "COMPOSE_PROJECT_NAME" || + opts.resolution.unbound.has(key) + ) { + throw refused( + spec.path, + "runtime-dependent sources cannot interpolate private file values", + opts.context.name + ); + } + return opts.resolution.resolved.get(key); + }, + }), + onUnresolved: (key, entry) => { + if (!shellVariableReadable(key)) { + throw refused( + spec.path, + "a source cannot be read from the caller's shell; use an explicit file value or credential adapter", + opts.context.name, + entry + ); + } + }, + }); + } catch (error: unknown) { + if (error instanceof NativeEnvFileRefusal) { + throw error; + } + throw syntaxRefusal(spec.path, error, opts.context.name); + } + } + return files; +} + +async function normalizeService(opts: { + readonly context: ServiceContext; + readonly sources: Sources; + readonly service: NativeComposeEnvironmentService; + readonly active: boolean; +}): Promise> { + const { document, environment: authored } = opts.service; + if (Object.hasOwn(document, UNSETS)) { + throw refused( + "environment", + "the native unset metadata field is reserved", + opts.context.name + ); + } + for (const [field, value] of Object.entries(document)) { + if ( + opts.context.filePrivacy && + field !== "environment" && + field !== "env_file" && + !ignoredExtension(field) + ) { + checkPublicFields({ + value, + lookup: opts.context.lookup, + service: opts.context.name, + field, + }); + } + } + const resolution = resolveService({ + context: opts.context, + environment: authored, + }); + const specs = fileSpecs(document.env_file, opts.context.name); + const files = opts.active + ? await serviceFiles({ ...opts, specs, resolution }) + : new Map(); + const environment: Record = Object.assign( + Object.create(null), + resolution.environment + ); + const fileBindings = new Set(); + for (const [key, value] of files) { + if (protectedName(key)) { + throw refused( + "env_file", + `protected variable ${key} cannot be delivered`, + opts.context.name + ); + } + if (Object.hasOwn(authored, key)) { + continue; + } + environment[key] = null; + resolution.delivered[key] = value; + fileBindings.add(key); + } + resolveFileBindingDefaults({ environment, resolution, fileBindings }); + resolvePublicFileBindingDefaults({ + document, + context: opts.context, + fileBindings, + }); + if (specs.length > 0 || resolution.changed) { + document.environment = environment; + } + if (resolution.unset.length > 0) { + // Names are ASCII identifiers: default code-unit order equals Rust BTreeSet order. + document[UNSETS] = resolution.unset.sort(); + } + Reflect.deleteProperty(document, "env_file"); + return resolution.delivered; +} + +/** Only executable fields consumed by the runtime get a frozen public default. */ +function resolvePublicFileBindingDefaults(opts: { + readonly document: Record; + readonly context: ServiceContext; + readonly fileBindings: ReadonlySet; +}): void { + if (opts.fileBindings.size === 0) { + return; + } + const managedNames = new Set(Object.keys(opts.context.managed)); + const prepare = (value: unknown, field: string) => + checkPublicFields({ + value, + field, + service: opts.context.name, + lookup: opts.context.lookup, + fileBindings: opts.fileBindings, + managedNames, + }); + for (const field of ["command", "entrypoint", "user", "extra_hosts"]) { + if (Object.hasOwn(opts.document, field)) { + opts.document[field] = prepare(opts.document[field], field); + } + } + if ( + isRecord(opts.document.healthcheck) && + Object.hasOwn(opts.document.healthcheck, "test") + ) { + opts.document.healthcheck.test = prepare( + opts.document.healthcheck.test, + "healthcheck.test" + ); + } +} + +/** + * YAML environment interpolation uses the project context, never env-file values. + * Freeze a public default before a newly private file name becomes runtime-owned. + * Re-escape dollars because the runtime still evaluates the public declarations. + */ +function resolveFileBindingDefaults(opts: { + readonly environment: Record; + readonly resolution: Resolution; + readonly fileBindings: ReadonlySet; +}): void { + for (const [key, references] of opts.resolution.references) { + const value = opts.resolution.resolved.get(key); + if ( + value !== undefined && + [...references].some((name) => opts.fileBindings.has(name)) + ) { + opts.environment[key] = value.replaceAll("$", () => "$$"); + } + } +} + +/** Fresh exec acquires selected-service values but cannot initiate a build. */ +function requireBuildSeparation( + opts: Options & { + readonly sources: Sources; + readonly ignoredRootDotenv: boolean; + } +): void { + if (opts.selectedServices === undefined) { + const privatePaths = Object.entries(opts.services).flatMap( + ([service, value]) => + fileSpecs(value.document.env_file, service).map( + (spec) => + nativeEnvFileSourceLocation({ + ...opts, + path: spec.path, + service, + }).path + ) + ); + if (privatePaths.length > 0 || opts.sources.automaticDotenv !== undefined) { + // Retain absent optional declarations and the automatic dotenv location. + // Nothing is read by this guard; ignore patterns do not grant exclusion. + privatePaths.push(join(dirname(opts.composeFile), ".env")); + } + if (opts.ignoredRootDotenv) { + // Ignored for interpolation does not make the raw Buildx context safe. + privatePaths.push(join(opts.projectRoot, ".env")); + } + if (privatePaths.length > 0) { + requireNativeEnvBuildSeparation({ + ...opts, + services: Object.fromEntries( + Object.entries(opts.services).map(([name, value]) => [ + name, + value.document, + ]) + ), + privatePaths, + }); + } + } +} + +/** + * Native-only normalization. Public declarations contain names, never file or + * shell values. Returned maps are private ephemeral inputs, not managed-env + * fingerprints. Start and preflight call this same preparation path. + */ +export async function normalizeNativeComposeEnvironment( + opts: Options +): Promise<{ + readonly shellEnvironment: Record>; + readonly notices: Notice[]; + readonly forwardedNames: Record; + readonly fileServices: Readonly>; + readonly ignoredRootDotenv: boolean; +}> { + const canonical = await canonicalSourceOptions(opts); + const notices: Notice[] = []; + const sources = await environmentSources(canonical, notices); + const ignoredRootDotenv = + dirname(canonical.composeFile) !== canonical.projectRoot && + (await present(join(canonical.projectRoot, ".env"))); + requireBuildSeparation({ ...canonical, sources, ignoredRootDotenv }); + const filePrivacy = + sources.automaticDotenv !== undefined || + Object.values(opts.services).some((service) => + Object.hasOwn(service.document, "env_file") + ); + if (ignoredRootDotenv) { + notices.push({ line: NATIVE_ROOT_DOTENV_IGNORED_NOTICE }); + } + for (const [field, value] of Object.entries(opts.compose)) { + if (filePrivacy && field !== "services" && !ignoredExtension(field)) { + checkPublicFields({ value, lookup: sources.lookup, field }); + } + } + const shellEnvironment: Record< + string, + Record + > = Object.create(null); + const forwardedNames: Record = Object.create(null); + const fileServices: Record = Object.create(null); + for (const [name, service] of Object.entries(opts.services)) { + sources.shellNames.clear(); + const active = + opts.selectedServices === undefined + ? nativeServiceActive(service.document, opts.profiles ?? []) + : opts.selectedServices.includes(name); + if (Object.hasOwn(service.document, "env_file")) { + fileServices[name] = active; + } + const delivered = await normalizeService({ + sources, + service, + active, + context: { + name, + global: opts.global, + managed: Object.hasOwn(opts.managed, name) + ? (opts.managed[name] ?? {}) + : {}, + lookup: sources.lookup, + notices, + filePrivacy, + composeFile: relative(canonical.projectRoot, canonical.composeFile), + }, + }); + if (Object.keys(delivered).length > 0) { + shellEnvironment[name] = delivered; + } + forwardedNames[name] = [...sources.shellNames].sort(); + } + return { + shellEnvironment, + notices, + forwardedNames, + fileServices, + ignoredRootDotenv, + }; +} diff --git a/src/backends/native-env-build-context.ts b/src/backends/native-env-build-context.ts new file mode 100644 index 000000000..7866ff89b --- /dev/null +++ b/src/backends/native-env-build-context.ts @@ -0,0 +1,98 @@ +import { dirname, relative, resolve, sep } from "node:path"; +import { HackCliError } from "../lib/cli-result.ts"; +import { NativeEnvFileRefusal } from "./native-env-file-source.ts"; +import { + admitNativeProjectBuild, + type NativeBuildRequest, +} from "./native-project-builds.ts"; + +const ASCII = /^[\x00-\x7f]*$/; + +/** + * Prove disjointness without consulting a mutable host directory. ASCII names + * compare case-insensitively. A non-ASCII component is conservatively ambiguous, + * covering every host Unicode alias without implementing another case-fold table. + */ +function maySharePrefix( + parent: readonly string[], + child: readonly string[] +): boolean { + return ( + parent.length <= child.length && + parent.every((part, index) => { + const other = child[index] ?? ""; + return ( + !(ASCII.test(part) && ASCII.test(other)) || + part.toLowerCase() === other.toLowerCase() + ); + }) + ); +} + +/** The project-relative components of a literal, already bounded source path. */ +function parts(root: string, path: string): string[] { + return relative(root, path) + .split(sep) + .filter((part) => part !== "" && part !== "."); +} + +/** + * Buildx reads raw host contexts and may use a Dockerfile-specific ignore file. + * Until builds can carry enforced private exclusions, reject overlapping inputs + * before any hook/provider/build effect, regardless of authored ignore patterns. + */ +export function requireNativeEnvBuildSeparation(opts: { + readonly projectRoot: string; + readonly composeFile: string; + readonly services: Readonly< + Record>> + >; + readonly privatePaths: readonly string[]; +}): void { + const root = resolve(opts.projectRoot); + const privatePaths = opts.privatePaths.map((path) => ({ + path, + parts: parts(root, path), + })); + for (const [name, service] of Object.entries(opts.services)) { + let request: NativeBuildRequest | undefined; + try { + request = admitNativeProjectBuild({ ...opts, name, service }); + } catch (error: unknown) { + // The dedicated build admission already refuses unsupported declarations. + if ( + error instanceof HackCliError && + error.code === "E_NATIVE_PROJECT_UNSUPPORTED" + ) { + continue; + } + throw error; + } + if (request === undefined) { + continue; + } + const context = parts(root, resolve(root, request.context)); + const dockerfile = resolve(root, request.context, request.dockerfile); + const definitions = [dockerfile, `${dockerfile}.dockerignore`].map((path) => + parts(root, path) + ); + const overlap = privatePaths.find( + (source) => + maySharePrefix(context, source.parts) || + definitions.some( + (definition) => + definition.length === source.parts.length && + maySharePrefix(definition, source.parts) + ) + ); + if (overlap !== undefined) { + throw new NativeEnvFileRefusal({ + service: name, + field: "build", + source: relative(dirname(opts.composeFile), overlap.path), + reason: + "native build inputs overlap an environment source; use disjoint build inputs, a prebuilt image, or the Compose backend", + }); + } + } +} diff --git a/src/backends/native-env-file-source.ts b/src/backends/native-env-file-source.ts new file mode 100644 index 000000000..862e23f9d --- /dev/null +++ b/src/backends/native-env-file-source.ts @@ -0,0 +1,253 @@ +import { constants, type Stats } from "node:fs"; +import { lstat, open } from "node:fs/promises"; +import { + basename, + dirname, + isAbsolute, + relative, + resolve, + sep, +} from "node:path"; +import { HackCliError } from "../lib/cli-result.ts"; +import { isRecord } from "../lib/guards.ts"; + +const MAX_FILE_BYTES = 256 * 1024; +const SERVICE = /^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$/; +const VARIABLE_PATH = /[$\0\r\n]/; +const FIELD = /^[A-Za-z0-9_.-]{1,128}$/; +const KEY = /^[A-Za-z_][A-Za-z0-9_]*$/; +const IGNORE_SOURCES = new Set([".gitignore", ".ignore"]); + +/** A closed native-only refusal; source paths and names are allowed, values are not. */ +export class NativeEnvFileRefusal extends HackCliError { + readonly service?: string; + readonly source: string; + readonly field: string; + readonly reason: string; + readonly line?: number; + readonly key?: string; + + constructor(opts: { + readonly source: string; + readonly reason: string; + readonly service?: string; + readonly field?: string; + readonly line?: number; + readonly key?: string; + }) { + const service = + opts.service !== undefined && SERVICE.test(opts.service) + ? opts.service + : undefined; + const fallbackField = + opts.source === "environment" ? "environment" : "env_file"; + const field = + opts.field !== undefined && FIELD.test(opts.field) + ? opts.field + : fallbackField; + const line = + Number.isSafeInteger(opts.line) && (opts.line ?? 0) > 0 + ? opts.line + : undefined; + const key = + opts.key !== undefined && KEY.test(opts.key) ? opts.key : undefined; + super({ + code: "E_NATIVE_PROJECT_UNSUPPORTED", + message: `Native environment input${service === undefined ? "" : ` for service ${service} field ${field}`} source ${JSON.stringify(opts.source)}${line === undefined ? "" : ` line ${line}`}${key === undefined ? "" : ` key ${key}`} refused: ${opts.reason}; values omitted.`, + }); + this.source = opts.source; + this.service = service; + this.field = field; + this.reason = opts.reason; + this.line = line; + this.key = key; + } +} + +function missing(error: unknown): boolean { + return isRecord(error) && error.code === "ENOENT"; +} + +function sameVersion(a: Stats, b: Stats): boolean { + return ( + a.dev === b.dev && + a.ino === b.ino && + a.size === b.size && + a.mtimeMs === b.mtimeMs && + a.ctimeMs === b.ctimeMs && + a.mode === b.mode && + a.uid === b.uid && + a.nlink === b.nlink + ); +} + +function safeFile(info: Stats): boolean { + return ( + info.isFile() && + info.uid === process.getuid?.() && + info.nlink === 1 && + (info.mode & 0o022) === 0 && + (info.mode & 0o400) !== 0 && + info.size <= MAX_FILE_BYTES + ); +} + +async function readStableBytes(opts: { + readonly path: string; + readonly observed: Stats; + readonly refusal: (reason: string) => Error; +}): Promise { + const file = await open( + opts.path, + constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK + ); + try { + const before = await file.stat(); + if (!sameVersion(opts.observed, before)) { + throw opts.refusal("the source changed before reading"); + } + const buffer = Buffer.alloc(before.size + 1); + let size = 0; + while (size < buffer.length) { + const chunk = await file.read(buffer, size, buffer.length - size, size); + if (chunk.bytesRead === 0) { + break; + } + size += chunk.bytesRead; + } + if ( + size !== before.size || + !sameVersion(before, await file.stat()) || + !sameVersion(before, await lstat(opts.path)) + ) { + throw opts.refusal("the source changed while reading"); + } + return buffer.subarray(0, size); + } finally { + await file.close(); + } +} + +/** Literal source admission also runs before reusing a cached read. */ +export function nativeEnvFileSourceLocation(opts: { + readonly projectRoot: string; + readonly composeFile: string; + readonly path: string; + readonly service?: string; +}): { readonly root: string; readonly path: string; readonly member: string } { + const refusal = (reason: string) => + new NativeEnvFileRefusal({ + source: opts.path, + service: opts.service, + reason, + }); + if ( + opts.path.length === 0 || + opts.path.length > 4096 || + VARIABLE_PATH.test(opts.path) || + opts.path.includes("~") + ) { + throw refusal("use a literal project-local path"); + } + const root = resolve(opts.projectRoot); + const path = resolve(dirname(opts.composeFile), opts.path); + if (IGNORE_SOURCES.has(basename(path))) { + throw refusal("an environment source cannot also own source ignore rules"); + } + const member = relative(root, path); + if ( + member === "" || + member === ".." || + member.startsWith(`..${sep}`) || + isAbsolute(member) + ) { + throw refusal("the source must stay inside this project root"); + } + return { root, path, member }; +} + +/** + * Read one project-owned env source without mutations. Every ancestor within + * the canonical project root is a stable, owned, non-writable-by-others real + * directory. The final entry is opened O_NOFOLLOW|O_NONBLOCK and must be a + * stable single-link regular file that other users cannot write. Optional + * ignores initial ENOENT only, never a disappearance during a read. + */ +export async function readNativeEnvFileSource(opts: { + readonly projectRoot: string; + readonly composeFile: string; + readonly path: string; + readonly required: boolean; + readonly service?: string; +}): Promise<{ readonly path: string; readonly text: string } | null> { + const { root, path, member } = nativeEnvFileSourceLocation(opts); + const refusal = (reason: string) => + new NativeEnvFileRefusal({ + source: opts.path, + service: opts.service, + reason, + }); + const directories: { readonly path: string; readonly info: Stats }[] = []; + const paths = [root]; + let parent = root; + for (const component of member.split(sep).slice(0, -1)) { + parent = resolve(parent, component); + paths.push(parent); + } + const recheckDirectories = async () => { + for (const directory of directories) { + const now = await lstat(directory.path); + if ( + !now.isDirectory() || + now.dev !== directory.info.dev || + now.ino !== directory.info.ino || + now.uid !== directory.info.uid || + (now.mode & 0o022) !== 0 + ) { + throw refusal("a source directory changed or is unsafe"); + } + } + }; + let observedSource = false; + try { + for (const directory of paths) { + const info = await lstat(directory); + if ( + !info.isDirectory() || + info.uid !== process.getuid?.() || + (info.mode & 0o022) !== 0 + ) { + throw refusal( + "source directories must be real, owned by this user and not writable by others" + ); + } + directories.push({ path: directory, info }); + } + const observed = await lstat(path); + observedSource = true; + if (!safeFile(observed)) { + throw refusal( + "use a single-link regular file owned by this user, owner-readable and not writable by others, at most 256 KiB" + ); + } + const buffer = await readStableBytes({ path, observed, refusal }); + await recheckDirectories(); + return { + path, + text: new TextDecoder("utf-8", { fatal: true }).decode(buffer), + }; + } catch (error: unknown) { + if (missing(error) && !observedSource && directories.length > 0) { + await recheckDirectories().catch(() => { + throw refusal("a source directory changed"); + }); + if (!opts.required) { + return null; + } + throw refusal("the required source is missing"); + } + throw error instanceof NativeEnvFileRefusal + ? error + : refusal("the source is unreadable, unsafe or not valid UTF-8"); + } +} diff --git a/src/backends/native-environment-unsets.ts b/src/backends/native-environment-unsets.ts new file mode 100644 index 000000000..873a1b514 --- /dev/null +++ b/src/backends/native-environment-unsets.ts @@ -0,0 +1,37 @@ +import { isRecord } from "../lib/guards.ts"; + +/** + * Old runtimes ignore extension metadata. Require the reviewed plan to echo + * every names-only removal before any effect, so a new CLI cannot silently + * retain an image default when paired with an older runtime binary. + */ +export function nativeEnvironmentUnsetsReviewed(opts: { + readonly compose: unknown; + readonly plan: unknown; +}): boolean { + if (!(isRecord(opts.compose) && isRecord(opts.compose.services))) { + return false; + } + for (const [name, service] of Object.entries(opts.compose.services)) { + if ( + !( + isRecord(service) && Array.isArray(service["x-hack-unset-environment"]) + ) || + service["x-hack-unset-environment"].length === 0 + ) { + continue; + } + if (!(isRecord(opts.plan) && isRecord(opts.plan.services))) { + return false; + } + const reviewed = opts.plan.services[name]; + if ( + !isRecord(reviewed) || + JSON.stringify(reviewed.environment_unsets) !== + JSON.stringify(service["x-hack-unset-environment"]) + ) { + return false; + } + } + return true; +} diff --git a/src/backends/native-exec-environment.ts b/src/backends/native-exec-environment.ts index 7c54b101b..763cf4397 100644 --- a/src/backends/native-exec-environment.ts +++ b/src/backends/native-exec-environment.ts @@ -1,7 +1,10 @@ import { HackCliError } from "../lib/cli-result.ts"; import { adaptNativeAwsEnvironment } from "./native-aws-environment.ts"; import { adaptNativeGcloudEnvironment } from "./native-gcloud-environment.ts"; -import { prepareNativeProjectInput } from "./native-project-input.ts"; +import { + mergeNativeShellEnvironment, + prepareNativeProjectInput, +} from "./native-project-input.ts"; import { describeNativeRunEnvironmentChange, type NativeProjectRun, @@ -36,6 +39,7 @@ export async function prepareNativeExecEnvironment(opts: { readonly run: NativeProjectRun; readonly composeFile: string; readonly service: string; + readonly env?: Readonly>; readonly prepare?: typeof prepareNativeProjectInput; readonly adaptAws?: typeof adaptNativeAwsEnvironment; readonly adaptGcloud?: typeof adaptNativeGcloudEnvironment; @@ -58,6 +62,9 @@ export async function prepareNativeExecEnvironment(opts: { projectDir: opts.scope.projectDir, composeFile: opts.composeFile, envName: opts.run.effectiveEnvName, + env: opts.env ?? process.env, + profiles: opts.run.profiles ?? [], + selectedServices: [opts.service], }); if ( !input.serviceNames.includes(opts.service) || @@ -84,6 +91,7 @@ export async function prepareNativeExecEnvironment(opts: { ) { throw new SelectionChanged(); } + input = mergeNativeShellEnvironment(input); if (opts.run.aws) { input = ( await (opts.adaptAws ?? adaptNativeAwsEnvironment)({ diff --git a/src/backends/native-exec-stale-env.ts b/src/backends/native-exec-stale-env.ts index 6e05dfa3e..84d440e46 100644 --- a/src/backends/native-exec-stale-env.ts +++ b/src/backends/native-exec-stale-env.ts @@ -62,6 +62,7 @@ export async function resolveNativeImageExecStaleEnvWarning(opts: { projectDir: opts.scope.projectDir, composeFile: opts.composeFile, envName: mapping.effectiveEnvName, + managedOnly: true, }); if (input.effectiveEnvName !== mapping.effectiveEnvName) { return null; diff --git a/src/backends/native-gcloud-environment.ts b/src/backends/native-gcloud-environment.ts index 340678234..ea746348a 100644 --- a/src/backends/native-gcloud-environment.ts +++ b/src/backends/native-gcloud-environment.ts @@ -148,6 +148,15 @@ function adcMount(value: unknown, home: string): string | undefined { return longAdcMount(value, sources); } +/** Exact authored mount forms consumed privately by ADC adaptation. No file is read. */ +export function nativeGcloudCredentialMount(value: unknown): boolean { + try { + return adcMount(value, "") !== undefined; + } catch { + return false; + } +} + function mountTarget(mount: unknown): string { if (typeof mount === "string") { const parts = mount.split(":"); diff --git a/src/backends/native-preflight.ts b/src/backends/native-preflight.ts index 04c7b5ed2..8098cac69 100644 --- a/src/backends/native-preflight.ts +++ b/src/backends/native-preflight.ts @@ -1,5 +1,5 @@ import { createHash } from "node:crypto"; -import { access, chmod, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { chmod, mkdtemp, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join, relative } from "node:path"; import { isRecord } from "../lib/guards.ts"; @@ -8,17 +8,20 @@ import { classifyComposeBackendReading, RESERVED_OWNERSHIP_LABEL_PREFIXES, } from "../lib/native-compose-backend-features.ts"; -import { - projectEnvConfigExists, - resolveProjectEnvMetadata, -} from "../lib/project-env-config.ts"; +import { NATIVE_ROOT_DOTENV_IGNORED_NOTICE } from "../lib/native-environment-notices.ts"; +import { NativeEnvFileRefusal } from "./native-env-file-source.ts"; +import { nativeEnvironmentUnsetsReviewed } from "./native-environment-unsets.ts"; import { admitNativeHttpProbe } from "./native-http-probe-admission.ts"; import { prepareNativeProjectAdaptation } from "./native-project-adaptation.ts"; import { admitNativeProjectBuild } from "./native-project-builds.ts"; import type { NativeProjectInput } from "./native-project-input.ts"; import { - nativeServiceEnvironment, + mergeNativeShellEnvironment, + nativeProjectInputRefusalMessage, + prepareNativeProjectInput, readNativeComposeBytes, + requireNativeShellDeliveryFits, + selectActiveNativeProjectInput, } from "./native-project-input.ts"; import { applyNativeInternalHostAliases, @@ -43,7 +46,6 @@ import { NativeRuntimeRequestError, type NativeRuntimeSelection, } from "./native-runtime-client.ts"; -import { shellReference } from "./native-shell-environment.ts"; import { NATIVE_START_REFUSAL_ENTRIES } from "./native-start-refusal-entries.ts"; import { readNativeStartRefusals } from "./native-start-refusals.ts"; @@ -286,6 +288,11 @@ const RUNTIME_FINDINGS: Readonly> = { message: "A mount source is outside the project.", fix: "Mount only paths inside the project root.", }, + source_case_collision: { + message: + "Project source paths collide under Unicode normalization and case folding.", + fix: "Rename the conflicting files or directories so their normalized, case-folded paths are distinct, then correct any env_file references.", + }, unresolved_source_path: { message: "A mount source cannot be resolved inside the project.", fix: "Use a plain path relative to the project.", @@ -370,7 +377,7 @@ const RUNTIME_FINDINGS: Readonly> = { }, unsupported_env_file_mode: { message: "env_file uses an option native does not accept.", - fix: "Move the variables into Hack managed env with hack env set --service.", + fix: "Use an omitted or empty format for Compose dotenv syntax, or raw.", }, invalid_compose_yaml: { message: "The Compose file is not valid YAML.", @@ -495,6 +502,7 @@ export const REVIEW_STOP_CODES: ReadonlySet = new Set([ "invalid_duration", "invalid_env_file", "invalid_environment_key", + "invalid_environment_unset", "invalid_extra_hosts", "invalid_healthcheck", "invalid_identifier", @@ -518,6 +526,7 @@ export const REVIEW_STOP_CODES: ReadonlySet = new Set([ "registry_template_unsafe", "service_budget", "source_budget", + "source_case_collision", "source_escape", "source_generated", "source_limit_config", @@ -729,17 +738,6 @@ function cliServiceFindings(opts: { builds.add(name); } } - if (isRecord(service) && Object.hasOwn(service, "env_file")) { - found.push( - finding( - { ...base, code: "env_file", field: "env_file" }, - { - message: "The service uses env_file, which native does not read.", - fix: "Move its variables into Hack managed env with hack env set --service, or run hack doctor --migrate-env-config.", - } - ) - ); - } for (const problem of nativeServicePreparationProblems({ name, service, @@ -784,13 +782,6 @@ function cliServiceFindings(opts: { return found; } -async function exists(path: string): Promise { - return await access(path).then( - () => true, - () => false - ); -} - /** * Reads the Compose file with the bounds native start applies * (`readCompose` in `native-project-input.ts`): a regular file opened without @@ -829,51 +820,37 @@ type AdaptedProject = { /** False when the adaptation was refused and the document is unadapted. */ readonly adaptationValid: boolean; readonly managedServices: readonly string[]; + /** In-memory check only. Captured private values never enter report JSON. */ + readonly checkDelivery: (plan: unknown) => void; }; }; -/** - * Normalize the same managed keys and shell references as start, without a - * decryption key or access to process.env. Only service names leave this helper. - * An unset shell reference is still private delivery (of an empty value) at start. - */ -async function preflightEnvironment(opts: { - readonly projectRoot: string; - readonly projectDir: string; - readonly effectiveEnvName: string | null; - readonly services: Record; -}): Promise { - const metadata = await resolveProjectEnvMetadata({ - projectRoot: opts.projectRoot, - projectDir: opts.projectDir, - envName: opts.effectiveEnvName, - serviceNames: Object.keys(opts.services), - }); - const delivered: string[] = []; - for (const [name, service] of Object.entries(opts.services)) { - if (!isRecord(service)) { - continue; - } - const managed = new Set( - Object.keys(metadata?.effectiveMetadata[name] ?? {}) - ); - const environment = nativeServiceEnvironment(service.environment, name); - let hasDelivery = managed.size > 0; - for (const key of managed) { - environment[key] = null; - } - for (const [key, value] of Object.entries(environment)) { - if (shellReference({ key, value, managed }) !== undefined) { - environment[key] = null; - hasDelivery = true; - } - } - if (hasDelivery) { - service.environment = environment; - delivered.push(name); +function environmentInputFinding( + error: unknown, + delivery = false +): NativePreflightFinding { + const source = error instanceof NativeEnvFileRefusal ? error : undefined; + const inputCode = + source === undefined ? "managed_env_unreadable" : "environment_file_input"; + return finding( + { + severity: "blocker", + gate: "cli", + code: delivery ? "environment_delivery" : inputCode, + service: source?.service, + field: source?.field, + }, + { + message: + source?.message ?? + nativeProjectInputRefusalMessage(error) ?? + "The project's environment declarations or Hack managed env could not be read or decrypted.", + fix: + source === undefined + ? "Check the environment declarations and run hack env list to see the problem." + : "Check the named source and environment declaration; use owned regular files that are not writable by others.", } - } - return delivered.sort(byCodePoint); + ); } /** @@ -892,6 +869,7 @@ async function adaptForPreflight(opts: { readonly internalHostSources: Readonly>; readonly hasDefaultDependencyFile: boolean; readonly effectiveEnvName: string | null; + readonly profiles: readonly string[]; }): Promise { const blocker = { severity: "blocker", gate: "cli" } as const; // One bounded read: the parsed document and the digest come from the same bytes. @@ -917,26 +895,6 @@ async function adaptForPreflight(opts: { ...structuredClone(compose), services: structuredClone(compose.services), }; - let managedServices: readonly string[] = []; - try { - managedServices = await preflightEnvironment({ - ...opts, - services: normalized.services, - }); - } catch { - findings.push( - finding( - { ...blocker, code: "managed_env_unreadable" }, - { - message: - "The project's service environment declarations or Hack managed env metadata could not be read.", - fix: "Check the service environment declarations and run hack env list to see the problem.", - } - ) - ); - } - // Only the public document is adapted; no private value is involved. Every - // field is named so a helper that starts reading one fails the type check. let input: NativeProjectInput = { originalSha256: createHash("sha256").update(bytes).digest("hex"), normalizedComposeJson: JSON.stringify(normalized), @@ -947,6 +905,47 @@ async function adaptForPreflight(opts: { environmentFiles: [], serviceNames: Object.keys(compose.services), }; + try { + const prepared = await prepareNativeProjectInput({ + ...opts, + envName: opts.effectiveEnvName, + env: process.env, + }); + if (prepared.originalSha256 !== input.originalSha256) { + throw new Error("Compose input changed during preflight"); + } + input = prepared; + for (const notice of prepared.environmentNotices ?? []) { + findings.push( + finding( + { + severity: "warning", + gate: "cli", + code: + prepared.ignoredRootDotenv && + notice.service === undefined && + notice.line === NATIVE_ROOT_DOTENV_IGNORED_NOTICE + ? "root_dotenv_ignored" + : "environment_notice", + service: notice.service, + field: notice.service === undefined ? undefined : "environment", + }, + { + message: notice.line, + fix: "Review the named environment input; values are omitted.", + } + ) + ); + } + } catch (error: unknown) { + findings.push(environmentInputFinding(error)); + } + const managedServices = [ + ...new Set([ + ...Object.keys(input.managedEnvironment), + ...Object.keys(input.shellEnvironment ?? {}), + ]), + ].sort(byCodePoint); let adaptationValid = true; try { input = await prepareNativeProjectAdaptation({ @@ -1004,6 +1003,13 @@ async function adaptForPreflight(opts: { originalSha256: input.originalSha256, originalCompose: compose, managedServices, + checkDelivery: (plan) => + requireNativeShellDeliveryFits( + selectActiveNativeProjectInput({ + input: mergeNativeShellEnvironment(input), + reviewedPlan: plan, + }) + ), normalizedComposeJson: input.normalizedComposeJson, services: isRecord(adapted) && isRecord(adapted.services) ? adapted.services : {}, @@ -1069,35 +1075,6 @@ async function collectNativeCliFindings(opts: { }) ); } - const managedEnv = await projectEnvConfigExists({ - projectDir: opts.projectDir, - }).catch(() => null); - if (managedEnv === null) { - found.push( - finding( - { ...blocker, code: "managed_env_unreadable" }, - { - message: "The project's Hack managed env files could not be read.", - fix: "Run hack env list to see the problem.", - } - ) - ); - } else if ( - !managedEnv && - ((await exists(join(opts.projectDir, ".env"))) || - (await exists(join(opts.projectRoot, ".env")))) - ) { - found.push( - finding( - { ...blocker, code: "dotenv_without_managed_env" }, - { - message: - "The project has a .env file and no Hack managed env. Native does not read .env files.", - fix: "Migrate it with hack doctor --migrate-env-config, or remove the file.", - } - ) - ); - } if (!opts.sharedSource) { found.push( finding( @@ -1297,6 +1274,33 @@ async function collectNativeRuntimeFindings(opts: { if (!(isRecord(reply) && isRecord(reply.plan))) { return { review: { status: "unavailable" }, findings: [] }; } + if ( + !nativeEnvironmentUnsetsReviewed({ + compose: { services: opts.adapted.services }, + plan: reply.plan, + }) + ) { + return { + review: { + status: "unavailable", + code: "native_environment_unset_unavailable", + }, + findings: [ + finding( + { + severity: "blocker", + gate: "runtime", + code: "native_environment_unset_unavailable", + }, + { + message: + "The selected native runtime cannot confirm explicit environment removals.", + fix: "Update the selected native runtime and retry; no VM was started.", + } + ), + ], + }; + } const services = Object.keys(opts.adapted.services).filter((name) => SERVICE.test(name) ); @@ -1462,6 +1466,18 @@ export type NativePreflightOptions = { readonly invoke?: typeof invokeNativeRuntime; }; +function deliveryFindings( + document: NonNullable, + plan: unknown +): NativePreflightFinding[] { + try { + document.checkDelivery(plan); + return []; + } catch (error: unknown) { + return [environmentInputFinding(error, true)]; + } +} + /** Runs both gates over one project and merges their findings. */ export async function collectNativePreflight( opts: NativePreflightOptions @@ -1500,6 +1516,7 @@ export async function collectNativePreflight( invoke: opts.invoke, }); if (runtime.plan) { + cli.push(...deliveryFindings(adapted.document, runtime.plan)); cli.push( ...routeFindings({ plan: runtime.plan, diff --git a/src/backends/native-project-builds.ts b/src/backends/native-project-builds.ts index e10516b5f..d0e2602ca 100644 --- a/src/backends/native-project-builds.ts +++ b/src/backends/native-project-builds.ts @@ -499,7 +499,8 @@ export function refuseEmulatedBuilds(opts: { /** * Refuse, before any hook, VM or build effect, a build argument whose value equals a - * secret value Hack manages for this project and environment (see below). + * nonempty value explicitly marked secret in this project's managed env. Plain + * env-file and shell values have no secret metadata and do not widen this guard. */ export function refuseManagedSecretBuildArguments(opts: { readonly builds: ReadonlyMap; diff --git a/src/backends/native-project-input.ts b/src/backends/native-project-input.ts index a47e44279..c8fa7e13c 100644 --- a/src/backends/native-project-input.ts +++ b/src/backends/native-project-input.ts @@ -1,7 +1,6 @@ import { createHash } from "node:crypto"; import { constants } from "node:fs"; -import { lstat, open } from "node:fs/promises"; -import { join } from "node:path"; +import { open } from "node:fs/promises"; import { HackCliError } from "../lib/cli-result.ts"; import { isRecord } from "../lib/guards.ts"; import { @@ -11,15 +10,17 @@ import { resolveProjectEnvSelection, selectProjectEnvValuesForExecutionTarget, } from "../lib/project-env-config.ts"; +import { + type NativeComposeEnvironmentService, + normalizeNativeComposeEnvironment, +} from "./native-compose-environment.ts"; +import { NativeEnvFileRefusal } from "./native-env-file-source.ts"; import { MAX_NATIVE_PRIVATE_INPUT_BYTES, nativePrivateEnvironmentPayload, } from "./native-private-environment-payload.ts"; import type { NativeEmulationPolicy } from "./native-project-platform.ts"; -import { - resolveShellEnvironment, - shellDeliveryProblem, -} from "./native-shell-environment.ts"; +import { shellDeliveryProblem } from "./native-shell-environment.ts"; const LIMIT = 256 * 1024; const KEY = /^[A-Za-z_][A-Za-z0-9_]*$/; @@ -36,7 +37,7 @@ export type NativeManagedFile = { /** One value-free note about a service's `environment`: names only. */ export type NativeEnvironmentNotice = { - readonly service: string; + readonly service?: string; readonly line: string; }; @@ -52,8 +53,8 @@ export type NativeProjectInput = { Record >; /** - * Private in-memory only: the values Hack stores as secrets for this project and - * environment, so build arguments can refuse to carry one. Never log or write it. + * Private in-memory disclosure guard: nonempty values explicitly marked secret + * in managed env. Plain file and shell inputs have no secret metadata here. */ readonly managedSecretValues?: readonly string[]; /** Private lifecycle environment; not container environment. */ @@ -65,16 +66,18 @@ export type NativeProjectInput = { readonly effectiveEnvName: string | null; readonly environmentFiles: readonly string[]; readonly serviceNames: readonly string[]; + /** Names-only activation expectation, checked against the runtime before effects. */ + readonly environmentFileServices?: Readonly>; /** - * Private in-memory transport only: values read from the caller's shell for - * service `environment` references, by service. Kept apart from + * Private in-memory transport only: file and caller-shell values for service + * environment inputs. Kept apart from * `managedEnvironment` so they are never fingerprinted or treated as values * Hack stores; see {@link mergeNativeShellEnvironment}. Never log this map. */ readonly shellEnvironment?: Readonly< Record>> >; - /** Services whose private delivery includes a value from the shell, once merged. */ + /** Services receiving ephemeral file/shell values, once merged. */ readonly shellServices?: readonly string[]; /** Forwarded source names by service, for value-free operator notices only. */ readonly forwardedShellVariables?: Readonly< @@ -85,6 +88,7 @@ export type NativeProjectInput = { * a referenced variable that is not set. Names only. */ readonly environmentNotices?: readonly NativeEnvironmentNotice[]; + readonly ignoredRootDotenv?: boolean; /** * Emulation policy of each service the native adaptation opted in to `linux/amd64` * (`amd64Emulation`); absent when none did. See `native-project-platform.ts`. @@ -101,6 +105,13 @@ const SERVICE_NAME = /^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$/; */ class NativeProjectInputRefusal extends HackCliError {} +/** Only this boundary's closed messages may reach a preflight finding. */ +export function nativeProjectInputRefusalMessage( + error: unknown +): string | undefined { + return error instanceof NativeProjectInputRefusal ? error.message : undefined; +} + function refused( cause = "unsupported or invalid Compose/environment configuration", code: @@ -189,7 +200,7 @@ export function nativeServiceEnvironment( if (!Array.isArray(value)) { throw invalid(); } - const result: Record = {}; + const result: Record = Object.create(null); for (const entry of value) { if (typeof entry !== "string") { throw invalid(); @@ -204,23 +215,14 @@ export function nativeServiceEnvironment( return result; } -async function exists(path: string): Promise { - try { - await lstat(path); - return true; - } catch (error) { - if (isRecord(error) && error.code === "ENOENT") { - return false; - } - throw error; - } -} - function normalizeManagedEnvironment( services: Record, resolved: Awaited> ): Record> { - const managedEnvironment: Record> = {}; + const managedEnvironment: Record< + string, + Record + > = Object.create(null); for (const name of Object.keys(services)) { const service = services[name]; if (!isRecord(service)) { @@ -253,59 +255,11 @@ function normalizeManagedEnvironment( } /** - * Resolves the shell references in each service's `environment` (see - * `native-shell-environment.ts`). The public document keeps only bare - * references; the values are returned separately and are not part of - * `managedEnvironment` until {@link mergeNativeShellEnvironment}. - */ -function resolveServiceShellEnvironment(opts: { - readonly services: Record; - readonly managedEnvironment: Record>; - readonly env?: Readonly>; -}): { - readonly shellEnvironment: Record>; - readonly notices: NativeEnvironmentNotice[]; - readonly forwardedNames: Readonly>; -} { - const shellEnvironment: Record> = {}; - const notices: NativeEnvironmentNotice[] = []; - const forwardedNames: Record = {}; - for (const [name, service] of Object.entries(opts.services)) { - // An `environment:` key with every entry commented out is YAML null. - if ( - !isRecord(service) || - service.environment === undefined || - service.environment === null - ) { - continue; - } - const resolution = resolveShellEnvironment({ - service: name, - environment: nativeServiceEnvironment(service.environment, name), - env: opts.env, - managed: new Set(Object.keys(opts.managedEnvironment[name] ?? {})), - }); - if (resolution === null) { - continue; - } - service.environment = resolution.environment; - shellEnvironment[name] = resolution.delivered; - forwardedNames[name] = resolution.forwardedNames; - if (SERVICE_NAME.test(name)) { - notices.push( - ...resolution.notices.map((line) => ({ service: name, line })) - ); - } - } - return { shellEnvironment, notices, forwardedNames }; -} - -/** - * Adds the values read from the caller's shell to the private delivery. Native + * Adds ephemeral file/shell values to the private delivery. Native * start and restart call this once, after the env fingerprints are taken, so a - * shell value is delivered to its service and never digested to disk. Commands - * that prepare input without calling it (exec, stale-env checks) deliver no - * shell value. Calling it again changes nothing. + * value is delivered to its service and never digested to disk. Commands + * Fresh exec also calls this after preparing its selected service. Metadata-only + * stale-env checks never acquire or merge these values. Calling it again changes nothing. */ export function mergeNativeShellEnvironment( input: NativeProjectInput @@ -440,8 +394,9 @@ export function nativeReviewedActiveServices( * Restrict private delivery to the reviewed, profile-active service set. * * Normalization declares managed keys on every authored service, including - * services outside the selected profiles, so public Compose and the reviewed - * plan stay identical across profile selections. The runtime refuses managed + * services outside the selected profiles. File declarations follow the selected + * profiles; the runtime's active set must not require any unprepared file inputs. + * The runtime refuses managed * values for a service it will not schedule, so an optional disabled service * must not receive them and block the selected graph. */ @@ -450,6 +405,13 @@ export function selectActiveNativeProjectInput(opts: { readonly reviewedPlan: unknown; }): NativeProjectInput { const active = nativeReviewedActiveServices(opts.reviewedPlan); + for (const name of active) { + if (opts.input.environmentFileServices?.[name] === false) { + throw refused( + `${serviceLabel(name)} env_file activation differs from the reviewed runtime; retry with a matching CLI/runtime` + ); + } + } return { ...opts.input, managedEnvironment: Object.fromEntries( @@ -479,11 +441,16 @@ export async function prepareNativeProjectInput(opts: { readonly projectDir: string; readonly composeFile: string; readonly envName?: string | null; + readonly profiles?: readonly string[]; + /** Fresh exec restricts file acquisition to the selected running service. */ + readonly selectedServices?: readonly string[]; + /** Advisory managed fingerprints do not acquire ephemeral file/shell inputs. */ + readonly managedOnly?: boolean; /** * The caller's environment, read only for the variables that service * `environment` entries reference. Omitted, nothing is read and a shell - * reference resolves as unset: only native start and restart pass the - * process environment. + * reference resolves as unset. Start, preflight and fresh exec pass the + * same allowed caller context. */ readonly env?: Readonly>; }): Promise { @@ -501,16 +468,13 @@ export async function prepareNativeProjectInput(opts: { throw refused("the Compose file has no services mapping"); } const serviceNames = Object.keys(compose.services); - // Refuse before resolving/decrypting any values. Never read env_file contents. + const services: Record = + Object.create(null); for (const [name, service] of Object.entries(compose.services)) { if (!isRecord(service)) { throw refused(`${serviceLabel(name)} must be a mapping`); } - if (Object.hasOwn(service, "env_file")) { - throw refused( - `${serviceLabel(name)} field env_file is not supported by native up; move its variables into Hack managed env with hack env set --service` - ); - } + services[name] = { document: service, environment: {} }; } const resolved = await resolveProjectEnvConfig({ ...opts, @@ -518,24 +482,41 @@ export async function prepareNativeProjectInput(opts: { }).catch((error: unknown) => { throw environmentRefused(error); }); - if ( - !resolved && - ((await exists(join(opts.projectDir, ".env"))) || - (await exists(join(opts.projectRoot, ".env")))) - ) { - throw refused( - "a project .env file is present without Hack managed env; native up does not read .env files. Migrate it with hack doctor --migrate-env-config or remove it" - ); - } const managedEnvironment = normalizeManagedEnvironment( compose.services, resolved ); - const shell = resolveServiceShellEnvironment({ - services: compose.services, - managedEnvironment, - env: opts.env, - }); + for (const [name, service] of Object.entries(services)) { + services[name] = { + document: service.document, + environment: nativeServiceEnvironment( + service.document.environment, + name + ), + }; + } + const shell = opts.managedOnly + ? { + shellEnvironment: {}, + notices: [], + forwardedNames: {}, + fileServices: {}, + ignoredRootDotenv: false, + } + : await normalizeNativeComposeEnvironment({ + ...opts, + compose, + services, + managed: managedEnvironment, + global: resolved + ? selectProjectEnvValuesForExecutionTarget({ + resolved, + scopeName: "global", + target: "compose", + }) + : {}, + env: opts.env, + }); const normalizedComposeJson = JSON.stringify(compose); if (Buffer.byteLength(normalizedComposeJson) > LIMIT) { throw refused("the normalized Compose project exceeds 256 KiB"); @@ -564,12 +545,17 @@ export async function prepareNativeProjectInput(opts: { effectiveEnvName: selection.effectiveEnv, environmentFiles: resolved?.files ?? [], serviceNames, + environmentFileServices: shell.fileServices, shellEnvironment: shell.shellEnvironment, forwardedShellVariables: shell.forwardedNames, environmentNotices: shell.notices, + ignoredRootDotenv: shell.ignoredRootDotenv, }; } catch (error: unknown) { // Only this module's closed refusals carry a cause; anything else stays generic. - throw error instanceof NativeProjectInputRefusal ? error : refused(); + throw error instanceof NativeProjectInputRefusal || + error instanceof NativeEnvFileRefusal + ? error + : refused(); } } diff --git a/src/backends/native-project-restart-preflight.ts b/src/backends/native-project-restart-preflight.ts index 3d941ef16..07e371fb4 100644 --- a/src/backends/native-project-restart-preflight.ts +++ b/src/backends/native-project-restart-preflight.ts @@ -451,6 +451,7 @@ export async function preflightNativeRestart(opts: { ...opts.scope, composeFile: opts.composeFile, envName: selected.envName, + profiles: selected.profiles, // As native start: `environment` references read the caller's shell. env: process.env, }) diff --git a/src/backends/native-project-review.ts b/src/backends/native-project-review.ts index 732d29476..61286b0de 100644 --- a/src/backends/native-project-review.ts +++ b/src/backends/native-project-review.ts @@ -3,6 +3,7 @@ import { tmpdir } from "node:os"; import { join, relative } from "node:path"; import { ClosedCliError, HackCliError } from "../lib/cli-result.ts"; import { isRecord } from "../lib/guards.ts"; +import { nativeEnvironmentUnsetsReviewed } from "./native-environment-unsets.ts"; import { nativeProjectBranchArgs, prepareNativeProjectBranch, @@ -339,6 +340,16 @@ export async function withNativeProjectReview(opts: { "Native project returned an invalid reviewed identity." ); } + if ( + !nativeEnvironmentUnsetsReviewed({ + compose: JSON.parse(opts.input.normalizedComposeJson), + plan: report.plan, + }) + ) { + throw new ClosedCliError( + "The selected native runtime cannot confirm explicit environment removals. Update it and retry; no VM was started." + ); + } return await opts.run({ planId: report.plan_id, namespace, diff --git a/src/backends/native-project-start-preflight.ts b/src/backends/native-project-start-preflight.ts index d18dd8d38..248ea3c62 100644 --- a/src/backends/native-project-start-preflight.ts +++ b/src/backends/native-project-start-preflight.ts @@ -1,6 +1,10 @@ import { ClosedCliError } from "../lib/cli-result.ts"; import { isRecord } from "../lib/guards.ts"; -import type { NativeProjectInput } from "./native-project-input.ts"; +import { + type NativeProjectInput, + requireNativeShellDeliveryFits, + selectActiveNativeProjectInput, +} from "./native-project-input.ts"; import { withNativeProjectReview } from "./native-project-review.ts"; import { invokeNativeRuntime, @@ -52,6 +56,12 @@ export async function requireNativeStaticStartReview(opts: { if (!(isRecord(plan) && isRecord(plan.services))) { throw new ClosedCliError("Native static start review is invalid."); } + requireNativeShellDeliveryFits( + selectActiveNativeProjectInput({ + input: opts.input, + reviewedPlan: plan, + }) + ); const services = Object.keys(plan.services); const refusals = readNativeStartRefusals(report.start_refusals, services); if (refusals === null) { diff --git a/src/backends/native-project-start.ts b/src/backends/native-project-start.ts index 37a77c100..72402572a 100644 --- a/src/backends/native-project-start.ts +++ b/src/backends/native-project-start.ts @@ -463,11 +463,13 @@ function reportNativeEnvironmentNotices( .flatMap(([, names]) => names) ), ].sort(); - if (!nativeDetachedUpShellVariables(names)) { + if ( + !nativeDetachedUpShellVariables(names, input.ignoredRootDotenv === true) + ) { warn(nativeShellForwardingNotice(names)); } for (const notice of input.environmentNotices ?? []) { - if (active.has(notice.service)) { + if (notice.service === undefined || active.has(notice.service)) { warn(notice.line); } } @@ -1968,26 +1970,23 @@ async function startNativeProjectAttempt( }); requireActiveStartup(opts.signal); } - // A down hook of an earlier generation with unknown completion blocks this - // one before any hook runs; any other leftover record is superseded. - await deps.admitDownHooks(opts.scope); const profiles = selection.profiles; progress({ phase: "input" }); - await preflightNativeProjectSource({ - runtime: opts.runtime, - projectRoot: opts.scope.projectRoot, - signal: opts.signal, - invoke: deps.invoke, - }); - await deps.prepareStorage(opts.scope); + // Keep ps aware of this frontend while input is read. This advisory marker + // carries no values and is retired even if admission refuses the input. await observeStartup(); + // Read and admit private project inputs before any pool-share or storage + // mutation. An unsafe env file or public interpolation must have no effects. let input = await deps.prepare({ ...opts.scope, composeFile: opts.composeFile, envName: selection.envName, - // Service `environment` references are resolved from the caller's shell. + profiles, env: process.env, }); + // A down hook of an earlier generation with unknown completion blocks this + // one before any hook runs; any other leftover record is superseded. + await deps.admitDownHooks(opts.scope); // Digest the selection now: later steps adapt the delivered values. const selectedEnvironment = opts.onEnvironmentReady ? withComposeServiceEnvFingerprints({ @@ -2061,6 +2060,13 @@ async function startNativeProjectAttempt( invoke: deps.invoke, signal: opts.signal, }); + await preflightNativeProjectSource({ + runtime: opts.runtime, + projectRoot: opts.scope.projectRoot, + signal: opts.signal, + invoke: deps.invoke, + }); + await deps.prepareStorage(opts.scope); // A graph that captures source (bind mounts or a dependency cache) measures it // before hooks, image work or runtime start, so an oversized project refuses with // its size and largest paths. An image-only graph never reads source content. diff --git a/src/backends/native-shell-environment.ts b/src/backends/native-shell-environment.ts index 7c09985d6..d1d7374f4 100644 --- a/src/backends/native-shell-environment.ts +++ b/src/backends/native-shell-environment.ts @@ -1,50 +1,9 @@ import { HACK_KEY_MATERIAL_VARIABLES } from "../lib/child-environment.ts"; import { COMPOSE_PROJECT_NAME_VARIABLE } from "./native-compose-project-name.ts"; -/** - * Shell references in a service's `environment`, resolved the way Compose - * resolves them, for the native backend. - * - * Compose reads a variable reference and a bare `NAME` entry from the shell - * that runs it. The native runtime deliberately consults no ambient - * environment, so such an entry used to fail the start. This module resolves - * the two forms that could never start, in the CLI, and hands each value to - * the private delivery that Hack managed env already uses. - * - * Invariants: - * - A value read from the shell is never written to the normalized Compose - * document, a plan, a receipt, an env fingerprint or a log. The entry - * becomes a bare reference (`null`) and the value travels only in the - * private delivery. - * - The reviewed document does not depend on the shell: a resolved entry is - * always a bare reference, whether or not the variable is set. So a start - * from another shell reviews the same plan. - * - Only variables an entry references are read, and only own string - * properties of the environment. - * - Nothing that starts today changes. A reference with a default, an - * alternative or an error operator is left to the runtime, as before. - * - A key Hack manages for the service is never taken from the shell, and a - * reference to a managed variable is left for the runtime, which refuses it. - * - A name the runtime supplies itself, Hack's own `HACK_*` variables and the - * cloud credential names the scoped adapters own are never read from the - * shell. - * - * Scope: an entry whose whole value is one plain reference, or a bare name. - * A reference inside a longer string, or in any field other than - * `environment`, is not resolved here. - */ - -/** A valid environment variable name. */ -const NAME = /^[A-Za-z_][A-Za-z0-9_]*$/; -/** The whole value is one braced reference with no operator. */ -const BRACED = /^\$\{([A-Za-z_][A-Za-z0-9_]*)\}$/; -/** The whole value is one unbraced reference. */ -const UNBRACED = /^\$([A-Za-z_][A-Za-z0-9_]*)$/; - -/** - * Names the native runtime binds itself during review. Taking one from the - * shell would give a service another instance's value and remove the - * reference the runtime looks for. +/** Shell source-name policy and private delivery limits. Resolution, including + * dotenv and unset semantics, is owned by native-compose-environment.ts. + * Values never enter this module except to measure a private delivery. */ const RUNTIME_SUPPLIED_NAMES: ReadonlySet = new Set([ COMPOSE_PROJECT_NAME_VARIABLE, @@ -79,110 +38,6 @@ export function shellVariableReadable(name: string): boolean { ); } -/** The shell's own string value for `name`, never an inherited property. */ -function shellValue( - env: Readonly>, - name: string -): string | undefined { - if (!Object.hasOwn(env, name)) { - return; - } - const value = env[name]; - return typeof value === "string" ? value : undefined; -} - -/** The variable an entry reads from the shell, or nothing when it reads none. */ -export function shellReference(opts: { - readonly key: string; - readonly value: unknown; - /** Names Hack delivers for this service. */ - readonly managed: ReadonlySet; -}): string | undefined { - if (opts.managed.has(opts.key) || !NAME.test(opts.key)) { - return; - } - let name: string | undefined; - if (opts.value === null) { - // A bare `NAME`: pass the shell's value through. - name = opts.key; - } else if (typeof opts.value === "string") { - name = BRACED.exec(opts.value)?.[1] ?? UNBRACED.exec(opts.value)?.[1]; - } - if (name === undefined || opts.managed.has(name)) { - return; - } - // The entry's own key is checked too: it is the name that gets delivered. - return shellVariableReadable(name) && shellVariableReadable(opts.key) - ? name - : undefined; -} - -export type ShellEnvironmentResolution = { - /** The service's environment as a mapping, with resolved entries set to `null`. */ - readonly environment: Record; - /** Values to deliver privately, by variable name. Never log this map. */ - readonly delivered: Record; - /** Value-free notes for the user: names only. */ - readonly notices: string[]; - /** Source names actually present in the caller's environment; never values. */ - readonly forwardedNames: readonly string[]; -}; - -/** - * Resolves every shell reference of one service. Returns `null` when the - * service has none, so the caller leaves its `environment` untouched. - * - * A referenced variable the shell does not set is delivered as an empty - * value, with a notice that names it. Compose substitutes an empty string for - * an unset reference; for a bare name Compose leaves the variable unset in the - * container, and native delivers it empty instead so the reviewed document - * stays the same from one shell to the next. - */ -export function resolveShellEnvironment(opts: { - readonly service: string; - /** The service's environment as a mapping (`NAME` → value or `null`). */ - readonly environment: Readonly>; - readonly env?: Readonly>; - readonly managed: ReadonlySet; -}): ShellEnvironmentResolution | null { - // Null-prototype maps: `__proto__` is a valid variable name. - const environment: Record = Object.create(null); - const delivered: Record = Object.create(null); - const notices: string[] = []; - const forwardedNames = new Set(); - let changed = false; - for (const [key, value] of Object.entries(opts.environment)) { - const name = shellReference({ key, value, managed: opts.managed }); - if (name === undefined) { - environment[key] = value; - continue; - } - changed = true; - const found = - opts.env === undefined ? undefined : shellValue(opts.env, name); - environment[key] = null; - delivered[key] = found ?? ""; - if (found !== undefined) { - forwardedNames.add(name); - } - if (found === undefined) { - notices.push( - name === key - ? `Service ${opts.service}: ${key} is empty because it is not set in your shell.` - : `Service ${opts.service}: ${key} is empty because ${name} is not set in your shell.` - ); - } - } - return changed - ? { - environment, - delivered, - notices, - forwardedNames: [...forwardedNames].sort(), - } - : null; -} - /** The runtime's limits on one service's private delivery (`provider/environment.rs`). */ const MAX_SERVICE_PAYLOAD_BYTES = 32 * 1024; const MAX_SERVICE_KEYS = 256; diff --git a/src/lib/native-detached-up.ts b/src/lib/native-detached-up.ts index 32f600c20..2c709a4f2 100644 --- a/src/lib/native-detached-up.ts +++ b/src/lib/native-detached-up.ts @@ -9,6 +9,7 @@ import { okResult, } from "./cli-result.ts"; import { isRecord } from "./guards.ts"; +import { NATIVE_ROOT_DOTENV_IGNORED_NOTICE } from "./native-environment-notices.ts"; export const NATIVE_DETACHED_OWNER_ARGUMENT = "--internal-native-detached-owner"; @@ -22,7 +23,9 @@ type Ready = { readonly completed?: readonly string[]; }; let publishReady: ((ready: Ready) => void) | undefined; -let publishShellVariables: ((names: readonly string[]) => void) | undefined; +let publishShellVariables: + | ((names: readonly string[], ignoredRootDotenv?: boolean) => void) + | undefined; type StartupFailure = { readonly code: string; readonly message?: string; @@ -140,12 +143,13 @@ function startupFailureError(failure: StartupFailure): HackCliError { /** Names-only notice: never accepts or forwards an environment map or output text. */ export function nativeDetachedUpShellVariables( - names: readonly string[] + names: readonly string[], + ignoredRootDotenv = false ): boolean { if (!publishShellVariables) { return false; } - publishShellVariables(names); + publishShellVariables(names, ignoredRootDotenv); return true; } @@ -202,7 +206,11 @@ export async function runNativeDetachedOwner(opts: { value: | string | { readonly kind: "ready"; readonly data: Ready } - | { readonly kind: "shell-variables"; readonly names: readonly string[] } + | { + readonly kind: "shell-variables"; + readonly names: readonly string[]; + readonly ignoredRootDotenv?: true; + } | { readonly kind: "failure"; readonly data: StartupFailure } ): Promise => new Promise((resolve) => { @@ -243,9 +251,13 @@ export async function runNativeDetachedOwner(opts: { ready = true; send({ kind: "ready", data }); }; - publishShellVariables = (names) => { + publishShellVariables = (names, ignoredRootDotenv) => { if (!stopping) { - send({ kind: "shell-variables", names }); + send({ + kind: "shell-variables", + names, + ...(ignoredRootDotenv ? { ignoredRootDotenv: true as const } : {}), + }); } }; // A fresh owner starts with no names from any earlier attempt. @@ -380,11 +392,18 @@ export async function startNativeDetachedUp(opts: { if (isRecord(value) && value.kind === "shell-variables") { if ( !shellVariablesReported && - Object.keys(value).length === 2 && + Object.keys(value).every((key) => + ["kind", "names", "ignoredRootDotenv"].includes(key) + ) && + (value.ignoredRootDotenv === undefined || + value.ignoredRootDotenv === true) && isShellVariables(value.names) ) { shellVariablesReported = true; process.stderr.write(`${nativeShellForwardingNotice(value.names)}\n`); + if (value.ignoredRootDotenv === true) { + process.stderr.write(`${NATIVE_ROOT_DOTENV_IGNORED_NOTICE}\n`); + } } return; } diff --git a/src/lib/native-environment-notices.ts b/src/lib/native-environment-notices.ts new file mode 100644 index 000000000..ac3149cbe --- /dev/null +++ b/src/lib/native-environment-notices.ts @@ -0,0 +1,3 @@ +/** Fixed, value-free wording shared with the detached owner's names-only IPC. */ +export const NATIVE_ROOT_DOTENV_IGNORED_NOTICE = + "Project-root .env is ignored: native interpolation uses .env beside the selected Compose file. Move required variables beside that file or into Hack managed env."; diff --git a/tests/background-command-journal.test.ts b/tests/background-command-journal.test.ts index 659beb680..8cafa42b3 100644 --- a/tests/background-command-journal.test.ts +++ b/tests/background-command-journal.test.ts @@ -132,7 +132,10 @@ async function expectCode(promise: Promise, code: string) { describe("journal events and cursors", () => { test("accepted is persisted first; writers keep seq dense and strictly increasing", async () => { - const { paths, meta } = await acceptCommand({ input: input(), root }); + const { paths, meta } = await acceptCommand({ + input: input({ caller: "test:env" }), + root, + }); expect((await stat(paths.dir)).mode & 0o777).toBe(0o700); expect((await stat(paths.meta)).mode & 0o777).toBe(0o600); expect((await stat(paths.events)).mode & 0o777).toBe(0o600); @@ -154,7 +157,10 @@ describe("journal events and cursors", () => { seq: 4, }) ); - expect(JSON.stringify(meta)).not.toContain("env"); + const persisted: unknown = JSON.parse(await readFile(paths.meta, "utf8")); + expect(persisted).toEqual(meta); + expect(persisted).not.toHaveProperty("env"); + expect(persisted).not.toHaveProperty("environment"); }); test("an unterminated final line is a write in progress, not an event", () => { diff --git a/tests/fixtures/native-detached-owner.ts b/tests/fixtures/native-detached-owner.ts index 65dd7a9bf..1354d7a77 100644 --- a/tests/fixtures/native-detached-owner.ts +++ b/tests/fixtures/native-detached-owner.ts @@ -219,6 +219,10 @@ if (role === "channel-holder") { if (mode === "shell-names-empty") { nativeDetachedUpShellVariables([]); } + if (mode === "shell-names-root-dotenv") { + nativeDetachedUpShellVariables([], true); + nativeDetachedUpShellVariables([], true); + } if (["provider-pin", "wrapped-provider-pin"].includes(mode)) { await nativeDetachedUpFailure(new Error("synthetic-private-value")); const failure = new NativeRuntimeRequestError({ @@ -359,6 +363,7 @@ if (role === "channel-holder") { "cancel-at-accept", "shell-names", "shell-names-empty", + "shell-names-root-dotenv", "unsafe-failure", ].includes(mode) ) { diff --git a/tests/native-compose-backend-features-cli.test.ts b/tests/native-compose-backend-features-cli.test.ts index c83ee430c..45e727f2f 100644 --- a/tests/native-compose-backend-features-cli.test.ts +++ b/tests/native-compose-backend-features-cli.test.ts @@ -37,9 +37,9 @@ const STACK_FRAME = /^\s+at /m; const SPAWN_TIMEOUT_MS = 60_000; const IMAGE = `sha256:${"e".repeat(64)}`; const ADMITTED_START_REQUESTS = [ - "runtime check-project-share", "project plan", "project plan", + "runtime check-project-share", "runtime up", ]; const SELECTION = @@ -416,8 +416,8 @@ test( : refusal; expectOneClosedLine(result, `E_NATIVE_PROJECT_UNSUPPORTED: ${expected}`); expect(result.output).not.toContain("HACK_RUNTIME_BACKEND=compose"); - // The check passed: native up went on to its own source preflight. - expect(await f.calls()).toEqual(["runtime check-project-share"]); + // Static input admission refuses before the provider share check. + expect(await f.calls()).toEqual([]); } }, SPAWN_TIMEOUT_MS @@ -479,7 +479,7 @@ test( expect(result.output).not.toContain(secret); // The Compose backend would record the value in the image history too. expect(result.output).not.toContain("HACK_RUNTIME_BACKEND=compose"); - expect(await f.calls()).toEqual(["runtime check-project-share"]); + expect(await f.calls()).toEqual([]); }, SPAWN_TIMEOUT_MS ); diff --git a/tests/native-compose-env-syntax.test.ts b/tests/native-compose-env-syntax.test.ts new file mode 100644 index 000000000..2d589657d --- /dev/null +++ b/tests/native-compose-env-syntax.test.ts @@ -0,0 +1,186 @@ +import { expect, test } from "bun:test"; +import { + composeEnvironmentReferences, + interpolateComposeEnvironment, + parseComposeEnvironmentFile, +} from "../src/backends/native-compose-env-syntax.ts"; + +const SENTINEL = "synthetic-private-environment-canary"; +const variables: Readonly> = { + SET: SENTINEL, + EMPTY: "", + OTHER: "second", +}; +const lookup = (name: string) => + Object.hasOwn(variables, name) ? variables[name] : undefined; + +test("line budget counts physical lines including a terminated last assignment", () => { + const text = "KEY=value\n".repeat(4096); + expect(parseComposeEnvironmentFile({ text, lookup }).get("KEY")).toBe( + "value" + ); + expect(() => + parseComposeEnvironmentFile({ text: `${text}KEY=value\n`, lookup }) + ).toThrow("line 4097"); +}); + +test("source names include unselected branches while escaped dollars stay literal", () => { + expect([ + ...composeEnvironmentReferences( + "${OTHER:+${HOST}} $$ESCAPED $${BRACED} $$$REAL ${MISSING:-$$LITERAL}" + ), + ]).toEqual(["OTHER", "HOST", "REAL", "MISSING"]); +}); + +test.each([ + ["$SET/${SET}", `${SENTINEL}/${SENTINEL}`], + ["${SET:-fallback}", SENTINEL], + ["${EMPTY:-fallback}", "fallback"], + ["${EMPTY-fallback}", ""], + ["${MISSING-fallback}", "fallback"], + ["${SET:+yes}${EMPTY:+no}${EMPTY+set}", "yesset"], + ["${MISSING:-${EMPTY:-${OTHER}}}", "second"], + ["$$SET/$5/end$", "$SET/$5/end$"], + ["${EMPTY?not used}", ""], + ["${SET:?not used}", SENTINEL], +])("Compose interpolation %s", (text, expected) => { + expect(interpolateComposeEnvironment({ text, lookup })).toBe(expected); +}); + +test.each([ + `\u0000${SENTINEL}`, + `\${MISSING:?${SENTINEL}}`, + `\${EMPTY:?${SENTINEL}}`, + `\${MISSING?${SENTINEL}}`, + `\${SET/old/${SENTINEL}}`, + "${SET", + "${A:-${A:-${A:-${A:-${A:-${A}}}}}}", + "${SET:-${A:-${A:-${A:-${A}}}}}", +])("refusals omit authored error words and private values", (text) => { + try { + interpolateComposeEnvironment({ text, lookup }); + throw new Error("unexpected success"); + } catch (error) { + expect(String(error)).toContain("values omitted"); + expect(String(error)).not.toContain(SENTINEL); + } +}); + +test("four nested expressions are admitted at the declared depth bound", () => { + expect( + interpolateComposeEnvironment({ + text: "${A:-${A:-${A:-${OTHER}}}}", + lookup, + }) + ).toBe("second"); +}); + +test("Compose quote delimiters and octal escapes follow the pinned parser", () => { + const result = parseComposeEnvironmentFile({ + text: String.raw`OCTAL="\0123/\x41/\u0041" +SINGLE='a\\' +DOUBLE="a\\" +`, + lookup, + }); + expect(Object.fromEntries(result)).toEqual({ + OCTAL: "S/\\x41/\\u0041", + SINGLE: "a\\\\", + DOUBLE: "a\\", + }); +}); + +test("Compose dotenv quoting, comments, multiline values and sequential lookups", () => { + const result = parseComposeEnvironmentFile({ + text: [ + "# comment", + "export FIRST=one # comment", + "SECOND=${FIRST}", + "FIRST=two", + "THIRD=${FIRST}", + "EXTERNAL=${SET}", + "COLON: value", + "HASH=a#literal", + "QUOTE='${SET} # literal'", + 'MULTI="line1', + 'line2\\tvalue" # comment', + 'ESCAPED="\\$SET/$$SET"', + 'LITERAL="{\\"key\\": \\"value\\"}"', + "BARE", + "EMPTY=", + "__proto__=own", + "", + ].join("\n"), + lookup, + }); + expect(Object.fromEntries(result)).toEqual({ + FIRST: "two", + SECOND: "one", + THIRD: "two", + EXTERNAL: SENTINEL, + COLON: "value", + HASH: "a#literal", + QUOTE: "${SET} # literal", + MULTI: "line1\nline2\tvalue", + ESCAPED: "$SET/$SET", + LITERAL: '{"key": "value"}', + EMPTY: "", + ["__proto__"]: "own", + }); + expect(result.has("BARE")).toBe(false); + expect(result.get("__proto__")).toBe("own"); +}); + +test("lookup takes precedence, current file overrides earlier files, unresolved bare does not erase", () => { + const result = parseComposeEnvironmentFile({ + text: "FIRST=new\nCOPY=${FIRST}\nKEPT\nSET\nFROM_PROJECT=${SET}\n", + initial: new Map([ + ["FIRST", "earlier"], + ["KEPT", "earlier"], + ]), + lookup, + }); + expect(Object.fromEntries(result)).toEqual({ + FIRST: "new", + COPY: "new", + KEPT: "earlier", + SET: SENTINEL, + FROM_PROJECT: SENTINEL, + }); +}); + +test("raw leaves dollar references, quote characters and comments literal", () => { + expect( + Object.fromEntries( + parseComposeEnvironmentFile({ + text: 'A="${SET}" # text\nB= $SET \n', + raw: true, + lookup, + }) + ) + ).toEqual({ A: '"${SET}" # text', B: " $SET " }); +}); + +test("pinned Compose removes the escaped single-quote delimiter backslash", () => { + expect( + parseComposeEnvironmentFile({ text: "A='it\\'s'\n", lookup }).get("A") + ).toBe("it's"); +}); + +test("raw does not accept a colon separator before an equals in the value", () => { + expect(() => + parseComposeEnvironmentFile({ text: "KEY: a=b\n", raw: true, lookup }) + ).toThrow("values omitted"); +}); + +test.each([ + `BAD KEY=${SENTINEL}`, + `BROKEN='${SENTINEL}`, + `A="${SENTINEL}"garbage`, + "\n".repeat(4097), + `A=${"a".repeat(256 * 1024)}`, +])("malformed or over-budget dotenv never echoes contents", (text) => { + expect(() => parseComposeEnvironmentFile({ text, lookup })).toThrow( + "values omitted" + ); +}); diff --git a/tests/native-compose-environment.test.ts b/tests/native-compose-environment.test.ts new file mode 100644 index 000000000..2dff99693 --- /dev/null +++ b/tests/native-compose-environment.test.ts @@ -0,0 +1,1191 @@ +import { afterEach, expect, test } from "bun:test"; +import { + chmod, + link, + mkdir, + mkdtemp, + readFile, + realpath, + rm, + symlink, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { adaptNativeAwsEnvironment } from "../src/backends/native-aws-environment.ts"; +import { adaptNativeGcloudEnvironment } from "../src/backends/native-gcloud-environment.ts"; +import { nativePrivateEnvironmentPayload } from "../src/backends/native-private-environment-payload.ts"; +import { + admitNativeProjectBuilds, + buildNativeProjectImages, + refuseManagedSecretBuildArguments, +} from "../src/backends/native-project-builds.ts"; +import { + mergeNativeShellEnvironment, + prepareNativeProjectInput, + requireNativeShellDeliveryFits, + selectActiveNativeProjectInput, +} from "../src/backends/native-project-input.ts"; +import { NATIVE_ROOT_DOTENV_IGNORED_NOTICE } from "../src/lib/native-environment-notices.ts"; +import { setProjectEnvValue } from "../src/lib/project-env-config.ts"; + +const FILE = "synthetic-file-value-canary"; +const DOTENV = "synthetic-dotenv-value-canary"; +const SHELL = "synthetic-shell-value-canary"; +const MANAGED = "synthetic-managed-value-canary"; +const roots: string[] = []; +afterEach(async () => { + for (const root of roots.splice(0)) { + await rm(root, { recursive: true, force: true }); + } +}); +async function fixture( + service: Record, + extra: Record = {} +) { + const projectRoot = await realpath( + await mkdtemp(join(tmpdir(), "native-compose-env-")) + ); + roots.push(projectRoot); + const projectDir = join(projectRoot, ".hack"); + await mkdir(projectDir, { mode: 0o700 }); + const composeFile = join(projectDir, "docker-compose.yml"); + await writeFile( + composeFile, + JSON.stringify({ + services: { app: { image: "alpine:3.21", ...service } }, + ...extra, + }) + ); + const source = async (path: string, text: string) => + await writeFile(join(projectDir, path), text, { mode: 0o600 }); + return { projectRoot, projectDir, composeFile, source }; +} +async function managed(projectDir: string, values: unknown) { + await writeFile( + join(projectDir, "hack.env.default.yaml"), + JSON.stringify({ + version: 1, + environment: "default", + secretsprovider: "project_key", + values, + }) + ); +} +function publicService( + input: Awaited> +) { + return JSON.parse(input.normalizedComposeJson).services.app; +} + +test("service environment beats ordered files; interpolation is private and dotenv does not inject", async () => { + const f = await fixture({ + env_file: "service.env", + environment: { + OVERRIDE: "explicit", + EMPTY: "", + PASSED: null, + ABSENT: null, + INTERPOLATED: "prefix-${SOURCE}", + }, + }); + await f.source(".env", `SOURCE=${DOTENV}\nNOT_INJECTED=${DOTENV}\n`); + await f.source( + "service.env", + `FILE=${FILE}\nOVERRIDE=${FILE}\nEMPTY=${FILE}\nABSENT=${FILE}\nCOPY=\${SOURCE}\n` + ); + const original = await readFile(f.composeFile); + const input = await prepareNativeProjectInput({ + ...f, + env: { SOURCE: SHELL, PASSED: SHELL }, + }); + expect(input.managedEnvironment).toEqual({}); + expect(input.shellEnvironment).toEqual({ + app: { FILE, COPY: SHELL, PASSED: SHELL, INTERPOLATED: `prefix-${SHELL}` }, + }); + expect(publicService(input)).toEqual({ + image: "alpine:3.21", + environment: { + FILE: null, + COPY: null, + OVERRIDE: "explicit", + EMPTY: "", + PASSED: null, + INTERPOLATED: null, + }, + "x-hack-unset-environment": ["ABSENT"], + }); + expect(publicService(input).environment).not.toHaveProperty("NOT_INJECTED"); + expect(await readFile(f.composeFile)).toEqual(original); + for (const canary of [FILE, DOTENV, SHELL]) { + expect(input.normalizedComposeJson).not.toContain(canary); + expect(input.originalSha256).not.toContain(canary); + expect(JSON.stringify(input.environmentNotices)).not.toContain(canary); + } + const delivered = mergeNativeShellEnvironment(input); + expect(() => requireNativeShellDeliveryFits(delivered)).not.toThrow(); + const wire = nativePrivateEnvironmentPayload({ + services: delivered.managedEnvironment, + planId: "a".repeat(64), + run: "b".repeat(32), + }); + expect(JSON.parse(wire.toString()).services.app.FILE).toBe(FILE); +}); + +test("short/list/long optional/raw and duplicate entries preserve Compose order", async () => { + const f = await fixture({ + env_file: [ + { path: "one.env", format: "" }, + { path: "missing.env", required: false }, + { path: "two.env", format: "raw" }, + "one.env", + ], + }); + await f.source("one.env", `VALUE=${FILE}\nBARE\n`); + await f.source( + "two.env", + 'VALUE="${SOURCE}" # literal\nRAW="${SOURCE}" # literal\n' + ); + const input = await prepareNativeProjectInput(f); + expect(input.shellEnvironment).toEqual({ + app: { VALUE: FILE, RAW: '"${SOURCE}" # literal' }, + }); + expect(publicService(input)).not.toHaveProperty("env_file"); +}); + +test("Compose rejects the literal default format name", async () => { + const f = await fixture({ + env_file: { path: "service.env", format: "default" }, + }); + await f.source("service.env", `TOKEN=${FILE}\n`); + await expect(prepareNativeProjectInput(f)).rejects.toThrow("format"); +}); + +test("explicit cloud values may seed env-file expansion without reading cloud shell variables", async () => { + const f = await fixture({ + env_file: "service.env", + environment: { AWS_REGION: "explicit-region" }, + }); + await f.source("service.env", "REGION=${AWS_REGION}\n"); + const input = await prepareNativeProjectInput({ + ...f, + env: { AWS_REGION: SHELL }, + }); + expect(input.shellEnvironment).toEqual({ + app: { REGION: "explicit-region" }, + }); + expect(input.normalizedComposeJson).not.toContain(SHELL); +}); + +test("file-backed names do not capture service defaults from the project interpolation context", async () => { + const f = await fixture({ + env_file: "service.env", + environment: { + URL: "${HOST:-localhost}", + ALT: "${HOST:+present}", + DOLLAR: "${HOST:-$$LITERAL}", + UNSELECTED: "${OTHER:+${HOST}}", + UNAFFECTED: "${OTHER:-public}", + }, + }); + await f.source("service.env", `HOST=${FILE}\n`); + const first = await prepareNativeProjectInput(f); + expect(first.shellEnvironment).toEqual({ app: { HOST: FILE } }); + expect(publicService(first).environment).toEqual({ + HOST: null, + URL: "localhost", + ALT: "", + DOLLAR: "$$LITERAL", + UNSELECTED: "", + UNAFFECTED: "${OTHER:-public}", + }); + expect(first.normalizedComposeJson).not.toContain(FILE); + await f.source("service.env", `HOST=${DOTENV}\n`); + const second = await prepareNativeProjectInput(f); + expect(second.normalizedComposeJson).toBe(first.normalizedComposeJson); + expect(second.shellEnvironment).toEqual({ app: { HOST: DOTENV } }); +}); + +test("a default combined with a private source stays private after file bindings are introduced", async () => { + const f = await fixture({ + env_file: "service.env", + environment: { URL: "${HOST:-localhost}/${SOURCE}" }, + }); + await f.source("service.env", `HOST=${FILE}\n`); + await f.source(".env", `SOURCE=${DOTENV}\n`); + const input = await prepareNativeProjectInput(f); + expect(publicService(input).environment).toEqual({ HOST: null, URL: null }); + expect(input.shellEnvironment).toEqual({ + app: { HOST: FILE, URL: `localhost/${DOTENV}` }, + }); + expect(input.normalizedComposeJson).not.toContain(FILE); + expect(input.normalizedComposeJson).not.toContain(DOTENV); +}); + +test.each([ + { + field: "command", + authored: [ + "echo", + "${HOST:-localhost}", + "${HOST:+present}", + "${OTHER:+${HOST}}", + "${HOST:-$$LITERAL}", + "${COMPOSE_PROJECT_NAME}", + "${OTHER:-untouched}", + "$$HOST", + ], + expected: [ + "echo", + "localhost", + "", + "", + "$$LITERAL", + "${COMPOSE_PROJECT_NAME}", + "${OTHER:-untouched}", + "$$HOST", + ], + }, + { + field: "command", + authored: "echo ${HOST:-localhost}", + expected: "echo localhost", + }, + { field: "entrypoint", authored: ["${HOST:-echo}"], expected: ["echo"] }, + { field: "entrypoint", authored: "${HOST:-echo}", expected: "echo" }, + { field: "user", authored: "${HOST:-1000}", expected: "1000" }, + { + field: "healthcheck", + authored: { test: ["CMD", "echo", "${HOST:-healthy}"] }, + expected: { test: ["CMD", "echo", "healthy"] }, + }, + { + field: "healthcheck", + authored: { test: "echo ${HOST:-healthy}" }, + expected: { test: "echo healthy" }, + }, + { + field: "extra_hosts", + authored: ["relay.hack:${HOST:-host-gateway}"], + expected: ["relay.hack:host-gateway"], + }, + { + field: "extra_hosts", + authored: { "relay.hack": "${HOST:-host-gateway}" }, + expected: { "relay.hack": "host-gateway" }, + }, + { + field: "extra_hosts", + authored: { "${HOST:-relay.hack}": "host-gateway" }, + expected: { "relay.hack": "host-gateway" }, + }, + { + field: "extra_hosts", + authored: { "${OTHER:+${HOST}}relay.hack": "host-gateway" }, + expected: { "relay.hack": "host-gateway" }, + }, +])("public executable defaults use project context before file bindings (%s)", async ({ + field, + authored, + expected, +}) => { + const f = await fixture({ env_file: "service.env", [field]: authored }); + await f.source("service.env", `HOST=${FILE}\n`); + const first = await prepareNativeProjectInput(f); + expect(publicService(first)[field]).toEqual(expected); + expect(first.shellEnvironment).toEqual({ app: { HOST: FILE } }); + expect(first.normalizedComposeJson).not.toContain(FILE); + await f.source("service.env", `HOST=${DOTENV}\n`); + const second = await prepareNativeProjectInput(f); + expect(second.normalizedComposeJson).toBe(first.normalizedComposeJson); + expect(second.shellEnvironment).toEqual({ app: { HOST: DOTENV } }); +}); + +test("extra_hosts keys that freeze to one hostname refuse without echoing file values", async () => { + const f = await fixture({ + env_file: "service.env", + extra_hosts: { + "${HOST:-relay.hack}": "host-gateway", + "relay.hack": "host-gateway", + }, + }); + await f.source("service.env", `HOST=${FILE}\n`); + const error: unknown = await prepareNativeProjectInput(f).catch( + (caught: unknown) => caught + ); + expect(String(error)).toContain("hostnames must remain distinct"); + expect(String(error)).not.toContain(FILE); +}); + +test("extra_hosts hostname interpolation refuses project values and runtime-bound compounds", async () => { + const f = await fixture({ + env_file: "service.env", + extra_hosts: { "${HOST:-relay.hack}": "host-gateway" }, + }); + await f.source("service.env", `HOST=${FILE}\n`); + const projectError: unknown = await prepareNativeProjectInput({ + ...f, + env: { HOST: SHELL }, + }).catch((caught: unknown) => caught); + expect(String(projectError)).toContain("private variables"); + expect(String(projectError)).not.toContain(SHELL); + expect(String(projectError)).not.toContain(FILE); + await writeFile( + f.composeFile, + JSON.stringify({ + services: { + app: { + image: "alpine", + env_file: "service.env", + extra_hosts: { + "${HOST:-relay}.${COMPOSE_PROJECT_NAME}.hack": "host-gateway", + }, + }, + }, + }) + ); + const runtimeError: unknown = await prepareNativeProjectInput(f).catch( + (caught: unknown) => caught + ); + expect(String(runtimeError)).toContain("runtime-bound interpolation"); + expect(String(runtimeError)).toContain("runtime-bound"); + expect(String(runtimeError)).not.toContain(FILE); +}); + +test("other public mapping keys keep their ordinary Compose semantics", async () => { + const f = await fixture({ + env_file: "service.env", + labels: { "${HOST:-literal-label}": "public" }, + }); + await f.source("service.env", `HOST=${FILE}\n`); + const input = await prepareNativeProjectInput(f); + expect(publicService(input).labels).toEqual({ + "${HOST:-literal-label}": "public", + }); + expect(input.normalizedComposeJson).not.toContain(FILE); +}); + +test.each([ + "COMPOSE_PROJECT_NAME", + "AWS_REGION", + "MANAGED", +])("public defaults cannot freeze runtime-bound %s into a different value", async (name) => { + const f = await fixture({ + env_file: "service.env", + command: ["echo", `\${HOST:-localhost}/\${${name}:-fallback}`], + }); + await f.source("service.env", `HOST=${FILE}\n`); + await managed(f.projectDir, { app: { MANAGED } }); + const error: unknown = await prepareNativeProjectInput(f).catch( + (caught: unknown) => caught + ); + expect(String(error)).toContain("runtime-bound"); + expect(String(error)).not.toContain(FILE); + expect(String(error)).not.toContain(MANAGED); +}); + +test("public executable defaults still refuse a defined project source beside a file binding", async () => { + const f = await fixture({ + env_file: "service.env", + command: ["echo", "${HOST:-localhost}/${SOURCE}"], + }); + await f.source("service.env", `HOST=${FILE}\n`); + await f.source(".env", `SOURCE=${DOTENV}\n`); + const error: unknown = await prepareNativeProjectInput(f).catch( + (caught: unknown) => caught + ); + expect(String(error)).toContain("private variables"); + expect(String(error)).not.toContain(FILE); + expect(String(error)).not.toContain(DOTENV); +}); + +test("env-file-local cloud values resolve without reading cloud shell values", async () => { + const f = await fixture({ env_file: ["one.env", "two.env"] }); + await f.source( + "one.env", + `AWS_REGION=${FILE}\nSAME=\${AWS_REGION}\nGOOGLE_PROJECT=${DOTENV}\nCLOUDSDK_CORE_PROJECT=${FILE}\n` + ); + await f.source( + "two.env", + "REGION=${AWS_REGION}\nPROJECT=${GOOGLE_PROJECT}\nCLOUD=${CLOUDSDK_CORE_PROJECT}\nAWS_REGION\n" + ); + const env: Record = {}; + for (const key of ["AWS_REGION", "GOOGLE_PROJECT", "CLOUDSDK_CORE_PROJECT"]) { + Object.defineProperty(env, key, { + enumerable: true, + get() { + throw new Error("cloud shell value must never be read"); + }, + }); + } + const input = await prepareNativeProjectInput({ ...f, env }); + expect(input.shellEnvironment).toEqual({ + app: { + AWS_REGION: FILE, + SAME: FILE, + GOOGLE_PROJECT: DOTENV, + CLOUDSDK_CORE_PROJECT: FILE, + REGION: FILE, + PROJECT: DOTENV, + CLOUD: FILE, + }, + }); + expect(input.forwardedShellVariables?.app).toEqual([]); + expect(input.normalizedComposeJson).not.toContain(FILE); + expect(input.normalizedComposeJson).not.toContain(DOTENV); + await f.source("one.env", "REGION=${AWS_REGION}\n"); + await expect(prepareNativeProjectInput({ ...f, env })).rejects.toThrow( + "cannot be read from the caller's shell" + ); +}); + +test("dotenv cloud references use earlier local values and refuse an unavailable source", async () => { + const f = await fixture({ environment: { REGION: "${REGION}" } }); + await f.source(".env", `AWS_REGION=${DOTENV}\nREGION=\${AWS_REGION}\n`); + const input = await prepareNativeProjectInput({ + ...f, + env: { AWS_REGION: SHELL }, + }); + expect(input.shellEnvironment).toEqual({ app: { REGION: DOTENV } }); + expect(input.normalizedComposeJson).not.toContain(DOTENV); + expect(input.normalizedComposeJson).not.toContain(SHELL); + await f.source(".env", "REGION=${AWS_REGION}\n"); + const error: unknown = await prepareNativeProjectInput({ + ...f, + env: { AWS_REGION: SHELL }, + }).catch((caught: unknown) => caught); + expect(String(error)).toContain('source ".env" line 1 key REGION'); + expect(String(error)).toContain("cannot be read"); + expect(String(error)).not.toContain("AWS_REGION"); + expect(String(error)).not.toContain(SHELL); +}); + +test("NUL private inputs refuse with closed diagnostics", async () => { + const f = await fixture({ env_file: "service.env" }); + await f.source("service.env", `TOKEN=${FILE}\0\n`); + const error: unknown = await prepareNativeProjectInput(f).catch( + (caught: unknown) => caught + ); + expect(String(error)).toContain("values omitted"); + expect(String(error)).not.toContain(FILE); +}); + +test("documented Compose precedence includes managed globals, service values, files and interpolation sources", async () => { + // Compose v5.6.1 / compose-go v2.16.1: project lookup > service environment + // > current/previous env-file assignments; service environment wins finally. + const f = await fixture({ + env_file: ["one.env", "two.env"], + environment: { + K: "service", + BARE: null, + EMPTY: "", + FROM_CONTEXT: "${CONTEXT}", + }, + }); + await f.source(".env", `CONTEXT=${DOTENV}\n`); + await f.source( + "one.env", + `K=${FILE}\nEARLIER=${FILE}\nBARE=${FILE}\nEMPTY=${FILE}\n` + ); + await f.source("two.env", "R=${K}\nS=${EARLIER}\nEARLIER\nT=${CONTEXT}\n"); + await managed(f.projectDir, { + global: { CONTEXT: MANAGED }, + app: { K: MANAGED }, + }); + const input = await prepareNativeProjectInput({ + ...f, + env: { CONTEXT: SHELL }, + }); + const final = mergeNativeShellEnvironment(input); + expect(final.managedEnvironment.app).toEqual({ + CONTEXT: MANAGED, + K: MANAGED, + FROM_CONTEXT: MANAGED, + EARLIER: FILE, + R: MANAGED, + S: FILE, + T: MANAGED, + }); + expect(publicService(input).environment.EMPTY).toBe(""); + expect(publicService(input)["x-hack-unset-environment"]).toEqual(["BARE"]); + expect(final.forwardedShellVariables?.app).not.toContain("CONTEXT"); +}); + +test("file value-only edits do not enter normalized identity inputs or managed fingerprints", async () => { + const f = await fixture({ + env_file: "service.env", + environment: { COPY: "${SOURCE}" }, + }); + const prepare = async (value: string) => { + await f.source(".env", `SOURCE=${value}\n`); + await f.source("service.env", `TOKEN=${value}\n`); + return await prepareNativeProjectInput(f); + }; + const a = await prepare(FILE); + const b = await prepare(DOTENV); + expect(a.normalizedComposeJson).toBe(b.normalizedComposeJson); + expect(a.originalSha256).toBe(b.originalSha256); + expect(a.managedEnvironment).toEqual(b.managedEnvironment); + expect(a.shellEnvironment).not.toEqual(b.shellEnvironment); +}); + +test("unreferenced dotenv does not create private delivery or rewrite pure defaults", async () => { + const f = await fixture({ + restart: "unless-stopped", + environment: { MODE: "${NOT_SET:-safe}", LITERAL: "public" }, + }); + await f.source(".env", `UNUSED=${DOTENV}\n`); + const input = await prepareNativeProjectInput(f); + expect(input.shellEnvironment).toEqual({}); + expect(publicService(input).environment).toEqual({ + MODE: "${NOT_SET:-safe}", + LITERAL: "public", + }); +}); + +test.each([ + { image: "example:${SOURCE:-fallback}" }, + { command: ["echo", "${SOURCE:-fallback}"] }, + { entrypoint: ["${SOURCE}"] }, + { ports: ["${SOURCE:-3000}:3000"] }, + { labels: { note: "${SOURCE}" } }, + { labels: { "x-note": "${SOURCE}" } }, + { user: "${SOURCE:-1000}" }, + { healthcheck: { test: ["CMD", "${SOURCE}"] } }, + { build: { args: { SOURCE: "${SOURCE}" } } }, +])("public-field interpolation refuses before values can leave private preparation", async (service) => { + const f = await fixture(service); + await f.source(".env", `SOURCE=${DOTENV}\n`); + const error: unknown = await prepareNativeProjectInput(f).catch( + (caught: unknown) => caught + ); + expect(error).toBeInstanceOf(Error); + expect(String(error)).toContain("private variables"); + expect(String(error)).not.toContain(DOTENV); +}); + +test("missing required references never echo the authored error message", async () => { + const f = await fixture({ env_file: "service.env" }); + await f.source("service.env", `TOKEN=\${MISSING:?${FILE}}\n`); + const error: unknown = await prepareNativeProjectInput(f).catch( + (caught: unknown) => caught + ); + expect(error).toBeInstanceOf(Error); + expect(String(error)).toContain("service.env"); + expect(String(error)).not.toContain(FILE); +}); + +test("possible materialized dotenv refuses; root dotenv is not an interpolation source", async () => { + const f = await fixture({ environment: { ROOT_ONLY: "${ROOT_ONLY}" } }); + await writeFile(join(f.projectRoot, ".env"), `ROOT_ONLY=${DOTENV}\n`, { + mode: 0o600, + }); + const input = await prepareNativeProjectInput(f); + expect(input.shellEnvironment?.app?.ROOT_ONLY).toBe(""); + await f.source(".env", `ROOT_ONLY=${DOTENV}\n`); + await f.source(".env.state.json", "not trusted even if readable"); + await expect(prepareNativeProjectInput(f)).rejects.toThrow("materialized"); +}); + +test.each([ + "COMPOSE_FILE", + "COMPOSE_PROFILES", + "COMPOSE_PROJECT_NAME", + "COMPOSE_ENV_FILES", + "HACK_ENV_SECRET_KEY", +])("dotenv cannot change project controls or forward protected %s", async (key) => { + const f = await fixture({}); + await f.source(".env", `${key}=${DOTENV}\n`); + const error: unknown = await prepareNativeProjectInput(f).catch( + (caught: unknown) => caught + ); + expect(error).toBeInstanceOf(Error); + expect(String(error)).toContain(key); + expect(String(error)).not.toContain(DOTENV); +}); + +test("file delivery is counted by the private service budget", async () => { + const f = await fixture({ env_file: "service.env" }); + await f.source("service.env", `LARGE=${"x".repeat(33 * 1024)}\n`); + const input = mergeNativeShellEnvironment(await prepareNativeProjectInput(f)); + expect(() => requireNativeShellDeliveryFits(input)).toThrow("32 KiB"); + await chmod(join(f.projectDir, "service.env"), 0o666); + await expect(prepareNativeProjectInput(f)).rejects.toThrow( + "not writable by others" + ); +}); + +test("unused extension nodes do not block aliased private environment", async () => { + const f = await fixture( + { environment: { TOKEN: "${SOURCE}" }, "x-env": { TOKEN: "${SOURCE}" } }, + { "x-env": { TOKEN: "${SOURCE}" } } + ); + await f.source(".env", `SOURCE=${DOTENV}\n`); + expect((await prepareNativeProjectInput(f)).shellEnvironment).toEqual({ + app: { TOKEN: DOTENV }, + }); +}); + +test("a project-root alias is canonicalized without permitting source symlinks", async () => { + const f = await fixture({ env_file: "service.env" }); + await f.source("service.env", `TOKEN=${FILE}\n`); + const alias = join(f.projectRoot, "root-alias"); + await symlink(f.projectRoot, alias); + const input = await prepareNativeProjectInput({ + projectRoot: alias, + projectDir: join(alias, ".hack"), + composeFile: join(alias, ".hack/docker-compose.yml"), + }); + expect(input.shellEnvironment).toEqual({ app: { TOKEN: FILE } }); +}); + +test.each([ + "AWS_REGION", + "GOOGLE_APPLICATION_CREDENTIALS", + "CLOUDSDK_CORE_PROJECT", + "COMPOSE_PROJECT_NAME", +])("unreadable bare %s remains runtime-owned rather than unset", async (key) => { + const f = await fixture({ environment: { [key]: null } }); + const input = await prepareNativeProjectInput({ + ...f, + env: { [key]: SHELL }, + }); + expect(publicService(input).environment).toEqual({ [key]: null }); + expect(publicService(input)).not.toHaveProperty("x-hack-unset-environment"); + expect(input.shellEnvironment).toEqual({}); +}); + +test("plain file and shell values do not widen the managed-secret build guard", async () => { + const f = await fixture({ + env_file: "service.env", + environment: { MODE: "explicit" }, + }); + await f.source(".env", "UNUSED=production\n"); + await f.source("service.env", `MODE=production\nTOKEN=${FILE}\n`); + const input = await prepareNativeProjectInput(f); + expect(input.managedSecretValues).toEqual([]); +}); + +test("ordinary env-file and shell values matching literal build args still build", async () => { + const f = await fixture({ + image: undefined, + env_file: "service.env", + environment: { PORT: null }, + build: { + context: "../app", + args: { NODE_ENV: "production", PORT: "3000" }, + }, + }); + await f.source("service.env", "NODE_ENV=production\n"); + const input = await prepareNativeProjectInput({ + ...f, + env: { PORT: "3000" }, + }); + expect(input.shellEnvironment).toEqual({ + app: { NODE_ENV: "production", PORT: "3000" }, + }); + expect(input.managedSecretValues).toEqual([]); + const specs = { app: publicService(input) }; + const builds = admitNativeProjectBuilds({ + ...f, + services: specs, + enabled: true, + }); + refuseManagedSecretBuildArguments({ + builds, + managedSecretValues: input.managedSecretValues ?? [], + }); + let invoked = false; + await buildNativeProjectImages({ + runtime: { + binary: "/unused/runtime", + home: join(f.projectRoot, "native-home"), + }, + projectRoot: f.projectRoot, + branch: null, + profiles: [], + builds, + specs, + retainedImages: new Map(), + restore: false, + rebuild: false, + signal: new AbortController().signal, + invoke: async ({ requestFile }) => { + expect(JSON.parse(await readFile(requestFile, "utf8")).args).toEqual({ + NODE_ENV: "production", + PORT: "3000", + }); + invoked = true; + return { + imageId: `sha256:${"a".repeat(64)}`, + buildKey: "b".repeat(64), + reused: false, + }; + }, + }); + expect(invoked).toBe(true); +}); + +test("only explicit managed secrets enter the build guard, including one-character values", async () => { + const f = await fixture({}); + await managed(f.projectDir, { global: { PLAIN: "production" } }); + await setProjectEnvValue({ + projectRoot: f.projectRoot, + projectDir: f.projectDir, + envName: null, + key: "MARKED", + value: "x", + secret: true, + scope: "global", + }); + const input = await prepareNativeProjectInput(f); + expect(input.managedSecretValues).toEqual(["x"]); + const builds = admitNativeProjectBuilds({ + ...f, + enabled: true, + services: { app: { build: { context: "../app", args: { A: "x" } } } }, + }); + expect(() => + refuseManagedSecretBuildArguments({ + builds, + managedSecretValues: input.managedSecretValues ?? [], + }) + ).toThrow("hack env secret value"); +}); + +test.each([ + "service.env", + ".env", +])("%s notices never expose dollar tokens from private values", async (path) => { + const f = await fixture({ env_file: path === ".env" ? undefined : path }); + await f.source(path, "# header\nDB_PASSWORD=abc$Tail9\n"); + const input = await prepareNativeProjectInput(f); + const notices = JSON.stringify(input.environmentNotices); + expect(notices).toContain(path); + expect(notices).toContain("line 2 key DB_PASSWORD"); + expect(notices).not.toContain("Tail9"); + expect(notices).not.toContain("abc"); + const withShell = await prepareNativeProjectInput({ + ...f, + env: { Tail9: SHELL }, + }); + expect(JSON.stringify(withShell.forwardedShellVariables)).not.toContain( + "Tail9" + ); + expect(JSON.stringify(withShell.environmentNotices)).not.toContain("Tail9"); +}); + +test.each([ + { token: "HACK_PrivateTail9", reason: "protected variables" }, + { token: "AWS_PrivateTail9", reason: "cannot be read" }, + { token: "RuntimeTail9", reason: "runtime-dependent" }, +])("private dollar-reference refusal identifies the assignment, never $token", async ({ + token, + reason, +}) => { + const f = await fixture({ + env_file: "service.env", + environment: { RuntimeTail9: "${COMPOSE_PROJECT_NAME}" }, + }); + await f.source("service.env", `# header\nDB_PASSWORD=abc$${token}\n`); + const error: unknown = await prepareNativeProjectInput(f).catch( + (caught: unknown) => caught + ); + expect(error).toBeInstanceOf(Error); + const message = String(error); + expect(message).toContain('source "service.env" line 2 key DB_PASSWORD'); + expect(message).toContain(reason); + expect(message).not.toContain(token); + expect(message).not.toContain("abc"); +}); + +test.each([ + "HACK_PrivateTail9", + "AWS_PrivateTail9", +])("dotenv refusal omits the private reference %s", async (token) => { + const f = await fixture({}); + await f.source(".env", `# header\nDB_PASSWORD=abc$${token}\n`); + const error: unknown = await prepareNativeProjectInput(f).catch( + (caught: unknown) => caught + ); + expect(error).toBeInstanceOf(Error); + expect(String(error)).toContain('source ".env" line 2 key DB_PASSWORD'); + expect(String(error)).not.toContain(token); + expect(String(error)).not.toContain("abc"); +}); + +test.each([ + "service.env", + ".env", +])("%s syntax refusals identify the physical line without excerpts", async (path) => { + const f = await fixture(path === ".env" ? {} : { env_file: path }); + for (const syntax of [ + `DOTTED.KEY=${FILE}`, + `DASHED-KEY=${FILE}`, + `TOKEN="${FILE}`, + `TOKEN=\${MissingTail9:?${FILE}}`, + `TOKEN=${FILE}\0`, + ]) { + await f.source(path, `# header\nOK=value\n${syntax}\n`); + const error: unknown = await prepareNativeProjectInput(f).catch( + (caught: unknown) => caught + ); + expect(error).toBeInstanceOf(Error); + expect(String(error)).toContain(`source "${path}" line 3`); + expect(String(error)).not.toContain(FILE); + expect(String(error)).not.toContain("MissingTail9"); + expect(String(error)).not.toContain("DOTTED.KEY"); + } +}); + +test("without file sources public fields keep their authored defaults despite exported shell names", async () => { + const declarations = { + ports: ["${PORT:-3000}:3000"], + image: "example:${TAG:-1}", + command: ["echo", "${PUBLIC:-fallback}"], + entrypoint: ["${ENTRY:-echo}"], + }; + const f = await fixture(declarations); + const env: Record = {}; + for (const name of ["PORT", "TAG", "PUBLIC", "ENTRY"]) { + Object.defineProperty(env, name, { + enumerable: true, + get() { + throw new Error("public fields must not read ambient values"); + }, + }); + } + const input = await prepareNativeProjectInput({ ...f, env }); + expect(publicService(input)).toEqual(declarations); + await f.source(".env", ""); + await expect( + prepareNativeProjectInput({ ...f, env: { PORT: "4000" } }) + ).rejects.toThrow("private variables"); + // Removing the ambient source retains the authored public fallback. + expect(publicService(await prepareNativeProjectInput(f))).toEqual( + declarations + ); +}); + +test("service defaults use shell presence without placing values in normalized identity", async () => { + const f = await fixture({ + environment: { MODE: "${NODE_ENV:-development}" }, + }); + const absent = await prepareNativeProjectInput(f); + const present = await prepareNativeProjectInput({ + ...f, + env: { NODE_ENV: SHELL }, + }); + expect(publicService(absent).environment.MODE).toBe( + "${NODE_ENV:-development}" + ); + expect(publicService(present).environment.MODE).toBeNull(); + expect(present.shellEnvironment?.app?.MODE).toBe(SHELL); + expect(present.normalizedComposeJson).not.toContain(SHELL); + expect(present.normalizedComposeJson).not.toBe(absent.normalizedComposeJson); +}); + +test("ignored root dotenv gets one fixed notice without opening the file", async () => { + const f = await fixture({}); + const path = join(f.projectRoot, ".env"); + await writeFile(path, FILE, { mode: 0o000 }); + const input = await prepareNativeProjectInput(f); + expect(input.ignoredRootDotenv).toBe(true); + expect(input.environmentNotices).toEqual([ + { line: NATIVE_ROOT_DOTENV_IGNORED_NOTICE }, + ]); + expect(input.normalizedComposeJson).not.toContain(FILE); +}); + +test.each([ + { context: "..", path: "service.env" }, + { context: ".", path: "service.env" }, + { context: "../APP", path: "../app/secret.vars" }, + { context: "../STRASSE", path: "../Straße/secret.vars" }, + { context: "../café", path: "../cafe\u0301/secret.vars" }, + { context: "../app", path: "../app/nested/secret.vars" }, +])("build contexts cannot overlap an optional env-file path (%s)", async ({ + context, + path, +}) => { + const f = await fixture( + {}, + { + services: { + app: { + image: "alpine", + env_file: { path, required: false }, + profiles: ["inactive"], + }, + builder: { build: { context } }, + }, + } + ); + const error: unknown = await prepareNativeProjectInput(f).catch( + (caught: unknown) => caught + ); + expect(String(error)).toContain( + "native build inputs overlap an environment source" + ); + expect(String(error)).toContain("service builder field build"); + expect(String(error)).not.toContain(FILE); +}); + +test.each([ + "../.hack/service.env", + "../.hack/service.env.dockerignore", +])("build definition inputs outside the context cannot read env-file %s", async (path) => { + const f = await fixture( + {}, + { + services: { + app: { + image: "alpine", + env_file: { + path: path.endsWith(".dockerignore") + ? "service.env.dockerignore" + : "service.env", + required: false, + }, + }, + builder: { + build: { context: "../app", dockerfile: "../.hack/service.env" }, + }, + }, + } + ); + const error: unknown = await prepareNativeProjectInput(f).catch( + (caught: unknown) => caught + ); + expect(String(error)).toContain( + "native build inputs overlap an environment source" + ); +}); + +test("a selected dotenv cannot enter the raw build context even with no delivered key", async () => { + const f = await fixture({}, { services: { builder: { build: ".." } } }); + await f.source(".env", `# ${DOTENV}\n`); + const error: unknown = await prepareNativeProjectInput(f).catch( + (caught: unknown) => caught + ); + expect(String(error)).toContain("overlap an environment source"); + expect(String(error)).not.toContain(DOTENV); +}); + +test("env-file sources stay private with disjoint native build inputs", async () => { + const f = await fixture( + {}, + { + services: { + app: { image: "alpine", env_file: "service.env" }, + builder: { build: { context: "../app", dockerfile: "Dockerfile" } }, + }, + } + ); + await f.source("service.env", `TOKEN=${FILE}\n`); + await f.source(".env", `UNUSED=${DOTENV}\n`); + const input = await prepareNativeProjectInput(f); + expect(input.shellEnvironment).toEqual({ app: { TOKEN: FILE } }); + expect(publicService(input).environment).toEqual({ TOKEN: null }); + expect(input.normalizedComposeJson).not.toContain(FILE); + expect(input.normalizedComposeJson).not.toContain(DOTENV); +}); + +test.each([ + { context: ".." }, + { context: "../app", dockerfile: "../.env" }, +])("ignored root dotenv stays out of native build inputs %s", async (build) => { + const f = await fixture({}, { services: { builder: { build } } }); + // No env_file or adjacent .env: this is the explicitly approved root-only exception. + await writeFile(join(f.projectRoot, ".env"), `TOKEN=${FILE}\n`, { + mode: 0o000, + }); + const error: unknown = await prepareNativeProjectInput(f).catch( + (caught: unknown) => caught + ); + expect(error).toBeInstanceOf(Error); + expect(String(error)).toContain('source "../.env"'); + expect(String(error)).toContain("service builder field build"); + expect(String(error)).toContain("overlap an environment source"); + expect(String(error)).not.toContain(FILE); +}); + +test("ignored root dotenv preserves disjoint builds and selected-service exec without reading its values", async () => { + const f = await fixture( + {}, + { services: { builder: { build: { context: "../app" } } } } + ); + await writeFile(join(f.projectRoot, ".env"), `TOKEN=${FILE}\n`, { + mode: 0o000, + }); + const input = await prepareNativeProjectInput(f); + expect(input.ignoredRootDotenv).toBe(true); + expect(input.shellEnvironment).toEqual({}); + expect(JSON.stringify(input.environmentNotices)).not.toContain(FILE); + await writeFile( + f.composeFile, + JSON.stringify({ services: { builder: { build: ".." } } }) + ); + const exec = await prepareNativeProjectInput({ + ...f, + selectedServices: ["builder"], + }); + expect(exec.ignoredRootDotenv).toBe(true); + expect(exec.shellEnvironment).toEqual({}); +}); + +test("source-free builds and selected-service exec retain their existing behavior", async () => { + const plain = await fixture({}, { services: { builder: { build: ".." } } }); + expect((await prepareNativeProjectInput(plain)).shellEnvironment).toEqual({}); + const f = await fixture( + {}, + { + services: { + app: { image: "alpine", env_file: "service.env" }, + builder: { build: ".." }, + }, + } + ); + await f.source("service.env", `TOKEN=${FILE}\n`); + const input = await prepareNativeProjectInput({ + ...f, + selectedServices: ["app"], + }); + expect(input.shellEnvironment).toEqual({ app: { TOKEN: FILE } }); + expect(input.normalizedComposeJson).not.toContain(FILE); +}); + +test("authored public runtime metadata literals survive while file/shell copies stay refused", async () => { + const f = await fixture({ + environment: { HACK_SERVICE_URL: "author-owned" }, + }); + const input = await prepareNativeProjectInput(f); + expect(publicService(input).environment.HACK_SERVICE_URL).toBe( + "author-owned" + ); + expect(input.shellEnvironment).toEqual({}); + await f.source(".env", ""); + await writeFile( + f.composeFile, + JSON.stringify({ + services: { + app: { + image: "alpine", + environment: { HACK_SERVICE_URL: "${SOURCE}" }, + }, + }, + }) + ); + await expect( + prepareNativeProjectInput({ ...f, env: { SOURCE: SHELL } }) + ).rejects.toThrow("reserved destination"); + await writeFile( + f.composeFile, + JSON.stringify({ + services: { + app: { image: "alpine", environment: { HACK_ENV_SECRET_KEY: FILE } }, + }, + }) + ); + const error: unknown = await prepareNativeProjectInput(f).catch( + (caught: unknown) => caught + ); + expect(String(error)).toContain("key material"); + expect(String(error)).not.toContain(FILE); +}); + +test("inactive service files are not read, but selected profiles require them", async () => { + const f = await fixture( + {}, + { + services: { + app: { image: "alpine" }, + worker: { + image: "alpine", + profiles: ["worker"], + env_file: "missing.env", + }, + }, + } + ); + const inactive = await prepareNativeProjectInput(f); + expect(inactive.shellEnvironment).toEqual({}); + expect(() => + selectActiveNativeProjectInput({ + input: inactive, + reviewedPlan: { + services: { app: { active: true }, worker: { active: true } }, + }, + }) + ).toThrow("env_file activation differs"); + expect(() => + selectActiveNativeProjectInput({ + input: inactive, + reviewedPlan: { + services: { app: { active: true }, worker: { active: false } }, + }, + }) + ).not.toThrow(); + await expect( + prepareNativeProjectInput({ ...f, profiles: ["worker"] }) + ).rejects.toThrow("required source is missing"); + await f.source("missing.env", `TOKEN=${FILE}\n`); + await chmod(join(f.projectDir, "missing.env"), 0o666); + expect((await prepareNativeProjectInput(f)).shellEnvironment).toEqual({}); +}); + +test("materialized dotenv identity aliases are refused before source reading", async () => { + const f = await fixture({ env_file: "alias.env" }); + await f.source(".env", `TOKEN=${DOTENV}\n`); + await f.source(".env.state.json", "{}\n"); + await link(join(f.projectDir, ".env"), join(f.projectDir, "alias.env")); + await expect(prepareNativeProjectInput(f)).rejects.toThrow("materialized"); +}); + +test("exact credential-adapter mounts remain eligible with HOME in the shell", async () => { + const aws = await fixture({ + volumes: ["${HOME}/.aws:/root/.aws:ro"], + environment: { AWS_REGION: null }, + }); + const awsInput = mergeNativeShellEnvironment( + await prepareNativeProjectInput({ ...aws, env: { HOME: SHELL } }) + ); + const adapted = await adaptNativeAwsEnvironment({ + input: awsInput, + profile: "fixture", + region: "us-east-1", + homeDirectory: aws.projectRoot, + exportSource: async () => ({ + credentials: { + Version: 1, + AccessKeyId: "synthetic-access", + SecretAccessKey: "synthetic-secret", + SessionToken: "synthetic-session", + Expiration: new Date(Date.now() + 3_600_000).toISOString(), + }, + region: "us-east-1", + }), + }); + expect( + JSON.parse(adapted.input.normalizedComposeJson).services.app + ).not.toHaveProperty("x-hack-unset-environment"); + const gcloud = await fixture({ + volumes: [ + "${HOME}/.config/gcloud/application_default_credentials.json:/tmp/adc.json:ro", + ], + environment: { GOOGLE_APPLICATION_CREDENTIALS: "/tmp/adc.json" }, + }); + const gcloudInput = await prepareNativeProjectInput({ + ...gcloud, + env: { HOME: SHELL }, + }); + const google = await adaptNativeGcloudEnvironment({ + input: gcloudInput, + mode: "file", + deliver: false, + homeDirectory: gcloud.projectRoot, + }); + expect( + JSON.parse(google.input.normalizedComposeJson).services.app.volumes ?? [] + ).toEqual([]); +}); diff --git a/tests/native-detached-up.test.ts b/tests/native-detached-up.test.ts index 48d62a581..458115614 100644 --- a/tests/native-detached-up.test.ts +++ b/tests/native-detached-up.test.ts @@ -3,6 +3,7 @@ import { mkdtemp, readFile, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { NATIVE_PROVIDER_PIN_MESSAGE } from "../src/backends/native-runtime-client.ts"; +import { NATIVE_ROOT_DOTENV_IGNORED_NOTICE } from "../src/lib/native-environment-notices.ts"; import { privateCliEnv } from "./helpers/private-cli-env.ts"; const fixture = join(import.meta.dir, "fixtures", "native-detached-owner.ts"); @@ -42,6 +43,7 @@ for (const mode of [ "ready", "shell-names", "shell-names-empty", + "shell-names-root-dotenv", "unsafe-failure", "fail", "provider-pin", @@ -82,13 +84,16 @@ for (const mode of [ ? "Forwarding shell variables: A_TOKEN, Z_TOKEN.\n" : mode === "shell-names-empty" ? "Forwarding shell variables: (none).\n" - : "" + : mode === "shell-names-root-dotenv" + ? `Forwarding shell variables: (none).\n${NATIVE_ROOT_DOTENV_IGNORED_NOTICE}\n` + : "" ); if ( [ "ready", "shell-names", "shell-names-empty", + "shell-names-root-dotenv", "unsafe-failure", ].includes(mode) ) { @@ -144,6 +149,7 @@ for (const mode of [ [ "ready", "shell-names", + "shell-names-root-dotenv", "shell-names-empty", "unsafe-failure", ].includes(mode) diff --git a/tests/native-env-file-source.test.ts b/tests/native-env-file-source.test.ts new file mode 100644 index 000000000..149729992 --- /dev/null +++ b/tests/native-env-file-source.test.ts @@ -0,0 +1,151 @@ +import { afterEach, expect, test } from "bun:test"; +import { + chmod, + link, + mkdir, + mkdtemp, + readFile, + realpath, + rm, + symlink, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { readNativeEnvFileSource } from "../src/backends/native-env-file-source.ts"; + +const roots: string[] = []; +const SENTINEL = "synthetic-private-file-source-canary"; +afterEach(async () => { + for (const root of roots.splice(0)) { + await rm(root, { recursive: true, force: true }); + } +}); +async function fixture() { + const projectRoot = await realpath( + await mkdtemp(join(tmpdir(), "native-env-source-")) + ); + roots.push(projectRoot); + await mkdir(join(projectRoot, ".hack"), { mode: 0o700 }); + return { + projectRoot, + composeFile: join(projectRoot, ".hack/docker-compose.yml"), + path: "service.env", + required: true, + service: "app", + }; +} +test("reads exact text privately, including empty files, without changing modes or bytes", async () => { + const opts = await fixture(); + const path = join(opts.projectRoot, ".hack/service.env"); + for (const mode of [0o644, 0o600, 0o400]) { + await writeFile(path, ` KEY=${SENTINEL} \n`); + await chmod(path, mode); + const result = await readNativeEnvFileSource(opts); + expect(result?.text).toBe(` KEY=${SENTINEL} \n`); + expect(await readFile(path, "utf8")).toBe(result?.text ?? ""); + } + await chmod(path, 0o600); + await writeFile(path, ""); + expect((await readNativeEnvFileSource(opts))?.text).toBe(""); +}); +test("only absent optional sources are ignored, including a missing parent", async () => { + const opts = await fixture(); + expect( + await readNativeEnvFileSource({ ...opts, required: false }) + ).toBeNull(); + expect( + await readNativeEnvFileSource({ + ...opts, + path: "missing/optional.env", + required: false, + }) + ).toBeNull(); + await expect(readNativeEnvFileSource(opts)).rejects.toThrow( + "required source is missing" + ); +}); +test.each([ + "symlink", + "dangling", + "hardlink", + "directory", + "unreadable", + "writable", + "oversize", + "invalid-utf8", +])("optional %s source refuses without reading or echoing its contents", async (kind) => { + const opts = await fixture(); + const path = join(opts.projectRoot, ".hack/service.env"); + const target = join(opts.projectRoot, "target"); + await writeFile(target, SENTINEL, { mode: 0o600 }); + switch (kind) { + case "symlink": + await symlink(target, path); + break; + case "dangling": + await symlink(join(opts.projectRoot, "absent"), path); + break; + case "hardlink": + await link(target, path); + break; + case "directory": + await mkdir(path); + break; + default: { + const content = + kind === "oversize" + ? Buffer.alloc(256 * 1024 + 1) + : Buffer.from(SENTINEL); + await writeFile( + path, + kind === "invalid-utf8" ? new Uint8Array([255]) : content, + { mode: 0o600 } + ); + if (kind === "unreadable") { + await chmod(path, 0); + } + if (kind === "writable") { + await chmod(path, 0o666); + } + } + } + const error: unknown = await readNativeEnvFileSource({ + ...opts, + required: false, + }).catch((failure: unknown) => failure); + expect(error).toBeInstanceOf(Error); + expect(String(error)).toContain("values omitted"); + expect(String(error)).toContain("service.env"); + expect(String(error)).not.toContain(SENTINEL); + expect(await readFile(target, "utf8")).toBe(SENTINEL); +}); +test("rejects a symlinked parent and a writable directory, even for an absent optional source", async () => { + const opts = await fixture(); + await mkdir(join(opts.projectRoot, "actual")); + await symlink( + join(opts.projectRoot, "actual"), + join(opts.projectRoot, ".hack/alias") + ); + await expect( + readNativeEnvFileSource({ ...opts, path: "alias/file", required: false }) + ).rejects.toThrow("source directories must be real"); + await chmod(join(opts.projectRoot, ".hack"), 0o777); + try { + await expect( + readNativeEnvFileSource({ ...opts, required: false }) + ).rejects.toThrow("source directories must be real"); + } finally { + await chmod(join(opts.projectRoot, ".hack"), 0o700); + } +}); +test.each([ + "../../external", + "${SOURCE}", + "~/file", + "bad\0path", +])("refuses unsupported paths before reading", async (path) => { + await expect( + readNativeEnvFileSource({ ...(await fixture()), path }) + ).rejects.toThrow("refused"); +}); diff --git a/tests/native-exec-environment.test.ts b/tests/native-exec-environment.test.ts index 17859d765..8a16a7170 100644 --- a/tests/native-exec-environment.test.ts +++ b/tests/native-exec-environment.test.ts @@ -70,6 +70,34 @@ test("fresh exec reads selected service overlay and ignores a changed default", }) ).toEqual({ VALUE: "base" }); }); + +test("fresh exec privately resolves selected-service env files with the caller context", async () => { + const opts = await fixture(); + const sentinel = "synthetic-fresh-exec-file-canary"; + await writeFile( + opts.composeFile, + JSON.stringify({ + services: { + web: { image: "fixture", env_file: "web.env" }, + other: { image: "fixture", env_file: "missing-other.env" }, + }, + }) + ); + await writeFile( + join(opts.scope.projectDir, "web.env"), + "TOKEN=${SOURCE:?}\nVALUE=overridden-file-value\n", + { mode: 0o600 } + ); + expect( + await prepareNativeExecEnvironment({ ...opts, env: { SOURCE: sentinel } }) + ).toEqual({ VALUE: "selected", TOKEN: sentinel }); + const error: unknown = await prepareNativeExecEnvironment({ + ...opts, + env: {}, + }).catch((caught: unknown) => caught); + expect(String(error)).toContain("Values omitted"); + expect(String(error)).not.toContain(sentinel); +}); test("fresh exec refuses missing selectors or deleted overlays before AWS export", async () => { const opts = await fixture(); let calls = 0; diff --git a/tests/native-exec-stale-env.test.ts b/tests/native-exec-stale-env.test.ts index 070fe60f1..a3ed4b4fb 100644 --- a/tests/native-exec-stale-env.test.ts +++ b/tests/native-exec-stale-env.test.ts @@ -318,6 +318,48 @@ function preparedInput( } describe("native image exec stale env check", () => { + test("managed stale checks do not acquire or evaluate ephemeral env files", async () => { + const f = await project(); + const composeFile = join(f.projectDir, "docker-compose.yml"); + await writeFile( + composeFile, + JSON.stringify({ + services: { + web: { + image: "fixture", + env_file: "missing.env", + environment: { FROM_SHELL: "${REQUIRES_SHELL:?}" }, + }, + }, + }) + ); + await writeFile( + join(f.projectDir, "hack.env.default.yaml"), + JSON.stringify({ + version: 1, + environment: "default", + secretsprovider: "project_key", + values: { web: { [KEY]: SECRET_AFTER } }, + }) + ); + await recordImage({ + projectDir: f.projectDir, + env: { web: { [KEY]: SECRET_BEFORE } }, + }); + const warning = await resolveNativeImageExecStaleEnvWarning({ + scope: { + projectRoot: f.root, + projectDir: f.projectDir, + nativeHome: f.root, + branch: null, + }, + service: "web", + composeFile, + loadRun: async () => imageMapping(), + }); + expect(warning).toContain("hack restart"); + expectValueFree(warning ?? ""); + }); async function resolve(opts: { readonly projectDir: string; readonly root: string; diff --git a/tests/native-preflight-cli.test.ts b/tests/native-preflight-cli.test.ts index 44dd53fff..c10751415 100644 --- a/tests/native-preflight-cli.test.ts +++ b/tests/native-preflight-cli.test.ts @@ -49,7 +49,7 @@ async function fixture(compose: Record) { const projectDir = join(project, ".hack"); const nativeHome = join(root, "native-home"); const stub = join(root, "stub"); - await mkdir(projectDir, { recursive: true }); + await mkdir(projectDir, { recursive: true, mode: 0o700 }); await mkdir(nativeHome, { mode: 0o700 }); await mkdir(stub); await writeFile( @@ -215,6 +215,214 @@ function codes(report: NativePreflightReport, gate: "cli" | "runtime") { const GRANT = { HACK_NATIVE_SHARED_SOURCE: "1" }; +test("preflight and start share env-file admission and never publish file values", async () => { + const f = await fixture({ + services: { + web: { + image: IMAGE, + env_file: "service.env", + environment: { COPY: "${SOURCE}" }, + }, + }, + }); + await chmod(f.project, 0o700); + await writeFile(join(f.projectDir, "service.env"), `TOKEN=${PRIVATE}\n`, { + mode: 0o600, + }); + await writeFile(join(f.projectDir, ".env"), `SOURCE=${PRIVATE}\n`, { + mode: 0o600, + }); + await f.review([]); + const env = { ...f.native, ...GRANT }; + const observed = await preflight(f, env); + expect(observed.report.ready).toBe(true); + const normalized = await readFile( + join(f.root, "stub/normalized.json"), + "utf8" + ); + expect(JSON.parse(normalized).services.web.environment).toEqual({ + TOKEN: null, + COPY: null, + }); + expect(normalized).not.toContain(PRIVATE); + expect(observed.output).not.toContain(PRIVATE); + expect((await f.argv()).join("\n")).not.toContain(PRIVATE); + for (const unsafe of ["missing", "writable", "public-reference"]) { + if (unsafe === "missing") { + await rm(join(f.projectDir, "service.env")); + } + if (unsafe === "writable") { + await writeFile(join(f.projectDir, "service.env"), `TOKEN=${PRIVATE}\n`, { + mode: 0o666, + }); + await chmod(join(f.projectDir, "service.env"), 0o666); + } + if (unsafe === "public-reference") { + await chmod(join(f.projectDir, "service.env"), 0o600); + await writeFile( + join(f.projectDir, "docker-compose.yml"), + JSON.stringify({ + services: { + web: { image: IMAGE, command: ["echo", "${SOURCE:-fallback}"] }, + }, + }) + ); + } + const report = await preflight(f, env); + expect(codes(report.report, "cli")).toContain("environment_file_input:web"); + const before = await f.calls(); + const up = await cli(f, ["up", "--json"], env); + expect(up.code).toBe(1); + expect(up.output).not.toContain(PRIVATE); + expect(report.output).not.toContain(PRIVATE); + expect((await f.calls()).slice(before.length)).toEqual([]); + } +}, 30_000); + +test("preflight and static start refuse oversized file delivery before pool effects", async () => { + const f = await fixture({ + services: { web: { image: IMAGE, env_file: "service.env" } }, + }); + await chmod(f.project, 0o700); + await writeFile( + join(f.projectDir, "service.env"), + `TOKEN=${"private-quota-canary".repeat(2500)}\n`, + { mode: 0o600 } + ); + await f.review([], { services: { web: { active: true } } }); + const env = { ...f.native, ...GRANT }; + const report = await preflight(f, env); + expect(codes(report.report, "cli")).toContain("environment_delivery"); + expect(report.output).not.toContain("private-quota-canary"); + const before = await f.calls(); + const up = await cli(f, ["up"], env); + expect(up.code).toBe(1); + expect(up.output).toContain("32 KiB"); + expect(up.output).not.toContain("private-quota-canary"); + expect( + (await f.calls()) + .slice(before.length) + .every((call) => call === "project plan") + ).toBe(true); +}); + +test( + "preflight and start refuse private sources in native build contexts before effects", + async () => { + const f = await fixture({ + services: { + web: { image: IMAGE, env_file: "service.env" }, + builder: { build: ".." }, + }, + }); + await chmod(f.project, 0o700); + await writeFile(join(f.projectDir, "service.env"), `TOKEN=${PRIVATE}\n`, { + mode: 0o600, + }); + await f.review([]); + const env = { ...f.native, ...GRANT, HACK_NATIVE_BUILDS: "1" }; + const report = await preflight(f, env); + expect(report.code).toBe(1); + expect(report.report.ready).toBe(false); + expect(codes(report.report, "cli")).toContain( + "environment_file_input:builder" + ); + expect(report.output).toContain("disjoint build inputs"); + const before = await f.calls(); + const up = await cli(f, ["up"], env); + expect(up.code).toBe(1); + expect(up.output).toContain("overlap an environment source"); + expect((await f.calls()).slice(before.length)).toEqual([]); + expect(report.output).not.toContain(PRIVATE); + expect(up.output).not.toContain(PRIVATE); + expect((await f.argv()).join("\n")).not.toContain(PRIVATE); + const detached = await cli(f, ["up", "--json"], env); + expect(detached.code).toBe(1); + expect(detached.output).not.toContain(PRIVATE); + expect((await f.calls()).slice(before.length)).toEqual([]); + }, + SPAWN_TIMEOUT_MS +); + +test("preflight and start refuse ignored root dotenv in a native build before effects", async () => { + const f = await fixture({ services: { builder: { build: ".." } } }); + await writeFile(join(f.project, ".env"), `TOKEN=${PRIVATE}\n`, { + mode: 0o000, + }); + await f.review([]); + const env = { ...f.native, ...GRANT, HACK_NATIVE_BUILDS: "1" }; + const report = await preflight(f, env); + expect(report.code).toBe(1); + expect(report.report.ready).toBe(false); + expect(codes(report.report, "cli")).toContain( + "environment_file_input:builder" + ); + expect( + report.report.blockers.find( + (entry) => entry.code === "environment_file_input" + )?.message + ).toContain('source "../.env"'); + expect(report.output).not.toContain(PRIVATE); + const before = await f.calls(); + for (const args of [["up"], ["up", "--json"]]) { + const started = await cli(f, args, env); + expect(started.code).toBe(1); + if (args.includes("--json")) { + expect(JSON.parse(started.stdout)).toMatchObject({ + ok: false, + error: { code: "E_STARTUP_INCOMPLETE" }, + }); + } else { + expect(started.output).toContain("overlap an environment source"); + } + expect(started.output).not.toContain(PRIVATE); + expect((await f.calls()).slice(before.length)).toEqual([]); + } +}); + +test("preflight requires the same decryption key as start and closes key errors", async () => { + const f = await fixture({ services: { web: { image: IMAGE } } }); + await writeFile( + join(f.projectDir, "hack.env.default.yaml"), + JSON.stringify({ + version: 1, + environment: "default", + secretsprovider: "project_key", + values: { global: { TOKEN: { secure: PRIVATE } } }, + }) + ); + await f.review([]); + const report = await preflight(f, { ...f.native, ...GRANT }); + expect(codes(report.report, "cli")).toEqual(["managed_env_unreadable"]); + expect(report.output).not.toContain(PRIVATE); + const before = await f.calls(); + const up = await cli(f, ["up"], { ...f.native, ...GRANT }); + expect(up.code).toBe(1); + expect(up.output).toContain("E_CONFIG_INVALID"); + expect(up.output).not.toContain(PRIVATE); + expect((await f.calls()).slice(before.length)).toEqual([]); +}); + +test("old runtimes cannot silently ignore explicit environment unsets", async () => { + const f = await fixture({ + services: { + web: { image: IMAGE, environment: { ABSENT_FIXTURE_NAME: null } }, + }, + }); + await f.review([]); + const report = await preflight(f, { ...f.native, ...GRANT }); + expect(report.report.runtime_review.code).toBe( + "native_environment_unset_unavailable" + ); + await f.review([], { + services: { + web: { active: true, environment_unsets: ["ABSENT_FIXTURE_NAME"] }, + }, + }); + const supported = await preflight(f, { ...f.native, ...GRANT }); + expect(supported.report.ready).toBe(true); +}); + /** Replies that carry native `up` past its source checks to service admission. */ async function scriptSourceShare(f: Fixture): Promise { await f.reply("runtime-check-project-share.out", { @@ -285,9 +493,7 @@ test( expect(report.runtime_review).toEqual({ status: "completed" }); expect(codes(report, "cli")).toEqual([ "build_requires_opt_in:web", - "dotenv_without_managed_env", - "env_file:api", - "env_file:jobs", + "environment_file_input:web", "external_network", "extra_hosts_without_dependencies:web", "invalid_restart:jobs", @@ -388,7 +594,7 @@ test( expect(code).toBe(1); expect(report.runtime_review).toEqual({ status: "skipped" }); expect(report.ready).toBe(false); - expect(codes(report, "cli")).toEqual(["env_file:web"]); + expect(codes(report, "cli")).toEqual(["environment_file_input:web"]); expect(await f.calls()).toEqual([]); // No blocker is still not "ready" when half the checks did not run. const clean = await fixture({ services: { web: { image: IMAGE } } }); @@ -418,6 +624,34 @@ test( SPAWN_TIMEOUT_MS ); +test( + "ambiguous env-file aliases are a preflight blocker in JSON and text", + async () => { + const f = await fixture({ services: { web: { image: IMAGE } } }); + await f.reviewStops("source_case_collision"); + const env = { ...f.native, ...GRANT }; + const result = await preflight(f, env); + expect(result.code).toBe(1); + expect(result.report.ready).toBe(false); + expect(result.report.runtime_review).toEqual({ + status: "stopped", + code: "source_case_collision", + }); + expect(result.report.blockers).toHaveLength(1); + expect(result.report.blockers[0]).toMatchObject({ + gate: "runtime", + code: "source_case_collision", + }); + expect(result.report.blockers[0]?.fix).toContain("Rename"); + const text = await cli(f, ["doctor", "--native-preflight"], env); + expect(text.code).toBe(1); + expect(text.output).toContain("Unicode normalization and case folding"); + expect(text.output).toContain("Rename"); + expect(text.output).not.toContain(PRIVATE); + }, + SPAWN_TIMEOUT_MS +); + test( "a review that stops early is reported as a blocker, and one that fails is reported as unavailable", async () => { @@ -486,7 +720,7 @@ test( "preflight agrees with native up: each CLI blocker is a refusal, and a clean project passes the gate", async () => { for (const [service, extra, expected] of [ - [{ image: IMAGE, env_file: ".env" }, {}, "env_file:web"], + [{ image: IMAGE, env_file: ".env" }, {}, "environment_file_input:web"], [ { image: IMAGE, logging: { driver: "syslog" } }, {}, @@ -556,7 +790,7 @@ test( expect(up.output).toContain( "Run `hack doctor --native-preflight` to list every blocker with its fix." ); - expect(await f.calls()).toEqual(["runtime check-project-share"]); + expect(await f.calls()).toEqual([]); }, SPAWN_TIMEOUT_MS ); @@ -1113,7 +1347,7 @@ test("preflight cannot report ready when a runtime omits or corrupts start-admis } }); -test("preflight supplies names-only private-delivery context without decrypting managed env or reading shell values", async () => { +test("preflight uses the same managed resolution and shell context but sends only names to review", async () => { const f = await fixture({ services: { managed: { image: IMAGE, restart: "always" }, @@ -1125,7 +1359,7 @@ test("preflight supplies names-only private-delivery context without decrypting literal: { image: IMAGE, environment: { TOKEN: PRIVATE } }, excluded: { image: IMAGE, - environment: ["AWS_SECRET_ACCESS_KEY", "COMPOSE_PROJECT_NAME"], + environment: { AWS_REGION: "explicit-literal" }, }, }, }); @@ -1135,7 +1369,7 @@ test("preflight supplies names-only private-delivery context without decrypting version: 1, environment: "default", secretsprovider: "project_key", - values: { managed: { TOKEN: { secure: "invalid-ciphertext-sentinel" } } }, + values: { managed: { TOKEN: PRIVATE } }, }) ); await f.review( @@ -1228,6 +1462,46 @@ test("preflight reports one project-wide managed env metadata error", async () = ).toHaveLength(1); }); +test("preflight stays usable with a missing managed key and reports its recovery hint", async () => { + const f = await fixture({ services: { web: { image: IMAGE } } }); + await writeFile( + join(f.projectDir, "hack.env.default.yaml"), + JSON.stringify({ + version: 1, + environment: "default", + secretsprovider: "project_key", + values: { global: { TOKEN: { secure: "synthetic-ciphertext-canary" } } }, + }) + ); + await f.review([]); + const result = await preflight(f, f.native); + expect(result.code).toBe(1); + const blockers = result.report.blockers.filter( + (entry) => entry.code === "managed_env_unreadable" + ); + expect(blockers).toHaveLength(1); + expect(blockers[0]?.message).toContain("project env key is missing"); + expect(blockers[0]?.message).toContain("Copy .hack.secret.key"); + expect(result.output).not.toContain("synthetic-ciphertext-canary"); +}); + +test("preflight reports ignored root dotenv once without its contents", async () => { + const f = await fixture({ services: { web: { image: IMAGE } } }); + await writeFile(join(f.project, ".env"), `ROOT_ONLY=${PRIVATE}\n`, { + mode: 0o000, + }); + await f.review([]); + const result = await preflight(f, { ...f.native, ...GRANT }); + expect(result.report.blockers).toEqual([]); + expect( + result.report.warnings.filter( + (entry) => entry.code === "root_dotenv_ignored" + ) + ).toHaveLength(1); + expect(result.output.match(/Project-root \.env is ignored/g)).toHaveLength(1); + expect(result.output).not.toContain(PRIVATE); +}); + for (const syntax of ["JSON null", "commented YAML"] as const) { for (const managed of [false, true]) { test(`empty service environments: ${syntax}, managed=${managed}`, async () => { diff --git a/tests/native-project-input.test.ts b/tests/native-project-input.test.ts index 52647b701..4975b928d 100644 --- a/tests/native-project-input.test.ts +++ b/tests/native-project-input.test.ts @@ -37,7 +37,7 @@ async function fixture(compose: unknown) { ); roots.push(projectRoot); const projectDir = join(projectRoot, ".hack"); - await mkdir(projectDir); + await mkdir(projectDir, { mode: 0o700 }); const composeFile = join(projectDir, "docker-compose.yml"); await writeFile(composeFile, JSON.stringify(compose)); return { projectRoot, projectDir, composeFile }; @@ -163,12 +163,12 @@ test("fourteen services and 110 managed variables each preserve overlay selectio expect(result.normalizedComposeJson).not.toContain("synthetic-"); expect(result.normalizedComposeJson).not.toContain("overlay-value"); }); -test("env_file and malformed modern input refuse without diagnostic contents", async () => { +test("missing env_file and malformed modern input refuse without diagnostic contents", async () => { const opts = await fixture({ services: { api: { env_file: "private-source", image: "bun:1" } }, }); await env(opts.projectDir, "default", { - global: { TOKEN: { secure: "invalid-secret-canary" } }, + global: { TOKEN: "managed-synthetic" }, }); await expect(prepareNativeProjectInput(opts)).rejects.toMatchObject({ code: "E_NATIVE_PROJECT_UNSUPPORTED", @@ -187,7 +187,7 @@ test("env_file and malformed modern input refuse without diagnostic contents", a expect(String(error)).toContain("values omitted"); } }); -test("legacy dotenv refuses and unmanaged public projects preserve original fields", async () => { +test("unreferenced dotenv does not inject values and public projects preserve original fields", async () => { const original = { services: { api: { image: "bun:1", environment: { STATIC: "tracked" } } }, }; @@ -195,11 +195,12 @@ test("legacy dotenv refuses and unmanaged public projects preserve original fiel expect( JSON.parse((await prepareNativeProjectInput(opts)).normalizedComposeJson) ).toEqual(original); - await writeFile(join(opts.projectDir, ".env"), "PRIVATE=synthetic\n"); - await expect(prepareNativeProjectInput(opts)).rejects.toMatchObject({ - code: "E_NATIVE_PROJECT_UNSUPPORTED", - message: expect.stringContaining("values omitted"), + await writeFile(join(opts.projectDir, ".env"), "PRIVATE=synthetic\n", { + mode: 0o644, }); + const withDotenv = await prepareNativeProjectInput(opts); + expect(JSON.parse(withDotenv.normalizedComposeJson)).toEqual(original); + expect(withDotenv.shellEnvironment).toEqual({}); }); test("the effective env is the requested or default env even without managed env files", async () => { @@ -448,7 +449,7 @@ function environments(result: { readonly normalizedComposeJson: string }) { * writing the value into the public document: the entry stays a reference and * the value is kept apart, for the private delivery. */ -test("plain shell references and bare names are resolved privately; everything else is left as written", async () => { +test("shell expressions are private, unresolved bare names unset, and public literals stay authored", async () => { const PRIVATE = "synthetic-shell-value"; const opts = await fixture({ services: { @@ -460,22 +461,11 @@ test("plain shell references and bare names are resolved privately; everything e UNSET: reference("SHELL_UNSET"), PASSED: null, NOT_PASSED: null, - // Forms that start today are not touched, whatever the shell holds. WITH_DEFAULT: reference("SHELL_SET", "fallback"), LITERAL: "plain", PARTIAL: `http://${reference("SHELL_SET")}:80`, ESCAPED: `$${reference("SHELL_SET")}`, - // Hack's own variables: the env decryption key must never be copyable. - LEAKED_KEY: reference("HACK_ENV_SECRET_KEY"), - HACK_ENV_SECRET_KEY: null, - HACK_GATEWAY_TOKEN: reference("HACK_GATEWAY_TOKEN"), - // Names the runtime supplies or the credential adapters own. PROJECT: reference("COMPOSE_PROJECT_NAME"), - COMPOSE_PROJECT_NAME: reference("SHELL_SET"), - AWS_SECRET_ACCESS_KEY: reference("AWS_SECRET_ACCESS_KEY"), - COPIED_KEY: reference("AWS_ACCESS_KEY_ID"), - GOOGLE_APPLICATION_CREDENTIALS: null, - // Valid names that are also Object.prototype properties. constructor: reference("constructor"), INHERITED: reference("toString"), }, @@ -491,7 +481,6 @@ test("plain shell references and bare names are resolved privately; everything e }, plain: { image: "bun:1" }, literal: { image: "bun:1", environment: ["A=1", "B=two"] }, - // Every entry commented out is YAML null; it must stay accepted. empty: { image: "bun:1", environment: null }, }, }); @@ -501,105 +490,86 @@ test("plain shell references and bare names are resolved privately; everything e SHELL_SET: PRIVATE, PASSED: `${PRIVATE}-passed`, COMPOSE_PROJECT_NAME: "other-instance", - AWS_SECRET_ACCESS_KEY: "long-lived-credential", - AWS_ACCESS_KEY_ID: "long-lived-id", - GOOGLE_APPLICATION_CREDENTIALS: "credential-path", - HACK_ENV_SECRET_KEY: "project-env-decryption-key", - HACK_GATEWAY_TOKEN: "gateway-token", - // Never read: no entry references it. UNRELATED: "unrelated-private", }, }); - // Shell values are kept apart from what Hack manages, and never fingerprinted. expect(result.managedEnvironment).toEqual({}); - expect(JSON.parse(JSON.stringify(result.shellEnvironment))).toEqual({ + expect(result.shellEnvironment).toEqual({ api: { WHOLE: PRIVATE, UNBRACED: PRIVATE, - // Compose substitutes an empty string for an unset variable. UNSET: "", PASSED: `${PRIVATE}-passed`, - // Delivered empty rather than left out, so the reviewed document is the - // same from a shell that sets it and one that does not. - NOT_PASSED: "", - // Unset in this shell, and not taken from Object.prototype. + WITH_DEFAULT: PRIVATE, + PARTIAL: `http://${PRIVATE}:80`, constructor: "", INHERITED: "", }, - worker: { PASSED: `${PRIVATE}-passed`, NOT_PASSED: "", WHOLE: PRIVATE }, - }); - expect(result.normalizedComposeJson).not.toContain(PRIVATE); - // Not read at all: neither public nor private. - for (const hidden of [ - "unrelated-private", - "other-instance", - "long-lived-credential", - "long-lived-id", - "credential-path", - "project-env-decryption-key", - "gateway-token", - ]) { - expect(result.normalizedComposeJson).not.toContain(hidden); - expect(JSON.stringify(result.shellEnvironment)).not.toContain(hidden); - } + worker: { PASSED: `${PRIVATE}-passed`, WHOLE: PRIVATE }, + }); const services = environments(result); - expect(JSON.parse(JSON.stringify(services.api?.environment))).toEqual({ + expect(services.api?.environment).toEqual({ WHOLE: null, UNBRACED: null, UNSET: null, PASSED: null, - NOT_PASSED: null, - constructor: null, - INHERITED: null, - WITH_DEFAULT: reference("SHELL_SET", "fallback"), + WITH_DEFAULT: null, LITERAL: "plain", - PARTIAL: `http://${reference("SHELL_SET")}:80`, - ESCAPED: `$${reference("SHELL_SET")}`, + PARTIAL: null, + ESCAPED: "$${SHELL_SET}", PROJECT: reference("COMPOSE_PROJECT_NAME"), - COMPOSE_PROJECT_NAME: reference("SHELL_SET"), - AWS_SECRET_ACCESS_KEY: reference("AWS_SECRET_ACCESS_KEY"), - COPIED_KEY: reference("AWS_ACCESS_KEY_ID"), - GOOGLE_APPLICATION_CREDENTIALS: null, - LEAKED_KEY: reference("HACK_ENV_SECRET_KEY"), - HACK_ENV_SECRET_KEY: null, - HACK_GATEWAY_TOKEN: reference("HACK_GATEWAY_TOKEN"), + constructor: null, + INHERITED: null, }); + expect( + JSON.parse(result.normalizedComposeJson).services.api[ + "x-hack-unset-environment" + ] + ).toEqual(["NOT_PASSED"]); expect(services.worker?.environment).toEqual({ PASSED: null, - NOT_PASSED: null, WHOLE: null, KEEP: "1", }); expect(services.plain?.environment).toBeUndefined(); - // No reference, no rewrite: the list form is exactly what was authored. expect(services.literal?.environment).toEqual(["A=1", "B=two"]); expect(services.empty?.environment).toBeNull(); - // Names only, never values. - expect(result.environmentNotices).toEqual([ - { - service: "api", - line: "Service api: UNSET is empty because SHELL_UNSET is not set in your shell.", - }, - { - service: "api", - line: "Service api: NOT_PASSED is empty because it is not set in your shell.", - }, - { - service: "api", - line: "Service api: constructor is empty because it is not set in your shell.", - }, - { - service: "api", - line: "Service api: INHERITED is empty because toString is not set in your shell.", - }, - { - service: "worker", - line: "Service worker: NOT_PASSED is empty because it is not set in your shell.", - }, - ]); + expect(result.environmentNotices).toHaveLength(5); + expect(JSON.stringify(result.environmentNotices)).toContain( + "key NOT_PASSED: an unresolved input is unset" + ); + for (const hidden of [PRIVATE, "other-instance", "unrelated-private"]) { + expect(result.normalizedComposeJson).not.toContain(hidden); + expect(JSON.stringify(result.environmentNotices)).not.toContain(hidden); + } +}); + +test("Hack key material stays runtime-owned without files and refuses with files, never reading the caller", async () => { + for (const key of ["HACK_ENV_SECRET_KEY", "HACK_GATEWAY_TOKEN"]) { + const opts = await fixture({ + services: { + api: { image: "bun:1", environment: { COPY: reference(key) } }, + }, + }); + const caller = Object.defineProperty({}, key, { + enumerable: true, + get() { + throw new Error("private getter must never be invoked"); + }, + }); + const input = await prepareNativeProjectInput({ ...opts, env: caller }); + expect(environments(input).api?.environment).toEqual({ + COPY: reference(key), + }); + expect(input.shellEnvironment).toEqual({}); + await writeFile(join(opts.projectDir, ".env"), "", { mode: 0o600 }); + await expect( + prepareNativeProjectInput({ ...opts, env: caller }) + ).rejects.toThrow("protected variable"); + } }); -test("the reviewed document does not depend on the shell, and nothing is read unless the caller passes an environment", async () => { +test("unset names affect only names in the document; ambient values require an explicit caller environment", async () => { const compose = { services: { api: { @@ -616,8 +586,13 @@ test("the reviewed document does not depend on the shell, and nothing is read un ...(await fixture(compose)), env: {}, }); - // Same public document from a shell that sets the variables and one that does not. - expect(set.normalizedComposeJson).toBe(unset.normalizedComposeJson); + // Bare unset is distinct from an empty string and only the name enters identity. + expect(set.normalizedComposeJson).not.toBe(unset.normalizedComposeJson); + expect( + JSON.parse(unset.normalizedComposeJson).services.api[ + "x-hack-unset-environment" + ] + ).toEqual(["PASSED"]); // Without `env`, as exec and the stale-env check call it, the process // environment is not consulted. const previous = process.env.SHELL_TOKEN; @@ -625,9 +600,9 @@ test("the reviewed document does not depend on the shell, and nothing is read un try { const ambient = await prepareNativeProjectInput(await fixture(compose)); expect(ambient.shellEnvironment).toEqual({ - api: { TOKEN: "", PASSED: "" }, + api: { TOKEN: "" }, }); - expect(ambient.normalizedComposeJson).toBe(set.normalizedComposeJson); + expect(ambient.normalizedComposeJson).toBe(unset.normalizedComposeJson); } finally { restoreEnv("SHELL_TOKEN", previous); } @@ -662,12 +637,11 @@ test("a key Hack manages is delivered from managed env, never from the shell", a TOKEN: "managed-private", FROM_MANAGED: "managed-too", }); - // Nothing of this service is taken from the shell. - expect(result.shellEnvironment).toEqual({}); - // A reference to a managed variable is left for the runtime to refuse. + // A global managed value participates in interpolation above shell and dotenv. + expect(result.shellEnvironment).toEqual({ api: { COPY: "managed-private" } }); expect( (environments(result).api?.environment as Record).COPY - ).toBe(reference("TOKEN")); + ).toBeNull(); for (const hidden of ["shell-private", "shell-loses"]) { expect(result.normalizedComposeJson).not.toContain(hidden); } @@ -713,7 +687,6 @@ test("merged shell values join the private delivery, and a delivery the runtime // JSON escaping counts: the runtime receives the escaped form. [1, '"'.repeat(17 * 1024), "are over 32 KiB together"], [257, "#", "more than 256 variables"], - [1, "bad\0value", "contains a NUL byte"], ] as const) { const input = await prepared(count, value); const delivery = mergeNativeShellEnvironment(input); diff --git a/tests/native-project-start.test.ts b/tests/native-project-start.test.ts index 5f35a1520..95377dfa0 100644 --- a/tests/native-project-start.test.ts +++ b/tests/native-project-start.test.ts @@ -4077,21 +4077,22 @@ test("pending retained restart refuses fresh startup before effects", async () = expect(events).toEqual([]); }); -test("startup prepares mapping storage before input and plan review", async () => { +test("startup admits private input before storage and prepares storage before plan effects", async () => { const { opts, events } = await fixture(false); const prepare = opts.dependencies.prepare!; const review = opts.dependencies.review!; opts.dependencies.prepareStorage = async (scope) => { expect(scope).toEqual(opts.scope); + expect(events).toContain("input"); events.push("storage"); }; opts.dependencies.prepare = async (request) => { - expect(events[0]).toBe("storage"); + expect(events).not.toContain("storage"); events.push("input"); return await prepare(request); }; opts.dependencies.review = async (request) => { - expect(events.indexOf("storage")).toBeLessThan(events.indexOf("input")); + expect(events.indexOf("input")).toBeLessThan(events.indexOf("storage")); return await review(request); }; expect(await startNativeProject(opts)).toBe(0); diff --git a/tests/native-refusal-cli.test.ts b/tests/native-refusal-cli.test.ts index 08b0351f2..09550284d 100644 --- a/tests/native-refusal-cli.test.ts +++ b/tests/native-refusal-cli.test.ts @@ -501,9 +501,9 @@ test( "E_NATIVE_PROJECT_UNSUPPORTED: Selected (non-ignored) source is 140 MiB in 3 files; the native source limit is 128 MiB (default). Native up refuses source over the limit; no runtime, image or build work was started. Largest directories: assets (140 MiB), assets/video (140 MiB). Largest files: assets/video/intro.bin (140 MiB). Exclude paths the containers do not need in the project's .gitignore or .ignore file (for example a line such as `assets/video/`) or raise native.sourceLimitMiB in .hack/hack.config.json (maximum 512)." ); expect(await f.calls()).toEqual([ - "runtime check-project-share", "project plan", "project plan", + "runtime check-project-share", "project source-size", ]); }, @@ -527,9 +527,9 @@ test( "E_CONFIG_INVALID: Native host dependencies refused: the selection must be {version: 1, dependencies: [1..128 entries]} with no other fields; listener values omitted." ); expect(await f.calls()).toEqual([ - "runtime check-project-share", "project plan", "project plan", + "runtime check-project-share", ]); }, SPAWN_TIMEOUT_MS @@ -546,10 +546,7 @@ test( result, "Native project input changed before review; prepare it again." ); - expect(await f.calls()).toEqual([ - "runtime check-project-share", - "project plan", - ]); + expect(await f.calls()).toEqual(["project plan"]); }, SPAWN_TIMEOUT_MS ); @@ -912,7 +909,7 @@ test( result, "E_CONFIG_INVALID: Native workspace cache layout conflicts with declared mounts or is invalid; values omitted." ); - expect(await f.calls()).toEqual(["runtime check-project-share"]); + expect(await f.calls()).toEqual([]); }, SPAWN_TIMEOUT_MS ); diff --git a/tests/native-shell-environment-cli.test.ts b/tests/native-shell-environment-cli.test.ts index a838e9bdd..4538a4415 100644 --- a/tests/native-shell-environment-cli.test.ts +++ b/tests/native-shell-environment-cli.test.ts @@ -16,7 +16,6 @@ import { prepareNativeProjectInput, } from "../src/backends/native-project-input.ts"; import { nativeEnvironmentDelivery } from "../src/backends/native-project-start.ts"; -import { resolveShellEnvironment } from "../src/backends/native-shell-environment.ts"; import { nativeSourceSizeReceipt } from "./helpers/native-source-size.ts"; import { privateCliEnv } from "./helpers/private-cli-env.ts"; @@ -145,6 +144,9 @@ for (const mode of ["foreground", "detached", "json"] as const) { `native ${mode} up reports forwarded shell names and prints no value`, async () => { const f = await fixture(); + await writeFile(join(f.project, ".env"), `ROOT_ONLY=${PRIVATE}\n`, { + mode: 0o600, + }); const child = Bun.spawn( [ process.execPath, @@ -190,7 +192,7 @@ for (const mode of ["foreground", "detached", "json"] as const) { if (!detached) { expect( output.split( - "Service web: REGION is empty because FIXTURE_REGION is not set in your shell.\n" + 'Service web source ".hack/docker-compose.yml" key REGION: an unresolved input is empty.\n' ) ).toHaveLength(2); } @@ -198,6 +200,7 @@ for (const mode of ["foreground", "detached", "json"] as const) { stderr.split("Forwarding shell variables: FIXTURE_API_KEY, PASSED.\n") ).toHaveLength(2); expect(output).not.toContain("UNREFERENCED_PRIVATE"); + expect(output.match(/Project-root \.env is ignored/g)).toHaveLength(1); // No value reaches the terminal. expect(output).not.toContain(PRIVATE); // The project passed the CLI's admission: the stub runtime was asked to start. @@ -254,28 +257,43 @@ test("shell values travel only in the private stdin payload, never in the public }); }); -test("a variable named like an Object.prototype property is an ordinary variable", () => { - const resolution = resolveShellEnvironment({ - service: "web", - environment: Object.fromEntries([ - ["__proto__", reference("FIXTURE_SET")], - ["constructor", null], - ["PLAIN", reference("toString")], - ]), +test("a variable named like an Object.prototype property is an ordinary variable", async () => { + const f = await fixture(); + await writeFile( + f.composeFile, + JSON.stringify({ + services: { + web: { + image: IMAGE, + environment: Object.fromEntries([ + ["__proto__", reference("FIXTURE_SET")], + ["constructor", null], + ["PLAIN", reference("toString")], + ]), + }, + }, + }) + ); + const resolution = await prepareNativeProjectInput({ + ...f, + projectRoot: f.project, env: { FIXTURE_SET: PRIVATE }, - managed: new Set(), }); - expect(Object.entries(resolution?.delivered ?? {})).toEqual([ + expect(Object.entries(resolution.shellEnvironment?.web ?? {})).toEqual([ ["__proto__", PRIVATE], // Not set in this shell; never the inherited function. - ["constructor", ""], ["PLAIN", ""], ]); - expect(Object.keys(resolution?.environment ?? {})).toEqual([ - "__proto__", - "constructor", - "PLAIN", - ]); + expect( + Object.keys( + JSON.parse(resolution.normalizedComposeJson).services.web.environment + ) + ).toEqual(["__proto__", "PLAIN"]); + expect( + JSON.parse(resolution.normalizedComposeJson).services.web[ + "x-hack-unset-environment" + ] + ).toEqual(["constructor"]); }); async function regularFiles(directory: string): Promise { @@ -305,7 +323,7 @@ async function regularFiles(directory: string): Promise { return result; } -test("only source names referenced by Compose are read and reported, never the ambient environment", () => { +test("only source names referenced by Compose are read and reported, never the ambient environment", async () => { const caller = Object.defineProperties( { REFERENCED: PRIVATE }, { @@ -317,16 +335,30 @@ test("only source names referenced by Compose are read and reported, never the a }, } ); - const resolved = resolveShellEnvironment({ - service: "web", - environment: { - API_KEY: reference("REFERENCED"), - COPY: reference("REFERENCED"), - }, + const f = await fixture(); + await writeFile( + f.composeFile, + JSON.stringify({ + services: { + web: { + image: IMAGE, + environment: { + API_KEY: reference("REFERENCED"), + COPY: reference("REFERENCED"), + }, + }, + }, + }) + ); + const resolved = await prepareNativeProjectInput({ + ...f, + projectRoot: f.project, env: caller, - managed: new Set(), }); - expect(resolved?.delivered).toEqual({ API_KEY: PRIVATE, COPY: PRIVATE }); - expect(resolved?.forwardedNames).toEqual(["REFERENCED"]); - expect(JSON.stringify(resolved?.environment)).not.toContain(PRIVATE); + expect(resolved.shellEnvironment?.web).toEqual({ + API_KEY: PRIVATE, + COPY: PRIVATE, + }); + expect(resolved.forwardedShellVariables?.web).toEqual(["REFERENCED"]); + expect(resolved.normalizedComposeJson).not.toContain(PRIVATE); });