From a34e50c18b17c3bbad68fe98851254880d1891d0 Mon Sep 17 00:00:00 2001 From: Sam Gbafa Date: Fri, 16 Oct 2026 19:18:30 +0000 Subject: [PATCH 1/4] ci: prepare reviewed node digest promotion --- .github/workflows/docker.yml | 448 ++++++++++++++++++++++++++--- ops/quota-overrides.json | 10 + scripts/ci/test-quota-overrides.py | 325 +++++++++++++++++++++ 3 files changed, 739 insertions(+), 44 deletions(-) create mode 100644 ops/quota-overrides.json create mode 100755 scripts/ci/test-quota-overrides.py diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 7bdc0b1f..2dc1b899 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -43,6 +43,11 @@ on: required: false type: boolean default: false + restore_n0_quota_snapshot: + description: 'One-dispatch N0 authorization to restore the reviewed Sam quota snapshot. Refresh config/authorization before reuse; never enable for unrelated same-image redeploys.' + required: false + type: boolean + default: false allow_non_descendant: description: 'Skip the production ancestry guard for a deliberate non-descendant deployment' required: false @@ -54,22 +59,22 @@ on: type: boolean default: true image_version: - description: 'Manual image version to tag/deploy, without leading v. Must equal the tinycloud-node version in tinycloud-node-server/Cargo.toml at the selected ref. Defaults to the selected ref name.' + description: 'Image version without leading v. Builds must match the selected crate; digest promotions must match the recorded image version (rollback_image_version when supplied).' required: false type: string default: '' deploy_image_digest: - description: 'Promote an existing sha256 digest without rebuilding. Requires deploy_phala, matching image_version, and default dstack features.' + description: 'Promote an existing sha256 digest without rebuilding, dispatched on main only. Requires truthful image_version, deploy_phala and default dstack features.' required: false type: string default: '' rollback_image_version: - description: 'Emergency rollback only: version label of a recorded digest whose revision is in main history' + description: 'Recorded digest version label when promoting an older image or performing an emergency rollback; revision must be in main history' required: false type: string default: '' rollback_image_revision: - description: 'Emergency rollback only: 40-character revision label of the recorded digest (must be an ancestor of main)' + description: 'Recorded digest 40-character revision label (must be an ancestor of main)' required: false type: string default: '' @@ -111,12 +116,15 @@ jobs: name: Version guard runs-on: ubuntu-latest outputs: - cargo_version: ${{ steps.check.outputs.cargo_version }} + image_version: ${{ steps.check.outputs.image_version }} steps: - uses: actions/checkout@v4 - name: Check published Docker tags run: python3 scripts/ci/test-docker-tags.py + - name: Test quota override restoration with synthetic HTTP + run: python3 scripts/ci/test-quota-overrides.py + - name: Validate image version against Cargo.toml id: check env: @@ -124,6 +132,7 @@ jobs: INPUT_IMAGE_VERSION: ${{ inputs.image_version }} INPUT_DEPLOY_DIGEST: ${{ inputs.deploy_image_digest }} INPUT_DEPLOY_PHALA: ${{ inputs.deploy_phala }} + N0_QUOTA_SNAPSHOT: ${{ inputs.restore_n0_quota_snapshot }} ROLLBACK_VERSION: ${{ inputs.rollback_image_version }} ROLLBACK_REVISION: ${{ inputs.rollback_image_revision }} INCLUDE_DUCKDB: ${{ inputs.include_duckdb }} @@ -139,8 +148,21 @@ jobs: echo "::error::could not read the tinycloud-node version from tinycloud-node-server/Cargo.toml" exit 1 fi - echo "cargo_version=${CARGO_VERSION}" >> "$GITHUB_OUTPUT" - echo "tinycloud-node-server/Cargo.toml version: ${CARGO_VERSION}" + # An image digest is not a compose identity or evergreen quota + # policy. This dispatch explicitly authorizes the reviewed N0 + # snapshot; an unrelated same-image redeploy must not reuse it. + if [ "${N0_QUOTA_SNAPSHOT:-false}" = "true" ] \ + || [ "${INPUT_DEPLOY_DIGEST}" = "sha256:4bdae4fc232d886bb56bfbce4ec62ed4b8fb4e1369a01f6afb73d06dd68c188f" ]; then + if [ "${N0_QUOTA_SNAPSHOT:-false}" != "true" ] \ + || [ "${EVENT_NAME}" != "workflow_dispatch" ] || [ "${GITHUB_REF}" != "refs/heads/main" ] \ + || [ "${INPUT_DEPLOY_PHALA}" != "true" ] \ + || [ "${INPUT_DEPLOY_DIGEST}" != "sha256:4bdae4fc232d886bb56bfbce4ec62ed4b8fb4e1369a01f6afb73d06dd68c188f" ] \ + || [ "${INPUT_IMAGE_VERSION}" != "1.20.6" ] || [ "${ROLLBACK_VERSION}" != "1.20.6" ] \ + || [ "${ROLLBACK_REVISION}" != "e18578ff9162986ddd813259c60f73866e3e60d1" ]; then + echo "::error::N0 promotion requires explicit reviewed quota snapshot authorization and exact 1.20.6/e18578 digest labels on main" + exit 1 + fi + fi check_matches() { label="$1" @@ -158,9 +180,6 @@ jobs: check_matches "git tag" "${GITHUB_REF_NAME}" fi - if [ -n "${INPUT_IMAGE_VERSION}" ]; then - check_matches "image_version input" "${INPUT_IMAGE_VERSION}" - fi # TC-940: every production revision is in main's history, so a # rollback dispatched on main passes the ancestry guard without # allow_non_descendant. The rollback revision itself must be an @@ -172,16 +191,32 @@ jobs: echo "::error::Rollback label override requires a digest and valid version/revision labels" exit 1 fi - echo "::warning::Emergency rollback to recorded ${ROLLBACK_VERSION} revision ${ROLLBACK_REVISION}" + echo "::notice::Using recorded image labels: ${ROLLBACK_VERSION} revision ${ROLLBACK_REVISION}" + fi + if [ -n "${INPUT_IMAGE_VERSION}" ]; then + if [ -n "${INPUT_DEPLOY_DIGEST}" ] && [ -n "${ROLLBACK_VERSION}" ]; then + if [ "${INPUT_IMAGE_VERSION}" != "${ROLLBACK_VERSION}" ]; then + echo "::error::Digest promotion image_version must equal the recorded image version" + exit 1 + fi + else + check_matches "image_version input" "${INPUT_IMAGE_VERSION}" + fi fi if [ -n "${INPUT_DEPLOY_DIGEST}" ]; then - if [ "${EVENT_NAME}" != "workflow_dispatch" ] || [ "${INPUT_DEPLOY_PHALA}" != "true" ] || [ -z "${INPUT_IMAGE_VERSION}" ] \ + if [ "${EVENT_NAME}" != "workflow_dispatch" ] || [ "${GITHUB_REF}" != "refs/heads/main" ] \ + || [ "${INPUT_DEPLOY_PHALA}" != "true" ] || [ -z "${INPUT_IMAGE_VERSION}" ] \ || [ "${INCLUDE_DUCKDB}" = "true" ] || [ "${INCLUDE_TC_BENCH_V1}" = "true" ] \ || ! [[ "${INPUT_DEPLOY_DIGEST}" =~ ^sha256:[0-9a-f]{64}$ ]]; then - echo "::error::Digest promotion requires deploy_phala=true, matching image_version, default dstack features and a sha256 digest." + echo "::error::Digest promotion requires workflow_dispatch on main, deploy_phala=true, truthful image_version, default dstack features and a sha256 digest." exit 1 fi fi + # Only the version of the selected binary enters deployment + # outputs/logs; the workflow checkout is not the promoted image. + SELECTED_VERSION="${ROLLBACK_VERSION:-${CARGO_VERSION}}" + echo "image_version=${SELECTED_VERSION}" >> "$GITHUB_OUTPUT" + echo "Selected image version: ${SELECTED_VERSION}" # TC-940: refuse a production deploy from anything but main or a vX.Y.Z # tag on main's history before spending time on image builds. The @@ -391,16 +426,20 @@ jobs: deploy-phala: runs-on: ubuntu-latest needs: [version-guard, build, build-dstack] - # TC-971: the deploy secrets live only in the production environment, whose - # deployment policy admits main and v* tags. A dispatch from any other ref - # is rejected before the job starts, and an old ref whose docker.yml lacks - # this line gets no deploy secrets at all. See docs/releasing.md. + # TC-971: production-environment policy admits main and v* tags. Most deploy + # secrets live there; PROD_TINYCLOUD_ADMIN_SECRET is repository-scoped. + # Digest promotion additionally requires main, not a release tag. environment: production if: >- ${{ vars.TC780_CUTOVER_READY == 'true' && !cancelled() && !inputs.rollback_audit_only && needs.version-guard.result == 'success' && github.event_name == 'workflow_dispatch' && inputs.deploy_phala && - ((inputs.deploy_image_digest != '' && needs.build.result == 'skipped' && needs.build-dstack.result == 'skipped') || + ((inputs.deploy_image_digest != '' && github.ref == 'refs/heads/main' && needs.build.result == 'skipped' && needs.build-dstack.result == 'skipped') || (inputs.deploy_image_digest == '' && needs.build.result == 'success' && needs.build-dstack.result == 'success')) }} + outputs: + image: ${{ steps.pin.outputs.image }} + version: ${{ steps.pin.outputs.version }} + revision: ${{ steps.pin.outputs.revision }} + image_label: ${{ steps.pin.outputs.image_label }} permissions: contents: read packages: read @@ -439,6 +478,11 @@ jobs: - name: Install Phala CLI run: npm install -g phala@1.1.19 + - name: Install Phala env synchronization SDK + # Same SDK generation as phala 1.1.19, isolated from the checkout. + # Install before any sealed values enter the step environment. + run: npm install --prefix "$RUNNER_TEMP/phala-env-sync" --no-save --ignore-scripts @phala/cloud@0.2.9 + - name: Guard production ancestry env: PHALA_CLOUD_API_KEY: ${{ secrets.PHALA_CLOUD_API_KEY }} @@ -521,10 +565,11 @@ jobs: echo "Running production revision ${RUNNING_REVISION} is an ancestor of ${GITHUB_SHA}." - name: Validate and pin deployment image + id: pin env: INPUT_DEPLOY_DIGEST: ${{ inputs.deploy_image_digest }} BUILD_DIGEST: ${{ needs.build-dstack.outputs.digest }} - CARGO_VERSION: ${{ needs.version-guard.outputs.cargo_version }} + DEPLOY_VERSION: ${{ needs.version-guard.outputs.image_version }} INCLUDE_DUCKDB: ${{ inputs.include_duckdb }} INCLUDE_TC_BENCH_V1: ${{ inputs.include_tc_bench_v1 }} ROLLBACK_VERSION: ${{ inputs.rollback_image_version }} @@ -537,12 +582,11 @@ jobs: docker pull --platform linux/amd64 "${RESOLVED_IMAGE}" REVISION="$(docker image inspect "${RESOLVED_IMAGE}" --format '{{ index .Config.Labels "org.opencontainers.image.revision" }}')" VERSION="$(docker image inspect "${RESOLVED_IMAGE}" --format '{{ index .Config.Labels "org.opencontainers.image.version" }}')" - EXPECTED="${CARGO_VERSION}-dstack" + EXPECTED="${DEPLOY_VERSION}-dstack" if [ "${INCLUDE_DUCKDB}" = "true" ]; then EXPECTED="${EXPECTED}-duckdb"; fi if [ "${INCLUDE_TC_BENCH_V1}" = "true" ]; then EXPECTED="${EXPECTED}-tc-bench-v1"; fi EXPECTED_REVISION="$(git rev-parse HEAD)" if [ -n "${ROLLBACK_VERSION}" ]; then - EXPECTED="${ROLLBACK_VERSION}-dstack" EXPECTED_REVISION="${ROLLBACK_REVISION}" fi if [ "${REVISION}" != "${EXPECTED_REVISION}" ] || [ "${VERSION}" != "${EXPECTED}" ]; then @@ -551,15 +595,32 @@ jobs: fi python3 - <<'PY' import os + import re from pathlib import Path path = Path('docker-compose.dstack-postgres.yaml') content = path.read_text() - original = 'ghcr.io/tinycloudlabs/tinycloud-node:dstack' - assert content.count(original) == 1, 'Expected exactly one native image reference' - path.write_text(content.replace(original, os.environ['RESOLVED_IMAGE'])) + native = 'ghcr.io/tinycloudlabs/tinycloud-node' + reviewed = native + '@sha256:4bdae4fc232d886bb56bfbce4ec62ed4b8fb4e1369a01f6afb73d06dd68c188f' + images = [ + image for image in re.finditer(r'(?m)^([ \t]+image:[ \t]+)([^\n]+)$', content) + if image[2].strip().strip('"\'').startswith((native + ':', native + '@')) + ] + if len(images) != 1 or images[0][2].strip().strip('"\'') not in (native + ':dstack', reviewed): + raise SystemExit('Expected exactly one :dstack or reviewed digest-pinned native image') + image = images[0] + if image[2].strip().strip('"\'') == reviewed and reviewed != os.environ['RESOLVED_IMAGE']: + raise SystemExit('Reviewed compose digest does not match the selected deployment image') + path.write_text(content[:image.start(2)] + os.environ['RESOLVED_IMAGE'] + content[image.end(2):]) PY + { + echo "image=${RESOLVED_IMAGE}" + echo "version=${VERSION%%-dstack*}" + echo "revision=${REVISION}" + echo "image_label=${VERSION}" + } >> "$GITHUB_OUTPUT" echo "Resolved image: ${RESOLVED_IMAGE} (revision ${REVISION}, version ${VERSION})" - cat docker-compose.dstack-postgres.yaml + printf 'Deployment image: `%s` — version `%s`, revision `%s`.\n' \ + "${RESOLVED_IMAGE}" "${VERSION}" "${REVISION}" >> "$GITHUB_STEP_SUMMARY" # TC-500: Phala considers a CVM ready before compose services have # started. Validate the production-sealed bundle using the exact image @@ -567,7 +628,9 @@ jobs: # without opening a database, resolving dstack, network, or socket # access, before the deploy step can replace the healthy CVM. - name: Preflight sealed runtime configuration - if: inputs.rollback_image_version == '' + # Older emergency images may lack --validate-config. This reviewed + # 1.20.6 image supports it and MUST NOT inherit that emergency skip. + if: inputs.rollback_image_version == '' || inputs.deploy_image_digest == 'sha256:4bdae4fc232d886bb56bfbce4ec62ed4b8fb4e1369a01f6afb73d06dd68c188f' env: TINYCLOUD_STORAGE__DATABASE: ${{ secrets.PROD_TINYCLOUD_DATABASE_URL }} TINYCLOUD_DATABASE__WRITE_FENCE: ${{ inputs.sql_identity_fence }} @@ -591,7 +654,123 @@ jobs: -e TINYCLOUD_KEYS__TYPE \ "${IMAGE}" --validate-config + # TC-959: only env_keys/encrypted_env may change here. Never copy the + # precedent's raw-response logs or its unguarded compose_hash retry. + # https://docs.phala.com/phala-cloud/references/cloud-js-sdk/update-cvm-envs + - name: Prepare allowed_envs synchronization + run: | + set -euo pipefail + cat > "$RUNNER_TEMP/phala-env-sync/sync.mjs" <<'JS' + import { readFileSync } from "node:fs"; + import { setTimeout as sleep } from "node:timers/promises"; + + // SDK DEBUG can print request bodies and server-supplied errors. + delete process.env.DEBUG; + let phase = "sealed env validation"; + const requireCondition = (condition) => { + if (!condition) throw new Error("guard failed"); + }; + const validName = (name) => typeof name === "string" && /^[A-Za-z_][A-Za-z0-9_]*$/.test(name); + const namesFrom = (compose) => { + const names = compose?.allowed_envs; + requireCondition(Array.isArray(names) && names.every(validName)); + requireCondition(new Set(names).size === names.length); + return names; + }; + + try { + const { + createClient, getCvmComposeFile, getCvmInfo, updateCvmEnvs, + encryptEnvVars, parseEnvVars, + } = await import("@phala/cloud"); + requireCondition(Boolean(process.env.PHALA_CLOUD_API_KEY)); + const source = readFileSync(".phala-prod.env", "utf8"); + const envs = []; + const keys = new Set(); + for (const line of source.split("\n")) { + if (line === "") continue; + const separator = line.indexOf("="); + const key = line.slice(0, separator); + const value = line.slice(separator + 1); + requireCondition(separator > 0 && validName(key) && !keys.has(key)); + requireCondition(value.length > 0 && !/[\r\n]/.test(value)); + keys.add(key); + envs.push({ key, value }); + } + requireCondition(keys.has("BETTERSTACK_SOURCE_TOKEN")); + // Match the CLI's dotenv interpretation exactly; no truncated, + // quoted or comment-stripped value may replace the intended one. + requireCondition(JSON.stringify(parseEnvVars(source)) === JSON.stringify(envs)); + const client = createClient({ + apiKey: process.env.PHALA_CLOUD_API_KEY, + baseURL: "https://cloud-api.phala.com/api/v1", + version: "2026-01-21", + timeout: 15000, + retry: 0, + }); + const ref = { id: "tinycloud-node" }; + phase = "complete whitelist validation"; + const allowed = namesFrom(await getCvmComposeFile(client, ref)); + // Retaining a name cannot recover a value absent from this file. + // Check before deploy AND again immediately before any env PATCH. + requireCondition(allowed.every((key) => keys.has(key))); + const missing = [...keys].filter((key) => !allowed.includes(key)); + if (process.argv.includes("--check")) { + console.log(`Validated complete sealed set: ${keys.size} env names.`); + } else if (missing.length === 0) { + console.log(`allowed_envs verified: ${keys.size} env names; no PATCH needed.`); + } else { + phase = "public encryption key validation"; + const info = await getCvmInfo(client, ref); + const pubkey = info.kms_info?.encrypted_env_pubkey ?? info.encrypted_env_pubkey; + requireCondition(typeof pubkey === "string" && /^(0x)?[0-9a-fA-F]{64}$/.test(pubkey)); + const encrypted = await encryptEnvVars(envs, pubkey); + phase = "env whitelist update"; + const result = await updateCvmEnvs(client, { + ...ref, encrypted_env: encrypted, env_keys: [...allowed, ...missing], + }); + // An on-chain precondition needs a separately reviewed + // addComposeHash transaction. Never assume hash-only approval. + requireCondition(result.status === "in_progress" && typeof result.correlation_id === "string" && result.correlation_id.length > 0); + console.log(`allowed_envs update accepted: ${missing.length} added names.`); + phase = "env whitelist convergence"; + const deadline = Date.now() + 600000; + let completed = false; + let observedOperation = false; + const pending = new Set(["starting", "stopping", "stopped", "restarting", "upgrading", "updating", "creating"]); + while (Date.now() < deadline) { + await sleep(5000); + const current = await getCvmInfo(client, ref); + requireCondition(current.status === "running" || pending.has(current.status)); + const after = namesFrom(await getCvmComposeFile(client, ref)); + const progress = current.progress; + if (progress?.correlation_id === result.correlation_id || pending.has(current.status)) observedOperation = true; + // progress is optional in the pinned SDK. Missing or null is + // settled only AFTER an update/progress transition was seen. + const settled = progress == null || ( + progress && !progress.target && !progress.started_at && !progress.correlation_id + ); + // A stored whitelist/running status can precede a queued + // restart. Require this operation to be observed and settled; + // an unobserved fast update fails closed at the deadline. + if (observedOperation && current.status === "running" && settled && [...keys].every((key) => after.includes(key))) { + completed = true; + break; + } + } + requireCondition(completed); + console.log(`allowed_envs verified after update: ${keys.size} env names.`); + } + } catch { + // Neither response message/JSON, ciphertext, nor exception detail + // is safe for public Actions logs. + console.error(`::error::allowed_envs synchronization failed during ${phase}; response details withheld.`); + process.exitCode = 1; + } + JS + - name: Deploy to Phala Cloud + id: deploy env: PHALA_CLOUD_API_KEY: ${{ secrets.PHALA_CLOUD_API_KEY }} PHALA_ENV_DATABASE_URL: ${{ secrets.PROD_TINYCLOUD_DATABASE_URL }} @@ -606,12 +785,16 @@ jobs: PHALA_ENV_CLOUDFLARE_API_TOKEN: ${{ secrets.PROD_TINYCLOUD_CLOUDFLARE_API_TOKEN }} PHALA_ENV_DSTACK_GATEWAY_DOMAIN: ${{ secrets.PROD_TINYCLOUD_DSTACK_GATEWAY_DOMAIN }} PHALA_ENV_CERTBOT_EMAIL: ${{ secrets.PROD_TINYCLOUD_CERTBOT_EMAIL }} + PHALA_ENV_BETTERSTACK_SOURCE_TOKEN: ${{ secrets.PROD_TINYCLOUD_BETTERSTACK_SOURCE_TOKEN }} PHALA_ENV_SQL_IDENTITY_FENCE: ${{ inputs.sql_identity_fence }} # Update the existing prod CVM (name: tinycloud-node) in the # Tiny Cloud workspace. The PHALA_CLOUD_API_KEY secret is # workspace-scoped and must be issued from that workspace. run: | set -euo pipefail + set +x + umask 077 + trap 'rm -f .phala-prod.env' EXIT required_vars=( PHALA_CLOUD_API_KEY @@ -627,6 +810,7 @@ jobs: PHALA_ENV_CLOUDFLARE_API_TOKEN PHALA_ENV_DSTACK_GATEWAY_DOMAIN PHALA_ENV_CERTBOT_EMAIL + PHALA_ENV_BETTERSTACK_SOURCE_TOKEN PHALA_ENV_SQL_IDENTITY_FENCE ) @@ -635,8 +819,16 @@ jobs: echo "::error::Missing required GitHub secret for Phala deploy: ${var#PHALA_ENV_}" exit 1 fi + if [[ "${!var}" == *$'\n'* || "${!var}" == *$'\r'* ]]; then + echo "::error::Phala deploy inputs must be single-line values" + exit 1 + fi done + # Also restrict a file left by an earlier failed runner step before + # writing any value; umask alone does not tighten an existing file. + : > .phala-prod.env + chmod 0600 .phala-prod.env { printf 'DATABASE_URL=%s\n' "$PHALA_ENV_DATABASE_URL" printf 'S3_BUCKET=%s\n' "$PHALA_ENV_S3_BUCKET" @@ -650,6 +842,7 @@ jobs: printf 'CLOUDFLARE_API_TOKEN=%s\n' "$PHALA_ENV_CLOUDFLARE_API_TOKEN" printf 'DSTACK_GATEWAY_DOMAIN=%s\n' "$PHALA_ENV_DSTACK_GATEWAY_DOMAIN" printf 'CERTBOT_EMAIL=%s\n' "$PHALA_ENV_CERTBOT_EMAIL" + printf 'BETTERSTACK_SOURCE_TOKEN=%s\n' "$PHALA_ENV_BETTERSTACK_SOURCE_TOKEN" printf 'SQL_IDENTITY_FENCE=%s\n' "$PHALA_ENV_SQL_IDENTITY_FENCE" # SSH public keys authorized on the CVM (checked into the repo, # see dstack-authorized-keys). The Phala Cloud pre-launch script @@ -658,13 +851,33 @@ jobs: printf 'DSTACK_AUTHORIZED_KEYS=%s\n' "$(cat dstack-authorized-keys)" } > .phala-prod.env + # Validate complete env names/whitelist before any CVM mutation. + node "$RUNNER_TEMP/phala-env-sync/sync.mjs" --check + # Fail before replacing the running CVM if a required compose input # is absent. Phala's control-plane ready state does not prove that # docker compose interpolation or container startup succeeded. - docker compose --env-file .phala-prod.env \ - -f docker-compose.dstack-postgres.yaml config --quiet + if ! docker compose --env-file .phala-prod.env \ + -f docker-compose.dstack-postgres.yaml config --quiet >/dev/null 2>&1; then + echo "::error::Sealed compose validation failed; deployment not attempted" + exit 1 + fi + + # The CLI/SDK can include request details in errors. Never forward + # those responses into Actions logs, even when deployment fails. + # A failing CLI/update can still have restarted the CVM. + echo "cvm_mutation_attempted=true" >> "$GITHUB_OUTPUT" + if ! phala deploy --cvm-id tinycloud-node -c docker-compose.dstack-postgres.yaml -e .phala-prod.env --no-public-logs --no-public-sysinfo --wait >/dev/null 2>&1; then + echo "::error::Phala deployment failed; inspect the private control plane" + exit 1 + fi - phala deploy --cvm-id tinycloud-node -c docker-compose.dstack-postgres.yaml -e .phala-prod.env --no-public-logs --no-public-sysinfo --wait + # Do not assume -e registered the whitelist; verify/sync it explicitly. + node "$RUNNER_TEMP/phala-env-sync/sync.mjs" + + - name: Remove sealed deployment environment + if: always() + run: rm -f .phala-prod.env # TC-312: prove the deploy actually converged. The image tag matching the # crate version is enforced at build time by version-guard, but nothing @@ -674,23 +887,14 @@ jobs: # just deployed. - name: Verify deployed version env: - EVENT_NAME: ${{ github.event_name }} - RELEASE_TAG: ${{ github.event.release.tag_name }} - INPUT_IMAGE_VERSION: ${{ inputs.image_version }} - ROLLBACK_VERSION: ${{ inputs.rollback_image_version }} + EXPECTED: ${{ steps.pin.outputs.version }} + IMAGE_REVISION: ${{ steps.pin.outputs.revision }} run: | set -euo pipefail - # Resolve the version we just deployed, mirroring the compose step. - if [ "${EVENT_NAME}" = "release" ]; then - TAG="${RELEASE_TAG}" - elif [ -n "${INPUT_IMAGE_VERSION}" ]; then - TAG="${INPUT_IMAGE_VERSION}" - else - TAG="${GITHUB_REF_NAME}" - fi - EXPECTED="${ROLLBACK_VERSION:-${TAG#v}}" - echo "Expecting node.tinycloud.xyz to report version ${EXPECTED}" + # Only validated image labels describe the binary, not Cargo.toml + # at the workflow checkout when an existing digest is promoted. + echo "Expecting node.tinycloud.xyz to report version ${EXPECTED} (revision ${IMAGE_REVISION})" # The CVM needs a short while to pull the new image and restart, so # retry with a fixed backoff (~5 min total) before giving up. @@ -776,6 +980,151 @@ jobs: echo "${route}: mounted (HTTP ${STATUS})" rm -f "${RESPONSE_FILE}" done + # Reviewed N0-only snapshot, not evergreen quota policy. Same deploy job + # means no second environment approval or watcher-scheduling gap. + # Future restarts require refreshed config and operator authorization. + + - name: Restore storage quota overrides + id: quotas + timeout-minutes: 5 + if: >- + ${{ success() && github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main' && inputs.deploy_phala && inputs.restore_n0_quota_snapshot && + inputs.deploy_image_digest == 'sha256:4bdae4fc232d886bb56bfbce4ec62ed4b8fb4e1369a01f6afb73d06dd68c188f' && + steps.pin.outputs.image == 'ghcr.io/tinycloudlabs/tinycloud-node@sha256:4bdae4fc232d886bb56bfbce4ec62ed4b8fb4e1369a01f6afb73d06dd68c188f' && + steps.pin.outputs.version == '1.20.6' && steps.pin.outputs.image_label == '1.20.6-dstack' && + steps.pin.outputs.revision == 'e18578ff9162986ddd813259c60f73866e3e60d1' }} + env: + # Repository-scoped; never read by an agent or passed in argv. + QUOTA_ADMIN_SECRET: ${{ secrets.PROD_TINYCLOUD_ADMIN_SECRET }} + run: | + set +x + python3 - <<'PY' + import http.client + import json + import os + from pathlib import Path + import re + import time + from urllib.parse import quote + + CONFIG_PATH = Path("ops/quota-overrides.json") + LIMITS = {"applications": 5368709120, "secrets": 268721121} + + class QuotaError(Exception): + pass + + def load_overrides(path): + try: + document = json.loads(path.read_text()) + if not isinstance(document, dict) or len(document) != 2: + raise ValueError + names = set() + overrides = {} + for space, row in document.items(): + match = re.fullmatch(r"tinycloud:[A-Za-z0-9:._-]+:(applications|secrets)", space) + if not isinstance(row, dict) or set(row) != {"limit_bytes", "_comment"}: + raise ValueError + limit = row["limit_bytes"] + if (not match or type(limit) is not int or limit != LIMITS[match[1]] + or not isinstance(row["_comment"], str) or not row["_comment"]): + raise ValueError + names.add(match[1]) + overrides[space] = limit + if names != set(LIMITS): + raise ValueError + return overrides + except (OSError, ValueError, TypeError): + raise QuotaError("reviewed quota configuration is invalid") from None + + def request_admin(method, path, limit, secret): + body = None if limit is None else json.dumps({"limit_bytes": limit}).encode() + headers = {"Authorization": "Bearer " + secret, "Content-Type": "application/json"} + # Fixed HTTPS origin, no redirects/proxies; authorization exists + # only in process memory and the TLS request header, never argv. + for attempt in range(3): + connection = http.client.HTTPSConnection("node.tinycloud.xyz", timeout=15) + try: + connection.request(method, path, body=body, headers=headers) + response = connection.getresponse() + payload = response.read(65537) + except (OSError, http.client.HTTPException): + if attempt == 2: + raise QuotaError("quota transport failed after bounded attempts") from None + time.sleep(attempt + 1) + continue + finally: + connection.close() + # HTTP failures (including 5xx) are NOT transport retries. + # Never include a response body/status detail in diagnostics. + if response.status != 200: + raise QuotaError("quota API rejected the request") + try: + result = json.loads(payload) if len(payload) <= 65536 else None + if not isinstance(result, dict): + raise ValueError + return result + except ValueError: + raise QuotaError("quota API returned an invalid response") from None + + def apply_overrides(overrides, secret): + if not secret or "\r" in secret or "\n" in secret: + raise QuotaError("admin authorization is missing or invalid") + # A quota GET echoes its supplied ID even for a nonexistent + # space. Only /admin/usage enumerates actual list_space_ids. + # Validate BOTH configured targets before ANY PUT. + inventory = request_admin("GET", "/admin/usage", None, secret) + spaces = inventory.get("spaces") + if (not isinstance(spaces, list) or type(inventory.get("count")) is not int + or inventory["count"] != len(spaces)): + raise QuotaError("space inventory is invalid; no overrides applied") + usage = {} + for row in spaces: + if (not isinstance(row, dict) or not isinstance(row.get("space_id"), str) + or row["space_id"] in usage): + raise QuotaError("space inventory is ambiguous; no overrides applied") + usage[row["space_id"]] = row.get("usage_bytes") + if any(type(usage.get(space)) is not int or usage[space] <= 0 for space in overrides): + raise QuotaError("both exact quota targets require positive known usage; no overrides applied") + for space, limit in overrides.items(): + path = "/admin/quota/" + quote(space, safe="") + request_admin("PUT", path, limit, secret) + actual = request_admin("GET", path, None, secret) + if (actual.get("space_id") != space + or type(actual.get("override_bytes")) is not int + or actual["override_bytes"] != limit + or type(actual.get("effective_limit_bytes")) is not int + or actual["effective_limit_bytes"] != limit): + raise QuotaError("quota override and effective limit did not both verify") + + def main(): + try: + overrides = load_overrides(CONFIG_PATH) + apply_overrides(overrides, os.environ.get("QUOTA_ADMIN_SECRET", "")) + message = "Quota overrides verified: applications=5368709120; secrets=268721121." + code = 0 + except QuotaError as error: + message = f"Quota restoration FAILED: {error}. Authorized TC-1161 fallback required." + code = 1 + except Exception: + message = "Quota restoration FAILED: unexpected error; details withheld. Authorized TC-1161 fallback required." + code = 1 + print(("::error::" if code else "") + message) + with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as summary: + summary.write(message + "\n") + return code + + if __name__ == "__main__": + raise SystemExit(main()) + PY + + - name: Report unverified storage overrides + if: >- + ${{ always() && steps.deploy.outputs.cvm_mutation_attempted == 'true' && steps.quotas.outcome != 'success' }} + run: | + MESSAGE="OVERRIDES UNVERIFIED: a CVM mutation was attempted without successful reviewed quota restoration. Authorized operator must resolve via TC-1161 before another dispatch." + echo "::error::${MESSAGE}" + printf '%s\n' "${MESSAGE}" >> "$GITHUB_STEP_SUMMARY" + exit 1 # TC-1176: the 2026-10-14 read hang began ~40 min after a promote. After any # Phala deploy attempt (including rollbacks and failed verifications, which @@ -794,4 +1143,15 @@ jobs: - name: Dispatch Write Probe promote watch env: GH_TOKEN: ${{ github.token }} - run: gh workflow run write-probe.yml -R "${GITHUB_REPOSITORY}" --ref main -f mode=dense + IMAGE: ${{ needs.deploy-phala.outputs.image }} + IMAGE_VERSION: ${{ needs.deploy-phala.outputs.version }} + IMAGE_LABEL: ${{ needs.deploy-phala.outputs.image_label }} + IMAGE_REVISION: ${{ needs.deploy-phala.outputs.revision }} + run: | + if [ -n "${IMAGE}" ]; then + echo "Promote watch for ${IMAGE_LABEL} (version ${IMAGE_VERSION}) revision ${IMAGE_REVISION}: ${IMAGE}" + else + echo "Promote watch after deployment attempt; image validation did not complete." + fi + gh workflow run write-probe.yml -R "${GITHUB_REPOSITORY}" --ref main -f mode=dense + diff --git a/ops/quota-overrides.json b/ops/quota-overrides.json new file mode 100644 index 00000000..929c914c --- /dev/null +++ b/ops/quota-overrides.json @@ -0,0 +1,10 @@ +{ + "tinycloud:pkh:eip155:1:0xd559CCd9EB87c530A9a349262669386dE93cf412:applications": { + "limit_bytes": 5368709120, + "_comment": "Confirmed by Amos's read-only TC-1161 D1 quota_overrides and quota_override_audit extract, 2026-10-16: applied 2026-10-16T17:37:16Z, last_node_status 200. N0 snapshot only: Sam authorizes values; Amos supplies reviewed target/value evidence. Refresh this config and authorization before reuse. The explicit restore_n0_quota_snapshot dispatch input is not compose-hash verification. The workflow must still prove exact space existence and positive usage before any PUT." + }, + "tinycloud:pkh:eip155:1:0xd559CCd9EB87c530A9a349262669386dE93cf412:secrets": { + "limit_bytes": 268721121, + "_comment": "Space ID verified read-only by Amos via Sam's tc-secrets release-keys and monitoring profiles; override value 268721121 was applied manually before the dashboard existed. No secret values were read. TC-1161 D1 record is pending Sam's board task. The authorized workflow must still prove this exact full ID with positive usage through GET /admin/usage before ANY PUT; a quota GET echo is not existence evidence." + } +} diff --git a/scripts/ci/test-quota-overrides.py b/scripts/ci/test-quota-overrides.py new file mode 100755 index 00000000..a301f2b4 --- /dev/null +++ b/scripts/ci/test-quota-overrides.py @@ -0,0 +1,325 @@ +#!/usr/bin/env python3 +"""Exercise docker.yml's quota restoration against synthetic HTTP only.""" + +import contextlib +import io +import json +from pathlib import Path +import secrets +import tempfile +import re +import subprocess +import textwrap +import unittest +from unittest.mock import patch + + +WORKFLOW = Path(__file__).resolve().parents[2] / ".github/workflows/docker.yml" +SOURCE = WORKFLOW.read_text() +MARKER = " - name: Restore storage quota overrides\n" +# Execute the actual inline implementation without its command entry point. +# No duplicated implementation or third-party YAML dependency is needed. +if MARKER not in SOURCE: + raise SystemExit("FAIL: quota restoration step is absent") +BLOCK = SOURCE.split(MARKER, 1)[1] +SCRIPT = BLOCK.split(" python3 - <<'PY'\n", 1)[1].split("\n PY", 1)[0] +NAMESPACE = {"__name__": "quota_workflow_test"} +exec(compile(textwrap.dedent(SCRIPT), str(WORKFLOW) + ":quota-restoration", "exec"), NAMESPACE) + +APPLICATIONS = "tinycloud:pkh:eip155:1:0x" + "1" * 40 + ":applications" +SECRETS = "tinycloud:pkh:eip155:1:0x" + "1" * 40 + ":secrets" +OVERRIDES = {APPLICATIONS: 5368709120, SECRETS: 268721121} +CANARIES = [] + + +class SecretSafeResult(unittest.TextTestResult): + def _exc_info_to_string(self, error, test): + output = super()._exc_info_to_string(error, test) + for canary in CANARIES: + output = output.replace(canary, "[sealed canary withheld]") + return output + + +def quota_restore_enabled(context): + condition = BLOCK.split(" if: >-\n", 1)[1].split("\n env:", 1)[0].strip()[3:-2] + return condition_enabled(condition, context) + + +def condition_enabled(condition, context): + literals = [] + def protect(match): + literals.append(match[0]) + return f"@{len(literals) - 1}@" + condition = re.sub(r"'[^']*'", protect, condition) + condition = re.sub(r"(?:github|inputs|steps|needs)\.[\w.-]+(?:\.[\w.-]+)*", + lambda match: repr(context.get(match[0], "")), condition) + condition = condition.replace("success()", repr(context.get("job.success", True))) + condition = condition.replace("always()", "True") + condition = condition.replace("&&", " and ").replace("||", " or ") + for index, literal in enumerate(literals): + condition = condition.replace(f"@{index}@", literal) + return bool(eval("(" + condition + ")", {"__builtins__": {}}, {})) + + +def config_document(overrides): + return {space: {"limit_bytes": limit, "_comment": "Synthetic reviewed target"} + for space, limit in overrides.items()} + + +class Response: + def __init__(self, status, payload): + self.status = status + self.payload = payload + + def read(self, count): + return self.payload[:count] + + +class QuotaTests(unittest.TestCase): + def setUp(self): + self.secret = secrets.token_urlsafe(32) + CANARIES.append(self.secret) + self.requests = [] + self.values = {} + self.status = 200 + self.transport_failures = 0 + self.fail_method = None + self.usage_rows = [{"space_id": space, "usage_bytes": 10} for space in OVERRIDES] + self.applied = True + self.wrong_effective = False + self.patchers = [ + patch.object(NAMESPACE["http"].client, "HTTPSConnection", self.connection), + patch.object(NAMESPACE["time"], "sleep"), + patch("socket.create_connection", side_effect=AssertionError("Real network forbidden")), + ] + for patcher in self.patchers: + patcher.start() + self.addCleanup(patcher.stop) + + def connection(self, host, timeout): + self.assertEqual(host, "node.tinycloud.xyz") + self.assertGreater(timeout, 0) + test = self + + class Connection: + def request(self, method, path, body=None, headers=None): + test.requests.append((method, path, body, headers)) + if test.transport_failures and (test.fail_method is None or method == test.fail_method): + test.transport_failures -= 1 + raise OSError(test.secret) + self.method = method + self.path = path + if method == "PUT" and test.applied: + test.values[path] = json.loads(body)["limit_bytes"] + + def getresponse(self): + if test.status != 200: + return Response(test.status, test.secret.encode()) + from urllib.parse import unquote + if self.path == "/admin/usage": + return Response(200, json.dumps({ + "spaces": test.usage_rows, "count": len(test.usage_rows), + }).encode()) + space = unquote(self.path.removeprefix("/admin/quota/")) + override = test.values.get(self.path) + effective = 1 if test.wrong_effective else override + return Response(200, json.dumps({ + "space_id": space, + "override_bytes": override, + "effective_limit_bytes": effective, + }).encode()) + + def close(self): + pass + + return Connection() + + def apply(self): + return NAMESPACE["apply_overrides"](OVERRIDES, self.secret) + + def test_applied_overrides_are_read_back_and_idempotent(self): + self.apply() + self.apply() + self.assertEqual(len(self.requests), 10) + self.assertEqual([r[0] for r in self.requests], ["GET", "PUT", "GET", "PUT", "GET"] * 2) + self.assertTrue(self.requests[0][1] == self.requests[5][1] == "/admin/usage") + self.assertEqual(sorted(self.values.values()), sorted(OVERRIDES.values())) + for method, path, body, headers in self.requests: + self.assertTrue(headers.get("Authorization") == "Bearer " + self.secret, + "Admin authorization must be passed as an HTTP header") + self.assertFalse(self.secret in path, "Sealed canary leaked in a URL") + if body: + self.assertFalse(self.secret.encode() in body, "Sealed canary leaked in a request body") + + def test_successful_put_does_not_replace_get_verification(self): + self.applied = False + with self.assertRaises(NAMESPACE["QuotaError"]): + self.apply() + self.assertEqual([r[0] for r in self.requests], ["GET", "PUT", "GET"]) + + def test_effective_limit_must_also_match(self): + self.wrong_effective = True + with self.assertRaises(NAMESPACE["QuotaError"]): + self.apply() + + def test_http_5xx_is_not_retried_or_logged(self): + self.status = 503 + captured = io.StringIO() + with contextlib.redirect_stdout(captured), contextlib.redirect_stderr(captured): + with self.assertRaises(NAMESPACE["QuotaError"]): + self.apply() + self.assertEqual(len(self.requests), 1) + self.assertFalse(self.secret in captured.getvalue(), "Sealed canary leaked in diagnostics") + + def test_transport_retry_is_bounded_and_idempotent(self): + self.transport_failures = 2 + self.fail_method = "PUT" + self.apply() + self.assertEqual([r[0] for r in self.requests[:4]], ["GET", "PUT", "PUT", "PUT"]) + self.assertEqual(sorted(self.values.values()), sorted(OVERRIDES.values())) + + def test_transport_exhaustion_fails_without_error_details(self): + self.transport_failures = 10 + with self.assertRaises(NAMESPACE["QuotaError"]) as error: + self.apply() + self.assertEqual(len(self.requests), 3) + self.assertFalse(self.secret in str(error.exception), "Sealed canary leaked in an exception") + + def test_missing_authorization_never_requests(self): + with self.assertRaises(NAMESPACE["QuotaError"]): + NAMESPACE["apply_overrides"](OVERRIDES, "") + self.assertEqual(self.requests, []) + + def test_reviewed_config_requires_exact_two_absolute_limits(self): + with tempfile.TemporaryDirectory() as directory: + config = Path(directory) / "overrides.json" + config.write_text(json.dumps(config_document(OVERRIDES))) + self.assertEqual(NAMESPACE["load_overrides"](config), OVERRIDES) + for bad in ({APPLICATIONS: 5368709120}, {APPLICATIONS: 5, SECRETS: 268721121}, + {APPLICATIONS: True, SECRETS: 268721121}): + config.write_text(json.dumps(config_document(bad))) + with self.assertRaises(NAMESPACE["QuotaError"]): + NAMESPACE["load_overrides"](config) + + def test_main_failure_is_nonzero_and_has_safe_summary(self): + self.status = 503 + with tempfile.TemporaryDirectory() as directory: + config = Path(directory) / "overrides.json" + summary = Path(directory) / "summary" + config.write_text(json.dumps(config_document(OVERRIDES))) + output = io.StringIO() + with patch.dict(NAMESPACE["os"].environ, { + "QUOTA_ADMIN_SECRET": self.secret, + "GITHUB_STEP_SUMMARY": str(summary), + }), patch.dict(NAMESPACE, {"CONFIG_PATH": config}): + with contextlib.redirect_stdout(output), contextlib.redirect_stderr(output): + self.assertEqual(NAMESPACE["main"](), 1) + self.assertIn("FAILED", summary.read_text()) + self.assertFalse(self.secret in output.getvalue() + summary.read_text(), "Sealed canary leaked") + + def test_usage_inventory_must_prove_both_targets_before_any_put(self): + bad_rows = [ + [{"space_id": APPLICATIONS, "usage_bytes": 10}], + [{"space_id": APPLICATIONS, "usage_bytes": 10}, + {"space_id": SECRETS, "usage_bytes": None}], + [{"space_id": APPLICATIONS, "usage_bytes": 10}, + {"space_id": SECRETS, "usage_bytes": 0}], + [{"space_id": APPLICATIONS, "usage_bytes": 10}, + {"space_id": SECRETS, "usage_bytes": True}], + [{"space_id": APPLICATIONS, "usage_bytes": 10}, + {"space_id": SECRETS, "usage_bytes": 10}, + {"space_id": SECRETS, "usage_bytes": 10}], + [{"space_id": APPLICATIONS, "usage_bytes": 10}, + {"space_id": SECRETS + "-typo", "usage_bytes": 10}], + ] + for index, rows in enumerate(bad_rows): + with self.subTest(case=index): + self.requests.clear() + self.usage_rows = rows + with self.assertRaises(NAMESPACE["QuotaError"]): + self.apply() + self.assertTrue(all(request[0] == "GET" for request in self.requests), + "No target may be mutated before both identities are proved") + + def test_checked_in_config_has_reviewed_absolute_limits(self): + overrides = NAMESPACE["load_overrides"](WORKFLOW.parents[2] / "ops/quota-overrides.json") + self.assertEqual(sorted(overrides.values()), sorted(OVERRIDES.values())) + + def test_inventory_failure_never_dumps_raw_space_rows(self): + self.usage_rows = [{"space_id": self.secret, "usage_bytes": 10}] + with tempfile.TemporaryDirectory() as directory: + config = Path(directory) / "overrides.json" + summary = Path(directory) / "summary" + config.write_text(json.dumps(config_document(OVERRIDES))) + output = io.StringIO() + with patch.dict(NAMESPACE["os"].environ, { + "QUOTA_ADMIN_SECRET": self.secret, "GITHUB_STEP_SUMMARY": str(summary), + }), patch.dict(NAMESPACE, {"CONFIG_PATH": config}): + with contextlib.redirect_stdout(output), contextlib.redirect_stderr(output): + self.assertEqual(NAMESPACE["main"](), 1) + self.assertTrue(all(request[0] == "GET" for request in self.requests)) + self.assertFalse(self.secret in output.getvalue() + summary.read_text(), "Raw inventory leaked") + + def test_snapshot_requires_explicit_n0_authorization_and_exact_image(self): + digest = "sha256:4bdae4fc232d886bb56bfbce4ec62ed4b8fb4e1369a01f6afb73d06dd68c188f" + selected = { + "github.event_name": "workflow_dispatch", "github.ref": "refs/heads/main", + "inputs.deploy_phala": True, "inputs.deploy_image_digest": digest, + "inputs.restore_n0_quota_snapshot": True, + "job.success": True, "needs.promote-watch.result": "success", + "steps.pin.outputs.image": "ghcr.io/tinycloudlabs/tinycloud-node@" + digest, + "steps.pin.outputs.version": "1.20.6", + "steps.pin.outputs.image_label": "1.20.6-dstack", + "steps.pin.outputs.revision": "e18578ff9162986ddd813259c60f73866e3e60d1", + } + self.assertTrue(quota_restore_enabled(selected)) + self.assertTrue(quota_restore_enabled({**selected, "needs.promote-watch.result": "failure"}), + "Watch scheduling must not block post-restart quota restoration") + for changes in ( + {"inputs.restore_n0_quota_snapshot": False}, + {"inputs.deploy_image_digest": ""}, + {"inputs.deploy_image_digest": "sha256:" + "f" * 64}, + {"github.event_name": "release"}, + {"github.event_name": "push"}, + {"github.ref": "refs/tags/v1.20.6"}, + {"steps.pin.outputs.version": "1.20.7"}, + {"steps.pin.outputs.revision": "f" * 40}, + {"steps.pin.outputs.image": "ghcr.io/tinycloudlabs/tinycloud-node@sha256:" + "f" * 64}, + {"job.success": False}, + ): + with self.subTest(fields=sorted(changes)): + self.assertFalse(quota_restore_enabled({**selected, **changes}), + "Unrelated deploy must not replay the reviewed snapshot") + + def test_partial_restart_failure_is_explicitly_unverified(self): + report = SOURCE.split(" - name: Report unverified storage overrides\n", 1)[1] + condition = report.split(" if: >-\n", 1)[1].split("\n run:", 1)[0].strip()[3:-2] + for attempted, outcome, enabled in ( + ("true", "failure", True), ("true", "skipped", True), + ("true", "cancelled", True), ("true", "success", False), + ("", "skipped", False), + ): + with self.subTest(attempted=attempted, outcome=outcome): + self.assertEqual(condition_enabled(condition, { + "job.success": False, + "steps.deploy.outputs.cvm_mutation_attempted": attempted, + "steps.quotas.outcome": outcome, + }), enabled) + lines = [] + for line in report.split(" run: |\n", 1)[1].splitlines(): + if line and not line.startswith(" "): + break + lines.append(line) + with tempfile.TemporaryDirectory() as directory: + summary = Path(directory) / "summary" + result = subprocess.run(["bash", "-e", "-c", textwrap.dedent("\n".join(lines))], + env={"GITHUB_STEP_SUMMARY": str(summary), "QUOTA_ADMIN_SECRET": self.secret}, + capture_output=True, text=True) + self.assertNotEqual(result.returncode, 0) + self.assertIn("OVERRIDES UNVERIFIED", summary.read_text()) + self.assertFalse(self.secret in result.stdout + result.stderr + summary.read_text(), + "Partial-failure diagnostics must not expose authorization") + + +if __name__ == "__main__": + unittest.main(testRunner=unittest.TextTestRunner(resultclass=SecretSafeResult, verbosity=2)) From 311103644a4591b3d6a2c7cae1ba489f2d022301 Mon Sep 17 00:00:00 2001 From: Sam Gbafa Date: Fri, 16 Oct 2026 19:36:19 +0000 Subject: [PATCH 2/4] ci: close quota restoration and async update safety gaps --- .github/workflows/docker.yml | 135 +++++++++++---------- scripts/check-deployment-policy-probes.mjs | 127 +++++++++++++------ scripts/ci/test-quota-overrides.py | 94 ++++++++++---- 3 files changed, 236 insertions(+), 120 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 2dc1b899..edc57197 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -744,15 +744,15 @@ jobs: requireCondition(current.status === "running" || pending.has(current.status)); const after = namesFrom(await getCvmComposeFile(client, ref)); const progress = current.progress; - if (progress?.correlation_id === result.correlation_id || pending.has(current.status)) observedOperation = true; - // progress is optional in the pinned SDK. Missing or null is - // settled only AFTER an update/progress transition was seen. + // Prefer this operation's correlation. If optional progress + // is absent, require a post-PATCH non-running/running cycle. + if (progress?.correlation_id === result.correlation_id || + (!progress?.correlation_id && pending.has(current.status))) observedOperation = true; + // Missing/null progress can mean a fast or still-queued + // restart. Stored names + running alone are not completion. const settled = progress == null || ( progress && !progress.target && !progress.started_at && !progress.correlation_id ); - // A stored whitelist/running status can precede a queued - // restart. Require this operation to be observed and settled; - // an unobserved fast update fails closed at the deadline. if (observedOperation && current.status === "running" && settled && [...keys].every((key) => after.includes(key))) { completed = true; break; @@ -914,72 +914,24 @@ jobs: echo "::error::node.tinycloud.xyz still reports '${REPORTED}' after $((ATTEMPTS * SLEEP))s, expected '${EXPECTED}'. The deploy did not converge." exit 1 - # TC-465: Share requires the complete Policy/v3 control plane before it - # can create or receive an exact-email share. A version-only check let a - # pre-TC-405 image look healthy while every required route returned 404. - # Empty JSON is deliberately rejected before any handler can mutate - # policy state, so this proves route mounting against the direct TEE - # origin without creating a binding, challenge, policy, or delegation. - - name: Verify deployed Policy/v3 routes + # Ingress TLS readiness is required health. Restore quotas before the + # separate Policy probes, which still fail the deployment if invalid. + - name: Verify deployed ingress health + env: + EXPECTED: ${{ steps.pin.outputs.version }} run: | set -euo pipefail - - NODE_ORIGIN="https://tee.node.tinycloud.xyz" - ROUTES=( - /policy/v3/enforcer-bindings - /policy/v3/policies - /policy/v3/challenges - /policy/v3/delegations - /policy/v3/deliveries/authorize - ) - - # The direct TEE ingress obtains/loads its certificate after the - # node's public /version endpoint is already healthy. Wait for that - # TLS endpoint before probing routes so a successful rollout does - # not fail on a transient SSL_ERROR_SYSCALL during ingress startup. for attempt in $(seq 1 30); do - if curl -fsS --max-time 10 "${NODE_ORIGIN}/version" > /dev/null; then - echo "Direct TEE ingress is ready." - break - fi - if [ "${attempt}" = "30" ]; then - echo "::error::Direct TEE ingress did not become ready after 150s." - exit 1 + REPORTED="$(curl -fsS --max-time 10 https://tee.node.tinycloud.xyz/version | jq -r '.version' || true)" + if [ "${REPORTED}" = "${EXPECTED}" ]; then + echo "Direct TEE ingress is serving ${EXPECTED}." + exit 0 fi sleep 5 done + echo "::error::Direct TEE ingress did not become healthy at version ${EXPECTED}." + exit 1 - for route in "${ROUTES[@]}"; do - RESPONSE_FILE="$(mktemp)" - STATUS="$(curl -sS --max-time 15 -o "${RESPONSE_FILE}" -w '%{http_code}' \ - -X POST \ - -H 'accept: application/json' \ - -H 'content-type: application/json' \ - --data '{}' \ - "${NODE_ORIGIN}${route}")" - - # Delivery is authenticated before request-body validation, so its - # intentionally unsigned probe must be exactly 401. Other - # control-plane routes reject the body first (400 or 422). Neither - # case may silently accept a missing route or a server failure. - if [ "${route}" = "/policy/v3/deliveries/authorize" ]; then - EXPECTED="401" - case "${STATUS}" in 401) VALID=true ;; *) VALID=false ;; esac - else - EXPECTED="400 or 422" - case "${STATUS}" in 400|422) VALID=true ;; *) VALID=false ;; esac - fi - - if [ "${VALID}" != true ]; then - echo "::error::${route} returned HTTP ${STATUS}; expected ${EXPECTED} for an intentionally invalid unauthenticated request." - head -c 1000 "${RESPONSE_FILE}" || true - rm -f "${RESPONSE_FILE}" - exit 1 - fi - - echo "${route}: mounted (HTTP ${STATUS})" - rm -f "${RESPONSE_FILE}" - done # Reviewed N0-only snapshot, not evergreen quota policy. Same deploy job # means no second environment approval or watcher-scheduling gap. # Future restarts require refreshed config and operator authorization. @@ -1009,6 +961,7 @@ jobs: CONFIG_PATH = Path("ops/quota-overrides.json") LIMITS = {"applications": 5368709120, "secrets": 268721121} + SNAPSHOT_OWNER = "tinycloud:pkh:eip155:1:0xd559CCd9EB87c530A9a349262669386dE93cf412" class QuotaError(Exception): pass @@ -1026,6 +979,7 @@ jobs: raise ValueError limit = row["limit_bytes"] if (not match or type(limit) is not int or limit != LIMITS[match[1]] + or space != f"{SNAPSHOT_OWNER}:{match[1]}" or not isinstance(row["_comment"], str) or not row["_comment"]): raise ValueError names.add(match[1]) @@ -1117,6 +1071,57 @@ jobs: raise SystemExit(main()) PY + # TC-465: Share requires the complete Policy/v3 control plane before it + # can create or receive an exact-email share. A version-only check let a + # pre-TC-405 image look healthy while every required route returned 404. + # Empty JSON is deliberately rejected before any handler can mutate + # policy state, so this proves route mounting against the direct TEE + # origin without creating a binding, challenge, policy, or delegation. + - name: Verify deployed Policy/v3 routes + run: | + set -euo pipefail + + NODE_ORIGIN="https://tee.node.tinycloud.xyz" + ROUTES=( + /policy/v3/enforcer-bindings + /policy/v3/policies + /policy/v3/challenges + /policy/v3/delegations + /policy/v3/deliveries/authorize + ) + + for route in "${ROUTES[@]}"; do + RESPONSE_FILE="$(mktemp)" + STATUS="$(curl -sS --max-time 15 -o "${RESPONSE_FILE}" -w '%{http_code}' \ + -X POST \ + -H 'accept: application/json' \ + -H 'content-type: application/json' \ + --data '{}' \ + "${NODE_ORIGIN}${route}")" + + # Delivery is authenticated before request-body validation, so its + # intentionally unsigned probe must be exactly 401. Other + # control-plane routes reject the body first (400 or 422). Neither + # case may silently accept a missing route or a server failure. + if [ "${route}" = "/policy/v3/deliveries/authorize" ]; then + EXPECTED="401" + case "${STATUS}" in 401) VALID=true ;; *) VALID=false ;; esac + else + EXPECTED="400 or 422" + case "${STATUS}" in 400|422) VALID=true ;; *) VALID=false ;; esac + fi + + if [ "${VALID}" != true ]; then + echo "::error::${route} returned HTTP ${STATUS}; expected ${EXPECTED} for an intentionally invalid unauthenticated request." + head -c 1000 "${RESPONSE_FILE}" || true + rm -f "${RESPONSE_FILE}" + exit 1 + fi + + echo "${route}: mounted (HTTP ${STATUS})" + rm -f "${RESPONSE_FILE}" + done + - name: Report unverified storage overrides if: >- ${{ always() && steps.deploy.outputs.cvm_mutation_attempted == 'true' && steps.quotas.outcome != 'success' }} diff --git a/scripts/check-deployment-policy-probes.mjs b/scripts/check-deployment-policy-probes.mjs index 932c055f..29dac8fe 100644 --- a/scripts/check-deployment-policy-probes.mjs +++ b/scripts/check-deployment-policy-probes.mjs @@ -1,5 +1,10 @@ -import { readFileSync } from "node:fs"; +import assert from "node:assert/strict"; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { spawnSync } from "node:child_process"; +const workflow = readFileSync(".github/workflows/docker.yml", "utf8"); const expectedRoutes = [ "/policy/v3/enforcer-bindings", "/policy/v3/policies", @@ -7,44 +12,98 @@ const expectedRoutes = [ "/policy/v3/delegations", "/policy/v3/deliveries/authorize", ]; -const workflow = readFileSync(".github/workflows/docker.yml", "utf8"); -const source = readFileSync("tinycloud-node-server/src/policy_v3.rs", "utf8"); -const probeStart = workflow.indexOf("- name: Verify deployed Policy/v3 routes"); -const probeEnd = workflow.indexOf("\n - name:", probeStart + 1); -if (probeStart === -1) { - throw new Error("Policy/v3 deployment probe step is missing or malformed"); +function shellForStep(name) { + const lines = workflow.split("\n"); + const start = lines.indexOf(` - name: ${name}`); + assert(start >= 0, `missing deployment check: ${name}`); + const run = lines.indexOf(" run: |", start); + assert(run > start, `missing deployment script: ${name}`); + const body = []; + for (const line of lines.slice(run + 1)) { + if (line && !line.startsWith(" ")) break; + body.push(line.slice(10)); + } + return body.join("\n"); } -const probe = workflow.slice(probeStart, probeEnd === -1 ? undefined : probeEnd); -const routesBlock = probe.match(/ROUTES=\(\n([\s\S]*?)\n \)/)?.[1]; -const routes = routesBlock?.match(/^\s+(\/\S+)$/gm)?.map((route) => route.trim()); +const directory = mkdtempSync(join(tmpdir(), "tinycloud-deploy-probes-")); +const logPath = join(directory, "requests.jsonl"); +const mockCurl = `#!${process.execPath} +const fs = require("node:fs"); +const args = process.argv.slice(2); +const url = new URL(args.at(-1)); +if (url.origin !== "https://tee.node.tinycloud.xyz") process.exit(90); +const valueAfter = flag => args[args.indexOf(flag) + 1]; +const post = args.includes("-X") && valueAfter("-X") === "POST"; +const headers = args.flatMap((arg, index) => arg === "-H" ? [args[index + 1].toLowerCase()] : []); +const safeProbe = post && valueAfter("--data") === "{}" && + headers.includes("accept: application/json") && headers.includes("content-type: application/json") && + !headers.some(header => header.startsWith("authorization:")); +fs.appendFileSync(process.env.PROBE_LOG, JSON.stringify({path:url.pathname, safeProbe}) + "\\n"); +if (url.pathname === "/version") { + if (process.env.TLS_UNREADY === "true") process.exit(22); + console.log(JSON.stringify({version:process.env.READY_VERSION})); +} else { + if (!safeProbe) process.exit(91); + const statuses = JSON.parse(process.env.PROBE_STATUSES); + const status = statuses[url.pathname] ?? 404; + if (args.includes("-o")) fs.writeFileSync(valueAfter("-o"), "{}"); + process.stdout.write(String(status)); +} +`; +const mockJq = `#!${process.execPath} +let input = ""; +process.stdin.on("data", chunk => input += chunk); +process.stdin.on("end", () => { + try { console.log(JSON.parse(input).version); } catch { process.exitCode = 1; } +}); +`; +let cases = 0; -if (JSON.stringify(routes) !== JSON.stringify(expectedRoutes)) { - throw new Error(`unexpected Policy/v3 deployment probe routes: ${JSON.stringify(routes)}`); +function run(name, statuses, success, options = {}) { + writeFileSync(logPath, ""); + const result = spawnSync("bash", ["-e", "-o", "pipefail", "-c", shellForStep(name)], { + env: { + PATH: `${directory}:${process.env.PATH}`, + EXPECTED: "1.20.6", + PROBE_LOG: logPath, + PROBE_STATUSES: JSON.stringify(statuses), + READY_VERSION: options.version ?? "1.20.6", + TLS_UNREADY: options.tlsUnready ? "true" : "false", + }, + encoding: "utf8", + timeout: 15000, + }); + // Do not forward captured output: workflow errors may contain server data. + assert.equal(result.error, undefined, "synthetic probe execution failed"); + assert.equal(result.status === 0, success, "deployment probe accepted/rejected the wrong response"); + cases++; + return readFileSync(logPath, "utf8").trim().split("\n").filter(Boolean).map(line => JSON.parse(line)); } -for (const route of expectedRoutes) { - if (!source.includes(`#[post("${route}"`)) { - throw new Error(`deployment probe route is not a Node-owned POST route: ${route}`); + +try { + writeFileSync(join(directory, "curl"), mockCurl, { mode: 0o700 }); + writeFileSync(join(directory, "jq"), mockJq, { mode: 0o700 }); + writeFileSync(join(directory, "sleep"), "#!/bin/sh\nexit 0\n", { mode: 0o700 }); + const defaults = Object.fromEntries(expectedRoutes.map(route => [route, route.endsWith("/authorize") ? 401 : 400])); + for (const general of [400, 422]) { + const statuses = Object.fromEntries(expectedRoutes.map(route => [route, route.endsWith("/authorize") ? 401 : general])); + const requests = run("Verify deployed Policy/v3 routes", statuses, true); + assert.deepEqual(new Set(requests.map(request => request.path)), new Set(expectedRoutes)); + assert.equal(requests.length, expectedRoutes.length, "each required route must be probed exactly once"); + assert(requests.every(request => request.safeProbe), "probe must be unsigned POST with intentionally empty JSON"); } -} -if (probe.includes("/share/")) { - throw new Error("deployment probe must not use a retired /share/* route"); -} -for (const invariant of [ - 'NODE_ORIGIN="https://tee.node.tinycloud.xyz"', - '"${NODE_ORIGIN}/version"', - "--data '{}'", - 'if [ "${route}" = "/policy/v3/deliveries/authorize" ]', - 'EXPECTED="401"', - 'case "${STATUS}" in 401) VALID=true', - 'EXPECTED="400 or 422"', - 'case "${STATUS}" in 400|422) VALID=true', -]) { - if (!probe.includes(invariant)) { - throw new Error(`Policy/v3 deployment probe lost required invariant: ${invariant}`); + for (const bad of [200, 400, 422, 500]) { + run("Verify deployed Policy/v3 routes", { ...defaults, [expectedRoutes.at(-1)]: bad }, false); } -} -if (/400\|401|401\|4|"\$\{STATUS\}" = "401"/.test(probe)) { - throw new Error("the unsigned delivery probe must require exactly HTTP 401"); + for (const bad of [200, 401, 404, 500]) { + run("Verify deployed Policy/v3 routes", { ...defaults, [expectedRoutes[0]]: bad }, false); + } + run("Verify deployed ingress health", {}, true); + run("Verify deployed ingress health", {}, false, { version: "1.20.5" }); + run("Verify deployed ingress health", {}, false, { tlsUnready: true }); + console.log(`Deployment probes: ${cases} synthetic response scenarios passed; no network requests.`); +} finally { + rmSync(directory, { recursive: true, force: true }); } diff --git a/scripts/ci/test-quota-overrides.py b/scripts/ci/test-quota-overrides.py index a301f2b4..a9e75d64 100755 --- a/scripts/ci/test-quota-overrides.py +++ b/scripts/ci/test-quota-overrides.py @@ -26,10 +26,30 @@ NAMESPACE = {"__name__": "quota_workflow_test"} exec(compile(textwrap.dedent(SCRIPT), str(WORKFLOW) + ":quota-restoration", "exec"), NAMESPACE) -APPLICATIONS = "tinycloud:pkh:eip155:1:0x" + "1" * 40 + ":applications" -SECRETS = "tinycloud:pkh:eip155:1:0x" + "1" * 40 + ":secrets" +APPLICATIONS = "tinycloud:pkh:eip155:1:0xd559CCd9EB87c530A9a349262669386dE93cf412:applications" +SECRETS = "tinycloud:pkh:eip155:1:0xd559CCd9EB87c530A9a349262669386dE93cf412:secrets" OVERRIDES = {APPLICATIONS: 5368709120, SECRETS: 268721121} CANARIES = [] +DIGEST = "sha256:4bdae4fc232d886bb56bfbce4ec62ed4b8fb4e1369a01f6afb73d06dd68c188f" +N0_CONTEXT = { + "github.event_name": "workflow_dispatch", "github.ref": "refs/heads/main", + "inputs.deploy_phala": True, "inputs.deploy_image_digest": DIGEST, + "inputs.restore_n0_quota_snapshot": True, "job.success": True, + "needs.promote-watch.result": "success", + "steps.pin.outputs.image": "ghcr.io/tinycloudlabs/tinycloud-node@" + DIGEST, + "steps.pin.outputs.version": "1.20.6", "steps.pin.outputs.image_label": "1.20.6-dstack", + "steps.pin.outputs.revision": "e18578ff9162986ddd813259c60f73866e3e60d1", +} + + +def shell_for_step(name): + block = SOURCE.split(" - name: " + name + "\n", 1)[1] + lines = [] + for line in block.split(" run: |\n", 1)[1].splitlines(): + if line and not line.startswith(" "): + break + lines.append(line) + return textwrap.dedent("\n".join(lines)) class SecretSafeResult(unittest.TextTestResult): @@ -37,6 +57,8 @@ def _exc_info_to_string(self, error, test): output = super()._exc_info_to_string(error, test) for canary in CANARIES: output = output.replace(canary, "[sealed canary withheld]") + for space in OVERRIDES: + output = output.replace(space, "[space id withheld]") return output @@ -241,9 +263,9 @@ def test_usage_inventory_must_prove_both_targets_before_any_put(self): self.assertTrue(all(request[0] == "GET" for request in self.requests), "No target may be mutated before both identities are proved") - def test_checked_in_config_has_reviewed_absolute_limits(self): + def test_checked_in_config_matches_verified_full_ids_and_limits(self): overrides = NAMESPACE["load_overrides"](WORKFLOW.parents[2] / "ops/quota-overrides.json") - self.assertEqual(sorted(overrides.values()), sorted(OVERRIDES.values())) + self.assertTrue(overrides == OVERRIDES, "Reviewed canonical target/value contract mismatch") def test_inventory_failure_never_dumps_raw_space_rows(self): self.usage_rows = [{"space_id": self.secret, "usage_bytes": 10}] @@ -261,17 +283,8 @@ def test_inventory_failure_never_dumps_raw_space_rows(self): self.assertFalse(self.secret in output.getvalue() + summary.read_text(), "Raw inventory leaked") def test_snapshot_requires_explicit_n0_authorization_and_exact_image(self): - digest = "sha256:4bdae4fc232d886bb56bfbce4ec62ed4b8fb4e1369a01f6afb73d06dd68c188f" - selected = { - "github.event_name": "workflow_dispatch", "github.ref": "refs/heads/main", - "inputs.deploy_phala": True, "inputs.deploy_image_digest": digest, - "inputs.restore_n0_quota_snapshot": True, - "job.success": True, "needs.promote-watch.result": "success", - "steps.pin.outputs.image": "ghcr.io/tinycloudlabs/tinycloud-node@" + digest, - "steps.pin.outputs.version": "1.20.6", - "steps.pin.outputs.image_label": "1.20.6-dstack", - "steps.pin.outputs.revision": "e18578ff9162986ddd813259c60f73866e3e60d1", - } + digest = DIGEST + selected = N0_CONTEXT self.assertTrue(quota_restore_enabled(selected)) self.assertTrue(quota_restore_enabled({**selected, "needs.promote-watch.result": "failure"}), "Watch scheduling must not block post-restart quota restoration") @@ -291,6 +304,30 @@ def test_snapshot_requires_explicit_n0_authorization_and_exact_image(self): self.assertFalse(quota_restore_enabled({**selected, **changes}), "Unrelated deploy must not replay the reviewed snapshot") + def test_policy_probe_failure_cannot_skip_verified_quota_restoration(self): + restored = False + healthy = True + names = ["Restore storage quota overrides", "Verify deployed Policy/v3 routes"] + names.sort(key=lambda name: SOURCE.index(" - name: " + name + "\n")) + with tempfile.TemporaryDirectory() as directory: + curl = Path(directory) / "curl" + curl.write_text("#!/bin/sh\nprintf '500'\n") + curl.chmod(0o700) + for name in names: + if name == "Restore storage quota overrides": + if quota_restore_enabled({**N0_CONTEXT, "job.success": healthy}): + self.apply() + restored = True + else: + result = subprocess.run(["bash", "-e", "-c", shell_for_step(name)], + env={"PATH": directory + ":/usr/bin:/bin"}, + capture_output=True, text=True) + self.assertNotEqual(result.returncode, 0, "Synthetic Policy probe must fail") + healthy = False + self.assertFalse(healthy) + self.assertTrue(restored, "Policy failure after restart must not prevent quota restoration") + self.assertEqual(sorted(self.values.values()), sorted(OVERRIDES.values())) + def test_partial_restart_failure_is_explicitly_unverified(self): report = SOURCE.split(" - name: Report unverified storage overrides\n", 1)[1] condition = report.split(" if: >-\n", 1)[1].split("\n run:", 1)[0].strip()[3:-2] @@ -305,14 +342,10 @@ def test_partial_restart_failure_is_explicitly_unverified(self): "steps.deploy.outputs.cvm_mutation_attempted": attempted, "steps.quotas.outcome": outcome, }), enabled) - lines = [] - for line in report.split(" run: |\n", 1)[1].splitlines(): - if line and not line.startswith(" "): - break - lines.append(line) + script = shell_for_step("Report unverified storage overrides") with tempfile.TemporaryDirectory() as directory: summary = Path(directory) / "summary" - result = subprocess.run(["bash", "-e", "-c", textwrap.dedent("\n".join(lines))], + result = subprocess.run(["bash", "-e", "-c", script], env={"GITHUB_STEP_SUMMARY": str(summary), "QUOTA_ADMIN_SECRET": self.secret}, capture_output=True, text=True) self.assertNotEqual(result.returncode, 0) @@ -320,6 +353,25 @@ def test_partial_restart_failure_is_explicitly_unverified(self): self.assertFalse(self.secret in result.stdout + result.stderr + summary.read_text(), "Partial-failure diagnostics must not expose authorization") + def test_positive_usage_wrong_owner_config_fails_before_any_request(self): + other = "tinycloud:pkh:eip155:1:0x" + "1" * 40 + wrong = {other + ":applications": 5368709120, other + ":secrets": 268721121} + self.usage_rows = [{"space_id": space, "usage_bytes": 10} for space in wrong] + with tempfile.TemporaryDirectory() as directory: + config = Path(directory) / "overrides.json" + summary = Path(directory) / "summary" + config.write_text(json.dumps(config_document(wrong))) + output = io.StringIO() + with patch.dict(NAMESPACE["os"].environ, { + "QUOTA_ADMIN_SECRET": self.secret, "GITHUB_STEP_SUMMARY": str(summary), + }), patch.dict(NAMESPACE, {"CONFIG_PATH": config}): + with contextlib.redirect_stdout(output), contextlib.redirect_stderr(output): + self.assertEqual(NAMESPACE["main"](), 1) + self.assertFalse(bool(self.requests), "Wrong-owner config must fail before any request") + logs = output.getvalue() + summary.read_text() + self.assertFalse(any(space in logs for space in [*wrong, *OVERRIDES])) + self.assertFalse(self.secret in logs, "Sealed canary leaked") + if __name__ == "__main__": unittest.main(testRunner=unittest.TextTestRunner(resultclass=SecretSafeResult, verbosity=2)) From fd1b1893c9b874be518397697c0daa3b899ea574 Mon Sep 17 00:00:00 2001 From: Sam Gbafa Date: Fri, 16 Oct 2026 20:19:30 +0000 Subject: [PATCH 3/4] ci: enforce strict shell baseline and expose pending compose gate --- .github/workflows/docker.yml | 3 + scripts/ci/test-quota-overrides.py | 131 +++++++++++++++++++++++++++++ 2 files changed, 134 insertions(+) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index edc57197..9dcdcf3b 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -949,6 +949,7 @@ jobs: # Repository-scoped; never read by an agent or passed in argv. QUOTA_ADMIN_SECRET: ${{ secrets.PROD_TINYCLOUD_ADMIN_SECRET }} run: | + set -euo pipefail set +x python3 - <<'PY' import http.client @@ -1126,6 +1127,7 @@ jobs: if: >- ${{ always() && steps.deploy.outputs.cvm_mutation_attempted == 'true' && steps.quotas.outcome != 'success' }} run: | + set -euo pipefail MESSAGE="OVERRIDES UNVERIFIED: a CVM mutation was attempted without successful reviewed quota restoration. Authorized operator must resolve via TC-1161 before another dispatch." echo "::error::${MESSAGE}" printf '%s\n' "${MESSAGE}" >> "$GITHUB_STEP_SUMMARY" @@ -1153,6 +1155,7 @@ jobs: IMAGE_LABEL: ${{ needs.deploy-phala.outputs.image_label }} IMAGE_REVISION: ${{ needs.deploy-phala.outputs.revision }} run: | + set -euo pipefail if [ -n "${IMAGE}" ]; then echo "Promote watch for ${IMAGE_LABEL} (version ${IMAGE_VERSION}) revision ${IMAGE_REVISION}: ${IMAGE}" else diff --git a/scripts/ci/test-quota-overrides.py b/scripts/ci/test-quota-overrides.py index a9e75d64..8832c873 100755 --- a/scripts/ci/test-quota-overrides.py +++ b/scripts/ci/test-quota-overrides.py @@ -353,6 +353,137 @@ def test_partial_restart_failure_is_explicitly_unverified(self): self.assertFalse(self.secret in result.stdout + result.stderr + summary.read_text(), "Partial-failure diagnostics must not expose authorization") + def test_new_shell_steps_enforce_strict_runtime_baseline(self): + probe = """ + require_strict_shell() { + for option in errexit nounset pipefail; do + case ":${SHELLOPTS}:" in + *:"${option}":*) ;; + *) builtin printf 'missing-%s\\n' "${option}" >> "$SHELL_PROBE"; return 89 ;; + esac + done + builtin printf 'strict\\n' >> "$SHELL_PROBE" + } + python3() { require_strict_shell; } + gh() { require_strict_shell; } + printf() { require_strict_shell || return; builtin printf "$@"; } + """ + for name in ("Restore storage quota overrides", "Report unverified storage overrides", + "Dispatch Write Probe promote watch"): + with self.subTest(step=name), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + startup = root / "bash-env" + trace = root / "probe" + summary = root / "summary" + startup.write_text(textwrap.dedent(probe)) + result = subprocess.run( + ["bash", "--noprofile", "--norc", "-c", shell_for_step(name)], + env={"PATH": "/usr/bin:/bin", "BASH_ENV": str(startup), "SHELL_PROBE": str(trace), + "GITHUB_STEP_SUMMARY": str(summary), "QUOTA_ADMIN_SECRET": self.secret, + "IMAGE": N0_CONTEXT["steps.pin.outputs.image"], "IMAGE_VERSION": "1.20.6", + "IMAGE_LABEL": "1.20.6-dstack", + "IMAGE_REVISION": N0_CONTEXT["steps.pin.outputs.revision"], + "GITHUB_REPOSITORY": "synthetic/repository"}, + capture_output=True, text=True) + logs = result.stdout + result.stderr + (summary.read_text() if summary.exists() else "") + self.assertFalse(self.secret in logs, "Shell diagnostics leaked a sealed canary") + self.assertEqual(trace.read_text().splitlines(), ["strict"], + "Each new shell block must enforce all three runtime options") + if name == "Report unverified storage overrides": + self.assertNotEqual(result.returncode, 0) + self.assertIn("OVERRIDES UNVERIFIED", summary.read_text()) + else: + self.assertEqual(result.returncode, 0) + + def test_pending_reviewed_compose_binding_rejects_same_image_other_configuration(self): + # Intentionally RED until #309 hosted acceptance and Amos's exact-head + # review authorize its compose bytes. No candidate/caller hash is trusted. + boundary = """#!/usr/bin/env python3 +import os +from pathlib import Path +import sys +tool = Path(sys.argv[0]).name +args = sys.argv[1:] +if tool == "docker": + if args[:1] == ["pull"]: + assert args[1:3] == ["--platform", "linux/amd64"] + elif args[:2] == ["image", "inspect"]: + label = args[-1] + if "org.opencontainers.image.revision" in label: + print(os.environ["ROLLBACK_REVISION"]) + elif "org.opencontainers.image.version" in label: + print("1.20.6-dstack") + else: + raise SystemExit("Unexpected synthetic label lookup") + elif args[:1] == ["run"]: + assert args[-1] == "--validate-config" and "none" in args and "--read-only" in args + elif args[:1] == ["compose"]: + assert args[-2:] == ["config", "--quiet"] + else: + raise SystemExit("Unexpected synthetic Docker command") +elif tool == "git": + assert args == ["rev-parse", "HEAD"] + print("f" * 40) +elif tool == "node": + assert args[0].endswith("/phala-env-sync/sync.mjs") and args[1:] in ([], ["--check"]) +elif tool == "phala": + assert args[:3] == ["deploy", "--cvm-id", "tinycloud-node"] + assert "--no-public-logs" in args and "--no-public-sysinfo" in args + assert Path(".phala-prod.env").stat().st_mode & 0o777 == 0o600 + with open(os.environ["MUTATION_TRACE"], "a") as output: + output.write("cvm-mutation\\n") +else: + raise SystemExit("Unexpected synthetic executable") +""" + deploy = shell_for_step("Deploy to Phala Cloud") + required = deploy.split("required_vars=(\n", 1)[1].split("\n)", 1)[0].split() + steps = [shell_for_step(name) for name in ( + "Validate image version against Cargo.toml", "Validate and pin deployment image", + "Preflight sealed runtime configuration", "Deploy to Phala Cloud")] + for configuration in ("alpha", "beta"): + with self.subTest(configuration=configuration), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + binaries = root / "bin" + binaries.mkdir() + for tool in ("docker", "git", "node", "phala"): + executable = binaries / tool + executable.write_text(boundary) + executable.chmod(0o700) + (root / "tinycloud-node-server").mkdir() + (root / "tinycloud-node-server/Cargo.toml").write_text('version = "1.20.7"\n') + (root / "dstack-authorized-keys").write_text("synthetic-public-key\n") + (root / "docker-compose.dstack-postgres.yaml").write_text( + "services:\n tinycloud:\n image: " + N0_CONTEXT["steps.pin.outputs.image"] + + "\n environment:\n SYNTHETIC_CONFIG: " + configuration + "\n") + trace = root / "mutation-trace" + summary = root / "summary" + output = root / "output" + env = { + "PATH": str(binaries) + ":/usr/bin:/bin", "RUNNER_TEMP": directory, + "EVENT_NAME": "workflow_dispatch", "GITHUB_REF": "refs/heads/main", + "GITHUB_REF_TYPE": "branch", "GITHUB_REF_NAME": "main", "RELEASE_TAG": "", + "INPUT_IMAGE_VERSION": "1.20.6", "INPUT_DEPLOY_DIGEST": DIGEST, + "INPUT_DEPLOY_PHALA": "true", "N0_QUOTA_SNAPSHOT": "true", + "INCLUDE_DUCKDB": "false", "INCLUDE_TC_BENCH_V1": "false", + "ROLLBACK_VERSION": "1.20.6", + "ROLLBACK_REVISION": N0_CONTEXT["steps.pin.outputs.revision"], + "DEPLOY_VERSION": "1.20.6", "BUILD_DIGEST": "", "REGISTRY": "ghcr.io", + "IMAGE_NAME": "tinycloudlabs/tinycloud-node", "MUTATION_TRACE": str(trace), + "GITHUB_OUTPUT": str(output), "GITHUB_STEP_SUMMARY": str(summary), + **{name: self.secret for name in required}, + } + result = subprocess.run(["bash", "--noprofile", "--norc", "-e", "-c", "\n".join(steps)], + cwd=root, env=env, capture_output=True, text=True) + logs = result.stdout + result.stderr + logs += summary.read_text() if summary.exists() else "" + logs += output.read_text() if output.exists() else "" + self.assertFalse(self.secret in logs, "Synthetic workflow leaked a sealed canary") + self.assertFalse((root / ".phala-prod.env").exists(), "Sealed temp file was not removed") + self.assertFalse(trace.exists(), + "PENDING reviewed #309 compose binding: unreviewed config reached CVM mutation") + self.assertNotEqual(result.returncode, 0, + "Same immutable image must not authorize unrelated compose bytes") + def test_positive_usage_wrong_owner_config_fails_before_any_request(self): other = "tinycloud:pkh:eip155:1:0x" + "1" * 40 wrong = {other + ":applications": 5368709120, other + ":secrets": 268721121} From 87b2a38b2ad523bf6587e3c08ef1e072138b56e7 Mon Sep 17 00:00:00 2001 From: Sam Gbafa Date: Fri, 16 Oct 2026 21:09:48 +0000 Subject: [PATCH 4/4] ci: bind N0 promotion to approved compose bytes --- .github/workflows/docker.yml | 16 +- scripts/ci/fixtures/tc959-n0-compose.yaml | 177 ++++++++++++++++++++++ scripts/ci/test-quota-overrides.py | 76 ++++++++-- 3 files changed, 254 insertions(+), 15 deletions(-) create mode 100644 scripts/ci/fixtures/tc959-n0-compose.yaml diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 9dcdcf3b..a401dacc 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -594,13 +594,22 @@ jobs: exit 1 fi python3 - <<'PY' + import hashlib import os import re from pathlib import Path path = Path('docker-compose.dstack-postgres.yaml') - content = path.read_text() + raw = path.read_bytes() + content = raw.decode('utf-8') native = 'ghcr.io/tinycloudlabs/tinycloud-node' reviewed = native + '@sha256:4bdae4fc232d886bb56bfbce4ec62ed4b8fb4e1369a01f6afb73d06dd68c188f' + n0_configuration = os.environ['RESOLVED_IMAGE'] == reviewed + if n0_configuration: + # #309 head 44eeb20a1461ef66b5af492e3be4c42be2728b67. + # Approved: https://github.com/TinyCloudLabs/tinycloud-node/pull/309#issuecomment-6224650050 + expected = 'ca110b0bae6d5a06a1002fb80df3f71cffec11659c9f9dbfa5d72624735546f6' + if hashlib.sha256(raw).hexdigest() != expected: + raise SystemExit('::error::N0 promotion requires the exact reviewed compose bytes; deployment not attempted') images = [ image for image in re.finditer(r'(?m)^([ \t]+image:[ \t]+)([^\n]+)$', content) if image[2].strip().strip('"\'').startswith((native + ':', native + '@')) @@ -610,7 +619,9 @@ jobs: image = images[0] if image[2].strip().strip('"\'') == reviewed and reviewed != os.environ['RESOLVED_IMAGE']: raise SystemExit('Reviewed compose digest does not match the selected deployment image') - path.write_text(content[:image.start(2)] + os.environ['RESOLVED_IMAGE'] + content[image.end(2):]) + path.write_bytes((content[:image.start(2)] + os.environ['RESOLVED_IMAGE'] + content[image.end(2):]).encode('utf-8')) + with open(os.environ['GITHUB_OUTPUT'], 'a') as output: + output.write(f'n0_configuration_verified={str(n0_configuration).lower()}\n') PY { echo "image=${RESOLVED_IMAGE}" @@ -943,6 +954,7 @@ jobs: ${{ success() && github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main' && inputs.deploy_phala && inputs.restore_n0_quota_snapshot && inputs.deploy_image_digest == 'sha256:4bdae4fc232d886bb56bfbce4ec62ed4b8fb4e1369a01f6afb73d06dd68c188f' && steps.pin.outputs.image == 'ghcr.io/tinycloudlabs/tinycloud-node@sha256:4bdae4fc232d886bb56bfbce4ec62ed4b8fb4e1369a01f6afb73d06dd68c188f' && + steps.pin.outputs.n0_configuration_verified == 'true' && steps.pin.outputs.version == '1.20.6' && steps.pin.outputs.image_label == '1.20.6-dstack' && steps.pin.outputs.revision == 'e18578ff9162986ddd813259c60f73866e3e60d1' }} env: diff --git a/scripts/ci/fixtures/tc959-n0-compose.yaml b/scripts/ci/fixtures/tc959-n0-compose.yaml new file mode 100644 index 00000000..d661d530 --- /dev/null +++ b/scripts/ci/fixtures/tc959-n0-compose.yaml @@ -0,0 +1,177 @@ +# dstack deployment - Topology 2: Postgres + S3 +# tinycloud-node in the CVM, Postgres external. +# Database credentials encrypted by dstack KMS. +# Column encryption enabled for sensitive data. +# +# All ${...} variables should be encrypted using dstack's KMS public key +# before deployment. The operator never sees plaintext values. + +services: + tinycloud: + # N0: unchanged production 1.20.6 binary; this is not a new app release. + image: ghcr.io/tinycloudlabs/tinycloud-node@sha256:4bdae4fc232d886bb56bfbce4ec62ed4b8fb4e1369a01f6afb73d06dd68c188f + restart: unless-stopped + labels: + tc.logship: tinycloud-node + # Host Docker daemon sends GELF; the app never joins Vector's network. + # Nonblocking UDP can lose logs. No raw Docker cache or /logs fallback. + logging: + driver: gelf + options: + gelf-address: udp://127.0.0.1:12201 + gelf-compression-type: none + labels: tc.logship + mode: non-blocking + max-buffer-size: 4m + cache-disabled: "true" + ports: + - "8000:8000" + volumes: + - /var/run/dstack.sock:/var/run/dstack.sock + environment: + TINYCLOUD_KEYS_TYPE: Dstack + TINYCLOUD_STORAGE_DATABASE: "${DATABASE_URL}" + TINYCLOUD_STORAGE_BLOCKS_TYPE: S3 + TINYCLOUD_STORAGE_BLOCKS_BUCKET: "${S3_BUCKET}" + TINYCLOUD_STORAGE_BLOCKS_ENDPOINT: "${S3_ENDPOINT}" + AWS_ACCESS_KEY_ID: "${AWS_KEY}" + AWS_SECRET_ACCESS_KEY: "${AWS_SECRET}" + AWS_DEFAULT_REGION: "${AWS_REGION}" + TINYCLOUD_LOG_LEVEL: normal + TINYCLOUD_LOG__FORMAT: Json + # Enable Prometheus telemetry (metrics, span histograms, pool gauges). + # The metrics handler is intentionally NOT gateway-exposed: port 8001 is + # not published here, so the unauthenticated /metrics endpoint stays + # internal to the CVM. + TINYCLOUD_TELEMETRY__ENABLED: "true" + # The managed Postgres instance allows max_connections = 25 (measured + # 2026-07-29, with some of those reserved for superuser roles). The pool + # default of 100 cannot be satisfied: the node opened and held slots + # until the instance ran out, and further requests failed with + # "Failed to acquire connection from pool: Connection pool timed out" + # after the 30s sqlx default. Keep this below the instance limit with + # headroom for admin/psql access, and raise it only alongside the + # instance's own max_connections. + TINYCLOUD_DATABASE__MAX_CONNECTIONS: "15" + # Required for every manual TC-780 deploy. Keep true through offline + # verification; only the final, explicitly reviewed redeploy sets false. + TINYCLOUD_DATABASE__WRITE_FENCE: "${SQL_IDENTITY_FENCE:?set true during cutover, false after verification}" + TINYCLOUD_CORS: "true" + # --- addressed sharing (TC-397) ------------------------------------ + # + # The public trust document shared with Share and OpenCredentials, + # delivered inline because nothing else can reach this container: the + # image is `FROM scratch` (no shell, so share-api's decode-to-tmpfs + # entrypoint cannot work here) and a dstack deployment uploads only + # this compose file (so there is no host path to bind-mount from). + # + # Seal the base64 of the reviewed trust-bundle.production.json — the + # same bytes share-api serves — into the CVM's encrypted environment as + # SHARE_TRUST_BUNDLE_BASE64. Same variable name and same wire form + # share-api already uses, so one sealed value feeds both services and + # cannot drift between them. + # + # The `:?` form is deliberate: compose refuses to interpolate a partial + # deployment, failing loudly at deploy time rather than letting the node + # die at boot with "share email trust bundle is required". + TINYCLOUD_SHARE_EMAIL__ENABLED: "true" + TINYCLOUD_SHARE_EMAIL__TRUST_BUNDLE_BASE64: "${SHARE_TRUST_BUNDLE_BASE64:?seal the reviewed share-email trust bundle, base64-encoded}" + # The sealed DATABASE_URL already carries sslmode=verify-full as its sole + # query parameter. ROOT_CERT_PATH stays unset so verify-full resolves + # against the compiled-in public root set (TC-363). + TINYCLOUD_SHARE_EMAIL__POSTGRES_TLS__SSLMODE: verify-full + TINYCLOUD_QUOTA_URL: "${TINYCLOUD_QUOTA_URL}" + TINYCLOUD_ADMIN_SECRET: "${TINYCLOUD_ADMIN_SECRET}" + ROCKET_ADDRESS: "0.0.0.0" + ROCKET_MAX_BLOCKING: "1024" + + dstack-ingress: + image: ghcr.io/tinycloudlabs/tinycloud-node-ingress@sha256:63c1c44e2ded5b73a5f174807b91ca677826974df0f9ab3045f6ec87865a5744 + ports: + - "443:443" + depends_on: + - tinycloud + environment: + - DOMAIN=tee.node.tinycloud.xyz + - TARGET_ENDPOINT=http://tinycloud:8000 + - CLOUDFLARE_API_TOKEN=${CLOUDFLARE_API_TOKEN} + # CNAME target the ingress publishes for DOMAIN. Must not start with `_`: + # Android's resolver (bionic res_hnok) rejects a CNAME target whose first + # label is `_`, so `_.` is unresolvable on Android (TC-513). + # `gateway.` is the target Phala Cloud advertises (gateway.cname) + # and upstream dstack-ingress documents; routing still uses SNI + the + # _dstack-app-address TXT record, so TLS stays end-to-end into the CVM. + - GATEWAY_DOMAIN=gateway.${DSTACK_GATEWAY_DOMAIN} + - CERTBOT_EMAIL=${CERTBOT_EMAIL} + - SET_CAA=true + - DNS_PROVIDER=cloudflare + - CLIENT_MAX_BODY_SIZE=32m + volumes: + - /var/run/dstack.sock:/var/run/dstack.sock + - cert-data:/etc/letsencrypt + restart: unless-stopped + labels: + tc.logship: tinycloud-ingress + logging: + driver: gelf + options: + gelf-address: udp://127.0.0.1:12201 + gelf-compression-type: none + labels: tc.logship + mode: non-blocking + max-buffer-size: 4m + cache-disabled: "true" + + # The dstack gateway's TLS-terminating routes (-.:443) + # only forward to compose-published ports, so guest sshd on :22 is + # unreachable via `phala ssh` even with authorized_keys installed. + # Publish a TCP forwarder on 10022 that relays to guest sshd via the + # docker host gateway. Port 10022 exposes ONLY sshd, which is key-gated. + ssh-gateway: + image: alpine/socat:1.8.1.3@sha256:188fe0a22182f81c16def9d1137930121eaa3023f427164e32c0d2f118f79dd6 + restart: unless-stopped + ports: + - "10022:10022" + extra_hosts: + - "host.docker.internal:host-gateway" + command: TCP-LISTEN:10022,fork,reuseaddr TCP:host.docker.internal:22 + + vector: + restart: unless-stopped + image: timberio/vector:0.59.0-alpine@sha256:0e36bc2a290039e9b6a5e92f04bd2f133223506f72d09682c1df16c015172b7a + init: true + user: 10001:10001 + read_only: true + cap_drop: + - ALL + security_opt: + - no-new-privileges:true + mem_limit: 64m + memswap_limit: 64m + cpus: 0.5 + ports: + - 127.0.0.1:12201:12201/udp + networks: + - logship + tmpfs: + - /var/lib/vector:rw,noexec,nosuid,size=8m,uid=10001,gid=10001,mode=0700 + environment: + TINYCLOUD_LOGSHIP_PROFILE: ${TINYCLOUD_LOGSHIP_PROFILE:-errors_slow} + BETTERSTACK_SOURCE_TOKEN: ${BETTERSTACK_SOURCE_TOKEN:?seal the node source token before enabling log shipping} + VECTOR_CONFIG_JSON: "{\"data_dir\":\"/var/lib/vector\",\"sources\":{\"gelf_in\":{\"type\":\"socket\",\"mode\":\"udp\",\"address\":\"0.0.0.0:12201\",\"max_length\":262144,\"receive_buffer_bytes\":8388608,\"framing\":{\"method\":\"chunked_gelf\",\"chunked_gelf\":{\"decompression\":\"Auto\"}},\"decoding\":{\"codec\":\"bytes\"}}},\"transforms\":{\"gelf_envelope\":{\"type\":\"remap\",\"inputs\":[\"gelf_in\"],\"source\":\"packet = string(.message) ?? \\\"\\\"\\nif length(packet) > 262144 {\\n . = {\\\"reason\\\": \\\"oversized\\\"}\\n} else {\\n outer, err = parse_json(packet)\\n if err != null || !is_object(outer) || (outer.version != \\\"1.1\\\" && outer.version != \\\"1.0\\\") {\\n . = {\\\"reason\\\": \\\"malformed_envelope\\\"}\\n } else if exists(outer.partial_message) || exists(outer._partial_message) {\\n . = {\\\"reason\\\": \\\"partial\\\"}\\n } else {\\n label = get(outer, [\\\"_tc.logship\\\"]) ?? null\\n . = {\\\"_tc.logship\\\": label, \\\"message\\\": outer.short_message}\\n }\\n}\",\"drop_on_error\":true,\"drop_on_abort\":true},\"logship_gate\":{\"type\":\"remap\",\"inputs\":[\"gelf_envelope\"],\"source\":\"if exists(.reason) {\\n . = {\\\"reason\\\": .reason}\\n} else {\\nlabel = string(get(., [\\\"_tc.logship\\\"]) ?? null) ?? \\\"\\\"\\nraw = .message\\nreason = \\\"\\\"\\nif label == \\\"\\\" { reason = \\\"unlabelled\\\" }\\nelse if !includes([\\\"tinycloud-node\\\", \\\"tinycloud-ingress\\\"], label) { reason = \\\"unexpected_label\\\" }\\nelse if exists(.partial_message) { reason = \\\"partial\\\" }\\nelse if !is_string(raw) { reason = \\\"malformed_envelope\\\" }\\nelse if length(string!(raw)) > 65536 { reason = \\\"oversized\\\" }\\nif reason != \\\"\\\" {\\n . = {\\\"reason\\\": reason}\\n} else {\\n parsed, err = parse_json(string!(raw))\\n if err != null || !is_object(parsed) { . = {\\\"reason\\\": \\\"non_json\\\"} }\\n else if exists(parsed.partial_message) { . = {\\\"reason\\\": \\\"partial\\\"} }\\n else {\\n target = string(parsed.target) ?? \\\"\\\"\\n level = downcase(string(parsed.level) ?? \\\"\\\")\\n if label == \\\"tinycloud-node\\\" && !includes([\\\"warn\\\", \\\"error\\\"], level) && target != \\\"tinycloud\\\" && !starts_with(target, \\\"tinycloud::\\\") && target != \\\"tinycloud_core\\\" && !starts_with(target, \\\"tinycloud_core::\\\") {\\n . = {\\\"reason\\\": \\\"unknown_target\\\"}\\n } else if label == \\\"tinycloud-node\\\" && !includes([\\\"warn\\\", \\\"error\\\"], level) && (target != \\\"tinycloud::request\\\" || level != \\\"info\\\") {\\n . = {\\\"reason\\\": \\\"filtered_level\\\"}\\n } else { . = {\\\"label\\\": label, \\\"parsed\\\": parsed} }\\n }\\n}\\n}\",\"drop_on_error\":true,\"drop_on_abort\":true},\"gate_route\":{\"type\":\"route\",\"inputs\":[\"logship_gate\"],\"route\":{\"candidate\":\"exists(.parsed)\",\"rejected\":\"exists(.reason)\"}},\"allowlist\":{\"type\":\"remap\",\"inputs\":[\"gate_route.candidate\"],\"source\":\"# Only this fresh object can leave the process boundary. No envelope, span,\\n# message, error, identity, URI or unknown key is copied into it.\\nj = .parsed\\nlabel = .label\\nout = {}\\nts = \\\"\\\"\\nif label == \\\"tinycloud-node\\\" {\\n target = string(j.target) ?? \\\"\\\"\\n level = downcase(string!(j.level))\\n f = object(j.fields) ?? {}\\n msg = string(f.message) ?? \\\"\\\"\\n ts = string!(j.timestamp)\\n if target == \\\"tinycloud::request\\\" && level == \\\"info\\\" && msg == \\\"request\\\" {\\n method = string!(f.method)\\n if includes([\\\"TRACE\\\", \\\"CONNECT\\\"], method) { method = \\\"\\\" }\\n route = string!(f.route)\\n # Literal Rocket template aliases only; never split an arbitrary URI.\\n aliases = {\\\"/attestation?\\\":\\\"/attestation\\\",\\\"/.well-known/encryption/network/?\\\":\\\"/.well-known/encryption/network/\\\",\\\"/hooks/events?\\\":\\\"/hooks/events\\\",\\\"/hooks/webhooks?\\\":\\\"/hooks/webhooks\\\",\\\"/public//kv?\\\":\\\"/public//kv\\\"}\\n canonical_route = get(aliases, [route]) ?? null\\n if is_string(canonical_route) { route = canonical_route }\\n if !includes([\\\"GET\\\",\\\"HEAD\\\",\\\"POST\\\",\\\"PUT\\\",\\\"PATCH\\\",\\\"DELETE\\\",\\\"OPTIONS\\\",\\\"\\\"], method) || !includes([\\\"/healthz\\\",\\\"/info\\\",\\\"/version\\\",\\\"/invoke\\\",\\\"/delegate\\\",\\\"/revoke\\\",\\\"/attestation\\\",\\\"/signed/kv\\\",\\\"/signed/kv/\\\",\\\"/admin/quota\\\",\\\"/admin/quota/\\\",\\\"/admin/usage\\\",\\\"/hooks/events\\\",\\\"/hooks/tickets\\\",\\\"/hooks/webhooks\\\",\\\"/hooks/webhooks/\\\",\\\"/delegation/query\\\",\\\"/delegation/status\\\",\\\"/encryption/networks\\\",\\\"/encryption/networks/\\\",\\\"/encryption/networks//revoke\\\",\\\"/encryption/networks//decrypt\\\",\\\"/policy/v3/status\\\",\\\"/policy/v3/status/\\\",\\\"/policy/v3/policies\\\",\\\"/policy/v3/challenges\\\",\\\"/policy/v3/delegations\\\",\\\"/policy/v3/enforcer-bindings\\\",\\\"/policy/v3/deliveries/authorize\\\",\\\"/.well-known/tinycloud/node-keys\\\",\\\"/.well-known/encryption/network/\\\",\\\"/public//kv\\\",\\\"/public//kv/\\\",\\\"/peer/generate/\\\",\\\"/evidences/\\\",\\\"/share/v2/readiness\\\",\\\"/share/v2/policies\\\",\\\"/share/v2/policy/challenges\\\",\\\"/share/v2/policy/session\\\",\\\"/share/v2/invoke\\\",\\\"/share/v2/deliveries/authorize\\\",\\\"/<_s..>\\\",\\\"/public/<_space_id>/kv/<_key..>\\\",\\\"\\\"], route) { abort }\\n status = f.status\\n latency = f.latency_us\\n if !is_integer(status) || !is_integer(latency) { abort }\\n status = int!(status)\\n latency = int!(latency)\\n if status < 100 || status > 599 || latency < 0 || latency > 9007199254740991 { abort }\\n out = {\\\"event\\\": \\\"node_request\\\", \\\"svc\\\": \\\"api\\\", \\\"level\\\": \\\"info\\\", \\\"method\\\": method, \\\"route\\\": route, \\\"status\\\": status, \\\"latency_us\\\": latency}\\n } else {\\n if !includes([\\\"warn\\\", \\\"error\\\"], level) { abort }\\n code = \\\"unclassified\\\"\\n if target == \\\"tinycloud\\\" || starts_with(target, \\\"tinycloud::\\\") || target == \\\"tinycloud_core\\\" || starts_with(target, \\\"tinycloud_core::\\\") {\\n codes = {\\\"invocation replay insert failed\\\":\\\"invocation_replay_insert_failed\\\",\\\"invocation replay cleanup failed\\\":\\\"invocation_replay_cleanup_failed\\\",\\\"share-v2 local-tee feature active: deriving a deterministic, non-attested TEE identity from node key material\\\":\\\"share_local_tee_identity\\\",\\\"retention: delivered hook delivery prune failed\\\":\\\"retention_hook_delivery_prune_failed\\\",\\\"retention: dead-letter hook delivery prune failed\\\":\\\"retention_dead_letter_prune_failed\\\",\\\"retention: signed KV ticket prune failed\\\":\\\"retention_signed_ticket_prune_failed\\\",\\\"failed to load tunnel state; tunnel not started\\\":\\\"tunnel_state_load_failed\\\",\\\"invalid loopback API address; tunnel not started\\\":\\\"tunnel_address_invalid\\\",\\\"failed to load link state; LAN listener not started\\\":\\\"link_state_load_failed\\\",\\\"invalid link bind address; LAN listener not started\\\":\\\"link_bind_invalid\\\",\\\"invalid loopback API address; LAN listener not started\\\":\\\"link_address_invalid\\\",\\\"failed to bind link LAN listener; LAN listener not started\\\":\\\"link_bind_failed\\\",\\\"failed to build TLS config; LAN listener not started\\\":\\\"link_tls_config_failed\\\",\\\"link LAN listener exited with error\\\":\\\"link_listener_error\\\",\\\"failed to record link listener bind state\\\":\\\"link_listener_state_write_failed\\\",\\\"failed to record link listener bind failure\\\":\\\"link_bind_state_write_failed\\\",\\\"failed to load config for auto-renew\\\":\\\"link_auto_renew_config_failed\\\",\\\"link auto-renew failed\\\":\\\"link_auto_renew_failed\\\",\\\"failed to hot-reload renewed link cert\\\":\\\"link_cert_reload_failed\\\",\\\"link auto-renew succeeded but no state.json found; skipping cert hot-reload\\\":\\\"link_cert_state_missing\\\",\\\"failed to reload TLS material after link auto-renew\\\":\\\"link_listener_cert_reload_failed\\\",\\\"link disable: service delete failed; local state removed anyway\\\":\\\"link_disable_failed\\\",\\\"link connection error\\\":\\\"link_connection_error\\\",\\\"link listener accept error\\\":\\\"link_listener_accept_failed\\\",\\\"webhook delivery attempt failed\\\":\\\"webhook_delivery_failed\\\",\\\"webhook dispatcher loop failed\\\":\\\"webhook_dispatcher_failed\\\",\\\"addressed delegation authorization failed\\\":\\\"share_authorization_failed\\\",\\\"native share authorization denied\\\":\\\"share_native_authorization_denied\\\",\\\"native share KV put failed\\\":\\\"share_native_kv_put_failed\\\",\\\"share v2 readiness probe failed\\\":\\\"share_v2_readiness_failed\\\",\\\"share v2 is enabled but shareEmail.invitationPublicKey is unset\\\":\\\"share_invitation_key_missing\\\",\\\"configured share invitation key does not match the node's derived signing key\\\":\\\"share_invitation_key_mismatch\\\",\\\"share/v2 policy challenge denied\\\":\\\"share_policy_denied\\\",\\\"failed to read committed hook events\\\":\\\"hook_events_read_failed\\\",\\\"failed to load kv write hook rows\\\":\\\"hook_rows_load_failed\\\",\\\"failed to load kv delete hook rows\\\":\\\"hook_rows_load_failed\\\",\\\"missing committed kv hook row for invocation\\\":\\\"missing_committed_hook_row\\\",\\\"database write committed but webhook enqueue failed\\\":\\\"webhook_enqueue_failed\\\",\\\"tunnel: failed to prepare connection attempt\\\":\\\"tunnel_connection_prepare_failed\\\",\\\"tunnel: no frame within idle timeout; treating connection as dead\\\":\\\"tunnel_idle_timeout\\\",\\\"tunnel: read error\\\":\\\"tunnel_read_failed\\\",\\\"tunnel: dropping malformed frame\\\":\\\"tunnel_malformed_frame\\\",\\\"tunnel: relay sent an error frame\\\":\\\"tunnel_relay_error\\\",\\\"tunnel: failed to write frame\\\":\\\"tunnel_write_failed\\\",\\\"tunnel: failed to record runtime state\\\":\\\"tunnel_state_record_failed\\\",\\\"tunnel: failed to clear runtime state after disable\\\":\\\"tunnel_state_clear_failed\\\",\\\"tunnel authentication rejected\\\":\\\"tunnel_auth_rejected\\\",\\\"network DID resolution enabled: did:web/did:tz fetches use the unhardened ssi client (TC-1080)\\\":\\\"network_did_resolution_enabled\\\"}\\n known = get(codes, [msg]) ?? null\\n if is_string(known) { code = known }\\n if starts_with(msg, \\\"quota service returned \\\") { code = \\\"quota_fallback\\\" }\\nif starts_with(msg, \\\"Failed to get TDX quote: \\\") { code = \\\"attestation_quote_failed\\\" }\\nif starts_with(msg, \\\"quota service returned an invalid body: \\\") { code = \\\"quota_invalid_body\\\" }\\nif starts_with(msg, \\\"dstack socket available but get_info failed: \\\") { code = \\\"dstack_get_info_failed\\\" }\\n }\\n event = if level == \\\"warn\\\" { \\\"node_warning\\\" } else { \\\"node_error\\\" }\\n out = {\\\"event\\\": event, \\\"svc\\\": \\\"api\\\", \\\"level\\\": level, \\\"event_code\\\": code, \\\"event_count\\\": 1}\\n }\\n} else {\\n ts = string!(j.time)\\n method = string!(j.method)\\n route = string!(j.route)\\n if route == \\\"\\\" { route = \\\"\\\" }\\n if !includes([\\\"GET\\\",\\\"HEAD\\\",\\\"POST\\\",\\\"PUT\\\",\\\"PATCH\\\",\\\"DELETE\\\",\\\"OPTIONS\\\",\\\"\\\"], method) || !includes([\\\"/healthz\\\",\\\"/info\\\",\\\"/version\\\",\\\"/invoke\\\",\\\"/delegate\\\",\\\"/revoke\\\",\\\"/attestation\\\",\\\"/signed/kv\\\",\\\"/signed/kv/\\\",\\\"/admin/quota\\\",\\\"/admin/quota/\\\",\\\"/admin/usage\\\",\\\"/hooks/events\\\",\\\"/hooks/tickets\\\",\\\"/hooks/webhooks\\\",\\\"/hooks/webhooks/\\\",\\\"/delegation/query\\\",\\\"/delegation/status\\\",\\\"/encryption/networks\\\",\\\"/encryption/networks/\\\",\\\"/encryption/networks//revoke\\\",\\\"/encryption/networks//decrypt\\\",\\\"/policy/v3/status\\\",\\\"/policy/v3/status/\\\",\\\"/policy/v3/policies\\\",\\\"/policy/v3/challenges\\\",\\\"/policy/v3/delegations\\\",\\\"/policy/v3/enforcer-bindings\\\",\\\"/policy/v3/deliveries/authorize\\\",\\\"/.well-known/tinycloud/node-keys\\\",\\\"/.well-known/encryption/network/\\\",\\\"/public//kv\\\",\\\"/public//kv/\\\",\\\"/peer/generate/\\\",\\\"/evidences/\\\",\\\"/share/v2/readiness\\\",\\\"/share/v2/policies\\\",\\\"/share/v2/policy/challenges\\\",\\\"/share/v2/policy/session\\\",\\\"/share/v2/invoke\\\",\\\"/share/v2/deliveries/authorize\\\",\\\"/<_s..>\\\",\\\"/public/<_space_id>/kv/<_key..>\\\",\\\"\\\"], route) { abort }\\n status = j.status\\n request_time = j.request_time\\n if !is_integer(status) || !(is_integer(request_time) || is_float(request_time)) { abort }\\n status = int!(status)\\n request_time = to_float!(request_time)\\n if status < 100 || status > 599 || request_time < 0 || request_time > 86400 { abort }\\n out = {\\\"event\\\": \\\"ingress_request\\\", \\\"svc\\\": \\\"ingress\\\", \\\"level\\\": \\\"info\\\", \\\"method\\\": method, \\\"route\\\": route, \\\"status\\\": status, \\\"request_time_seconds\\\": request_time}\\n}\\nif length(ts) > 64 { abort }\\nparsed_ts = parse_timestamp!(ts, format: \\\"%+\\\")\\nout.ts = format_timestamp!(parsed_ts, format: \\\"%Y-%m-%dT%H:%M:%S%.3fZ\\\")\\n. = out\\n\",\"drop_on_error\":true,\"drop_on_abort\":true,\"reroute_dropped\":true},\"gate_counts\":{\"type\":\"remap\",\"inputs\":[\"gate_route.rejected\",\"gate_route._unmatched\",\"allowlist.dropped\",\"event_route._unmatched\"],\"source\":\"reason = .reason\\nif !includes([\\\"malformed_envelope\\\",\\\"unexpected_label\\\",\\\"unlabelled\\\",\\\"non_json\\\",\\\"oversized\\\",\\\"partial\\\",\\\"invalid_fields\\\",\\\"unknown_target\\\",\\\"filtered_level\\\"], reason) { reason = \\\"invalid_fields\\\" }\\n. = {\\\"reason\\\": reason, \\\"count\\\": 1}\",\"drop_on_error\":true,\"drop_on_abort\":true},\"gate_reduce\":{\"type\":\"reduce\",\"inputs\":[\"gate_counts\"],\"group_by\":[\"reason\"],\"end_every_period_ms\":60000,\"expire_after_ms\":75000,\"flush_period_ms\":1000,\"merge_strategies\":{\"count\":\"sum\"}},\"gate_output\":{\"type\":\"remap\",\"inputs\":[\"gate_reduce\"],\"source\":\". = {\\\"ts\\\": format_timestamp!(now(), format: \\\"%Y-%m-%dT%H:%M:%S%.3fZ\\\"), \\\"level\\\": \\\"warn\\\", \\\"event\\\": \\\"logship_gate_summary_total\\\", \\\"svc\\\": \\\"logship\\\", \\\"reason\\\": .reason, \\\"count\\\": .count}\",\"drop_on_error\":true,\"drop_on_abort\":true},\"request_profile\":{\"type\":\"filter\",\"inputs\":[\"allowlist\"],\"condition\":\"\\\"$${TINYCLOUD_LOGSHIP_PROFILE}\\\" == \\\"all_requests\\\" || includes([\\\"node_warning\\\", \\\"node_error\\\"], .event) || (int(.status) ?? 0) >= 400 || (.event == \\\"node_request\\\" && (int(.latency_us) ?? 0) > 1000000) || (.event == \\\"ingress_request\\\" && (to_float(.request_time_seconds) ?? 0.0) > 1.000000)\"},\"event_route\":{\"type\":\"route\",\"inputs\":[\"request_profile\"],\"route\":{\"request\":\".event == \\\"node_request\\\" || .event == \\\"ingress_request\\\"\",\"diagnostic\":\".event == \\\"node_warning\\\" || .event == \\\"node_error\\\"\"}},\"diagnostic_reduce\":{\"type\":\"reduce\",\"inputs\":[\"event_route.diagnostic\"],\"group_by\":[\"event\",\"event_code\",\"level\",\"svc\"],\"end_every_period_ms\":60000,\"expire_after_ms\":75000,\"flush_period_ms\":1000,\"merge_strategies\":{\"event_count\":\"sum\",\"ts\":\"discard\"}},\"diagnostic_output\":{\"type\":\"remap\",\"inputs\":[\"diagnostic_reduce\"],\"source\":\". = {\\\"ts\\\": format_timestamp!(now(), format: \\\"%Y-%m-%dT%H:%M:%S%.3fZ\\\"), \\\"level\\\": .level, \\\"event\\\": .event, \\\"svc\\\": \\\"api\\\", \\\"event_code\\\": .event_code, \\\"event_count\\\": .event_count}\",\"drop_on_error\":true,\"drop_on_abort\":true}},\"sinks\":{\"betterstack\":{\"type\":\"http\",\"inputs\":[\"event_route.request\",\"diagnostic_output\",\"gate_output\"],\"method\":\"post\",\"uri\":\"https://s2795437.us-west-2a.betterstackdata.com/\",\"encoding\":{\"codec\":\"json\"},\"framing\":{\"method\":\"newline_delimited\"},\"compression\":\"gzip\",\"auth\":{\"strategy\":\"bearer\",\"token\":\"SECRET[bs.bs_token]\"},\"batch\":{\"max_bytes\":262144,\"timeout_secs\":1},\"buffer\":{\"type\":\"memory\",\"max_events\":1000,\"when_full\":\"block\"},\"request\":{\"timeout_secs\":10,\"retry_max_duration_secs\":300},\"healthcheck\":{\"enabled\":false}}},\"secret\":{\"bs\":{\"type\":\"directory\",\"path\":\"/var/lib/vector/secrets\",\"remove_trailing_whitespace\":true}}}" + entrypoint: + - /bin/sh + - -ec + - "umask 077\nmkdir -p /var/lib/vector/secrets\ncase \"$$TINYCLOUD_LOGSHIP_PROFILE\" in\n errors_slow|all_requests) ;;\n *) printf '%s\\n' 'logship profile invalid' >&2; exit 78 ;;\nesac\nif [ -z \"$$BETTERSTACK_SOURCE_TOKEN\" ]; then\n printf '%s\\n' 'logship source token missing' >&2\n exit 78\nfi\nprintf '%s' \"$$BETTERSTACK_SOURCE_TOKEN\" > /var/lib/vector/secrets/bs_token\nprintf '%s' \"$$VECTOR_CONFIG_JSON\" > /var/lib/vector/runtime.json\nunset BETTERSTACK_SOURCE_TOKEN VECTOR_CONFIG_JSON\nexec vector --config /var/lib/vector/runtime.json\n" + logging: + driver: json-file + options: + max-size: 10m + max-file: '3' + +volumes: + cert-data: + +networks: + logship: + driver: bridge diff --git a/scripts/ci/test-quota-overrides.py b/scripts/ci/test-quota-overrides.py index 8832c873..9573fb39 100755 --- a/scripts/ci/test-quota-overrides.py +++ b/scripts/ci/test-quota-overrides.py @@ -2,6 +2,7 @@ """Exercise docker.yml's quota restoration against synthetic HTTP only.""" import contextlib +import hashlib import io import json from pathlib import Path @@ -31,6 +32,10 @@ OVERRIDES = {APPLICATIONS: 5368709120, SECRETS: 268721121} CANARIES = [] DIGEST = "sha256:4bdae4fc232d886bb56bfbce4ec62ed4b8fb4e1369a01f6afb73d06dd68c188f" +# #309 exact head 44eeb20a1461ef66b5af492e3be4c42be2728b67, +# approved at https://github.com/TinyCloudLabs/tinycloud-node/pull/309#issuecomment-6224650050. +REVIEWED_COMPOSE = Path(__file__).with_name("fixtures") / "tc959-n0-compose.yaml" +REVIEWED_COMPOSE_SHA256 = "ca110b0bae6d5a06a1002fb80df3f71cffec11659c9f9dbfa5d72624735546f6" N0_CONTEXT = { "github.event_name": "workflow_dispatch", "github.ref": "refs/heads/main", "inputs.deploy_phala": True, "inputs.deploy_image_digest": DIGEST, @@ -39,6 +44,7 @@ "steps.pin.outputs.image": "ghcr.io/tinycloudlabs/tinycloud-node@" + DIGEST, "steps.pin.outputs.version": "1.20.6", "steps.pin.outputs.image_label": "1.20.6-dstack", "steps.pin.outputs.revision": "e18578ff9162986ddd813259c60f73866e3e60d1", + "steps.pin.outputs.n0_configuration_verified": "true", } @@ -290,6 +296,8 @@ def test_snapshot_requires_explicit_n0_authorization_and_exact_image(self): "Watch scheduling must not block post-restart quota restoration") for changes in ( {"inputs.restore_n0_quota_snapshot": False}, + {"steps.pin.outputs.n0_configuration_verified": ""}, + {"steps.pin.outputs.n0_configuration_verified": "false"}, {"inputs.deploy_image_digest": ""}, {"inputs.deploy_image_digest": "sha256:" + "f" * 64}, {"github.event_name": "release"}, @@ -395,9 +403,9 @@ def test_new_shell_steps_enforce_strict_runtime_baseline(self): else: self.assertEqual(result.returncode, 0) - def test_pending_reviewed_compose_binding_rejects_same_image_other_configuration(self): - # Intentionally RED until #309 hosted acceptance and Amos's exact-head - # review authorize its compose bytes. No candidate/caller hash is trusted. + def test_reviewed_compose_binding_rejects_same_image_other_configuration(self): + # Execute the actual pre-mutation path against frozen reviewed bytes + # and unrelated same-image configs, without any network or real CLI. boundary = """#!/usr/bin/env python3 import os from pathlib import Path @@ -410,9 +418,9 @@ def test_pending_reviewed_compose_binding_rejects_same_image_other_configuration elif args[:2] == ["image", "inspect"]: label = args[-1] if "org.opencontainers.image.revision" in label: - print(os.environ["ROLLBACK_REVISION"]) + print(os.environ["BOUNDARY_REVISION"]) elif "org.opencontainers.image.version" in label: - print("1.20.6-dstack") + print(os.environ["BOUNDARY_LABEL"]) else: raise SystemExit("Unexpected synthetic label lookup") elif args[:1] == ["run"]: @@ -440,7 +448,24 @@ def test_pending_reviewed_compose_binding_rejects_same_image_other_configuration steps = [shell_for_step(name) for name in ( "Validate image version against Cargo.toml", "Validate and pin deployment image", "Preflight sealed runtime configuration", "Deploy to Phala Cloud")] - for configuration in ("alpha", "beta"): + reviewed = REVIEWED_COMPOSE.read_bytes() + self.assertEqual(hashlib.sha256(reviewed).hexdigest(), REVIEWED_COMPOSE_SHA256) + native = "ghcr.io/tinycloudlabs/tinycloud-node" + legacy = ("services:\n tinycloud:\n image: " + native + ":dstack\n").encode() + cases = [("reviewed", reviewed, True, True)] + cases += [ + (name, ("services:\n tinycloud:\n image: " + N0_CONTEXT["steps.pin.outputs.image"] + + "\n environment:\n SYNTHETIC_CONFIG: " + name + "\n").encode(), False, True) + for name in ("alpha", "beta") + ] + cases += [ + ("profile-change", reviewed.replace(b":-errors_slow}", b":-all_requests}"), False, True), + ("crlf-change", reviewed.replace(b"\n", b"\r\n"), False, True), + ("n0-unreviewed-tag", reviewed.replace( + N0_CONTEXT["steps.pin.outputs.image"].encode(), (native + ":dstack").encode()), False, True), + ("normal-dstack", legacy, True, False), + ] + for configuration, compose, permitted, n0 in cases: with self.subTest(configuration=configuration), tempfile.TemporaryDirectory() as directory: root = Path(directory) binaries = root / "bin" @@ -452,9 +477,8 @@ def test_pending_reviewed_compose_binding_rejects_same_image_other_configuration (root / "tinycloud-node-server").mkdir() (root / "tinycloud-node-server/Cargo.toml").write_text('version = "1.20.7"\n') (root / "dstack-authorized-keys").write_text("synthetic-public-key\n") - (root / "docker-compose.dstack-postgres.yaml").write_text( - "services:\n tinycloud:\n image: " + N0_CONTEXT["steps.pin.outputs.image"] - + "\n environment:\n SYNTHETIC_CONFIG: " + configuration + "\n") + compose_path = root / "docker-compose.dstack-postgres.yaml" + compose_path.write_bytes(compose) trace = root / "mutation-trace" summary = root / "summary" output = root / "output" @@ -472,6 +496,14 @@ def test_pending_reviewed_compose_binding_rejects_same_image_other_configuration "GITHUB_OUTPUT": str(output), "GITHUB_STEP_SUMMARY": str(summary), **{name: self.secret for name in required}, } + if not n0: + env.update({ + "INPUT_IMAGE_VERSION": "1.20.7", "INPUT_DEPLOY_DIGEST": "", + "BUILD_DIGEST": "sha256:" + "f" * 64, "N0_QUOTA_SNAPSHOT": "false", + "ROLLBACK_VERSION": "", "ROLLBACK_REVISION": "", "DEPLOY_VERSION": "1.20.7", + }) + env["BOUNDARY_REVISION"] = N0_CONTEXT["steps.pin.outputs.revision"] if n0 else "f" * 40 + env["BOUNDARY_LABEL"] = "1.20.6-dstack" if n0 else "1.20.7-dstack" result = subprocess.run(["bash", "--noprofile", "--norc", "-e", "-c", "\n".join(steps)], cwd=root, env=env, capture_output=True, text=True) logs = result.stdout + result.stderr @@ -479,10 +511,28 @@ def test_pending_reviewed_compose_binding_rejects_same_image_other_configuration logs += output.read_text() if output.exists() else "" self.assertFalse(self.secret in logs, "Synthetic workflow leaked a sealed canary") self.assertFalse((root / ".phala-prod.env").exists(), "Sealed temp file was not removed") - self.assertFalse(trace.exists(), - "PENDING reviewed #309 compose binding: unreviewed config reached CVM mutation") - self.assertNotEqual(result.returncode, 0, - "Same immutable image must not authorize unrelated compose bytes") + selected = dict(line.split("=", 1) for line in output.read_text().splitlines()) + if permitted: + self.assertEqual(result.returncode, 0, "Authorized configuration was rejected") + self.assertEqual(trace.read_text().splitlines(), ["cvm-mutation"]) + context = { + **N0_CONTEXT, "inputs.restore_n0_quota_snapshot": n0, + **{"steps.pin.outputs." + key: value for key, value in selected.items()}, + } + self.assertEqual(quota_restore_enabled(context), n0) + if n0: + self.assertEqual(selected.get("n0_configuration_verified"), "true", + "Reviewed bytes must authorize quota restoration") + self.assertEqual(compose_path.read_bytes(), reviewed) + self.assertNotIn("1.20.7", logs, "Checkout version leaked into the old-image promotion") + else: + self.assertNotEqual(selected.get("n0_configuration_verified"), "true") + self.assertIn((native + "@sha256:" + "f" * 64).encode(), compose_path.read_bytes()) + else: + self.assertFalse(trace.exists(), "Unreviewed configuration reached CVM mutation") + self.assertNotEqual(result.returncode, 0, "Unreviewed configuration was authorized") + self.assertNotEqual(selected.get("n0_configuration_verified"), "true") + self.assertEqual(compose_path.read_bytes(), compose, "Rejected compose was modified") def test_positive_usage_wrong_owner_config_fails_before_any_request(self): other = "tinycloud:pkh:eip155:1:0x" + "1" * 40