diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 7bdc0b1f..49ccae86 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -43,6 +43,22 @@ on: required: false type: boolean default: false + authorize_quota_values: + description: 'Fresh authorization of the reviewed quota values/authority for this deployment; required on every deploy_phala path.' + required: false + type: boolean + default: false + rollback_pre_n0: + description: 'Deploy only the captured pre-N0 1.20.6 compose; requires an explicit Sam-approved rollback privacy mode.' + required: false + type: boolean + default: false + rollback_privacy_mode: + description: 'Explicit rollback-only policy; stop-future-local-logs does NOT delete existing files. No production default.' + required: false + type: choice + options: [unselected, keep-json-file, stop-future-local-logs] + default: unselected allow_non_descendant: description: 'Skip the production ancestry guard for a deliberate non-descendant deployment' required: false @@ -54,22 +70,22 @@ on: type: boolean default: true image_version: - description: 'Manual image version to tag/deploy, without leading v. Must equal the tinycloud-node version in tinycloud-node-server/Cargo.toml at the selected ref. Defaults to the selected ref name.' + description: 'Image version without leading v. Builds must match the selected crate; digest promotions must match the recorded image version (rollback_image_version when supplied).' required: false type: string default: '' deploy_image_digest: - description: 'Promote an existing sha256 digest without rebuilding. Requires deploy_phala, matching image_version, and default dstack features.' + description: 'Promote an existing sha256 digest without rebuilding, dispatched on main only. Requires truthful image_version, deploy_phala and default dstack features.' required: false type: string default: '' rollback_image_version: - description: 'Emergency rollback only: version label of a recorded digest whose revision is in main history' + description: 'Recorded immutable digest version label, for recovery, rollback or forward digest promotion; revision must be in main history' required: false type: string default: '' rollback_image_revision: - description: 'Emergency rollback only: 40-character revision label of the recorded digest (must be an ancestor of main)' + description: 'Recorded digest 40-character revision label (must be an ancestor of main)' required: false type: string default: '' @@ -111,12 +127,17 @@ jobs: name: Version guard runs-on: ubuntu-latest outputs: - cargo_version: ${{ steps.check.outputs.cargo_version }} + image_version: ${{ steps.check.outputs.image_version }} steps: - uses: actions/checkout@v4 - name: Check published Docker tags run: python3 scripts/ci/test-docker-tags.py + - name: Test quota authority with synthetic HTTP + run: python3 scripts/ci/test-quota-authority.py + - name: Test recovery deployment boundaries and documented commands + run: python3 scripts/ci/test-deploy-recovery.py + - name: Validate image version against Cargo.toml id: check env: @@ -124,6 +145,10 @@ jobs: INPUT_IMAGE_VERSION: ${{ inputs.image_version }} INPUT_DEPLOY_DIGEST: ${{ inputs.deploy_image_digest }} INPUT_DEPLOY_PHALA: ${{ inputs.deploy_phala }} + AUTHORIZE_QUOTA_VALUES: ${{ inputs.authorize_quota_values }} + ROLLBACK_PRE_N0: ${{ inputs.rollback_pre_n0 }} + ROLLBACK_PRIVACY_MODE: ${{ inputs.rollback_privacy_mode }} + APPROVED_ROLLBACK_PRIVACY_MODE: ${{ vars.TC959_ROLLBACK_PRIVACY_MODE }} ROLLBACK_VERSION: ${{ inputs.rollback_image_version }} ROLLBACK_REVISION: ${{ inputs.rollback_image_revision }} INCLUDE_DUCKDB: ${{ inputs.include_duckdb }} @@ -131,6 +156,15 @@ jobs: RELEASE_TAG: ${{ github.event.release.tag_name }} run: | set -euo pipefail + # #310 and this recovery change must land as one lineage, before + # #309's later logging window. Never merge #310 alone. + if [ "${INPUT_DEPLOY_PHALA}" = "true" ]; then + if [ "${EVENT_NAME}" != "workflow_dispatch" ] || [ "${AUTHORIZE_QUOTA_VALUES:-false}" != "true" ]; then + echo "::error::Every deployment requires explicit authorize_quota_values=true before any mutation" + exit 1 + fi + fi + python3 scripts/ci/reconcile-quota.py --validate-only # `|| true` so a missing match reaches the explicit check below # instead of dying under `set -e` with no explanation. @@ -139,8 +173,27 @@ jobs: echo "::error::could not read the tinycloud-node version from tinycloud-node-server/Cargo.toml" exit 1 fi - echo "cargo_version=${CARGO_VERSION}" >> "$GITHUB_OUTPUT" - echo "tinycloud-node-server/Cargo.toml version: ${CARGO_VERSION}" + if [ "${ROLLBACK_PRE_N0:-false}" = "true" ]; then + if [ "${EVENT_NAME}" != "workflow_dispatch" ] || [ "${GITHUB_REF}" != "refs/heads/main" ] \ + || [ "${INPUT_DEPLOY_PHALA}" != "true" ] \ + || [ "${INPUT_DEPLOY_DIGEST}" != "sha256:4bdae4fc232d886bb56bfbce4ec62ed4b8fb4e1369a01f6afb73d06dd68c188f" ] \ + || [ "${INPUT_IMAGE_VERSION}" != "1.20.6" ] || [ "${ROLLBACK_VERSION}" != "1.20.6" ] \ + || [ "${ROLLBACK_REVISION}" != "e18578ff9162986ddd813259c60f73866e3e60d1" ]; then + echo "::error::Captured pre-N0 rollback requires main-only dispatch and the exact recorded 1.20.6 digest labels" + exit 1 + fi + case "${ROLLBACK_PRIVACY_MODE:-unselected}" in + keep-json-file|stop-future-local-logs) ;; + *) echo "::error::Choose an explicit Sam-approved rollback privacy mode"; exit 1 ;; + esac + if [ "${APPROVED_ROLLBACK_PRIVACY_MODE:-}" != "${ROLLBACK_PRIVACY_MODE}" ]; then + echo "::error::Rollback privacy mode does not match the recorded Sam card decision" + exit 1 + fi + elif [ "${ROLLBACK_PRIVACY_MODE:-unselected}" != "unselected" ]; then + echo "::error::Rollback privacy selection is mutually exclusive from current-compose promotion" + exit 1 + fi check_matches() { label="$1" @@ -158,9 +211,6 @@ jobs: check_matches "git tag" "${GITHUB_REF_NAME}" fi - if [ -n "${INPUT_IMAGE_VERSION}" ]; then - check_matches "image_version input" "${INPUT_IMAGE_VERSION}" - fi # TC-940: every production revision is in main's history, so a # rollback dispatched on main passes the ancestry guard without # allow_non_descendant. The rollback revision itself must be an @@ -172,16 +222,32 @@ jobs: echo "::error::Rollback label override requires a digest and valid version/revision labels" exit 1 fi - echo "::warning::Emergency rollback to recorded ${ROLLBACK_VERSION} revision ${ROLLBACK_REVISION}" + echo "::notice::Using recorded image labels: ${ROLLBACK_VERSION} revision ${ROLLBACK_REVISION}" + fi + if [ -n "${INPUT_IMAGE_VERSION}" ]; then + if [ -n "${INPUT_DEPLOY_DIGEST}" ] && [ -n "${ROLLBACK_VERSION}" ]; then + if [ "${INPUT_IMAGE_VERSION}" != "${ROLLBACK_VERSION}" ]; then + echo "::error::Digest promotion image_version must equal the recorded image version" + exit 1 + fi + else + check_matches "image_version input" "${INPUT_IMAGE_VERSION}" + fi fi if [ -n "${INPUT_DEPLOY_DIGEST}" ]; then - if [ "${EVENT_NAME}" != "workflow_dispatch" ] || [ "${INPUT_DEPLOY_PHALA}" != "true" ] || [ -z "${INPUT_IMAGE_VERSION}" ] \ + if [ "${EVENT_NAME}" != "workflow_dispatch" ] || [ "${GITHUB_REF}" != "refs/heads/main" ] \ + || [ "${INPUT_DEPLOY_PHALA}" != "true" ] || [ -z "${INPUT_IMAGE_VERSION}" ] \ || [ "${INCLUDE_DUCKDB}" = "true" ] || [ "${INCLUDE_TC_BENCH_V1}" = "true" ] \ || ! [[ "${INPUT_DEPLOY_DIGEST}" =~ ^sha256:[0-9a-f]{64}$ ]]; then - echo "::error::Digest promotion requires deploy_phala=true, matching image_version, default dstack features and a sha256 digest." + echo "::error::Digest promotion requires workflow_dispatch on main, deploy_phala=true, truthful image_version, default dstack features and a sha256 digest." exit 1 fi fi + # Only the version of the selected binary enters deployment + # outputs/logs; the workflow checkout is not the promoted image. + SELECTED_VERSION="${ROLLBACK_VERSION:-${CARGO_VERSION}}" + echo "image_version=${SELECTED_VERSION}" >> "$GITHUB_OUTPUT" + echo "Selected image version: ${SELECTED_VERSION}" # TC-940: refuse a production deploy from anything but main or a vX.Y.Z # tag on main's history before spending time on image builds. The @@ -391,16 +457,20 @@ jobs: deploy-phala: runs-on: ubuntu-latest needs: [version-guard, build, build-dstack] - # TC-971: the deploy secrets live only in the production environment, whose - # deployment policy admits main and v* tags. A dispatch from any other ref - # is rejected before the job starts, and an old ref whose docker.yml lacks - # this line gets no deploy secrets at all. See docs/releasing.md. + # TC-971: production-environment policy admits main and v* tags. Most deploy + # secrets live there; PROD_TINYCLOUD_ADMIN_SECRET is repository-scoped. + # Digest promotion additionally requires main, not a release tag. environment: production if: >- ${{ vars.TC780_CUTOVER_READY == 'true' && !cancelled() && !inputs.rollback_audit_only && needs.version-guard.result == 'success' && github.event_name == 'workflow_dispatch' && inputs.deploy_phala && - ((inputs.deploy_image_digest != '' && needs.build.result == 'skipped' && needs.build-dstack.result == 'skipped') || + ((inputs.deploy_image_digest != '' && github.ref == 'refs/heads/main' && needs.build.result == 'skipped' && needs.build-dstack.result == 'skipped') || (inputs.deploy_image_digest == '' && needs.build.result == 'success' && needs.build-dstack.result == 'success')) }} + outputs: + image: ${{ steps.pin.outputs.image }} + version: ${{ steps.pin.outputs.version }} + revision: ${{ steps.pin.outputs.revision }} + image_label: ${{ steps.pin.outputs.image_label }} permissions: contents: read packages: read @@ -439,6 +509,11 @@ jobs: - name: Install Phala CLI run: npm install -g phala@1.1.19 + - name: Install Phala env synchronization SDK + # Same SDK generation as phala 1.1.19, isolated from the checkout. + # Install before any sealed values enter the step environment. + run: npm install --prefix "$RUNNER_TEMP/phala-env-sync" --no-save --ignore-scripts @phala/cloud@0.2.9 + - name: Guard production ancestry env: PHALA_CLOUD_API_KEY: ${{ secrets.PHALA_CLOUD_API_KEY }} @@ -520,15 +595,31 @@ jobs: fi echo "Running production revision ${RUNNING_REVISION} is an ancestor of ${GITHUB_SHA}." + - name: Preflight quota authority + env: + QUOTA_ADMIN_SECRET: ${{ secrets.PROD_TINYCLOUD_ADMIN_SECRET }} + # Billing's storage-comp guard checks the same shared node/billing + # admin value (billing TINYCLOUD_ADMIN_SECRET); see docs/releasing.md. + QUOTA_BILLING_ADMIN_SECRET: ${{ secrets.PROD_TINYCLOUD_ADMIN_SECRET }} + QUOTA_BILLING_URL: ${{ secrets.PROD_TINYCLOUD_QUOTA_URL }} + run: | + set -euo pipefail + set +x + python3 scripts/ci/reconcile-quota.py --preflight + - name: Validate and pin deployment image + id: pin env: INPUT_DEPLOY_DIGEST: ${{ inputs.deploy_image_digest }} BUILD_DIGEST: ${{ needs.build-dstack.outputs.digest }} - CARGO_VERSION: ${{ needs.version-guard.outputs.cargo_version }} + DEPLOY_VERSION: ${{ needs.version-guard.outputs.image_version }} INCLUDE_DUCKDB: ${{ inputs.include_duckdb }} INCLUDE_TC_BENCH_V1: ${{ inputs.include_tc_bench_v1 }} ROLLBACK_VERSION: ${{ inputs.rollback_image_version }} ROLLBACK_REVISION: ${{ inputs.rollback_image_revision }} + ROLLBACK_PRE_N0: ${{ inputs.rollback_pre_n0 }} + ROLLBACK_PRIVACY_MODE: ${{ inputs.rollback_privacy_mode }} + APPROVED_ROLLBACK_PRIVACY_MODE: ${{ vars.TC959_ROLLBACK_PRIVACY_MODE }} run: | set -euo pipefail DIGEST="${INPUT_DEPLOY_DIGEST:-${BUILD_DIGEST}}" @@ -537,12 +628,11 @@ jobs: docker pull --platform linux/amd64 "${RESOLVED_IMAGE}" REVISION="$(docker image inspect "${RESOLVED_IMAGE}" --format '{{ index .Config.Labels "org.opencontainers.image.revision" }}')" VERSION="$(docker image inspect "${RESOLVED_IMAGE}" --format '{{ index .Config.Labels "org.opencontainers.image.version" }}')" - EXPECTED="${CARGO_VERSION}-dstack" + EXPECTED="${DEPLOY_VERSION}-dstack" if [ "${INCLUDE_DUCKDB}" = "true" ]; then EXPECTED="${EXPECTED}-duckdb"; fi if [ "${INCLUDE_TC_BENCH_V1}" = "true" ]; then EXPECTED="${EXPECTED}-tc-bench-v1"; fi EXPECTED_REVISION="$(git rev-parse HEAD)" if [ -n "${ROLLBACK_VERSION}" ]; then - EXPECTED="${ROLLBACK_VERSION}-dstack" EXPECTED_REVISION="${ROLLBACK_REVISION}" fi if [ "${REVISION}" != "${EXPECTED_REVISION}" ] || [ "${VERSION}" != "${EXPECTED}" ]; then @@ -550,16 +640,79 @@ jobs: exit 1 fi python3 - <<'PY' + import hashlib import os + import re from pathlib import Path + def read_artifact(relative): + path = Path(relative) + if path.is_symlink() or path.resolve() != Path.cwd().resolve() / relative: + raise SystemExit('::error::Deployment artifact path is not the fixed reviewed path') + try: + return path.read_bytes() + except OSError: + raise SystemExit('::error::Reviewed deployment artifact is unavailable') from None + path = Path('docker-compose.dstack-postgres.yaml') - content = path.read_text() - original = 'ghcr.io/tinycloudlabs/tinycloud-node:dstack' - assert content.count(original) == 1, 'Expected exactly one native image reference' - path.write_text(content.replace(original, os.environ['RESOLVED_IMAGE'])) + if path.is_symlink() or path.resolve() != Path.cwd().resolve() / path: + raise SystemExit('::error::Deployment destination is not the fixed reviewed path') + native = 'ghcr.io/tinycloudlabs/tinycloud-node' + reviewed = native + '@sha256:4bdae4fc232d886bb56bfbce4ec62ed4b8fb4e1369a01f6afb73d06dd68c188f' + n0_hash = 'ca110b0bae6d5a06a1002fb80df3f71cffec11659c9f9dbfa5d72624735546f6' + pre_n0_hash = 'df239749ca0a9c09893a855ae2a8abb94090b8556515ad002105c58ac91f4c91' + pre_n0_template_hash = '28302d6761d841b7cc5c11314aed2d377735e6bbdcaf85972a752e03bc9fdc93' + rollback = os.environ.get('ROLLBACK_PRE_N0') == 'true' + mode = os.environ.get('ROLLBACK_PRIVACY_MODE') or 'unselected' + if rollback: + raw = read_artifact('ops/rollback/pre-n0-compose.yaml') + if os.environ['RESOLVED_IMAGE'] != reviewed or hashlib.sha256(raw).hexdigest() != pre_n0_hash: + raise SystemExit('::error::Captured pre-N0 image/compose pin mismatch; deployment not attempted') + if mode not in ('keep-json-file', 'stop-future-local-logs') or mode != os.environ.get('APPROVED_ROLLBACK_PRIVACY_MODE'): + raise SystemExit('::error::Rollback privacy decision is missing or mismatched') + if mode == 'stop-future-local-logs': + raw = read_artifact('ops/rollback/pre-n0-stop-local-logs.yaml') + if hashlib.sha256(raw).hexdigest() != '315ceca54bf8bfefe2b395b4097fe38ed8ec847fa0688bc873220bd65efc99ec': + raise SystemExit('::error::Reviewed stop-future-logs artifact pin mismatch') + n0_configuration = False + else: + if mode != 'unselected': + raise SystemExit('::error::Rollback privacy cannot modify a forward/current-compose deployment') + raw = read_artifact(str(path)) + digest = hashlib.sha256(raw).hexdigest() + n0_configuration = digest == n0_hash + if n0_configuration and os.environ['RESOLVED_IMAGE'] != reviewed: + raise SystemExit('::error::Reviewed N0 compose cannot substitute a different image') + if os.environ['RESOLVED_IMAGE'] == reviewed and digest not in (n0_hash, pre_n0_hash, pre_n0_template_hash): + raise SystemExit('::error::Same-image recovery requires the exact current pre-N0 or reviewed N0 compose') + content = raw.decode('utf-8') + images = [ + image for image in re.finditer(r'(?m)^([ \t]+image:[ \t]+)([^\n]+)$', content) + if image[2].strip().strip('"\'').startswith((native + ':', native + '@')) + ] + if len(images) != 1 or images[0][2].strip().strip('"\'') not in (native + ':dstack', reviewed): + raise SystemExit('Expected exactly one :dstack or reviewed digest-pinned native image') + image = images[0] + if image[2].strip().strip('"\'') == reviewed and reviewed != os.environ['RESOLVED_IMAGE']: + raise SystemExit('Reviewed compose digest does not match the selected deployment image') + path.write_bytes((content[:image.start(2)] + os.environ['RESOLVED_IMAGE'] + content[image.end(2):]).encode('utf-8')) + with open(os.environ['GITHUB_OUTPUT'], 'a') as output: + output.write(f'n0_configuration_verified={str(n0_configuration).lower()}\n') + output.write(f'rollback_pre_n0_verified={str(rollback).lower()}\n') + if rollback: + message = f'Rollback privacy: {mode}; no deletion of existing log files is requested.' + print(message) + with open(os.environ['GITHUB_STEP_SUMMARY'], 'a') as summary: + summary.write(message + '\n') PY + { + echo "image=${RESOLVED_IMAGE}" + echo "version=${VERSION%%-dstack*}" + echo "revision=${REVISION}" + echo "image_label=${VERSION}" + } >> "$GITHUB_OUTPUT" echo "Resolved image: ${RESOLVED_IMAGE} (revision ${REVISION}, version ${VERSION})" - cat docker-compose.dstack-postgres.yaml + printf 'Deployment image: `%s` — version `%s`, revision `%s`.\n' \ + "${RESOLVED_IMAGE}" "${VERSION}" "${REVISION}" >> "$GITHUB_STEP_SUMMARY" # TC-500: Phala considers a CVM ready before compose services have # started. Validate the production-sealed bundle using the exact image @@ -567,7 +720,6 @@ jobs: # without opening a database, resolving dstack, network, or socket # access, before the deploy step can replace the healthy CVM. - name: Preflight sealed runtime configuration - if: inputs.rollback_image_version == '' env: TINYCLOUD_STORAGE__DATABASE: ${{ secrets.PROD_TINYCLOUD_DATABASE_URL }} TINYCLOUD_DATABASE__WRITE_FENCE: ${{ inputs.sql_identity_fence }} @@ -591,7 +743,123 @@ jobs: -e TINYCLOUD_KEYS__TYPE \ "${IMAGE}" --validate-config + # TC-959: only env_keys/encrypted_env may change here. Never copy the + # precedent's raw-response logs or its unguarded compose_hash retry. + # https://docs.phala.com/phala-cloud/references/cloud-js-sdk/update-cvm-envs + - name: Prepare allowed_envs synchronization + run: | + set -euo pipefail + cat > "$RUNNER_TEMP/phala-env-sync/sync.mjs" <<'JS' + import { readFileSync } from "node:fs"; + import { setTimeout as sleep } from "node:timers/promises"; + + // SDK DEBUG can print request bodies and server-supplied errors. + delete process.env.DEBUG; + let phase = "sealed env validation"; + const requireCondition = (condition) => { + if (!condition) throw new Error("guard failed"); + }; + const validName = (name) => typeof name === "string" && /^[A-Za-z_][A-Za-z0-9_]*$/.test(name); + const namesFrom = (compose) => { + const names = compose?.allowed_envs; + requireCondition(Array.isArray(names) && names.every(validName)); + requireCondition(new Set(names).size === names.length); + return names; + }; + + try { + const { + createClient, getCvmComposeFile, getCvmInfo, updateCvmEnvs, + encryptEnvVars, parseEnvVars, + } = await import("@phala/cloud"); + requireCondition(Boolean(process.env.PHALA_CLOUD_API_KEY)); + const source = readFileSync(".phala-prod.env", "utf8"); + const envs = []; + const keys = new Set(); + for (const line of source.split("\n")) { + if (line === "") continue; + const separator = line.indexOf("="); + const key = line.slice(0, separator); + const value = line.slice(separator + 1); + requireCondition(separator > 0 && validName(key) && !keys.has(key)); + requireCondition(value.length > 0 && !/[\r\n]/.test(value)); + keys.add(key); + envs.push({ key, value }); + } + requireCondition(keys.has("BETTERSTACK_SOURCE_TOKEN")); + // Match the CLI's dotenv interpretation exactly; no truncated, + // quoted or comment-stripped value may replace the intended one. + requireCondition(JSON.stringify(parseEnvVars(source)) === JSON.stringify(envs)); + const client = createClient({ + apiKey: process.env.PHALA_CLOUD_API_KEY, + baseURL: "https://cloud-api.phala.com/api/v1", + version: "2026-01-21", + timeout: 15000, + retry: 0, + }); + const ref = { id: "tinycloud-node" }; + phase = "complete whitelist validation"; + const allowed = namesFrom(await getCvmComposeFile(client, ref)); + // Retaining a name cannot recover a value absent from this file. + // Check before deploy AND again immediately before any env PATCH. + requireCondition(allowed.every((key) => keys.has(key))); + const missing = [...keys].filter((key) => !allowed.includes(key)); + if (process.argv.includes("--check")) { + console.log(`Validated complete sealed set: ${keys.size} env names.`); + } else if (missing.length === 0) { + console.log(`allowed_envs verified: ${keys.size} env names; no PATCH needed.`); + } else { + phase = "public encryption key validation"; + const info = await getCvmInfo(client, ref); + const pubkey = info.kms_info?.encrypted_env_pubkey ?? info.encrypted_env_pubkey; + requireCondition(typeof pubkey === "string" && /^(0x)?[0-9a-fA-F]{64}$/.test(pubkey)); + const encrypted = await encryptEnvVars(envs, pubkey); + phase = "env whitelist update"; + const result = await updateCvmEnvs(client, { + ...ref, encrypted_env: encrypted, env_keys: [...allowed, ...missing], + }); + // An on-chain precondition needs a separately reviewed + // addComposeHash transaction. Never assume hash-only approval. + requireCondition(result.status === "in_progress" && typeof result.correlation_id === "string" && result.correlation_id.length > 0); + console.log(`allowed_envs update accepted: ${missing.length} added names.`); + phase = "env whitelist convergence"; + const deadline = Date.now() + 600000; + let completed = false; + let observedOperation = false; + const pending = new Set(["starting", "stopping", "stopped", "restarting", "upgrading", "updating", "creating"]); + while (Date.now() < deadline) { + await sleep(5000); + const current = await getCvmInfo(client, ref); + requireCondition(current.status === "running" || pending.has(current.status)); + const after = namesFrom(await getCvmComposeFile(client, ref)); + const progress = current.progress; + // Prefer this operation's correlation. If optional progress + // is absent, require a post-PATCH non-running/running cycle. + if (progress?.correlation_id === result.correlation_id || + (!progress?.correlation_id && pending.has(current.status))) observedOperation = true; + // Missing/null progress can mean a fast or still-queued + // restart. Stored names + running alone are not completion. + const settled = progress == null || ( + progress && !progress.target && !progress.started_at && !progress.correlation_id + ); + if (observedOperation && current.status === "running" && settled && [...keys].every((key) => after.includes(key))) { + completed = true; + break; + } + } + requireCondition(completed); + console.log(`allowed_envs verified after update: ${keys.size} env names.`); + } + } catch { + // Neither response message/JSON, ciphertext, nor exception detail + // is safe for public Actions logs. + console.error(`::error::allowed_envs synchronization failed during ${phase}; response details withheld.`); + process.exitCode = 1; + } + JS + - name: Deploy to Phala Cloud + id: deploy env: PHALA_CLOUD_API_KEY: ${{ secrets.PHALA_CLOUD_API_KEY }} PHALA_ENV_DATABASE_URL: ${{ secrets.PROD_TINYCLOUD_DATABASE_URL }} @@ -606,12 +874,16 @@ jobs: PHALA_ENV_CLOUDFLARE_API_TOKEN: ${{ secrets.PROD_TINYCLOUD_CLOUDFLARE_API_TOKEN }} PHALA_ENV_DSTACK_GATEWAY_DOMAIN: ${{ secrets.PROD_TINYCLOUD_DSTACK_GATEWAY_DOMAIN }} PHALA_ENV_CERTBOT_EMAIL: ${{ secrets.PROD_TINYCLOUD_CERTBOT_EMAIL }} + PHALA_ENV_BETTERSTACK_SOURCE_TOKEN: ${{ secrets.PROD_TINYCLOUD_BETTERSTACK_SOURCE_TOKEN }} PHALA_ENV_SQL_IDENTITY_FENCE: ${{ inputs.sql_identity_fence }} # Update the existing prod CVM (name: tinycloud-node) in the # Tiny Cloud workspace. The PHALA_CLOUD_API_KEY secret is # workspace-scoped and must be issued from that workspace. run: | set -euo pipefail + set +x + umask 077 + trap 'rm -f .phala-prod.env' EXIT required_vars=( PHALA_CLOUD_API_KEY @@ -627,6 +899,7 @@ jobs: PHALA_ENV_CLOUDFLARE_API_TOKEN PHALA_ENV_DSTACK_GATEWAY_DOMAIN PHALA_ENV_CERTBOT_EMAIL + PHALA_ENV_BETTERSTACK_SOURCE_TOKEN PHALA_ENV_SQL_IDENTITY_FENCE ) @@ -635,8 +908,16 @@ jobs: echo "::error::Missing required GitHub secret for Phala deploy: ${var#PHALA_ENV_}" exit 1 fi + if [[ "${!var}" == *$'\n'* || "${!var}" == *$'\r'* ]]; then + echo "::error::Phala deploy inputs must be single-line values" + exit 1 + fi done + # Also restrict a file left by an earlier failed runner step before + # writing any value; umask alone does not tighten an existing file. + : > .phala-prod.env + chmod 0600 .phala-prod.env { printf 'DATABASE_URL=%s\n' "$PHALA_ENV_DATABASE_URL" printf 'S3_BUCKET=%s\n' "$PHALA_ENV_S3_BUCKET" @@ -650,6 +931,7 @@ jobs: printf 'CLOUDFLARE_API_TOKEN=%s\n' "$PHALA_ENV_CLOUDFLARE_API_TOKEN" printf 'DSTACK_GATEWAY_DOMAIN=%s\n' "$PHALA_ENV_DSTACK_GATEWAY_DOMAIN" printf 'CERTBOT_EMAIL=%s\n' "$PHALA_ENV_CERTBOT_EMAIL" + printf 'BETTERSTACK_SOURCE_TOKEN=%s\n' "$PHALA_ENV_BETTERSTACK_SOURCE_TOKEN" printf 'SQL_IDENTITY_FENCE=%s\n' "$PHALA_ENV_SQL_IDENTITY_FENCE" # SSH public keys authorized on the CVM (checked into the repo, # see dstack-authorized-keys). The Phala Cloud pre-launch script @@ -658,13 +940,33 @@ jobs: printf 'DSTACK_AUTHORIZED_KEYS=%s\n' "$(cat dstack-authorized-keys)" } > .phala-prod.env + # Validate complete env names/whitelist before any CVM mutation. + node "$RUNNER_TEMP/phala-env-sync/sync.mjs" --check + # Fail before replacing the running CVM if a required compose input # is absent. Phala's control-plane ready state does not prove that # docker compose interpolation or container startup succeeded. - docker compose --env-file .phala-prod.env \ - -f docker-compose.dstack-postgres.yaml config --quiet + if ! docker compose --env-file .phala-prod.env \ + -f docker-compose.dstack-postgres.yaml config --quiet >/dev/null 2>&1; then + echo "::error::Sealed compose validation failed; deployment not attempted" + exit 1 + fi - phala deploy --cvm-id tinycloud-node -c docker-compose.dstack-postgres.yaml -e .phala-prod.env --no-public-logs --no-public-sysinfo --wait + # The CLI/SDK can include request details in errors. Never forward + # those responses into Actions logs, even when deployment fails. + # A failing CLI/update can still have restarted the CVM. + echo "cvm_mutation_attempted=true" >> "$GITHUB_OUTPUT" + if ! phala deploy --cvm-id tinycloud-node -c docker-compose.dstack-postgres.yaml -e .phala-prod.env --no-public-logs --no-public-sysinfo --wait >/dev/null 2>&1; then + echo "::error::Phala deployment failed; inspect the private control plane" + exit 1 + fi + + # Do not assume -e registered the whitelist; verify/sync it explicitly. + node "$RUNNER_TEMP/phala-env-sync/sync.mjs" + + - name: Remove sealed deployment environment + if: always() + run: rm -f .phala-prod.env # TC-312: prove the deploy actually converged. The image tag matching the # crate version is enforced at build time by version-guard, but nothing @@ -674,23 +976,14 @@ jobs: # just deployed. - name: Verify deployed version env: - EVENT_NAME: ${{ github.event_name }} - RELEASE_TAG: ${{ github.event.release.tag_name }} - INPUT_IMAGE_VERSION: ${{ inputs.image_version }} - ROLLBACK_VERSION: ${{ inputs.rollback_image_version }} + EXPECTED: ${{ steps.pin.outputs.version }} + IMAGE_REVISION: ${{ steps.pin.outputs.revision }} run: | set -euo pipefail - # Resolve the version we just deployed, mirroring the compose step. - if [ "${EVENT_NAME}" = "release" ]; then - TAG="${RELEASE_TAG}" - elif [ -n "${INPUT_IMAGE_VERSION}" ]; then - TAG="${INPUT_IMAGE_VERSION}" - else - TAG="${GITHUB_REF_NAME}" - fi - EXPECTED="${ROLLBACK_VERSION:-${TAG#v}}" - echo "Expecting node.tinycloud.xyz to report version ${EXPECTED}" + # Only validated image labels describe the binary, not Cargo.toml + # at the workflow checkout when an existing digest is promoted. + echo "Expecting node.tinycloud.xyz to report version ${EXPECTED} (revision ${IMAGE_REVISION})" # The CVM needs a short while to pull the new image and restart, so # retry with a fixed backoff (~5 min total) before giving up. @@ -710,6 +1003,42 @@ jobs: echo "::error::node.tinycloud.xyz still reports '${REPORTED}' after $((ATTEMPTS * SLEEP))s, expected '${EXPECTED}'. The deploy did not converge." exit 1 + # Ingress TLS readiness is required health. Restore quotas before the + # separate Policy probes, which still fail the deployment if invalid. + - name: Verify deployed ingress health + env: + EXPECTED: ${{ steps.pin.outputs.version }} + run: | + set -euo pipefail + for attempt in $(seq 1 30); do + REPORTED="$(curl -fsS --max-time 10 https://tee.node.tinycloud.xyz/version | jq -r '.version' || true)" + if [ "${REPORTED}" = "${EXPECTED}" ]; then + echo "Direct TEE ingress is serving ${EXPECTED}." + exit 0 + fi + sleep 5 + done + echo "::error::Direct TEE ingress did not become healthy at version ${EXPECTED}." + exit 1 + + # General reviewed authority/values, freshly authorized for every deploy. + # Same job, after the final env operation and required health, before + # ancillary Policy probes. Billing mode is verification-only. + + - name: Restore storage quota overrides + id: quotas + timeout-minutes: 5 + if: ${{ success() && inputs.authorize_quota_values }} + env: + # Repository-scoped; never read by an agent or passed in argv. + QUOTA_ADMIN_SECRET: ${{ secrets.PROD_TINYCLOUD_ADMIN_SECRET }} + QUOTA_BILLING_ADMIN_SECRET: ${{ secrets.PROD_TINYCLOUD_ADMIN_SECRET }} + QUOTA_BILLING_URL: ${{ secrets.PROD_TINYCLOUD_QUOTA_URL }} + run: | + set -euo pipefail + set +x + python3 scripts/ci/reconcile-quota.py + # TC-465: Share requires the complete Policy/v3 control plane before it # can create or receive an exact-email share. A version-only check let a # pre-TC-405 image look healthy while every required route returned 404. @@ -729,22 +1058,6 @@ jobs: /policy/v3/deliveries/authorize ) - # The direct TEE ingress obtains/loads its certificate after the - # node's public /version endpoint is already healthy. Wait for that - # TLS endpoint before probing routes so a successful rollout does - # not fail on a transient SSL_ERROR_SYSCALL during ingress startup. - for attempt in $(seq 1 30); do - if curl -fsS --max-time 10 "${NODE_ORIGIN}/version" > /dev/null; then - echo "Direct TEE ingress is ready." - break - fi - if [ "${attempt}" = "30" ]; then - echo "::error::Direct TEE ingress did not become ready after 150s." - exit 1 - fi - sleep 5 - done - for route in "${ROUTES[@]}"; do RESPONSE_FILE="$(mktemp)" STATUS="$(curl -sS --max-time 15 -o "${RESPONSE_FILE}" -w '%{http_code}' \ @@ -777,6 +1090,16 @@ jobs: rm -f "${RESPONSE_FILE}" done + - name: Report unverified storage overrides + if: >- + ${{ always() && steps.deploy.outputs.cvm_mutation_attempted == 'true' && steps.quotas.outcome != 'success' }} + run: | + set -euo pipefail + MESSAGE="OVERRIDES UNVERIFIED: a CVM mutation was attempted without successful authorized quota reconciliation. Authorized operator must resolve before another dispatch." + echo "::error::${MESSAGE}" + printf '%s\n' "${MESSAGE}" >> "$GITHUB_STEP_SUMMARY" + exit 1 + # TC-1176: the 2026-10-14 read hang began ~40 min after a promote. After any # Phala deploy attempt (including rollbacks and failed verifications, which # may still have changed prod), start the Write Probe's dense promote watch: @@ -794,4 +1117,16 @@ jobs: - name: Dispatch Write Probe promote watch env: GH_TOKEN: ${{ github.token }} - run: gh workflow run write-probe.yml -R "${GITHUB_REPOSITORY}" --ref main -f mode=dense + IMAGE: ${{ needs.deploy-phala.outputs.image }} + IMAGE_VERSION: ${{ needs.deploy-phala.outputs.version }} + IMAGE_LABEL: ${{ needs.deploy-phala.outputs.image_label }} + IMAGE_REVISION: ${{ needs.deploy-phala.outputs.revision }} + run: | + set -euo pipefail + if [ -n "${IMAGE}" ]; then + echo "Promote watch for ${IMAGE_LABEL} (version ${IMAGE_VERSION}) revision ${IMAGE_REVISION}: ${IMAGE}" + else + echo "Promote watch after deployment attempt; image validation did not complete." + fi + gh workflow run write-probe.yml -R "${GITHUB_REPOSITORY}" --ref main -f mode=dense + diff --git a/docs/releasing.md b/docs/releasing.md index d8004ef5..3558bf88 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -60,12 +60,18 @@ built from a `1.9.0` tree); the `version-guard` job now blocks the second one. ## Promoting to production (main-only) Production is built and promoted **only from `main`** (TC-940, 2026-10-13). -There is no release branch. The `deploy-phala` job refuses to run unless the -dispatched ref is `refs/heads/main` or a `refs/tags/vX.Y.Z` tag whose commit is -an ancestor of `origin/main` (`scripts/ci/check-deploy-ref.sh`, run both in -`version-guard` before any build and again at the start of `deploy-phala`). A -dispatch from any other branch or tag fails with -`Deploy refused (main-only)`. +There is no release branch. Immutable digest promotions require +`refs/heads/main`; normal build dispatches may also use a `vX.Y.Z` tag in +main's history (`scripts/ci/check-deploy-ref.sh`, before builds and again +inside `deploy-phala`). Other refs fail `Deploy refused (main-only)`. + +**Release order:** approved #310 must never merge alone. Land #310 and its +stacked recovery change atomically as one lineage, preserving emergency +re-promotion and security rollback first. #309's sidecar compose belongs +only to the later approved logging window, not this workflow/security +release. The security card remains rollback-not-ready until the combined +path is reviewed and landed. These docs grant no merge or dispatch authority. +Merging either workflow change is a push, not a deployment dispatch. ### The `production` environment (TC-971) @@ -73,8 +79,9 @@ The ref check above runs inside the workflow file of the dispatched ref, so it only protects refs whose `docker.yml` already has it. `workflow_dispatch` runs the workflow file at the selected ref, and repository or organization secrets are readable by every ref. To close that, every job that reads production -secrets runs in the GitHub `production` environment, and those secrets exist -only there: +secrets runs in GitHub's `production` environment. Most deploy secrets +exist only there; `PROD_TINYCLOUD_ADMIN_SECRET` is the approved repository- +scoped exception and is consumed only inside the deploy job. | Workflow | Job | Production secrets | | --- | --- | --- | @@ -92,10 +99,11 @@ Environment settings (Settings → Environments → `production`): `PROD_TINYCLOUD_S3_BUCKET`, `PROD_TINYCLOUD_S3_ENDPOINT`, `PROD_TINYCLOUD_AWS_ACCESS_KEY_ID`, `PROD_TINYCLOUD_AWS_SECRET_ACCESS_KEY`, `PROD_TINYCLOUD_AWS_REGION`, `PROD_TINYCLOUD_QUOTA_URL`, - `PROD_TINYCLOUD_ADMIN_SECRET`, `PROD_TINYCLOUD_SHARE_TRUST_BUNDLE_BASE64`, + `PROD_TINYCLOUD_SHARE_TRUST_BUNDLE_BASE64`, `PROD_TINYCLOUD_CLOUDFLARE_API_TOKEN`, `PROD_TINYCLOUD_DSTACK_GATEWAY_DOMAIN`, - `PROD_TINYCLOUD_CERTBOT_EMAIL`, `PROD_TINYCLOUD_KEYS_SECRET`. -- None of these may also exist as a repository secret, and this repository + `PROD_TINYCLOUD_CERTBOT_EMAIL`, `PROD_TINYCLOUD_KEYS_SECRET`, + `PROD_TINYCLOUD_BETTERSTACK_SOURCE_TOKEN`. +- Except for the explicit node admin exception, none may also exist as a repository secret; this repository must not be selected for the organization secret `PHALA_CLOUD_API_KEY`. A same-named repository or organization secret is still readable by a job without `environment: production`. @@ -119,6 +127,47 @@ closes that gap. `TC780_CUTOVER_READY` stays a repository variable: the build jobs, which have no environment, read it too. `GITHUB_TOKEN` is the automatic token. +The quota steps send `PROD_TINYCLOUD_ADMIN_SECRET` to billing's +`/api/admin/storage-comp/:space_id` as well as to the node's admin API. +Billing's storage-comp guard checks the Worker secret +`TINYCLOUD_ADMIN_SECRET`, which is the node/billing shared admin value: +billing uses it for its own calls to the node's `/admin/*` routes, and a +rotation sets the GitHub secret and the `tinycloud-billing` Worker secret +to one new value. No separate billing runner secret exists; rotating the +shared value must update both holders together, or the quota steps fail +closed with `quota API rejected the request`. The value is never passed in +argv/logs and never added to the CVM env or `allowed_envs` under a billing +name. A missing credential or an incompatible billing URL fails before +compose/CVM mutation. Billing URLs derive strictly from the same HTTPS origin and +`/api/quota` path of `PROD_TINYCLOUD_QUOTA_URL`; no URL is printed. + +Every `deploy_phala=true` command must explicitly include +**`-f authorize_quota_values=true`**. This authorizes the reviewed values +and authority in `ops/quota-overrides.json` for that one dispatch; it is +not N0-specific or permission to silently reuse old values. Missing +authorization or missing/unknown authority fails before any compose, +CVM or whitelist mutation. + +`authority=node-override` currently applies the reviewed limits after the +last restart and required health. Both exact spaces must exist once with +positive usage in authenticated `/admin/usage`. Before ANY PUT, both billing +storage-comp reads must prove no active comp: 404 logs only +`billing storage-comp unavailable`; valid `active:false` also permits the +legacy restore. Active comp fails `authority mismatch: flip to billing`. +Auth/5xx/transport/malformed responses fail closed, never count as absence. +Each PUT requires GET readback of both override and effective values. + +The later reviewed authority flip to `billing` is separate from this PR and +follows TC-1162 migration. Billing mode never PUTs/DELETEs a node override: +active absolute per-space `storage_limit_bytes` must cover each reviewed +floor, and node GET must show `override_bytes=null`. A non-null override +fails `stale legacy override`. A cold node cache is not a failure: billing +is authoritative and the node fetches on the next write (≤60 s TTL). +Do not compare incremental account `comp_bytes` with absolute space limits. +No raw response, identity list, credential or error body is logged. +Any attempted restart without verified reconciliation is fatal +**OVERRIDES UNVERIFIED**, requiring authorized operator/TC-1161 fallback. + If a job that names the environment ever runs before the environment exists, GitHub creates `production` with no branch policy. Create the environment and its branch policy before any workflow that names it runs. @@ -133,20 +182,25 @@ its branch policy before any workflow that names it runs. --format '{{ json .Image.Config.Labels }}' # revision == tag commit, version == X.Y.Z-dstack ``` -2. **Promote from the tag:** +2. **Promote the recorded immutable digest from main:** ```sh - gh workflow run docker.yml -R TinyCloudLabs/tinycloud-node --ref vX.Y.Z \ - -f deploy_phala=true -f image_version=X.Y.Z \ - -f deploy_image_digest=sha256: \ + gh workflow run docker.yml -R TinyCloudLabs/tinycloud-node --ref main \ + -f deploy_phala=true -f authorize_quota_values=true \ + -f image_version=X.Y.Z \ + -f deploy_image_digest=sha256: \ + -f rollback_image_version=X.Y.Z \ + -f rollback_image_revision= \ -f sql_identity_fence=false ``` - - Dispatch on the **tag**, not `main`. The deploy requires the image's - `org.opencontainers.image.revision` label to equal `HEAD` of the dispatched - ref; the tag always points at the build commit, while `main` only does - until the next merge. (`--ref main` works only while `main`'s tip is the - tag commit.) + - `image_version` and recorded labels describe the immutable image, + never an unrelated current crate version. The revision must be in + main history. A future non-N0 digest uses this same contract. + - The reviewed N0 compose remains raw-hash locked to its exact image; + it cannot substitute a future security image. Before the later + logging window, the ordinary main `:dstack` compose supports forward + image promotions without installing the sidecar. - `sql_identity_fence=false` is required. The input defaults to `true`, which re-fences SQL/DuckDB in production (503s). - The production ancestry guard still applies: the revision prod is running @@ -206,41 +260,103 @@ its branch policy before any workflow that names it runs. ## Rolling back -Roll back by promoting an older **recorded** digest, dispatched on `main`. -Keep a record (digest, version, revision) of every digest you promote. +Keep the digest, actual OCI version and actual revision of every promoted +image. Sealed-runtime preflight is mandatory on recovery, rollback and +forward paths; an image that cannot validate config fails before restart. +The same general quota authority/reconciliation gate runs after the final +env operation plus version/ingress health, before ancillary Policy probes. +The downstream watcher cannot skip quota restoration. + +### Emergency same-image recovery (12c) + +After the combined workflow is reviewed/landed and a named operator has fresh +values authorization, this is the single copy-paste recovery command. It uses +the **current main compose**, whether the exact pre-N0 template or the exact +reviewed N0 bytes. It does not opt into an N0-only snapshot or change the +current logging policy. No command has been dispatched by this work. + +```sh +gh workflow run docker.yml -R TinyCloudLabs/tinycloud-node --ref main \ + -f deploy_phala=true \ + -f authorize_quota_values=true \ + -f image_version=1.20.6 \ + -f deploy_image_digest=sha256:4bdae4fc232d886bb56bfbce4ec62ed4b8fb4e1369a01f6afb73d06dd68c188f \ + -f rollback_image_version=1.20.6 \ + -f rollback_image_revision=e18578ff9162986ddd813259c60f73866e3e60d1 \ + -f sql_identity_fence=false +``` + +The served/OCI identity remains **1.20.6-dstack / e18578ff9162986ddd813259c60f73866e3e60d1**. +The N0 raw lock is unchanged: +`ca110b0bae6d5a06a1002fb80df3f71cffec11659c9f9dbfa5d72624735546f6`, +from #309 reviewed head `44eeb20a1461ef66b5af492e3be4c42be2728b67`, +[approval](https://github.com/TinyCloudLabs/tinycloud-node/pull/309#issuecomment-6224650050). +Current pre-N0 tag/pinned equivalents are separately hash-checked; arbitrary +same-image config substitutions fail before mutation. + +### TC-1302 captured pre-N0 rollback + +This is a compose rollback, not merely a node digest selection. The source +`ops/rollback/pre-n0-compose.yaml` is byte-identical to Amos's capture: +raw SHA256 **`df239749ca0a9c09893a855ae2a8abb94090b8556515ad002105c58ac91f4c91`**. +Its provenance was reconstructed from the frozen public template with only +the immutable node pin; the private saved file's contents were not read. +The node/ingress images, app env, storage and ingress topology stay fixed. +Other paths, symlinks, source drift and edited variants fail before mutation. + +**Privacy choice remains pending Sam's explicit card answer.** The default +is `unselected`, which refuses rollback. Amos records the approved choice in +repository variable `TC959_ROLLBACK_PRIVACY_MODE`; the chosen input must match. +Neither command below is authorization to choose that policy. Use exactly +the command matching Sam's answer; do not add an override/bypass flag. + +If Sam explicitly selects **keep prior json-file logging**, retain the +captured 3×50 MB local logging policy: ```sh gh workflow run docker.yml -R TinyCloudLabs/tinycloud-node --ref main \ -f deploy_phala=true \ - -f image_version= \ - -f deploy_image_digest=sha256: \ - -f rollback_image_version= \ - -f rollback_image_revision= \ + -f authorize_quota_values=true \ + -f rollback_pre_n0=true \ + -f rollback_privacy_mode=keep-json-file \ + -f image_version=1.20.6 \ + -f deploy_image_digest=sha256:4bdae4fc232d886bb56bfbce4ec62ed4b8fb4e1369a01f6afb73d06dd68c188f \ + -f rollback_image_version=1.20.6 \ + -f rollback_image_revision=e18578ff9162986ddd813259c60f73866e3e60d1 \ -f sql_identity_fence=false ``` -- The `rollback_image_*` inputs replace the "revision label equals HEAD" - check with "labels equal these recorded values". The rollback revision must - be an ancestor of `main`, which every production revision since TC-940 is. -- `allow_non_descendant` is **not** needed. The ancestry guard compares the - revision prod is currently running with `main`, and that revision is on - `main`. Keep `allow_non_descendant` as break-glass for when Phala or GHCR - cannot be queried for the running revision; the main-only check still - applies with it set. -- The compose file and the ingress image come from `main`, not from the old - release; only the node binary rolls back. Check - `tinycloud-core/src/migrations` between the two revisions first: a binary - rollback across a schema migration is not DB-safe. -- The sealed-config preflight is skipped for rollbacks, and the post-deploy - `/version` check expects `rollback_image_version`. Run the same manual - verification as for a release. -- Rollback runs in the `production` environment like any other deploy, and - `main` is admitted by its branch policy. No environment change is needed. -- Do not roll back by dispatching on an old tag or a retired branch. Old refs - carry an older `docker.yml` without the main-only guard or the `production` - environment, so they get no deploy secrets and fail. The retired deploy - branches (`feat/tc-780-release-line`, - `Codex/roman/rollback-meeting-node-20260915`) are also locked. +If Sam explicitly selects **stop FUTURE local logs**, the reviewed derivative +sets only the node and ingress logging drivers to `none`: + +```sh +gh workflow run docker.yml -R TinyCloudLabs/tinycloud-node --ref main \ + -f deploy_phala=true \ + -f authorize_quota_values=true \ + -f rollback_pre_n0=true \ + -f rollback_privacy_mode=stop-future-local-logs \ + -f image_version=1.20.6 \ + -f deploy_image_digest=sha256:4bdae4fc232d886bb56bfbce4ec62ed4b8fb4e1369a01f6afb73d06dd68c188f \ + -f rollback_image_version=1.20.6 \ + -f rollback_image_revision=e18578ff9162986ddd813259c60f73866e3e60d1 \ + -f sql_identity_fence=false +``` + +That derivative's raw SHA256 is +`315ceca54bf8bfefe2b395b4097fe38ed8ec847fa0688bc873220bd65efc99ec`. +Both the original capture and the selected derivative are fixed-path/hash +validated. **Stopping future logs does not delete existing files. There is +NO truncation/deletion path in this PR.** Any destructive action needs a +separate explicit authorization and reviewed procedure. + +Never silently restore raw retention on forward/current-compose promotion: +the privacy input is rollback-only and mutually exclusive. Existing private +logs/private sysinfo flags remain; operator readback must preserve +logs/sysinfo/tcbinfo = false/false/true. Do not use retired branches or tags +for these digest commands. `allow_non_descendant` is not normally needed; +all recorded revisions must remain in main history. Binary rollback across +a database schema migration still needs a separate DB-safety decision. + ## Why the version bump is manual diff --git a/ops/quota-overrides.json b/ops/quota-overrides.json new file mode 100644 index 00000000..b5899847 --- /dev/null +++ b/ops/quota-overrides.json @@ -0,0 +1,13 @@ +{ + "authority": "node-override", + "overrides": { + "tinycloud:pkh:eip155:1:0xd559CCd9EB87c530A9a349262669386dE93cf412:applications": { + "limit_bytes": 5368709120, + "_comment": "Confirmed by Amos's read-only TC-1161 D1 quota_overrides and quota_override_audit extract, 2026-10-16: applied 2026-10-16T17:37:16Z, last_node_status 200. Sam authorizes these values; every deployment requires fresh authorize_quota_values opt-in. Flip authority to billing only through a separate reviewed PR after TC-1162 migration. Prove both exact spaces and positive usage, and reject active billing comp before any legacy node PUT." + }, + "tinycloud:pkh:eip155:1:0xd559CCd9EB87c530A9a349262669386dE93cf412:secrets": { + "limit_bytes": 268721121, + "_comment": "Space ID verified read-only by Amos via Sam's tc-secrets release-keys and monitoring profiles; value applied manually before the dashboard existed. No secret values were read. TC-1161 D1 parity remains pending Sam's card. Every authorized deployment must prove this full ID with positive usage; a quota GET echo is not existence evidence. Existing node/billing authority may never be silently overridden." + } + } +} diff --git a/ops/rollback/pre-n0-compose.yaml b/ops/rollback/pre-n0-compose.yaml new file mode 100644 index 00000000..f93e9f0b --- /dev/null +++ b/ops/rollback/pre-n0-compose.yaml @@ -0,0 +1,134 @@ +# dstack deployment - Topology 2: Postgres + S3 +# tinycloud-node in the CVM, Postgres external. +# Database credentials encrypted by dstack KMS. +# Column encryption enabled for sensitive data. +# +# All ${...} variables should be encrypted using dstack's KMS public key +# before deployment. The operator never sees plaintext values. + +services: + tinycloud: + # DuckDB-enabled images are opt-in and use the -duckdb suffix, for example: + # ghcr.io/tinycloudlabs/tinycloud-node:1.4.0-dstack-duckdb + image: ghcr.io/tinycloudlabs/tinycloud-node@sha256:4bdae4fc232d886bb56bfbce4ec62ed4b8fb4e1369a01f6afb73d06dd68c188f + restart: unless-stopped + # TC-970: cap the node's Docker log (stdout: one line per request plus + # application logs) at 3 x 50 MB instead of the guest daemon default + # (current dstack OS images: json-file, 10 x 100 MB). json-file keeps the + # dstack log API (`/logs/`) working. + logging: + driver: json-file + options: + max-size: "50m" + max-file: "3" + ports: + - "8000:8000" + volumes: + - /var/run/dstack.sock:/var/run/dstack.sock + environment: + TINYCLOUD_KEYS_TYPE: Dstack + TINYCLOUD_STORAGE_DATABASE: "${DATABASE_URL}" + TINYCLOUD_STORAGE_BLOCKS_TYPE: S3 + TINYCLOUD_STORAGE_BLOCKS_BUCKET: "${S3_BUCKET}" + TINYCLOUD_STORAGE_BLOCKS_ENDPOINT: "${S3_ENDPOINT}" + AWS_ACCESS_KEY_ID: "${AWS_KEY}" + AWS_SECRET_ACCESS_KEY: "${AWS_SECRET}" + AWS_DEFAULT_REGION: "${AWS_REGION}" + TINYCLOUD_LOG_LEVEL: normal + # Enable Prometheus telemetry (metrics, span histograms, pool gauges). + # The metrics handler is intentionally NOT gateway-exposed: port 8001 is + # not published here, so the unauthenticated /metrics endpoint stays + # internal to the CVM. + TINYCLOUD_TELEMETRY__ENABLED: "true" + # The managed Postgres instance allows max_connections = 25 (measured + # 2026-07-29, with some of those reserved for superuser roles). The pool + # default of 100 cannot be satisfied: the node opened and held slots + # until the instance ran out, and further requests failed with + # "Failed to acquire connection from pool: Connection pool timed out" + # after the 30s sqlx default. Keep this below the instance limit with + # headroom for admin/psql access, and raise it only alongside the + # instance's own max_connections. + TINYCLOUD_DATABASE__MAX_CONNECTIONS: "15" + # Required for every manual TC-780 deploy. Keep true through offline + # verification; only the final, explicitly reviewed redeploy sets false. + TINYCLOUD_DATABASE__WRITE_FENCE: "${SQL_IDENTITY_FENCE:?set true during cutover, false after verification}" + TINYCLOUD_CORS: "true" + # --- addressed sharing (TC-397) ------------------------------------ + # + # The public trust document shared with Share and OpenCredentials, + # delivered inline because nothing else can reach this container: the + # image is `FROM scratch` (no shell, so share-api's decode-to-tmpfs + # entrypoint cannot work here) and a dstack deployment uploads only + # this compose file (so there is no host path to bind-mount from). + # + # Seal the base64 of the reviewed trust-bundle.production.json — the + # same bytes share-api serves — into the CVM's encrypted environment as + # SHARE_TRUST_BUNDLE_BASE64. Same variable name and same wire form + # share-api already uses, so one sealed value feeds both services and + # cannot drift between them. + # + # The `:?` form is deliberate: compose refuses to interpolate a partial + # deployment, failing loudly at deploy time rather than letting the node + # die at boot with "share email trust bundle is required". + TINYCLOUD_SHARE_EMAIL__ENABLED: "true" + TINYCLOUD_SHARE_EMAIL__TRUST_BUNDLE_BASE64: "${SHARE_TRUST_BUNDLE_BASE64:?seal the reviewed share-email trust bundle, base64-encoded}" + # The sealed DATABASE_URL already carries sslmode=verify-full as its sole + # query parameter. ROOT_CERT_PATH stays unset so verify-full resolves + # against the compiled-in public root set (TC-363). + TINYCLOUD_SHARE_EMAIL__POSTGRES_TLS__SSLMODE: verify-full + TINYCLOUD_QUOTA_URL: "${TINYCLOUD_QUOTA_URL}" + TINYCLOUD_ADMIN_SECRET: "${TINYCLOUD_ADMIN_SECRET}" + ROCKET_ADDRESS: "0.0.0.0" + ROCKET_MAX_BLOCKING: "1024" + + dstack-ingress: + image: ghcr.io/tinycloudlabs/tinycloud-node-ingress@sha256:63c1c44e2ded5b73a5f174807b91ca677826974df0f9ab3045f6ec87865a5744 + ports: + - "443:443" + depends_on: + - tinycloud + environment: + - DOMAIN=tee.node.tinycloud.xyz + - TARGET_ENDPOINT=http://tinycloud:8000 + - CLOUDFLARE_API_TOKEN=${CLOUDFLARE_API_TOKEN} + # CNAME target the ingress publishes for DOMAIN. Must not start with `_`: + # Android's resolver (bionic res_hnok) rejects a CNAME target whose first + # label is `_`, so `_.` is unresolvable on Android (TC-513). + # `gateway.` is the target Phala Cloud advertises (gateway.cname) + # and upstream dstack-ingress documents; routing still uses SNI + the + # _dstack-app-address TXT record, so TLS stays end-to-end into the CVM. + - GATEWAY_DOMAIN=gateway.${DSTACK_GATEWAY_DOMAIN} + - CERTBOT_EMAIL=${CERTBOT_EMAIL} + - SET_CAA=true + - DNS_PROVIDER=cloudflare + - CLIENT_MAX_BODY_SIZE=32m + volumes: + - /var/run/dstack.sock:/var/run/dstack.sock + - cert-data:/etc/letsencrypt + restart: unless-stopped + # TC-970: the ingress logs one JSON line per request to stdout. Cap its + # Docker log at 3 x 50 MB instead of relying on the guest daemon default + # (current dstack OS images ship json-file with 10 x 100 MB). json-file + # keeps the dstack log API (`/logs/`) working. + logging: + driver: json-file + options: + max-size: "50m" + max-file: "3" + + # The dstack gateway's TLS-terminating routes (-.:443) + # only forward to compose-published ports, so guest sshd on :22 is + # unreachable via `phala ssh` even with authorized_keys installed. + # Publish a TCP forwarder on 10022 that relays to guest sshd via the + # docker host gateway. Port 10022 exposes ONLY sshd, which is key-gated. + ssh-gateway: + image: alpine/socat:1.8.1.3@sha256:188fe0a22182f81c16def9d1137930121eaa3023f427164e32c0d2f118f79dd6 + restart: unless-stopped + ports: + - "10022:10022" + extra_hosts: + - "host.docker.internal:host-gateway" + command: TCP-LISTEN:10022,fork,reuseaddr TCP:host.docker.internal:22 + +volumes: + cert-data: diff --git a/ops/rollback/pre-n0-stop-local-logs.yaml b/ops/rollback/pre-n0-stop-local-logs.yaml new file mode 100644 index 00000000..3e38240b --- /dev/null +++ b/ops/rollback/pre-n0-stop-local-logs.yaml @@ -0,0 +1,120 @@ +# dstack deployment - Topology 2: Postgres + S3 +# tinycloud-node in the CVM, Postgres external. +# Database credentials encrypted by dstack KMS. +# Column encryption enabled for sensitive data. +# +# All ${...} variables should be encrypted using dstack's KMS public key +# before deployment. The operator never sees plaintext values. + +services: + tinycloud: + # DuckDB-enabled images are opt-in and use the -duckdb suffix, for example: + # ghcr.io/tinycloudlabs/tinycloud-node:1.4.0-dstack-duckdb + image: ghcr.io/tinycloudlabs/tinycloud-node@sha256:4bdae4fc232d886bb56bfbce4ec62ed4b8fb4e1369a01f6afb73d06dd68c188f + restart: unless-stopped + logging: + driver: none + ports: + - "8000:8000" + volumes: + - /var/run/dstack.sock:/var/run/dstack.sock + environment: + TINYCLOUD_KEYS_TYPE: Dstack + TINYCLOUD_STORAGE_DATABASE: "${DATABASE_URL}" + TINYCLOUD_STORAGE_BLOCKS_TYPE: S3 + TINYCLOUD_STORAGE_BLOCKS_BUCKET: "${S3_BUCKET}" + TINYCLOUD_STORAGE_BLOCKS_ENDPOINT: "${S3_ENDPOINT}" + AWS_ACCESS_KEY_ID: "${AWS_KEY}" + AWS_SECRET_ACCESS_KEY: "${AWS_SECRET}" + AWS_DEFAULT_REGION: "${AWS_REGION}" + TINYCLOUD_LOG_LEVEL: normal + # Enable Prometheus telemetry (metrics, span histograms, pool gauges). + # The metrics handler is intentionally NOT gateway-exposed: port 8001 is + # not published here, so the unauthenticated /metrics endpoint stays + # internal to the CVM. + TINYCLOUD_TELEMETRY__ENABLED: "true" + # The managed Postgres instance allows max_connections = 25 (measured + # 2026-07-29, with some of those reserved for superuser roles). The pool + # default of 100 cannot be satisfied: the node opened and held slots + # until the instance ran out, and further requests failed with + # "Failed to acquire connection from pool: Connection pool timed out" + # after the 30s sqlx default. Keep this below the instance limit with + # headroom for admin/psql access, and raise it only alongside the + # instance's own max_connections. + TINYCLOUD_DATABASE__MAX_CONNECTIONS: "15" + # Required for every manual TC-780 deploy. Keep true through offline + # verification; only the final, explicitly reviewed redeploy sets false. + TINYCLOUD_DATABASE__WRITE_FENCE: "${SQL_IDENTITY_FENCE:?set true during cutover, false after verification}" + TINYCLOUD_CORS: "true" + # --- addressed sharing (TC-397) ------------------------------------ + # + # The public trust document shared with Share and OpenCredentials, + # delivered inline because nothing else can reach this container: the + # image is `FROM scratch` (no shell, so share-api's decode-to-tmpfs + # entrypoint cannot work here) and a dstack deployment uploads only + # this compose file (so there is no host path to bind-mount from). + # + # Seal the base64 of the reviewed trust-bundle.production.json — the + # same bytes share-api serves — into the CVM's encrypted environment as + # SHARE_TRUST_BUNDLE_BASE64. Same variable name and same wire form + # share-api already uses, so one sealed value feeds both services and + # cannot drift between them. + # + # The `:?` form is deliberate: compose refuses to interpolate a partial + # deployment, failing loudly at deploy time rather than letting the node + # die at boot with "share email trust bundle is required". + TINYCLOUD_SHARE_EMAIL__ENABLED: "true" + TINYCLOUD_SHARE_EMAIL__TRUST_BUNDLE_BASE64: "${SHARE_TRUST_BUNDLE_BASE64:?seal the reviewed share-email trust bundle, base64-encoded}" + # The sealed DATABASE_URL already carries sslmode=verify-full as its sole + # query parameter. ROOT_CERT_PATH stays unset so verify-full resolves + # against the compiled-in public root set (TC-363). + TINYCLOUD_SHARE_EMAIL__POSTGRES_TLS__SSLMODE: verify-full + TINYCLOUD_QUOTA_URL: "${TINYCLOUD_QUOTA_URL}" + TINYCLOUD_ADMIN_SECRET: "${TINYCLOUD_ADMIN_SECRET}" + ROCKET_ADDRESS: "0.0.0.0" + ROCKET_MAX_BLOCKING: "1024" + + dstack-ingress: + image: ghcr.io/tinycloudlabs/tinycloud-node-ingress@sha256:63c1c44e2ded5b73a5f174807b91ca677826974df0f9ab3045f6ec87865a5744 + ports: + - "443:443" + depends_on: + - tinycloud + environment: + - DOMAIN=tee.node.tinycloud.xyz + - TARGET_ENDPOINT=http://tinycloud:8000 + - CLOUDFLARE_API_TOKEN=${CLOUDFLARE_API_TOKEN} + # CNAME target the ingress publishes for DOMAIN. Must not start with `_`: + # Android's resolver (bionic res_hnok) rejects a CNAME target whose first + # label is `_`, so `_.` is unresolvable on Android (TC-513). + # `gateway.` is the target Phala Cloud advertises (gateway.cname) + # and upstream dstack-ingress documents; routing still uses SNI + the + # _dstack-app-address TXT record, so TLS stays end-to-end into the CVM. + - GATEWAY_DOMAIN=gateway.${DSTACK_GATEWAY_DOMAIN} + - CERTBOT_EMAIL=${CERTBOT_EMAIL} + - SET_CAA=true + - DNS_PROVIDER=cloudflare + - CLIENT_MAX_BODY_SIZE=32m + volumes: + - /var/run/dstack.sock:/var/run/dstack.sock + - cert-data:/etc/letsencrypt + restart: unless-stopped + logging: + driver: none + + # The dstack gateway's TLS-terminating routes (-.:443) + # only forward to compose-published ports, so guest sshd on :22 is + # unreachable via `phala ssh` even with authorized_keys installed. + # Publish a TCP forwarder on 10022 that relays to guest sshd via the + # docker host gateway. Port 10022 exposes ONLY sshd, which is key-gated. + ssh-gateway: + image: alpine/socat:1.8.1.3@sha256:188fe0a22182f81c16def9d1137930121eaa3023f427164e32c0d2f118f79dd6 + restart: unless-stopped + ports: + - "10022:10022" + extra_hosts: + - "host.docker.internal:host-gateway" + command: TCP-LISTEN:10022,fork,reuseaddr TCP:host.docker.internal:22 + +volumes: + cert-data: diff --git a/scripts/check-deployment-policy-probes.mjs b/scripts/check-deployment-policy-probes.mjs index 932c055f..29dac8fe 100644 --- a/scripts/check-deployment-policy-probes.mjs +++ b/scripts/check-deployment-policy-probes.mjs @@ -1,5 +1,10 @@ -import { readFileSync } from "node:fs"; +import assert from "node:assert/strict"; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { spawnSync } from "node:child_process"; +const workflow = readFileSync(".github/workflows/docker.yml", "utf8"); const expectedRoutes = [ "/policy/v3/enforcer-bindings", "/policy/v3/policies", @@ -7,44 +12,98 @@ const expectedRoutes = [ "/policy/v3/delegations", "/policy/v3/deliveries/authorize", ]; -const workflow = readFileSync(".github/workflows/docker.yml", "utf8"); -const source = readFileSync("tinycloud-node-server/src/policy_v3.rs", "utf8"); -const probeStart = workflow.indexOf("- name: Verify deployed Policy/v3 routes"); -const probeEnd = workflow.indexOf("\n - name:", probeStart + 1); -if (probeStart === -1) { - throw new Error("Policy/v3 deployment probe step is missing or malformed"); +function shellForStep(name) { + const lines = workflow.split("\n"); + const start = lines.indexOf(` - name: ${name}`); + assert(start >= 0, `missing deployment check: ${name}`); + const run = lines.indexOf(" run: |", start); + assert(run > start, `missing deployment script: ${name}`); + const body = []; + for (const line of lines.slice(run + 1)) { + if (line && !line.startsWith(" ")) break; + body.push(line.slice(10)); + } + return body.join("\n"); } -const probe = workflow.slice(probeStart, probeEnd === -1 ? undefined : probeEnd); -const routesBlock = probe.match(/ROUTES=\(\n([\s\S]*?)\n \)/)?.[1]; -const routes = routesBlock?.match(/^\s+(\/\S+)$/gm)?.map((route) => route.trim()); +const directory = mkdtempSync(join(tmpdir(), "tinycloud-deploy-probes-")); +const logPath = join(directory, "requests.jsonl"); +const mockCurl = `#!${process.execPath} +const fs = require("node:fs"); +const args = process.argv.slice(2); +const url = new URL(args.at(-1)); +if (url.origin !== "https://tee.node.tinycloud.xyz") process.exit(90); +const valueAfter = flag => args[args.indexOf(flag) + 1]; +const post = args.includes("-X") && valueAfter("-X") === "POST"; +const headers = args.flatMap((arg, index) => arg === "-H" ? [args[index + 1].toLowerCase()] : []); +const safeProbe = post && valueAfter("--data") === "{}" && + headers.includes("accept: application/json") && headers.includes("content-type: application/json") && + !headers.some(header => header.startsWith("authorization:")); +fs.appendFileSync(process.env.PROBE_LOG, JSON.stringify({path:url.pathname, safeProbe}) + "\\n"); +if (url.pathname === "/version") { + if (process.env.TLS_UNREADY === "true") process.exit(22); + console.log(JSON.stringify({version:process.env.READY_VERSION})); +} else { + if (!safeProbe) process.exit(91); + const statuses = JSON.parse(process.env.PROBE_STATUSES); + const status = statuses[url.pathname] ?? 404; + if (args.includes("-o")) fs.writeFileSync(valueAfter("-o"), "{}"); + process.stdout.write(String(status)); +} +`; +const mockJq = `#!${process.execPath} +let input = ""; +process.stdin.on("data", chunk => input += chunk); +process.stdin.on("end", () => { + try { console.log(JSON.parse(input).version); } catch { process.exitCode = 1; } +}); +`; +let cases = 0; -if (JSON.stringify(routes) !== JSON.stringify(expectedRoutes)) { - throw new Error(`unexpected Policy/v3 deployment probe routes: ${JSON.stringify(routes)}`); +function run(name, statuses, success, options = {}) { + writeFileSync(logPath, ""); + const result = spawnSync("bash", ["-e", "-o", "pipefail", "-c", shellForStep(name)], { + env: { + PATH: `${directory}:${process.env.PATH}`, + EXPECTED: "1.20.6", + PROBE_LOG: logPath, + PROBE_STATUSES: JSON.stringify(statuses), + READY_VERSION: options.version ?? "1.20.6", + TLS_UNREADY: options.tlsUnready ? "true" : "false", + }, + encoding: "utf8", + timeout: 15000, + }); + // Do not forward captured output: workflow errors may contain server data. + assert.equal(result.error, undefined, "synthetic probe execution failed"); + assert.equal(result.status === 0, success, "deployment probe accepted/rejected the wrong response"); + cases++; + return readFileSync(logPath, "utf8").trim().split("\n").filter(Boolean).map(line => JSON.parse(line)); } -for (const route of expectedRoutes) { - if (!source.includes(`#[post("${route}"`)) { - throw new Error(`deployment probe route is not a Node-owned POST route: ${route}`); + +try { + writeFileSync(join(directory, "curl"), mockCurl, { mode: 0o700 }); + writeFileSync(join(directory, "jq"), mockJq, { mode: 0o700 }); + writeFileSync(join(directory, "sleep"), "#!/bin/sh\nexit 0\n", { mode: 0o700 }); + const defaults = Object.fromEntries(expectedRoutes.map(route => [route, route.endsWith("/authorize") ? 401 : 400])); + for (const general of [400, 422]) { + const statuses = Object.fromEntries(expectedRoutes.map(route => [route, route.endsWith("/authorize") ? 401 : general])); + const requests = run("Verify deployed Policy/v3 routes", statuses, true); + assert.deepEqual(new Set(requests.map(request => request.path)), new Set(expectedRoutes)); + assert.equal(requests.length, expectedRoutes.length, "each required route must be probed exactly once"); + assert(requests.every(request => request.safeProbe), "probe must be unsigned POST with intentionally empty JSON"); } -} -if (probe.includes("/share/")) { - throw new Error("deployment probe must not use a retired /share/* route"); -} -for (const invariant of [ - 'NODE_ORIGIN="https://tee.node.tinycloud.xyz"', - '"${NODE_ORIGIN}/version"', - "--data '{}'", - 'if [ "${route}" = "/policy/v3/deliveries/authorize" ]', - 'EXPECTED="401"', - 'case "${STATUS}" in 401) VALID=true', - 'EXPECTED="400 or 422"', - 'case "${STATUS}" in 400|422) VALID=true', -]) { - if (!probe.includes(invariant)) { - throw new Error(`Policy/v3 deployment probe lost required invariant: ${invariant}`); + for (const bad of [200, 400, 422, 500]) { + run("Verify deployed Policy/v3 routes", { ...defaults, [expectedRoutes.at(-1)]: bad }, false); } -} -if (/400\|401|401\|4|"\$\{STATUS\}" = "401"/.test(probe)) { - throw new Error("the unsigned delivery probe must require exactly HTTP 401"); + for (const bad of [200, 401, 404, 500]) { + run("Verify deployed Policy/v3 routes", { ...defaults, [expectedRoutes[0]]: bad }, false); + } + run("Verify deployed ingress health", {}, true); + run("Verify deployed ingress health", {}, false, { version: "1.20.5" }); + run("Verify deployed ingress health", {}, false, { tlsUnready: true }); + console.log(`Deployment probes: ${cases} synthetic response scenarios passed; no network requests.`); +} finally { + rmSync(directory, { recursive: true, force: true }); } diff --git a/scripts/ci/fixtures/tc959-n0-compose.yaml b/scripts/ci/fixtures/tc959-n0-compose.yaml new file mode 100644 index 00000000..d661d530 --- /dev/null +++ b/scripts/ci/fixtures/tc959-n0-compose.yaml @@ -0,0 +1,177 @@ +# dstack deployment - Topology 2: Postgres + S3 +# tinycloud-node in the CVM, Postgres external. +# Database credentials encrypted by dstack KMS. +# Column encryption enabled for sensitive data. +# +# All ${...} variables should be encrypted using dstack's KMS public key +# before deployment. The operator never sees plaintext values. + +services: + tinycloud: + # N0: unchanged production 1.20.6 binary; this is not a new app release. + image: ghcr.io/tinycloudlabs/tinycloud-node@sha256:4bdae4fc232d886bb56bfbce4ec62ed4b8fb4e1369a01f6afb73d06dd68c188f + restart: unless-stopped + labels: + tc.logship: tinycloud-node + # Host Docker daemon sends GELF; the app never joins Vector's network. + # Nonblocking UDP can lose logs. No raw Docker cache or /logs fallback. + logging: + driver: gelf + options: + gelf-address: udp://127.0.0.1:12201 + gelf-compression-type: none + labels: tc.logship + mode: non-blocking + max-buffer-size: 4m + cache-disabled: "true" + ports: + - "8000:8000" + volumes: + - /var/run/dstack.sock:/var/run/dstack.sock + environment: + TINYCLOUD_KEYS_TYPE: Dstack + TINYCLOUD_STORAGE_DATABASE: "${DATABASE_URL}" + TINYCLOUD_STORAGE_BLOCKS_TYPE: S3 + TINYCLOUD_STORAGE_BLOCKS_BUCKET: "${S3_BUCKET}" + TINYCLOUD_STORAGE_BLOCKS_ENDPOINT: "${S3_ENDPOINT}" + AWS_ACCESS_KEY_ID: "${AWS_KEY}" + AWS_SECRET_ACCESS_KEY: "${AWS_SECRET}" + AWS_DEFAULT_REGION: "${AWS_REGION}" + TINYCLOUD_LOG_LEVEL: normal + TINYCLOUD_LOG__FORMAT: Json + # Enable Prometheus telemetry (metrics, span histograms, pool gauges). + # The metrics handler is intentionally NOT gateway-exposed: port 8001 is + # not published here, so the unauthenticated /metrics endpoint stays + # internal to the CVM. + TINYCLOUD_TELEMETRY__ENABLED: "true" + # The managed Postgres instance allows max_connections = 25 (measured + # 2026-07-29, with some of those reserved for superuser roles). The pool + # default of 100 cannot be satisfied: the node opened and held slots + # until the instance ran out, and further requests failed with + # "Failed to acquire connection from pool: Connection pool timed out" + # after the 30s sqlx default. Keep this below the instance limit with + # headroom for admin/psql access, and raise it only alongside the + # instance's own max_connections. + TINYCLOUD_DATABASE__MAX_CONNECTIONS: "15" + # Required for every manual TC-780 deploy. Keep true through offline + # verification; only the final, explicitly reviewed redeploy sets false. + TINYCLOUD_DATABASE__WRITE_FENCE: "${SQL_IDENTITY_FENCE:?set true during cutover, false after verification}" + TINYCLOUD_CORS: "true" + # --- addressed sharing (TC-397) ------------------------------------ + # + # The public trust document shared with Share and OpenCredentials, + # delivered inline because nothing else can reach this container: the + # image is `FROM scratch` (no shell, so share-api's decode-to-tmpfs + # entrypoint cannot work here) and a dstack deployment uploads only + # this compose file (so there is no host path to bind-mount from). + # + # Seal the base64 of the reviewed trust-bundle.production.json — the + # same bytes share-api serves — into the CVM's encrypted environment as + # SHARE_TRUST_BUNDLE_BASE64. Same variable name and same wire form + # share-api already uses, so one sealed value feeds both services and + # cannot drift between them. + # + # The `:?` form is deliberate: compose refuses to interpolate a partial + # deployment, failing loudly at deploy time rather than letting the node + # die at boot with "share email trust bundle is required". + TINYCLOUD_SHARE_EMAIL__ENABLED: "true" + TINYCLOUD_SHARE_EMAIL__TRUST_BUNDLE_BASE64: "${SHARE_TRUST_BUNDLE_BASE64:?seal the reviewed share-email trust bundle, base64-encoded}" + # The sealed DATABASE_URL already carries sslmode=verify-full as its sole + # query parameter. ROOT_CERT_PATH stays unset so verify-full resolves + # against the compiled-in public root set (TC-363). + TINYCLOUD_SHARE_EMAIL__POSTGRES_TLS__SSLMODE: verify-full + TINYCLOUD_QUOTA_URL: "${TINYCLOUD_QUOTA_URL}" + TINYCLOUD_ADMIN_SECRET: "${TINYCLOUD_ADMIN_SECRET}" + ROCKET_ADDRESS: "0.0.0.0" + ROCKET_MAX_BLOCKING: "1024" + + dstack-ingress: + image: ghcr.io/tinycloudlabs/tinycloud-node-ingress@sha256:63c1c44e2ded5b73a5f174807b91ca677826974df0f9ab3045f6ec87865a5744 + ports: + - "443:443" + depends_on: + - tinycloud + environment: + - DOMAIN=tee.node.tinycloud.xyz + - TARGET_ENDPOINT=http://tinycloud:8000 + - CLOUDFLARE_API_TOKEN=${CLOUDFLARE_API_TOKEN} + # CNAME target the ingress publishes for DOMAIN. Must not start with `_`: + # Android's resolver (bionic res_hnok) rejects a CNAME target whose first + # label is `_`, so `_.` is unresolvable on Android (TC-513). + # `gateway.` is the target Phala Cloud advertises (gateway.cname) + # and upstream dstack-ingress documents; routing still uses SNI + the + # _dstack-app-address TXT record, so TLS stays end-to-end into the CVM. + - GATEWAY_DOMAIN=gateway.${DSTACK_GATEWAY_DOMAIN} + - CERTBOT_EMAIL=${CERTBOT_EMAIL} + - SET_CAA=true + - DNS_PROVIDER=cloudflare + - CLIENT_MAX_BODY_SIZE=32m + volumes: + - /var/run/dstack.sock:/var/run/dstack.sock + - cert-data:/etc/letsencrypt + restart: unless-stopped + labels: + tc.logship: tinycloud-ingress + logging: + driver: gelf + options: + gelf-address: udp://127.0.0.1:12201 + gelf-compression-type: none + labels: tc.logship + mode: non-blocking + max-buffer-size: 4m + cache-disabled: "true" + + # The dstack gateway's TLS-terminating routes (-.:443) + # only forward to compose-published ports, so guest sshd on :22 is + # unreachable via `phala ssh` even with authorized_keys installed. + # Publish a TCP forwarder on 10022 that relays to guest sshd via the + # docker host gateway. Port 10022 exposes ONLY sshd, which is key-gated. + ssh-gateway: + image: alpine/socat:1.8.1.3@sha256:188fe0a22182f81c16def9d1137930121eaa3023f427164e32c0d2f118f79dd6 + restart: unless-stopped + ports: + - "10022:10022" + extra_hosts: + - "host.docker.internal:host-gateway" + command: TCP-LISTEN:10022,fork,reuseaddr TCP:host.docker.internal:22 + + vector: + restart: unless-stopped + image: timberio/vector:0.59.0-alpine@sha256:0e36bc2a290039e9b6a5e92f04bd2f133223506f72d09682c1df16c015172b7a + init: true + user: 10001:10001 + read_only: true + cap_drop: + - ALL + security_opt: + - no-new-privileges:true + mem_limit: 64m + memswap_limit: 64m + cpus: 0.5 + ports: + - 127.0.0.1:12201:12201/udp + networks: + - logship + tmpfs: + - /var/lib/vector:rw,noexec,nosuid,size=8m,uid=10001,gid=10001,mode=0700 + environment: + TINYCLOUD_LOGSHIP_PROFILE: ${TINYCLOUD_LOGSHIP_PROFILE:-errors_slow} + BETTERSTACK_SOURCE_TOKEN: ${BETTERSTACK_SOURCE_TOKEN:?seal the node source token before enabling log shipping} + VECTOR_CONFIG_JSON: "{\"data_dir\":\"/var/lib/vector\",\"sources\":{\"gelf_in\":{\"type\":\"socket\",\"mode\":\"udp\",\"address\":\"0.0.0.0:12201\",\"max_length\":262144,\"receive_buffer_bytes\":8388608,\"framing\":{\"method\":\"chunked_gelf\",\"chunked_gelf\":{\"decompression\":\"Auto\"}},\"decoding\":{\"codec\":\"bytes\"}}},\"transforms\":{\"gelf_envelope\":{\"type\":\"remap\",\"inputs\":[\"gelf_in\"],\"source\":\"packet = string(.message) ?? \\\"\\\"\\nif length(packet) > 262144 {\\n . = {\\\"reason\\\": \\\"oversized\\\"}\\n} else {\\n outer, err = parse_json(packet)\\n if err != null || !is_object(outer) || (outer.version != \\\"1.1\\\" && outer.version != \\\"1.0\\\") {\\n . = {\\\"reason\\\": \\\"malformed_envelope\\\"}\\n } else if exists(outer.partial_message) || exists(outer._partial_message) {\\n . = {\\\"reason\\\": \\\"partial\\\"}\\n } else {\\n label = get(outer, [\\\"_tc.logship\\\"]) ?? null\\n . = {\\\"_tc.logship\\\": label, \\\"message\\\": outer.short_message}\\n }\\n}\",\"drop_on_error\":true,\"drop_on_abort\":true},\"logship_gate\":{\"type\":\"remap\",\"inputs\":[\"gelf_envelope\"],\"source\":\"if exists(.reason) {\\n . = {\\\"reason\\\": .reason}\\n} else {\\nlabel = string(get(., [\\\"_tc.logship\\\"]) ?? null) ?? \\\"\\\"\\nraw = .message\\nreason = \\\"\\\"\\nif label == \\\"\\\" { reason = \\\"unlabelled\\\" }\\nelse if !includes([\\\"tinycloud-node\\\", \\\"tinycloud-ingress\\\"], label) { reason = \\\"unexpected_label\\\" }\\nelse if exists(.partial_message) { reason = \\\"partial\\\" }\\nelse if !is_string(raw) { reason = \\\"malformed_envelope\\\" }\\nelse if length(string!(raw)) > 65536 { reason = \\\"oversized\\\" }\\nif reason != \\\"\\\" {\\n . = {\\\"reason\\\": reason}\\n} else {\\n parsed, err = parse_json(string!(raw))\\n if err != null || !is_object(parsed) { . = {\\\"reason\\\": \\\"non_json\\\"} }\\n else if exists(parsed.partial_message) { . = {\\\"reason\\\": \\\"partial\\\"} }\\n else {\\n target = string(parsed.target) ?? \\\"\\\"\\n level = downcase(string(parsed.level) ?? \\\"\\\")\\n if label == \\\"tinycloud-node\\\" && !includes([\\\"warn\\\", \\\"error\\\"], level) && target != \\\"tinycloud\\\" && !starts_with(target, \\\"tinycloud::\\\") && target != \\\"tinycloud_core\\\" && !starts_with(target, \\\"tinycloud_core::\\\") {\\n . = {\\\"reason\\\": \\\"unknown_target\\\"}\\n } else if label == \\\"tinycloud-node\\\" && !includes([\\\"warn\\\", \\\"error\\\"], level) && (target != \\\"tinycloud::request\\\" || level != \\\"info\\\") {\\n . = {\\\"reason\\\": \\\"filtered_level\\\"}\\n } else { . = {\\\"label\\\": label, \\\"parsed\\\": parsed} }\\n }\\n}\\n}\",\"drop_on_error\":true,\"drop_on_abort\":true},\"gate_route\":{\"type\":\"route\",\"inputs\":[\"logship_gate\"],\"route\":{\"candidate\":\"exists(.parsed)\",\"rejected\":\"exists(.reason)\"}},\"allowlist\":{\"type\":\"remap\",\"inputs\":[\"gate_route.candidate\"],\"source\":\"# Only this fresh object can leave the process boundary. No envelope, span,\\n# message, error, identity, URI or unknown key is copied into it.\\nj = .parsed\\nlabel = .label\\nout = {}\\nts = \\\"\\\"\\nif label == \\\"tinycloud-node\\\" {\\n target = string(j.target) ?? \\\"\\\"\\n level = downcase(string!(j.level))\\n f = object(j.fields) ?? {}\\n msg = string(f.message) ?? \\\"\\\"\\n ts = string!(j.timestamp)\\n if target == \\\"tinycloud::request\\\" && level == \\\"info\\\" && msg == \\\"request\\\" {\\n method = string!(f.method)\\n if includes([\\\"TRACE\\\", \\\"CONNECT\\\"], method) { method = \\\"\\\" }\\n route = string!(f.route)\\n # Literal Rocket template aliases only; never split an arbitrary URI.\\n aliases = {\\\"/attestation?\\\":\\\"/attestation\\\",\\\"/.well-known/encryption/network/?\\\":\\\"/.well-known/encryption/network/\\\",\\\"/hooks/events?\\\":\\\"/hooks/events\\\",\\\"/hooks/webhooks?\\\":\\\"/hooks/webhooks\\\",\\\"/public//kv?\\\":\\\"/public//kv\\\"}\\n canonical_route = get(aliases, [route]) ?? null\\n if is_string(canonical_route) { route = canonical_route }\\n if !includes([\\\"GET\\\",\\\"HEAD\\\",\\\"POST\\\",\\\"PUT\\\",\\\"PATCH\\\",\\\"DELETE\\\",\\\"OPTIONS\\\",\\\"\\\"], method) || !includes([\\\"/healthz\\\",\\\"/info\\\",\\\"/version\\\",\\\"/invoke\\\",\\\"/delegate\\\",\\\"/revoke\\\",\\\"/attestation\\\",\\\"/signed/kv\\\",\\\"/signed/kv/\\\",\\\"/admin/quota\\\",\\\"/admin/quota/\\\",\\\"/admin/usage\\\",\\\"/hooks/events\\\",\\\"/hooks/tickets\\\",\\\"/hooks/webhooks\\\",\\\"/hooks/webhooks/\\\",\\\"/delegation/query\\\",\\\"/delegation/status\\\",\\\"/encryption/networks\\\",\\\"/encryption/networks/\\\",\\\"/encryption/networks//revoke\\\",\\\"/encryption/networks//decrypt\\\",\\\"/policy/v3/status\\\",\\\"/policy/v3/status/\\\",\\\"/policy/v3/policies\\\",\\\"/policy/v3/challenges\\\",\\\"/policy/v3/delegations\\\",\\\"/policy/v3/enforcer-bindings\\\",\\\"/policy/v3/deliveries/authorize\\\",\\\"/.well-known/tinycloud/node-keys\\\",\\\"/.well-known/encryption/network/\\\",\\\"/public//kv\\\",\\\"/public//kv/\\\",\\\"/peer/generate/\\\",\\\"/evidences/\\\",\\\"/share/v2/readiness\\\",\\\"/share/v2/policies\\\",\\\"/share/v2/policy/challenges\\\",\\\"/share/v2/policy/session\\\",\\\"/share/v2/invoke\\\",\\\"/share/v2/deliveries/authorize\\\",\\\"/<_s..>\\\",\\\"/public/<_space_id>/kv/<_key..>\\\",\\\"\\\"], route) { abort }\\n status = f.status\\n latency = f.latency_us\\n if !is_integer(status) || !is_integer(latency) { abort }\\n status = int!(status)\\n latency = int!(latency)\\n if status < 100 || status > 599 || latency < 0 || latency > 9007199254740991 { abort }\\n out = {\\\"event\\\": \\\"node_request\\\", \\\"svc\\\": \\\"api\\\", \\\"level\\\": \\\"info\\\", \\\"method\\\": method, \\\"route\\\": route, \\\"status\\\": status, \\\"latency_us\\\": latency}\\n } else {\\n if !includes([\\\"warn\\\", \\\"error\\\"], level) { abort }\\n code = \\\"unclassified\\\"\\n if target == \\\"tinycloud\\\" || starts_with(target, \\\"tinycloud::\\\") || target == \\\"tinycloud_core\\\" || starts_with(target, \\\"tinycloud_core::\\\") {\\n codes = {\\\"invocation replay insert failed\\\":\\\"invocation_replay_insert_failed\\\",\\\"invocation replay cleanup failed\\\":\\\"invocation_replay_cleanup_failed\\\",\\\"share-v2 local-tee feature active: deriving a deterministic, non-attested TEE identity from node key material\\\":\\\"share_local_tee_identity\\\",\\\"retention: delivered hook delivery prune failed\\\":\\\"retention_hook_delivery_prune_failed\\\",\\\"retention: dead-letter hook delivery prune failed\\\":\\\"retention_dead_letter_prune_failed\\\",\\\"retention: signed KV ticket prune failed\\\":\\\"retention_signed_ticket_prune_failed\\\",\\\"failed to load tunnel state; tunnel not started\\\":\\\"tunnel_state_load_failed\\\",\\\"invalid loopback API address; tunnel not started\\\":\\\"tunnel_address_invalid\\\",\\\"failed to load link state; LAN listener not started\\\":\\\"link_state_load_failed\\\",\\\"invalid link bind address; LAN listener not started\\\":\\\"link_bind_invalid\\\",\\\"invalid loopback API address; LAN listener not started\\\":\\\"link_address_invalid\\\",\\\"failed to bind link LAN listener; LAN listener not started\\\":\\\"link_bind_failed\\\",\\\"failed to build TLS config; LAN listener not started\\\":\\\"link_tls_config_failed\\\",\\\"link LAN listener exited with error\\\":\\\"link_listener_error\\\",\\\"failed to record link listener bind state\\\":\\\"link_listener_state_write_failed\\\",\\\"failed to record link listener bind failure\\\":\\\"link_bind_state_write_failed\\\",\\\"failed to load config for auto-renew\\\":\\\"link_auto_renew_config_failed\\\",\\\"link auto-renew failed\\\":\\\"link_auto_renew_failed\\\",\\\"failed to hot-reload renewed link cert\\\":\\\"link_cert_reload_failed\\\",\\\"link auto-renew succeeded but no state.json found; skipping cert hot-reload\\\":\\\"link_cert_state_missing\\\",\\\"failed to reload TLS material after link auto-renew\\\":\\\"link_listener_cert_reload_failed\\\",\\\"link disable: service delete failed; local state removed anyway\\\":\\\"link_disable_failed\\\",\\\"link connection error\\\":\\\"link_connection_error\\\",\\\"link listener accept error\\\":\\\"link_listener_accept_failed\\\",\\\"webhook delivery attempt failed\\\":\\\"webhook_delivery_failed\\\",\\\"webhook dispatcher loop failed\\\":\\\"webhook_dispatcher_failed\\\",\\\"addressed delegation authorization failed\\\":\\\"share_authorization_failed\\\",\\\"native share authorization denied\\\":\\\"share_native_authorization_denied\\\",\\\"native share KV put failed\\\":\\\"share_native_kv_put_failed\\\",\\\"share v2 readiness probe failed\\\":\\\"share_v2_readiness_failed\\\",\\\"share v2 is enabled but shareEmail.invitationPublicKey is unset\\\":\\\"share_invitation_key_missing\\\",\\\"configured share invitation key does not match the node's derived signing key\\\":\\\"share_invitation_key_mismatch\\\",\\\"share/v2 policy challenge denied\\\":\\\"share_policy_denied\\\",\\\"failed to read committed hook events\\\":\\\"hook_events_read_failed\\\",\\\"failed to load kv write hook rows\\\":\\\"hook_rows_load_failed\\\",\\\"failed to load kv delete hook rows\\\":\\\"hook_rows_load_failed\\\",\\\"missing committed kv hook row for invocation\\\":\\\"missing_committed_hook_row\\\",\\\"database write committed but webhook enqueue failed\\\":\\\"webhook_enqueue_failed\\\",\\\"tunnel: failed to prepare connection attempt\\\":\\\"tunnel_connection_prepare_failed\\\",\\\"tunnel: no frame within idle timeout; treating connection as dead\\\":\\\"tunnel_idle_timeout\\\",\\\"tunnel: read error\\\":\\\"tunnel_read_failed\\\",\\\"tunnel: dropping malformed frame\\\":\\\"tunnel_malformed_frame\\\",\\\"tunnel: relay sent an error frame\\\":\\\"tunnel_relay_error\\\",\\\"tunnel: failed to write frame\\\":\\\"tunnel_write_failed\\\",\\\"tunnel: failed to record runtime state\\\":\\\"tunnel_state_record_failed\\\",\\\"tunnel: failed to clear runtime state after disable\\\":\\\"tunnel_state_clear_failed\\\",\\\"tunnel authentication rejected\\\":\\\"tunnel_auth_rejected\\\",\\\"network DID resolution enabled: did:web/did:tz fetches use the unhardened ssi client (TC-1080)\\\":\\\"network_did_resolution_enabled\\\"}\\n known = get(codes, [msg]) ?? null\\n if is_string(known) { code = known }\\n if starts_with(msg, \\\"quota service returned \\\") { code = \\\"quota_fallback\\\" }\\nif starts_with(msg, \\\"Failed to get TDX quote: \\\") { code = \\\"attestation_quote_failed\\\" }\\nif starts_with(msg, \\\"quota service returned an invalid body: \\\") { code = \\\"quota_invalid_body\\\" }\\nif starts_with(msg, \\\"dstack socket available but get_info failed: \\\") { code = \\\"dstack_get_info_failed\\\" }\\n }\\n event = if level == \\\"warn\\\" { \\\"node_warning\\\" } else { \\\"node_error\\\" }\\n out = {\\\"event\\\": event, \\\"svc\\\": \\\"api\\\", \\\"level\\\": level, \\\"event_code\\\": code, \\\"event_count\\\": 1}\\n }\\n} else {\\n ts = string!(j.time)\\n method = string!(j.method)\\n route = string!(j.route)\\n if route == \\\"\\\" { route = \\\"\\\" }\\n if !includes([\\\"GET\\\",\\\"HEAD\\\",\\\"POST\\\",\\\"PUT\\\",\\\"PATCH\\\",\\\"DELETE\\\",\\\"OPTIONS\\\",\\\"\\\"], method) || !includes([\\\"/healthz\\\",\\\"/info\\\",\\\"/version\\\",\\\"/invoke\\\",\\\"/delegate\\\",\\\"/revoke\\\",\\\"/attestation\\\",\\\"/signed/kv\\\",\\\"/signed/kv/\\\",\\\"/admin/quota\\\",\\\"/admin/quota/\\\",\\\"/admin/usage\\\",\\\"/hooks/events\\\",\\\"/hooks/tickets\\\",\\\"/hooks/webhooks\\\",\\\"/hooks/webhooks/\\\",\\\"/delegation/query\\\",\\\"/delegation/status\\\",\\\"/encryption/networks\\\",\\\"/encryption/networks/\\\",\\\"/encryption/networks//revoke\\\",\\\"/encryption/networks//decrypt\\\",\\\"/policy/v3/status\\\",\\\"/policy/v3/status/\\\",\\\"/policy/v3/policies\\\",\\\"/policy/v3/challenges\\\",\\\"/policy/v3/delegations\\\",\\\"/policy/v3/enforcer-bindings\\\",\\\"/policy/v3/deliveries/authorize\\\",\\\"/.well-known/tinycloud/node-keys\\\",\\\"/.well-known/encryption/network/\\\",\\\"/public//kv\\\",\\\"/public//kv/\\\",\\\"/peer/generate/\\\",\\\"/evidences/\\\",\\\"/share/v2/readiness\\\",\\\"/share/v2/policies\\\",\\\"/share/v2/policy/challenges\\\",\\\"/share/v2/policy/session\\\",\\\"/share/v2/invoke\\\",\\\"/share/v2/deliveries/authorize\\\",\\\"/<_s..>\\\",\\\"/public/<_space_id>/kv/<_key..>\\\",\\\"\\\"], route) { abort }\\n status = j.status\\n request_time = j.request_time\\n if !is_integer(status) || !(is_integer(request_time) || is_float(request_time)) { abort }\\n status = int!(status)\\n request_time = to_float!(request_time)\\n if status < 100 || status > 599 || request_time < 0 || request_time > 86400 { abort }\\n out = {\\\"event\\\": \\\"ingress_request\\\", \\\"svc\\\": \\\"ingress\\\", \\\"level\\\": \\\"info\\\", \\\"method\\\": method, \\\"route\\\": route, \\\"status\\\": status, \\\"request_time_seconds\\\": request_time}\\n}\\nif length(ts) > 64 { abort }\\nparsed_ts = parse_timestamp!(ts, format: \\\"%+\\\")\\nout.ts = format_timestamp!(parsed_ts, format: \\\"%Y-%m-%dT%H:%M:%S%.3fZ\\\")\\n. = out\\n\",\"drop_on_error\":true,\"drop_on_abort\":true,\"reroute_dropped\":true},\"gate_counts\":{\"type\":\"remap\",\"inputs\":[\"gate_route.rejected\",\"gate_route._unmatched\",\"allowlist.dropped\",\"event_route._unmatched\"],\"source\":\"reason = .reason\\nif !includes([\\\"malformed_envelope\\\",\\\"unexpected_label\\\",\\\"unlabelled\\\",\\\"non_json\\\",\\\"oversized\\\",\\\"partial\\\",\\\"invalid_fields\\\",\\\"unknown_target\\\",\\\"filtered_level\\\"], reason) { reason = \\\"invalid_fields\\\" }\\n. = {\\\"reason\\\": reason, \\\"count\\\": 1}\",\"drop_on_error\":true,\"drop_on_abort\":true},\"gate_reduce\":{\"type\":\"reduce\",\"inputs\":[\"gate_counts\"],\"group_by\":[\"reason\"],\"end_every_period_ms\":60000,\"expire_after_ms\":75000,\"flush_period_ms\":1000,\"merge_strategies\":{\"count\":\"sum\"}},\"gate_output\":{\"type\":\"remap\",\"inputs\":[\"gate_reduce\"],\"source\":\". = {\\\"ts\\\": format_timestamp!(now(), format: \\\"%Y-%m-%dT%H:%M:%S%.3fZ\\\"), \\\"level\\\": \\\"warn\\\", \\\"event\\\": \\\"logship_gate_summary_total\\\", \\\"svc\\\": \\\"logship\\\", \\\"reason\\\": .reason, \\\"count\\\": .count}\",\"drop_on_error\":true,\"drop_on_abort\":true},\"request_profile\":{\"type\":\"filter\",\"inputs\":[\"allowlist\"],\"condition\":\"\\\"$${TINYCLOUD_LOGSHIP_PROFILE}\\\" == \\\"all_requests\\\" || includes([\\\"node_warning\\\", \\\"node_error\\\"], .event) || (int(.status) ?? 0) >= 400 || (.event == \\\"node_request\\\" && (int(.latency_us) ?? 0) > 1000000) || (.event == \\\"ingress_request\\\" && (to_float(.request_time_seconds) ?? 0.0) > 1.000000)\"},\"event_route\":{\"type\":\"route\",\"inputs\":[\"request_profile\"],\"route\":{\"request\":\".event == \\\"node_request\\\" || .event == \\\"ingress_request\\\"\",\"diagnostic\":\".event == \\\"node_warning\\\" || .event == \\\"node_error\\\"\"}},\"diagnostic_reduce\":{\"type\":\"reduce\",\"inputs\":[\"event_route.diagnostic\"],\"group_by\":[\"event\",\"event_code\",\"level\",\"svc\"],\"end_every_period_ms\":60000,\"expire_after_ms\":75000,\"flush_period_ms\":1000,\"merge_strategies\":{\"event_count\":\"sum\",\"ts\":\"discard\"}},\"diagnostic_output\":{\"type\":\"remap\",\"inputs\":[\"diagnostic_reduce\"],\"source\":\". = {\\\"ts\\\": format_timestamp!(now(), format: \\\"%Y-%m-%dT%H:%M:%S%.3fZ\\\"), \\\"level\\\": .level, \\\"event\\\": .event, \\\"svc\\\": \\\"api\\\", \\\"event_code\\\": .event_code, \\\"event_count\\\": .event_count}\",\"drop_on_error\":true,\"drop_on_abort\":true}},\"sinks\":{\"betterstack\":{\"type\":\"http\",\"inputs\":[\"event_route.request\",\"diagnostic_output\",\"gate_output\"],\"method\":\"post\",\"uri\":\"https://s2795437.us-west-2a.betterstackdata.com/\",\"encoding\":{\"codec\":\"json\"},\"framing\":{\"method\":\"newline_delimited\"},\"compression\":\"gzip\",\"auth\":{\"strategy\":\"bearer\",\"token\":\"SECRET[bs.bs_token]\"},\"batch\":{\"max_bytes\":262144,\"timeout_secs\":1},\"buffer\":{\"type\":\"memory\",\"max_events\":1000,\"when_full\":\"block\"},\"request\":{\"timeout_secs\":10,\"retry_max_duration_secs\":300},\"healthcheck\":{\"enabled\":false}}},\"secret\":{\"bs\":{\"type\":\"directory\",\"path\":\"/var/lib/vector/secrets\",\"remove_trailing_whitespace\":true}}}" + entrypoint: + - /bin/sh + - -ec + - "umask 077\nmkdir -p /var/lib/vector/secrets\ncase \"$$TINYCLOUD_LOGSHIP_PROFILE\" in\n errors_slow|all_requests) ;;\n *) printf '%s\\n' 'logship profile invalid' >&2; exit 78 ;;\nesac\nif [ -z \"$$BETTERSTACK_SOURCE_TOKEN\" ]; then\n printf '%s\\n' 'logship source token missing' >&2\n exit 78\nfi\nprintf '%s' \"$$BETTERSTACK_SOURCE_TOKEN\" > /var/lib/vector/secrets/bs_token\nprintf '%s' \"$$VECTOR_CONFIG_JSON\" > /var/lib/vector/runtime.json\nunset BETTERSTACK_SOURCE_TOKEN VECTOR_CONFIG_JSON\nexec vector --config /var/lib/vector/runtime.json\n" + logging: + driver: json-file + options: + max-size: 10m + max-file: '3' + +volumes: + cert-data: + +networks: + logship: + driver: bridge diff --git a/scripts/ci/reconcile-quota.py b/scripts/ci/reconcile-quota.py new file mode 100755 index 00000000..70e580e1 --- /dev/null +++ b/scripts/ci/reconcile-quota.py @@ -0,0 +1,193 @@ +#!/usr/bin/env python3 +"""Reconcile reviewed quota authority; secrets are accepted only through env.""" +import argparse +import http.client +import json +import os +from pathlib import Path +import sys +import time +from urllib.parse import quote, urlsplit + +CONFIG_PATH = Path("ops/quota-overrides.json") +OWNER = "tinycloud:pkh:eip155:1:0xd559CCd9EB87c530A9a349262669386dE93cf412" +LIMITS = {"applications": 5368709120, "secrets": 268721121} +NODE_HOST = "node.tinycloud.xyz" + + +class QuotaError(Exception): + """A fixed, safe diagnostic; never wrap response or credential contents.""" + + +def load_configuration(path): + """Require an explicit reviewed authority and the two canonical targets.""" + try: + document = json.loads(path.read_text()) + if not isinstance(document, dict) or set(document) != {"authority", "overrides"}: + raise ValueError + authority = document["authority"] + if authority not in ("node-override", "billing"): + raise ValueError + rows = document["overrides"] + expected = {f"{OWNER}:{name}": limit for name, limit in LIMITS.items()} + if not isinstance(rows, dict) or set(rows) != set(expected): + raise ValueError + for space, row in rows.items(): + if (not isinstance(row, dict) or set(row) != {"limit_bytes", "_comment"} + or type(row["limit_bytes"]) is not int or row["limit_bytes"] != expected[space] + or not isinstance(row["_comment"], str) or not row["_comment"]): + raise ValueError + return authority, expected + except (OSError, ValueError, TypeError): + raise QuotaError("reviewed quota configuration is invalid") from None + + +def credentials(): + """Validate presence before compose mutation, without contacting services.""" + node = os.environ.get("QUOTA_ADMIN_SECRET", "") + billing = os.environ.get("QUOTA_BILLING_ADMIN_SECRET", "") + if not node or not billing or any("\r" in value or "\n" in value for value in (node, billing)): + raise QuotaError("required runner quota credentials are missing or invalid") + try: + url = urlsplit(os.environ.get("QUOTA_BILLING_URL", "")) + if (url.scheme != "https" or not url.hostname or url.username or url.password + or url.path.rstrip("/") != "/api/quota" or url.query or url.fragment): + raise ValueError + port = url.port + except ValueError: + raise QuotaError("billing quota URL has an incompatible shape") from None + return node, billing, (url.hostname, port) + + +def request_json(host, method, path, secret, limit=None, port=None, allow_404=False): + """Fixed TLS requests, no redirects; retry transport only, never HTTP errors.""" + headers = {"Authorization": "Bearer " + secret, "Content-Type": "application/json"} + body = None if limit is None else json.dumps({"limit_bytes": limit}).encode() + for attempt in range(3): + connection = None + try: + kwargs = {"timeout": 15} + if port is not None: + kwargs["port"] = port + connection = http.client.HTTPSConnection(host, **kwargs) + connection.request(method, path, body=body, headers=headers) + response = connection.getresponse() + payload = response.read(65537) + except (OSError, http.client.HTTPException): + if attempt == 2: + raise QuotaError("quota transport failed after bounded attempts") from None + time.sleep(attempt + 1) + continue + finally: + if connection is not None: + connection.close() + if response.status == 404 and allow_404: + return None + if response.status != 200: + raise QuotaError("quota API rejected the request") + try: + result = json.loads(payload) if len(payload) <= 65536 else None + if not isinstance(result, dict): + raise ValueError + return result + except (ValueError, UnicodeError): + raise QuotaError("quota API returned an invalid response") from None + + +def prove_inventory(overrides, node_secret): + """A quota GET echoes arbitrary IDs; only usage enumerates existing spaces.""" + inventory = request_json(NODE_HOST, "GET", "/admin/usage", node_secret) + rows = inventory.get("spaces") + if (not isinstance(rows, list) or type(inventory.get("count")) is not int + or inventory["count"] != len(rows)): + raise QuotaError("space inventory is invalid; no overrides applied") + usage = {} + for row in rows: + if (not isinstance(row, dict) or not isinstance(row.get("space_id"), str) + or row["space_id"] in usage): + raise QuotaError("space inventory is ambiguous; no overrides applied") + usage[row["space_id"]] = row.get("usage_bytes") + if any(type(usage.get(space)) is not int or usage[space] <= 0 for space in overrides): + raise QuotaError("both exact quota targets require positive known usage; no overrides applied") + + +def billing_comp(space, billing_secret, origin): + """Read PR48's absolute per-space limit, not incremental account comp_bytes.""" + result = request_json(origin[0], "GET", "/api/admin/storage-comp/" + quote(space, safe=""), + billing_secret, port=origin[1], allow_404=True) + if result is None: + print("billing storage-comp unavailable") + return None + if (type(result.get("active")) is not bool + or type(result.get("comp_bytes")) is not int or result["comp_bytes"] < 0 + or type(result.get("account_limit_bytes")) is not int or result["account_limit_bytes"] < 0 + or "storage_limit_bytes" not in result + or (result["storage_limit_bytes"] is not None + and (type(result["storage_limit_bytes"]) is not int or result["storage_limit_bytes"] < 0))): + raise QuotaError("billing storage-comp returned an unexpected body") + print("billing storage-comp active" if result["active"] else "billing storage-comp inactive") + return result + + +def reconcile(authority, overrides, node_secret, billing_secret, billing_origin): + """Prove both identities and both authority answers before any node PUT.""" + prove_inventory(overrides, node_secret) + comps = {space: billing_comp(space, billing_secret, billing_origin) for space in overrides} + if authority == "node-override": + if any(comp is not None and comp["active"] for comp in comps.values()): + raise QuotaError("authority mismatch: flip to billing") + for space, limit in overrides.items(): + path = "/admin/quota/" + quote(space, safe="") + request_json(NODE_HOST, "PUT", path, node_secret, limit) + actual = request_json(NODE_HOST, "GET", path, node_secret) + if (actual.get("space_id") != space or type(actual.get("override_bytes")) is not int + or actual["override_bytes"] != limit or type(actual.get("effective_limit_bytes")) is not int + or actual["effective_limit_bytes"] != limit): + raise QuotaError("quota override and effective limit did not both verify") + return "Quota overrides verified: applications=5368709120; secrets=268721121." + for space, floor in overrides.items(): + comp = comps[space] + if (comp is None or not comp["active"] or type(comp["storage_limit_bytes"]) is not int + or comp["storage_limit_bytes"] < floor): + raise QuotaError("billing authority does not cover both reviewed quota floors") + actual = request_json(NODE_HOST, "GET", "/admin/quota/" + quote(space, safe=""), node_secret) + if actual.get("space_id") != space or "override_bytes" not in actual: + raise QuotaError("node quota readback is invalid") + if actual["override_bytes"] is not None: + raise QuotaError("stale legacy override") + return "billing-authoritative; the node fetches on the next write (≤60 s TTL)" + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + modes = parser.add_mutually_exclusive_group() + modes.add_argument("--validate-only", action="store_true") + modes.add_argument("--preflight", action="store_true") + args = parser.parse_args(argv) + try: + authority, overrides = load_configuration(CONFIG_PATH) + if args.validate_only: + print("Reviewed quota authority and targets validated.") + return 0 + node_secret, billing_secret, origin = credentials() + if args.preflight: + print("Runner quota credentials and billing URL validated; no service requests.") + return 0 + message = reconcile(authority, overrides, node_secret, billing_secret, origin) + code = 0 + except QuotaError as error: + message = f"Quota restoration FAILED: {error}. Authorized operator fallback required." + code = 1 + except Exception: + message = "Quota restoration FAILED: unexpected error; details withheld. Authorized operator fallback required." + code = 1 + print(("::error::" if code else "") + message) + summary_path = os.environ.get("GITHUB_STEP_SUMMARY") + if summary_path: + with open(summary_path, "a") as summary: + summary.write(message + "\n") + return code + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/ci/test-deploy-recovery.py b/scripts/ci/test-deploy-recovery.py new file mode 100755 index 00000000..d5c6bd18 --- /dev/null +++ b/scripts/ci/test-deploy-recovery.py @@ -0,0 +1,424 @@ +#!/usr/bin/env python3 +"""Run actual workflow Bash against offline CLI/HTTP boundaries only.""" +import json +from pathlib import Path +import re +import secrets +import shlex +import subprocess +import sys +import tempfile +import textwrap +import unittest + +ROOT = Path(__file__).resolve().parents[2] +SOURCE = (ROOT / ".github/workflows/docker.yml").read_text() +DIGEST = "sha256:4bdae4fc232d886bb56bfbce4ec62ed4b8fb4e1369a01f6afb73d06dd68c188f" +REVISION = "e18578ff9162986ddd813259c60f73866e3e60d1" +N0 = (ROOT / "scripts/ci/fixtures/tc959-n0-compose.yaml").read_bytes() +PRE_N0 = (ROOT / "ops/rollback/pre-n0-compose.yaml").read_bytes() +STOP_LOGS = (ROOT / "ops/rollback/pre-n0-stop-local-logs.yaml").read_bytes() +CURRENT = (ROOT / "docker-compose.dstack-postgres.yaml").read_bytes() +CANARIES = [] + + +def shell_for_step(name): + marker = " - name: " + name + "\n" + block = SOURCE.split(marker, 1)[1] + lines = [] + for line in block.split(" run: |\n", 1)[1].splitlines(): + if line and not line.startswith(" "): + break + lines.append(line) + return textwrap.dedent("\n".join(lines)) + + +def enabled(condition, context): + literals = [] + def protect(match): + literals.append(match[0]) + return f"@{len(literals)-1}@" + condition = re.sub(r"'[^']*'", protect, condition) + condition = re.sub(r"(?:github|inputs|steps|needs|vars)\.[\w.-]+(?:\.[\w.-]+)*", + lambda match: repr(context.get(match[0], "")), condition) + condition = condition.replace("success()", repr(context.get("job.success", True))) + condition = condition.replace("always()", "True").replace("cancelled()", "False") + condition = re.sub(r"!(?!=)", "not ", condition).replace("&&", " and ").replace("||", " or ") + for index, literal in enumerate(literals): + condition = condition.replace(f"@{index}@", literal) + return bool(eval("(" + condition + ")", {"__builtins__": {}}, {})) + + +def step_enabled(name, context): + block = SOURCE.split(" - name: " + name + "\n", 1)[1] + prefix = block.split(" run:", 1)[0] + if not context["job.success"] and "always()" not in prefix: + return False + if " if: >-\n" in prefix: + condition = prefix.split(" if: >-\n", 1)[1].split("\n env:", 1)[0].strip()[3:-2] + elif " if: " in prefix: + condition = prefix.split(" if: ", 1)[1].split("\n", 1)[0].strip() + if condition.startswith("${{"): + condition = condition[3:-2] + else: + return context["job.success"] + return enabled(condition, context) + + +CLI = r'''#!/usr/bin/env python3 +import json, os, sys +from pathlib import Path +args=sys.argv[1:]; name=Path(sys.argv[0]).name +state=Path(os.environ['TEST_STATE']) +def event(value): + data=json.loads(state.read_text()); data['events'].append(value); state.write_text(json.dumps(data)) +if name=='docker': + if args[0]=='pull': event('image-pull') + elif args[:2]==['image','inspect']: + print(os.environ['OCI_REVISION'] if 'revision' in args[-1] else os.environ['OCI_LABEL']) + elif args[0]=='run': + assert args[-1]=='--validate-config' and 'none' in args and '--read-only' in args; event('sealed-preflight') + elif args[0]=='compose': + assert args[-2:]==['config','--quiet']; assert Path('.phala-prod.env').stat().st_mode & 0o777==0o600 + else: raise SystemExit('Unexpected Docker boundary') +elif name=='git': + assert args==['rev-parse','HEAD']; print('f'*40) +elif name=='node': + names=[line.split('=',1)[0] for line in Path('.phala-prod.env').read_text().splitlines()] + assert 'BETTERSTACK_SOURCE_TOKEN' in names and 'QUOTA_BILLING_ADMIN_SECRET' not in names + if args[-1]=='--check': event('whitelist-check') + else: + event('whitelist-sync') + if os.environ['FAIL_AT']=='sync': raise SystemExit('Fixed synchronization failure') + event('final-restart') +elif name=='phala': + assert args[:3]==['deploy','--cvm-id','tinycloud-node'] + assert '--no-public-logs' in args and '--no-public-sysinfo' in args + event('cvm-deploy') + if os.environ['FAIL_AT']=='deploy': + print(os.environ['PHALA_ENV_BETTERSTACK_SOURCE_TOKEN'],file=sys.stderr); sys.exit(1) +elif name=='curl': + url=next(value for value in args if value.startswith('https://')) + if url.endswith('/version'): + event('ingress-health' if 'tee.node.' in url else 'version-health') + print(json.dumps({'version':'not-ready' if os.environ['FAIL_AT']=='health' else os.environ['OCI_LABEL'].split('-dstack')[0]})) + else: + assert url.startswith('https://tee.node.tinycloud.xyz/policy/v3/') + if '-o' in args: Path(args[args.index('-o')+1]).write_text('{}') + event('policy-probe'); print('500' if os.environ['FAIL_AT']=='policy' else ('401' if url.endswith('/deliveries/authorize') else '400'),end='') +elif name=='sleep': pass +else: raise SystemExit('Unexpected CLI boundary') +''' + +HTTP = r'''import http.client, json, os, socket +from pathlib import Path +from urllib.parse import unquote +socket.create_connection=lambda *a,**k: (_ for _ in ()).throw(AssertionError('Real network forbidden')) +class Connection: + def __init__(self,host,timeout,**kwargs): + assert host in ('node.tinycloud.xyz','billing.example.invalid'); self.host=host + def request(self,method,path,body=None,headers=None): + expected=os.environ['QUOTA_BILLING_ADMIN_SECRET'] if self.host=='billing.example.invalid' else os.environ['QUOTA_ADMIN_SECRET'] + assert headers['Authorization']=='Bearer '+expected + self.method,self.path=method,path + state=Path(os.environ['TEST_STATE']); data=json.loads(state.read_text()) + if self.host=='billing.example.invalid': data['events'].append('billing-comp') + elif path=='/admin/usage': data['events'].append('quota-inventory') + elif method=='PUT': data['events'].append('quota-put'); data['values'][path]=json.loads(body)['limit_bytes'] + elif method=='DELETE': raise AssertionError('Node override deletion is forbidden') + else: data['events'].append('quota-readback') + state.write_text(json.dumps(data)) + def getresponse(self): + data=json.loads(Path(os.environ['TEST_STATE']).read_text()) + rows=json.loads(Path('ops/quota-overrides.json').read_text()) + targets=rows.get('overrides',rows) + status=200 + if self.host=='billing.example.invalid': + if rows.get('authority')=='billing': + floor=targets[unquote(self.path.rsplit('/',1)[1])]['limit_bytes'] + payload={'active':True,'comp_bytes':1,'account_limit_bytes':floor+100,'storage_limit_bytes':floor+100} + else: status,payload=404,os.environ['QUOTA_BILLING_ADMIN_SECRET'] + elif os.environ['FAIL_AT']=='quota': status,payload=503,os.environ['QUOTA_ADMIN_SECRET'] + elif self.path=='/admin/usage': payload={'count':len(targets),'spaces':[{'space_id':space,'usage_bytes':10} for space in targets]} + else: + value=data['values'].get(self.path) + payload={'space_id':unquote(self.path.removeprefix('/admin/quota/')),'override_bytes':value,'effective_limit_bytes':value if value is not None else 123} + class Response: + def read(self,count): return json.dumps(payload).encode()[:count] + response=Response(); response.status=status; return response + def close(self): pass +http.client.HTTPSConnection=Connection +''' + + +class SafeResult(unittest.TextTestResult): + def _exc_info_to_string(self, error, test): + text = super()._exc_info_to_string(error, test) + for value in CANARIES: + text = text.replace(value, "[withheld]") + return text + + +class RecoveryTests(unittest.TestCase): + def run_deploy(self, compose=CURRENT, changes=None, *, fail_at="", artifact_error="", authority="node-override", + missing_billing=False, missing_authorization=False, approved_privacy="", event="workflow_dispatch", + ref="refs/heads/main", oci_label=None, oci_revision=None, build_digest=""): + inputs = {"deploy_phala": "true", "authorize_quota_values": "true", "image_version": "1.20.6", + "deploy_image_digest": DIGEST, "rollback_image_version": "1.20.6", "rollback_image_revision": REVISION, + "rollback_pre_n0": "false", "rollback_privacy_mode": "unselected", "sql_identity_fence": "false"} + inputs.update(changes or {}) + if missing_authorization: + inputs.pop("authorize_quota_values", None) + canary = secrets.token_urlsafe(32) + CANARIES.append(canary) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory); binaries = root / "bin"; binaries.mkdir() + for name in ("docker", "git", "node", "phala", "curl", "sleep"): + binary = binaries / name; binary.write_text(CLI); binary.chmod(0o700) + (root / "sitecustomize.py").write_text(HTTP) + (root / "tinycloud-node-server").mkdir() + (root / "tinycloud-node-server/Cargo.toml").write_text('version = "1.20.7"\n') + (root / "dstack-authorized-keys").write_text("synthetic-public-key\n") + (root / "phala-env-sync").mkdir() + (root / "scripts/ci").mkdir(parents=True) + (root / "scripts/ci/reconcile-quota.py").write_bytes((ROOT / "scripts/ci/reconcile-quota.py").read_bytes()) + (root / "ops/rollback").mkdir(parents=True) + for filename, content in (("pre-n0-compose.yaml", PRE_N0), ("pre-n0-stop-local-logs.yaml", STOP_LOGS)): + (root / "ops/rollback" / filename).write_bytes(content) + base_artifact = root / "ops/rollback/pre-n0-compose.yaml" + if artifact_error == "drift": base_artifact.write_bytes(PRE_N0 + b"\n# drift\n") + if artifact_error == "missing": base_artifact.unlink() + if artifact_error == "symlink": + base_artifact.unlink(); (root / "alternate.yaml").write_bytes(PRE_N0); base_artifact.symlink_to(root / "alternate.yaml") + if artifact_error == "stop-drift": (root / "ops/rollback/pre-n0-stop-local-logs.yaml").write_bytes(STOP_LOGS + b"\n# drift\n") + config = json.loads((ROOT / "ops/quota-overrides.json").read_text()) + config["authority"] = authority + if authority is None: config.pop("authority") + (root / "ops/quota-overrides.json").write_text(json.dumps(config)) + compose_path = root / "docker-compose.dstack-postgres.yaml"; compose_path.write_bytes(compose) + if artifact_error == "root-symlink": + compose_path.unlink(); (root / "destination.yaml").write_bytes(compose); compose_path.symlink_to(root / "destination.yaml") + state = root / "state"; state.write_text(json.dumps({"events": [], "values": {}})) + output = root / "output"; summary = root / "summary" + deploy = shell_for_step("Deploy to Phala Cloud") + required = deploy.split("required_vars=(\n", 1)[1].split("\n)", 1)[0].split() + env = {name: canary for name in required} + env.update({"PATH": str(binaries) + ":" + str(Path(sys.executable).parent) + ":/usr/bin:/bin", + "PYTHONPATH": directory, "TMPDIR": directory, "RUNNER_TEMP": directory, "TEST_STATE": str(state), + "FAIL_AT": fail_at, "GITHUB_OUTPUT": str(output), "GITHUB_STEP_SUMMARY": str(summary), + "REGISTRY": "ghcr.io", "IMAGE_NAME": "tinycloudlabs/tinycloud-node", "BUILD_DIGEST": build_digest, + "EVENT_NAME": event, "GITHUB_REF": ref, "GITHUB_REF_TYPE": "branch", "GITHUB_REF_NAME": "main", "RELEASE_TAG": "", + "INPUT_DEPLOY_PHALA": inputs["deploy_phala"], "AUTHORIZE_QUOTA_VALUES": inputs.get("authorize_quota_values", "false"), + "INPUT_IMAGE_VERSION": inputs["image_version"], "INPUT_DEPLOY_DIGEST": inputs["deploy_image_digest"], + "ROLLBACK_VERSION": inputs.get("rollback_image_version", ""), "ROLLBACK_REVISION": inputs.get("rollback_image_revision", ""), + "ROLLBACK_PRE_N0": inputs["rollback_pre_n0"], "ROLLBACK_PRIVACY_MODE": inputs["rollback_privacy_mode"], + "APPROVED_ROLLBACK_PRIVACY_MODE": approved_privacy, "INCLUDE_DUCKDB": "false", "INCLUDE_TC_BENCH_V1": "false", + "DEPLOY_VERSION": inputs["image_version"], "OCI_LABEL": oci_label or inputs["image_version"] + "-dstack", + "OCI_REVISION": oci_revision or inputs.get("rollback_image_revision") or "f" * 40, + # Production maps both from PROD_TINYCLOUD_ADMIN_SECRET: one shared value. + "QUOTA_ADMIN_SECRET": canary, "QUOTA_BILLING_ADMIN_SECRET": canary, + "QUOTA_BILLING_URL": "https://billing.example.invalid/api/quota"}) + if missing_billing: env.pop("QUOTA_BILLING_ADMIN_SECRET") + context = {"github.event_name": event, "github.ref": ref, "job.success": True, + **{"inputs." + key: (value == "true" if value in ("true", "false") else value) for key, value in inputs.items()}} + logs = "" + names = ["Validate image version against Cargo.toml", "Preflight quota authority", "Validate and pin deployment image", + "Preflight sealed runtime configuration", "Prepare allowed_envs synchronization", "Deploy to Phala Cloud", + "Verify deployed version", "Verify deployed ingress health", "Restore storage quota overrides", + "Verify deployed Policy/v3 routes", "Report unverified storage overrides"] + names.sort(key=lambda name: SOURCE.index(" - name: " + name + "\n")) + for name in names: + script = shell_for_step(name) + selected = dict(line.split("=", 1) for line in output.read_text().splitlines()) if output.exists() else {} + context.update({"steps.pin.outputs." + key: value for key, value in selected.items()}) + context["steps.deploy.outputs.cvm_mutation_attempted"] = selected.get("cvm_mutation_attempted", "") + if not step_enabled(name, context): + if name == "Restore storage quota overrides": context["steps.quotas.outcome"] = "skipped" + continue + env.update({"EXPECTED": selected.get("version", inputs["image_version"]), "IMAGE_REVISION": selected.get("revision", REVISION)}) + result = subprocess.run(["bash", "--noprofile", "--norc", "-e", "-c", script], cwd=root, env=env, capture_output=True, text=True) + logs += result.stdout + result.stderr + if name == "Restore storage quota overrides": context["steps.quotas.outcome"] = "success" if result.returncode == 0 else "failure" + if result.returncode != 0: context["job.success"] = False + report = summary.read_text() if summary.exists() else "" + self.assertNotIn(canary, logs + report, "Sealed canary leaked") + self.assertFalse((root / ".phala-prod.env").exists(), "Sealed env was not cleaned") + return {"success": context["job.success"], "events": json.loads(state.read_text())["events"], + "compose": compose_path.read_bytes(), "summary": report, "logs": logs} + + def test_recovery_current_pre_n0_and_reviewed_n0(self): + for compose in (CURRENT, N0): + with self.subTest(n0=compose == N0): + result = self.run_deploy(compose) + self.assertTrue(result["success"], "Authorized same-image recovery must work") + self.assertIn("sealed-preflight", result["events"]) + self.assertEqual(result["events"].count("quota-put"), 2) + self.assertEqual(result["compose"], N0 if compose == N0 else PRE_N0) + self.assertNotIn("1.20.7", result["logs"] + result["summary"]) + + def test_future_digest_gets_preflight_and_general_quota_restore(self): + result = self.run_deploy(changes={"deploy_image_digest": "sha256:" + "a" * 64, + "image_version": "1.20.8", "rollback_image_version": "1.20.8", "rollback_image_revision": "a" * 40}) + self.assertTrue(result["success"]) + self.assertIn("sealed-preflight", result["events"]) + self.assertEqual(result["events"].count("quota-put"), 2) + self.assertIn(b"@sha256:" + b"a" * 64, result["compose"]) + + def test_explicit_rollback_privacy_variants(self): + for mode, expected in (("keep-json-file", PRE_N0), ("stop-future-local-logs", STOP_LOGS)): + with self.subTest(mode=mode): + result = self.run_deploy(N0, {"rollback_pre_n0": "true", "rollback_privacy_mode": mode}, approved_privacy=mode) + self.assertTrue(result["success"]) + self.assertEqual(result["compose"], expected) + self.assertEqual(result["events"].count("quota-put"), 2) + self.assertIn(mode, result["summary"]) + + def test_missing_authorization_never_reaches_any_mutation(self): + for compose, changes, mode in ((CURRENT, {}, ""), (N0, {}, ""), + (N0, {"rollback_pre_n0": "true", "rollback_privacy_mode": "keep-json-file"}, "keep-json-file"), + (CURRENT, {"deploy_image_digest": "sha256:" + "a" * 64, "image_version": "1.20.8", + "rollback_image_version": "1.20.8", "rollback_image_revision": "a" * 40}, "")): + result = self.run_deploy(compose, {**changes, "authorize_quota_values": "false"}, approved_privacy=mode) + self.assertFalse(result["success"]) + self.assertEqual(result["events"], []) + self.assertEqual(result["compose"], compose) + + def test_wrong_hash_and_artifact_paths_refuse_before_mutation(self): + wrong = N0.replace(b":-errors_slow}", b":-all_requests}") + self.assertFalse(self.run_deploy(wrong)["success"]) + for error in ("drift", "missing", "symlink", "stop-drift", "root-symlink"): + result = self.run_deploy(N0, {"rollback_pre_n0": "true", "rollback_privacy_mode": "stop-future-local-logs"}, + artifact_error=error, approved_privacy="stop-future-local-logs") + self.assertFalse(result["success"]) + self.assertNotIn("cvm-deploy", result["events"]) + self.assertEqual(result["compose"], N0) + + def test_rollback_privacy_is_explicit_approved_and_exclusive(self): + for changes, approval in (({"rollback_pre_n0": "true"}, ""), + ({"rollback_pre_n0": "true", "rollback_privacy_mode": "keep-json-file"}, ""), + ({"rollback_pre_n0": "true", "rollback_privacy_mode": "keep-json-file"}, "stop-future-local-logs"), + ({"rollback_privacy_mode": "stop-future-local-logs"}, "stop-future-local-logs")): + result = self.run_deploy(N0, changes, approved_privacy=approval) + self.assertFalse(result["success"]) + self.assertEqual(result["events"], []) + + def test_missing_authority_or_billing_secret_is_pre_mutation_failure(self): + for authority in (None, "unknown"): + result = self.run_deploy(N0, authority=authority) + self.assertFalse(result["success"]) + self.assertEqual(result["events"], []) + result = self.run_deploy(N0, missing_billing=True) + self.assertFalse(result["success"]) + self.assertEqual(result["events"], []) + self.assertEqual(result["compose"], N0) + + def test_dispatch_main_and_digest_pins_are_required(self): + for event, ref, changes in (("push", "refs/heads/main", {}), ("workflow_dispatch", "refs/tags/v1.20.6", {}), + ("workflow_dispatch", "refs/heads/main", {"deploy_image_digest": ""})): + result = self.run_deploy(N0, changes, event=event, ref=ref) + self.assertFalse(result["success"]) + self.assertEqual(result["events"], []) + + def test_wrong_oci_labels_refuse_before_compose_or_cvm_mutation(self): + for label, revision in (("1.20.6-dstack-duckdb", None), (None, "f" * 40)): + result = self.run_deploy(N0, oci_label=label, oci_revision=revision) + self.assertFalse(result["success"]) + self.assertNotIn("cvm-deploy", result["events"]) + self.assertEqual(result["compose"], N0) + + def test_normal_build_deployment_keeps_preflight_and_general_quota(self): + result = self.run_deploy(changes={"deploy_image_digest": "", "image_version": "1.20.7", + "rollback_image_version": "", "rollback_image_revision": ""}, + build_digest="sha256:" + "f" * 64) + self.assertTrue(result["success"]) + self.assertIn("sealed-preflight", result["events"]) + self.assertEqual(result["events"].count("quota-put"), 2) + + def test_forward_n0_lock_and_captured_rollback_are_not_substitutable(self): + future = {"deploy_image_digest": "sha256:" + "a" * 64, "image_version": "1.20.8", + "rollback_image_version": "1.20.8", "rollback_image_revision": "a" * 40} + for changes, mode in ((future, ""), ({**future, "rollback_pre_n0": "true", + "rollback_privacy_mode": "keep-json-file"}, "keep-json-file")): + result = self.run_deploy(N0, changes, approved_privacy=mode) + self.assertFalse(result["success"]) + self.assertNotIn("cvm-deploy", result["events"]) + self.assertEqual(result["compose"], N0) + + def test_last_restart_precedes_health_inventory_and_quota(self): + result = self.run_deploy(N0, {"rollback_pre_n0": "true", "rollback_privacy_mode": "keep-json-file"}, approved_privacy="keep-json-file") + self.assertTrue(result["success"]) + events = result["events"] + self.assertLess(events.index("final-restart"), events.index("ingress-health")) + self.assertLess(events.index("ingress-health"), events.index("quota-inventory")) + self.assertLess(events.index("quota-inventory"), events.index("quota-put")) + self.assertLess(events.index("quota-readback"), events.index("policy-probe")) + + def test_partial_restart_or_quota_failure_is_loudly_unverified(self): + for failure in ("deploy", "sync", "health", "quota"): + with self.subTest(phase=failure): + result = self.run_deploy(N0, {"rollback_pre_n0": "true", "rollback_privacy_mode": "keep-json-file"}, + fail_at=failure, approved_privacy="keep-json-file") + self.assertFalse(result["success"]) + self.assertIn("OVERRIDES UNVERIFIED", result["summary"]) + + def test_billing_authority_after_restart_is_read_only_and_cold_cache_safe(self): + result = self.run_deploy(authority="billing") + self.assertTrue(result["success"]) + self.assertNotIn("quota-put", result["events"]) + self.assertIn("billing-authoritative", result["summary"]) + + def test_policy_failure_cannot_skip_already_verified_quota(self): + result = self.run_deploy(N0, fail_at="policy") + self.assertFalse(result["success"]) + self.assertEqual(result["events"].count("quota-put"), 2) + self.assertNotIn("OVERRIDES UNVERIFIED", result["summary"]) + + def test_three_shell_blocks_keep_strict_runtime_options(self): + probe = """ + require_strict() { + for option in errexit nounset pipefail; do + case ":${SHELLOPTS}:" in + *:"${option}":*) ;; + *) return 89 ;; + esac + done + builtin printf 'strict\\n' >> "$SHELL_PROBE" + } + python3() { require_strict; } + gh() { require_strict; } + printf() { require_strict || return; builtin printf "$@"; } + """ + for name in ("Restore storage quota overrides", "Report unverified storage overrides", "Dispatch Write Probe promote watch"): + with self.subTest(step=name), tempfile.TemporaryDirectory() as directory: + startup = Path(directory) / "startup" + startup.write_text(textwrap.dedent(probe)) + trace = Path(directory) / "probe" + subprocess.run(["bash", "--noprofile", "--norc", "-c", shell_for_step(name)], + env={"PATH": "/usr/bin:/bin", "BASH_ENV": str(startup), "SHELL_PROBE": str(trace), + "GITHUB_STEP_SUMMARY": str(Path(directory) / "summary"), "IMAGE": "", "GITHUB_REPOSITORY": "synthetic/repository"}, + capture_output=True, text=True) + self.assertEqual(trace.read_text().splitlines(), ["strict"]) + + def test_documented_recovery_and_both_rollback_commands(self): + text = (ROOT / "docs/releasing.md").read_text() + for heading in ("### Emergency same-image recovery (12c)", "### TC-1302 captured pre-N0 rollback"): + self.assertTrue(heading in text, "Copy-paste recovery contract is missing") + section = text.split(heading, 1)[1].split("\n### ", 1)[0].split("\n## ", 1)[0] + commands = re.findall(r"```sh\n(gh workflow run docker.yml.*?)\n```", section, re.S) + self.assertEqual(len(commands), 1 if "12c" in heading else 2) + for command in commands: + tokens = shlex.split(command.replace("\\\n", " ")) + self.assertEqual(tokens[:6], ["gh", "workflow", "run", "docker.yml", "-R", "TinyCloudLabs/tinycloud-node"]) + self.assertEqual(tokens[tokens.index("--ref") + 1], "main") + inputs = dict(tokens[index + 1].split("=", 1) for index, value in enumerate(tokens) if value == "-f") + mode = inputs.get("rollback_privacy_mode", "") + result = self.run_deploy(changes=inputs, approved_privacy=mode) + self.assertTrue(result["success"], "Documented command failed actual input validation") + inputs.pop("authorize_quota_values") + denied = self.run_deploy(changes=inputs, missing_authorization=True, approved_privacy=mode) + self.assertFalse(denied["success"]) + self.assertEqual(denied["events"], []) + + +if __name__ == "__main__": + unittest.main(testRunner=unittest.TextTestRunner(resultclass=SafeResult, verbosity=2)) diff --git a/scripts/ci/test-quota-authority.py b/scripts/ci/test-quota-authority.py new file mode 100755 index 00000000..98b3bdbb --- /dev/null +++ b/scripts/ci/test-quota-authority.py @@ -0,0 +1,311 @@ +#!/usr/bin/env python3 +"""Synthetic HTTP contracts for reviewed node/billing quota authority.""" +import contextlib +import http.client +import io +import json +from pathlib import Path +import secrets +import tempfile +import unittest +from unittest.mock import patch +from urllib.parse import unquote + +ROOT = Path(__file__).resolve().parents[2] +SCRIPT = ROOT / "scripts/ci/reconcile-quota.py" +NS = {"__name__": "quota_authority_test", "__file__": str(SCRIPT)} +exec(compile(SCRIPT.read_text(), str(SCRIPT), "exec"), NS) + +OWNER = "tinycloud:pkh:eip155:1:0xd559CCd9EB87c530A9a349262669386dE93cf412" +LIMITS = {OWNER + ":applications": 5368709120, OWNER + ":secrets": 268721121} +CANARIES = [] + + +class SafeResult(unittest.TextTestResult): + def _exc_info_to_string(self, error, test): + output = super()._exc_info_to_string(error, test) + for value in [*CANARIES, *LIMITS]: + output = output.replace(value, "[withheld]") + return output + + +class AuthorityTests(unittest.TestCase): + def setUp(self): + self.node_secret = secrets.token_urlsafe(32) + self.billing_secret = secrets.token_urlsafe(32) + CANARIES.extend([self.node_secret, self.billing_secret]) + self.calls = [] + self.values = {} + self.billing_status = {space: 404 for space in LIMITS} + self.billing_body = {space: {"active": False, "comp_bytes": 0, + "account_limit_bytes": 104857600, "storage_limit_bytes": None} + for space in LIMITS} + self.transport_failures = 0 + self.apply_put = True + self.node_status = 200 + self.bad_effective = False + self.usage = [{"space_id": space, "usage_bytes": 10} for space in LIMITS] + self.directory = tempfile.TemporaryDirectory() + self.addCleanup(self.directory.cleanup) + self.config = Path(self.directory.name) / "quota.json" + self.summary = Path(self.directory.name) / "summary" + self.configure("node-override") + self.environment = { + "QUOTA_ADMIN_SECRET": self.node_secret, + "QUOTA_BILLING_ADMIN_SECRET": self.billing_secret, + "QUOTA_BILLING_URL": "https://billing.example.invalid/api/quota", + "GITHUB_STEP_SUMMARY": str(self.summary), + } + for patcher in ( + patch.object(http.client, "HTTPSConnection", self.connection), + patch("socket.create_connection", side_effect=AssertionError("Real network forbidden")), + patch.dict(NS, {"CONFIG_PATH": self.config}), + ): + patcher.start() + self.addCleanup(patcher.stop) + if "time" in NS: + patcher = patch.object(NS["time"], "sleep") + patcher.start() + self.addCleanup(patcher.stop) + + def configure(self, authority): + self.config.write_text(json.dumps({"authority": authority, "overrides": { + space: {"limit_bytes": limit, "_comment": "Reviewed synthetic target"} + for space, limit in LIMITS.items() + }})) + + def connection(self, host, timeout, **kwargs): + test = self + self.assertIn(host, ["node.tinycloud.xyz", "billing.example.invalid"]) + self.assertGreater(timeout, 0) + + class Connection: + def request(self, method, path, body=None, headers=None): + expected = test.node_secret if host == "node.tinycloud.xyz" else test.billing_secret + test.assertTrue(headers.get("Authorization") == "Bearer " + expected, + "Distinct credentials must remain in their own TLS headers") + test.assertFalse(test.node_secret in path or test.billing_secret in path) + test.calls.append((host, method, path)) + if host == "billing.example.invalid" and test.transport_failures: + test.transport_failures -= 1 + raise OSError(test.billing_secret) + self.method, self.path = method, path + if method == "PUT" and test.apply_put: + test.values[unquote(path.rsplit("/", 1)[1])] = json.loads(body)["limit_bytes"] + + def getresponse(self): + if host == "billing.example.invalid": + space = unquote(self.path.rsplit("/", 1)[1]) + status = test.billing_status[space] + body = test.billing_body[space] if status == 200 else test.billing_secret + elif test.node_status != 200: + status, body = test.node_status, test.node_secret + elif self.path == "/admin/usage": + status, body = 200, {"count": len(test.usage), "spaces": test.usage} + else: + space = unquote(self.path.rsplit("/", 1)[1]) + value = test.values.get(space) + status, body = 200, {"space_id": space, "override_bytes": value, + "effective_limit_bytes": (value + int(test.bad_effective)) if value is not None else 123, + "current_usage_bytes": 10} + + class Response: + def read(self, count): + return json.dumps(body).encode()[:count] + response = Response() + response.status = status + return response + + def close(self): + pass + return Connection() + + def run_main(self, args=()): + output = io.StringIO() + with patch.dict(NS["os"].environ, self.environment, clear=True): + with contextlib.redirect_stdout(output), contextlib.redirect_stderr(output): + code = NS["main"](list(args)) + logs = output.getvalue() + (self.summary.read_text() if self.summary.exists() else "") + self.assertFalse(any(value in logs for value in [*CANARIES, *LIMITS]), + "Secret, raw response or full space identity leaked") + return code, logs + + def puts(self): + return [call for call in self.calls if call[1] in ("PUT", "DELETE")] + + def billing_active(self): + for space, limit in LIMITS.items(): + self.billing_status[space] = 200 + # Incremental comp intentionally below the absolute per-space floor. + self.billing_body[space] = {"active": True, "comp_bytes": 1, + "account_limit_bytes": limit + 100, + "storage_limit_bytes": limit + 100} + + def test_legacy_404_checks_both_authorities_before_any_put(self): + code, logs = self.run_main() + self.assertEqual(code, 0) + self.assertIn("billing storage-comp unavailable", logs) + self.assertEqual(sorted(self.values.values()), sorted(LIMITS.values())) + first_put = next(index for index, call in enumerate(self.calls) if call[1] == "PUT") + self.assertEqual(sum(call[0] == "billing.example.invalid" for call in self.calls[:first_put]), 2) + self.assertEqual([call[1] for call in self.calls], ["GET"] * 3 + ["PUT", "GET", "PUT", "GET"]) + + def test_valid_inactive_200_allows_reviewed_legacy_restore(self): + for space in LIMITS: + self.billing_status[space] = 200 + self.assertEqual(self.run_main()[0], 0) + self.assertEqual(len(self.puts()), 2) + + def test_active_comp_on_second_space_blocks_every_legacy_put(self): + self.billing_active() + first = next(iter(LIMITS)) + self.billing_body[first]["active"] = False + code, logs = self.run_main() + self.assertEqual(code, 1) + self.assertEqual(self.puts(), []) + self.assertIn("authority mismatch: flip to billing", logs) + + def test_billing_http_errors_are_not_absence_or_retried(self): + first = next(iter(LIMITS)) + for status in (401, 403, 500, 503): + with self.subTest(status=status): + self.calls.clear() + self.billing_status[first] = status + self.assertEqual(self.run_main()[0], 1) + self.assertEqual(self.puts(), []) + self.assertEqual(sum(call[0] == "billing.example.invalid" for call in self.calls), 1) + + def test_malformed_billing_body_is_not_absence(self): + first = next(iter(LIMITS)) + self.billing_status[first] = 200 + for body in ({}, {"active": "false"}, {"active": False}, + {"active": False, "comp_bytes": True, "account_limit_bytes": 1, "storage_limit_bytes": None}): + with self.subTest(case=type(body.get("active")).__name__): + self.calls.clear() + self.billing_body[first] = body + self.assertEqual(self.run_main()[0], 1) + self.assertEqual(self.puts(), []) + + def test_transport_retry_is_bounded_without_details(self): + self.transport_failures = 2 + self.assertEqual(self.run_main()[0], 0) + self.calls.clear() + self.transport_failures = 10 + self.assertEqual(self.run_main()[0], 1) + self.assertEqual(sum(call[0] == "billing.example.invalid" for call in self.calls), 3) + self.assertEqual(self.puts(), []) + + def test_billing_mode_uses_absolute_limit_and_accepts_cold_node_cache(self): + self.configure("billing") + self.billing_active() + code, logs = self.run_main() + self.assertEqual(code, 0) + self.assertEqual(self.puts(), []) + self.assertIn("billing-authoritative; the node fetches on the next write (≤60 s TTL)", logs) + + def test_billing_absent_inactive_or_insufficient_fails_without_mutation(self): + self.configure("billing") + for variant in ("404", "inactive", "insufficient", "unknown-limit"): + with self.subTest(case=variant): + self.billing_active() + first = next(iter(LIMITS)) + if variant == "404": + self.billing_status[first] = 404 + elif variant == "inactive": + self.billing_body[first]["active"] = False + elif variant == "insufficient": + self.billing_body[first]["storage_limit_bytes"] = LIMITS[first] - 1 + else: + self.billing_body[first]["storage_limit_bytes"] = None + self.assertEqual(self.run_main()[0], 1) + self.assertEqual(self.puts(), []) + + def test_stale_local_override_rejects_billing_mode(self): + self.configure("billing") + self.billing_active() + self.values[next(iter(LIMITS))] = 1 + code, logs = self.run_main() + self.assertEqual(code, 1) + self.assertIn("stale legacy override", logs) + self.assertEqual(self.puts(), []) + + def test_unapplied_legacy_put_fails_readback(self): + self.apply_put = False + self.assertEqual(self.run_main()[0], 1) + self.assertEqual(len(self.puts()), 1) + + def test_missing_unknown_authority_fails_validate_only_without_requests(self): + for value in (None, "unknown"): + with self.subTest(case=value): + self.configure(value) + if value is None: + document = json.loads(self.config.read_text()) + del document["authority"] + self.config.write_text(json.dumps(document)) + self.assertEqual(self.run_main(["--validate-only"])[0], 1) + self.assertEqual(self.calls, []) + + def test_missing_billing_secret_fails_preflight_without_requests(self): + self.environment.pop("QUOTA_BILLING_ADMIN_SECRET") + self.assertEqual(self.run_main(["--preflight"])[0], 1) + self.assertEqual(self.calls, []) + + def test_preflight_does_not_require_a_healthy_node_or_mutate(self): + self.assertEqual(self.run_main(["--preflight"])[0], 0) + self.assertEqual(self.calls, []) + + def test_incompatible_or_credential_bearing_billing_url_fails_without_requests(self): + for url in ("http://billing.example.invalid/api/quota", "https://billing.example.invalid/unreviewed", + "https://user:password@billing.example.invalid/api/quota", "https://billing.example.invalid/api/quota?key=bad"): + self.environment["QUOTA_BILLING_URL"] = url + self.assertEqual(self.run_main(["--preflight"])[0], 1) + self.assertEqual(self.calls, []) + + def test_effective_limit_and_override_must_both_match(self): + self.bad_effective = True + self.assertEqual(self.run_main()[0], 1) + self.assertEqual(len(self.puts()), 1) + + def test_invalid_inventory_never_applies_either_override(self): + first, second = list(LIMITS) + for rows in ([{"space_id": first, "usage_bytes": 10}], + [{"space_id": first, "usage_bytes": 10}, {"space_id": second, "usage_bytes": None}], + [{"space_id": first, "usage_bytes": 10}, {"space_id": second, "usage_bytes": 0}], + [{"space_id": first, "usage_bytes": 10}, {"space_id": second, "usage_bytes": True}], + [{"space_id": first, "usage_bytes": 10}, {"space_id": first, "usage_bytes": 10}], + [{"space_id": self.node_secret, "usage_bytes": 10}]): + with self.subTest(rows=len(rows)): + self.calls.clear() + self.usage = rows + self.assertEqual(self.run_main()[0], 1) + self.assertEqual(self.puts(), []) + + def test_wrong_owner_config_fails_before_any_request(self): + document = json.loads(self.config.read_text()) + document["overrides"] = { + space.replace(OWNER, "tinycloud:pkh:eip155:1:0x" + "1" * 40): row + for space, row in document["overrides"].items() + } + self.config.write_text(json.dumps(document)) + self.assertEqual(self.run_main()[0], 1) + self.assertEqual(self.calls, []) + + def test_checked_in_full_target_and_value_contract(self): + authority, overrides = NS["load_configuration"](ROOT / "ops/quota-overrides.json") + self.assertIn(authority, ("node-override", "billing")) + self.assertTrue(overrides == LIMITS, "Canonical Sam targets/values changed") + + def test_missing_node_secret_is_preflight_failure(self): + self.environment.pop("QUOTA_ADMIN_SECRET") + self.assertEqual(self.run_main(["--preflight"])[0], 1) + self.assertEqual(self.calls, []) + + def test_node_http_error_does_not_retry_or_leak(self): + self.node_status = 503 + self.assertEqual(self.run_main()[0], 1) + self.assertEqual(len(self.calls), 1) + self.assertEqual(self.puts(), []) + + +if __name__ == "__main__": + unittest.main(testRunner=unittest.TextTestRunner(resultclass=SafeResult, verbosity=2))