From 82a7cd07d2cd878eef4c821f4126559eacfb1b37 Mon Sep 17 00:00:00 2001 From: Sam Gbafa Date: Sat, 17 Oct 2026 04:15:25 +0000 Subject: [PATCH 1/2] test(TC-959): use pre-N0 template fixture in recovery tests Future-digest and normal-build recovery cases ran against the repository's current compose. Since #309 that file is the reviewed N0 compose, which the workflow intentionally refuses to pair with any image other than the reviewed 1.20.6 digest. Run these cases against a frozen copy of the pre-N0 template (sha256 28302d67...) instead, and assert the N0 compose refuses a normal build image as it already refuses a future digest. --- .../ci/fixtures/pre-n0-template-compose.yaml | 134 ++++++++++++++++++ scripts/ci/test-deploy-recovery.py | 20 ++- 2 files changed, 149 insertions(+), 5 deletions(-) create mode 100644 scripts/ci/fixtures/pre-n0-template-compose.yaml diff --git a/scripts/ci/fixtures/pre-n0-template-compose.yaml b/scripts/ci/fixtures/pre-n0-template-compose.yaml new file mode 100644 index 00000000..9afa4196 --- /dev/null +++ b/scripts/ci/fixtures/pre-n0-template-compose.yaml @@ -0,0 +1,134 @@ +# dstack deployment - Topology 2: Postgres + S3 +# tinycloud-node in the CVM, Postgres external. +# Database credentials encrypted by dstack KMS. +# Column encryption enabled for sensitive data. +# +# All ${...} variables should be encrypted using dstack's KMS public key +# before deployment. The operator never sees plaintext values. + +services: + tinycloud: + # DuckDB-enabled images are opt-in and use the -duckdb suffix, for example: + # ghcr.io/tinycloudlabs/tinycloud-node:1.4.0-dstack-duckdb + image: ghcr.io/tinycloudlabs/tinycloud-node:dstack + restart: unless-stopped + # TC-970: cap the node's Docker log (stdout: one line per request plus + # application logs) at 3 x 50 MB instead of the guest daemon default + # (current dstack OS images: json-file, 10 x 100 MB). json-file keeps the + # dstack log API (`/logs/`) working. + logging: + driver: json-file + options: + max-size: "50m" + max-file: "3" + ports: + - "8000:8000" + volumes: + - /var/run/dstack.sock:/var/run/dstack.sock + environment: + TINYCLOUD_KEYS_TYPE: Dstack + TINYCLOUD_STORAGE_DATABASE: "${DATABASE_URL}" + TINYCLOUD_STORAGE_BLOCKS_TYPE: S3 + TINYCLOUD_STORAGE_BLOCKS_BUCKET: "${S3_BUCKET}" + TINYCLOUD_STORAGE_BLOCKS_ENDPOINT: "${S3_ENDPOINT}" + AWS_ACCESS_KEY_ID: "${AWS_KEY}" + AWS_SECRET_ACCESS_KEY: "${AWS_SECRET}" + AWS_DEFAULT_REGION: "${AWS_REGION}" + TINYCLOUD_LOG_LEVEL: normal + # Enable Prometheus telemetry (metrics, span histograms, pool gauges). + # The metrics handler is intentionally NOT gateway-exposed: port 8001 is + # not published here, so the unauthenticated /metrics endpoint stays + # internal to the CVM. + TINYCLOUD_TELEMETRY__ENABLED: "true" + # The managed Postgres instance allows max_connections = 25 (measured + # 2026-07-29, with some of those reserved for superuser roles). The pool + # default of 100 cannot be satisfied: the node opened and held slots + # until the instance ran out, and further requests failed with + # "Failed to acquire connection from pool: Connection pool timed out" + # after the 30s sqlx default. Keep this below the instance limit with + # headroom for admin/psql access, and raise it only alongside the + # instance's own max_connections. + TINYCLOUD_DATABASE__MAX_CONNECTIONS: "15" + # Required for every manual TC-780 deploy. Keep true through offline + # verification; only the final, explicitly reviewed redeploy sets false. + TINYCLOUD_DATABASE__WRITE_FENCE: "${SQL_IDENTITY_FENCE:?set true during cutover, false after verification}" + TINYCLOUD_CORS: "true" + # --- addressed sharing (TC-397) ------------------------------------ + # + # The public trust document shared with Share and OpenCredentials, + # delivered inline because nothing else can reach this container: the + # image is `FROM scratch` (no shell, so share-api's decode-to-tmpfs + # entrypoint cannot work here) and a dstack deployment uploads only + # this compose file (so there is no host path to bind-mount from). + # + # Seal the base64 of the reviewed trust-bundle.production.json — the + # same bytes share-api serves — into the CVM's encrypted environment as + # SHARE_TRUST_BUNDLE_BASE64. Same variable name and same wire form + # share-api already uses, so one sealed value feeds both services and + # cannot drift between them. + # + # The `:?` form is deliberate: compose refuses to interpolate a partial + # deployment, failing loudly at deploy time rather than letting the node + # die at boot with "share email trust bundle is required". + TINYCLOUD_SHARE_EMAIL__ENABLED: "true" + TINYCLOUD_SHARE_EMAIL__TRUST_BUNDLE_BASE64: "${SHARE_TRUST_BUNDLE_BASE64:?seal the reviewed share-email trust bundle, base64-encoded}" + # The sealed DATABASE_URL already carries sslmode=verify-full as its sole + # query parameter. ROOT_CERT_PATH stays unset so verify-full resolves + # against the compiled-in public root set (TC-363). + TINYCLOUD_SHARE_EMAIL__POSTGRES_TLS__SSLMODE: verify-full + TINYCLOUD_QUOTA_URL: "${TINYCLOUD_QUOTA_URL}" + TINYCLOUD_ADMIN_SECRET: "${TINYCLOUD_ADMIN_SECRET}" + ROCKET_ADDRESS: "0.0.0.0" + ROCKET_MAX_BLOCKING: "1024" + + dstack-ingress: + image: ghcr.io/tinycloudlabs/tinycloud-node-ingress@sha256:63c1c44e2ded5b73a5f174807b91ca677826974df0f9ab3045f6ec87865a5744 + ports: + - "443:443" + depends_on: + - tinycloud + environment: + - DOMAIN=tee.node.tinycloud.xyz + - TARGET_ENDPOINT=http://tinycloud:8000 + - CLOUDFLARE_API_TOKEN=${CLOUDFLARE_API_TOKEN} + # CNAME target the ingress publishes for DOMAIN. Must not start with `_`: + # Android's resolver (bionic res_hnok) rejects a CNAME target whose first + # label is `_`, so `_.` is unresolvable on Android (TC-513). + # `gateway.` is the target Phala Cloud advertises (gateway.cname) + # and upstream dstack-ingress documents; routing still uses SNI + the + # _dstack-app-address TXT record, so TLS stays end-to-end into the CVM. + - GATEWAY_DOMAIN=gateway.${DSTACK_GATEWAY_DOMAIN} + - CERTBOT_EMAIL=${CERTBOT_EMAIL} + - SET_CAA=true + - DNS_PROVIDER=cloudflare + - CLIENT_MAX_BODY_SIZE=32m + volumes: + - /var/run/dstack.sock:/var/run/dstack.sock + - cert-data:/etc/letsencrypt + restart: unless-stopped + # TC-970: the ingress logs one JSON line per request to stdout. Cap its + # Docker log at 3 x 50 MB instead of relying on the guest daemon default + # (current dstack OS images ship json-file with 10 x 100 MB). json-file + # keeps the dstack log API (`/logs/`) working. + logging: + driver: json-file + options: + max-size: "50m" + max-file: "3" + + # The dstack gateway's TLS-terminating routes (-.:443) + # only forward to compose-published ports, so guest sshd on :22 is + # unreachable via `phala ssh` even with authorized_keys installed. + # Publish a TCP forwarder on 10022 that relays to guest sshd via the + # docker host gateway. Port 10022 exposes ONLY sshd, which is key-gated. + ssh-gateway: + image: alpine/socat:1.8.1.3@sha256:188fe0a22182f81c16def9d1137930121eaa3023f427164e32c0d2f118f79dd6 + restart: unless-stopped + ports: + - "10022:10022" + extra_hosts: + - "host.docker.internal:host-gateway" + command: TCP-LISTEN:10022,fork,reuseaddr TCP:host.docker.internal:22 + +volumes: + cert-data: diff --git a/scripts/ci/test-deploy-recovery.py b/scripts/ci/test-deploy-recovery.py index d5c6bd18..5728fadf 100755 --- a/scripts/ci/test-deploy-recovery.py +++ b/scripts/ci/test-deploy-recovery.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 """Run actual workflow Bash against offline CLI/HTTP boundaries only.""" +import hashlib import json from pathlib import Path import re @@ -19,6 +20,9 @@ PRE_N0 = (ROOT / "ops/rollback/pre-n0-compose.yaml").read_bytes() STOP_LOGS = (ROOT / "ops/rollback/pre-n0-stop-local-logs.yaml").read_bytes() CURRENT = (ROOT / "docker-compose.dstack-postgres.yaml").read_bytes() +# Frozen ordinary (pre-N0) template from main before #309; the workflow accepts it by raw hash. +PRE_N0_TEMPLATE = (ROOT / "scripts/ci/fixtures/pre-n0-template-compose.yaml").read_bytes() +assert hashlib.sha256(PRE_N0_TEMPLATE).hexdigest() == "28302d6761d841b7cc5c11314aed2d377735e6bbdcaf85972a752e03bc9fdc93" CANARIES = [] @@ -247,7 +251,7 @@ def run_deploy(self, compose=CURRENT, changes=None, *, fail_at="", artifact_erro "compose": compose_path.read_bytes(), "summary": report, "logs": logs} def test_recovery_current_pre_n0_and_reviewed_n0(self): - for compose in (CURRENT, N0): + for compose in (PRE_N0_TEMPLATE, N0): with self.subTest(n0=compose == N0): result = self.run_deploy(compose) self.assertTrue(result["success"], "Authorized same-image recovery must work") @@ -257,7 +261,7 @@ def test_recovery_current_pre_n0_and_reviewed_n0(self): self.assertNotIn("1.20.7", result["logs"] + result["summary"]) def test_future_digest_gets_preflight_and_general_quota_restore(self): - result = self.run_deploy(changes={"deploy_image_digest": "sha256:" + "a" * 64, + result = self.run_deploy(PRE_N0_TEMPLATE, {"deploy_image_digest": "sha256:" + "a" * 64, "image_version": "1.20.8", "rollback_image_version": "1.20.8", "rollback_image_revision": "a" * 40}) self.assertTrue(result["success"]) self.assertIn("sealed-preflight", result["events"]) @@ -274,9 +278,9 @@ def test_explicit_rollback_privacy_variants(self): self.assertIn(mode, result["summary"]) def test_missing_authorization_never_reaches_any_mutation(self): - for compose, changes, mode in ((CURRENT, {}, ""), (N0, {}, ""), + for compose, changes, mode in ((PRE_N0_TEMPLATE, {}, ""), (N0, {}, ""), (N0, {"rollback_pre_n0": "true", "rollback_privacy_mode": "keep-json-file"}, "keep-json-file"), - (CURRENT, {"deploy_image_digest": "sha256:" + "a" * 64, "image_version": "1.20.8", + (PRE_N0_TEMPLATE, {"deploy_image_digest": "sha256:" + "a" * 64, "image_version": "1.20.8", "rollback_image_version": "1.20.8", "rollback_image_revision": "a" * 40}, "")): result = self.run_deploy(compose, {**changes, "authorize_quota_values": "false"}, approved_privacy=mode) self.assertFalse(result["success"]) @@ -327,7 +331,7 @@ def test_wrong_oci_labels_refuse_before_compose_or_cvm_mutation(self): self.assertEqual(result["compose"], N0) def test_normal_build_deployment_keeps_preflight_and_general_quota(self): - result = self.run_deploy(changes={"deploy_image_digest": "", "image_version": "1.20.7", + result = self.run_deploy(PRE_N0_TEMPLATE, {"deploy_image_digest": "", "image_version": "1.20.7", "rollback_image_version": "", "rollback_image_revision": ""}, build_digest="sha256:" + "f" * 64) self.assertTrue(result["success"]) @@ -343,6 +347,12 @@ def test_forward_n0_lock_and_captured_rollback_are_not_substitutable(self): self.assertFalse(result["success"]) self.assertNotIn("cvm-deploy", result["events"]) self.assertEqual(result["compose"], N0) + result = self.run_deploy(N0, {"deploy_image_digest": "", "image_version": "1.20.7", + "rollback_image_version": "", "rollback_image_revision": ""}, + build_digest="sha256:" + "f" * 64) + self.assertFalse(result["success"]) + self.assertNotIn("cvm-deploy", result["events"]) + self.assertEqual(result["compose"], N0) def test_last_restart_precedes_health_inventory_and_quota(self): result = self.run_deploy(N0, {"rollback_pre_n0": "true", "rollback_privacy_mode": "keep-json-file"}, approved_privacy="keep-json-file") From 36265f9a705747b1dfe5dcb5e1ddfcf83f067028 Mon Sep 17 00:00:00 2001 From: Sam Gbafa Date: Sat, 17 Oct 2026 04:16:38 +0000 Subject: [PATCH 2/2] test(TC-959): derive pre-N0 template from the rollback artifact Same bytes (sha256 28302d67...) as the frozen copy, without adding a fixture file; matches the derivation used by #315. --- .../ci/fixtures/pre-n0-template-compose.yaml | 134 ------------------ scripts/ci/test-deploy-recovery.py | 5 +- 2 files changed, 3 insertions(+), 136 deletions(-) delete mode 100644 scripts/ci/fixtures/pre-n0-template-compose.yaml diff --git a/scripts/ci/fixtures/pre-n0-template-compose.yaml b/scripts/ci/fixtures/pre-n0-template-compose.yaml deleted file mode 100644 index 9afa4196..00000000 --- a/scripts/ci/fixtures/pre-n0-template-compose.yaml +++ /dev/null @@ -1,134 +0,0 @@ -# dstack deployment - Topology 2: Postgres + S3 -# tinycloud-node in the CVM, Postgres external. -# Database credentials encrypted by dstack KMS. -# Column encryption enabled for sensitive data. -# -# All ${...} variables should be encrypted using dstack's KMS public key -# before deployment. The operator never sees plaintext values. - -services: - tinycloud: - # DuckDB-enabled images are opt-in and use the -duckdb suffix, for example: - # ghcr.io/tinycloudlabs/tinycloud-node:1.4.0-dstack-duckdb - image: ghcr.io/tinycloudlabs/tinycloud-node:dstack - restart: unless-stopped - # TC-970: cap the node's Docker log (stdout: one line per request plus - # application logs) at 3 x 50 MB instead of the guest daemon default - # (current dstack OS images: json-file, 10 x 100 MB). json-file keeps the - # dstack log API (`/logs/`) working. - logging: - driver: json-file - options: - max-size: "50m" - max-file: "3" - ports: - - "8000:8000" - volumes: - - /var/run/dstack.sock:/var/run/dstack.sock - environment: - TINYCLOUD_KEYS_TYPE: Dstack - TINYCLOUD_STORAGE_DATABASE: "${DATABASE_URL}" - TINYCLOUD_STORAGE_BLOCKS_TYPE: S3 - TINYCLOUD_STORAGE_BLOCKS_BUCKET: "${S3_BUCKET}" - TINYCLOUD_STORAGE_BLOCKS_ENDPOINT: "${S3_ENDPOINT}" - AWS_ACCESS_KEY_ID: "${AWS_KEY}" - AWS_SECRET_ACCESS_KEY: "${AWS_SECRET}" - AWS_DEFAULT_REGION: "${AWS_REGION}" - TINYCLOUD_LOG_LEVEL: normal - # Enable Prometheus telemetry (metrics, span histograms, pool gauges). - # The metrics handler is intentionally NOT gateway-exposed: port 8001 is - # not published here, so the unauthenticated /metrics endpoint stays - # internal to the CVM. - TINYCLOUD_TELEMETRY__ENABLED: "true" - # The managed Postgres instance allows max_connections = 25 (measured - # 2026-07-29, with some of those reserved for superuser roles). The pool - # default of 100 cannot be satisfied: the node opened and held slots - # until the instance ran out, and further requests failed with - # "Failed to acquire connection from pool: Connection pool timed out" - # after the 30s sqlx default. Keep this below the instance limit with - # headroom for admin/psql access, and raise it only alongside the - # instance's own max_connections. - TINYCLOUD_DATABASE__MAX_CONNECTIONS: "15" - # Required for every manual TC-780 deploy. Keep true through offline - # verification; only the final, explicitly reviewed redeploy sets false. - TINYCLOUD_DATABASE__WRITE_FENCE: "${SQL_IDENTITY_FENCE:?set true during cutover, false after verification}" - TINYCLOUD_CORS: "true" - # --- addressed sharing (TC-397) ------------------------------------ - # - # The public trust document shared with Share and OpenCredentials, - # delivered inline because nothing else can reach this container: the - # image is `FROM scratch` (no shell, so share-api's decode-to-tmpfs - # entrypoint cannot work here) and a dstack deployment uploads only - # this compose file (so there is no host path to bind-mount from). - # - # Seal the base64 of the reviewed trust-bundle.production.json — the - # same bytes share-api serves — into the CVM's encrypted environment as - # SHARE_TRUST_BUNDLE_BASE64. Same variable name and same wire form - # share-api already uses, so one sealed value feeds both services and - # cannot drift between them. - # - # The `:?` form is deliberate: compose refuses to interpolate a partial - # deployment, failing loudly at deploy time rather than letting the node - # die at boot with "share email trust bundle is required". - TINYCLOUD_SHARE_EMAIL__ENABLED: "true" - TINYCLOUD_SHARE_EMAIL__TRUST_BUNDLE_BASE64: "${SHARE_TRUST_BUNDLE_BASE64:?seal the reviewed share-email trust bundle, base64-encoded}" - # The sealed DATABASE_URL already carries sslmode=verify-full as its sole - # query parameter. ROOT_CERT_PATH stays unset so verify-full resolves - # against the compiled-in public root set (TC-363). - TINYCLOUD_SHARE_EMAIL__POSTGRES_TLS__SSLMODE: verify-full - TINYCLOUD_QUOTA_URL: "${TINYCLOUD_QUOTA_URL}" - TINYCLOUD_ADMIN_SECRET: "${TINYCLOUD_ADMIN_SECRET}" - ROCKET_ADDRESS: "0.0.0.0" - ROCKET_MAX_BLOCKING: "1024" - - dstack-ingress: - image: ghcr.io/tinycloudlabs/tinycloud-node-ingress@sha256:63c1c44e2ded5b73a5f174807b91ca677826974df0f9ab3045f6ec87865a5744 - ports: - - "443:443" - depends_on: - - tinycloud - environment: - - DOMAIN=tee.node.tinycloud.xyz - - TARGET_ENDPOINT=http://tinycloud:8000 - - CLOUDFLARE_API_TOKEN=${CLOUDFLARE_API_TOKEN} - # CNAME target the ingress publishes for DOMAIN. Must not start with `_`: - # Android's resolver (bionic res_hnok) rejects a CNAME target whose first - # label is `_`, so `_.` is unresolvable on Android (TC-513). - # `gateway.` is the target Phala Cloud advertises (gateway.cname) - # and upstream dstack-ingress documents; routing still uses SNI + the - # _dstack-app-address TXT record, so TLS stays end-to-end into the CVM. - - GATEWAY_DOMAIN=gateway.${DSTACK_GATEWAY_DOMAIN} - - CERTBOT_EMAIL=${CERTBOT_EMAIL} - - SET_CAA=true - - DNS_PROVIDER=cloudflare - - CLIENT_MAX_BODY_SIZE=32m - volumes: - - /var/run/dstack.sock:/var/run/dstack.sock - - cert-data:/etc/letsencrypt - restart: unless-stopped - # TC-970: the ingress logs one JSON line per request to stdout. Cap its - # Docker log at 3 x 50 MB instead of relying on the guest daemon default - # (current dstack OS images ship json-file with 10 x 100 MB). json-file - # keeps the dstack log API (`/logs/`) working. - logging: - driver: json-file - options: - max-size: "50m" - max-file: "3" - - # The dstack gateway's TLS-terminating routes (-.:443) - # only forward to compose-published ports, so guest sshd on :22 is - # unreachable via `phala ssh` even with authorized_keys installed. - # Publish a TCP forwarder on 10022 that relays to guest sshd via the - # docker host gateway. Port 10022 exposes ONLY sshd, which is key-gated. - ssh-gateway: - image: alpine/socat:1.8.1.3@sha256:188fe0a22182f81c16def9d1137930121eaa3023f427164e32c0d2f118f79dd6 - restart: unless-stopped - ports: - - "10022:10022" - extra_hosts: - - "host.docker.internal:host-gateway" - command: TCP-LISTEN:10022,fork,reuseaddr TCP:host.docker.internal:22 - -volumes: - cert-data: diff --git a/scripts/ci/test-deploy-recovery.py b/scripts/ci/test-deploy-recovery.py index 5728fadf..ae1d30ed 100755 --- a/scripts/ci/test-deploy-recovery.py +++ b/scripts/ci/test-deploy-recovery.py @@ -20,8 +20,9 @@ PRE_N0 = (ROOT / "ops/rollback/pre-n0-compose.yaml").read_bytes() STOP_LOGS = (ROOT / "ops/rollback/pre-n0-stop-local-logs.yaml").read_bytes() CURRENT = (ROOT / "docker-compose.dstack-postgres.yaml").read_bytes() -# Frozen ordinary (pre-N0) template from main before #309; the workflow accepts it by raw hash. -PRE_N0_TEMPLATE = (ROOT / "scripts/ci/fixtures/pre-n0-template-compose.yaml").read_bytes() +# Ordinary (pre-N0) template from main before #309: the captured rollback compose with its digest pin +# reverted to the :dstack tag. The workflow accepts these bytes by raw hash. +PRE_N0_TEMPLATE = PRE_N0.replace(b"@" + DIGEST.encode(), b":dstack") assert hashlib.sha256(PRE_N0_TEMPLATE).hexdigest() == "28302d6761d841b7cc5c11314aed2d377735e6bbdcaf85972a752e03bc9fdc93" CANARIES = []