diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml index 9ecbff66f..9a105c9c0 100644 --- a/.github/workflows/format.yml +++ b/.github/workflows/format.yml @@ -4,7 +4,8 @@ on: push: branches: [main] pull_request: - branches: [main] + # Keep lint gates active for native-store and Geo stacked PRs. + branches: [main, feat/native-celld-stores, feature/pluggable-geo-providers] permissions: contents: read @@ -117,6 +118,12 @@ jobs: feature_sets: |- axum axum,test-utils + geo-cloudfront + geo-cloudfront,test-utils + geo-maxmind + geo-maxmind,test-utils + geo-cloudfront,geo-maxmind + geo-cloudfront,geo-maxmind,test-utils steps: - uses: actions/checkout@v6 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index adc4d2c70..1155297e1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -4,7 +4,14 @@ on: push: branches: [main] pull_request: - branches: [main] + # Keep checks active for native-store and Geo stacked PRs. + branches: + [ + main, + docs/outbound-http-spec, + feat/native-celld-stores, + feature/pluggable-geo-providers, + ] permissions: contents: read @@ -114,6 +121,61 @@ jobs: - name: Run workspace tests run: cargo test --workspace --all-targets + - name: Native store dependency isolation + run: bash scripts/check_native_store_dependency_contract.sh + + - name: Geo dependency isolation + run: bash scripts/check_geo_dependency_contract.sh + + - name: Synthetic GeoIP fixture reproducibility + run: | + node --check scripts/generate_geoip_fixtures.mjs + node scripts/generate_geoip_fixtures.mjs --check + + - name: Immutable core Geo outcome + run: scripts/run_test_nonzero.sh geo_owned_outcome_survives_public_request_mutation cargo test --offline --locked -p edgezero-core --lib geo_ + + - name: Native Geo disabled implementation and policy unit contracts + run: | + scripts/run_test_nonzero.sh geo_protected_policy_reuses_network_bounds cargo test --offline --locked -p edgezero-adapter-axum --no-default-features --features axum --lib geo:: + scripts/run_test_nonzero.sh geo_unavailable_selection_fails_before_runner_resources cargo test --offline --locked -p edgezero-adapter-axum --no-default-features --features axum --lib geo_ + + - name: Native Geo supplied-provider unit contract without built-in + run: scripts/run_test_nonzero.sh source_admission_precedes_header_inspection_and_provider_calls cargo test --offline --locked -p edgezero-adapter-axum --no-default-features --features axum --lib source_admission_precedes_header_inspection_and_provider_calls + + - name: Native Geo enabled implementation and managed dispatch unit contracts + run: | + scripts/run_test_nonzero.sh geo_cloudfront_parsing_matches_spec cargo test --offline --locked -p edgezero-adapter-axum --no-default-features --features geo-cloudfront --lib geo:: + scripts/run_test_nonzero.sh geo_managed_resolution_runs_once_from_owned_ingress cargo test --offline --locked -p edgezero-adapter-axum --no-default-features --features geo-cloudfront --lib geo_ + cargo test --offline --locked -p edgezero-adapter-axum --features geo-cloudfront --lib service::tests + + - name: Native Geo disabled built-in hosting (loopback only) + run: scripts/run_test_nonzero.sh geo_hosting_rejects_unavailable_cloudfront cargo test --offline --locked -p edgezero-adapter-axum --no-default-features --features axum,test-utils --test production_hosting geo_hosting_rejects_unavailable_cloudfront + + - name: Native Geo supplied provider without built-in (loopback only) + run: scripts/run_test_nonzero.sh geo_hosting_supplied_provider_without_builtin cargo test --offline --locked -p edgezero-adapter-axum --no-default-features --features axum,test-utils --test production_hosting geo_hosting_supplied_provider_without_builtin + + - name: Native Geo real-peer hosting (loopback only) + run: scripts/run_test_nonzero.sh geo_hosting_proves_real_peer_and_shared_outcome cargo test --offline --locked -p edgezero-adapter-axum --no-default-features --features geo-cloudfront,test-utils --test production_hosting geo_hosting_proves_real_peer_and_shared_outcome + + - name: Native AWS provider and startup contracts (loopback only) + run: | + cargo test --offline --locked -p edgezero-store-aws --no-default-features --features appconfig-agent --lib + cargo test --offline --locked -p edgezero-store-aws --no-default-features --features secrets-manager --lib + cargo test --offline --locked -p edgezero-store-aws --no-default-features --features dynamodb-kv --lib + cargo test --offline --locked -p edgezero-store-aws --all-features --lib + scripts/run_test_nonzero.sh production_runner_loads_envelope_and_real_sdk_secret_before_serving_mixed_stores cargo test --offline --locked -p edgezero-adapter-axum --features aws-stores,test-utils --test native_aws_startup + + - name: Native celld independent provider and startup contracts (loopback only) + run: | + cargo test --offline --locked -p edgezero-store-celld --no-default-features --lib + cargo test --offline --locked -p edgezero-store-celld --no-default-features --features config --lib + cargo test --offline --locked -p edgezero-store-celld --no-default-features --features kv --lib + cargo test --offline --locked -p edgezero-store-celld --all-features --lib + cargo test --offline --locked -p edgezero-adapter-axum --no-default-features --features native-bindings --lib + cargo test --offline --locked -p edgezero-adapter-axum --features celld-config,celld-kv,test-utils --test native_celld_startup + cargo test --offline --locked -p edgezero-adapter-axum --features aws-stores,aws-dynamodb,celld-config,celld-kv,test-utils --lib --test native_aws_startup --test native_celld_startup + - name: Bundled demo logging contract run: scripts/run_test_nonzero.sh demo_respects_off_without_hiding_terminal_errors cargo test --offline --locked -p edgezero-cli --features demo-example --test demo_logging @@ -147,6 +209,29 @@ jobs: bash scripts/check_adapter_feature_matrix.sh "${adapter}" native done + - name: Generated native AWS startup (loopback only) + env: + EDGEZERO_GENERATED_TEST_TARGET_DIR: ${{ github.workspace }}/target + run: scripts/run_test_nonzero.sh --ignored generated_cli_build_features_and_native_aws_startup_are_proven_together cargo test --offline --locked -p edgezero-cli --test native_store_generated_proof + + - name: Generated native DynamoDB KV (loopback only) + env: + EDGEZERO_GENERATED_TEST_TARGET_DIR: ${{ github.workspace }}/target + run: scripts/run_test_nonzero.sh --ignored generated_cli_default_off_and_enabled_dynamodb_crud_list_ttl_restart cargo test --offline --locked -p edgezero-cli --test dynamodb_generated_proof + + - name: Generated native celld startup and KV (deterministic loopback fixture) + env: + EDGEZERO_GENERATED_TEST_TARGET_DIR: ${{ github.workspace }}/target + run: scripts/run_test_nonzero.sh --ignored generated_celld_default_off_kv_and_config_restart cargo test --offline --locked -p edgezero-cli --no-default-features --features cli,edgezero-adapter-axum --test celld_generated_proof generated_celld_default_off_kv_and_config_restart + + - name: Native provider integration runner tests + run: node --test scripts/native_aws_test_admin.test.mjs scripts/run_native_aws_integration.test.mjs scripts/run_native_celld_integration.test.mjs + + - name: Generated native AWS independent service contracts + env: + EDGEZERO_GENERATED_TEST_TARGET_DIR: ${{ github.workspace }}/target + run: node scripts/run_native_aws_integration.mjs + - name: Verify a generated project compiles run: scripts/run_test_nonzero.sh --ignored generated_workspace_compiles cargo test --offline --locked -p edgezero-cli --test generated_project_builds @@ -177,6 +262,81 @@ jobs: cargo check --locked -p app-demo-adapter-fastly --target wasm32-wasip1 --no-default-features --features fastly cargo check --locked -p app-demo-adapter-spin --target wasm32-wasip2 --no-default-features --features spin + geo-features: + name: Geo isolated features [${{ matrix.features }}] + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - features: axum + cache_key: axum + builtin_sentinel: geo_unavailable_selection_fails_before_runner_resources + - features: geo-cloudfront + cache_key: cloudfront + builtin_sentinel: geo_cloudfront_parsing_matches_spec + - features: geo-maxmind + cache_key: maxmind + builtin_sentinel: geo_maxmind_structural_pass_closes_upstream_data_gaps + - features: geo-cloudfront,geo-maxmind + cache_key: cloudfront-maxmind + builtin_sentinel: geo_maxmind_structural_pass_closes_upstream_data_gaps + steps: + - uses: actions/checkout@v6 + + - name: Cache Cargo dependencies + uses: actions/cache@v5 + with: + path: | + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + key: ${{ runner.os }}-cargo-geo-${{ matrix.cache_key }}-${{ hashFiles('**/Cargo.lock') }} + + - name: Retrieve Rust version + id: rust-version + run: echo "rust-version=$(grep '^rust ' .tool-versions | awk '{print $2}')" >> "$GITHUB_OUTPUT" + shell: bash + + - name: Set up Rust tool chain + uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + toolchain: ${{ steps.rust-version.outputs.rust-version }} + + - name: Fetch dependencies (locked) + run: cargo fetch --locked + + - name: Isolated Geo unit contracts + env: + FEATURES: ${{ matrix.features }} + SENTINEL: ${{ matrix.builtin_sentinel }} + run: | + scripts/run_test_nonzero.sh "$SENTINEL" cargo test --offline --locked -p edgezero-adapter-axum --no-default-features --features "$FEATURES" --lib geo_ + scripts/run_test_nonzero.sh geo_reader_admission_uses_private_checked_provenance cargo test --offline --locked -p edgezero-adapter-axum --no-default-features --features "$FEATURES" --lib geo_reader_admission_uses_private_checked_provenance + + - name: Supplied reader hosting without required built-ins + env: + FEATURES: ${{ matrix.features }} + run: scripts/run_test_nonzero.sh geo_hosting_supplied_reader_without_builtin cargo test --offline --locked -p edgezero-adapter-axum --no-default-features --features "$FEATURES,test-utils" --test production_hosting geo_hosting_supplied_reader_without_builtin + + - name: Disabled MaxMind fails before hosting + if: "!contains(matrix.features, 'geo-maxmind')" + env: + FEATURES: ${{ matrix.features }} + run: scripts/run_test_nonzero.sh geo_hosting_rejects_unavailable_maxmind cargo test --offline --locked -p edgezero-adapter-axum --no-default-features --features "$FEATURES,test-utils" --test production_hosting geo_hosting_rejects_unavailable_maxmind + + - name: MaxMind snapshot and restart hosting + if: contains(matrix.features, 'geo-maxmind') + env: + FEATURES: ${{ matrix.features }} + run: scripts/run_test_nonzero.sh geo_hosting_maxmind_snapshot_and_restart cargo test --offline --locked -p edgezero-adapter-axum --no-default-features --features "$FEATURES,test-utils" --test production_hosting geo_hosting_ + + - name: Public examples with isolated features + env: + FEATURES: ${{ matrix.features }} + run: cargo test --offline --locked -p edgezero-adapter-axum --no-default-features --features "$FEATURES" --doc + adapter-wasm-tests: name: ${{ matrix.adapter }} wasm tests runs-on: ubuntu-latest diff --git a/CHANGELOG.md b/CHANGELOG.md index 0473fa462..42209bc04 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,32 @@ ## Unreleased +### Axum local GeoIP and breaking Geo input migration + +Draft implementation, not accepted for release. Full metadata-corruption rejection remains blocked +by the pinned reader API, with an explicit ignored regression. File acquisition is Unix-only; +non-Unix selection reports implementation unavailability. Production-size measurements remain open. + +- Add default-off `geo-maxmind` for an operator-supplied GeoLite2 City or Country MMDB. + Prepare and validate a bounded, read-only in-memory snapshot before initialization and binding. + Updates require restart; active processes retain their snapshot after source replacement/deletion. + Preparation precedes runtime signal/stop polling and has no filesystem I/O deadline. +- Add explicit `PeerAllowed`/`ForwardedRequired` checked-reader selection and + `SuppliedReaderIp` composition, which works with `axum` alone. No geographic-header fallback, + downloader, remote lookup or automatic permission decision is added. +- Breaking pre-release migration: replace `GeoInput { country, subdivision }` with + `GeoInput::Assertions { country, subdivision }`, match fields by variant, and explicitly handle + or reject `ReaderIp { address, source }`. Update exhaustive matches for resolution `NoMatch`, + unavailable `MissingReaderIp`/`NoMatch`, source `LocalDatabase`/`CheckedReaderIp` and trust basis + `CheckedReaderIp(GeoIpSource)`. Unsupported inputs return `ProviderFailure`. +- Replace capability's `Selected { provider, source, trust_basis }` with + `Selected { provider, input: NativeGeoInputContract }`. Capability validates configuration + without database I/O and does not describe request provenance. Reserve custom provider ID + `maxmind` alongside `cloudfront`. No compatibility aliases or unrelated version bumps. +- Document mounts, proxy trust, redaction, uncertainty, dataset attribution and operator-owned + update/deletion duties, including running snapshots and rollback copies, in the + [native Geo guide](docs/guide/native-geo.md). + ### Portable HTTP And Request Lifecycles (PR #275) The changes below cover [PR #275](https://github.com/stackpop/edgezero/pull/275), including its diff --git a/Cargo.lock b/Cargo.lock index 1ea350eb7..8406c0cd4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -121,6 +121,15 @@ dependencies = [ "validator", ] +[[package]] +name = "arc-swap" +version = "1.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c049c0be4daef0b145cb3555416b3b8ef5b7888a38aea1a3a155801fe7b0810b" +dependencies = [ + "rustversion", +] + [[package]] name = "async-compression" version = "0.4.43" @@ -178,6 +187,49 @@ version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" +[[package]] +name = "aws-config" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8d7b388a9fc3a6db15a5ec778c38b354eff1364882c94d08e0252f7a47dcaa4" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-sdk-sso", + "aws-sdk-ssooidc", + "aws-sdk-sts", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "hex", + "http 1.4.2", + "sha1", + "time", + "tokio", + "tracing", + "url", + "zeroize", +] + +[[package]] +name = "aws-credential-types" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e93964ffdaf57857f544be3666a5f57570bb699e934700f11b49708f61bb556e" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "zeroize", +] + [[package]] name = "aws-lc-rs" version = "1.17.0" @@ -200,6 +252,383 @@ dependencies = [ "fs_extra", ] +[[package]] +name = "aws-runtime" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b8a9911551b4ea6ca13805ef52ed96f7d2bbb43cc3b4a14cb0776a71f33cfaa" +dependencies = [ + "aws-credential-types", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "bytes-utils", + "fastrand", + "http 1.4.2", + "http-body", + "percent-encoding", + "pin-project-lite", + "tracing", + "uuid", +] + +[[package]] +name = "aws-sdk-dynamodb" +version = "1.120.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f578638cafe182113da770e9798e7063d6de96cbbfe06f647d0026f197c9b97c" +dependencies = [ + "arc-swap", + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-observability", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 0.2.12", + "http 1.4.2", + "regex-lite", + "tracing", + "url", +] + +[[package]] +name = "aws-sdk-secretsmanager" +version = "1.120.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "832dc9d5dbd19c0ac6b24e21ef9f3e553695d0f3928de7134567407b3b7a447d" +dependencies = [ + "arc-swap", + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-observability", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 1.4.2", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-sso" +version = "1.114.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12be2f9c8eef7f5fc919c96d538e629698469a02b4cb75408b26b1bd984ebe79" +dependencies = [ + "arc-swap", + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-observability", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 1.4.2", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-ssooidc" +version = "1.116.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7645db8724ea3b82fdccfb67e1b0f637c9d8ab0e7ef29d84884d8d8d73f805d" +dependencies = [ + "arc-swap", + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-observability", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 1.4.2", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-sts" +version = "1.119.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03e490aa904849b38e770922faac779dd245f5cf8be81f19065acf18d276e4ad" +dependencies = [ + "arc-swap", + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-observability", + "aws-smithy-query", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "fastrand", + "http 1.4.2", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sigv4" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2312577f088c9fbf4206dfdb884cf1de9407b43e1a923cbed5237775116fc24b" +dependencies = [ + "aws-credential-types", + "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "form_urlencoded", + "hex", + "hmac", + "http 1.4.2", + "percent-encoding", + "sha2 0.11.0", + "time", + "tracing", +] + +[[package]] +name = "aws-smithy-async" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f02e407fb3b54891734224b9ffac8a71fdd35f542500fa1af95754a6b2beb316" +dependencies = [ + "futures-util", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "aws-smithy-http" +version = "0.64.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "639b4d8f8555f24a9be649811c3eb0b4d4616f4d61daf0c32e28873bc1ea9af1" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "bytes-utils", + "futures-core", + "futures-util", + "http 1.4.2", + "http-body", + "http-body-util", + "percent-encoding", + "pin-project-lite", + "pin-utils", + "tracing", +] + +[[package]] +name = "aws-smithy-http-client" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51c89cc3f1f281d659a67a519a1b5c6d445b5ce09fa7e5aee40c2c2707e9509d" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "h2", + "http 1.4.2", + "hyper", + "hyper-rustls", + "hyper-util", + "pin-project-lite", + "rustls", + "rustls-native-certs", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tower", + "tracing", +] + +[[package]] +name = "aws-smithy-json" +version = "0.63.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3385d469edbe8b60cc72002784652b5efca39178192aa9cc4b44c9875c6bdc18" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", +] + +[[package]] +name = "aws-smithy-observability" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e86338c869539a581bf161247762a6e87f92c5c075060057b5ed6d06632ed0c" +dependencies = [ + "aws-smithy-runtime-api", +] + +[[package]] +name = "aws-smithy-query" +version = "0.62.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1d1d71f6562be974caa85442ecd90194c40fdb5df045f182a6c2e872ce95056" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "aws-smithy-xml", + "urlencoding", +] + +[[package]] +name = "aws-smithy-runtime" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6e302ac1d88b99652489df31abdec6ac42a2ab2ac3982ad0ac49f64dfaf28ba" +dependencies = [ + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-http-client", + "aws-smithy-observability", + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "bytes", + "fastrand", + "http 1.4.2", + "http-body", + "http-body-util", + "pin-project-lite", + "pin-utils", + "tokio", + "tracing", +] + +[[package]] +name = "aws-smithy-runtime-api" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0730c16f91124c6a2abb4932c77e299288b3dd9f967ea2e9ec48cc6731e87a4" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api-macros", + "aws-smithy-types", + "bytes", + "http 1.4.2", + "pin-project-lite", + "tokio", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-smithy-runtime-api-macros" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "221eaa237ddf1ca79b60d1372aad77e47f9c0ea5b3ce5099da8c61d027dc77b3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "aws-smithy-schema" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8f395d93304280b64b7632fea798d177e74897fe7f063416ce627cd6fa24829" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-types", + "http 1.4.2", +] + +[[package]] +name = "aws-smithy-types" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69bb407740a197147da48238ecc94498493c9e85445732360cec180296ca45f1" +dependencies = [ + "base64-simd", + "bytes", + "bytes-utils", + "futures-core", + "http 1.4.2", + "http-body", + "http-body-util", + "itoa", + "num-integer", + "pin-project-lite", + "pin-utils", + "ryu", + "serde", + "time", + "tokio", + "tokio-util", +] + +[[package]] +name = "aws-smithy-xml" +version = "0.62.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b932c8d6dc127fc980eecd78f8694ae9b9551b69a93a7def2a199c1c0033daf" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "xmlparser", +] + +[[package]] +name = "aws-types" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "209f3a6d82a6e9e5f94abbed94c7a26e1c052341002bf57a5fb5481f625896fc" +dependencies = [ + "aws-credential-types", + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "rustc_version", + "tracing", +] + [[package]] name = "axum" version = "0.8.9" @@ -211,7 +640,7 @@ dependencies = [ "bytes", "form_urlencoded", "futures-util", - "http", + "http 1.4.2", "http-body", "http-body-util", "hyper", @@ -242,7 +671,7 @@ checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" dependencies = [ "bytes", "futures-core", - "http", + "http 1.4.2", "http-body", "http-body-util", "mime", @@ -270,6 +699,16 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64-simd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "339abbe78e73178762e23bea9dfd08e697eb3f3301cd4be981c0f78ba5859195" +dependencies = [ + "outref", + "vsimd", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -300,6 +739,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + [[package]] name = "brotli" version = "8.0.4" @@ -333,6 +781,16 @@ version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" +[[package]] +name = "bytes-utils" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dafe3a8757b027e2be6e4e5601ed563c55989fcf1546e933c66c8eb3a058d35" +dependencies = [ + "bytes", + "either", +] + [[package]] name = "cast" version = "0.3.0" @@ -436,6 +894,12 @@ dependencies = [ "cc", ] +[[package]] +name = "cmov" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" + [[package]] name = "colorchoice" version = "1.0.5" @@ -479,6 +943,12 @@ version = "0.4.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cc14f565cf027a105f7a44ccf9e5b424348421a1d8952a8fc9d499d313107789" +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + [[package]] name = "core-foundation" version = "0.10.1" @@ -532,6 +1002,15 @@ dependencies = [ "typenum", ] +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + [[package]] name = "ctor" version = "1.0.13" @@ -542,6 +1021,15 @@ dependencies = [ "linktime-proc-macro", ] +[[package]] +name = "ctutils" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03bb0e1cc970d482d121d9a1744999169b69a07470b3d644a7894e53fcaf4574" +dependencies = [ + "cmov", +] + [[package]] name = "darling" version = "0.20.11" @@ -634,7 +1122,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer 0.10.4", - "crypto-common", + "crypto-common 0.1.6", +] + +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "const-oid", + "crypto-common 0.2.2", + "ctutils", ] [[package]] @@ -681,26 +1181,35 @@ dependencies = [ "ctor", "edgezero-adapter", "edgezero-core", + "edgezero-store-aws", + "edgezero-store-celld", "flate2", "futures", "futures-util", - "http", + "getrandom 0.4.2", + "http 1.4.2", "http-body", "hyper", "hyper-util", + "ipnet", "libc", "log", + "maxminddb", "redb", "reqwest", + "rustls", "serde", "serde_json", + "sha2 0.10.9", "simple_logger", "tempfile", "thiserror 2.0.20", "tokio", + "tokio-rustls", "toml", "tower", "tracing", + "validator", "walkdir", ] @@ -725,7 +1234,7 @@ dependencies = [ "toml_edit", "walkdir", "wasm-bindgen-test", - "wasm-streams", + "wasm-streams 0.6.0", "web-sys", "worker", ] @@ -808,12 +1317,14 @@ dependencies = [ "proc-macro2", "serde", "serde_json", + "sha2 0.10.9", "similar", "simple_logger", "syn 3.0.3", "tempfile", "thiserror 2.0.20", "toml", + "url", "validator", "walkdir", ] @@ -839,7 +1350,7 @@ dependencies = [ "flate2", "futures", "futures-util", - "http", + "http 1.4.2", "http-body", "log", "matchit 0.9.2", @@ -880,6 +1391,49 @@ dependencies = [ "validator", ] +[[package]] +name = "edgezero-store-aws" +version = "0.1.0" +dependencies = [ + "async-trait", + "aws-config", + "aws-sdk-dynamodb", + "aws-sdk-secretsmanager", + "base64", + "bytes", + "edgezero-core", + "futures", + "futures-util", + "percent-encoding", + "reqwest", + "serde", + "serde_json", + "tempfile", + "thiserror 2.0.20", + "tokio", + "toml", + "url", +] + +[[package]] +name = "edgezero-store-celld" +version = "0.1.0" +dependencies = [ + "async-trait", + "base64", + "bytes", + "edgezero-core", + "futures-util", + "reqwest", + "serde", + "serde_json", + "sha2 0.10.9", + "thiserror 2.0.20", + "tokio", + "tokio-rustls", + "url", +] + [[package]] name = "either" version = "1.16.0" @@ -937,7 +1491,7 @@ dependencies = [ "fastly-macros", "fastly-shared", "fastly-sys", - "http", + "http 1.4.2", "itertools", "mime", "serde", @@ -969,7 +1523,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "191c2cf082f5d17f4ca0685cedaa0642ad4ade949fac8d1dbdd62ff756b13500" dependencies = [ "bitflags 1.3.2", - "http", + "http 1.4.2", ] [[package]] @@ -980,7 +1534,7 @@ checksum = "239c77706cf9110b423a89b6b03b8011729b144390d2a7f641d95ef66b2768b8" dependencies = [ "bitflags 1.3.2", "fastly-shared", - "http", + "http 1.4.2", "wasip2", "wit-bindgen 0.51.0", ] @@ -1194,6 +1748,25 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" +[[package]] +name = "h2" +version = "0.4.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d29020232d6aa3fb1daca64c1127cf662cf97f254ae16c18c05b8ab635fc118" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http 1.4.2", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "handlebars" version = "6.4.3" @@ -1252,6 +1825,32 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hmac" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" +dependencies = [ + "digest 0.11.3", +] + +[[package]] +name = "http" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + [[package]] name = "http" version = "1.4.2" @@ -1269,7 +1868,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", - "http", + "http 1.4.2", ] [[package]] @@ -1280,7 +1879,7 @@ checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" dependencies = [ "bytes", "futures-core", - "http", + "http 1.4.2", "http-body", "pin-project-lite", ] @@ -1297,6 +1896,15 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +[[package]] +name = "hybrid-array" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17" +dependencies = [ + "typenum", +] + [[package]] name = "hyper" version = "1.12.0" @@ -1305,8 +1913,10 @@ checksum = "2c3e324da4c95177d6291d4c8730197c0d1822f8a9766814a4a44fa5ab797c9c" dependencies = [ "atomic-waker", "bytes", + "futures-channel", "futures-core", - "http", + "h2", + "http 1.4.2", "http-body", "httparse", "httpdate", @@ -1323,10 +1933,11 @@ version = "0.27.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" dependencies = [ - "http", + "http 1.4.2", "hyper", "hyper-util", "rustls", + "rustls-native-certs", "tokio", "tokio-rustls", "tower-service", @@ -1342,7 +1953,7 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "http", + "http 1.4.2", "http-body", "hyper", "ipnet", @@ -1512,6 +2123,12 @@ version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" +[[package]] +name = "ipnetwork" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf370abdafd54d13e54a620e8c3e1145f28e46cc9d704bc6d94414559df41763" + [[package]] name = "is_terminal_polyfill" version = "1.70.2" @@ -1698,6 +2315,18 @@ version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8863b587001c1b9a8a4e36008cebc6b3612cb1226fe2de94858e06092687b608" +[[package]] +name = "maxminddb" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b43d2458d977b610b04a3858e6504f06133663d303a1a0606601b530bfc9bc6" +dependencies = [ + "ipnetwork", + "memchr", + "serde", + "thiserror 2.0.20", +] + [[package]] name = "memchr" version = "2.8.1" @@ -1756,6 +2385,15 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" +[[package]] +name = "num-integer" +version = "0.1.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" +dependencies = [ + "num-traits", +] + [[package]] name = "num-modular" version = "0.6.1" @@ -1820,6 +2458,12 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" +[[package]] +name = "outref" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" + [[package]] name = "percent-encoding" version = "2.3.2" @@ -1895,6 +2539,12 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" +[[package]] +name = "pin-utils" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13bee6c73da26345c729282832b60b0363cf3dd9f4bfd81d8551b7a1c889a113" + [[package]] name = "pkg-config" version = "0.3.33" @@ -2093,6 +2743,12 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "regex-lite" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973" + [[package]] name = "regex-syntax" version = "0.8.10" @@ -2110,7 +2766,7 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "http", + "http 1.4.2", "http-body", "http-body-util", "hyper", @@ -2129,12 +2785,14 @@ dependencies = [ "sync_wrapper", "tokio", "tokio-rustls", + "tokio-util", "tower", "tower-http", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", + "wasm-streams 0.5.0", "web-sys", ] @@ -2426,6 +3084,17 @@ dependencies = [ "serde", ] +[[package]] +name = "sha1" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + [[package]] name = "sha2" version = "0.9.9" @@ -2450,6 +3119,17 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "digest 0.11.3", +] + [[package]] name = "shlex" version = "2.0.1" @@ -2548,7 +3228,7 @@ dependencies = [ "anyhow", "bytes", "futures", - "http", + "http 1.4.2", "http-body", "http-body-util", "spin-macro", @@ -2811,6 +3491,20 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-util" +version = "0.7.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "libc", + "pin-project-lite", + "tokio", +] + [[package]] name = "toml" version = "1.1.2+spec-1.1.0" @@ -2888,7 +3582,7 @@ dependencies = [ "bitflags 2.11.1", "bytes", "futures-util", - "http", + "http 1.4.2", "http-body", "pin-project-lite", "tower", @@ -3004,6 +3698,12 @@ dependencies = [ "serde", ] +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + [[package]] name = "utf8_iter" version = "1.0.4" @@ -3016,6 +3716,16 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "uuid" +version = "1.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97277d36b9c3ace13e58fa6e753f8b0bbbf302a18dd193240d70f9e29681059a" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "validator" version = "0.20.0" @@ -3058,6 +3768,12 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "vsimd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" + [[package]] name = "walkdir" version = "2.5.0" @@ -3108,7 +3824,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75e9ed91ee5a995b6fe16c89d4511a8c12eb76967be20b57557618e02e781bcc" dependencies = [ "bytes", - "http", + "http 1.4.2", "http-body", "thiserror 2.0.20", "wit-bindgen 0.57.1", @@ -3252,6 +3968,19 @@ dependencies = [ "wasmparser 0.247.0", ] +[[package]] +name = "wasm-streams" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1ec4f6517c9e11ae630e200b2b65d193279042e28edd4a2cda233e46670bbb" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "wasm-streams" version = "0.6.0" @@ -3742,7 +4471,7 @@ dependencies = [ "chrono", "futures-channel", "futures-util", - "http", + "http 1.4.2", "http-body", "js-sys", "matchit 0.7.3", @@ -3756,7 +4485,7 @@ dependencies = [ "url", "wasm-bindgen", "wasm-bindgen-futures", - "wasm-streams", + "wasm-streams 0.6.0", "web-sys", "worker-macros", "worker-sys", @@ -3796,6 +4525,12 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" +[[package]] +name = "xmlparser" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" + [[package]] name = "yoke" version = "0.8.2" diff --git a/Cargo.toml b/Cargo.toml index 0837012a9..319a6c6a5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,6 +8,8 @@ members = [ "crates/edgezero-cli", "crates/edgezero-core", "crates/edgezero-macros", + "crates/edgezero-store-aws", + "crates/edgezero-store-celld", ] exclude = ["examples/app-demo"] resolver = "2" @@ -39,7 +41,11 @@ async-compression = { version = "=0.4.43", default-features = false, features = ] } async-stream = "0.3" async-trait = "0.1" +aws-config = { version = "=1.12.0", default-features = false, features = ["default-https-client", "rt-tokio", "credentials-process", "sso"] } +aws-sdk-dynamodb = { version = "=1.120.0", default-features = false, features = ["default-https-client", "rt-tokio"] } +aws-sdk-secretsmanager = { version = "=1.120.0", default-features = false, features = ["default-https-client", "rt-tokio"] } axum = { version = "0.8", default-features = true } +base64 = "=0.22.1" brotli = "=8.0.4" brotli-decompressor = "=5.0.1" bytes = "=1.12.1" @@ -54,6 +60,8 @@ edgezero-adapter-cloudflare = { path = "crates/edgezero-adapter-cloudflare", def edgezero-adapter-fastly = { path = "crates/edgezero-adapter-fastly", default-features = false } edgezero-adapter-spin = { path = "crates/edgezero-adapter-spin", default-features = false } edgezero-core = { path = "crates/edgezero-core", default-features = false } +edgezero-store-aws = { path = "crates/edgezero-store-aws", default-features = false } +edgezero-store-celld = { path = "crates/edgezero-store-celld", default-features = false } edgezero-cli = { path = "crates/edgezero-cli", default-features = false } fastly = "=0.13.1" fastly-shared = "=0.13.1" @@ -62,17 +70,20 @@ fern = "0.7" flate2 = { version = "=1.1.9", default-features = false, features = ["rust_backend"] } futures = { version = "0.3", features = ["std", "executor"] } futures-util = { version = "0.3", features = ["alloc", "io"] } +getrandom = "0.4.2" handlebars = "6" http = "=1.4.2" http-body = "1" http-body-util = "0.1" hyper = { version = "=1.12.0", features = ["http1", "server"] } hyper-util = { version = "0.1", features = ["tokio"] } +ipnet = "2.12.0" log = "0.4" libc = "0.2" miniz_oxide = { version = "=0.8.9", default-features = false, features = ["simd", "with-alloc"] } log-fastly = "=0.13.1" matchit = "0.9" +maxminddb = { version = "=0.32.0", default-features = false } once_cell = "1" redb = "4.1.0" reqwest = { version = "=0.13.4", default-features = false, features = ["rustls", "blocking", "json"] } @@ -83,6 +94,7 @@ simd-adler32 = { version = "=0.3.9", default-features = false } # file (`/.spin/sqlite_key_value.db`) using the schema # vendored from spinframework/spin's `crates/key-value-spin/src/store.rs`. rusqlite = { version = "0.32", default-features = false, features = ["bundled"] } +rustls = { version = "=0.23.40", default-features = false, features = ["std", "tls12", "aws_lc_rs"] } ryu = "1" serde = { version = "1", features = ["derive"] } serde_json = "=1.0.150" @@ -91,6 +103,7 @@ serde_urlencoded = "0.7" sha2 = "0.10" similar = "2" simple_logger = "5" +percent-encoding = "=2.3.2" proc-macro2 = { version = "1", features = ["span-locations"] } quote = "1" syn = { version = "3", features = ["full", "extra-traits", "visit"] } @@ -102,6 +115,7 @@ tempfile = "3" toml_edit = "0.25" thiserror = "2" tokio = { version = "1", features = ["io-util", "macros", "net", "rt-multi-thread", "signal", "sync", "time"] } +tokio-rustls = { version = "=0.26.4", default-features = false, features = ["aws_lc_rs", "tls12"] } trybuild = "1" toml = { version = "1.1" } tower = { version = "0.5", features = ["util"] } diff --git a/crates/edgezero-adapter-axum/Cargo.toml b/crates/edgezero-adapter-axum/Cargo.toml index ea2f57bdd..24bf2ff5d 100644 --- a/crates/edgezero-adapter-axum/Cargo.toml +++ b/crates/edgezero-adapter-axum/Cargo.toml @@ -13,25 +13,37 @@ workspace = true [features] default = ["axum"] axum = [ + "native-bindings", "dep:async-stream", "dep:axum", "dep:tokio", "dep:tower", "dep:futures-util", + "dep:getrandom", + "dep:ipnet", "dep:http-body", "dep:hyper", "dep:hyper-util", "dep:reqwest", "dep:redb", ] +native-bindings = ["dep:toml", "dep:edgezero-store-aws", "dep:edgezero-store-celld"] +aws-appconfig-agent = ["axum", "edgezero-store-aws/appconfig-agent"] +aws-secrets-manager = ["axum", "edgezero-store-aws/secrets-manager"] +aws-stores = ["aws-appconfig-agent", "aws-secrets-manager"] +aws-dynamodb = ["axum", "edgezero-store-aws/dynamodb-kv"] +celld-config = ["axum", "edgezero-store-celld/config"] +celld-kv = ["axum", "edgezero-store-celld/kv"] +geo-cloudfront = ["axum"] +geo-maxmind = ["axum", "dep:maxminddb", "dep:sha2"] cli = [ + "native-bindings", "dep:edgezero-adapter", "edgezero-adapter/cli", "dep:ctor", - "dep:toml", "dep:walkdir", ] -test-utils = [] +test-utils = ["dep:validator"] [dependencies] edgezero-adapter = { path = "../edgezero-adapter", optional = true, features = [ @@ -46,6 +58,8 @@ bytes = { workspace = true } ctor = { workspace = true, optional = true } futures = { workspace = true } futures-util = { workspace = true, optional = true } +getrandom = { workspace = true, optional = true } +ipnet = { workspace = true, optional = true } http = { workspace = true } http-body = { workspace = true, optional = true } hyper = { workspace = true, optional = true } @@ -55,6 +69,7 @@ redb = { workspace = true, optional = true } reqwest = { workspace = true, optional = true } serde = { workspace = true } serde_json = { workspace = true } +validator = { workspace = true, optional = true } simple_logger = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true, optional = true } @@ -63,9 +78,43 @@ tower = { workspace = true, optional = true } tracing = { workspace = true } walkdir = { workspace = true, optional = true } +[target.'cfg(not(target_arch = "wasm32"))'.dependencies] +maxminddb = { workspace = true, optional = true } +sha2 = { workspace = true, optional = true } +edgezero-store-aws = { workspace = true, optional = true } +edgezero-store-celld = { workspace = true, optional = true } + [target.'cfg(unix)'.dependencies] libc = { workspace = true } +[[bin]] +name = "edgezero-hosting-fixture" +path = "tests/fixtures/production_host.rs" +required-features = ["axum", "test-utils"] +test = false + +[[bin]] +name = "edgezero-native-aws-host" +path = "tests/fixtures/native_aws_host.rs" +required-features = ["aws-stores", "test-utils"] +test = false + +[[test]] +name = "native_aws_startup" +path = "tests/native_aws_startup.rs" +required-features = ["aws-stores", "test-utils"] + +[[bin]] +name = "edgezero-native-celld-host" +path = "tests/fixtures/native_celld_host.rs" +required-features = ["celld-config", "celld-kv", "test-utils"] +test = false + +[[test]] +name = "native_celld_startup" +path = "tests/native_celld_startup.rs" +required-features = ["celld-config", "celld-kv", "test-utils"] + [dev-dependencies] async-trait = { workspace = true } axum = { workspace = true, features = ["macros"] } @@ -74,6 +123,8 @@ flate2 = { workspace = true } http-body = { workspace = true } hyper = { workspace = true } hyper-util = { workspace = true } +rustls = { workspace = true } serde = { workspace = true } tempfile = { workspace = true } tokio = { workspace = true, features = ["io-util", "macros", "net", "rt", "rt-multi-thread", "test-util"] } +tokio-rustls = { workspace = true } diff --git a/crates/edgezero-adapter-axum/src/cli.rs b/crates/edgezero-adapter-axum/src/cli.rs index 111e06d67..d2d7a3ac2 100644 --- a/crates/edgezero-adapter-axum/src/cli.rs +++ b/crates/edgezero-adapter-axum/src/cli.rs @@ -24,6 +24,8 @@ use edgezero_core::{Capability, CapabilitySupport, PlatformMetadata}; use toml::Value; use walkdir::WalkDir; +use crate::native_build::NativeCargoSelection; + static AXUM_TEMPLATE_REGISTRATIONS: &[TemplateRegistration] = &[ TemplateRegistration { name: "axum_Cargo_toml", @@ -116,6 +118,7 @@ struct AxumProject { axum_host: Option, axum_manifest: PathBuf, axum_port: Option, + build_features: Vec, cargo_manifest: PathBuf, crate_dir: PathBuf, crate_name: String, @@ -131,7 +134,7 @@ struct EdgezeroAxumConfig { #[expect( clippy::missing_trait_methods, - reason = "axum has no validate_app_config_keys / validate_adapter_manifest / validate_typed_secrets requirements; those three trait defaults are intentionally inherited. `read_config_entry` delegates to `read_config_entry_local` (axum is local-only). `single_store_kinds` IS overridden below (returns `&[\"secrets\"]`)." + reason = "axum has no validate_app_config_keys / validate_adapter_manifest / validate_typed_secrets requirements; those three trait defaults are intentionally inherited. `read_config_entry` delegates to `read_config_entry_local` (axum is local-only). `single_store_kinds` explicitly declares Multi support for every store kind." )] impl Adapter for AxumCliAdapter { fn capability(&self, capability: Capability) -> CapabilitySupport { @@ -409,9 +412,9 @@ impl Adapter for AxumCliAdapter { } fn single_store_kinds(&self) -> &'static [&'static str] { - //: axum is Multi for KV (local file dirs) and Config - // (local JSON files), Single for Secrets (env vars). - &["secrets"] + // Every kind supports multiple logical IDs. Local secret bindings may + // share the environment backend; that does not restrict their ID count. + &[] } } @@ -466,7 +469,15 @@ fn deploy(_extra_args: &[String]) -> Result<(), String> { fn locate_project() -> Result { let cwd = env::current_dir().map_err(|err| err.to_string())?; let manifest = find_axum_manifest(&cwd)?; - read_axum_project(&manifest) + let mut project = read_axum_project(&manifest)?; + if let Some(edgezero_manifest) = find_manifest_upwards(&cwd, "edgezero.toml") { + let loader = ManifestLoader::from_path(&edgezero_manifest) + .map_err(|err| format!("failed to load {}: {err}", edgezero_manifest.display()))?; + if let Some((_name, adapter)) = loader.manifest().adapter_entry("axum") { + project.build_features.clone_from(&adapter.build.features); + } + } + Ok(project) } fn cargo_command( @@ -495,7 +506,17 @@ fn cargo_command( .to_str() .ok_or_else(|| format!("invalid manifest path {}", project.cargo_manifest.display()))?, ); - command.args(extra_args); + let selection = NativeCargoSelection::normalize_args(&project.build_features, extra_args)?; + command.args(selection.cargo_args()); + match subcommand { + "run" if !selection.run_args.is_empty() => { + command.arg("--").args(&selection.run_args); + } + "build" if !selection.run_args.is_empty() => { + return Err("`cargo build` does not accept runtime arguments after `--`".to_owned()); + } + _ => {} + } command.current_dir(&project.crate_dir); // Canonical env vars. The runtime's `EnvConfig` reads only the // `EDGEZERO__*` form (see `crates/edgezero-core/src/env_config.rs`); @@ -776,6 +797,9 @@ fn read_axum_project_with_env( axum_host: config_host, axum_manifest: manifest.to_path_buf(), axum_port: config_port, + // Pinned targets use only the feature inputs selected by the caller; + // the CLI dispatcher has already forwarded manifest build features. + build_features: Vec::new(), cargo_manifest, crate_dir, crate_name, @@ -791,6 +815,11 @@ mod tests { use std::net::Ipv6Addr; use tempfile::tempdir; + #[test] + fn all_store_kinds_accept_multiple_logical_ids() { + assert!(AXUM_ADAPTER.single_store_kinds().is_empty()); + } + #[test] fn cargo_commands_preserve_ingress_defaults_and_bind_precedence() { let dir = tempdir().unwrap(); @@ -799,6 +828,7 @@ mod tests { axum_host: None, axum_manifest: dir.path().join("axum.toml"), axum_port: None, + build_features: Vec::new(), cargo_manifest: dir.path().join("Cargo.toml"), crate_dir: dir.path().to_path_buf(), crate_name: "fixture".to_owned(), diff --git a/crates/edgezero-adapter-axum/src/config_store.rs b/crates/edgezero-adapter-axum/src/config_store.rs index 41e7e9d08..c30f8d72d 100644 --- a/crates/edgezero-adapter-axum/src/config_store.rs +++ b/crates/edgezero-adapter-axum/src/config_store.rs @@ -114,6 +114,23 @@ impl AxumConfigStore { path: &Path, limits: ConfigStoreLimits, existing: usize, + ) -> Result { + Self::load_snapshot(path, limits, existing, false) + } + + pub(crate) fn load_required_at_startup( + path: &Path, + limits: ConfigStoreLimits, + existing: usize, + ) -> Result { + Self::load_snapshot(path, limits, existing, true) + } + + fn load_snapshot( + path: &Path, + limits: ConfigStoreLimits, + existing: usize, + required: bool, ) -> Result { let mut options = fs::OpenOptions::new(); options.read(true); @@ -137,7 +154,12 @@ impl AxumConfigStore { } Snapshot::load(&mut file, limits, existing)? } - Err(err) if err.kind() == ErrorKind::NotFound => Snapshot::empty(limits, existing)?, + Err(err) if err.kind() == ErrorKind::NotFound && !required => { + Snapshot::empty(limits, existing)? + } + Err(err) if err.kind() == ErrorKind::NotFound => { + return Err(ConfigStoreError::unavailable("config file missing")); + } Err(_io) => { return Err(ConfigStoreError::unavailable("config snapshot open failed")); } diff --git a/crates/edgezero-adapter-axum/src/connection.rs b/crates/edgezero-adapter-axum/src/connection.rs index e99c2a7f2..16e77ff68 100644 --- a/crates/edgezero-adapter-axum/src/connection.rs +++ b/crates/edgezero-adapter-axum/src/connection.rs @@ -2,6 +2,7 @@ use std::error::Error as StdError; use std::future::{self, Future}; use std::io; +use edgezero_core::probe::LifecyclePhase; use http_body::Body as HttpBody; use hyper::Request; use hyper::body::Incoming; @@ -9,6 +10,7 @@ use hyper::server::conn::http1::Builder as Http1Builder; use hyper::service::Service; use hyper_util::rt::TokioIo; use tokio::net::TcpStream; +use tokio::sync::watch; use tokio::time::{Instant as TokioInstant, sleep_until}; use crate::header_deadline::{HeaderDeadline, HeaderDeadlineExceeded, HeaderIo, HeaderService}; @@ -24,15 +26,33 @@ pub(crate) enum ConnectionExit { DeadlineExceeded, } -/// Owns HTTP/1 parsing, idle/header expiry and the existing response deadline race. +#[cfg(test)] +pub(crate) fn serve_http1( + stream: TcpStream, + service: ServiceType, + responses: EgressConnection, + limits: AxumIngressConfig, +) -> impl Future> +where + ServiceType: Service, Response = hyper::Response>, + ServiceType::Error: Into>, + ServiceType::Future: 'static, + ResponseBody: HttpBody + 'static, + ResponseBody::Error: Into>, +{ + serve_http1_with_shutdown(stream, service, responses, None, limits) +} + +/// Owns bounded HTTP/1 parsing, response deadlines and graceful lifecycle drain. #[expect( clippy::integer_division_remainder_used, reason = "tokio::select! expands to internal randomized branch selection arithmetic" )] -pub(crate) fn serve_http1( +pub(crate) fn serve_http1_with_shutdown( stream: TcpStream, service: ServiceType, responses: EgressConnection, + mut phase: Option>, limits: AxumIngressConfig, ) -> impl Future> where @@ -59,6 +79,22 @@ where let mut connection = Box::pin(connection_future); loop { + if phase.as_ref().is_some_and(|reader| { + matches!( + *reader.borrow(), + LifecyclePhase::Draining | LifecyclePhase::Stopped + ) + }) { + connection.as_mut().graceful_shutdown(); + phase = None; + } + let draining = async { + if let Some(reader) = phase.as_mut() { + let _changed = reader.changed().await; + } else { + future::pending::<()>().await; + } + }; if head .deadline() .is_some_and(|deadline| TokioInstant::now() >= deadline) @@ -95,6 +131,7 @@ where } () = head.changed() => {} () = responses.changed() => {} + () = draining => {} } } } diff --git a/crates/edgezero-adapter-axum/src/dev_server.rs b/crates/edgezero-adapter-axum/src/dev_server.rs index 5c0031689..0c7c5f48a 100644 --- a/crates/edgezero-adapter-axum/src/dev_server.rs +++ b/crates/edgezero-adapter-axum/src/dev_server.rs @@ -1,45 +1,105 @@ +#![expect( + clippy::pub_use, + reason = "validated options retain the existing dev_server API location" +)] +#![expect( + clippy::arbitrary_source_item_ordering, + reason = "compatibility entrypoints, normalization, native ownership and startup policy remain grouped by responsibility" +)] + use std::any::Any; +use std::collections::BTreeMap; +use std::env::vars_os; use std::fs; -use std::future; +use std::future::{self, Future}; +use std::io::ErrorKind; #[cfg(test)] use std::iter; use std::net::{SocketAddr, TcpListener as StdTcpListener}; -use std::panic::AssertUnwindSafe; +use std::panic::{AssertUnwindSafe, catch_unwind}; use std::path::{Path, PathBuf}; +use std::pin::Pin; +use std::process; use std::sync::Arc; +use std::time::Duration; use anyhow::Context as _; -use futures_util::FutureExt as _; -use futures_util::StreamExt as _; -use futures_util::stream::FuturesUnordered; -use tokio::net::TcpListener as TokioTcpListener; -use tokio::runtime::Builder as RuntimeBuilder; -use tokio::signal; -use tokio::sync::oneshot::{Receiver as ShutdownReceiver, Sender as ShutdownSender, channel}; -use tokio::task::{LocalSet, spawn_blocking}; - use edgezero_core::addr; use edgezero_core::app::{App, Hooks, StoreMetadata, StoresMetadata}; use edgezero_core::config_store::{ConfigStoreError, ConfigStoreHandle, ConfigStoreOpenFailure}; use edgezero_core::env_config::EnvConfig; use edgezero_core::key_value_store::KvHandle; use edgezero_core::logging::{BOOT_LOG_LEVEL, BOOT_LOG_TARGET, resolve_logging_level}; +use edgezero_core::probe::LifecyclePhase; use edgezero_core::router::RouterService; use edgezero_core::secret_store::SecretHandle; use edgezero_core::store_registry::{ BoundSecretStore, ConfigRegistry, ConfigStoreBinding, KvRegistry, SecretRegistry, StoreRegistry, }; +use futures_util::{FutureExt as _, StreamExt as _, stream::FuturesUnordered}; +use log::{Level, LevelFilter}; use simple_logger::SimpleLogger; -use std::collections::BTreeMap; +use tokio::net::TcpListener as TokioTcpListener; +use tokio::runtime::Builder as RuntimeBuilder; +use tokio::signal; +#[cfg(test)] +use tokio::sync::oneshot::{Receiver as ShutdownReceiver, Sender as ShutdownSender, channel}; +use tokio::sync::watch; +#[cfg(test)] +use tokio::task::spawn_blocking; +use tokio::task::{LocalSet, yield_now}; +use tokio::time::{Instant as NativeInstant, sleep_until}; use crate::config_store::AxumConfigStore; use crate::config_store_limits::ConfigStoreLimits; -use crate::connection::{ConnectionExit, serve_http1}; +use crate::connection::{ConnectionExit, serve_http1_with_shutdown}; +use crate::diagnostics::{NativeDiagnosticsHandle, NativeSession}; +use crate::geo::NativeGeo; use crate::ingress_config::AxumIngressConfig; use crate::key_value_store::PersistentKvStore; +use crate::native_stores::{self, NativeSourcePlan, NativeStoreOverrides, NativeStoreSource}; +use crate::outbound::AxumOutboundClient; +use crate::proxy::TrustedProxyPolicy; use crate::response::EgressConnection; +pub use crate::run_options::AxumRunOptions; +use crate::run_options::{failure, native_settings, shutdown_grace, strip_native_config_prefix}; use crate::secret_store::EnvSecretStore; -use crate::service::AxumServiceState; +use crate::service::{ + AxumServiceState, legacy_config_registry, legacy_kv_registry, legacy_secret_registry, +}; + +/// Local application-owned startup work, borrowing the unshared App and prepared stores. +pub type LocalStartupFuture<'startup> = + Pin> + 'startup>>; + +/// The exact registries prepared for requests. Handles may be cloned by the application; +/// retained KV clones must be released by their owners before reopening or backing up. +#[derive(Default)] +pub struct PreparedStores { + config: Option, + kv: Option, + secrets: Option, +} + +impl PreparedStores { + #[must_use] + #[inline] + pub fn config(&self) -> Option<&ConfigRegistry> { + self.config.as_ref() + } + + #[must_use] + #[inline] + pub fn kv(&self) -> Option<&KvRegistry> { + self.kv.as_ref() + } + + #[must_use] + #[inline] + pub fn secrets(&self) -> Option<&SecretRegistry> { + self.secrets.as_ref() + } +} #[derive(Clone, Copy, Debug, Eq, PartialEq)] enum KvInitRequirement { @@ -47,8 +107,6 @@ enum KvInitRequirement { Required, } -type ConnectionResult = Result, Box>; - /// Configuration used when running the dev server embedding `EdgeZero` into Axum. #[derive(Clone)] pub struct AxumDevServerConfig { @@ -86,6 +144,7 @@ struct Stores { /// Blocking dev server runner used by the `EdgeZero` CLI. pub struct AxumDevServer { config: AxumDevServerConfig, + native: NativeHosting, router: RouterService, stores: Stores, } @@ -96,6 +155,7 @@ impl AxumDevServer { pub fn new(router: RouterService) -> Self { Self { config: AxumDevServerConfig::default(), + native: NativeHosting::default(), router, stores: Stores::default(), } @@ -105,39 +165,44 @@ impl AxumDevServer { /// Returns an error if the dev server fails to bind, the Tokio runtime fails to start, or the underlying request loop returns an error. #[inline] pub fn run(self) -> anyhow::Result<()> { - let runtime = RuntimeBuilder::new_multi_thread() - .enable_all() - .build() - .context("failed to build tokio runtime")?; - - runtime.block_on(async move { self.run_async().await }) - } - - async fn run_async(self) -> anyhow::Result<()> { - let AxumDevServer { + let Self { router, config, stores, + native, } = self; - - // Allow binding to already-open listener if caller created one to surface errors early. - let std_listener = StdTcpListener::bind(config.addr) - .with_context(|| format!("failed to bind dev server to {}", config.addr))?; - std_listener - .set_nonblocking(true) - .context("failed to set listener to non-blocking")?; - - let listener = TokioTcpListener::from_std(std_listener) - .context("failed to adopt std listener into tokio")?; - - serve_with_stores( - App::new(router), - listener, + run_local( + ListenerSource::Bind(config.addr), config.enable_ctrl_c, + Duration::from_secs(10), config.ingress, - stores, + future::pending::<()>(), + async move { prepare_transport(App::new(router), stores.into(), native) }, ) - .await + } + + /// Installs explicit checked forwarding policy without ambient settings. + #[must_use] + #[inline] + pub fn with_trusted_proxy_policy(mut self, policy: TrustedProxyPolicy) -> Self { + self.native.policy = policy; + self + } + + /// Controls default records, without disabling counters or observers. + #[must_use] + #[inline] + pub fn with_request_records(mut self, enabled: bool) -> Self { + self.native.request_records = enabled; + self + } + + /// Shares the managed native observer and counters. + #[must_use] + #[inline] + pub fn with_diagnostics(mut self, diagnostics: NativeDiagnosticsHandle) -> Self { + self.native.diagnostics = diagnostics; + self } #[cfg(test)] @@ -146,13 +211,15 @@ impl AxumDevServer { router, config, stores, + native, } = self; - serve_with_stores( + serve_with_native_stores( App::new(router), listener, config.enable_ctrl_c, config.ingress, stores, + native, ) .await } @@ -162,6 +229,7 @@ impl AxumDevServer { pub fn with_config(router: RouterService, config: AxumDevServerConfig) -> Self { Self { config, + native: NativeHosting::default(), router, stores: Stores::default(), } @@ -228,18 +296,16 @@ fn kv_init_requirement(stores: StoresMetadata) -> KvInitRequirement { } } -fn kv_store_path(store_name: &str) -> PathBuf { - // Every declared id gets its own slug-based filename. The - // pre-rewrite hard-coded `.edgezero/kv.redb` shortcut for - // store_name == "EDGEZERO_KV" is gone -- the runtime no longer - // hands out a default name; if you reach here you have a real - // declared id and the slug encoding handles every shape - // uniformly. - PathBuf::from(".edgezero").join(format!( +fn kv_store_file_name(store_name: &str) -> String { + format!( "kv-{}-{:016x}.redb", store_name_slug(store_name), stable_store_name_hash(store_name) - )) + ) +} + +fn kv_store_path(store_name: &str) -> PathBuf { + PathBuf::from(".edgezero").join(kv_store_file_name(store_name)) } fn store_name_slug(store_name: &str) -> String { @@ -299,12 +365,50 @@ fn kv_handle_from_path(kv_path: &Path) -> anyhow::Result { Ok(KvHandle::new(kv_store)) } +impl From for PreparedStores { + #[inline] + fn from(stores: Stores) -> Self { + Self { + config: stores + .config_registry + .or_else(|| stores.config_store.map(legacy_config_registry)), + kv: stores + .kv_registry + .or_else(|| stores.kv.map(legacy_kv_registry)), + secrets: stores + .secret_registry + .or_else(|| stores.secrets.map(legacy_secret_registry)), + } + } +} + +#[cfg(test)] async fn serve_with_stores( app: App, listener: TokioTcpListener, enable_ctrl_c: bool, ingress: AxumIngressConfig, stores: Stores, +) -> anyhow::Result<()> { + serve_with_native_stores( + app, + listener, + enable_ctrl_c, + ingress, + stores, + NativeHosting::default(), + ) + .await +} + +#[cfg(test)] +async fn serve_with_native_stores( + app: App, + listener: TokioTcpListener, + enable_ctrl_c: bool, + ingress: AxumIngressConfig, + stores: Stores, + native: NativeHosting, ) -> anyhow::Result<()> { struct ShutdownOnDrop(Option>); @@ -328,6 +432,7 @@ async fn serve_with_stores( enable_ctrl_c, ingress, stores, + native, shutdown_receiver, ) }); @@ -336,86 +441,355 @@ async fn serve_with_stores( result } +#[cfg(test)] fn serve_local( app: App, std_listener: StdTcpListener, enable_ctrl_c: bool, ingress: AxumIngressConfig, stores: Stores, + native: NativeHosting, shutdown_receiver: ShutdownReceiver<()>, ) -> anyhow::Result<()> { - let mut service = AxumServiceState::from_app(app); - if let Some(registry) = stores.config_registry { - service = service.with_config_registry(registry); - } - if let Some(handle) = stores.config_store { - service = service.with_config_store_handle(handle); - } - if let Some(registry) = stores.kv_registry { - service = service.with_kv_registry(registry); + run_local( + ListenerSource::Prebound(std_listener), + enable_ctrl_c, + Duration::ZERO, + ingress, + async move { + let _closed = shutdown_receiver.await; + }, + async move { prepare_transport(app, stores.into(), native) }, + ) +} + +#[derive(Clone)] +struct NativeHosting { + diagnostics: NativeDiagnosticsHandle, + geo: NativeGeo, + policy: TrustedProxyPolicy, + request_records: bool, +} + +impl Default for NativeHosting { + fn default() -> Self { + Self { + diagnostics: NativeDiagnosticsHandle::new(), + policy: TrustedProxyPolicy::no_trust(), + request_records: true, + geo: NativeGeo::default(), + } } - if let Some(handle) = stores.kv { - service = service.with_kv_handle(handle); +} + +type PreparedApp = (App, PreparedStores, Arc, NativeHosting); +type ConnectionResult = Result, Box>; +type Connections = FuturesUnordered>>>; + +enum ListenerSource { + Bind(SocketAddr), + #[cfg(test)] + Prebound(StdTcpListener), +} + +#[cfg_attr( + not(test), + expect( + clippy::needless_pass_by_value, + reason = "test embedding consumes a prebound listener through the same source" + ) +)] +fn adopt_listener(source: ListenerSource) -> anyhow::Result { + let listener = match source { + ListenerSource::Bind(addr) => StdTcpListener::bind(addr) + .map_err(|_error| failure("bind", "listener", "address unavailable"))?, + #[cfg(test)] + ListenerSource::Prebound(listener) => listener, + }; + listener + .set_nonblocking(true) + .map_err(|_error| failure("bind", "listener", "nonblocking setup failed"))?; + TokioTcpListener::from_std(listener) + .map_err(|_error| failure("bind", "listener", "runtime adoption failed")) +} + +fn prepare_transport( + app: App, + stores: PreparedStores, + native: NativeHosting, +) -> anyhow::Result { + let transport = AxumOutboundClient::try_transport() + .map_err(|_error| failure("transport", "HTTP client", "initialization failed"))?; + Ok((app, stores, transport, native)) +} + +struct Signals { + #[cfg(unix)] + interrupt: signal::unix::Signal, + #[cfg(unix)] + terminate: signal::unix::Signal, +} + +#[expect( + clippy::arbitrary_source_item_ordering, + reason = "registration precedes observation in this lifecycle owner" +)] +impl Signals { + fn register(enabled: bool) -> anyhow::Result> { + if !enabled { + return Ok(None); + } + #[cfg(unix)] + { + let interrupt = signal::unix::signal(signal::unix::SignalKind::interrupt()) + .map_err(|_error| failure("signals", "SIGINT", "registration failed"))?; + let terminate = signal::unix::signal(signal::unix::SignalKind::terminate()) + .map_err(|_error| failure("signals", "SIGTERM", "registration failed"))?; + Ok(Some(Self { + interrupt, + terminate, + })) + } + #[cfg(not(unix))] + { + Ok(Some(Self {})) + } } - if let Some(registry) = stores.secret_registry { - service = service.with_secret_registry(registry); + + #[expect( + clippy::integer_division_remainder_used, + reason = "tokio select branch arithmetic" + )] + async fn next(&mut self) -> anyhow::Result<()> { + #[cfg(unix)] + { + let received = tokio::select! { + received = self.interrupt.recv() => received, + received = self.terminate.recv() => received, + }; + received.ok_or_else(|| failure("signals", "termination", "observation closed")) + } + #[cfg(not(unix))] + { + signal::ctrl_c() + .await + .map_err(|_error| failure("signals", "SIGINT", "registration failed")) + } } - if let Some(handle) = stores.secrets { - service = service.with_secret_handle(handle); +} + +async fn next_signal(signals: &mut Option) -> anyhow::Result<()> { + match signals { + Some(source) => source.next().await, + None => future::pending().await, } +} +fn run_local( + listener: ListenerSource, + enable_signals: bool, + grace: Duration, + ingress: AxumIngressConfig, + stop: Stop, + prepare: Prepare, +) -> anyhow::Result<()> +where + Prepare: Future>, + Stop: Future, +{ let runtime = RuntimeBuilder::new_current_thread() .enable_all() .build() - .context("failed to build axum connection runtime")?; + .map_err(|_error| failure("runtime", "local runner", "creation failed"))?; let local = LocalSet::new(); - #[expect( - clippy::integer_division_remainder_used, - reason = "tokio::select! expands to internal randomized branch selection arithmetic" - )] - let serve = async move { - let listener = TokioTcpListener::from_std(std_listener) - .context("failed to adopt listener on axum connection runtime")?; - let shutdown = async move { - let _closed = shutdown_receiver.await; - }; - let ctrl_c = async move { - if enable_ctrl_c { - let _signal = signal::ctrl_c().await; + runtime.block_on(local.run_until(async move { + let mut signals = Signals::register(enable_signals)?; + let (phase, reader) = watch::channel(LifecyclePhase::Starting); + log_native_lifecycle(LifecyclePhase::Starting, "starting", grace, false); + let result = run_owned( + listener, + grace, + ingress, + stop, + prepare, + &mut signals, + &phase, + reader, + ) + .await; + phase.send_replace(LifecyclePhase::Stopped); + log_native_lifecycle( + LifecyclePhase::Stopped, + if result.is_ok() { + "stopped" } else { - future::pending::<()>().await; - } - }; - tokio::pin!(shutdown); - tokio::pin!(ctrl_c); - let mut connections = FuturesUnordered::new(); + "runner_failed" + }, + grace, + result.is_err(), + ); + result + })) +} - loop { - tokio::select! { - biased; - () = &mut shutdown => break, - () = &mut ctrl_c => break, - Some(result) = connections.next(), if !connections.is_empty() => { - report_connection_exit(&result); - } - accepted = listener.accept() => { - let (stream, remote_addr) = accepted.context("axum listener accept failed")?; - // One transient accepted socket; no parser, task or waiter on refusal. - if connections.len() >= ingress.max_connections() { - drop(stream); - continue; +#[expect( + clippy::integer_division_remainder_used, + reason = "tokio select branch arithmetic" +)] +#[expect( + clippy::too_many_arguments, + reason = "one lifecycle owner carries the validated ingress policy and stop sources" +)] +async fn run_owned( + listener_source: ListenerSource, + grace: Duration, + ingress: AxumIngressConfig, + stop: Stop, + prepare: Prepare, + signals: &mut Option, + phase: &watch::Sender, + reader: watch::Receiver, +) -> anyhow::Result<()> +where + Prepare: Future>, + Stop: Future, +{ + tokio::pin!(stop); + tokio::pin!(prepare); + let prepared = tokio::select! { + biased; + signal = next_signal(signals) => { + signal?; + phase.send_replace(LifecyclePhase::Draining); + return Ok(()); + } + () = &mut stop => { + phase.send_replace(LifecyclePhase::Draining); + return Ok(()); + } + prepared = &mut prepare => prepared.inspect_err(|_error| log_native_lifecycle(LifecyclePhase::Starting, "startup_failed", grace, true))?, + }; + let (app, stores, transport, native) = prepared; + let session = NativeSession::new(native.diagnostics, native.request_records)?; + let mut service = + AxumServiceState::with_native_transport(app, transport, session, native.policy, native.geo) + .with_phase(reader.clone()); + if let Some(registry) = stores.config { + service = service.with_config_registry(registry); + } + if let Some(registry) = stores.kv { + service = service.with_kv_registry(registry); + } + if let Some(registry) = stores.secrets { + service = service.with_secret_registry(registry); + } + // Give stop observation another poll before bind/adoption and readiness. + let listener = tokio::select! { + biased; + signal = next_signal(signals) => { signal?; phase.send_replace(LifecyclePhase::Draining); return Ok(()); } + () = &mut stop => { phase.send_replace(LifecyclePhase::Draining); return Ok(()); } + result = async { yield_now().await; adopt_listener(listener_source) } => result?, + }; + phase.send_replace(LifecyclePhase::Ready); + log::info!("[edgezero] native server ready"); + log_native_lifecycle(LifecyclePhase::Ready, "ready", grace, false); + let mut tasks = Connections::new(); + let mut outcome = Ok(()); + loop { + tokio::select! { + biased; + signal = next_signal(signals) => { outcome = signal; break; } + () = &mut stop => break, + Some(result) = tasks.next(), if !tasks.is_empty() => { + report_connection_exit(&result); + } + accepted = listener.accept() => { + match accepted { + Ok((stream, remote_addr)) => { + if tasks.len() >= ingress.max_connections() { + drop(stream); + continue; + } + let responses = EgressConnection::default(); + let (activity, connection_guard) = service.connection_activity(); + let connection_service = service.for_connection(remote_addr, responses.clone()).with_activity(activity); + let connection = serve_http1_with_shutdown(stream, connection_service, responses, Some(reader.clone()), ingress); + let owned_connection = async move { let _guard = connection_guard; connection.await }; + tasks.push(Box::pin(AssertUnwindSafe(owned_connection).catch_unwind())); + } + Err(_error) => { + outcome = Err(failure("runner", "listener", "accept failed")); + break; } - let responses = EgressConnection::default(); - let connection_service = service.for_connection(remote_addr, responses.clone()); - connections.push(AssertUnwindSafe(serve_http1(stream, connection_service, responses, ingress)).catch_unwind()); } } } - // Dropping the bounded owner releases sockets, parser buffers and egress attempts. - drop(connections); - Ok(()) + } + phase.send_replace(LifecyclePhase::Draining); + log_native_lifecycle( + LifecyclePhase::Draining, + if outcome.is_ok() { + "drain_requested" + } else { + "runner_failed" + }, + grace, + outcome.is_err(), + ); + drop(listener); + let result = drain_tasks(&mut tasks, grace, signals, outcome).await; + // Service/registry clones are dropped only after every owned connection settles. + drop(service); + result +} + +#[expect( + clippy::integer_division_remainder_used, + reason = "tokio select branch arithmetic" +)] +async fn drain_tasks( + tasks: &mut Connections, + grace: Duration, + signals: &mut Option, + mut outcome: anyhow::Result<()>, +) -> anyhow::Result<()> { + let Some(deadline) = NativeInstant::now().checked_add(grace) else { + tasks.clear(); + return Err(failure("shutdown", "budget", "unusable deadline")); + }; + while !tasks.is_empty() { + tokio::select! { + biased; + signal = next_signal(signals) => { + log_native_lifecycle(LifecyclePhase::Draining, "forced_stop", grace, true); + outcome = Err(signal.err().unwrap_or_else(|| failure("shutdown", "signal", "forced stop"))); + break; + } + () = sleep_until(deadline) => { + log_native_lifecycle(LifecyclePhase::Draining, "grace_exhausted", grace, true); + outcome = Err(failure("shutdown", "budget", "grace exhausted")); + break; + } + Some(result) = tasks.next() => { + report_connection_exit(&result); + } + } + } + tasks.clear(); + outcome +} + +fn log_native_lifecycle(phase: LifecyclePhase, reason: &'static str, grace: Duration, fault: bool) { + let phase_name = match phase { + LifecyclePhase::Starting => "starting", + LifecyclePhase::Ready => "ready", + LifecyclePhase::Draining => "draining", + LifecyclePhase::Stopped => "stopped", }; - runtime.block_on(local.run_until(serve)) + let level = if fault { Level::Error } else { Level::Info }; + let _logged = catch_unwind( + || log::log!(target: "edgezero::native", level, "event=lifecycle phase={phase_name} reason={reason} grace_ms={}", grace.as_millis()), + ); } fn report_connection_exit(result: &ConnectionResult) { @@ -434,7 +808,8 @@ fn report_connection_exit(result: &ConnectionResult) { /// # Errors /// Returns an error if the dev server fails to bind or any required store handle cannot be initialised. fn build_app_for_dispatch() -> anyhow::Result { - App::build::(crate::AXUM_PLATFORM).context("application configuration failed") + App::build::(crate::AXUM_PLATFORM) + .map_err(|_error| anyhow::anyhow!("application configuration failed")) } fn app_logger(env: &EnvConfig) -> SimpleLogger { @@ -443,6 +818,33 @@ fn app_logger(env: &EnvConfig) -> SimpleLogger { .with_module_level(BOOT_LOG_TARGET, BOOT_LOG_LEVEL) } +/// Generated application entrypoint: retain the development runner unless an +/// explicit native bindings file is selected. A selected file always enters +/// strict production startup, where a provider compiled out of the binary fails +/// closed instead of falling back to local stores. +/// +/// # Errors +/// Returns startup or listener errors from the selected runner. +#[inline] +pub fn run_generated_app() -> anyhow::Result<()> { + let bindings_selected = + vars_os().any(|(key, _value)| key.to_str().is_some_and(is_store_bindings_locator_key)); + if bindings_selected { + // `run_production_app` captures all bootstrap settings and the selected + // bindings document once, then rejects unavailable compiled providers. + run_production_app::() + } else { + // Preserve existing generated local-development behavior exactly when + // no native bindings locator was selected. + run_app::() + } +} + +fn is_store_bindings_locator_key(key: &str) -> bool { + strip_native_config_prefix(key) + .is_some_and(|path| path.eq_ignore_ascii_case("STORE_BINDINGS_FILE")) +} + /// Runs an application with the Axum development server. /// /// # Errors @@ -453,21 +855,23 @@ pub fn run_app() -> anyhow::Result<()> { run_app_with_preflight::(|| Ok(())) } -/// Run a startup check after logger initialization and before application configuration. -/// -/// Useful for capability validation in embedded runners. Applications that own logging -/// must install their backend and filters before calling this function. +/// Runs a startup check after logging and policy validation, before app configuration. /// /// # Errors -/// Returns an error if logger setup, `preflight`, application configuration, runtime setup, -/// store initialization, listener binding, or connection serving fails. +/// Returns an error for logging, preflight, configuration, stores, or serving failure. #[inline] pub fn run_app_with_preflight(preflight: F) -> anyhow::Result<()> where A: Hooks, F: FnOnce() -> anyhow::Result<()>, { - let env = EnvConfig::from_env(); + let env = capture_dev_environment()?; + let (policy, request_records) = native_settings(&env)?; + let native = NativeHosting { + policy, + request_records, + ..NativeHosting::default() + }; if !A::owns_logging() { app_logger(&env) .init() @@ -478,44 +882,400 @@ where log::info!("[edgezero] axum HTTP/1 ingress limits: {ingress:?}"); log::info!("[edgezero] axum config snapshot limits: {config_limits:?}"); preflight()?; - let app = build_app_for_dispatch::()?; - let stores = A::stores(); - let kv_init_requirement = kv_init_requirement(stores); - let kv_registry = build_kv_registry(stores.kv, &env, kv_init_requirement)?; - let config_registry = build_config_registry(stores.config, &env, config_limits)?; - let secret_registry = build_secret_registry(stores.secrets, &env); - + let grace = shutdown_grace(&env)?; let resolution = resolve_addr(&env); - for warning in &resolution.warnings { - log::warn!(target: BOOT_LOG_TARGET, "{warning}"); + run_local( + ListenerSource::Bind(resolution.addr), + true, + grace, + ingress, + future::pending::<()>(), + async move { + let app = build_app_for_dispatch::()?; + let metadata = A::stores(); + for warning in &resolution.warnings { + log::warn!(target: BOOT_LOG_TARGET, "{warning}"); + } + let stores = PreparedStores { + kv: build_kv_registry(metadata.kv, &env, kv_init_requirement(metadata))?, + config: build_config_registry(metadata.config, &env, config_limits)?, + secrets: build_secret_registry(metadata.secrets, &env), + }; + prepare_transport(app, stores, native) + }, + ) +} + +fn capture_dev_environment() -> anyhow::Result { + let mut pairs = Vec::new(); + for (os_key, os_value) in vars_os() { + let Some(key) = os_key.to_str() else { + continue; + }; + let recognized = strip_native_config_prefix(key).is_some_and(|path| { + [ + "ADAPTER__TRUSTED_PROXY_CIDRS", + "ADAPTER__FORWARDING_HEADER_FAMILY", + "LOGGING__REQUEST_RECORDS", + ] + .iter() + .any(|setting| path.eq_ignore_ascii_case(setting)) + }); + match (os_value.to_str(), recognized) { + (Some(value), _) => pairs.push((key.to_owned(), value.to_owned())), + (None, true) => { + return Err(failure( + "settings", + "native proxy/record policy", + "non-Unicode value", + )); + } + (None, false) => {} + } } - let addr = resolution.addr; - log::info!("[edgezero] starting axum server on http://{addr}"); + Ok(EnvConfig::from_vars(pairs)) +} - let runtime = RuntimeBuilder::new_multi_thread() - .enable_all() - .build() - .context("failed to build tokio runtime")?; - - runtime.block_on(async move { - let std_listener = StdTcpListener::bind(addr) - .with_context(|| format!("failed to bind dev server to {addr}"))?; - std_listener - .set_nonblocking(true) - .context("failed to set listener to non-blocking")?; - let listener = TokioTcpListener::from_std(std_listener) - .context("failed to adopt std listener into tokio")?; - - let request_stores = Stores { - config_registry, - kv_registry, - secret_registry, - ..Stores::default() +fn init_logging(level: LevelFilter) -> anyhow::Result<()> { + if !A::owns_logging() { + SimpleLogger::new() + .with_level(level) + .with_module_level(BOOT_LOG_TARGET, BOOT_LOG_LEVEL) + .init() + .map_err(|_error| failure("logging", "application", "initialization failed"))?; + } + Ok(()) +} + +fn no_initialize<'startup>( + _app: &'startup mut App, + _stores: &'startup PreparedStores, +) -> LocalStartupFuture<'startup> { + Box::pin(future::ready(Ok(()))) +} + +/// Runs strict framework startup with SIGINT and Unix SIGTERM. No callback is +/// required. Ready does not claim application-specific schema/bootstrap checks. +/// +/// # Errors +/// Returns classified startup/runner errors or non-success for forced shutdown. +#[inline] +pub fn run_production_app() -> anyhow::Result<()> { + run_production_selected::( + AxumRunOptions::from_env()?, + NativeStoreOverrides::default(), + true, + future::pending::<()>(), + no_initialize, + ) +} + +/// Runs strict startup plus exactly one application-owned local initializer. +/// Pass an inline `|app, stores| Box::pin(async move { ... })` or a boxed function +/// so Rust can infer the higher-ranked borrow lifetime. Futures need not be Send. +/// +/// # Errors +/// Returns safe classified errors; callback/provider source chains are discarded. +#[inline] +pub fn run_production_app_with_initializer( + initialize: Initialize, +) -> anyhow::Result<()> +where + A: Hooks, + Initialize: for<'startup> FnOnce( + &'startup mut App, + &'startup PreparedStores, + ) -> LocalStartupFuture<'startup>, +{ + run_production_selected::( + AxumRunOptions::from_env()?, + NativeStoreOverrides::default(), + true, + future::pending::<()>(), + initialize, + ) +} + +/// Runs production with explicit options and caller-owned stop, installing no +/// process-wide signal handlers and merging no ambient hosting/store settings. +/// +/// # Errors +/// Returns classified startup/runner errors or grace exhaustion. +#[inline] +pub fn run_app_with_options(options: AxumRunOptions, stop: Stop) -> anyhow::Result<()> +where + A: Hooks, + Stop: Future, +{ + run_app_with_options_and_initializer::(options, stop, no_initialize) +} + +/// Runs production embedding with app-selected startup checks before acceptance. +/// Retained cloned KV handles remain the caller's responsibility after return. +/// +/// # Errors +/// Returns classified startup/runner errors or grace exhaustion. +#[inline] +pub fn run_app_with_options_and_initializer( + options: AxumRunOptions, + stop: Stop, + initialize: Initialize, +) -> anyhow::Result<()> +where + A: Hooks, + Stop: Future, + Initialize: for<'startup> FnOnce( + &'startup mut App, + &'startup PreparedStores, + ) -> LocalStartupFuture<'startup>, +{ + run_app_with_native_stores::( + options, + NativeStoreOverrides::default(), + stop, + initialize, + ) +} + +/// Runs explicit native bindings and app-owned checks before accepting requests. +/// Setup futures are polled sequentially on the existing current-thread runtime. +/// +/// # Errors +/// Rejects invalid sources before opening local providers or polling deferred setup. +/// Preparation failures discard all staged handles and skip initializer/readiness. +#[inline] +pub fn run_app_with_native_stores( + options: AxumRunOptions, + overrides: NativeStoreOverrides, + stop: Stop, + initialize: Initialize, +) -> anyhow::Result<()> +where + A: Hooks, + Stop: Future, + Initialize: for<'startup> FnOnce( + &'startup mut App, + &'startup PreparedStores, + ) -> LocalStartupFuture<'startup>, +{ + run_production_selected::(options, overrides, false, stop, initialize) +} + +fn run_production_selected( + options: AxumRunOptions, + overrides: NativeStoreOverrides, + signals: bool, + stop: Stop, + initialize: Initialize, +) -> anyhow::Result<()> +where + A: Hooks, + Stop: Future, + Initialize: for<'startup> FnOnce( + &'startup mut App, + &'startup PreparedStores, + ) -> LocalStartupFuture<'startup>, +{ + let geo = options.native_geo()?; + let native = NativeHosting { + diagnostics: options.diagnostics(), + policy: options.trusted_proxy_policy(), + request_records: options.request_records(), + geo, + }; + run_local( + ListenerSource::Bind(options.addr()), + signals, + options.shutdown_grace(), + AxumIngressConfig::from_env(options.env_config())?, + stop, + async move { + let metadata = A::stores(); + let sources = NativeSourcePlan::validate(metadata, &options, overrides)?; + init_logging::(options.logging_level())?; + native.geo.log_snapshot(); + let mut app = build_app_for_dispatch::()?; + let stores = build_production_stores(metadata, &options, sources).await?; + let transport = AxumOutboundClient::try_transport() + .map_err(|_error| failure("transport", "HTTP client", "initialization failed"))?; + initialize(&mut app, &stores).await.map_err(|error| { + let category = error.store_extraction_reason().map_or_else( + || "application checks failed".to_owned(), + |reason| format!("{reason:?}"), + ); + failure("initializer", "application", &category) + })?; + Ok((app, stores, transport, native)) + }, + ) +} + +fn check_data_write_access(root: &Path) -> anyhow::Result<()> { + use std::sync::atomic::{AtomicU64, Ordering}; + static NEXT: AtomicU64 = AtomicU64::new(0); + for _attempt in 0..8_u8 { + let sequence = NEXT.fetch_add(1, Ordering::Relaxed); + let path = root.join(format!( + ".edgezero-write-check-{}-{sequence}", + process::id() + )); + match fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&path) + { + Ok(file) => { + drop(file); + fs::remove_file(path).map_err(|_error| { + failure("stores", "DATA_DIR", "write-probe cleanup failed") + })?; + return Ok(()); + } + Err(error) if error.kind() == ErrorKind::AlreadyExists => {} + Err(_error) => return Err(failure("stores", "DATA_DIR", "directory is not writable")), + } + } + Err(failure("stores", "DATA_DIR", "write-probe unavailable")) +} + +#[expect( + clippy::too_many_lines, + reason = "keep all sequential staging in one ownership scope before atomic publication" +)] +async fn build_production_stores( + metadata: StoresMetadata, + options: &AxumRunOptions, + sources: NativeSourcePlan, +) -> anyhow::Result { + let mut config = BTreeMap::new(); + let mut kv = BTreeMap::new(); + let mut secrets = BTreeMap::new(); + let mut resident = 0_usize; + let mut sessions = sources.sessions()?; + for (id, source) in sources.config { + let binding = match source { + NativeStoreSource::BuiltIn(builtin) => sessions.prepare_config(builtin, id).await?, + local @ (NativeStoreSource::Local + | NativeStoreSource::Ready(_) + | NativeStoreSource::Prepare(_)) => { + native_stores::prepare(local, "config", id, || { + let root = options + .config_dir() + .ok_or_else(|| failure("settings", "CONFIG_DIR", "required"))?; + let store = AxumConfigStore::load_required_at_startup( + &root.join(format!("local-config-{id}.json")), + sources.config_limits, + resident, + ) + .map_err(|error| { + let category = match error { + ConfigStoreError::ValueTooLarge => "snapshot size limit", + ConfigStoreError::Unavailable { message } + if message == "config file missing" => + { + "required snapshot missing" + } + ConfigStoreError::Unavailable { message } + if message == "config snapshot requires a regular file" => + { + "snapshot requires a regular file" + } + ConfigStoreError::DeadlineExceeded + | ConfigStoreError::Internal { .. } + | ConfigStoreError::InvalidKey { .. } + | ConfigStoreError::Unavailable { .. } + | _ => "required snapshot unavailable or malformed", + }; + failure("config", id, category) + })?; + resident = resident + .checked_add(store.resident_allocation_bytes()) + .ok_or_else(|| failure("config", id, "snapshot allocation limit"))?; + Ok(ConfigStoreBinding { + handle: ConfigStoreHandle::new(Arc::new(store)), + default_key: options.env_config().store_key("config", id), + }) + }) + .await? + } }; - serve_with_stores(app, listener, true, ingress, request_stores).await + config.insert(id.to_owned(), binding); + } + let mut handles: BTreeMap = BTreeMap::new(); + if sources + .kv + .values() + .any(|source| matches!(source, NativeStoreSource::Local)) + { + check_data_write_access( + options + .data_dir() + .ok_or_else(|| failure("settings", "DATA_DIR", "required"))?, + )?; + } + for (id, source) in sources.kv { + let handle = match source { + NativeStoreSource::BuiltIn(builtin) => sessions.prepare_kv(builtin, id).await?, + local @ (NativeStoreSource::Local + | NativeStoreSource::Ready(_) + | NativeStoreSource::Prepare(_)) => { + native_stores::prepare(local, "kv", id, || { + let root = options + .data_dir() + .ok_or_else(|| failure("settings", "DATA_DIR", "required"))?; + let name = options.env_config().store_name("kv", id); + if let Some(handle) = handles.get(&name) { + return Ok(handle.clone()); + } + let store = PersistentKvStore::new(root.join(kv_store_file_name(&name))) + .map_err(|_error| failure("kv", id, "database unavailable or locked"))?; + let handle = KvHandle::new(Arc::new(store)); + handles.insert(name, handle.clone()); + Ok(handle) + }) + .await? + } + }; + kv.insert(id.to_owned(), handle); + } + let environment_secrets = SecretHandle::new(Arc::new(EnvSecretStore::new())); + for (id, source) in sources.secrets { + let binding = match source { + NativeStoreSource::BuiltIn(builtin) => sessions.prepare_secrets(builtin, id).await?, + local @ (NativeStoreSource::Local + | NativeStoreSource::Ready(_) + | NativeStoreSource::Prepare(_)) => { + native_stores::prepare(local, "secrets", id, || { + Ok(BoundSecretStore::new( + environment_secrets.clone(), + options.env_config().store_name("secrets", id), + )) + }) + .await? + } + }; + secrets.insert(id.to_owned(), binding); + } + Ok(PreparedStores { + config: finish_registry(metadata.config, config, "config")?, + kv: finish_registry(metadata.kv, kv, "kv")?, + secrets: finish_registry(metadata.secrets, secrets, "secrets")?, }) } +fn finish_registry( + metadata: Option, + handles: BTreeMap, + kind: &str, +) -> anyhow::Result>> { + metadata + .map(|meta| { + StoreRegistry::from_parts(handles, meta.default.to_owned()) + .ok_or_else(|| failure(kind, meta.default, "invalid declared default binding")) + }) + .transpose() +} + /// Build the per-request KV registry from baked store metadata. /// /// Each declared id resolves to a [`PersistentKvStore`] at @@ -531,33 +1291,27 @@ fn build_kv_registry( }; let mut by_id: BTreeMap = BTreeMap::new(); + let mut handles: BTreeMap = BTreeMap::new(); for id in meta.ids { let store_name = env.store_name("kv", id); + if let Some(handle) = handles.get(&store_name) { + by_id.insert((*id).to_owned(), handle.clone()); + continue; + } let kv_path = kv_store_path(&store_name); let handle = match kv_handle_from_path(&kv_path) { Ok(handle) => handle, - Err(err) => match init { + Err(_error) => match init { KvInitRequirement::Optional => { - log::warn!( - target: BOOT_LOG_TARGET, - "KV store '{}' (id `{}`) could not be initialized at {}: {}", - store_name, - id, - kv_path.display(), - err - ); + log::warn!(target: BOOT_LOG_TARGET, "KV store id `{id}` unavailable; omitting optional binding"); continue; } KvInitRequirement::Required => { - return Err(err.context(format!( - "KV store '{}' (id `{}`) is explicitly configured for axum but could not be initialized at {}", - store_name, - id, - kv_path.display() - ))); + return Err(failure("KV", id, "declared database unavailable or locked")); } }, }; + handles.insert(store_name, handle.clone()); by_id.insert((*id).to_owned(), handle); } @@ -642,8 +1396,8 @@ fn build_config_registry_at( Ok(StoreRegistry::from_parts(by_id, default_id)) } -/// Build the per-request secret registry. Axum is `Single` for secrets — every -/// declared id maps to the same env-backed [`EnvSecretStore`]. Each binding +/// Build the per-request local secret registry. Every declared ID maps to the +/// shared env-backed [`EnvSecretStore`], without limiting the number of IDs. Each binding /// captures the platform store name resolved from /// `EDGEZERO__STORES__SECRETS____NAME` (defaulting to the logical id); /// the axum env-secret backend ignores the name on lookup, so the binding @@ -689,6 +1443,97 @@ mod tests { use std::process::Command; use std::sync::atomic::{AtomicUsize, Ordering}; + #[test] + fn generated_entry_selects_native_mode_only_for_bindings_locator() { + assert!(is_store_bindings_locator_key( + "EDGEZERO__STORE_BINDINGS_FILE" + )); + assert!(is_store_bindings_locator_key( + "EDGEZERO__store_bindings_file" + )); + assert!(!is_store_bindings_locator_key( + "EDGEZERO__ADAPTER__DATA_DIR" + )); + assert!(!is_store_bindings_locator_key( + "EDGEZERO_STORE_BINDINGS_FILE" + )); + for key in [ + "edgezero__store_bindings_file", + "EdGeZeRo__STORE_BINDINGS_FILE", + ] { + assert_eq!(is_store_bindings_locator_key(key), cfg!(windows)); + } + } + + #[test] + fn generated_bindings_locator_child() { + let Ok(expected) = env::var("EDGEZERO_TEST_BINDINGS_ERROR") else { + return; + }; + // This hook fails construction even if dispatch incorrectly selects local defaults. + let error = run_generated_app::().expect_err("startup failure"); + assert_eq!(error.to_string(), expected); + assert_eq!(error.chain().count(), 1); + } + + #[cfg(any(unix, windows))] + #[test] + fn generated_locator_casing_and_non_unicode_fail_closed() { + use std::ffi::OsString; + #[cfg(unix)] + use std::os::unix::ffi::OsStringExt as _; + #[cfg(windows)] + use std::os::windows::ffi::OsStringExt as _; + + let root = tempfile::tempdir().expect("root"); + let missing = root.path().join("missing.toml"); + #[cfg(unix)] + let non_unicode = OsString::from_vec(vec![0xff]); + #[cfg(windows)] + let non_unicode = OsString::from_wide(&[0xd800]); + for prefix in ["EDGEZERO__", "edgezero__", "EdGeZeRo__"] { + for (value, selected_error) in [ + ( + missing.clone().into_os_string(), + "startup settings: bindings: native bindings file unavailable", + ), + ( + non_unicode.clone(), + "startup settings: STORE_BINDINGS_FILE: non-Unicode value", + ), + ] { + let recognized = prefix == "EDGEZERO__" || cfg!(windows); + // Non-Unicode rejection belongs to selected strict startup; do not exercise + // the unchanged development runner's Unicode-only environment collection. + if !recognized && value.to_str().is_none() { + continue; + } + let expected = if recognized { + selected_error + } else { + "application configuration failed" + }; + let output = Command::new(env::current_exe().expect("test executable")) + .args([ + "--exact", + "dev_server::tests::generated_bindings_locator_child", + ]) + .env_clear() + .env("EDGEZERO_TEST_BINDINGS_ERROR", expected) + .env(format!("{prefix}store_bindings_file"), value) + .output() + .expect("isolated generated entrypoint"); + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(output.status.success(), "{stdout}\n{stderr}"); + assert!( + stdout.contains("1 passed"), + "child test must execute: {stdout}" + ); + } + } + } + struct FailingConfiguration; struct LoggingConfiguration; @@ -794,12 +1639,44 @@ mod tests { format!("{stdout}\n{stderr}") } + fn strict_logging_options() -> AxumRunOptions { + AxumRunOptions::from_vars([( + "EDGEZERO__LOGGING__LEVEL", + env::var("EDGEZERO__LOGGING__LEVEL").expect("probe logging level"), + )]) + .expect("strict probe options") + } + #[test] fn startup_logging_child() { let Ok(scenario) = env::var("EDGEZERO_TEST_LOGGING_SCENARIO") else { return; }; let result = match scenario.as_str() { + "strict-managed" => run_app_with_options::( + strict_logging_options(), + future::pending::<()>(), + ), + "strict-invalid-sources" => { + SimpleLogger::new().init().expect("existing logger"); + let mut overrides = NativeStoreOverrides::default(); + overrides + .prepare_config("undeclared", async { panic!("setup must not be polled") }) + .expect("deferred source"); + let result = run_app_with_native_stores::( + strict_logging_options(), + overrides, + future::pending::<()>(), + no_initialize, + ); + assert!( + result + .expect_err("source error must precede logger conflict") + .to_string() + .contains("undeclared binding"), + ); + return; + } "invalid-ingress" => { let result = run_app_with_preflight::(|| { panic!( @@ -814,30 +1691,50 @@ mod tests { ); return; } - "installed" => { + "installed" | "strict-installed" => { SimpleLogger::new() .with_level(LevelFilter::Trace) .init() .expect("existing logger"); - let result = run_app_with_preflight::(|| { - log::warn!(target: BOOT_LOG_TARGET, "unexpected-preflight-probe"); - Ok(()) - }); + let strict = scenario == "strict-installed"; + let result = if strict { + run_app_with_options::( + strict_logging_options(), + future::pending::<()>(), + ) + } else { + run_app_with_preflight::(|| { + log::warn!(target: BOOT_LOG_TARGET, "unexpected-preflight-probe"); + Ok(()) + }) + }; + let expected = if strict { + "startup logging: application: initialization failed" + } else { + "failed to initialize application logger" + }; assert_eq!( result .expect_err("logger conflict must stop startup") .to_string(), - "failed to initialize application logger" + expected ); return; } - "owned" => { + "owned" | "strict-owned" => { SimpleLogger::new() .with_level(LevelFilter::Trace) .init() .expect("owned logger"); log::set_max_level(LevelFilter::Debug); - let result = run_app::(); + let result = if scenario == "strict-owned" { + run_app_with_options::( + strict_logging_options(), + future::pending::<()>(), + ) + } else { + run_app::() + }; assert_eq!(log::max_level(), LevelFilter::Debug); result } @@ -875,6 +1772,44 @@ mod tests { } } + #[test] + fn strict_logging_is_ready_before_configure_with_separate_boot_filter() { + for level in ["off", "error", "trace"] { + let output = logging_probe("strict-managed", level); + assert!(output.contains("boot-warning-probe"), "{output}"); + assert!(!output.contains("boot-info-probe"), "{output}"); + assert_eq!( + output.contains("runtime-warning-probe"), + level == "trace", + "{output}" + ); + assert_eq!( + output.contains("runtime-debug-probe"), + level == "trace", + "{output}" + ); + } + } + + #[test] + fn strict_logger_conflict_prevents_configuration() { + let output = logging_probe("strict-installed", "trace"); + assert!(!output.contains("boot-warning-probe"), "{output}"); + } + + #[test] + fn strict_owned_logging_preserves_backend_and_facade_filter() { + let output = logging_probe("strict-owned", "off"); + assert!(output.contains("runtime-debug-probe"), "{output}"); + assert!(output.contains("boot-info-probe"), "{output}"); + } + + #[test] + fn strict_source_validation_precedes_logging_and_configuration() { + let output = logging_probe("strict-invalid-sources", "trace"); + assert!(!output.contains("boot-warning-probe"), "{output}"); + } + #[test] fn owned_logging_preserves_backend_and_facade_filter() { let output = logging_probe("owned", "off"); @@ -918,6 +1853,45 @@ mod tests { assert_eq!(app.platform(), crate::AXUM_PLATFORM); } + #[test] + fn initializer_borrows_inputs_across_await_and_captures_rc() { + use futures::executor::block_on; + use std::cell::Cell; + use std::rc::Rc; + + async fn initialize( + app: &mut App, + stores: &PreparedStores, + callback: Initialize, + ) -> Result<(), EdgeError> + where + Initialize: for<'startup> FnOnce( + &'startup mut App, + &'startup PreparedStores, + ) -> LocalStartupFuture<'startup>, + { + callback(app, stores).await + } + let calls = Rc::new(Cell::new(0_u32)); + let recorded = Rc::clone(&calls); + let mut owned_app = App::new(RouterService::builder().build()); + block_on(initialize( + &mut owned_app, + &PreparedStores::default(), + move |app, stores| { + Box::pin(async move { + future::ready(()).await; + assert!(stores.config().is_none()); + app.insert_state(Arc::::from("initialized")); + recorded.set(recorded.get() + 1); + Ok(()) + }) + }, + )) + .expect("initialize"); + assert_eq!(calls.get(), 1); + } + #[test] fn default_config_uses_expected_address() { let config = AxumDevServerConfig::default(); @@ -977,6 +1951,87 @@ mod tests { assert!(!server.config.enable_ctrl_c); } + #[test] + fn prepared_stores_preserve_legacy_defaults_and_explicit_registry_precedence() { + let temp = tempfile::tempdir().expect("database directory"); + let config = ConfigStoreHandle::new(Arc::new( + AxumConfigStore::from_map([], ConfigStoreLimits::default()).expect("config store"), + )); + let kv = KvHandle::new(Arc::new( + PersistentKvStore::new(temp.path().join("registry.redb")).expect("KV store"), + )); + let secrets = SecretHandle::new(Arc::new(EnvSecretStore::new())); + let with_handles = |server: AxumDevServer| { + server + .with_config_store(config.clone()) + .with_kv_handle(kv.clone()) + .with_secret_handle(secrets.clone()) + }; + let legacy: PreparedStores = + with_handles(AxumDevServer::new(RouterService::builder().build())) + .stores + .into(); + let legacy_config = legacy.config().expect("config registry"); + let legacy_kv = legacy.kv().expect("KV registry"); + let legacy_secrets = legacy.secrets().expect("secret registry"); + assert_eq!(legacy_config.ids().collect::>(), ["default"]); + assert_eq!(legacy_config.default_id(), "default"); + assert_eq!( + legacy_config.default_ref().expect("binding").default_key, + "default" + ); + assert_eq!(legacy_kv.ids().collect::>(), ["default"]); + assert_eq!(legacy_kv.default_id(), "default"); + assert_eq!(legacy_secrets.ids().collect::>(), ["default"]); + assert_eq!(legacy_secrets.default_id(), "default"); + assert_eq!( + legacy_secrets.default_ref().expect("binding").store_name(), + "default" + ); + + let with_registries = |server: AxumDevServer| { + server + .with_config_registry(ConfigRegistry::single_id( + "application".to_owned(), + ConfigStoreBinding { + handle: config.clone(), + default_key: "settings".to_owned(), + }, + )) + .with_kv_registry(KvRegistry::single_id("sessions".to_owned(), kv.clone())) + .with_secret_registry(SecretRegistry::single_id( + "runtime".to_owned(), + BoundSecretStore::new(secrets.clone(), "vault".to_owned()), + )) + }; + for registry_first in [true, false] { + let unwired = AxumDevServer::new(RouterService::builder().build()); + let server = if registry_first { + with_handles(with_registries(unwired)) + } else { + with_registries(with_handles(unwired)) + }; + let prepared: PreparedStores = server.stores.into(); + let config_registry = prepared.config().expect("config registry"); + let kv_registry = prepared.kv().expect("KV registry"); + let secret_registry = prepared.secrets().expect("secret registry"); + assert_eq!(config_registry.default_id(), "application"); + assert_eq!( + config_registry.default_ref().expect("binding").default_key, + "settings" + ); + assert_eq!(kv_registry.default_id(), "sessions"); + assert_eq!(secret_registry.default_id(), "runtime"); + assert_eq!( + secret_registry.default_ref().expect("binding").store_name(), + "vault" + ); + assert!(config_registry.named("default").is_none()); + assert!(kv_registry.named("default").is_none()); + assert!(secret_registry.named("default").is_none()); + } + } + #[test] fn every_store_name_gets_a_slug_based_path() { // The pre-rewrite shortcut hard-coded `.edgezero/kv.redb` @@ -2262,3 +3317,737 @@ mod integration_tests { server.handle.abort(); } } + +#[cfg(test)] +mod native_store_tests { + use super::*; + use crate::native_stores::NativeStorePreparationError; + use bytes::Bytes; + use edgezero_core::config_store::{ConfigStore, finish_bounded_config_read}; + use edgezero_core::context::RequestContext; + use edgezero_core::extractor::State; + use edgezero_core::key_value_store::NoopKvStore; + use edgezero_core::secret_store::InMemorySecretStore; + use edgezero_core::{ConfigValue, EdgeError, action}; + use futures::executor::block_on; + use std::cell::Cell; + use std::rc::Rc; + use std::sync::atomic::{AtomicUsize, Ordering}; + use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; + use tokio::net::TcpStream; + use tokio::time::timeout; + + struct ProbeConfig { + drops: Arc, + reads: Arc, + } + + impl Drop for ProbeConfig { + fn drop(&mut self) { + self.drops.fetch_add(1, Ordering::SeqCst); + } + } + + #[async_trait::async_trait(?Send)] + impl ConfigStore for ProbeConfig { + async fn get(&self, key: &str) -> Result, ConfigStoreError> { + self.reads.fetch_add(1, Ordering::SeqCst); + Ok((key == "owned_key").then(|| ConfigValue::from("snapshot"))) + } + + async fn get_bounded( + &self, + key: &str, + clock: &edgezero_core::MonotonicClock, + deadline: edgezero_core::Deadline, + max_backend_bytes: u64, + max_value_bytes: u64, + ) -> Result, ConfigStoreError> { + if deadline.is_expired_at(clock.now()) { + return Err(ConfigStoreError::DeadlineExceeded); + } + finish_bounded_config_read( + self.get(key).await, + clock, + deadline, + max_backend_bytes, + max_value_bytes, + ) + } + } + + fn binding(drops: &Arc, reads: &Arc) -> ConfigStoreBinding { + ConfigStoreBinding { + default_key: "owned_key".to_owned(), + handle: ConfigStoreHandle::new(Arc::new(ProbeConfig { + drops: Arc::clone(drops), + reads: Arc::clone(reads), + })), + } + } + + fn options() -> AxumRunOptions { + let listener = StdTcpListener::bind("127.0.0.1:0").unwrap(); + AxumRunOptions::new(listener.local_addr().unwrap()).unwrap() + } + + fn metadata() -> StoresMetadata { + StoresMetadata { + config: Some(StoreMetadata { + ids: &["first", "second"], + default: "first", + }), + ..StoresMetadata::default() + } + } + + struct CustomApp; + + #[expect( + clippy::missing_trait_methods, + reason = "fixture overrides only routes, store declarations and logging ownership" + )] + impl Hooks for CustomApp { + fn owns_logging() -> bool { + true + } + fn stores() -> StoresMetadata { + metadata() + } + fn routes() -> RouterService { + RouterService::builder().get("/", read_snapshot).build() + } + } + + #[action] + async fn read_snapshot( + ctx: RequestContext, + State(state): State>, + ) -> Result { + let selected = ctx + .config_store_default_binding() + .ok_or_else(|| EdgeError::internal(anyhow::anyhow!("missing fixture binding")))?; + let value = selected + .handle + .get(&selected.default_key) + .await? + .unwrap_or_default(); + Ok(format!("{}:{state}", value.as_str())) + } + + #[cfg(not(feature = "geo-cloudfront"))] + #[tokio::test(flavor = "current_thread")] + async fn geo_unavailable_selection_fails_before_runner_resources() { + use crate::geo::{NativeGeoSelection, ProtectedCloudFrontAlb}; + let root = tempfile::tempdir().unwrap(); + let options = options() + .with_config_dir(root.path()) + .unwrap() + .with_data_dir(root.path()) + .unwrap() + .with_geo(NativeGeoSelection::CloudFront { + policy: ProtectedCloudFrontAlb::new(["127.0.0.1/32"], "x-origin-verify-fixture") + .unwrap(), + }); + let address = options.addr(); + let prepared = Rc::new(Cell::new(false)); + let initialized = Rc::new(Cell::new(false)); + let prepare_observed = Rc::clone(&prepared); + let initialize_observed = Rc::clone(&initialized); + let mut overrides = NativeStoreOverrides::default(); + overrides + .prepare_config("first", async move { + prepare_observed.set(true); + Err(NativeStorePreparationError::Unavailable) + }) + .unwrap(); + // Existing Tokio runtime would reject nested runner construction if validation were late. + let error = run_app_with_native_stores::( + options, + overrides, + future::pending::<()>(), + move |_app, _stores| { + Box::pin(async move { + initialize_observed.set(true); + Ok(()) + }) + }, + ) + .unwrap_err(); + assert!(error.to_string().contains("unavailable")); + assert!(!prepared.get()); + assert!(!initialized.get()); + assert_eq!(fs::read_dir(root.path()).unwrap().count(), 0); + let listener = StdTcpListener::bind(address).expect("runner never bound listener"); + drop(listener); + } + + #[cfg(feature = "geo-maxmind")] + #[test] + fn geo_maxmind_prepared_before_ready_stop_skips_initializer_and_bind() { + use crate::geo::{MaxMindGeoConfig, NativeGeoSelection, ReaderIpExpectation}; + let options = options().with_geo(NativeGeoSelection::MaxMind { + config: MaxMindGeoConfig::new( + Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/geoip/city.mmdb"), + ReaderIpExpectation::PeerAllowed, + ), + }); + let address = options.addr(); + let initialized = Rc::new(Cell::new(false)); + let observed = Rc::clone(&initialized); + run_app_with_options_and_initializer::( + options, + future::ready(()), + move |_app, _stores| { + Box::pin(async move { + observed.set(true); + Ok(()) + }) + }, + ) + .unwrap(); + assert!(!initialized.get()); + drop(StdTcpListener::bind(address).expect("ready stop never binds")); + } + + #[cfg(feature = "geo-maxmind")] + #[tokio::test(flavor = "current_thread")] + async fn geo_maxmind_invalid_database_fails_before_runner_resources() { + use crate::geo::{MaxMindGeoConfig, NativeGeoSelection, ReaderIpExpectation}; + let root = tempfile::tempdir().unwrap(); + let path = root.path().join("PRIVATE_DATABASE_PATH"); + fs::write(&path, b"not an MMDB").unwrap(); + let options = options().with_geo(NativeGeoSelection::MaxMind { + config: MaxMindGeoConfig::new(path, ReaderIpExpectation::PeerAllowed), + }); + let address = options.addr(); + let error = run_app_with_options_and_initializer::( + options, + future::pending::<()>(), + |_app, _stores| Box::pin(async { panic!("initializer must not run") }), + ) + .unwrap_err(); + assert_eq!(error.to_string(), "native_geo: invalid database"); + assert!(!format!("{error:?}").contains("PRIVATE_DATABASE_PATH")); + drop(StdTcpListener::bind(address).expect("invalid database never binds")); + } + + #[test] + fn ready_and_rc_deferred_inputs_share_completed_registries_with_requests() { + let options = options(); + let address = options.addr(); + let drops = Arc::new(AtomicUsize::new(0)); + let reads = Arc::new(AtomicUsize::new(0)); + let initialized = Rc::new(Cell::new(false)); + let polls = Rc::new(Cell::new(0_usize)); + let mut overrides = NativeStoreOverrides::default(); + let deferred = binding(&drops, &reads); + let counted = Rc::clone(&polls); + overrides + .prepare_config("first", async move { + counted.set(counted.get() + 1); + yield_now().await; + Ok(deferred) + }) + .unwrap(); + overrides + .insert_config("second", binding(&drops, &Arc::new(AtomicUsize::new(0)))) + .unwrap(); + let completed = Rc::clone(&initialized); + run_app_with_native_stores::( + options, + overrides, + async { + timeout(Duration::from_secs(5), async { + while !initialized.get() { + yield_now().await; + } + let mut socket = loop { + if let Ok(socket) = TcpStream::connect(address).await { + break socket; + } + yield_now().await; + }; + socket + .write_all( + b"GET / HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n", + ) + .await + .unwrap(); + let mut response = Vec::new(); + socket.read_to_end(&mut response).await.unwrap(); + assert!(response.ends_with(b"snapshot:initialized")); + }) + .await + .unwrap(); + }, + move |app, stores| { + Box::pin(async move { + let registry = stores.config().unwrap(); + assert_eq!(registry.default_id(), "first"); + let selected = registry.default_ref().unwrap(); + assert_eq!(selected.default_key, "owned_key"); + assert_eq!( + selected + .handle + .get(&selected.default_key) + .await + .unwrap() + .unwrap() + .as_str(), + "snapshot" + ); + assert!(registry.named("second").is_some()); + app.insert_state(Arc::::from("initialized")); + completed.set(true); + Ok(()) + }) + }, + ) + .unwrap(); + assert_eq!(polls.get(), 1); + assert_eq!(reads.load(Ordering::SeqCst), 2); + assert_eq!(drops.load(Ordering::SeqCst), 2); + } + + #[test] + fn ready_and_deferred_kv_need_no_local_data_root() { + let opened = Rc::new(Cell::new(0_usize)); + let tracked = Rc::clone(&opened); + let mut overrides = NativeStoreOverrides::default(); + overrides + .insert_kv("first", KvHandle::new(Arc::new(NoopKvStore))) + .unwrap(); + overrides + .prepare_kv("second", async move { + tracked.set(tracked.get() + 1); + Ok(KvHandle::new(Arc::new(NoopKvStore))) + }) + .unwrap(); + let metadata = StoresMetadata { + kv: Some(StoreMetadata { + ids: &["first", "second"], + default: "second", + }), + ..StoresMetadata::default() + }; + let options = options(); + let plan = NativeSourcePlan::validate(metadata, &options, overrides).unwrap(); + let stores = block_on(build_production_stores(metadata, &options, plan)).unwrap(); + assert_eq!(opened.get(), 1); + assert_eq!(stores.kv().unwrap().default_id(), "second"); + assert!(stores.kv().unwrap().named("first").is_some()); + } + + #[test] + fn malformed_metadata_is_rejected_before_any_deferred_poll() { + for declaration in [ + StoreMetadata { + default: "missing", + ids: &["first"], + }, + StoreMetadata { + default: "first", + ids: &["first", "first"], + }, + StoreMetadata { + default: "bad/id", + ids: &["bad/id"], + }, + StoreMetadata { + default: "first", + ids: &[], + }, + ] { + let polled = Rc::new(Cell::new(false)); + let observed = Rc::clone(&polled); + let mut overrides = NativeStoreOverrides::default(); + overrides + .prepare_kv("first", async move { + observed.set(true); + Ok(KvHandle::new(Arc::new(NoopKvStore))) + }) + .unwrap(); + let metadata = StoresMetadata { + kv: Some(declaration), + ..StoresMetadata::default() + }; + assert!(NativeSourcePlan::validate(metadata, &options(), overrides).is_err()); + assert!(!polled.get()); + } + } + + #[test] + fn duplicate_inputs_never_replace_an_existing_source_and_debug_is_redacted() { + let mut overrides = NativeStoreOverrides::default(); + overrides + .insert_kv("SENTINEL", KvHandle::new(Arc::new(NoopKvStore))) + .unwrap(); + assert_eq!( + overrides.prepare_kv("SENTINEL", async { panic!("must not poll") }), + Err(NativeStorePreparationError::DuplicateBinding) + ); + let drops = Arc::new(AtomicUsize::new(0)); + overrides + .insert_config("SENTINEL", binding(&drops, &drops)) + .unwrap(); + assert_eq!( + overrides.insert_config("SENTINEL", binding(&drops, &drops)), + Err(NativeStorePreparationError::DuplicateBinding) + ); + let secret = BoundSecretStore::new( + SecretHandle::new(Arc::new(InMemorySecretStore::new([("right/key", "value")]))), + "right".to_owned(), + ); + overrides + .insert_secrets("SENTINEL", secret.clone()) + .unwrap(); + assert_eq!( + overrides.prepare_secrets("SENTINEL", async move { Ok(secret) }), + Err(NativeStorePreparationError::DuplicateBinding) + ); + assert!(!format!("{overrides:?}").contains("SENTINEL")); + } + + #[test] + fn file_and_inline_source_errors_precede_deferred_polls_and_local_opens() { + use crate::native_bindings::{NativeBindingsLimits, NativeStoreBindings}; + + for file_input in [false, true] { + for invalid in ["duplicate", "undeclared", "overlay", "disabled"] { + if invalid == "disabled" && cfg!(feature = "aws-appconfig-agent") { + continue; + } + let data = tempfile::tempdir().unwrap(); + let bootstrap = tempfile::tempdir().unwrap(); + let mut selected = options().with_data_dir(data.path()).unwrap(); + if invalid == "overlay" { + selected = AxumRunOptions::from_vars([( + "EDGEZERO__STORES__CONFIG__FIRST__KEY", + "OVERLAY_SENTINEL", + )]) + .unwrap() + .with_data_dir(data.path()) + .unwrap(); + } + let input = "version = 1\n[config.first]\nprovider = 'aws-appconfig-agent'\ndefault_key = 'settings'\nendpoint = 'http://127.0.0.1:2772'\n[config.first.documents.settings]\napplication = 'example'\nenvironment = 'production'\nprofile = 'PROFILE_SENTINEL'\n"; + let document = if invalid == "undeclared" { + input.replace("config.first", "config.UNKNOWN_SENTINEL") + } else { + input.to_owned() + }; + if file_input { + let path = bootstrap.path().join("bindings.toml"); + fs::write(&path, document).unwrap(); + selected = selected.with_store_bindings_file(&path).unwrap(); + fs::remove_file(path).unwrap(); + } else { + selected = selected + .with_store_bindings( + NativeStoreBindings::parse(&document, NativeBindingsLimits::default()) + .unwrap(), + ) + .unwrap(); + } + let polls = Rc::new(Cell::new(0_usize)); + let tracked = Rc::clone(&polls); + let drops = Arc::new(AtomicUsize::new(0)); + let handle = binding(&drops, &drops); + let mut overrides = NativeStoreOverrides::default(); + let id = if invalid == "duplicate" { + "first" + } else { + "second" + }; + overrides + .prepare_config(id, async move { + tracked.set(tracked.get() + 1); + Ok(handle) + }) + .unwrap(); + let initialized = Rc::new(Cell::new(false)); + let called = Rc::clone(&initialized); + let error = run_app_with_native_stores::( + selected, + overrides, + future::pending(), + move |_, _| { + called.set(true); + Box::pin(future::ready(Ok(()))) + }, + ) + .unwrap_err(); + assert_eq!(polls.get(), 0); + assert!(!initialized.get()); + assert_eq!(fs::read_dir(data.path()).unwrap().count(), 0); + assert_eq!(drops.load(Ordering::SeqCst), 1); + assert!(!format!("{error:?}").contains("SENTINEL")); + let expected = match invalid { + "duplicate" => "duplicate", + "undeclared" => "undeclared", + "overlay" => "NAME/KEY", + _ => "not compiled", + }; + assert!(error.to_string().contains(expected), "{error}"); + } + } + } + + #[test] + fn source_errors_do_not_poll_futures_open_databases_or_run_initializer() { + for invalid in ["undeclared", "name", "key", "config_root", "data_root"] { + let root = tempfile::tempdir().unwrap(); + let mut options = options().with_data_dir(root.path()).unwrap(); + if matches!(invalid, "name" | "key") { + options = AxumRunOptions::from_vars([( + format!( + "EDGEZERO__STORES__CONFIG__FIRST__{}", + invalid.to_uppercase() + ), + "OVERLAY_SENTINEL".to_owned(), + )]) + .unwrap() + .with_data_dir(root.path()) + .unwrap(); + } + if invalid == "data_root" { + options = AxumRunOptions::new(options.addr()).unwrap(); + } + let polled = Rc::new(Cell::new(0_usize)); + let tracked = Rc::clone(&polled); + let drops = Arc::new(AtomicUsize::new(0)); + let supplied = binding(&drops, &Arc::new(AtomicUsize::new(0))); + let mut overrides = NativeStoreOverrides::default(); + overrides + .prepare_config("first", async move { + tracked.set(tracked.get() + 1); + Ok(supplied) + }) + .unwrap(); + if invalid != "config_root" { + overrides + .insert_config("second", binding(&drops, &Arc::new(AtomicUsize::new(0)))) + .unwrap(); + } + if invalid == "undeclared" { + overrides + .insert_kv("OTHER_SENTINEL", KvHandle::new(Arc::new(NoopKvStore))) + .unwrap(); + } + let calls = Rc::new(Cell::new(0_usize)); + let called = Rc::clone(&calls); + let result = run_app_with_native_stores::( + options, + overrides, + future::pending(), + move |_, _| { + called.set(called.get() + 1); + Box::pin(future::ready(Ok(()))) + }, + ); + let error = result.unwrap_err(); + assert_eq!(polled.get(), 0, "{invalid}"); + assert_eq!(calls.get(), 0); + assert_eq!(fs::read_dir(root.path()).unwrap().count(), 0); + assert!(!format!("{error:?}").contains("SENTINEL")); + } + } + + struct WithLocalKv; + + #[expect( + clippy::missing_trait_methods, + reason = "fixture overrides only routes, store declarations and logging ownership" + )] + impl Hooks for WithLocalKv { + fn owns_logging() -> bool { + true + } + fn stores() -> StoresMetadata { + StoresMetadata { + kv: Some(StoreMetadata { + ids: &["state"], + default: "state", + }), + ..metadata() + } + } + fn routes() -> RouterService { + RouterService::builder().build() + } + } + + #[test] + fn local_roots_are_required_only_for_remaining_local_ids() { + let root = tempfile::tempdir().unwrap(); + fs::write( + root.path().join("local-config-second.json"), + r#"{"second":"local"}"#, + ) + .unwrap(); + let drops = Arc::new(AtomicUsize::new(0)); + let config = binding(&drops, &Arc::new(AtomicUsize::new(0))); + let mut overrides = NativeStoreOverrides::default(); + overrides.insert_config("first", config).unwrap(); + overrides + .insert_kv("state", KvHandle::new(Arc::new(NoopKvStore))) + .unwrap(); + let metadata = WithLocalKv::stores(); + let options = options().with_config_dir(root.path()).unwrap(); + let plan = NativeSourcePlan::validate(metadata, &options, overrides).unwrap(); + let stores = block_on(build_production_stores(metadata, &options, plan)).unwrap(); + let registry = stores.config().unwrap(); + assert_eq!( + registry.named_ref("first").unwrap().default_key, + "owned_key" + ); + let local = registry.named_ref("second").unwrap(); + assert_eq!(local.default_key, "second"); + assert_eq!( + block_on(local.handle.get("second")) + .unwrap() + .unwrap() + .as_str(), + "local" + ); + assert!(stores.kv().unwrap().named("state").is_some()); + assert_eq!(fs::read_dir(root.path()).unwrap().count(), 1); + } + + #[test] + fn supplied_secret_namespace_is_preserved_and_wrong_namespaces_miss() { + let handle = + SecretHandle::new(Arc::new(InMemorySecretStore::new([("right/key", "value")]))); + let mut overrides = NativeStoreOverrides::default(); + overrides + .insert_secrets( + "first", + BoundSecretStore::new(handle.clone(), "right".to_owned()), + ) + .unwrap(); + overrides + .prepare_secrets("second", async move { + Ok(BoundSecretStore::new(handle, "wrong".to_owned())) + }) + .unwrap(); + let metadata = StoresMetadata { + secrets: Some(StoreMetadata { + ids: &["first", "second"], + default: "second", + }), + ..StoresMetadata::default() + }; + let options = options(); + let plan = NativeSourcePlan::validate(metadata, &options, overrides).unwrap(); + let stores = block_on(build_production_stores(metadata, &options, plan)).unwrap(); + let registry = stores.secrets().unwrap(); + assert_eq!(registry.default_id(), "second"); + assert_eq!( + block_on(registry.named("first").unwrap().get_bytes("key")).unwrap(), + Some(Bytes::from("value")) + ); + assert_eq!( + block_on(registry.default().unwrap().get_bytes("key")).unwrap(), + None + ); + } + + #[test] + fn later_failure_drops_staged_handles_and_unpolled_futures_without_readiness() { + let options = options(); + let address = options.addr(); + let drops = Arc::new(AtomicUsize::new(0)); + let order = Rc::new(Cell::new(0_usize)); + let first = binding(&drops, &Arc::new(AtomicUsize::new(0))); + let recorded = Rc::clone(&order); + let mut overrides = NativeStoreOverrides::default(); + overrides + .prepare_config("first", async move { + assert_eq!(recorded.get(), 0); + recorded.set(1); + Ok(first) + }) + .unwrap(); + let failing_order = Rc::clone(&order); + overrides + .prepare_config("second", async move { + assert_eq!(failing_order.get(), 1); + failing_order.set(2); + Err(NativeStorePreparationError::Unavailable) + }) + .unwrap(); + let unused = binding(&drops, &Arc::new(AtomicUsize::new(0))); + overrides + .prepare_secrets("third", async move { + let _retained = unused; + panic!("later future must not poll after preparation failure") + }) + .unwrap(); + let result = run_app_with_native_stores::( + options, + overrides, + future::pending(), + |_, _| panic!("initializer must not run after failed preparation"), + ); + assert!(result.is_err()); + assert_eq!(order.get(), 2); + assert_eq!(drops.load(Ordering::SeqCst), 2); + StdTcpListener::bind(address).unwrap(); + } + + struct WithSecrets; + + #[expect( + clippy::missing_trait_methods, + reason = "fixture overrides only routes, store declarations and logging ownership" + )] + impl Hooks for WithSecrets { + fn owns_logging() -> bool { + true + } + fn stores() -> StoresMetadata { + StoresMetadata { + secrets: Some(StoreMetadata { + ids: &["third"], + default: "third", + }), + ..metadata() + } + } + fn routes() -> RouterService { + RouterService::builder().build() + } + } + + #[test] + fn stop_during_pending_preparation_drops_staged_state_without_binding() { + let options = options(); + let address = options.addr(); + let drops = Arc::new(AtomicUsize::new(0)); + let mut overrides = NativeStoreOverrides::default(); + overrides + .insert_config("first", binding(&drops, &Arc::new(AtomicUsize::new(0)))) + .unwrap(); + let entered = Rc::new(Cell::new(false)); + let observed = Rc::clone(&entered); + overrides + .prepare_config("second", async move { + observed.set(true); + future::pending().await + }) + .unwrap(); + run_app_with_native_stores::( + options, + overrides, + async { + while !entered.get() { + yield_now().await; + } + }, + |_, _| panic!("initializer must not run after startup stop"), + ) + .unwrap(); + assert_eq!(drops.load(Ordering::SeqCst), 1); + StdTcpListener::bind(address).unwrap(); + } +} diff --git a/crates/edgezero-adapter-axum/src/diagnostics.rs b/crates/edgezero-adapter-axum/src/diagnostics.rs new file mode 100644 index 000000000..c07f5a0a7 --- /dev/null +++ b/crates/edgezero-adapter-axum/src/diagnostics.rs @@ -0,0 +1,1521 @@ +//! Safe native request observations and managed-lifetime accounting. + +#![expect( + clippy::arbitrary_source_item_ordering, + reason = "Public record fields follow their diagnostic meaning; native identity and lifetime owners remain grouped with their transitions rather than alphabetized" +)] + +use std::fmt; +use std::net::{IpAddr, SocketAddr}; +use std::panic::{AssertUnwindSafe, catch_unwind}; +use std::sync::atomic::{AtomicU16, AtomicU64, Ordering}; +use std::sync::{Arc, Mutex, MutexGuard, OnceLock, PoisonError}; +use std::time::Duration; + +use edgezero_core::http::{Method, StatusCode}; +use edgezero_core::ingress::{ + CheckedEffectiveHost, CheckedHostSource, NormalizedIngressHeadSummary, +}; +use edgezero_core::response_egress::{ + ResponseEgressBodyKind, ResponseEgressCompletion, ResponseEgressFailureClassification, + ResponseEgressFailureKind, ResponseEgressFallbackDisposition, ResponseEgressOutcome, + ResponseEgressReport, +}; +use edgezero_core::{MonotonicClock, MonotonicInstant}; + +use crate::proxy::{CheckedProxyMetadata, ForwardingResult, Scheme}; + +/// Framework-generated correlation value, not a credential or visitor-supplied ID. +#[derive(Clone, Copy, Eq, Hash, PartialEq)] +pub struct RequestId { + nonce: [u8; 16], + host: u64, + request: u64, +} + +impl fmt::Display for RequestId { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + for byte in self.nonce { + write!(f, "{byte:02x}")?; + } + write!(f, "{:016x}{:016x}", self.host, self.request) + } +} + +impl fmt::Debug for RequestId { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(self, f) + } +} + +/// Immutable checked facts captured before managed admission. +#[derive(Clone)] +pub struct NativeIngressMetadata { + request_id: RequestId, + proxy: CheckedProxyMetadata, + received: NormalizedIngressHeadSummary, +} + +impl NativeIngressMetadata { + pub(crate) fn new( + request_id: RequestId, + proxy: CheckedProxyMetadata, + received: NormalizedIngressHeadSummary, + ) -> Self { + Self { + request_id, + proxy, + received, + } + } + + #[must_use] + #[inline] + pub fn request_id(&self) -> RequestId { + self.request_id + } + + #[must_use] + #[inline] + pub fn direct_peer(&self) -> Option { + self.proxy.direct_peer() + } + + #[must_use] + #[inline] + pub fn effective_client(&self) -> Option { + self.proxy.effective_client() + } + + #[must_use] + #[inline] + pub fn effective_host(&self) -> &CheckedEffectiveHost { + self.proxy.effective_host() + } + + #[must_use] + #[inline] + pub fn effective_scheme(&self) -> Scheme { + self.proxy.effective_scheme() + } + + #[must_use] + #[inline] + pub fn client_source(&self) -> CheckedHostSource { + self.proxy.client_source() + } + + #[must_use] + #[inline] + pub fn host_source(&self) -> CheckedHostSource { + self.proxy.effective_host().source() + } + + #[must_use] + #[inline] + pub fn scheme_source(&self) -> CheckedHostSource { + self.proxy.scheme_source() + } + + #[must_use] + #[inline] + pub fn forwarding_result(&self) -> ForwardingResult { + self.proxy.forwarding_result() + } + + #[must_use] + #[inline] + pub fn received_normalized_head(&self) -> &NormalizedIngressHeadSummary { + &self.received + } +} + +/// One coherent, momentary observation of managed guards, not kernel or detached work. +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub struct NativeDiagnosticsSnapshot { + pub active_requests: u64, + /// Includes idle keep-alive connection tasks. + pub active_connections: u64, + pub idle_connections: u64, +} + +pub trait NativeRequestObserver: Send + Sync + 'static { + fn observe(&self, record: &NativeRequestRecord); +} + +/// Sharing this handle explicitly aggregates counters across managed runners. +#[derive(Clone, Default)] +pub struct NativeDiagnosticsHandle { + counters: Arc>, + observer: Option>, +} + +impl NativeDiagnosticsHandle { + #[must_use] + #[inline] + pub fn new() -> Self { + Self::default() + } + + #[must_use] + #[inline] + pub fn with_request_observer(mut self, observer: O) -> Self + where + O: NativeRequestObserver, + { + self.observer = Some(Arc::new(observer)); + self + } + + #[must_use] + #[inline] + pub fn snapshot(&self) -> NativeDiagnosticsSnapshot { + *lock(&self.counters) + } +} + +impl fmt::Debug for NativeDiagnosticsHandle { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("NativeDiagnosticsHandle") + .field("snapshot", &self.snapshot()) + .finish_non_exhaustive() + } +} + +/// A no-response disposition, deliberately distinct from response-egress outcomes. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum IngressDisposition { + Aborted, + Draining, + Abandoned, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum NativeRequestOutcome { + Egress(ResponseEgressOutcome), + Ingress(IngressDisposition), +} + +/// Categories captured by owning framework boundaries, never error messages or sources. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum NativeRequestFailure { + FrameworkRouting { kind: &'static str }, + PropagatedError { kind: &'static str }, + FallbackBodyExceeded, + FallbackReadTimedOut, + AdmissionRefused, +} + +/// Allowlisted observation. Bytes and selected status do not establish client receipt. +#[derive(Clone, Debug)] +pub struct NativeRequestRecord { + pub event: &'static str, + pub request_id: RequestId, + pub method: &'static str, + /// Registered template only, or the fixed `unknown` value. + pub route: String, + pub status: Option, + /// Absent when the owning monotonic clock moved backwards. + pub duration: Option, + pub outcome: NativeRequestOutcome, + pub failure: Option, + pub forwarding_result: ForwardingResult, + pub bytes_written: Option, + pub body_kind: Option, + pub fallback_disposition: Option, +} + +impl NativeRequestRecord { + /// Severity follows typed ownership, never status alone. + #[must_use] + #[inline] + pub fn level(&self) -> log::Level { + if self.duration.is_none() { + return log::Level::Error; + } + let ordinary_failure = matches!( + self.failure, + None | Some(NativeRequestFailure::FrameworkRouting { .. }) + ); + let ordinary_outcome = matches!( + self.outcome, + NativeRequestOutcome::Egress( + ResponseEgressOutcome::Completed | ResponseEgressOutcome::HostHandoff + ) | NativeRequestOutcome::Ingress(IngressDisposition::Draining) + ); + if ordinary_failure + && ordinary_outcome + && self.forwarding_result != ForwardingResult::Malformed + { + log::Level::Info + } else { + log::Level::Warn + } + } +} + +#[derive(Clone)] +pub(crate) struct NativeSession { + identity: Arc, + handle: NativeDiagnosticsHandle, + request_records: bool, +} + +struct SessionIdentity { + nonce: [u8; 16], + host: u64, + requests: AtomicU64, +} + +static PROCESS_NONCE: OnceLock> = OnceLock::new(); +static HOST_SEQUENCE: AtomicU64 = AtomicU64::new(0); + +#[derive(Clone, Copy, Debug, thiserror::Error)] +enum IdentityFault { + #[error("native_identity_entropy_failed")] + Entropy, + #[error("native_identity_host_sequence_exhausted")] + HostSequence, + #[error("native_identity_request_sequence_exhausted")] + RequestSequence, + #[error("native_diagnostics_connection_scope_mismatch")] + ConnectionScope, +} + +impl NativeSession { + pub(crate) fn new( + handle: NativeDiagnosticsHandle, + request_records: bool, + ) -> anyhow::Result { + let nonce = *PROCESS_NONCE.get_or_init(|| nonce_from(getrandom::fill)); + Self::from_sources(handle, request_records, nonce, &HOST_SEQUENCE).map_err(Into::into) + } + + fn from_sources( + handle: NativeDiagnosticsHandle, + request_records: bool, + nonce: Result<[u8; 16], IdentityFault>, + hosts: &AtomicU64, + ) -> Result { + Ok(Self { + identity: Arc::new(SessionIdentity { + nonce: nonce?, + host: next_sequence(hosts).ok_or(IdentityFault::HostSequence)?, + requests: AtomicU64::new(0), + }), + handle, + request_records, + }) + } + + /// Construct before submitting the owned task, including tasks never polled. + pub(crate) fn connection(&self) -> (ConnectionActivity, ConnectionGuard) { + let activity = ConnectionActivity { + state: Arc::new(Mutex::new(ConnectionState { + requests: 0, + closed: false, + })), + counters: Arc::clone(&self.handle.counters), + }; + let mut counts = lock(&activity.counters); + // Metrics must not introduce a panic in managed guard ownership. + counts.active_connections = counts.active_connections.saturating_add(1); + counts.idle_connections = counts.idle_connections.saturating_add(1); + drop(counts); + let guard = ConnectionGuard { + activity: activity.clone(), + }; + (activity, guard) + } + + pub(crate) fn request( + &self, + method: &Method, + start: MonotonicInstant, + clock: MonotonicClock, + activity: Option, + ) -> anyhow::Result { + if activity + .as_ref() + .is_some_and(|link| !Arc::ptr_eq(&link.counters, &self.handle.counters)) + { + return Err(IdentityFault::ConnectionScope.into()); + } + let Some(sequence) = next_sequence(&self.identity.requests) else { + isolated( + || log::error!(target: "edgezero::native", "lifecycle reason=native_identity_request_sequence_exhausted"), + ); + return Err(IdentityFault::RequestSequence.into()); + }; + if let Some(link) = &activity { + let mut connection = lock(&link.state); + let mut counts = lock(&link.counters); + if !connection.closed { + if connection.requests == 0 { + counts.idle_connections = counts.idle_connections.saturating_sub(1); + } + connection.requests = connection.requests.saturating_add(1); + } + counts.active_requests = counts.active_requests.saturating_add(1); + } else { + let mut counts = lock(&self.handle.counters); + counts.active_requests = counts.active_requests.saturating_add(1); + } + Ok(RequestGuard { + session: self.clone(), + id: RequestId { + nonce: self.identity.nonce, + host: self.identity.host, + request: sequence, + }, + method: method_category(method), + start, + clock, + activity, + forwarding_result: ForwardingResult::Absent, + failure: None, + released: false, + }) + } +} + +fn nonce_from(fill: impl FnOnce(&mut [u8]) -> Result<(), E>) -> Result<[u8; 16], IdentityFault> { + let mut nonce = [0; 16]; + fill(&mut nonce).map_err(|_entropy_error| IdentityFault::Entropy)?; + Ok(nonce) +} + +fn next_sequence(sequence: &AtomicU64) -> Option { + sequence + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |value| { + value.checked_add(1) + }) + .ok() +} + +struct ConnectionState { + requests: u64, + closed: bool, +} + +#[derive(Clone)] +pub(crate) struct ConnectionActivity { + state: Arc>, + counters: Arc>, +} + +pub(crate) struct ConnectionGuard { + activity: ConnectionActivity, +} + +impl Drop for ConnectionGuard { + fn drop(&mut self) { + let mut connection = lock(&self.activity.state); + let mut counts = lock(&self.activity.counters); + connection.closed = true; + counts.active_connections = counts.active_connections.saturating_sub(1); + if connection.requests == 0 { + counts.idle_connections = counts.idle_connections.saturating_sub(1); + } + } +} + +#[derive(Clone, Default)] +pub(crate) struct SelectedStatus(Arc); + +impl SelectedStatus { + pub(crate) fn select(&self, status: StatusCode) { + self.0.store(status.as_u16(), Ordering::Relaxed); + } + + fn get(&self) -> Option { + StatusCode::from_u16(self.0.load(Ordering::Relaxed)).ok() + } +} + +pub(crate) struct RequestGuard { + session: NativeSession, + id: RequestId, + method: &'static str, + start: MonotonicInstant, + clock: MonotonicClock, + activity: Option, + forwarding_result: ForwardingResult, + failure: Option, + released: bool, +} + +impl RequestGuard { + pub(crate) fn request_id(&self) -> RequestId { + self.id + } + + pub(crate) fn set_forwarding_result(&mut self, result: ForwardingResult) { + self.forwarding_result = result; + } + + pub(crate) fn set_failure(&mut self, failure: Option) { + self.failure = failure.map(|classification| match classification.kind() { + ResponseEgressFailureKind::PropagatedError => { + let kind = classification.error_kind().unwrap_or("unknown"); + if classification.is_framework_route_error() { + NativeRequestFailure::FrameworkRouting { kind } + } else { + NativeRequestFailure::PropagatedError { kind } + } + } + ResponseEgressFailureKind::FallbackBodyExceeded => { + NativeRequestFailure::FallbackBodyExceeded + } + ResponseEgressFailureKind::FallbackReadTimedOut => { + NativeRequestFailure::FallbackReadTimedOut + } + _ => NativeRequestFailure::PropagatedError { kind: "unknown" }, + }); + } + + pub(crate) fn set_ingress_error_kind(&mut self, kind: &'static str) { + self.failure = Some(NativeRequestFailure::PropagatedError { kind }); + } + + pub(crate) fn refused(&mut self) { + self.failure = Some(NativeRequestFailure::AdmissionRefused); + } + + pub(crate) fn completion(mut self) -> (ResponseEgressCompletion, SelectedStatus) { + let selected = SelectedStatus::default(); + let captured = selected.clone(); + let completion = + ResponseEgressCompletion::new_with_terminal_time(move |report, terminal| { + self.release(); + let record = self.record( + NativeRequestOutcome::Egress(report.outcome), + terminal, + captured.get(), + Some(report), + ); + self.emit(&record); + }); + (completion, selected) + } + + pub(crate) fn finish_ingress(mut self, disposition: IngressDisposition) { + self.finish_ingress_at(disposition, self.clock.now()); + } + + fn finish_ingress_at(&mut self, disposition: IngressDisposition, terminal: MonotonicInstant) { + self.release(); + let record = self.record( + NativeRequestOutcome::Ingress(disposition), + terminal, + None, + None, + ); + self.emit(&record); + } + + fn record( + &self, + outcome: NativeRequestOutcome, + terminal: MonotonicInstant, + status: Option, + report: Option<&ResponseEgressReport>, + ) -> NativeRequestRecord { + NativeRequestRecord { + event: if report.is_some() { + "request_terminal" + } else { + "request_ingress" + }, + request_id: self.id, + method: self.method, + route: report + .and_then(|terminal_report| terminal_report.route.as_ref()) + .map_or_else(|| "unknown".into(), |route| route.pattern().to_owned()), + status, + duration: terminal.checked_duration_since(self.start), + outcome, + failure: self.failure, + forwarding_result: self.forwarding_result, + bytes_written: report.map(|terminal_report| terminal_report.bytes_written), + body_kind: report.map(|terminal_report| terminal_report.body_kind), + fallback_disposition: report + .and_then(|terminal_report| terminal_report.fallback_disposition), + } + } + + fn emit(&self, record: &NativeRequestRecord) { + emit_with( + record, + self.session.request_records, + |safe_record| log::log!(target: "edgezero::native", safe_record.level(), "{}", SafeRecord(safe_record)), + self.session.handle.observer.as_deref(), + ); + } + + fn release(&mut self) { + if self.released { + return; + } + self.released = true; + if let Some(activity) = &self.activity { + let mut connection = lock(&activity.state); + let mut counts = lock(&activity.counters); + counts.active_requests = counts.active_requests.saturating_sub(1); + if !connection.closed { + connection.requests = connection.requests.saturating_sub(1); + if connection.requests == 0 { + counts.idle_connections = counts.idle_connections.saturating_add(1); + } + } + } else { + let mut counts = lock(&self.session.handle.counters); + counts.active_requests = counts.active_requests.saturating_sub(1); + } + } +} + +impl Drop for RequestGuard { + fn drop(&mut self) { + if !self.released { + // No report/status for cancellation or an unbegun envelope's abandonment. + self.release(); + isolated(|| self.finish_ingress_at(IngressDisposition::Abandoned, self.clock.now())); + } + } +} + +fn lock(mutex: &Mutex) -> MutexGuard<'_, T> { + mutex.lock().unwrap_or_else(PoisonError::into_inner) +} + +fn isolated(action: impl FnOnce()) { + drop(catch_unwind(AssertUnwindSafe(action))); +} + +fn emit_with( + record: &NativeRequestRecord, + enabled: bool, + log_sink: impl FnOnce(&NativeRequestRecord), + observer: Option<&dyn NativeRequestObserver>, +) { + if enabled { + isolated(|| log_sink(record)); + } + if let Some(sink) = observer { + isolated(|| sink.observe(record)); + } +} + +fn method_category(method: &Method) -> &'static str { + match *method { + Method::GET => "GET", + Method::HEAD => "HEAD", + Method::POST => "POST", + Method::PUT => "PUT", + Method::DELETE => "DELETE", + Method::CONNECT => "CONNECT", + Method::OPTIONS => "OPTIONS", + Method::TRACE => "TRACE", + Method::PATCH => "PATCH", + _ => "other", + } +} + +struct SafeRecord<'record>(&'record NativeRequestRecord); + +impl fmt::Display for SafeRecord<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let record = self.0; + write!( + f, + "{} request_id={} method={} route=", + record.event, record.request_id, record.method + )?; + write!(f, "\"{}\"", record.route.escape_debug())?; + match record.status { + Some(status) => write!(f, " status={}", status.as_u16())?, + None => f.write_str(" status=none")?, + } + match record.duration { + Some(duration) => write!(f, " duration_us={}", duration.as_micros())?, + None => f.write_str(" duration_us=none clock_fault=backwards")?, + } + write!( + f, + " outcome={} failure={} forwarding_result={}", + outcome_name(record.outcome), + failure_name(record.failure), + record.forwarding_result.as_str() + )?; + if let Some(bytes) = record.bytes_written { + write!(f, " bytes_written={bytes}")?; + } + if let Some(kind) = record.body_kind { + f.write_str(match kind { + ResponseEgressBodyKind::Application => " body_kind=application", + ResponseEgressBodyKind::Fallback => " body_kind=fallback", + _ => " body_kind=unknown", + })?; + } + if let Some(disposition) = record.fallback_disposition { + f.write_str(match disposition { + ResponseEgressFallbackDisposition::Aborted => " fallback_disposition=aborted", + ResponseEgressFallbackDisposition::Completed => " fallback_disposition=completed", + _ => " fallback_disposition=unknown", + })?; + } + Ok(()) + } +} + +fn outcome_name(outcome: NativeRequestOutcome) -> &'static str { + match outcome { + NativeRequestOutcome::Ingress(IngressDisposition::Aborted) => "ingress_aborted", + NativeRequestOutcome::Ingress(IngressDisposition::Draining) => "ingress_draining", + NativeRequestOutcome::Ingress(IngressDisposition::Abandoned) => "ingress_abandoned", + NativeRequestOutcome::Egress(egress_outcome) => match egress_outcome { + ResponseEgressOutcome::ClientDisconnected => "client_disconnected", + ResponseEgressOutcome::Completed => "completed", + ResponseEgressOutcome::ConversionError => "conversion_error", + ResponseEgressOutcome::DeadlineExceeded => "deadline_exceeded", + ResponseEgressOutcome::HostHandoff => "host_handoff", + ResponseEgressOutcome::RequestCancelled => "request_cancelled", + ResponseEgressOutcome::SourceError => "source_error", + ResponseEgressOutcome::TransportError => "transport_error", + ResponseEgressOutcome::Unspecified => "unspecified", + _ => "unknown", + }, + } +} + +fn failure_name(failure: Option) -> &'static str { + match failure { + None => "none", + Some( + NativeRequestFailure::FrameworkRouting { kind } + | NativeRequestFailure::PropagatedError { kind }, + ) => kind, + Some(NativeRequestFailure::FallbackBodyExceeded) => "fallback_body_exceeded", + Some(NativeRequestFailure::FallbackReadTimedOut) => "fallback_read_timed_out", + Some(NativeRequestFailure::AdmissionRefused) => "admission_refused", + } +} + +#[cfg(test)] +mod tests { + use std::collections::HashSet; + use std::iter::repeat_with; + use std::sync::atomic::AtomicUsize; + use std::thread; + + use edgezero_core::http::{HeaderMap, Version}; + use edgezero_core::response_egress::{ + ResponseEgressAttempt, ResponseEgressHead, ResponseEgressObserverHandle, + }; + use edgezero_core::router::RouteMetadata; + + use super::*; + + #[derive(Clone, Default)] + struct RecordingObserver { + records: Arc>>, + snapshots: Arc>>, + handle: NativeDiagnosticsHandle, + panic: bool, + } + + impl NativeRequestObserver for RecordingObserver { + fn observe(&self, record: &NativeRequestRecord) { + lock(&self.records).push(record.clone()); + lock(&self.snapshots).push(self.handle.snapshot()); + assert!(!self.panic, "observer fixture panic"); + } + } + + fn session(handle: NativeDiagnosticsHandle, enabled: bool) -> NativeSession { + NativeSession::from_sources(handle, enabled, Ok([0x19; 16]), &AtomicU64::new(0)) + .expect("test identity") + } + + fn observed_session(enabled: bool) -> (NativeSession, RecordingObserver) { + let observer = RecordingObserver::default(); + let handle = observer + .handle + .clone() + .with_request_observer(observer.clone()); + (session(handle, enabled), observer) + } + + fn request(session: &NativeSession, activity: Option) -> RequestGuard { + session + .request( + &Method::GET, + MonotonicInstant::now(), + MonotonicClock::default(), + activity, + ) + .expect("request") + } + + fn attempt( + completion: ResponseEgressCompletion, + start: MonotonicInstant, + egress_start: MonotonicInstant, + clock: MonotonicClock, + route: Option<&RouteMetadata>, + ) -> ResponseEgressAttempt { + let headers = HeaderMap::new(); + let head = ResponseEgressHead::new( + StatusCode::OK, + Version::HTTP_11, + &headers, + None, + start, + route, + ); + ResponseEgressAttempt::new( + &head, + egress_start, + completion, + ResponseEgressObserverHandle::default(), + clock, + ) + } + + #[test] + fn identity_is_fixed_hex_and_sequences_are_shared_only_with_session_clones() { + let hosts = AtomicU64::new(0); + let first = NativeSession::from_sources( + NativeDiagnosticsHandle::new(), + false, + Ok([0xab; 16]), + &hosts, + ) + .unwrap(); + let second = NativeSession::from_sources( + NativeDiagnosticsHandle::new(), + false, + Ok([0xab; 16]), + &hosts, + ) + .unwrap(); + let ids = [ + request(&first, None).request_id(), + request(&first.clone(), None).request_id(), + request(&second, None).request_id(), + ]; + assert_eq!(ids.into_iter().collect::>().len(), 3); + assert_eq!( + ids[0].to_string(), + format!("{}{:016x}{:016x}", "ab".repeat(16), 0_u64, 0_u64) + ); + assert_eq!(ids[1].request, 1); + assert_eq!(ids[2].host, 1); + assert_eq!(ids[2].request, 0); + for id in ids { + let rendered = id.to_string(); + assert_eq!(rendered.len(), 64); + assert!(rendered.bytes().all(|byte| byte.is_ascii_hexdigit())); + assert_eq!(format!("{id:?}"), rendered); + } + } + + #[test] + fn entropy_and_host_exhaustion_fail_without_fallback_or_wrap() { + let nonce = nonce_from(|bytes| { + bytes.fill(0x12); + Ok::<_, ()>(()) + }) + .unwrap(); + assert_eq!(nonce, [0x12; 16]); + let failure = nonce_from(|_| Err("secret entropy source")); + assert_eq!( + failure.unwrap_err().to_string(), + "native_identity_entropy_failed" + ); + let hosts = AtomicU64::new(u64::MAX - 1); + let first = + NativeSession::from_sources(NativeDiagnosticsHandle::new(), false, Ok(nonce), &hosts) + .unwrap(); + assert_eq!(first.identity.host, u64::MAX - 1); + let error = + NativeSession::from_sources(NativeDiagnosticsHandle::new(), false, Ok(nonce), &hosts) + .err() + .unwrap(); + assert_eq!(error.to_string(), "native_identity_host_sequence_exhausted"); + assert_eq!(hosts.load(Ordering::Relaxed), u64::MAX); + let unused_hosts = AtomicU64::new(0); + assert!( + NativeSession::from_sources( + NativeDiagnosticsHandle::new(), + false, + failure, + &unused_hosts + ) + .is_err() + ); + assert_eq!(unused_hosts.load(Ordering::Relaxed), 0); + let other = NativeSession::from_sources( + NativeDiagnosticsHandle::new(), + false, + Ok([0x13; 16]), + &AtomicU64::new(0), + ) + .unwrap(); + assert_ne!( + request(&first, None).request_id(), + request(&other, None).request_id() + ); + } + + #[test] + fn request_exhaustion_preserves_counters_and_never_reuses_an_id() { + let session = session(NativeDiagnosticsHandle::new(), false); + session + .identity + .requests + .store(u64::MAX - 1, Ordering::Relaxed); + let guard = request(&session, None); + assert_eq!(guard.request_id().request, u64::MAX - 1); + drop(guard); + assert!( + session + .request( + &Method::GET, + MonotonicInstant::now(), + MonotonicClock::default(), + None + ) + .is_err() + ); + assert_eq!(session.identity.requests.load(Ordering::Relaxed), u64::MAX); + assert_eq!( + session.handle.snapshot(), + NativeDiagnosticsSnapshot::default() + ); + } + + #[test] + fn concurrent_requests_share_checked_sequence_without_collisions() { + let session = session(NativeDiagnosticsHandle::new(), false); + let workers: Vec<_> = repeat_with(|| { + let worker_session = session.clone(); + thread::spawn(move || { + repeat_with(|| request(&worker_session, None).request_id()) + .take(64) + .collect::>() + }) + }) + .take(8) + .collect(); + let ids: HashSet<_> = workers + .into_iter() + .flat_map(|worker| worker.join().unwrap()) + .collect(); + assert_eq!(ids.len(), 512); + assert_eq!(session.handle.snapshot().active_requests, 0); + } + + #[test] + fn connection_counts_exist_before_polling_and_track_two_request_lifetimes() { + let session = session(NativeDiagnosticsHandle::new(), false); + let (activity, connection) = session.connection(); + assert_eq!( + session.handle.snapshot(), + NativeDiagnosticsSnapshot { + active_requests: 0, + active_connections: 1, + idle_connections: 1 + } + ); + let first = request(&session, Some(activity.clone())); + let second = request(&session, Some(activity)); + assert_eq!( + session.handle.snapshot(), + NativeDiagnosticsSnapshot { + active_requests: 2, + active_connections: 1, + idle_connections: 0 + } + ); + drop(first); + assert_eq!( + session.handle.snapshot(), + NativeDiagnosticsSnapshot { + active_requests: 1, + active_connections: 1, + idle_connections: 0 + } + ); + drop(second); + assert_eq!( + session.handle.snapshot(), + NativeDiagnosticsSnapshot { + active_requests: 0, + active_connections: 1, + idle_connections: 1 + } + ); + // No future needs to have been polled for task cancellation to release this guard. + drop(connection); + assert_eq!( + session.handle.snapshot(), + NativeDiagnosticsSnapshot::default() + ); + } + + #[test] + fn closed_connection_late_release_cannot_recreate_idle_connections() { + let session = session(NativeDiagnosticsHandle::new(), false); + let (activity, connection) = session.connection(); + let first = request(&session, Some(activity.clone())); + drop(connection); + let late = request(&session, Some(activity)); + assert_eq!( + session.handle.snapshot(), + NativeDiagnosticsSnapshot { + active_requests: 2, + active_connections: 0, + idle_connections: 0 + } + ); + drop(first); + drop(late); + assert_eq!( + session.handle.snapshot(), + NativeDiagnosticsSnapshot::default() + ); + } + + #[test] + fn explicit_handle_sharing_aggregates_connections_and_requests() { + let handle = NativeDiagnosticsHandle::new(); + let first = session(handle.clone(), false); + let second = session(handle.clone(), false); + let (_, connection) = first.connection(); + let guard = request(&second, None); + assert_eq!( + handle.snapshot(), + NativeDiagnosticsSnapshot { + active_requests: 1, + active_connections: 1, + idle_connections: 1 + } + ); + drop((connection, guard)); + assert_eq!(handle.snapshot(), NativeDiagnosticsSnapshot::default()); + } + + #[test] + fn activity_from_a_different_handle_fails_without_incrementing_either_scope() { + let first = session(NativeDiagnosticsHandle::new(), false); + let second = session(NativeDiagnosticsHandle::new(), false); + let (activity, connection) = first.connection(); + assert!( + second + .request( + &Method::GET, + MonotonicInstant::now(), + MonotonicClock::default(), + Some(activity) + ) + .is_err() + ); + assert_eq!(first.handle.snapshot().active_requests, 0); + assert_eq!( + second.handle.snapshot(), + NativeDiagnosticsSnapshot::default() + ); + drop(connection); + } + + #[test] + fn ingress_and_unbegun_completion_abandonment_have_no_status_or_report_fields() { + let (session, observer) = observed_session(false); + request(&session, None).finish_ingress(IngressDisposition::Aborted); + request(&session, None).finish_ingress(IngressDisposition::Draining); + let (completion, status) = request(&session, None).completion(); + status.select(StatusCode::OK); + drop(completion); + let records = lock(&observer.records); + assert_eq!(records.len(), 3); + assert_eq!( + records[0].outcome, + NativeRequestOutcome::Ingress(IngressDisposition::Aborted) + ); + assert_eq!( + records[1].outcome, + NativeRequestOutcome::Ingress(IngressDisposition::Draining) + ); + assert_eq!( + records[2].outcome, + NativeRequestOutcome::Ingress(IngressDisposition::Abandoned) + ); + for record in records.iter() { + assert_eq!(record.event, "request_ingress"); + assert_eq!(record.status, None); + assert_eq!(record.bytes_written, None); + assert_eq!(record.body_kind, None); + assert_eq!(record.fallback_disposition, None); + } + assert!( + lock(&observer.snapshots) + .iter() + .all(|snapshot| snapshot.active_requests == 0) + ); + } + + #[test] + fn authoritative_terminal_time_excludes_prior_app_callback_and_sink_delays() { + let (session, observer) = observed_session(false); + let start = MonotonicInstant::now(); + let egress_start = start + Duration::from_secs(7); + let terminal = start + Duration::from_secs(11); + let now = Arc::new(Mutex::new(terminal)); + let source = Arc::clone(&now); + let clock = MonotonicClock::new(move || *lock(&source)); + let guard = session + .request(&Method::GET, start, clock.clone(), None) + .unwrap(); + let private_id = guard.request_id(); + let (native, status) = guard.completion(); + status.select(StatusCode::OK); + let app_source = Arc::clone(&now); + let app_reports = Arc::new(Mutex::new(Vec::new())); + let captured_reports = Arc::clone(&app_reports); + let app = ResponseEgressCompletion::new(move |report| { + lock(&captured_reports).push(report.clone()); + *lock(&app_source) = terminal + Duration::from_secs(100); + }); + let route = RouteMetadata::new(Method::GET, "/orders/{id}"); + let mut attempt = attempt(app.join(native), start, egress_start, clock, Some(&route)); + assert!(attempt.begin_writing()); + assert!(attempt.account_bytes(9, terminal)); + assert!(attempt.complete(terminal)); + assert!(!attempt.complete(terminal)); + drop(attempt); + let records = lock(&observer.records); + assert_eq!(records.len(), 1); + let record = &records[0]; + assert_eq!(record.request_id, private_id); + assert_eq!(record.duration, Some(Duration::from_secs(11))); + assert_eq!(lock(&app_reports)[0].elapsed, Duration::from_secs(4)); + assert_eq!(record.status, Some(StatusCode::OK)); + assert_eq!(record.route, "/orders/{id}"); + assert_eq!(record.bytes_written, Some(9)); + assert_eq!( + record.outcome, + NativeRequestOutcome::Egress(ResponseEgressOutcome::Completed) + ); + assert_eq!(lock(&observer.snapshots)[0].active_requests, 0); + emit_with( + record, + true, + |_| *lock(&now) = terminal + Duration::from_secs(200), + None, + ); + assert_eq!(record.duration, Some(Duration::from_secs(11))); + } + + #[test] + fn failing_attempt_retains_selected_status_and_fallback_updates_use_same_slot() { + for fallback in [false, true] { + let (session, observer) = observed_session(false); + let start = MonotonicInstant::now(); + let (completion, status) = session + .request(&Method::GET, start, MonotonicClock::default(), None) + .unwrap() + .completion(); + status.select(StatusCode::OK); + let mut attempt = attempt(completion, start, start, MonotonicClock::default(), None); + if fallback { + assert!(attempt.begin_fallback(ResponseEgressOutcome::ConversionError)); + status.select(StatusCode::INTERNAL_SERVER_ERROR); + assert!(attempt.begin_writing()); + assert!( + attempt.finish_fallback(ResponseEgressFallbackDisposition::Completed, start) + ); + } else { + assert!(attempt.begin_writing()); + assert!(attempt.account_bytes(3, start)); + assert!(attempt.terminate(ResponseEgressOutcome::SourceError, start)); + } + drop(attempt); + let records = lock(&observer.records); + assert_eq!(records.len(), 1); + assert_eq!( + records[0].status, + Some(if fallback { + StatusCode::INTERNAL_SERVER_ERROR + } else { + StatusCode::OK + }) + ); + assert_eq!( + records[0].outcome, + NativeRequestOutcome::Egress(if fallback { + ResponseEgressOutcome::ConversionError + } else { + ResponseEgressOutcome::SourceError + }) + ); + assert_eq!(records[0].level(), log::Level::Warn); + assert_eq!(session.handle.snapshot().active_requests, 0); + } + } + + #[test] + fn drop_and_completion_release_before_panicking_observer() { + let observer = RecordingObserver { + panic: true, + ..RecordingObserver::default() + }; + let session = session( + observer + .handle + .clone() + .with_request_observer(observer.clone()), + false, + ); + catch_unwind(AssertUnwindSafe(|| drop(request(&session, None)))) + .expect("observer panic must not escape request Drop"); + let start = MonotonicInstant::now(); + let (completion, status) = request(&session, None).completion(); + status.select(StatusCode::OK); + let mut attempt = attempt(completion, start, start, MonotonicClock::default(), None); + attempt.begin_writing(); + assert!(attempt.complete(start + Duration::from_secs(1))); + drop(attempt); + assert_eq!(lock(&observer.records).len(), 2); + assert!( + lock(&observer.snapshots) + .iter() + .all(|snapshot| snapshot.active_requests == 0) + ); + assert_eq!(session.handle.snapshot().active_requests, 0); + } + + #[test] + fn sink_panics_are_independent_and_never_retry_logging() { + let (session, observer) = observed_session(false); + drop(request(&session, None)); + let record = lock(&observer.records)[0].clone(); + let logs = AtomicUsize::new(0); + emit_with( + &record, + true, + |_| { + logs.fetch_add(1, Ordering::Relaxed); + panic!("logger fixture panic"); + }, + Some(&observer), + ); + assert_eq!(logs.load(Ordering::Relaxed), 1); + assert_eq!(lock(&observer.records).len(), 2); + let panic_observer = RecordingObserver { + panic: true, + ..RecordingObserver::default() + }; + catch_unwind(AssertUnwindSafe(|| { + emit_with( + &record, + true, + |_| { + logs.fetch_add(1, Ordering::Relaxed); + panic!("logger fixture panic"); + }, + Some(&panic_observer), + ); + })) + .expect("logger and observer panics must be contained independently"); + assert_eq!(logs.load(Ordering::Relaxed), 2); + assert_eq!(lock(&panic_observer.records).len(), 1); + } + + #[test] + fn opt_out_suppresses_only_log_sink_and_keeps_observers_and_accounting() { + let (session, observer) = observed_session(false); + drop(request(&session, None)); + let record = lock(&observer.records)[0].clone(); + emit_with( + &record, + false, + |_| panic!("disabled log sink must not run"), + Some(&observer), + ); + assert_eq!(lock(&observer.records).len(), 2); + assert_eq!( + session.handle.snapshot(), + NativeDiagnosticsSnapshot::default() + ); + } + + #[test] + fn severity_is_typed_not_status_and_extension_methods_never_leak() { + let (session, observer) = observed_session(false); + let start = MonotonicInstant::now(); + let extension = Method::from_bytes(b"SECRET-METHOD").unwrap(); + let (completion, status) = session + .request(&extension, start, MonotonicClock::default(), None) + .unwrap() + .completion(); + status.select(StatusCode::PAYLOAD_TOO_LARGE); + let mut attempt = attempt(completion, start, start, MonotonicClock::default(), None); + attempt.begin_writing(); + attempt.complete(start); + let ordinary = lock(&observer.records)[0].clone(); + assert_eq!(ordinary.method, "other"); + assert_eq!(ordinary.level(), log::Level::Info); + assert!(!SafeRecord(&ordinary).to_string().contains("SECRET")); + let mut classified = ordinary.clone(); + classified.failure = Some(NativeRequestFailure::AdmissionRefused); + assert_eq!(classified.level(), log::Level::Warn); + classified.failure = Some(NativeRequestFailure::FrameworkRouting { kind: "not_found" }); + assert_eq!(classified.level(), log::Level::Info); + classified.failure = Some(NativeRequestFailure::PropagatedError { kind: "not_found" }); + assert_eq!(classified.level(), log::Level::Warn); + classified.failure = Some(NativeRequestFailure::FrameworkRouting { + kind: "method_not_allowed", + }); + assert_eq!(classified.level(), log::Level::Info); + classified.failure = Some(NativeRequestFailure::PropagatedError { + kind: "method_not_allowed", + }); + assert_eq!(classified.level(), log::Level::Warn); + classified.failure = Some(NativeRequestFailure::PropagatedError { kind: "internal" }); + assert_eq!(classified.level(), log::Level::Warn); + classified.failure = Some(NativeRequestFailure::FallbackBodyExceeded); + assert_eq!(classified.level(), log::Level::Warn); + classified.failure = Some(NativeRequestFailure::FallbackReadTimedOut); + assert_eq!(classified.level(), log::Level::Warn); + classified.failure = None; + classified.forwarding_result = ForwardingResult::Malformed; + assert_eq!(classified.level(), log::Level::Warn); + classified.duration = None; + assert_eq!(classified.level(), log::Level::Error); + } + + #[test] + fn backwards_clock_is_an_explicit_safe_fault_not_wall_time_or_zero_duration() { + let (session, observer) = observed_session(false); + let start = MonotonicInstant::now(); + let clock = MonotonicClock::new(move || start); + let guard = session + .request(&Method::GET, start + Duration::from_secs(1), clock, None) + .unwrap(); + guard.finish_ingress(IngressDisposition::Aborted); + let record = lock(&observer.records)[0].clone(); + assert_eq!(record.duration, None); + assert_eq!(record.level(), log::Level::Error); + assert!( + SafeRecord(&record) + .to_string() + .contains("clock_fault=backwards") + ); + assert_eq!(session.handle.snapshot().active_requests, 0); + } + + #[test] + fn metadata_preserves_original_summary_and_public_replacement_cannot_change_private_identity() { + use crate::proxy::{TrustedProxyPolicy, normalize, strip_forwarding_headers}; + use edgezero_core::http::{HeaderValue, Request}; + use edgezero_core::ingress::{ + IngressHeadLimits, validate_normalized_ingress_parts_with_summary, + }; + + let (session, observer) = observed_session(false); + let guard = request(&session, None); + let authoritative = guard.request_id(); + let other_guard = request(&session, None); + let mut received = Request::new(edgezero_core::Body::empty()); + *received.uri_mut() = "/orders/SECRET-PATH?token=SECRET-QUERY".parse().unwrap(); + received + .headers_mut() + .insert("host", HeaderValue::from_static("direct.example:8080")); + received.headers_mut().insert( + "forwarded", + HeaderValue::from_static("for=192.0.2.99;host=spoof.example;proto=https"), + ); + received.headers_mut().insert( + "x-request-id", + HeaderValue::from_static("SECRET-VISITOR-ID"), + ); + let (mut parts, _) = received.into_parts(); + let summary = + validate_normalized_ingress_parts_with_summary(&parts, IngressHeadLimits::default()) + .unwrap(); + let peer: SocketAddr = "192.0.2.1:4567".parse().unwrap(); + let checked = normalize(&TrustedProxyPolicy::no_trust(), Some(peer), &parts); + let metadata = NativeIngressMetadata::new(authoritative, checked.clone(), summary); + assert_eq!(metadata.request_id(), authoritative); + assert_eq!(metadata.direct_peer(), Some(peer)); + assert_eq!(metadata.effective_client(), Some(peer.ip())); + assert_eq!( + metadata.effective_host().authority(), + Some("direct.example:8080") + ); + assert_eq!(metadata.effective_scheme(), Scheme::Http); + assert_eq!(metadata.client_source(), CheckedHostSource::Direct); + assert_eq!(metadata.host_source(), CheckedHostSource::Direct); + assert_eq!(metadata.scheme_source(), CheckedHostSource::Direct); + assert_eq!( + metadata.forwarding_result(), + ForwardingResult::UntrustedPeer + ); + assert_eq!(*metadata.received_normalized_head(), summary); + strip_forwarding_headers(&mut parts.headers); + let stripped = + validate_normalized_ingress_parts_with_summary(&parts, IngressHeadLimits::default()) + .unwrap(); + assert!(summary.header_bytes() > stripped.header_bytes()); + assert_eq!(summary.header_count(), stripped.header_count() + 1); + assert_eq!(*metadata.received_normalized_head(), summary); + parts.extensions.insert(metadata); + // A caller can replace its public extension with another request's copy. + parts.extensions.insert(NativeIngressMetadata::new( + other_guard.request_id(), + checked, + summary, + )); + assert_ne!( + parts + .extensions + .get::() + .unwrap() + .request_id(), + authoritative + ); + guard.finish_ingress(IngressDisposition::Aborted); + let records = lock(&observer.records); + assert_eq!(records.len(), 1); + assert_eq!(records[0].request_id, authoritative); + let safe = SafeRecord(&records[0]).to_string(); + assert!(!safe.contains("SECRET")); + assert!(!safe.contains("example")); + drop(records); + drop(other_guard); + assert_eq!( + session.handle.snapshot(), + NativeDiagnosticsSnapshot::default() + ); + } + + #[test] + fn cancellation_unwind_releases_counts_even_if_observer_also_panics() { + let observer = RecordingObserver { + panic: true, + ..RecordingObserver::default() + }; + let session = session( + observer + .handle + .clone() + .with_request_observer(observer.clone()), + false, + ); + let (activity, connection) = session.connection(); + let unwind = catch_unwind(AssertUnwindSafe(|| { + let _guard = request(&session, Some(activity)); + panic!("cancel managed request fixture"); + })); + assert!(unwind.is_err()); + assert_eq!( + session.handle.snapshot(), + NativeDiagnosticsSnapshot { + active_requests: 0, + active_connections: 1, + idle_connections: 1 + } + ); + assert_eq!(lock(&observer.records).len(), 1); + drop(connection); + assert_eq!( + session.handle.snapshot(), + NativeDiagnosticsSnapshot::default() + ); + } + + #[test] + fn snapshots_remain_coherent_during_connection_request_updates() { + let handle = NativeDiagnosticsHandle::new(); + let session = session(handle.clone(), false); + let finished = Arc::new(AtomicUsize::new(0)); + let workers: Vec<_> = repeat_with(|| { + let worker_session = session.clone(); + let worker_finished = Arc::clone(&finished); + thread::spawn(move || { + for _ in 0_usize..128 { + let (activity, connection) = worker_session.connection(); + let first = request(&worker_session, Some(activity.clone())); + let second = request(&worker_session, Some(activity)); + drop(first); + drop(connection); + drop(second); + } + worker_finished.fetch_add(1, Ordering::Release); + }) + }) + .take(4) + .collect(); + while finished.load(Ordering::Acquire) < 4 { + let snapshot = handle.snapshot(); + assert!(snapshot.idle_connections <= snapshot.active_connections); + if snapshot.active_requests == 0 { + assert_eq!(snapshot.idle_connections, snapshot.active_connections); + } + thread::yield_now(); + } + for worker in workers { + worker.join().unwrap(); + } + assert_eq!(handle.snapshot(), NativeDiagnosticsSnapshot::default()); + } + + #[test] + fn metric_saturation_and_underflow_cannot_panic_during_guard_ownership() { + let saturated = session(NativeDiagnosticsHandle::new(), false); + // Inject impossible-scale telemetry to exercise arithmetic boundaries, + // without changing the checked identity sequence or real guard count. + *lock(&saturated.handle.counters) = NativeDiagnosticsSnapshot { + active_requests: u64::MAX, + active_connections: u64::MAX, + idle_connections: u64::MAX, + }; + let (_, saturated_connection) = saturated.connection(); + let saturated_request = request(&saturated, None); + assert_eq!(saturated.handle.snapshot().active_requests, u64::MAX); + assert_eq!(saturated.handle.snapshot().active_connections, u64::MAX); + assert_eq!(saturated.handle.snapshot().idle_connections, u64::MAX); + drop(saturated_request); + drop(saturated_connection); + assert_eq!(saturated.handle.snapshot().active_requests, u64::MAX - 1); + assert_eq!(saturated.handle.snapshot().active_connections, u64::MAX - 1); + assert_eq!(saturated.handle.snapshot().idle_connections, u64::MAX - 1); + + let underflow = session(NativeDiagnosticsHandle::new(), false); + let (activity, connection) = underflow.connection(); + let guard = request(&underflow, Some(activity)); + *lock(&underflow.handle.counters) = NativeDiagnosticsSnapshot::default(); + drop(guard); + drop(connection); + assert_eq!( + underflow.handle.snapshot(), + NativeDiagnosticsSnapshot::default() + ); + } + + #[test] + fn route_template_output_is_escaped_for_one_line() { + let (session, observer) = observed_session(false); + drop(request(&session, None)); + let mut record = lock(&observer.records)[0].clone(); + record.route = "/orders/{id}\n\r\t\"\\".into(); + let rendered = SafeRecord(&record).to_string(); + assert!(!rendered.contains(['\n', '\r', '\t'])); + assert!(rendered.contains("/orders/{id}\\n\\r\\t\\\"\\\\")); + } +} diff --git a/crates/edgezero-adapter-axum/src/geo.rs b/crates/edgezero-adapter-axum/src/geo.rs new file mode 100644 index 000000000..02aea65b6 --- /dev/null +++ b/crates/edgezero-adapter-axum/src/geo.rs @@ -0,0 +1,931 @@ +//! Explicit native Geo composition for local `GeoLite` databases and protected assertions. +//! +//! Local database selection works without `CloudFront`, AWS stores or credentials. +//! Select `geo-maxmind` to load an operator-supplied read-only snapshot before the +//! runner initializes. Request lookup uses only checked ingress and memory. +//! Database preparation is synchronous, before signal registration/stop polling. +//! +//! Direct-reader configuration, with no file I/O during capability inspection: +//! ```no_run +//! use edgezero_adapter_axum::dev_server::AxumRunOptions; +//! use edgezero_adapter_axum::geo::{MaxMindGeoConfig, NativeGeoSelection, ReaderIpExpectation}; +//! +//! let options = AxumRunOptions::new("0.0.0.0:8787".parse()?)? +//! .with_geo(NativeGeoSelection::MaxMind { +//! config: MaxMindGeoConfig::new( +//! "/var/lib/edgezero/geo/GeoLite2-City.mmdb", +//! ReaderIpExpectation::PeerAllowed, +//! ), +//! }); +//! let _configured = options.geo_capability()?; +//! # Ok::<(), anyhow::Error>(()) +//! ``` +//! +//! A reverse proxy must resolve the reader through the existing trusted policy: +//! ```no_run +//! use edgezero_adapter_axum::dev_server::AxumRunOptions; +//! use edgezero_adapter_axum::geo::{MaxMindGeoConfig, NativeGeoSelection, ReaderIpExpectation}; +//! use edgezero_adapter_axum::proxy::{ForwardingHeaderFamily, TrustedProxyPolicy}; +//! +//! let proxy = TrustedProxyPolicy::new( +//! ForwardingHeaderFamily::Forwarded, ["10.20.0.8/32"], +//! )?; +//! let options = AxumRunOptions::new("0.0.0.0:8787".parse()?)? +//! .with_trusted_proxy_policy(proxy) +//! .with_geo(NativeGeoSelection::MaxMind { +//! config: MaxMindGeoConfig::new( +//! "/var/lib/edgezero/geo/GeoLite2-City.mmdb", +//! ReaderIpExpectation::ForwardedRequired, +//! ), +//! }); +//! # Ok::<(), anyhow::Error>(()) +//! ``` +//! +//! The `CloudFront` policy declares external infrastructure protection. A matching socket peer +//! is not proof of an ALB credential rule or a particular `CloudFront` distribution. +//! See the native Geo guide for the required network, header and cache controls. +//! +//! Off is valid and is the default, with no ambient Geo selector: +//! ``` +//! use edgezero_adapter_axum::dev_server::AxumRunOptions; +//! use edgezero_adapter_axum::geo::{NativeGeoCapability, NativeGeoSelection}; +//! let options = AxumRunOptions::new("127.0.0.1:8787".parse()?)? +//! .with_geo(NativeGeoSelection::Off); +//! assert_eq!(options.geo_capability()?, NativeGeoCapability::Off); +//! # Ok::<(), anyhow::Error>(()) +//! ``` +//! +//! Capture capability from finalized options in the existing local initializer. +//! The selector compiles without `geo-cloudfront`, but startup then rejects it: +//! ```no_run +//! use edgezero_adapter_axum::dev_server::{AxumRunOptions, run_app_with_options_and_initializer}; +//! use edgezero_adapter_axum::geo::{NativeGeoSelection, ProtectedCloudFrontAlb}; +//! use edgezero_core::app::Hooks; +//! +//! fn run() -> anyhow::Result<()> { +//! let policy = ProtectedCloudFrontAlb::new( +//! ["10.42.1.0/24", "10.42.2.0/24"], "x-origin-verify-randomsuffix", +//! )?; +//! let options = AxumRunOptions::new("0.0.0.0:8787".parse()?)? +//! .with_geo(NativeGeoSelection::CloudFront { policy }); +//! let capability = options.geo_capability()?; +//! run_app_with_options_and_initializer::( +//! options, std::future::pending::<()>(), move |_app, _stores| { +//! Box::pin(async move { +//! let _configured_capability = capability; +//! // Validate application requirements here, without inferring +//! // that every request will have geographic metadata. +//! Ok(()) +//! }) +//! }, +//! ) +//! } +//! ``` +//! +//! Supplied providers work without the built-in feature and use the same bounded, +//! synchronous, local-only contract. Construction belongs to the application: +//! ``` +//! use std::sync::Arc; +//! use edgezero_adapter_axum::dev_server::AxumRunOptions; +//! use edgezero_adapter_axum::geo::{NativeGeoSelection, ProtectedCloudFrontAlb}; +//! use edgezero_core::geo::{GeoError, GeoInput, GeoProvider, GeoProviderId, GeoResolution}; +//! +//! struct ExampleProvider; +//! impl GeoProvider for ExampleProvider { +//! fn resolve(&self, input: &GeoInput) -> GeoResolution { +//! match input { +//! GeoInput::Assertions { .. } => GeoResolution::MissingMetadata, +//! GeoInput::ReaderIp { .. } => GeoResolution::Error(GeoError::ProviderFailure), +//! } +//! } +//! } +//! let policy = ProtectedCloudFrontAlb::new( +//! ["10.42.1.0/24"], "x-origin-verify-randomsuffix", +//! )?; +//! let options = AxumRunOptions::new("127.0.0.1:8787".parse()?)? +//! .with_geo(NativeGeoSelection::Supplied { +//! policy, +//! id: GeoProviderId::new("example.local")?, +//! provider: Arc::new(ExampleProvider), +//! }); +//! let _capability = options.geo_capability()?; +//! # Ok::<(), anyhow::Error>(()) +//! ``` +//! +//! Consumers distinguish every outcome. Even a located country/subdivision pair +//! is an estimate, and subdivision membership is not verified: +//! ``` +//! use edgezero_core::geo::{GeoError, GeoOutcome, GeoStatus, GeoUnavailableReason}; +//! fn describe(outcome: &GeoOutcome) -> &'static str { +//! match outcome { +//! GeoOutcome::NoSelection => "not configured", +//! GeoOutcome::Selected { status, .. } => match status { +//! GeoStatus::Unavailable(GeoUnavailableReason::UntrustedIngress) => "source not admitted", +//! GeoStatus::Unavailable(GeoUnavailableReason::MissingMetadata) => "metadata missing", +//! GeoStatus::Unavailable(GeoUnavailableReason::MissingReaderIp) => "reader unavailable", +//! GeoStatus::Unavailable(GeoUnavailableReason::NoMatch) => "no geographic match", +//! GeoStatus::Error(GeoError::InvalidAssertion) => "assertion rejected", +//! GeoStatus::Error(GeoError::ProviderFailure) => "provider failed", +//! GeoStatus::Located { location, .. } if location.subdivision().is_none() => "country only", +//! GeoStatus::Located { .. } => "country and unverified subdivision assertion", +//! }, +//! } +//! } +//! ``` + +#![expect( + clippy::arbitrary_source_item_ordering, + reason = "public composition types precede private request-bound resolution and tests" +)] +#![cfg_attr( + any(feature = "geo-cloudfront", feature = "geo-maxmind"), + expect( + clippy::self_named_module_files, + reason = "repo convention uses geo.rs with submodules under geo/, not mod.rs" + ) +)] + +#[cfg(feature = "geo-cloudfront")] +mod cloudfront; +#[cfg(feature = "geo-maxmind")] +mod maxmind; +#[cfg(test)] +#[path = "geo/reader_tests.rs"] +mod reader_tests; + +use std::fmt; +use std::net::SocketAddr; +use std::path::PathBuf; +use std::sync::Arc; + +use edgezero_core::geo::{ + GeoAssertion, GeoInput, GeoIpSource, GeoOutcome, GeoProvider, GeoProviderId, GeoResolution, + GeoSource, GeoStatus, GeoTrustBasis, GeoUnavailableReason, +}; +use edgezero_core::http::{HeaderMap, HeaderName}; +use edgezero_core::ingress::CheckedHostSource; + +use crate::peer_networks::PeerNetworks; +use crate::proxy::CheckedProxyMetadata; + +/// Bounded startup configuration categories. No policy inputs are retained. +#[derive(Clone, Copy, Debug, Eq, PartialEq, thiserror::Error)] +pub enum NativeGeoConfigError { + #[error("native_geo: invalid protected source policy")] + InvalidProtectedPolicy, + #[error("native_geo: invalid credential removal policy")] + InvalidCredentialHeader, + #[error("native_geo: cloudfront implementation unavailable")] + UnavailableCloudFront, + #[error("native_geo: reserved supplied provider identity")] + ReservedProviderId, + #[error("native_geo: maxmind implementation unavailable")] + UnavailableMaxMind, + #[error("native_geo: invalid database preparation limits")] + InvalidPreparationLimits, + #[error("native_geo: database unavailable")] + UnavailableDatabase, + #[error("native_geo: invalid database")] + InvalidDatabase, + #[error("native_geo: database preparation limit exceeded")] + DatabaseLimit, + #[error("native_geo: unsupported database")] + UnsupportedDatabase, + #[error("native_geo: incompatible database schema")] + InvalidDatabaseSchema, +} + +/// Whether checked ingress must explicitly resolve a reader through a proxy. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ReaderIpExpectation { + PeerAllowed, + ForwardedRequired, +} + +/// Configurable startup bounds. They do not impose an I/O deadline or RSS quota. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct MaxMindPreparationLimits { + pub max_database_bytes: usize, + pub max_validation_networks: usize, +} + +impl Default for MaxMindPreparationLimits { + #[inline] + fn default() -> Self { + Self { + max_database_bytes: 128 * 1024 * 1024, + max_validation_networks: 32_000_000, + } + } +} + +impl MaxMindPreparationLimits { + fn validate(self) -> Result<(), NativeGeoConfigError> { + if self.max_database_bytes == 0 + || self.max_database_bytes.checked_add(1).is_none() + || self.max_validation_networks == 0 + { + return Err(NativeGeoConfigError::InvalidPreparationLimits); + } + Ok(()) + } +} + +/// Operator-owned database location and explicit reader expectation. No I/O on construction. +#[derive(Clone)] +pub struct MaxMindGeoConfig { + pub path: PathBuf, + pub limits: MaxMindPreparationLimits, + pub expectation: ReaderIpExpectation, +} + +impl MaxMindGeoConfig { + #[inline] + #[must_use] + pub fn new>(path: P, expectation: ReaderIpExpectation) -> Self { + Self { + path: path.into(), + limits: MaxMindPreparationLimits::default(), + expectation, + } + } +} + +impl fmt::Debug for MaxMindGeoConfig { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("MaxMindGeoConfig") + .field("path", &"[redacted]") + .field("limits", &self.limits) + .field("expectation", &self.expectation) + .finish() + } +} + +/// An operator declaration of a credential-gated, isolated CloudFront/ALB path. +/// +/// This is distinct from ordinary proxy trust. The policy contains no credential +/// value. The ALB must validate the credential before forwarding to the target. +#[derive(Clone)] +pub struct ProtectedCloudFrontAlb { + networks: PeerNetworks, + credential_header: HeaderName, +} + +impl ProtectedCloudFrontAlb { + /// Validate a nonempty bounded literal peer allowlist and dedicated header name. + /// + /// Names must be literal HTTP tokens of at most 128 ASCII bytes beginning with + /// `x-`. Forwarding, AWS, `EdgeZero`, real-IP and correlation names are reserved. + /// Neither malformed names nor network inputs are repaired or logged. + /// + /// # Errors + /// Returns a fixed category for invalid networks or an unsafe header name. + #[inline] + pub fn new(raw_networks: I, credential_name: &str) -> Result + where + I: IntoIterator, + S: AsRef, + { + let networks = PeerNetworks::new(raw_networks) + .map_err(|_error| NativeGeoConfigError::InvalidProtectedPolicy)?; + if networks.is_empty() { + return Err(NativeGeoConfigError::InvalidProtectedPolicy); + } + if credential_name.len() > 128 || !credential_name.is_ascii() { + return Err(NativeGeoConfigError::InvalidCredentialHeader); + } + let normalized = credential_name.to_ascii_lowercase(); + if !normalized.starts_with("x-") + || ["x-forwarded-", "x-amz-", "x-amzn-", "x-edge-"] + .iter() + .any(|prefix| normalized.starts_with(prefix)) + || ["x-real-ip", "x-request-id", "x-correlation-id"].contains(&normalized.as_str()) + { + return Err(NativeGeoConfigError::InvalidCredentialHeader); + } + let credential_header = HeaderName::from_bytes(normalized.as_bytes()) + .map_err(|_error| NativeGeoConfigError::InvalidCredentialHeader)?; + Ok(Self { + networks, + credential_header, + }) + } + + fn admit<'request>( + &self, + peer: Option, + headers: &'request HeaderMap, + ) -> Option> { + peer.filter(|actual_peer| self.networks.trusts(actual_peer.ip())) + .map(|_peer| ProtectedAssertions { headers }) + } +} + +impl fmt::Debug for ProtectedCloudFrontAlb { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ProtectedCloudFrontAlb") + .field("network_count", &self.networks.len()) + .finish_non_exhaustive() + } +} + +/// One complete deployment selection. Repeated `with_geo` calls replace it. +#[derive(Clone, Default)] +pub enum NativeGeoSelection { + #[default] + Off, + CloudFront { + policy: ProtectedCloudFrontAlb, + }, + MaxMind { + config: MaxMindGeoConfig, + }, + SuppliedReaderIp { + expectation: ReaderIpExpectation, + id: GeoProviderId, + provider: Arc, + }, + Supplied { + policy: ProtectedCloudFrontAlb, + id: GeoProviderId, + provider: Arc, + }, +} + +impl fmt::Debug for NativeGeoSelection { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Off => f.write_str("Off"), + Self::CloudFront { policy } => f + .debug_struct("CloudFront") + .field("policy", policy) + .finish(), + Self::MaxMind { config } => f.debug_struct("MaxMind").field("config", config).finish(), + Self::SuppliedReaderIp { + expectation, id, .. + } => f + .debug_struct("SuppliedReaderIp") + .field("expectation", expectation) + .field("id", id) + .finish_non_exhaustive(), + Self::Supplied { policy, id, .. } => f + .debug_struct("Supplied") + .field("policy", policy) + .field("id", id) + .finish_non_exhaustive(), + } + } +} + +/// Configured input contract, without eventual request provenance or database readiness. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum NativeGeoInputContract { + ProtectedCloudFrontAlb, + ReaderIp { expectation: ReaderIpExpectation }, +} + +/// Owned startup configuration capability. Inspection performs no database I/O. +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum NativeGeoCapability { + Off, + Selected { + provider: GeoProviderId, + input: NativeGeoInputContract, + }, +} + +impl NativeGeoSelection { + pub(crate) fn capability(&self) -> Result { + let (provider, input) = match self { + Self::Off => return Ok(NativeGeoCapability::Off), + Self::CloudFront { .. } => { + if !cfg!(feature = "geo-cloudfront") { + return Err(NativeGeoConfigError::UnavailableCloudFront); + } + ( + GeoProviderId::cloudfront(), + NativeGeoInputContract::ProtectedCloudFrontAlb, + ) + } + Self::MaxMind { config } => { + config.limits.validate()?; + if !cfg!(all(feature = "geo-maxmind", unix)) { + return Err(NativeGeoConfigError::UnavailableMaxMind); + } + ( + GeoProviderId::maxmind(), + NativeGeoInputContract::ReaderIp { + expectation: config.expectation, + }, + ) + } + Self::Supplied { id, .. } | Self::SuppliedReaderIp { id, .. } => { + if id == &GeoProviderId::cloudfront() || id == &GeoProviderId::maxmind() { + return Err(NativeGeoConfigError::ReservedProviderId); + } + let input = if let Self::SuppliedReaderIp { expectation, .. } = self { + NativeGeoInputContract::ReaderIp { + expectation: *expectation, + } + } else { + NativeGeoInputContract::ProtectedCloudFrontAlb + }; + (id.clone(), input) + } + }; + Ok(NativeGeoCapability::Selected { provider, input }) + } +} + +/// Source admission remains framework-owned and independent of provider identity. +#[derive(Clone)] +pub(crate) enum NativeGeoAdmission { + Assertions(ProtectedCloudFrontAlb), + ReaderIp { + expectation: ReaderIpExpectation, + source: GeoSource, + }, +} + +/// Prepared runner-owned composition, never installed in public extensions. +#[derive(Clone, Default)] +pub(crate) enum NativeGeo { + #[default] + Off, + Selected { + admission: NativeGeoAdmission, + id: GeoProviderId, + provider: Arc, + #[cfg(feature = "geo-maxmind")] + snapshot: Option, + }, +} + +impl NativeGeo { + pub(crate) fn prepare(selection: &NativeGeoSelection) -> Result { + let _capability = selection.capability()?; + let (admission, id, provider): (_, _, Arc) = match selection + { + NativeGeoSelection::Off => return Ok(Self::Off), + #[cfg(feature = "geo-cloudfront")] + NativeGeoSelection::CloudFront { policy } => ( + NativeGeoAdmission::Assertions(policy.clone()), + GeoProviderId::cloudfront(), + Arc::new(cloudfront::CloudFrontProvider), + ), + #[cfg(not(feature = "geo-cloudfront"))] + NativeGeoSelection::CloudFront { .. } => { + return Err(NativeGeoConfigError::UnavailableCloudFront); + } + #[cfg(feature = "geo-maxmind")] + NativeGeoSelection::MaxMind { config } => { + let provider = maxmind::MaxMindProvider::prepare(config)?; + let snapshot = Some(provider.metadata.clone()); + return Ok(Self::Selected { + admission: NativeGeoAdmission::ReaderIp { + expectation: config.expectation, + source: GeoSource::LocalDatabase, + }, + id: GeoProviderId::maxmind(), + provider: Arc::new(provider), + snapshot, + }); + } + #[cfg(not(feature = "geo-maxmind"))] + NativeGeoSelection::MaxMind { .. } => { + return Err(NativeGeoConfigError::UnavailableMaxMind); + } + NativeGeoSelection::Supplied { + policy, + id, + provider, + } => ( + NativeGeoAdmission::Assertions(policy.clone()), + id.clone(), + Arc::clone(provider), + ), + NativeGeoSelection::SuppliedReaderIp { + expectation, + id, + provider, + } => ( + NativeGeoAdmission::ReaderIp { + expectation: *expectation, + source: GeoSource::CheckedReaderIp, + }, + id.clone(), + Arc::clone(provider), + ), + }; + Ok(Self::Selected { + admission, + id, + provider, + #[cfg(feature = "geo-maxmind")] + snapshot: None, + }) + } + + #[cfg_attr( + not(feature = "geo-maxmind"), + expect( + clippy::unused_self, + reason = "the stable runner hook has no snapshot when the built-in reader is absent" + ) + )] + pub(crate) fn log_snapshot(&self) { + #[cfg(feature = "geo-maxmind")] + if let Self::Selected { + snapshot: Some(snapshot), + .. + } = self + { + log::info!(target: "edgezero::native", "event=geo_database provider=maxmind database_type={} build_epoch={} sha256={}", snapshot.database_type, snapshot.build_epoch, snapshot.sha256); + } + } + + pub(crate) fn strip_credential(&self, headers: &mut HeaderMap) { + if let Self::Selected { + admission: NativeGeoAdmission::Assertions(policy), + .. + } = self + { + let _removed = headers.remove(&policy.credential_header); + } + } + + pub(crate) fn reader_input(&self, checked: &CheckedProxyMetadata) -> Option { + let Self::Selected { + admission: NativeGeoAdmission::ReaderIp { expectation, .. }, + .. + } = self + else { + return None; + }; + let _peer = checked.direct_peer()?; + let source = match checked.client_source() { + CheckedHostSource::Direct if *expectation == ReaderIpExpectation::PeerAllowed => { + GeoIpSource::DirectPeer + } + CheckedHostSource::TrustedForwarded => GeoIpSource::TrustedForwarded, + CheckedHostSource::TrustedXForwarded => GeoIpSource::TrustedXForwarded, + CheckedHostSource::Direct | CheckedHostSource::Unavailable | _ => return None, + }; + Some(GeoInput::ReaderIp { + address: checked.effective_client()?, + source, + }) + } + + pub(crate) fn resolve( + &self, + peer: Option, + headers: &HeaderMap, + reader: Option<&GeoInput>, + ) -> GeoOutcome { + let Self::Selected { + admission, + id, + provider, + .. + } = self + else { + return GeoOutcome::NoSelection; + }; + let admitted = match admission { + NativeGeoAdmission::Assertions(policy) => policy + .admit(peer, headers) + .map(|admitted| { + ( + admitted.input(), + GeoSource::CloudFront, + GeoTrustBasis::DeclaredProtectedAlb, + ) + }) + .ok_or(GeoUnavailableReason::UntrustedIngress), + NativeGeoAdmission::ReaderIp { source, .. } => match reader { + Some( + input @ GeoInput::ReaderIp { + source: reader_source, + .. + }, + ) => Ok(( + input.clone(), + *source, + GeoTrustBasis::CheckedReaderIp(*reader_source), + )), + _ => Err(GeoUnavailableReason::MissingReaderIp), + }, + }; + let status = match admitted { + Err(reason) => GeoStatus::Unavailable(reason), + Ok((input, source, trust_basis)) => match provider.resolve(&input) { + GeoResolution::MissingMetadata => { + GeoStatus::Unavailable(GeoUnavailableReason::MissingMetadata) + } + GeoResolution::NoMatch => GeoStatus::Unavailable(GeoUnavailableReason::NoMatch), + GeoResolution::Error(error) => GeoStatus::Error(error), + GeoResolution::Located(location) => GeoStatus::Located { + location, + source, + trust_basis, + }, + }, + }; + GeoOutcome::Selected { + provider: id.clone(), + status, + } + } +} + +/// Private witness tied to the admitted request's still-owned header snapshot. +struct ProtectedAssertions<'request> { + headers: &'request HeaderMap, +} + +impl ProtectedAssertions<'_> { + fn input(&self) -> GeoInput { + GeoInput::Assertions { + country: assertion(self.headers, "cloudfront-viewer-country"), + subdivision: assertion(self.headers, "cloudfront-viewer-country-region"), + } + } +} + +fn assertion(headers: &HeaderMap, name: &'static str) -> GeoAssertion { + let mut values = headers.get_all(name).iter(); + let Some(first) = values.next() else { + return GeoAssertion::absent(); + }; + if values.next().is_some() { + return GeoAssertion::invalid(); + } + GeoAssertion::from_bytes(first.as_bytes()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::run_options::AxumRunOptions; + use edgezero_core::geo::{CountryCode, GeoError, GeoLocation}; + use edgezero_core::http::HeaderValue; + use std::iter::empty; + use std::sync::atomic::{AtomicUsize, Ordering}; + + fn policy() -> ProtectedCloudFrontAlb { + ProtectedCloudFrontAlb::new(["127.0.0.1/32"], "x-origin-verify-fixture").unwrap() + } + + #[test] + fn geo_protected_policy_reuses_network_bounds() { + assert_eq!( + ProtectedCloudFrontAlb::new(empty::<&str>(), "x-origin-verify-fixture").unwrap_err(), + NativeGeoConfigError::InvalidProtectedPolicy + ); + ProtectedCloudFrontAlb::new(["127.0.0.1"; 256], "x-origin-verify-fixture").unwrap(); + assert_eq!( + ProtectedCloudFrontAlb::new(["127.0.0.1"; 257], "x-origin-verify-fixture").unwrap_err(), + NativeGeoConfigError::InvalidProtectedPolicy + ); + let exact = format!("127.0.0.1{}", " ".repeat(0x4000 - 9)); + ProtectedCloudFrontAlb::new([exact.as_str()], "x-origin-verify-fixture").unwrap(); + assert_eq!( + ProtectedCloudFrontAlb::new([format!("{exact} ")], "x-origin-verify-fixture") + .unwrap_err(), + NativeGeoConfigError::InvalidProtectedPolicy + ); + for ranges in [ + vec!["0.0.0.0/0"], + vec!["::/0"], + vec!["0.0.0.0/1", "128.0.0.0/1"], + vec!["::/1", "8000::/1"], + vec!["::ffff:0:0/96"], + vec!["not-a-network"], + ] { + assert_eq!( + ProtectedCloudFrontAlb::new(ranges, "x-origin-verify-fixture").unwrap_err(), + NativeGeoConfigError::InvalidProtectedPolicy + ); + } + let selected = ProtectedCloudFrontAlb::new( + ["192.0.2.0/24", "2001:db8::/32"], + "x-origin-verify-fixture", + ) + .unwrap(); + let headers = HeaderMap::new(); + for peer in ["192.0.2.1:80", "[2001:db8::1]:80", "[::ffff:192.0.2.1]:80"] { + assert!( + selected + .admit(Some(peer.parse().unwrap()), &headers) + .is_some() + ); + } + assert!(selected.admit(None, &headers).is_none()); + assert!( + selected + .admit(Some("192.0.3.1:80".parse().unwrap()), &headers) + .is_none() + ); + ProtectedCloudFrontAlb::new(["192.0.2.0/24"; 256], "x-origin-verify-fixture").unwrap(); + } + + #[test] + fn credential_names_are_dedicated_bounded_and_redacted() { + for name in [ + "authorization", + "cookie", + "host", + "forwarded", + "CloudFront-Viewer-Country", + "CloudFront-Viewer-Country-Region", + "X-FORWARDED-FOR", + "x-forwarded-host", + "X-Real-IP", + "X-Amz-Token", + "x-amzn-trace-id", + "x-edge-request", + "x-request-id", + "X-Correlation-ID", + "x-bad name", + "x-bad\r\nname", + " x-secret", + "x-secret ", + "x-\u{e9}", + "", + ] { + assert!( + ProtectedCloudFrontAlb::new(["127.0.0.1"], name).is_err(), + "{name:?}" + ); + } + let exact = format!("x-{}", "a".repeat(126)); + ProtectedCloudFrontAlb::new(["127.0.0.1"], &exact).unwrap(); + assert_eq!( + ProtectedCloudFrontAlb::new(["127.0.0.1"], &format!("{exact}a")).unwrap_err(), + NativeGeoConfigError::InvalidCredentialHeader + ); + let policy = + ProtectedCloudFrontAlb::new(["192.0.2.0/24"], "X-Origin-Private-Sentinel").unwrap(); + let debug = format!("{policy:?}"); + assert!(!debug.contains("192.0.2")); + assert!(!debug.to_ascii_lowercase().contains("sentinel")); + let error = + ProtectedCloudFrontAlb::new(["not-a-network-secret"], "x-private-secret").unwrap_err(); + assert!(!format!("{error:?} {error}").contains("secret")); + } + + struct CountingProvider { + calls: Arc, + result: GeoResolution, + } + impl GeoProvider for CountingProvider { + fn resolve(&self, _input: &GeoInput) -> GeoResolution { + self.calls.fetch_add(1, Ordering::SeqCst); + self.result.clone() + } + } + + #[test] + fn source_admission_precedes_header_inspection_and_provider_calls() { + let calls = Arc::new(AtomicUsize::new(0)); + let selected = NativeGeo::prepare(&NativeGeoSelection::Supplied { + policy: policy(), + id: GeoProviderId::new("counting").unwrap(), + provider: Arc::new(CountingProvider { + calls: Arc::clone(&calls), + result: GeoResolution::Error(GeoError::ProviderFailure), + }), + }) + .unwrap(); + let mut headers = HeaderMap::new(); + headers.append("cloudfront-viewer-country", HeaderValue::from_static("US")); + headers.append("cloudfront-viewer-country", HeaderValue::from_static("CA")); + for peer in [None, Some("192.0.2.1:80".parse().unwrap())] { + assert_eq!( + selected.resolve(peer, &headers, None), + GeoOutcome::Selected { + provider: GeoProviderId::new("counting").unwrap(), + status: GeoStatus::Unavailable(GeoUnavailableReason::UntrustedIngress) + } + ); + } + assert_eq!( + NativeGeo::Off.resolve(Some("127.0.0.1:80".parse().unwrap()), &headers, None), + GeoOutcome::NoSelection + ); + assert_eq!(calls.load(Ordering::SeqCst), 0); + for values in [&headers, &HeaderMap::new()] { + assert_eq!( + selected.resolve(Some("127.0.0.1:80".parse().unwrap()), values, None), + GeoOutcome::Selected { + provider: GeoProviderId::new("counting").unwrap(), + status: GeoStatus::Error(GeoError::ProviderFailure) + } + ); + } + assert_eq!(calls.load(Ordering::SeqCst), 2); + } + + #[test] + fn duplicate_and_oversize_assertions_are_invalid_before_copying() { + let mut headers = HeaderMap::new(); + headers.append("cloudfront-viewer-country", HeaderValue::from_static("US")); + headers.append("cloudfront-viewer-country", HeaderValue::from_static("US")); + assert_eq!( + assertion::<2>(&headers, "cloudfront-viewer-country").as_bytes(), + Err(GeoError::InvalidAssertion) + ); + headers.remove("cloudfront-viewer-country"); + headers.insert("cloudfront-viewer-country", HeaderValue::from_static("USA")); + assert_eq!( + assertion::<2>(&headers, "cloudfront-viewer-country").as_bytes(), + Err(GeoError::InvalidAssertion) + ); + headers.insert( + "cloudfront-viewer-country-region", + HeaderValue::from_static("ABCD"), + ); + assert_eq!( + assertion::<3>(&headers, "cloudfront-viewer-country-region").as_bytes(), + Err(GeoError::InvalidAssertion) + ); + } + + #[test] + fn credential_removal_strips_all_instances_and_preserves_geography() { + let selected = NativeGeo::prepare(&NativeGeoSelection::Supplied { + policy: policy(), + id: GeoProviderId::new("custom").unwrap(), + provider: Arc::new(CountingProvider { + calls: Arc::default(), + result: GeoResolution::MissingMetadata, + }), + }) + .unwrap(); + let mut headers = HeaderMap::new(); + headers.append("x-origin-verify-fixture", HeaderValue::from_static("first")); + headers.append( + "x-origin-verify-fixture", + HeaderValue::from_static("second"), + ); + headers.insert("cloudfront-viewer-country", HeaderValue::from_static("US")); + selected.strip_credential(&mut headers); + assert_eq!(headers.get_all("x-origin-verify-fixture").iter().count(), 0); + assert_eq!(headers["cloudfront-viewer-country"], "US"); + } + + #[test] + fn supplied_identity_cannot_claim_builtin_and_options_keep_thread_bounds() { + fn assert_bounds() {} + assert_bounds::(); + let error = NativeGeo::prepare(&NativeGeoSelection::Supplied { + policy: policy(), + id: GeoProviderId::cloudfront(), + provider: Arc::new(CountingProvider { + calls: Arc::default(), + result: GeoResolution::Located(GeoLocation::new( + CountryCode::new("US").unwrap(), + None, + )), + }), + }) + .err() + .unwrap(); + assert_eq!(error, NativeGeoConfigError::ReservedProviderId); + let options = AxumRunOptions::new("127.0.0.1:8080".parse().unwrap()).unwrap(); + assert_eq!(options.geo_capability().unwrap(), NativeGeoCapability::Off); + let supplied = NativeGeoSelection::Supplied { + policy: policy(), + id: GeoProviderId::new("custom").unwrap(), + provider: Arc::new(CountingProvider { + calls: Arc::default(), + result: GeoResolution::MissingMetadata, + }), + }; + let selected_options = options.with_geo(supplied); + assert!( + matches!(selected_options.geo_capability().unwrap(), NativeGeoCapability::Selected { provider, .. } if provider.as_str() == "custom") + ); + assert_eq!( + selected_options + .with_geo(NativeGeoSelection::Off) + .geo_capability() + .unwrap(), + NativeGeoCapability::Off + ); + } + + #[cfg(not(feature = "geo-cloudfront"))] + #[test] + fn compiled_out_builtin_is_an_error_not_off() { + let selected = NativeGeoSelection::CloudFront { policy: policy() }; + assert_eq!( + NativeGeo::prepare(&selected).err(), + Some(NativeGeoConfigError::UnavailableCloudFront) + ); + } +} diff --git a/crates/edgezero-adapter-axum/src/geo/cloudfront.rs b/crates/edgezero-adapter-axum/src/geo/cloudfront.rs new file mode 100644 index 000000000..67c63a5a7 --- /dev/null +++ b/crates/edgezero-adapter-axum/src/geo/cloudfront.rs @@ -0,0 +1,153 @@ +//! Local normalization of source-admitted `CloudFront` geographic assertions. + +use std::str; + +use edgezero_core::geo::{ + CountryCode, GeoError, GeoInput, GeoLocation, GeoProvider, GeoResolution, SubdivisionCode, +}; + +pub(super) struct CloudFrontProvider; + +impl GeoProvider for CloudFrontProvider { + #[inline] + fn resolve(&self, input: &GeoInput) -> GeoResolution { + let GeoInput::Assertions { + country, + subdivision, + } = input + else { + return GeoResolution::Error(GeoError::ProviderFailure); + }; + let (Ok(country_slot), Ok(subdivision_slot)) = (country.as_bytes(), subdivision.as_bytes()) + else { + return GeoResolution::Error(GeoError::InvalidAssertion); + }; + let Some(country_bytes) = country_slot else { + return if subdivision_slot.is_none() { + GeoResolution::MissingMetadata + } else { + GeoResolution::Error(GeoError::InvalidAssertion) + }; + }; + let parsed_country = str::from_utf8(country_bytes) + .ok() + .and_then(|value| CountryCode::new(value).ok()); + let Some(country_code) = parsed_country else { + return GeoResolution::Error(GeoError::InvalidAssertion); + }; + let subdivision_code = match subdivision_slot { + None => None, + Some(bytes) => { + let parsed_subdivision = str::from_utf8(bytes) + .ok() + .and_then(|value| SubdivisionCode::new(value).ok()); + let Some(component) = parsed_subdivision else { + return GeoResolution::Error(GeoError::InvalidAssertion); + }; + Some(component) + } + }; + GeoResolution::Located(GeoLocation::new(country_code, subdivision_code)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use edgezero_core::geo::{GeoAssertion, GeoIpSource}; + + fn input(country: Option<&[u8]>, subdivision: Option<&[u8]>) -> GeoInput { + GeoInput::Assertions { + country: country.map_or_else(GeoAssertion::absent, GeoAssertion::from_bytes), + subdivision: subdivision.map_or_else(GeoAssertion::absent, GeoAssertion::from_bytes), + } + } + + fn located(country: &str, subdivision: Option<&str>) -> GeoResolution { + GeoResolution::Located(GeoLocation::new( + CountryCode::new(country).unwrap(), + subdivision.map(|value| SubdivisionCode::new(value).unwrap()), + )) + } + + #[test] + fn geo_cloudfront_parsing_matches_spec() { + let invalid = GeoResolution::Error(GeoError::InvalidAssertion); + let cases = [ + ( + GeoInput::ReaderIp { + address: "192.0.2.1".parse().unwrap(), + source: GeoIpSource::DirectPeer, + }, + GeoResolution::Error(GeoError::ProviderFailure), + ), + (input(None, None), GeoResolution::MissingMetadata), + (input(Some(b"us"), Some(b"ca")), located("US", Some("CA"))), + (input(Some(b"US"), None), located("US", None)), + (input(Some(b"US"), Some(b"ZZ")), located("US", Some("ZZ"))), + (input(Some(b"GB"), Some(b"eng")), located("GB", Some("ENG"))), + (input(Some(b"JP"), Some(b"01")), located("JP", Some("01"))), + (input(Some(b"BR"), Some(b"1")), located("BR", Some("1"))), + (input(Some(b"AU"), Some(b"001")), located("AU", Some("001"))), + (input(Some(b"US"), Some(b"a1B")), located("US", Some("A1B"))), + (input(None, Some(b"CA")), invalid.clone()), + (input(Some(b""), None), invalid.clone()), + (input(Some(b"U"), None), invalid.clone()), + (input(Some(b"USA"), None), invalid.clone()), + (input(Some(b"USAA"), None), invalid.clone()), + (input(Some(b"XX"), None), invalid.clone()), + (input(Some(b"ZZ"), None), invalid.clone()), + (input(Some(b"UK"), None), invalid.clone()), + (input(Some(b"XK"), None), invalid.clone()), + (input(Some(b"U1"), None), invalid.clone()), + (input(Some(b" US"), None), invalid.clone()), + (input(Some(b"US "), None), invalid.clone()), + (input(Some(b" U"), None), invalid.clone()), + (input(Some(b"U\t"), None), invalid.clone()), + (input(Some(b"U\0"), None), invalid.clone()), + (input(Some(b"\xc3\xa9"), None), invalid.clone()), + (input(Some(b"\xff\xfe"), None), invalid.clone()), + (input(Some(b"US,CA"), None), invalid.clone()), + (input(Some(b"%55S"), None), invalid.clone()), + (input(Some(b"US"), Some(b"")), invalid.clone()), + (input(Some(b"US"), Some(b"ABCD")), invalid.clone()), + (input(Some(b"US"), Some(b"US-CA")), invalid.clone()), + (input(Some(b"US"), Some(b" CA")), invalid.clone()), + (input(Some(b"US"), Some(b"CA ")), invalid.clone()), + (input(Some(b"US"), Some(b"A B")), invalid.clone()), + (input(Some(b"US"), Some(b"A\t")), invalid.clone()), + (input(Some(b"US"), Some(b"C\0")), invalid.clone()), + (input(Some(b"US"), Some(b"\xc3\xa9")), invalid.clone()), + (input(Some(b"US"), Some(b"\xff")), invalid.clone()), + (input(Some(b"US"), Some(b"A,B")), invalid.clone()), + (input(Some(b"US"), Some(b"A-B")), invalid.clone()), + (input(Some(b"US"), Some(b"A_B")), invalid.clone()), + (input(Some(b"US"), Some(b"%41")), invalid.clone()), + ( + GeoInput::Assertions { + country: GeoAssertion::invalid(), + subdivision: GeoAssertion::absent(), + }, + invalid.clone(), + ), + ( + GeoInput::Assertions { + country: GeoAssertion::from_bytes(b"US"), + subdivision: GeoAssertion::invalid(), + }, + invalid.clone(), + ), + ( + GeoInput::Assertions { + country: GeoAssertion::absent(), + subdivision: GeoAssertion::invalid(), + }, + invalid, + ), + ]; + let provider = CloudFrontProvider; + for (index, (input, expected)) in cases.iter().enumerate() { + assert_eq!(&provider.resolve(input), expected, "case {index}"); + } + } +} diff --git a/crates/edgezero-adapter-axum/src/geo/maxmind.rs b/crates/edgezero-adapter-axum/src/geo/maxmind.rs new file mode 100644 index 000000000..402cc2a02 --- /dev/null +++ b/crates/edgezero-adapter-axum/src/geo/maxmind.rs @@ -0,0 +1,255 @@ +//! Immutable local MMDB snapshots. Preparation is synchronous and Unix-only. +//! +//! File bytes and network visits are caller-bounded. Before verification the +//! upstream tree allowance `3 * node_count + 1` must fit that visit budget. +//! Upstream verification charges at most eight times each section's bytes and +//! memoizes shared values. Each additional structural decode and borrowed path +//! decode has the reader's 65,536-value, payload and depth limits. Structural +//! decoding deliberately expands shared values through the normal decoder. +//! Thus total decode work is bounded per unique record, not globally by bytes. +//! The dedup set is bounded by network visits; upstream tables are O(file bytes). +//! These bounds are not wall-clock or fixed-RSS promises. +//! +//! Metadata receives upstream typed decoding and verification. Version 0.32.0 +//! exposes neither a raw metadata decoder nor a metadata `LookupResult`. Therefore +//! unknown metadata fields retain upstream verifier gaps for non-string map keys +//! and pointer chains. Reachable data records receive the stricter pass below. + +#![expect( + clippy::field_scoped_visibility_modifiers, + reason = "snapshot metadata is shared only with the native composition parent" +)] +#![expect( + clippy::map_err_ignore, + clippy::wildcard_enum_match_arm, + reason = "vendor and I/O details deliberately collapse to source-free error categories" +)] + +use std::collections::HashSet; +#[cfg(any(unix, test))] +use std::io::{ErrorKind, Read}; + +use edgezero_core::geo::{ + CountryCode, GeoError, GeoInput, GeoLocation, GeoProvider, GeoResolution, SubdivisionCode, +}; +use maxminddb::{LookupResult, MaxMindDbError, Reader, WithinOptions, path}; +use sha2::{Digest as _, Sha256}; + +use super::{MaxMindGeoConfig, NativeGeoConfigError}; + +mod structure; +#[cfg(test)] +mod tests; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct MaxMindSnapshot { + pub(super) database_type: String, + pub(super) build_epoch: u64, + pub(super) sha256: String, +} + +#[derive(Clone, Copy)] +enum Dataset { + City, + Country, +} + +pub(super) struct MaxMindProvider { + reader: Reader>, + dataset: Dataset, + pub(super) metadata: MaxMindSnapshot, +} + +impl MaxMindProvider { + pub(super) fn prepare(config: &MaxMindGeoConfig) -> Result { + let byte_cap = config.limits.max_database_bytes; + let visit_cap = config.limits.max_validation_networks; + if byte_cap == 0 || visit_cap == 0 || byte_cap.checked_add(1).is_none() { + return Err(NativeGeoConfigError::DatabaseLimit); + } + let bytes = acquire(config)?; + let checksum = format!("{:x}", Sha256::digest(&bytes)); + let reader = Reader::from_source(bytes).map_err(database_error)?; + let reader_metadata = reader.metadata(); + let dataset = match reader_metadata.database_type.as_str() { + "GeoLite2-City" => Dataset::City, + "GeoLite2-Country" => Dataset::Country, + _ => return Err(NativeGeoConfigError::UnsupportedDatabase), + }; + if !matches!(reader_metadata.ip_version, 4 | 6) { + return Err(NativeGeoConfigError::UnsupportedDatabase); + } + let node_count = usize::try_from(reader_metadata.node_count) + .map_err(|_| NativeGeoConfigError::DatabaseLimit)?; + let tree_allowance = node_count + .checked_mul(3) + .and_then(|n| n.checked_add(1)) + .ok_or(NativeGeoConfigError::DatabaseLimit)?; + if tree_allowance > visit_cap { + return Err(NativeGeoConfigError::DatabaseLimit); + } + reader.verify().map_err(database_error)?; + let options = WithinOptions::default() + .include_aliased_networks() + .include_networks_without_data(); + let networks = reader.networks(options).map_err(database_error)?; + let mut offsets = HashSet::new(); + for (index, next_network) in networks.enumerate() { + // Count no-data leaves and aliases before deduplication. + if index >= visit_cap { + return Err(NativeGeoConfigError::DatabaseLimit); + } + let result = next_network.map_err(database_error)?; + if let Some(offset) = result.offset() + && offsets.insert(offset) + { + result + .decode::() + .map_err(database_error)?; + projection(&result, dataset).map_err(|error| match error { + MaxMindDbError::ResourceLimit { .. } => NativeGeoConfigError::DatabaseLimit, + _ => NativeGeoConfigError::InvalidDatabaseSchema, + })?; + } + } + let metadata = MaxMindSnapshot { + database_type: reader_metadata.database_type.clone(), + build_epoch: reader_metadata.build_epoch, + sha256: checksum, + }; + Ok(Self { + reader, + dataset, + metadata, + }) + } +} + +#[expect( + clippy::needless_pass_by_value, + reason = "map_err transfers ownership and intentionally drops vendor diagnostics" +)] +fn database_error(error: MaxMindDbError) -> NativeGeoConfigError { + match error { + MaxMindDbError::ResourceLimit { .. } => NativeGeoConfigError::DatabaseLimit, + _ => NativeGeoConfigError::InvalidDatabase, + } +} + +// The identical borrowed paths validate consumed types during preparation and +// resolve them during lookup. Duplicate keys consequently use the first value. +fn projection<'database>( + result: &LookupResult<'database, Vec>, + dataset: Dataset, +) -> Result<(Option<&'database str>, Option<&'database str>), MaxMindDbError> { + let country = result.decode_path::<&str>(&path!["country", "iso_code"])?; + let subdivision = match dataset { + Dataset::City => result.decode_path::<&str>(&path!["subdivisions", 0_usize, "iso_code"])?, + Dataset::Country => None, + }; + Ok((country, subdivision)) +} + +impl GeoProvider for MaxMindProvider { + fn resolve(&self, input: &GeoInput) -> GeoResolution { + let GeoInput::ReaderIp { address, .. } = input else { + return GeoResolution::Error(GeoError::ProviderFailure); + }; + // Checked ingress already normalizes mapped addresses. Do not invent + // source admission or address rewriting inside the database provider. + if address.is_ipv6() && self.reader.metadata().ip_version == 4 { + return GeoResolution::NoMatch; + } + let Ok(result) = self.reader.lookup(*address) else { + return GeoResolution::Error(GeoError::ProviderFailure); + }; + let Ok((country_slot, subdivision_slot)) = projection(&result, self.dataset) else { + return GeoResolution::Error(GeoError::ProviderFailure); + }; + let Some(country_value) = country_slot else { + return if subdivision_slot.is_none() { + GeoResolution::NoMatch + } else { + GeoResolution::Error(GeoError::InvalidAssertion) + }; + }; + let Ok(country) = CountryCode::new(country_value) else { + return GeoResolution::Error(GeoError::InvalidAssertion); + }; + let Ok(subdivision) = subdivision_slot.map(SubdivisionCode::new).transpose() else { + return GeoResolution::Error(GeoError::InvalidAssertion); + }; + GeoResolution::Located(GeoLocation::new(country, subdivision)) + } +} + +#[cfg(unix)] +#[expect( + clippy::filetype_is_file, + reason = "only regular files are accepted; all special file kinds must fail" +)] +fn acquire(config: &MaxMindGeoConfig) -> Result, NativeGeoConfigError> { + use std::fs::OpenOptions; + use std::os::unix::fs::OpenOptionsExt as _; + + // Opening first avoids pathname races; NONBLOCK prevents waiting for a FIFO + // writer. Descriptor metadata rejects pipes/devices before any read. + let file = OpenOptions::new() + .read(true) + .custom_flags(libc::O_NONBLOCK) + .open(&config.path) + .map_err(|_| NativeGeoConfigError::UnavailableDatabase)?; + let metadata = file + .metadata() + .map_err(|_| NativeGeoConfigError::UnavailableDatabase)?; + if !metadata.file_type().is_file() { + return Err(NativeGeoConfigError::UnavailableDatabase); + } + let cap = config.limits.max_database_bytes; + let file_cap = u64::try_from(cap).map_err(|_| NativeGeoConfigError::DatabaseLimit)?; + if metadata.len() > file_cap { + return Err(NativeGeoConfigError::DatabaseLimit); + } + read_bounded(file, cap) +} + +// No special-file-safe acquisition has been proven for other platforms. Fail +// before opening anything rather than potentially blocking on a named pipe. +#[cfg(not(unix))] +fn acquire(_config: &MaxMindGeoConfig) -> Result, NativeGeoConfigError> { + Err(NativeGeoConfigError::UnavailableDatabase) +} + +#[cfg(any(unix, test))] +#[expect( + clippy::indexing_slicing, + reason = "slice ends are bounded by chunk length and Read's returned count" +)] +fn read_bounded(mut source: impl Read, cap: usize) -> Result, NativeGeoConfigError> { + let limit = cap + .checked_add(1) + .ok_or(NativeGeoConfigError::DatabaseLimit)?; + let mut bytes = Vec::new(); + // Incremental fixed-size reads avoid reserving the full operator limit for a + // tiny database. Vec capacity can exceed length; it is O(cap), not cap exact. + let mut chunk = [0; 8192]; + while bytes.len() < limit { + let remaining = chunk.len().min(limit.saturating_sub(bytes.len())); + let count = match source.read(&mut chunk[..remaining]) { + Ok(count) => count, + Err(error) if error.kind() == ErrorKind::Interrupted => continue, + Err(_) => return Err(NativeGeoConfigError::UnavailableDatabase), + }; + if count == 0 { + break; + } + bytes + .try_reserve(count) + .map_err(|_| NativeGeoConfigError::DatabaseLimit)?; + bytes.extend_from_slice(&chunk[..count]); + } + if bytes.len() > cap { + return Err(NativeGeoConfigError::DatabaseLimit); + } + Ok(bytes) +} diff --git a/crates/edgezero-adapter-axum/src/geo/maxmind/structure.rs b/crates/edgezero-adapter-axum/src/geo/maxmind/structure.rs new file mode 100644 index 000000000..53fb77594 --- /dev/null +++ b/crates/edgezero-adapter-axum/src/geo/maxmind/structure.rs @@ -0,0 +1,89 @@ +//! Non-materializing traversal through the reader's bounded normal decoder. +//! Unlike `IgnoredAny`, this follows every pointer. The identifier entry point +//! rejects pointer chains before dispatching non-string values to `deserialize_any`. +//! Calling `deserialize_any` directly would retain the upstream pointer-chain gap. + +#![expect( + clippy::missing_trait_methods, + reason = "Serde defaults reject types that are not emitted by the pinned MMDB decoder" +)] + +use std::fmt; + +use serde::de::{Error, MapAccess, SeqAccess, Visitor}; +use serde::{Deserialize, Deserializer}; + +pub(super) struct StructuralValue; + +impl<'de> Deserialize<'de> for StructuralValue { + fn deserialize>(deserializer: D) -> Result { + // Pinned reader's try_read_identifier_bytes checks pointer targets even + // for non-strings, then rewinds and uses normal any decoding. Strings + // arrive as borrowed bytes; verify() has already checked their UTF-8. + deserializer.deserialize_identifier(StructuralVisitor) + } +} + +struct StructuralVisitor; + +impl<'de> Visitor<'de> for StructuralVisitor { + type Value = StructuralValue; + + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("an MMDB value with string map keys") + } + + fn visit_map>(self, mut map: A) -> Result { + // &str uses strict string decoding, not deserialize_identifier, which + // may accept numeric struct field identifiers. + while map.next_key::<&'de str>()?.is_some() { + map.next_value::()?; + } + Ok(StructuralValue) + } + + fn visit_seq>(self, mut seq: A) -> Result { + while seq.next_element::()?.is_some() {} + Ok(StructuralValue) + } + + fn visit_borrowed_str(self, _: &'de str) -> Result { + Ok(StructuralValue) + } + + fn visit_borrowed_bytes(self, _: &'de [u8]) -> Result { + Ok(StructuralValue) + } + + fn visit_bool(self, _: bool) -> Result { + Ok(StructuralValue) + } + + fn visit_i32(self, _: i32) -> Result { + Ok(StructuralValue) + } + + fn visit_u16(self, _: u16) -> Result { + Ok(StructuralValue) + } + + fn visit_u32(self, _: u32) -> Result { + Ok(StructuralValue) + } + + fn visit_u64(self, _: u64) -> Result { + Ok(StructuralValue) + } + + fn visit_u128(self, _: u128) -> Result { + Ok(StructuralValue) + } + + fn visit_f32(self, _: f32) -> Result { + Ok(StructuralValue) + } + + fn visit_f64(self, _: f64) -> Result { + Ok(StructuralValue) + } +} diff --git a/crates/edgezero-adapter-axum/src/geo/maxmind/tests.rs b/crates/edgezero-adapter-axum/src/geo/maxmind/tests.rs new file mode 100644 index 000000000..cbace7d7a --- /dev/null +++ b/crates/edgezero-adapter-axum/src/geo/maxmind/tests.rs @@ -0,0 +1,375 @@ +#![cfg_attr( + unix, + expect( + clippy::absolute_paths, + reason = "filesystem tests spell out standard-library operations" + ) +)] + +use super::*; +use crate::geo::{MaxMindPreparationLimits, ReaderIpExpectation}; +#[cfg(unix)] +use edgezero_core::geo::{GeoAssertion, GeoIpSource}; +use std::path::PathBuf; + +fn fixture(name: &str) -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/geoip") + .join(format!("{name}.mmdb")) +} + +fn config(name: &str) -> MaxMindGeoConfig { + MaxMindGeoConfig { + path: fixture(name), + limits: MaxMindPreparationLimits { + max_database_bytes: 128 * 1024 * 1024, + max_validation_networks: 32_000_000, + }, + expectation: ReaderIpExpectation::PeerAllowed, + } +} + +#[cfg(unix)] +fn input(address: &str) -> GeoInput { + GeoInput::ReaderIp { + address: address.parse().unwrap(), + source: GeoIpSource::DirectPeer, + } +} + +#[cfg(unix)] +fn located(country: &str, subdivision: Option<&str>) -> GeoResolution { + GeoResolution::Located(GeoLocation::new( + CountryCode::new(country).unwrap(), + subdivision.map(|code| SubdivisionCode::new(code).unwrap()), + )) +} + +#[cfg(unix)] +#[test] +fn geo_maxmind_city_projection_and_request_code_validation() { + let provider = MaxMindProvider::prepare(&config("city")).unwrap(); + let invalid = GeoResolution::Error(GeoError::InvalidAssertion); + for (address, expected) in [ + ("127.0.0.1", located("US", Some("CA"))), + ("192.0.2.1", located("US", Some("CA"))), + ("192.0.2.2", located("GB", None)), + ("192.0.2.3", located("JP", Some("01"))), + ("192.0.2.4", located("US", Some("ZZ"))), + ("192.0.2.5", located("US", None)), + ("192.0.2.6", located("US", None)), + ("192.0.2.7", GeoResolution::NoMatch), + ("192.0.2.8", invalid.clone()), + ("192.0.2.9", invalid.clone()), + ("192.0.2.10", invalid.clone()), + ("192.0.2.11", invalid.clone()), + ("192.0.2.12", invalid.clone()), + ("192.0.2.13", GeoResolution::NoMatch), + ("192.0.2.14", GeoResolution::NoMatch), + ("192.0.2.15", located("US", None)), + ("192.0.2.16", located("US", Some("CA"))), + ("192.0.2.17", located("US", Some("A1B"))), + ("192.0.2.18", invalid.clone()), + ("192.0.2.19", invalid.clone()), + ("192.0.2.20", invalid), + ("192.0.2.254", GeoResolution::NoMatch), + ("10.0.0.1", GeoResolution::NoMatch), + ("2001:db8::1", GeoResolution::NoMatch), + ("::ffff:192.0.2.1", GeoResolution::NoMatch), + ] { + assert_eq!(provider.resolve(&input(address)), expected, "{address}"); + } + for source in [ + GeoIpSource::DirectPeer, + GeoIpSource::TrustedForwarded, + GeoIpSource::TrustedXForwarded, + ] { + assert_eq!( + provider.resolve(&GeoInput::ReaderIp { + address: "192.0.2.1".parse().unwrap(), + source + }), + located("US", Some("CA")) + ); + } + assert_eq!( + provider.resolve(&GeoInput::Assertions { + country: GeoAssertion::from_bytes(b"US"), + subdivision: GeoAssertion::absent() + }), + GeoResolution::Error(GeoError::ProviderFailure) + ); +} + +#[cfg(unix)] +#[test] +fn geo_maxmind_country_ignores_subdivisions_and_record_sizes_work() { + let provider = MaxMindProvider::prepare(&config("country")).unwrap(); + assert_eq!(provider.resolve(&input("192.0.2.1")), located("US", None)); + for name in ["record-28", "record-32"] { + let sized_provider = MaxMindProvider::prepare(&config(name)).unwrap(); + assert_eq!( + sized_provider.resolve(&input("192.0.2.1")), + located("US", Some("CA")) + ); + assert_eq!( + sized_provider.resolve(&input("198.51.100.1")), + GeoResolution::NoMatch + ); + } +} + +#[cfg(unix)] +#[test] +fn geo_maxmind_ipv6_and_ipv4_subtree() { + let provider = MaxMindProvider::prepare(&config("ipv6")).unwrap(); + assert_eq!( + provider.resolve(&input("2001:db8::7")), + located("GB", Some("ENG")) + ); + assert_eq!( + provider.resolve(&input("192.0.2.1")), + located("US", Some("CA")) + ); + assert_eq!( + provider.resolve(&input("2001:db9::7")), + GeoResolution::NoMatch + ); + assert_eq!( + provider.resolve(&input("192.0.2.2")), + GeoResolution::NoMatch + ); +} + +#[cfg(unix)] +#[test] +fn geo_maxmind_consumed_type_failures_are_startup_errors_even_off_path() { + for name in [ + "bad-country-parent", + "bad-country-code", + "bad-subdivisions-parent", + "bad-first-subdivision", + "bad-subdivision-code", + ] { + assert!( + matches!( + MaxMindProvider::prepare(&config(name)), + Err(NativeGeoConfigError::InvalidDatabaseSchema) + ), + "{name}" + ); + } +} + +#[cfg(unix)] +#[test] +#[ignore = "#438 acceptance blocker: maxminddb 0.32.0 exposes no strict raw metadata decoding"] +fn geo_maxmind_rejects_unused_metadata_nonstring_keys() { + assert!( + MaxMindProvider::prepare(&config("bad-metadata-key")).is_err(), + "selected malformed metadata must fail startup; upstream verification currently accepts it" + ); +} + +#[cfg(unix)] +#[test] +fn geo_maxmind_structural_pass_closes_upstream_data_gaps() { + for name in ["bad-unrelated-key", "bad-unrelated-chain", "bad-root-chain"] { + let reader = Reader::from_source(std::fs::read(fixture(name)).unwrap()).unwrap(); + reader.verify().unwrap(); // Regression: verification alone is insufficient. + assert!( + matches!( + MaxMindProvider::prepare(&config(name)), + Err(NativeGeoConfigError::InvalidDatabase) + ), + "{name}" + ); + } +} + +#[cfg(unix)] +#[test] +fn geo_maxmind_off_path_corruption_and_metadata_fail_preparation() { + for name in [ + "bad-offpath-pointer", + "bad-offpath-utf8", + "bad-metadata", + "bad-tree-cycle", + "bad-separator", + "truncated", + ] { + assert!( + matches!( + MaxMindProvider::prepare(&config(name)), + Err(NativeGeoConfigError::InvalidDatabase) + ), + "{name}" + ); + } + assert!(matches!( + MaxMindProvider::prepare(&config("unsupported")), + Err(NativeGeoConfigError::UnsupportedDatabase) + )); +} + +#[cfg(unix)] +#[test] +fn geo_maxmind_limits_fail_without_partial_publication() { + let mut settings = config("city"); + for bytes in [0, 1, usize::MAX] { + settings.limits.max_database_bytes = bytes; + assert!(matches!( + MaxMindProvider::prepare(&settings), + Err(NativeGeoConfigError::DatabaseLimit) + )); + } + settings = config("city"); + let bytes = std::fs::read(&settings.path).unwrap(); + settings.limits.max_database_bytes = bytes.len(); + MaxMindProvider::prepare(&settings).unwrap(); + settings.limits.max_database_bytes = bytes.len().saturating_sub(1); + assert!(matches!( + MaxMindProvider::prepare(&settings), + Err(NativeGeoConfigError::DatabaseLimit) + )); + settings = config("city"); + let reader = Reader::from_source(bytes).unwrap(); + let allowance = usize::try_from(reader.metadata().node_count) + .unwrap() + .saturating_mul(3) + .saturating_add(1); + settings.limits.max_validation_networks = allowance.saturating_sub(1); + assert!(matches!( + MaxMindProvider::prepare(&settings), + Err(NativeGeoConfigError::DatabaseLimit) + )); + settings.limits.max_validation_networks = allowance; + MaxMindProvider::prepare(&settings).unwrap(); + settings.limits.max_validation_networks = 0; + assert!(matches!( + MaxMindProvider::prepare(&settings), + Err(NativeGeoConfigError::DatabaseLimit) + )); + let expansion = + Reader::from_source(std::fs::read(fixture("expansion-limit")).unwrap()).unwrap(); + expansion.verify().unwrap(); + for name in [ + "expansion-limit", + "verification-limit", + "metadata-verification-limit", + ] { + assert!( + matches!( + MaxMindProvider::prepare(&config(name)), + Err(NativeGeoConfigError::DatabaseLimit) + ), + "{name}" + ); + } + let scalar_provider = MaxMindProvider::prepare(&config("all-values")).unwrap(); + assert_eq!( + scalar_provider.resolve(&input("192.0.2.1")), + located("US", None) + ); +} + +#[test] +fn geo_maxmind_actual_read_limit_checks_the_extra_byte() { + use std::io::Cursor; + assert_eq!(read_bounded(Cursor::new(b"1234"), 4).unwrap(), b"1234"); + assert!(matches!( + read_bounded(Cursor::new(b"12345"), 4), + Err(NativeGeoConfigError::DatabaseLimit) + )); + let mut source = Cursor::new(b"123456789"); + assert!(matches!( + read_bounded(&mut source, 4), + Err(NativeGeoConfigError::DatabaseLimit) + )); + assert_eq!(source.position(), 5); + assert!(read_bounded(Cursor::new(b""), 4).unwrap().is_empty()); +} + +#[cfg(unix)] +#[test] +fn geo_maxmind_rejects_missing_directory_device_and_fifo_without_reading() { + let mut settings = config("does-not-exist"); + assert!(matches!( + MaxMindProvider::prepare(&settings), + Err(NativeGeoConfigError::UnavailableDatabase) + )); + settings.path = fixture("city").parent().unwrap().to_owned(); + assert!(matches!( + MaxMindProvider::prepare(&settings), + Err(NativeGeoConfigError::UnavailableDatabase) + )); + settings.path = PathBuf::from("/dev/zero"); + assert!(matches!( + MaxMindProvider::prepare(&settings), + Err(NativeGeoConfigError::UnavailableDatabase) + )); + let path = temporary("fifo"); + assert!( + std::process::Command::new("mkfifo") + .arg(&path) + .status() + .unwrap() + .success() + ); + settings.path = path.clone(); + let result = MaxMindProvider::prepare(&settings); + std::fs::remove_file(path).unwrap(); + assert!(matches!( + result, + Err(NativeGeoConfigError::UnavailableDatabase) + )); +} + +#[cfg(unix)] +fn temporary(label: &str) -> PathBuf { + use std::sync::atomic::{AtomicUsize, Ordering}; + static NEXT: AtomicUsize = AtomicUsize::new(0); + std::env::temp_dir().join(format!( + "edgezero-geo-{label}-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )) +} + +#[cfg(unix)] +#[test] +fn geo_maxmind_owned_snapshot_survives_replacement_and_deletion() { + use std::os::unix::fs::PermissionsExt as _; + let path = temporary("snapshot.mmdb"); + std::fs::copy(fixture("city"), &path).unwrap(); + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o444)).unwrap(); + let mut settings = config("city"); + settings.path = path.clone(); + let old = MaxMindProvider::prepare(&settings).unwrap(); + let original = old.metadata.clone(); + assert_eq!(original.database_type, "GeoLite2-City"); + assert_eq!(original.build_epoch, 1_793_750_400); + assert_eq!( + original.sha256, + format!("{:x}", Sha256::digest(std::fs::read(&path).unwrap())) + ); + let replacement = temporary("replacement.mmdb"); + std::fs::copy(fixture("city-replacement"), &replacement).unwrap(); + std::fs::rename(replacement, &path).unwrap(); + let new = MaxMindProvider::prepare(&settings).unwrap(); + assert_ne!(new.metadata.sha256, original.sha256); + assert_ne!(new.metadata.build_epoch, original.build_epoch); + std::fs::remove_file(path).unwrap(); + assert_eq!(old.metadata, original); + assert_eq!(old.resolve(&input("127.0.0.1")), located("US", Some("CA"))); + assert_eq!(new.resolve(&input("127.0.0.1")), located("US", Some("NY"))); +} + +#[cfg(not(unix))] +#[test] +fn geo_maxmind_unsupported_platform_does_not_open_source() { + assert!(matches!( + MaxMindProvider::prepare(&config("city")), + Err(NativeGeoConfigError::UnavailableDatabase) + )); +} diff --git a/crates/edgezero-adapter-axum/src/geo/reader_tests.rs b/crates/edgezero-adapter-axum/src/geo/reader_tests.rs new file mode 100644 index 000000000..ecdb48bb6 --- /dev/null +++ b/crates/edgezero-adapter-axum/src/geo/reader_tests.rs @@ -0,0 +1,258 @@ +//! Checked-IP composition tests run without either built-in implementation. + +use super::*; +use crate::proxy::{ForwardingHeaderFamily, TrustedProxyPolicy, normalize}; +use edgezero_core::geo::GeoError; +use edgezero_core::http::{HeaderValue, request_builder}; +use std::sync::Mutex; + +struct Counted(Arc>>); + +impl GeoProvider for Counted { + fn resolve(&self, input: &GeoInput) -> GeoResolution { + self.0.lock().unwrap().push(input.clone()); + GeoResolution::NoMatch + } +} + +fn selected(expectation: ReaderIpExpectation, calls: &Arc>>) -> NativeGeo { + NativeGeo::prepare(&NativeGeoSelection::SuppliedReaderIp { + expectation, + id: GeoProviderId::new("counted-reader").unwrap(), + provider: Arc::new(Counted(Arc::clone(calls))), + }) + .unwrap() +} + +#[expect( + clippy::too_many_lines, + reason = "the table keeps both forwarding families and reader expectations under the same counted admission assertions" +)] +#[test] +fn geo_reader_admission_uses_private_checked_provenance() { + for (family, name, source) in [ + ( + ForwardingHeaderFamily::Forwarded, + "forwarded", + GeoIpSource::TrustedForwarded, + ), + ( + ForwardingHeaderFamily::XForwarded, + "x-forwarded-for", + GeoIpSource::TrustedXForwarded, + ), + ] { + let policy = TrustedProxyPolicy::new(family, ["127.0.0.0/8"]).unwrap(); + for expectation in [ + ReaderIpExpectation::PeerAllowed, + ReaderIpExpectation::ForwardedRequired, + ] { + let calls = Arc::new(Mutex::new(Vec::new())); + let geo = selected(expectation, &calls); + for (raw_peer, value, address, provenance) in [ + ( + Some("127.0.0.1:80"), + Some(if name == "forwarded" { + "for=203.0.113.9" + } else { + "203.0.113.9" + }), + Some("203.0.113.9"), + Some(source), + ), + ( + Some("127.0.0.1:80"), + Some(if name == "forwarded" { + "for=\"[2001:db8::9]\"" + } else { + "2001:db8::9" + }), + Some("2001:db8::9"), + Some(source), + ), + ( + Some("127.0.0.1:80"), + Some(if name == "forwarded" { + "for=\"[::ffff:203.0.113.9]\"" + } else { + "::ffff:203.0.113.9" + }), + Some("203.0.113.9"), + Some(source), + ), + (Some("127.0.0.1:80"), Some("malformed"), None, None), + ( + Some("127.0.0.1:80"), + Some(if name == "forwarded" { + "for=127.0.0.2" + } else { + "127.0.0.2" + }), + None, + None, + ), + (Some("127.0.0.1:80"), None, None, None), + ( + Some("192.0.2.1:80"), + Some(if name == "forwarded" { + "for=203.0.113.9" + } else { + "203.0.113.9" + }), + None, + None, + ), + (Some("[::ffff:192.0.2.1]:80"), None, None, None), + (Some("[2001:db8::1]:80"), None, None, None), + ( + None, + Some(if name == "forwarded" { + "for=203.0.113.9" + } else { + "203.0.113.9" + }), + None, + None, + ), + ] { + let peer: Option = raw_peer.map(|socket| socket.parse().unwrap()); + let (mut parts, ()) = request_builder().uri("/").body(()).unwrap().into_parts(); + if let Some(assertion) = value { + parts + .headers + .insert(name, HeaderValue::from_str(assertion).unwrap()); + } + parts + .headers + .insert("cloudfront-viewer-country", HeaderValue::from_static("DE")); + parts.extensions.insert(GeoInput::ReaderIp { + address: "198.51.100.1".parse().unwrap(), + source, + }); + let checked = normalize(&policy, peer, &parts); + let input = geo.reader_input(&checked); + let expected = match (address, provenance) { + (Some(expected_address), Some(expected_source)) => Some(GeoInput::ReaderIp { + address: expected_address.parse().unwrap(), + source: expected_source, + }), + _ if expectation == ReaderIpExpectation::PeerAllowed && peer.is_some() => { + Some(GeoInput::ReaderIp { + address: checked.effective_client().unwrap(), + source: GeoIpSource::DirectPeer, + }) + } + _ => None, + }; + assert_eq!(input, expected, "{family:?} {expectation:?} {value:?}"); + let before = calls.lock().unwrap().len(); + let outcome = geo.resolve(peer, &parts.headers, input.as_ref()); + let reason = if input.is_some() { + GeoUnavailableReason::NoMatch + } else { + GeoUnavailableReason::MissingReaderIp + }; + assert_eq!( + outcome, + GeoOutcome::Selected { + provider: GeoProviderId::new("counted-reader").unwrap(), + status: GeoStatus::Unavailable(reason) + } + ); + assert_eq!( + calls.lock().unwrap().len(), + before + usize::from(input.is_some()) + ); + if let Some(admitted_input) = input { + assert_eq!(calls.lock().unwrap().last(), Some(&admitted_input)); + } + } + } + } +} + +#[test] +fn geo_reader_capability_is_pure_and_configuration_debug_is_redacted() { + let config = MaxMindGeoConfig::new( + "MISSING_PATH_SENTINEL/not-present.mmdb", + ReaderIpExpectation::ForwardedRequired, + ); + let selection = NativeGeoSelection::MaxMind { config }; + assert!(!format!("{selection:?}").contains("MISSING_PATH_SENTINEL")); + for _ in 0_u8..3 { + let actual = selection.capability(); + if cfg!(all(feature = "geo-maxmind", unix)) { + assert_eq!( + actual.unwrap(), + NativeGeoCapability::Selected { + provider: GeoProviderId::maxmind(), + input: NativeGeoInputContract::ReaderIp { + expectation: ReaderIpExpectation::ForwardedRequired + } + } + ); + } else { + assert_eq!( + actual.unwrap_err(), + NativeGeoConfigError::UnavailableMaxMind + ); + } + } + let mut invalid_config = + MaxMindGeoConfig::new("PRIVATE_PATH", ReaderIpExpectation::PeerAllowed); + invalid_config.limits.max_database_bytes = 0; + assert_eq!( + NativeGeoSelection::MaxMind { + config: invalid_config + } + .capability() + .unwrap_err(), + NativeGeoConfigError::InvalidPreparationLimits + ); +} + +#[test] +fn geo_reader_supplied_rejects_both_reserved_ids() { + for id in [GeoProviderId::cloudfront(), GeoProviderId::maxmind()] { + let selection = NativeGeoSelection::SuppliedReaderIp { + expectation: ReaderIpExpectation::PeerAllowed, + id, + provider: Arc::new(Counted(Arc::default())), + }; + assert_eq!( + selection.capability().unwrap_err(), + NativeGeoConfigError::ReservedProviderId + ); + assert_eq!( + NativeGeo::prepare(&selection).err(), + Some(NativeGeoConfigError::ReservedProviderId) + ); + } +} + +#[test] +fn geo_reader_provider_failures_remain_selected_outcomes() { + struct Failed; + impl GeoProvider for Failed { + fn resolve(&self, _input: &GeoInput) -> GeoResolution { + GeoResolution::Error(GeoError::ProviderFailure) + } + } + let geo = NativeGeo::prepare(&NativeGeoSelection::SuppliedReaderIp { + expectation: ReaderIpExpectation::PeerAllowed, + id: GeoProviderId::new("failed-reader").unwrap(), + provider: Arc::new(Failed), + }) + .unwrap(); + let input = GeoInput::ReaderIp { + address: "192.0.2.7".parse().unwrap(), + source: GeoIpSource::DirectPeer, + }; + assert_eq!( + geo.resolve(None, &HeaderMap::new(), Some(&input)), + GeoOutcome::Selected { + provider: GeoProviderId::new("failed-reader").unwrap(), + status: GeoStatus::Error(GeoError::ProviderFailure) + } + ); +} diff --git a/crates/edgezero-adapter-axum/src/key_value_store.rs b/crates/edgezero-adapter-axum/src/key_value_store.rs index bb375cf39..c4cf72b08 100644 --- a/crates/edgezero-adapter-axum/src/key_value_store.rs +++ b/crates/edgezero-adapter-axum/src/key_value_store.rs @@ -158,11 +158,9 @@ impl PersistentKvStore { /// Returns an error if the database file cannot be opened or initialised (corrupted file, locked by another process, or insufficient permissions). #[inline] pub fn new>(path: P) -> Result { - let db_path = path.as_ref().display().to_string(); - let db = Database::create(path).map_err(|err| { + let db = Database::create(path).map_err(|_error| { KvError::Internal(anyhow::anyhow!( - "Failed to open KV database at {db_path}. If the file is corrupted or locked \ - by another process, try deleting it and restarting: {err}" + "KV database unavailable; release other owners or check storage permissions, capacity and integrity" )) })?; diff --git a/crates/edgezero-adapter-axum/src/lib.rs b/crates/edgezero-adapter-axum/src/lib.rs index 789e35da7..b79afd5bb 100644 --- a/crates/edgezero-adapter-axum/src/lib.rs +++ b/crates/edgezero-adapter-axum/src/lib.rs @@ -12,17 +12,33 @@ pub mod context; #[cfg(feature = "axum")] pub mod dev_server; #[cfg(feature = "axum")] +pub mod diagnostics; +#[cfg(feature = "axum")] +pub mod geo; +#[cfg(feature = "axum")] mod header_deadline; pub mod ingress_config; #[cfg(feature = "axum")] pub mod key_value_store; +#[cfg(all(feature = "native-bindings", not(target_arch = "wasm32")))] +pub mod native_bindings; +#[cfg(all(feature = "native-bindings", not(target_arch = "wasm32")))] +pub mod native_build; +#[cfg(feature = "axum")] +pub mod native_stores; #[cfg(feature = "axum")] pub mod outbound; #[cfg(feature = "axum")] +mod peer_networks; +#[cfg(feature = "axum")] +pub mod proxy; +#[cfg(feature = "axum")] pub mod request; #[cfg(feature = "axum")] mod response; #[cfg(feature = "axum")] +mod run_options; +#[cfg(feature = "axum")] pub mod secret_store; #[cfg(feature = "axum")] mod service; diff --git a/crates/edgezero-adapter-axum/src/native_bindings.rs b/crates/edgezero-adapter-axum/src/native_bindings.rs new file mode 100644 index 000000000..3a4849def --- /dev/null +++ b/crates/edgezero-adapter-axum/src/native_bindings.rs @@ -0,0 +1,1441 @@ +//! Native bootstrap schema and pure per-ID source decisions, without service clients. + +#![expect( + clippy::arbitrary_source_item_ordering, + reason = "schema, bootstrap I/O and source validation are grouped" +)] + +#[cfg(unix)] +use libc::O_NONBLOCK; +use std::collections::{BTreeMap, BTreeSet}; +use std::fmt; +use std::fs::OpenOptions; +use std::io::Read as _; +#[cfg(unix)] +use std::os::unix::fs::OpenOptionsExt as _; +use std::path::{Path, PathBuf}; + +use edgezero_core::app::{StoreMetadata, StoresMetadata}; +use edgezero_core::env_config::EnvConfig; +use edgezero_core::manifest::{ManifestStores, StoreDeclaration}; +use edgezero_core::secret_store::MAX_NAME_LEN; +use edgezero_store_aws::settings::{ + AppConfigAgentSettings, AwsPreparationLimits, DynamoDbKvSettings, SecretsManagerSettings, +}; +use edgezero_store_celld::settings::{ + CelldConfigSettings, CelldKvSettings, CelldPreparationLimits, +}; +use serde::Deserialize; + +/// Closed bootstrap failures. File, TOML and provider inputs never become source chains. +#[derive(Clone, Copy, Debug, Eq, PartialEq, thiserror::Error)] +pub enum NativeBindingsError { + #[error("duplicate binding source")] + Duplicate, + #[error("invalid native bindings")] + Invalid, + #[error("invalid declared metadata")] + InvalidMetadata, + #[error("explicit binding conflicts with NAME/KEY overlay")] + OverlayConflict, + #[error("undeclared binding")] + Undeclared, + #[error("malformed native bindings")] + Malformed, + #[error("native bindings size limit")] + SizeLimit, + #[error("native bindings file unavailable")] + Unavailable, + #[error("provider not compiled")] + UnsupportedProvider, +} + +/// Trusted bootstrap bounds. The file cannot raise its own read/ID limits. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct NativeBindingsLimits { + pub max_file_bytes: usize, + pub max_explicit_ids: usize, +} + +impl Default for NativeBindingsLimits { + #[inline] + fn default() -> Self { + Self { + max_file_bytes: 256 * 1_024, + max_explicit_ids: 128, + } + } +} + +impl NativeBindingsLimits { + /// Checks positive counters and the cap+1 overflow probe. + /// + /// # Errors + /// Rejects zero or unusable counter values. + #[inline] + pub fn validate(&self) -> Result<(), NativeBindingsError> { + if self.max_file_bytes == 0 + || self.max_explicit_ids == 0 + || self.max_file_bytes.checked_add(1).is_none() + || u64::try_from(self.max_file_bytes).is_err() + { + return Err(NativeBindingsError::Invalid); + } + Ok(()) + } +} + +/// Providers present in a resolved target, independent of the running CLI's features. +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +#[expect( + clippy::struct_excessive_bools, + reason = "each provider feature is independent, not a state-machine flag" +)] +pub struct NativeProviderCapabilities { + pub appconfig_agent: bool, + pub secrets_manager: bool, + pub dynamodb: bool, + pub celld_config: bool, + pub celld_kv: bool, +} + +/// Strict configuration dispatch. Each provider validates its own settings. +#[derive(Clone, Debug, Deserialize)] +#[serde(tag = "provider", deny_unknown_fields)] +pub enum NativeConfigBinding { + #[serde(rename = "aws-appconfig-agent")] + AppConfigAgent(AppConfigAgentSettings), + #[serde(rename = "celld-config")] + Celld(CelldConfigSettings), +} + +/// Strict KV dispatch. The physical namespace is independent of the declared ID. +#[derive(Clone, Debug, Deserialize)] +#[serde(tag = "provider", deny_unknown_fields)] +pub enum NativeKvBinding { + #[serde(rename = "aws-dynamodb")] + DynamoDb(DynamoDbKvSettings), + #[serde(rename = "celld-kv")] + Celld(CelldKvSettings), +} + +/// Strict secret dispatch. No arbitrary remote names or per-request discovery. +#[derive(Clone, Debug, Deserialize)] +#[serde(tag = "provider", deny_unknown_fields)] +pub enum NativeSecretBinding { + #[serde(rename = "aws-secrets-manager")] + SecretsManager(SecretsManagerSettings), +} + +/// Independent explicit preparation policy sources. Same-provider sources cannot compete. +#[derive(Clone, Copy, Debug, Default)] +pub struct NativePreparationPolicySources { + pub aws: Option, + pub celld: Option, +} + +/// Resolved provider policies, including defaults for providers not selected. +#[derive(Clone, Copy, Debug)] +pub struct NativePreparationPolicies { + pub aws: AwsPreparationLimits, + pub celld: CelldPreparationLimits, +} + +#[derive(Clone, Debug, Default, Deserialize)] +#[serde(deny_unknown_fields)] +struct LimitsTable { + aws: Option, + celld: Option, +} + +/// Version 1 native document. Logical IDs and lookup keys remain case-sensitive. +#[derive(Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct NativeStoreBindings { + version: u32, + #[serde(default)] + config: BTreeMap, + #[serde(default)] + secrets: BTreeMap, + #[serde(default)] + kv: BTreeMap, + limits: Option, +} + +impl fmt::Debug for NativeStoreBindings { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("NativeStoreBindings") + .finish_non_exhaustive() + } +} + +impl NativeStoreBindings { + /// Builds a typed document with no file or ambient environment access. + #[inline] + #[must_use] + pub fn new( + config: BTreeMap, + secrets: BTreeMap, + ) -> Self { + Self { + version: 1, + config, + secrets, + kv: BTreeMap::new(), + limits: None, + } + } + + /// Parses strict TOML and validates resource shapes and trusted metadata bounds. + /// + /// # Errors + /// Returns fixed categories, never TOML excerpts or raw settings. + #[inline] + pub fn parse(input: &str, limits: NativeBindingsLimits) -> Result { + Self::parse_with_policy(input, limits, None) + } + + /// Parses using exactly one document or explicit remote policy source. + /// + /// # Errors + /// Rejects competing policies and malformed/invalid/over-limit documents. + #[inline] + pub fn parse_with_policy( + input: &str, + limits: NativeBindingsLimits, + aws: Option, + ) -> Result { + Self::parse_with_policies( + input, + limits, + NativePreparationPolicySources { aws, celld: None }, + ) + } + + /// Parses under independent explicit provider policies, before applying mapping caps. + /// + /// # Errors + /// Rejects malformed, conflicting or over-limit inputs without source excerpts. + #[inline] + pub fn parse_with_policies( + input: &str, + limits: NativeBindingsLimits, + policies: NativePreparationPolicySources, + ) -> Result { + limits.validate()?; + if input.len() > limits.max_file_bytes { + return Err(NativeBindingsError::SizeLimit); + } + let bindings: Self = + toml::from_str(input).map_err(|_error| NativeBindingsError::Malformed)?; + bindings.validate_with_policies(limits, policies)?; + Ok(bindings) + } + + /// Reads one explicitly selected absolute regular file, capped before parsing. + /// + /// # Errors + /// Missing, non-file, malformed and cap+1 inputs fail without echoing paths/contents. + #[inline] + pub fn read(path: &Path, limits: NativeBindingsLimits) -> Result { + Self::read_with_policy(path, limits, None) + } + + /// Captures one capped file under an explicit remote limits policy if supplied. + /// + /// # Errors + /// Rejects competing policies, unusable files and malformed/over-limit documents. + #[inline] + pub fn read_with_policy( + path: &Path, + limits: NativeBindingsLimits, + aws: Option, + ) -> Result { + Self::read_with_policies( + path, + limits, + NativePreparationPolicySources { aws, celld: None }, + ) + } + + /// Captures one capped file using independent explicit provider policies. + /// + /// # Errors + /// Rejects unusable files, conflicting policies and invalid documents. + #[inline] + pub fn read_with_policies( + path: &Path, + limits: NativeBindingsLimits, + policies: NativePreparationPolicySources, + ) -> Result { + limits.validate()?; + if !path.is_absolute() { + return Err(NativeBindingsError::Invalid); + } + let mut open = OpenOptions::new(); + open.read(true); + // A named pipe must not block before metadata can reject it. The check + // uses the opened handle, so replacing a path cannot bypass its type. + #[cfg(unix)] + open.custom_flags(O_NONBLOCK); + let file = open + .open(path) + .map_err(|_error| NativeBindingsError::Unavailable)?; + if !file + .metadata() + .map_err(|_error| NativeBindingsError::Unavailable)? + .is_file() + { + return Err(NativeBindingsError::Unavailable); + } + let probe = limits + .max_file_bytes + .checked_add(1) + .ok_or(NativeBindingsError::Invalid)?; + let mut bytes = Vec::new(); + file.take(u64::try_from(probe).map_err(|_error| NativeBindingsError::Invalid)?) + .read_to_end(&mut bytes) + .map_err(|_error| NativeBindingsError::Unavailable)?; + if bytes.len() > limits.max_file_bytes { + return Err(NativeBindingsError::SizeLimit); + } + let input = String::from_utf8(bytes).map_err(|_error| NativeBindingsError::Malformed)?; + Self::parse_with_policies(&input, limits, policies) + } + + /// Checks the whole settings document before constructing providers. + /// + /// # Errors + /// Rejects unsupported schema, mappings, limits and competing policy sources. + #[inline] + pub fn validate( + &self, + limits: NativeBindingsLimits, + explicit_aws: Option, + ) -> Result { + self.validate_with_policies( + limits, + NativePreparationPolicySources { + aws: explicit_aws, + celld: None, + }, + ) + .map(|policies| policies.aws) + } + + /// Validates all settings and resolves each provider policy independently. + /// + /// # Errors + /// Rejects unsupported schema, invalid settings, limits and same-provider conflicts. + #[inline] + pub fn validate_with_policies( + &self, + limits: NativeBindingsLimits, + explicit: NativePreparationPolicySources, + ) -> Result { + limits.validate()?; + if self.version != 1 { + return Err(NativeBindingsError::Invalid); + } + if self + .config + .len() + .checked_add(self.secrets.len()) + .and_then(|count| count.checked_add(self.kv.len())) + .ok_or(NativeBindingsError::SizeLimit)? + > limits.max_explicit_ids + { + return Err(NativeBindingsError::SizeLimit); + } + let file_aws = self.limits.as_ref().and_then(|table| table.aws); + let file_celld = self.limits.as_ref().and_then(|table| table.celld); + if (file_aws.is_some() && explicit.aws.is_some()) + || (file_celld.is_some() && explicit.celld.is_some()) + { + return Err(NativeBindingsError::Duplicate); + } + let policies = NativePreparationPolicies { + aws: explicit.aws.or(file_aws).unwrap_or_default(), + celld: explicit.celld.or(file_celld).unwrap_or_default(), + }; + policies + .aws + .validate() + .map_err(|_category| NativeBindingsError::Invalid)?; + policies + .celld + .validate() + .map_err(|_category| NativeBindingsError::Invalid)?; + let mut aws_mappings = 0_u64; + let mut celld_mappings = 0_u64; + for binding in self.kv.values() { + let counter = match binding { + NativeKvBinding::DynamoDb(settings) => { + settings + .validate() + .map_err(|_category| NativeBindingsError::Invalid)?; + &mut aws_mappings + } + NativeKvBinding::Celld(settings) => { + settings + .validate() + .map_err(|_category| NativeBindingsError::Invalid)?; + &mut celld_mappings + } + }; + charge_mappings(counter, 1)?; + } + for binding in self.config.values() { + let (counter, count) = match binding { + NativeConfigBinding::AppConfigAgent(settings) => { + settings + .validate() + .map_err(|_category| NativeBindingsError::Invalid)?; + (&mut aws_mappings, settings.documents.len()) + } + NativeConfigBinding::Celld(settings) => { + settings + .validate() + .map_err(|_category| NativeBindingsError::Invalid)?; + (&mut celld_mappings, settings.documents.len()) + } + }; + charge_mappings(counter, count)?; + } + for binding in self.secrets.values() { + let NativeSecretBinding::SecretsManager(settings) = binding; + settings + .validate() + .map_err(|_category| NativeBindingsError::Invalid)?; + charge_mappings(&mut aws_mappings, settings.entries.len())?; + } + if aws_mappings > policies.aws.max_remote_values + || celld_mappings > policies.celld.max_remote_values + { + return Err(NativeBindingsError::SizeLimit); + } + Ok(policies) + } + + /// Explicit configuration mappings. + #[inline] + #[must_use] + pub fn config(&self) -> &BTreeMap { + &self.config + } + + /// Explicit live KV bindings. + #[inline] + #[must_use] + pub fn kv(&self) -> &BTreeMap { + &self.kv + } + + /// Adds KV bindings without changing the original config/secrets constructor. + #[inline] + #[must_use] + pub fn with_kv(mut self, kv: BTreeMap) -> Self { + self.kv = kv; + self + } + + /// Explicit secret mappings. + #[inline] + #[must_use] + pub fn secrets(&self) -> &BTreeMap { + &self.secrets + } + + /// Selects one typed remote limits policy; it does not merge policy sources. + #[inline] + #[must_use] + pub fn with_aws_limits(mut self, limits: AwsPreparationLimits) -> Self { + self.limits.get_or_insert_with(LimitsTable::default).aws = Some(limits); + self + } + + /// Selects a celld document policy without discarding an AWS policy. + #[inline] + #[must_use] + pub fn with_celld_limits(mut self, limits: CelldPreparationLimits) -> Self { + self.limits.get_or_insert_with(LimitsTable::default).celld = Some(limits); + self + } +} + +fn charge_mappings(counter: &mut u64, count: usize) -> Result<(), NativeBindingsError> { + *counter = counter + .checked_add(u64::try_from(count).map_err(|_error| NativeBindingsError::SizeLimit)?) + .ok_or(NativeBindingsError::SizeLimit)?; + Ok(()) +} + +/// Cloneable bootstrap selection. File contents are captured once at selection. +#[derive(Clone, Default)] +pub enum NativeStoreBindingsInput { + #[default] + LocalDefaults, + File { + path: PathBuf, + bindings: NativeStoreBindings, + }, + Inline(NativeStoreBindings), +} + +impl NativeStoreBindingsInput { + /// Captured settings, without reopening the selected file. + #[inline] + #[must_use] + pub fn document(&self) -> Option<&NativeStoreBindings> { + match self { + Self::LocalDefaults => None, + Self::File { bindings, .. } | Self::Inline(bindings) => Some(bindings), + } + } +} + +impl fmt::Debug for NativeStoreBindingsInput { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(match self { + Self::LocalDefaults => "LocalDefaults", + Self::File { .. } => "File([redacted])", + Self::Inline(_) => "Inline([redacted])", + }) + } +} + +/// Pure source classification. Custom handles are not inspected or quota-accounted. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum NativeBindingSource { + Local, + Supplied, + BuiltIn, +} + +/// Supplied IDs only. This carries no handles/futures and is usable by offline CLI checks. +#[derive(Default)] +pub struct NativeSuppliedIds { + pub config: BTreeSet, + pub kv: BTreeSet, + pub secrets: BTreeSet, +} + +/// Borrowed declarations for compiled metadata or a CLI-owned portable manifest. +pub struct NativeStoreDeclarations<'declarations> { + pub config: Option>, + pub kv: Option>, + pub secrets: Option>, +} + +/// Names remain borrowed from the caller; validation does not leak or intern IDs. +pub struct NativeStoreDeclaration<'declarations> { + pub ids: Vec<&'declarations str>, + pub default: &'declarations str, +} + +impl From for NativeStoreDeclaration<'static> { + #[inline] + fn from(metadata: StoreMetadata) -> Self { + Self { + ids: metadata.ids.to_vec(), + default: metadata.default, + } + } +} + +impl From for NativeStoreDeclarations<'static> { + #[inline] + fn from(metadata: StoresMetadata) -> Self { + Self { + config: metadata.config.map(Into::into), + kv: metadata.kv.map(Into::into), + secrets: metadata.secrets.map(Into::into), + } + } +} + +impl<'declarations> TryFrom<&'declarations StoreDeclaration> + for NativeStoreDeclaration<'declarations> +{ + type Error = NativeBindingsError; + + #[inline] + fn try_from(declaration: &'declarations StoreDeclaration) -> Result { + if !declaration.legacy.is_empty() + || (declaration.ids.len() > 1 && declaration.default.is_none()) + { + return Err(NativeBindingsError::InvalidMetadata); + } + Ok(Self { + ids: declaration.ids.iter().map(String::as_str).collect(), + default: declaration.default_id(), + }) + } +} + +impl<'declarations> TryFrom<&'declarations ManifestStores> + for NativeStoreDeclarations<'declarations> +{ + type Error = NativeBindingsError; + + #[inline] + fn try_from(stores: &'declarations ManifestStores) -> Result { + Ok(Self { + config: stores + .config + .as_ref() + .map(NativeStoreDeclaration::try_from) + .transpose()?, + kv: stores + .kv + .as_ref() + .map(NativeStoreDeclaration::try_from) + .transpose()?, + secrets: stores + .secrets + .as_ref() + .map(NativeStoreDeclaration::try_from) + .transpose()?, + }) + } +} + +/// Complete per-ID decisions. Registries keep their original metadata/defaults. +pub struct NativeSourceSelection<'declarations> { + pub config: BTreeMap<&'declarations str, NativeBindingSource>, + pub kv: BTreeMap<&'declarations str, NativeBindingSource>, + pub secrets: BTreeMap<&'declarations str, NativeBindingSource>, +} + +impl<'declarations> NativeSourceSelection<'declarations> { + /// Validates metadata, every source and overlay, then checks resolved capabilities. + /// + /// # Errors + /// Duplicate/undeclared sources, scalar collisions and absent target providers fail closed. + #[inline] + pub fn validate>>( + metadata: Declarations, + bindings: &NativeStoreBindings, + supplied: &NativeSuppliedIds, + env: &EnvConfig, + capabilities: NativeProviderCapabilities, + limits: NativeBindingsLimits, + aws: Option, + ) -> Result { + Self::validate_with_policies( + metadata, + bindings, + supplied, + env, + capabilities, + limits, + NativePreparationPolicySources { aws, celld: None }, + ) + } + + /// Validates all sources with independent provider policies and actual-variant capabilities. + /// + /// # Errors + /// Rejects invalid declarations, settings, conflicts and unavailable selected providers. + #[inline] + pub fn validate_with_policies>>( + metadata: Declarations, + bindings: &NativeStoreBindings, + supplied: &NativeSuppliedIds, + env: &EnvConfig, + capabilities: NativeProviderCapabilities, + limits: NativeBindingsLimits, + policies: NativePreparationPolicySources, + ) -> Result { + bindings.validate_with_policies(limits, policies)?; + let declarations = metadata.into(); + let selection = Self { + config: resolve( + declarations.config, + &supplied.config, + bindings.config.keys(), + env, + "config", + )?, + kv: resolve(declarations.kv, &supplied.kv, bindings.kv.keys(), env, "kv")?, + secrets: resolve( + declarations.secrets, + &supplied.secrets, + bindings.secrets.keys(), + env, + "secrets", + )?, + }; + if bindings.config.values().any(|binding| match binding { + NativeConfigBinding::AppConfigAgent(_) => !capabilities.appconfig_agent, + NativeConfigBinding::Celld(_) => !capabilities.celld_config, + }) || (!bindings.secrets.is_empty() && !capabilities.secrets_manager) + || bindings.kv.values().any(|binding| match binding { + NativeKvBinding::DynamoDb(_) => !capabilities.dynamodb, + NativeKvBinding::Celld(_) => !capabilities.celld_kv, + }) + { + return Err(NativeBindingsError::UnsupportedProvider); + } + Ok(selection) + } +} + +fn resolve<'declarations, 'settings, Ids: Iterator>( + metadata: Option>, + supplied: &BTreeSet, + bound: Ids, + env: &EnvConfig, + kind: &str, +) -> Result, NativeBindingsError> { + let settings: BTreeSet<&str> = bound.map(String::as_str).collect(); + let Some(meta) = metadata else { + return if supplied.is_empty() && settings.is_empty() { + Ok(BTreeMap::new()) + } else { + Err(NativeBindingsError::Undeclared) + }; + }; + if meta.ids.is_empty() + || !meta.ids.contains(&meta.default) + || meta.ids.iter().enumerate().any(|(index, id)| { + id.is_empty() + || !id + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || byte == b'_') + || meta.ids.iter().take(index).any(|previous| previous == id) + }) + { + return Err(NativeBindingsError::InvalidMetadata); + } + if supplied.iter().any(|id| !meta.ids.contains(&id.as_str())) + || settings.iter().any(|id| !meta.ids.contains(id)) + { + return Err(NativeBindingsError::Undeclared); + } + let mut result = BTreeMap::new(); + for id in meta.ids { + let source = match (supplied.contains(id), settings.contains(id)) { + (true, true) => return Err(NativeBindingsError::Duplicate), + (true, false) => NativeBindingSource::Supplied, + (false, true) => NativeBindingSource::BuiltIn, + (false, false) => NativeBindingSource::Local, + }; + if source != NativeBindingSource::Local + && ["name", "key"] + .iter() + .any(|field| env.store_setting(kind, id, field).is_some()) + { + return Err(NativeBindingsError::OverlayConflict); + } + if source == NativeBindingSource::Local + && ["name", "key"].iter().any(|field| { + env.store_setting(kind, id, field).is_some_and(|value| { + value.trim().is_empty() || value.chars().any(char::is_control) + }) + }) + { + return Err(NativeBindingsError::Invalid); + } + if kind == "secrets" && source != NativeBindingSource::Supplied { + let namespace = if source == NativeBindingSource::Local { + env.store_setting(kind, id, "name").map_or(id, str::trim) + } else { + id + }; + if namespace.len() > MAX_NAME_LEN { + return Err( + if source == NativeBindingSource::Local + && env.store_setting(kind, id, "name").is_some() + { + NativeBindingsError::Invalid + } else { + NativeBindingsError::InvalidMetadata + }, + ); + } + } + result.insert(id, source); + } + Ok(result) +} + +#[cfg(test)] +mod tests { + #![expect( + clippy::assertions_on_result_states, + reason = "strict parser tests compare accepted/rejected input matrices" + )] + #![expect( + clippy::panic_in_result_fn, + reason = "tests use fallible setup and assertions" + )] + + use std::collections::{BTreeMap, BTreeSet}; + use std::error::Error; + use std::fmt::Write as _; + use std::fs; + use std::path::Path; + + use edgezero_core::app::{StoreMetadata, StoresMetadata}; + use edgezero_core::env_config::EnvConfig; + use edgezero_core::manifest::ManifestStores; + use edgezero_store_aws::settings::AwsPreparationLimits; + use edgezero_store_celld::settings::CelldPreparationLimits; + + use super::{ + NativeBindingSource, NativeBindingsError, NativeBindingsLimits, + NativePreparationPolicySources, NativeProviderCapabilities, NativeSourceSelection, + NativeStoreBindings, NativeStoreDeclarations, NativeSuppliedIds, + }; + + const CONFIG: &str = "version = 1\n[config.app]\nprovider = 'aws-appconfig-agent'\ndefault_key = 'settings'\nendpoint = 'http://127.0.0.1:2772'\n[config.app.documents.settings]\napplication = 'example-app'\nenvironment = 'production'\nprofile = 'envelope'\n"; + + const CELLD: &str = "version=1\n[config.app]\nprovider='celld-config'\nendpoint='http://127.0.0.1:8788'\ndefault_key='settings'\n[config.app.access]\nmode='trusted-network'\n[config.app.documents.settings]\nstore='config-fixture'\nkey='settings'\n[kv.state]\nprovider='celld-kv'\nendpoint='http://127.0.0.1:8788'\nstore='state-fixture'\n[kv.state.access]\nmode='trusted-network'\n"; + + #[test] + fn celld_schema_is_structural_strict_and_settings_only() -> Result<(), NativeBindingsError> { + let limits = NativeBindingsLimits::default(); + for input in [ + CELLD.to_owned(), + CELLD + .replace("http://127.0.0.1:8788", "https://unreachable.invalid") + .replace( + "mode='trusted-network'", + "mode='authenticated'\ntoken_env='ABSENT_TOKEN'", + ), + ] { + let document = NativeStoreBindings::parse(&input, limits)?; + assert_eq!(document.config().len(), 1); + assert_eq!(document.kv().len(), 1); + assert!(!format!("{document:?}").contains("fixture")); + } + for bad in [ + CELLD.replace("celld-config", "celld-unknown"), + CELLD.replace("[config.app]", "[secrets.app]"), + CELLD.replace("default_key='settings'", "default_key='missing'"), + CELLD.replace("store='state-fixture'", "store='bad/store'"), + CELLD.replace("mode='trusted-network'", "mode='authenticated'"), + CELLD.replace( + "mode='trusted-network'", + "mode='trusted-network'\ntoken_env='SENTINEL'", + ), + CELLD.replace("http://127.0.0.1:8788", "http://private.invalid"), + format!("{CELLD}\n[limits.celld]\noperation_timeout_ms=0"), + format!("{CELLD}\n[limits.celld]\nunknown=1"), + ] { + let error = NativeStoreBindings::parse(&bad, limits) + .err() + .ok_or(NativeBindingsError::Invalid)?; + assert!(error.source().is_none()); + assert!(!format!("{error:?}").contains("SENTINEL")); + } + Ok(()) + } + + #[test] + fn capabilities_follow_actual_variants() -> Result<(), NativeBindingsError> { + let limits = NativeBindingsLimits::default(); + let aws_kv = "\n[kv.state]\nprovider='aws-dynamodb'\nregion='us-east-1'\ntable='fixture-table'\nnamespace='fixture-space'\n"; + let celld_kv = CELLD + .split("[kv.state]") + .nth(1) + .ok_or(NativeBindingsError::Invalid)?; + let celld_config = CELLD + .split("[kv.state]") + .next() + .ok_or(NativeBindingsError::Invalid)?; + for (input, aws_config, aws_state, celld_app, celld_state) in [ + (CELLD.to_owned(), false, false, true, true), + (format!("{CONFIG}{aws_kv}"), true, true, false, false), + ( + format!("{CONFIG}[kv.state]{celld_kv}"), + true, + false, + false, + true, + ), + (format!("{celld_config}{aws_kv}"), false, true, true, false), + ] { + let document = NativeStoreBindings::parse(&input, limits)?; + for mask in 0_u8..16 { + let caps = NativeProviderCapabilities { + appconfig_agent: mask & 1 != 0, + dynamodb: mask & 2 != 0, + celld_config: mask & 4 != 0, + celld_kv: mask & 8 != 0, + secrets_manager: false, + }; + let accepted = (!aws_config || caps.appconfig_agent) + && (!aws_state || caps.dynamodb) + && (!celld_app || caps.celld_config) + && (!celld_state || caps.celld_kv); + assert_eq!( + NativeSourceSelection::validate( + metadata(), + &document, + &NativeSuppliedIds::default(), + &EnvConfig::default(), + caps, + limits, + None + ) + .is_ok(), + accepted + ); + } + } + Ok(()) + } + + #[test] + fn provider_policy_sources_and_mapping_counters_are_independent() + -> Result<(), NativeBindingsError> { + let limits = NativeBindingsLimits::default(); + let aws = AwsPreparationLimits { + max_remote_values: 1, + ..Default::default() + }; + let celld = CelldPreparationLimits { + max_remote_values: 2, + ..Default::default() + }; + let policies = NativePreparationPolicySources { + aws: Some(aws), + celld: Some(celld), + }; + let mixed = format!( + "{CONFIG}{}", + CELLD + .replace("version=1\n", "") + .replace("config.app", "config.remote") + ); + NativeStoreBindings::parse_with_policies(&mixed, limits, policies)?; + let too_many = format!( + "{mixed}\n[config.remote.documents.alias]\nstore='config-fixture'\nkey='settings'\n" + ); + assert_eq!( + NativeStoreBindings::parse_with_policies(&too_many, limits, policies).err(), + Some(NativeBindingsError::SizeLimit) + ); + let reverse = NativePreparationPolicySources { + aws: Some(AwsPreparationLimits { + max_remote_values: 2, + ..Default::default() + }), + celld: Some(CelldPreparationLimits { + max_remote_values: 1, + ..Default::default() + }), + }; + assert_eq!( + NativeStoreBindings::parse_with_policies(&mixed, limits, reverse).err(), + Some(NativeBindingsError::SizeLimit) + ); + for (table, explicit) in [ + ( + "[limits.aws]\nmax_remote_values=1", + NativePreparationPolicySources { + aws: None, + celld: Some(celld), + }, + ), + ( + "[limits.celld]\nmax_remote_values=2", + NativePreparationPolicySources { + aws: Some(aws), + celld: None, + }, + ), + ( + "[limits.aws]\nmax_remote_values=1\n[limits.celld]\nmax_remote_values=2", + NativePreparationPolicySources::default(), + ), + ] { + let document = NativeStoreBindings::parse_with_policies( + &format!("{mixed}\n{table}"), + limits, + explicit, + )?; + let resolved = document.validate_with_policies(limits, explicit)?; + assert_eq!(resolved.aws.max_remote_values, 1); + assert_eq!(resolved.celld.max_remote_values, 2); + assert!(matches!( + document.validate_with_policies(limits, policies), + Err(NativeBindingsError::Duplicate) + )); + } + let document = NativeStoreBindings::parse(&mixed, limits)?; + for typed in [ + document + .clone() + .with_aws_limits(aws) + .with_celld_limits(celld), + document.with_celld_limits(celld).with_aws_limits(aws), + ] { + let resolved = + typed.validate_with_policies(limits, NativePreparationPolicySources::default())?; + assert_eq!(resolved.aws.max_remote_values, 1); + assert_eq!(resolved.celld.max_remote_values, 2); + } + Ok(()) + } + + #[test] + fn explicit_celld_policy_raises_caps_during_file_parse() -> Result<(), Box> { + let mut input = CELLD.to_owned(); + for index in 0..1_024_u32 { + writeln!( + input, + "\n[config.app.documents.alias{index}]\nstore='config-fixture'\nkey='settings'" + )?; + } + let limits = NativeBindingsLimits::default(); + assert_eq!( + NativeStoreBindings::parse(&input, limits).err(), + Some(NativeBindingsError::SizeLimit) + ); + let explicit = NativePreparationPolicySources { + celld: Some(CelldPreparationLimits { + max_remote_values: 2_048, + ..Default::default() + }), + aws: None, + }; + let root = tempfile::tempdir()?; + let path = root.path().join("bindings.toml"); + fs::write(&path, input)?; + NativeStoreBindings::read_with_policies(&path, limits, explicit)?; + Ok(()) + } + + fn metadata() -> StoresMetadata { + StoresMetadata { + config: Some(StoreMetadata { + ids: &["app", "local"], + default: "app", + }), + kv: Some(StoreMetadata { + ids: &["state"], + default: "state", + }), + secrets: None, + } + } + + #[test] + fn dynamodb_binding_source_is_per_id_and_fails_closed() -> Result<(), NativeBindingsError> { + let limits = NativeBindingsLimits::default(); + let bindings = NativeStoreBindings::parse( + "version=1\n[kv.state]\nprovider='aws-dynamodb'\nregion='us-east-1'\ntable='edgezero-kv'\nnamespace='private-space'", + limits, + )?; + let capabilities = NativeProviderCapabilities { + dynamodb: true, + ..Default::default() + }; + let selection = NativeSourceSelection::validate( + metadata(), + &bindings, + &NativeSuppliedIds::default(), + &EnvConfig::default(), + capabilities, + limits, + None, + )?; + assert_eq!( + selection.kv.get("state"), + Some(&NativeBindingSource::BuiltIn) + ); + assert!(matches!( + NativeSourceSelection::validate( + metadata(), + &bindings, + &NativeSuppliedIds::default(), + &EnvConfig::default(), + NativeProviderCapabilities::default(), + limits, + None + ), + Err(NativeBindingsError::UnsupportedProvider) + )); + let supplied = NativeSuppliedIds { + kv: ["state".to_owned()].into_iter().collect(), + ..Default::default() + }; + assert!(matches!( + NativeSourceSelection::validate( + metadata(), + &bindings, + &supplied, + &EnvConfig::default(), + capabilities, + limits, + None + ), + Err(NativeBindingsError::Duplicate) + )); + assert!(!format!("{bindings:?}").contains("private-space")); + Ok(()) + } + + #[test] + fn manifest_ids_remain_borrowed_and_invalid_declarations_fail() -> Result<(), Box> { + let owned: ManifestStores = + toml::from_str("[config]\nids = ['app', 'local']\ndefault = 'app'\n")?; + let bindings = NativeStoreBindings::parse(CONFIG, NativeBindingsLimits::default())?; + let selection = NativeSourceSelection::validate( + NativeStoreDeclarations::try_from(&owned)?, + &bindings, + &NativeSuppliedIds::default(), + &EnvConfig::default(), + NativeProviderCapabilities { + appconfig_agent: true, + secrets_manager: false, + dynamodb: false, + ..Default::default() + }, + NativeBindingsLimits::default(), + None, + )?; + assert_eq!( + selection.config.get("app"), + Some(&NativeBindingSource::BuiltIn) + ); + let stored_id = selection + .config + .get_key_value("app") + .ok_or(NativeBindingsError::InvalidMetadata)? + .0; + let original_id = &owned + .config + .as_ref() + .ok_or(NativeBindingsError::InvalidMetadata)? + .ids[0]; + assert_eq!(stored_id.as_ptr(), original_id.as_ptr()); + for input in [ + "[config]\nids = ['app', 'local']\n", + "[config]\nids = ['app']\nname = 'legacy'\n", + ] { + let invalid: ManifestStores = toml::from_str(input)?; + assert!(matches!( + NativeStoreDeclarations::try_from(&invalid), + Err(NativeBindingsError::InvalidMetadata) + )); + } + Ok(()) + } + + #[test] + fn secret_namespace_limits_are_checked_before_preparation() { + let metadata = StoresMetadata { + config: None, + kv: None, + secrets: Some(StoreMetadata { + ids: &["signing"], + default: "signing", + }), + }; + let bindings = NativeStoreBindings::new(BTreeMap::new(), BTreeMap::new()); + for (bytes, accepted) in [(512, true), (513, false)] { + let env = EnvConfig::from_vars([( + "EDGEZERO__STORES__SECRETS__SIGNING__NAME", + "x".repeat(bytes), + )]); + let result = NativeSourceSelection::validate( + metadata, + &bindings, + &NativeSuppliedIds::default(), + &env, + NativeProviderCapabilities::default(), + NativeBindingsLimits::default(), + None, + ); + assert_eq!(result.is_ok(), accepted); + } + } + + #[test] + fn strict_tagged_schema_and_case_sensitive_mappings() { + let limits = NativeBindingsLimits::default(); + assert!(NativeStoreBindings::parse(CONFIG, limits).is_ok()); + for input in [ + CONFIG.replace("version = 1", "version = 2"), + CONFIG.replace("version = 1", "version = 1\nunknown = 'sentinel'"), + CONFIG.replace("aws-appconfig-agent", "dynamodb"), + CONFIG.replace("default_key = 'settings'", "default_key = 'Settings'"), + CONFIG.replace("endpoint =", "unknown = 'sentinel'\nendpoint ="), + CONFIG.replace("application =", "unknown = 'sentinel'\napplication ="), + CONFIG.replace("[config.app.documents.settings]", "[config.app.documents]"), + format!("{CONFIG}\n[kv.state]\nprovider = 'dynamodb'"), + format!("{CONFIG}\n[unknown]\nvalue = 'sentinel'"), + CONFIG.replace("version = 1", "version = 1\nversion = 1"), + format!("{CONFIG}\n[config.app]\nprovider = 'aws-appconfig-agent'"), + format!("{CONFIG}\n[config.app.documents.settings]\nprofile = 'sentinel'"), + ] { + let error = NativeStoreBindings::parse(&input, limits).err(); + assert!(error.is_some(), "accepted malformed bindings"); + assert!(!format!("{error:?}").contains("sentinel")); + } + } + + #[test] + fn capped_file_read_and_trusted_id_limit() -> Result<(), Box> { + let directory = tempfile::tempdir()?; + let path = directory.path().join("sentinel.toml"); + let limits = NativeBindingsLimits { + max_file_bytes: CONFIG.len(), + max_explicit_ids: 1, + }; + fs::write(&path, CONFIG)?; + assert!(NativeStoreBindings::read(&path, limits).is_ok()); + fs::write(&path, format!("{CONFIG} "))?; + assert_eq!( + NativeStoreBindings::read(&path, limits).err(), + Some(NativeBindingsError::SizeLimit) + ); + fs::write(&path, [0xff])?; + let malformed = NativeStoreBindings::read(&path, limits) + .err() + .ok_or(NativeBindingsError::Invalid)?; + assert_eq!(malformed, NativeBindingsError::Malformed); + assert!(!format!("{malformed:?}").contains("sentinel")); + assert!(malformed.source().is_none()); + assert_eq!( + NativeStoreBindings::read(directory.path(), limits).err(), + Some(NativeBindingsError::Unavailable) + ); + assert_eq!( + NativeStoreBindings::read(Path::new("relative.toml"), limits).err(), + Some(NativeBindingsError::Invalid) + ); + let mut many = String::from("version = 1\n"); + for index in 0..=128_u32 { + many.push_str( + &CONFIG + .replace("version = 1\n", "") + .replace("config.app", &format!("config.store{index}")), + ); + if index == 127 { + assert!(NativeStoreBindings::parse(&many, NativeBindingsLimits::default()).is_ok()); + } + } + assert_eq!( + NativeStoreBindings::parse(&many, NativeBindingsLimits::default()).err(), + Some(NativeBindingsError::SizeLimit) + ); + Ok(()) + } + + #[test] + fn whole_source_validation_and_target_capabilities() -> Result<(), NativeBindingsError> { + let limits = NativeBindingsLimits::default(); + let bindings = NativeStoreBindings::parse(CONFIG, limits)?; + let available = NativeProviderCapabilities { + appconfig_agent: true, + secrets_manager: false, + dynamodb: false, + ..Default::default() + }; + let mut supplied = NativeSuppliedIds::default(); + let env = EnvConfig::default(); + let selection = NativeSourceSelection::validate( + metadata(), + &bindings, + &supplied, + &env, + available, + limits, + None, + )?; + assert_eq!( + selection.config.get("app"), + Some(&NativeBindingSource::BuiltIn) + ); + assert_eq!( + selection.config.get("local"), + Some(&NativeBindingSource::Local) + ); + assert_eq!(selection.kv.get("state"), Some(&NativeBindingSource::Local)); + assert!(matches!( + NativeSourceSelection::validate( + metadata(), + &bindings, + &supplied, + &env, + NativeProviderCapabilities::default(), + limits, + None + ), + Err(NativeBindingsError::UnsupportedProvider) + )); + supplied.config = BTreeSet::from(["app".to_owned()]); + assert!(matches!( + NativeSourceSelection::validate( + metadata(), + &bindings, + &supplied, + &env, + available, + limits, + None + ), + Err(NativeBindingsError::Duplicate) + )); + supplied.config = BTreeSet::from(["APP".to_owned()]); + assert!(matches!( + NativeSourceSelection::validate( + metadata(), + &bindings, + &supplied, + &env, + available, + limits, + None + ), + Err(NativeBindingsError::Undeclared) + )); + supplied.config.clear(); + for field in ["NAME", "KEY"] { + let overlay = + EnvConfig::from_vars([(format!("EDGEZERO__STORES__CONFIG__APP__{field}"), "")]); + assert!(matches!( + NativeSourceSelection::validate( + metadata(), + &bindings, + &supplied, + &overlay, + available, + limits, + None + ), + Err(NativeBindingsError::OverlayConflict) + )); + } + Ok(()) + } + + #[test] + fn explicit_policy_can_raise_mapping_caps_before_file_parsing() -> Result<(), Box> { + let mut input = CONFIG.to_owned(); + for index in 0..1_024_u32 { + writeln!( + input, + "\n[config.app.documents.alias{index}]\napplication = 'example-app'\nenvironment = 'production'\nprofile = 'envelope'" + )?; + } + let limits = NativeBindingsLimits::default(); + assert_eq!( + NativeStoreBindings::parse(&input, limits).err(), + Some(NativeBindingsError::SizeLimit) + ); + let aws = AwsPreparationLimits { + max_remote_values: 2_048, + ..AwsPreparationLimits::default() + }; + let root = tempfile::tempdir()?; + let path = root.path().join("bindings.toml"); + fs::write(&path, input)?; + let bindings = NativeStoreBindings::read_with_policy(&path, limits, Some(aws))?; + assert_eq!( + bindings.validate(limits, Some(aws))?.max_remote_values, + 2_048 + ); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn named_pipe_is_rejected_without_waiting_for_a_writer() -> Result<(), Box> { + use std::process::Command; + let root = tempfile::tempdir()?; + let path = root.path().join("bindings.fifo"); + assert!(Command::new("mkfifo").arg(&path).status()?.success()); + assert_eq!( + NativeStoreBindings::read(&path, NativeBindingsLimits::default()).err(), + Some(NativeBindingsError::Unavailable) + ); + Ok(()) + } + + #[test] + fn secret_schema_rejects_irrelevant_fields_and_preserves_selectors() + -> Result<(), NativeBindingsError> { + let input = "version = 1\n[secrets.signing]\nprovider = 'aws-secrets-manager'\nregion = 'us-east-1'\n[secrets.signing.entries.active]\nsecret_id = 'arn:aws:secretsmanager:us-east-1:111122223333:secret:SENTINEL-AbCdEf'\nversion_stage = 'AWSCURRENT'\n"; + let limits = NativeBindingsLimits::default(); + let document = NativeStoreBindings::parse(input, limits)?; + assert_eq!(document.secrets().len(), 1); + assert!(!format!("{document:?}").contains("SENTINEL")); + for bad in [ + input.replace("region =", "endpoint = 'http://127.0.0.1'\nregion ="), + input.replace("version_stage =", "unknown = 'SENTINEL'\nversion_stage ="), + input.replace("us-east-1'\n[secrets", "us-west-2'\n[secrets"), + input.replace( + "version_stage =", + "version_id = '12345678901234567890123456789012'\nversion_stage =", + ), + input.replace("secret_id =", "secret_id = 'partial-name'\nother ="), + ] { + let error = NativeStoreBindings::parse(&bad, limits) + .err() + .ok_or(NativeBindingsError::Invalid)?; + assert!(!format!("{error:?}").contains("SENTINEL")); + assert!(error.source().is_none()); + } + Ok(()) + } + + #[test] + fn exact_mapping_boundary_uses_one_file_or_explicit_policy() -> Result<(), NativeBindingsError> + { + let limits = NativeBindingsLimits::default(); + let aws = AwsPreparationLimits { + max_remote_values: 1, + ..AwsPreparationLimits::default() + }; + let explicit = NativeStoreBindings::parse_with_policy(CONFIG, limits, Some(aws))?; + assert_eq!(explicit.validate(limits, Some(aws))?, aws); + let file = NativeStoreBindings::parse( + &format!("{CONFIG}\n[limits.aws]\nmax_remote_values = 1"), + limits, + )?; + assert_eq!(file.validate(limits, None)?, aws); + let alias = "\n[config.app.documents.alias]\napplication = 'example-app'\nenvironment = 'production'\nprofile = 'envelope'\n"; + assert_eq!( + NativeStoreBindings::parse_with_policy(&format!("{CONFIG}{alias}"), limits, Some(aws)) + .err(), + Some(NativeBindingsError::SizeLimit) + ); + assert_eq!( + NativeStoreBindings::parse( + &format!("{CONFIG}{alias}\n[limits.aws]\nmax_remote_values = 1"), + limits + ) + .err(), + Some(NativeBindingsError::SizeLimit) + ); + Ok(()) + } + + #[test] + fn limit_policy_sources_do_not_merge_and_aliases_count_before_dedup() + -> Result<(), NativeBindingsError> { + let limits = NativeBindingsLimits::default(); + let with_policy = format!("{CONFIG}\n[limits.aws]\nmax_snapshot_bytes = 16777216"); + let bindings = NativeStoreBindings::parse(&with_policy, limits)?; + assert_eq!( + bindings.validate(limits, Some(AwsPreparationLimits::default())), + Err(NativeBindingsError::Duplicate) + ); + let aliases = format!( + "{CONFIG}\n[config.app.documents.alias]\napplication = 'example-app'\nenvironment = 'production'\nprofile = 'envelope'\n[limits.aws]\nmax_remote_values = 1" + ); + assert_eq!( + NativeStoreBindings::parse(&aliases, limits).err(), + Some(NativeBindingsError::SizeLimit) + ); + Ok(()) + } +} diff --git a/crates/edgezero-adapter-axum/src/native_build.rs b/crates/edgezero-adapter-axum/src/native_build.rs new file mode 100644 index 000000000..d76d62399 --- /dev/null +++ b/crates/edgezero-adapter-axum/src/native_build.rs @@ -0,0 +1,310 @@ +//! SDK-free native Cargo selection and exact generated Axum command matching. + +use std::collections::BTreeSet; + +/// Normalized Cargo feature and target inputs shared by native CLI paths. +#[derive(Clone, Debug, Default, Eq, PartialEq)] +#[must_use] +pub struct NativeCargoSelection { + /// Whether Cargo should enable every package feature. + pub all_features: bool, + /// Non-feature Cargo args before `--`. + pub cargo_args: Vec, + /// Whether Cargo should suppress package defaults. + pub no_default_features: bool, + /// Explicit additions, including `[adapters.axum.build].features`. + pub requested_features: Vec, + /// Arguments after Cargo's `--`, for `cargo run` only. + pub run_args: Vec, + /// Cargo target triple. `None` means Cargo's host target. + pub target: Option, +} + +impl NativeCargoSelection { + /// Cargo arguments in normalized form; runtime args are intentionally omitted. + #[inline] + #[must_use] + pub fn cargo_args(&self) -> Vec { + let mut args = self.cargo_args.clone(); + if !self.requested_features.is_empty() { + args.extend(["--features".to_owned(), self.requested_features.join(",")]); + } + args + } + + /// Merge manifest features with Cargo feature controls, preserving unrelated + /// arguments and separating runtime args after `--`. + /// + /// # Errors + /// Returns an error for missing/empty feature lists, missing or conflicting + /// targets, or the non-Cargo `native` target sentinel. + #[inline] + pub fn normalize_args(manifest_features: &[String], args: &[String]) -> Result { + let mut selection = Self::default(); + let mut feature_values = Vec::new(); + let mut pending_value: Option<&str> = None; + let mut after_separator = false; + + for arg in args { + if let Some(option) = pending_value.take() { + if option == "--target" { + selection.set_target(arg)?; + selection + .cargo_args + .extend([option.to_owned(), arg.clone()]); + } else { + feature_values.extend(split_features(arg)?); + } + continue; + } + if after_separator { + selection.run_args.push(arg.clone()); + continue; + } + match arg.as_str() { + "--" => after_separator = true, + "--features" | "-F" | "--target" => pending_value = Some(arg), + "--all-features" => { + selection.all_features = true; + selection.cargo_args.push(arg.clone()); + } + "--no-default-features" => { + selection.no_default_features = true; + selection.cargo_args.push(arg.clone()); + } + _ => { + if let Some(value) = arg.strip_prefix("--features=") { + feature_values.extend(split_features(value)?); + } else if let Some(value) = arg.strip_prefix("-F") { + if value.is_empty() { + selection.cargo_args.push(arg.clone()); + } else { + feature_values.extend(split_features(value)?); + } + } else if let Some(value) = arg.strip_prefix("--target=") { + selection.set_target(value)?; + selection.cargo_args.push(arg.clone()); + } else { + selection.cargo_args.push(arg.clone()); + } + } + } + } + if let Some(option) = pending_value { + return Err(if option == "--target" { + "--target requires a triple".to_owned() + } else { + format!("{option} requires a feature list") + }); + } + + let mut seen = BTreeSet::new(); + for feature in manifest_features.iter().chain(feature_values.iter()) { + for normalized in split_features(feature)? { + if seen.insert(normalized.clone()) { + selection.requested_features.push(normalized); + } + } + } + Ok(selection) + } + + fn set_target(&mut self, target: &str) -> Result<(), String> { + if target.trim().is_empty() { + return Err("--target requires a non-empty target triple".to_owned()); + } + if target == "native" { + return Err("`native` selects Cargo's host target; omit `--target`".to_owned()); + } + match &self.target { + Some(previous) if previous != target => { + Err("conflicting --target selections; refusing to silently choose one".to_owned()) + } + _ => { + self.target = Some(target.to_owned()); + Ok(()) + } + } + } +} + +/// Recognize only the exact generated Axum command base, never arbitrary shell text. +/// +/// # Examples +/// ``` +/// assert_eq!( +/// edgezero_adapter_axum::native_build::generated_axum_cargo_action("cargo build -p app"), +/// Some(("build", "app")), +/// ); +/// ``` +#[inline] +#[must_use] +pub fn generated_axum_cargo_action(command: &str) -> Option<(&'static str, &str)> { + let mut words = command.split_ascii_whitespace(); + let (Some("cargo"), Some(action_word), Some("-p"), Some(package), None) = ( + words.next(), + words.next(), + words.next(), + words.next(), + words.next(), + ) else { + return None; + }; + let action = match action_word { + "build" => "build", + "run" => "run", + _ => return None, + }; + if !valid_package_token(package) || command != format!("cargo {action} -p {package}") { + return None; + } + Some((action, package)) +} + +fn split_features(value: &str) -> Result, String> { + let values: Vec<_> = value + .split(|ch: char| ch == ',' || ch.is_ascii_whitespace()) + .filter(|feature| !feature.is_empty()) + .map(str::to_owned) + .collect(); + if values.is_empty() { + return Err("Cargo feature list must not be empty".to_owned()); + } + Ok(values) +} + +fn valid_package_token(package: &str) -> bool { + let mut bytes = package.bytes(); + let Some(first) = bytes.next() else { + return false; + }; + first.is_ascii_alphanumeric() + && bytes.all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_')) +} + +#[cfg(test)] +mod tests { + use super::{NativeCargoSelection, generated_axum_cargo_action}; + + fn strings(values: &[&str]) -> Vec { + values.iter().map(|value| (*value).to_owned()).collect() + } + + #[test] + fn controls_after_separator_are_runtime_args() { + let selection = NativeCargoSelection::normalize_args( + &[], + &strings(&["--features", "compile", "--", "--features", "runtime"]), + ) + .expect("feature controls should parse"); + assert_eq!(selection.requested_features, strings(&["compile"])); + assert_eq!(selection.run_args, strings(&["--features", "runtime"])); + } + + #[test] + fn normalizes_repeated_features_and_preserves_cargo_controls() { + let selection = NativeCargoSelection::normalize_args( + &strings(&["aws-appconfig-agent", "shared"]), + &strings(&[ + "--features", + "shared,aws-secrets-manager", + "-F", + "logging extra\ttabbed", + "--features=metrics", + "-Fcache", + "--all-features", + "--no-default-features", + "--target", + "aarch64-unknown-linux-gnu", + "--locked", + ]), + ) + .expect("Cargo controls should normalize"); + assert_eq!( + selection.requested_features, + strings(&[ + "aws-appconfig-agent", + "shared", + "aws-secrets-manager", + "logging", + "extra", + "tabbed", + "metrics", + "cache", + ]) + ); + assert!(selection.all_features); + assert!(selection.no_default_features); + assert_eq!( + selection.target.as_deref(), + Some("aarch64-unknown-linux-gnu") + ); + assert_eq!( + selection.cargo_args(), + strings(&[ + "--all-features", + "--no-default-features", + "--target", + "aarch64-unknown-linux-gnu", + "--locked", + "--features", + "aws-appconfig-agent,shared,aws-secrets-manager,logging,extra,tabbed,metrics,cache", + ]) + ); + } + + #[test] + fn recognizes_only_exact_generated_package_commands() { + assert_eq!( + generated_axum_cargo_action("cargo build -p app_cli"), + Some(("build", "app_cli")) + ); + assert_eq!( + generated_axum_cargo_action("cargo run -p app-cli"), + Some(("run", "app-cli")) + ); + for command in [ + "cargo build -p app; touch /tmp/x", + "cargo build -p app | cat", + "cargo build -p 'app'", + "cargo build -p caf\u{e9}", + "cargo build -p app.extra", + "CARGO build -p app", + "cargo run -p app -- --help", + "cargo\tbuild -p app", + ] { + assert_eq!(generated_axum_cargo_action(command), None, "{command}"); + } + } + + #[test] + fn rejects_missing_features_and_invalid_or_conflicting_targets() { + assert_eq!( + NativeCargoSelection::normalize_args(&[], &strings(&["--features"])) + .expect_err("missing --features value must fail"), + "--features requires a feature list" + ); + assert_eq!( + NativeCargoSelection::normalize_args(&[], &strings(&["-F"])) + .expect_err("missing -F value must fail"), + "-F requires a feature list" + ); + assert_eq!( + NativeCargoSelection::normalize_args( + &[], + &strings(&[ + "--target", + "x86_64-unknown-linux-gnu", + "--target=aarch64-unknown-linux-gnu", + ]), + ) + .expect_err("conflicting targets must fail"), + "conflicting --target selections; refusing to silently choose one" + ); + assert_eq!( + NativeCargoSelection::normalize_args(&[], &strings(&["--target=native"])) + .expect_err("native is not a Cargo target triple"), + "`native` selects Cargo's host target; omit `--target`" + ); + } +} diff --git a/crates/edgezero-adapter-axum/src/native_stores.rs b/crates/edgezero-adapter-axum/src/native_stores.rs new file mode 100644 index 000000000..f06f5150e --- /dev/null +++ b/crates/edgezero-adapter-axum/src/native_stores.rs @@ -0,0 +1,1008 @@ +//! Native per-ID provider inputs. Deferred opening belongs inside the future body. + +#![expect( + clippy::arbitrary_source_item_ordering, + reason = "typed inputs and per-kind insertion methods precede source validation and preparation" +)] + +use std::collections::{BTreeMap, btree_map::Entry}; +use std::convert::Infallible; +use std::fmt; +use std::future::Future; +use std::pin::Pin; + +use edgezero_core::app::StoresMetadata; +use edgezero_core::key_value_store::KvHandle; +use edgezero_core::store_registry::{BoundSecretStore, ConfigStoreBinding}; +#[cfg(any( + feature = "aws-appconfig-agent", + feature = "aws-secrets-manager", + feature = "aws-dynamodb", + feature = "celld-config", + feature = "celld-kv" +))] +use edgezero_core::time::MonotonicClock; +#[cfg(any( + feature = "aws-appconfig-agent", + feature = "aws-secrets-manager", + feature = "aws-dynamodb" +))] +use edgezero_store_aws::AwsPreparation; +#[cfg(not(any( + feature = "aws-appconfig-agent", + feature = "aws-secrets-manager", + feature = "aws-dynamodb" +)))] +use edgezero_store_aws::settings::AwsPreparationLimits; +use edgezero_store_aws::settings::{AgentAccessToken, AppConfigAgentSettings, DynamoDbKvSettings}; +#[cfg(any(feature = "celld-config", feature = "celld-kv"))] +use edgezero_store_celld::CelldPreparation; +#[cfg(not(any(feature = "celld-config", feature = "celld-kv")))] +use edgezero_store_celld::settings::CelldPreparationLimits; +use edgezero_store_celld::settings::{CelldAccessToken, CelldConfigSettings, CelldKvSettings}; + +use crate::config_store_limits::ConfigStoreLimits; +use crate::native_bindings::{ + NativeBindingSource, NativeConfigBinding, NativeKvBinding, NativePreparationPolicies, + NativeProviderCapabilities, NativeSecretBinding, NativeSourceSelection, NativeStoreBindings, + NativeSuppliedIds, +}; +use crate::run_options::{AxumRunOptions, failure}; + +/// Closed preparation categories. Never wrap provider errors, settings or payloads. +#[derive(Clone, Copy, Debug, Eq, PartialEq, thiserror::Error)] +pub enum NativeStorePreparationError { + #[error("invalid binding")] + InvalidBinding, + #[error("duplicate binding")] + DuplicateBinding, + #[error("provider unavailable")] + Unavailable, + #[error("preparation deadline")] + Deadline, + #[error("size limit")] + SizeLimit, + #[error("malformed value")] + Malformed, +} + +/// Owned native setup. No `Send` bound; provider handles retain their core bounds. +pub type NativeStoreFuture = + Pin> + 'static>>; + +/// Supplied inputs have no built-in payload; validated plans select one per kind. +pub(crate) enum NativeStoreSource { + Local, + Ready(H), + Prepare(NativeStoreFuture), + BuiltIn(B), +} + +pub(crate) enum NativeConfigSource { + AppConfigAgent { + settings: AppConfigAgentSettings, + token: Option, + }, + Celld { + settings: CelldConfigSettings, + token: Option, + }, +} + +pub(crate) enum NativeKvSource { + DynamoDb(DynamoDbKvSettings), + Celld { + settings: CelldKvSettings, + token: Option, + }, +} + +#[cfg(any(feature = "celld-config", feature = "celld-kv"))] +pub(crate) type NativeCelldSession = Option; +#[cfg(not(any(feature = "celld-config", feature = "celld-kv")))] +pub(crate) struct NativeCelldSession; + +pub(crate) struct NativePreparationSessions { + aws: NativeAwsSession, + celld: NativeCelldSession, +} + +#[cfg(any( + feature = "aws-appconfig-agent", + feature = "aws-secrets-manager", + feature = "aws-dynamodb" +))] +pub(crate) type NativeAwsSession = Option; +#[cfg(not(any( + feature = "aws-appconfig-agent", + feature = "aws-secrets-manager", + feature = "aws-dynamodb" +)))] +pub(crate) struct NativeAwsSession; + +/// Move-only inputs, separate from cloneable hosting options. +/// +/// Deferred construction must be side-effect-free. The runner promises not to poll +/// setup until source validation succeeds; it cannot undo earlier caller-owned work. +#[derive(Default)] +pub struct NativeStoreOverrides { + config: BTreeMap>, + kv: BTreeMap>, + secrets: BTreeMap>, +} + +impl fmt::Debug for NativeStoreOverrides { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("NativeStoreOverrides") + .finish_non_exhaustive() + } +} + +impl NativeStoreOverrides { + /// Supplies an existing config binding, including its own default key. + /// + /// # Errors + /// Rejects a second supplied input for the same config ID. + #[inline] + pub fn insert_config>( + &mut self, + id: Id, + binding: ConfigStoreBinding, + ) -> Result<(), NativeStorePreparationError> { + insert( + &mut self.config, + id.into(), + NativeStoreSource::Ready(binding), + ) + } + + /// Defers config opening until the runner has validated all sources. + /// + /// # Errors + /// Rejects a second supplied input for the same config ID. + #[inline] + pub fn prepare_config( + &mut self, + id: Id, + future: Setup, + ) -> Result<(), NativeStorePreparationError> + where + Id: Into, + Setup: Future> + 'static, + { + insert( + &mut self.config, + id.into(), + NativeStoreSource::Prepare(Box::pin(future)), + ) + } + + /// Supplies an already-open KV handle. + /// + /// # Errors + /// Rejects a second supplied input for the same KV ID. + #[inline] + pub fn insert_kv>( + &mut self, + id: Id, + handle: KvHandle, + ) -> Result<(), NativeStorePreparationError> { + insert(&mut self.kv, id.into(), NativeStoreSource::Ready(handle)) + } + + /// Defers KV opening until the runner has validated all sources. + /// + /// # Errors + /// Rejects a second supplied input for the same KV ID. + #[inline] + pub fn prepare_kv( + &mut self, + id: Id, + future: Setup, + ) -> Result<(), NativeStorePreparationError> + where + Id: Into, + Setup: Future> + 'static, + { + insert( + &mut self.kv, + id.into(), + NativeStoreSource::Prepare(Box::pin(future)), + ) + } + + /// Supplies a secret binding, preserving its explicit namespace. + /// + /// # Errors + /// Rejects a second supplied input for the same secret ID. + #[inline] + pub fn insert_secrets>( + &mut self, + id: Id, + binding: BoundSecretStore, + ) -> Result<(), NativeStorePreparationError> { + insert( + &mut self.secrets, + id.into(), + NativeStoreSource::Ready(binding), + ) + } + + /// Defers secret opening until the runner has validated all sources. + /// + /// # Errors + /// Rejects a second supplied input for the same secret ID. + #[inline] + pub fn prepare_secrets( + &mut self, + id: Id, + future: Setup, + ) -> Result<(), NativeStorePreparationError> + where + Id: Into, + Setup: Future> + 'static, + { + insert( + &mut self.secrets, + id.into(), + NativeStoreSource::Prepare(Box::pin(future)), + ) + } +} + +fn insert( + map: &mut BTreeMap>, + id: String, + source: NativeStoreSource, +) -> Result<(), NativeStorePreparationError> { + match map.entry(id) { + Entry::Vacant(entry) => { + entry.insert(source); + Ok(()) + } + Entry::Occupied(_) => Err(NativeStorePreparationError::DuplicateBinding), + } +} + +pub(crate) struct NativeSourcePlan { + pub config_limits: ConfigStoreLimits, + pub policies: NativePreparationPolicies, + pub config: BTreeMap<&'static str, NativeStoreSource>, + pub kv: BTreeMap<&'static str, NativeStoreSource>, + pub secrets: BTreeMap<&'static str, NativeStoreSource>, +} + +impl NativeSourcePlan { + pub(crate) fn validate( + metadata: StoresMetadata, + options: &AxumRunOptions, + overrides: NativeStoreOverrides, + ) -> anyhow::Result { + options.validate_store_overlays(metadata)?; + let local_defaults = NativeStoreBindings::new(BTreeMap::new(), BTreeMap::new()); + let bindings = options.bindings().document().unwrap_or(&local_defaults); + let supplied = NativeSuppliedIds { + config: overrides.config.keys().cloned().collect(), + kv: overrides.kv.keys().cloned().collect(), + secrets: overrides.secrets.keys().cloned().collect(), + }; + let selection = NativeSourceSelection::validate_with_policies( + metadata, + bindings, + &supplied, + options.env_config(), + NativeProviderCapabilities { + appconfig_agent: cfg!(feature = "aws-appconfig-agent"), + secrets_manager: cfg!(feature = "aws-secrets-manager"), + dynamodb: cfg!(feature = "aws-dynamodb"), + celld_config: cfg!(feature = "celld-config"), + celld_kv: cfg!(feature = "celld-kv"), + }, + options.bindings_limits(), + options.preparation_policies(), + ) + .map_err(|category| failure("sources", "bindings", &category.to_string()))?; + let plan = Self { + config_limits: ConfigStoreLimits::from_env(options.env_config())?, + policies: bindings + .validate_with_policies(options.bindings_limits(), options.preparation_policies()) + .map_err(|category| failure("sources", "bindings", &category.to_string()))?, + config: resolve( + selection.config, + overrides.config, + "config", + |id| match bindings + .config() + .get(id) + .ok_or_else(|| failure("config", id, "invalid binding"))? + { + NativeConfigBinding::AppConfigAgent(settings) => { + Ok(NativeConfigSource::AppConfigAgent { + token: settings + .access_token_env + .as_deref() + .map(|name| options.agent_token(name)) + .transpose()?, + settings: settings.clone(), + }) + } + NativeConfigBinding::Celld(settings) => Ok(NativeConfigSource::Celld { + token: settings + .access + .token_env() + .map(|name| options.celld_token(name)) + .transpose()?, + settings: settings.clone(), + }), + }, + )?, + kv: resolve(selection.kv, overrides.kv, "kv", |id| { + match bindings + .kv() + .get(id) + .ok_or_else(|| failure("kv", id, "invalid binding"))? + { + NativeKvBinding::DynamoDb(settings) => { + Ok(NativeKvSource::DynamoDb(settings.clone())) + } + NativeKvBinding::Celld(settings) => Ok(NativeKvSource::Celld { + token: settings + .access + .token_env() + .map(|name| options.celld_token(name)) + .transpose()?, + settings: settings.clone(), + }), + } + })?, + secrets: resolve(selection.secrets, overrides.secrets, "secrets", |id| { + bindings + .secrets() + .get(id) + .cloned() + .ok_or_else(|| failure("secrets", id, "invalid binding")) + })?, + }; + options.validate_local_roots( + plan.config + .values() + .any(|source| matches!(source, NativeStoreSource::Local)), + plan.kv + .values() + .any(|source| matches!(source, NativeStoreSource::Local)), + )?; + Ok(plan) + } + + #[cfg_attr( + not(any( + feature = "aws-appconfig-agent", + feature = "aws-secrets-manager", + feature = "aws-dynamodb", + feature = "celld-config", + feature = "celld-kv" + )), + expect( + clippy::unnecessary_wraps, + reason = "compiled-out sessions retain the fallible preparation boundary" + ) + )] + pub(crate) fn sessions(&self) -> anyhow::Result { + // Budgets start together and include time spent preparing the other provider. + #[cfg(any( + feature = "aws-appconfig-agent", + feature = "aws-secrets-manager", + feature = "aws-dynamodb" + ))] + let aws = if self.config.values().any(|source| { + matches!( + source, + NativeStoreSource::BuiltIn(NativeConfigSource::AppConfigAgent { .. }) + ) + }) || self.kv.values().any(|source| { + matches!( + source, + NativeStoreSource::BuiltIn(NativeKvSource::DynamoDb(_)) + ) + }) || self + .secrets + .values() + .any(|source| matches!(source, NativeStoreSource::BuiltIn(_))) + { + Some( + AwsPreparation::new(MonotonicClock::default(), self.policies.aws) + .map_err(|category| failure("sources", "bindings", &category.to_string()))?, + ) + } else { + None + }; + #[cfg(not(any( + feature = "aws-appconfig-agent", + feature = "aws-secrets-manager", + feature = "aws-dynamodb" + )))] + let aws = { + let _: AwsPreparationLimits = self.policies.aws; + NativeAwsSession + }; + #[cfg(any(feature = "celld-config", feature = "celld-kv"))] + let celld = if self.config.values().any(|source| { + matches!( + source, + NativeStoreSource::BuiltIn(NativeConfigSource::Celld { .. }) + ) + }) || self.kv.values().any(|source| { + matches!( + source, + NativeStoreSource::BuiltIn(NativeKvSource::Celld { .. }) + ) + }) { + Some( + CelldPreparation::new(MonotonicClock::default(), self.policies.celld) + .map_err(|category| failure("sources", "bindings", &category.to_string()))?, + ) + } else { + None + }; + #[cfg(not(any(feature = "celld-config", feature = "celld-kv")))] + let celld = { + let _: CelldPreparationLimits = self.policies.celld; + NativeCelldSession + }; + Ok(NativePreparationSessions { aws, celld }) + } +} + +fn resolve anyhow::Result>( + selection: BTreeMap<&'static str, NativeBindingSource>, + mut supplied: BTreeMap>, + kind: &str, + builtin: BuiltIn, +) -> anyhow::Result>> { + selection + .into_iter() + .map(|(id, selected)| { + let source = match selected { + NativeBindingSource::Local => NativeStoreSource::Local, + NativeBindingSource::Supplied => match supplied + .remove(id) + .ok_or_else(|| failure(kind, id, "invalid binding"))? + { + NativeStoreSource::Local => NativeStoreSource::Local, + NativeStoreSource::Ready(handle) => NativeStoreSource::Ready(handle), + NativeStoreSource::Prepare(future) => NativeStoreSource::Prepare(future), + NativeStoreSource::BuiltIn(never) => match never {}, + }, + NativeBindingSource::BuiltIn => NativeStoreSource::BuiltIn(builtin(id)?), + }; + Ok((id, source)) + }) + .collect() +} + +pub(crate) async fn prepare( + source: NativeStoreSource, + kind: &str, + id: &str, + local: Open, +) -> anyhow::Result +where + Open: FnOnce() -> anyhow::Result, +{ + match source { + NativeStoreSource::Local => local(), + NativeStoreSource::Ready(handle) => Ok(handle), + NativeStoreSource::Prepare(future) => future + .await + .map_err(|category| failure(kind, id, &category.to_string())), + NativeStoreSource::BuiltIn(_) => Err(failure(kind, id, "provider not compiled")), + } +} + +impl NativePreparationSessions { + #[cfg_attr( + not(any(feature = "aws-appconfig-agent", feature = "celld-config")), + expect( + clippy::unused_async, + reason = "compiled-out config dispatch keeps the asynchronous boundary" + ) + )] + pub(crate) async fn prepare_config( + &mut self, + builtin: NativeConfigSource, + id: &str, + ) -> anyhow::Result { + match builtin { + NativeConfigSource::AppConfigAgent { settings, token } => { + #[cfg(feature = "aws-appconfig-agent")] + { + let handle = self + .aws + .as_mut() + .ok_or_else(|| failure("config", id, "invalid preparation session"))? + .prepare_appconfig(&settings, token.as_ref()) + .await + .map_err(|category| failure("config", id, &category.to_string()))?; + Ok(ConfigStoreBinding { + default_key: settings.default_key, + handle, + }) + } + #[cfg(not(feature = "aws-appconfig-agent"))] + { + let _: ( + AppConfigAgentSettings, + Option, + &mut NativeAwsSession, + ) = (settings, token, &mut self.aws); + Err(failure("config", id, "provider not compiled")) + } + } + NativeConfigSource::Celld { settings, token } => { + #[cfg(feature = "celld-config")] + { + let handle = self + .celld + .as_mut() + .ok_or_else(|| failure("config", id, "invalid preparation session"))? + .prepare_config(&settings, token.as_ref()) + .await + .map_err(|category| failure("config", id, &category.to_string()))?; + Ok(ConfigStoreBinding { + default_key: settings.default_key, + handle, + }) + } + #[cfg(not(feature = "celld-config"))] + { + let _: ( + CelldConfigSettings, + Option, + &mut NativeCelldSession, + ) = (settings, token, &mut self.celld); + Err(failure("config", id, "provider not compiled")) + } + } + } + } + + #[cfg_attr( + not(any(feature = "aws-dynamodb", feature = "celld-kv")), + expect( + clippy::unused_async, + reason = "compiled-out KV dispatch keeps the asynchronous boundary" + ) + )] + pub(crate) async fn prepare_kv( + &mut self, + builtin: NativeKvSource, + id: &str, + ) -> anyhow::Result { + match builtin { + NativeKvSource::DynamoDb(settings) => { + #[cfg(feature = "aws-dynamodb")] + { + self.aws + .as_mut() + .ok_or_else(|| failure("kv", id, "invalid preparation session"))? + .prepare_dynamodb(&settings) + .await + .map_err(|category| failure("kv", id, &category.to_string())) + } + #[cfg(not(feature = "aws-dynamodb"))] + { + let _: (DynamoDbKvSettings, &mut NativeAwsSession) = (settings, &mut self.aws); + Err(failure("kv", id, "provider not compiled")) + } + } + NativeKvSource::Celld { settings, token } => { + #[cfg(feature = "celld-kv")] + { + self.celld + .as_mut() + .ok_or_else(|| failure("kv", id, "invalid preparation session"))? + .prepare_kv(&settings, token.as_ref()) + .await + .map_err(|category| failure("kv", id, &category.to_string())) + } + #[cfg(not(feature = "celld-kv"))] + { + let _: ( + CelldKvSettings, + Option, + &mut NativeCelldSession, + ) = (settings, token, &mut self.celld); + Err(failure("kv", id, "provider not compiled")) + } + } + } + } + + #[cfg_attr( + not(feature = "aws-secrets-manager"), + expect( + clippy::unused_async, + reason = "compiled-out secret dispatch keeps the asynchronous boundary" + ) + )] + pub(crate) async fn prepare_secrets( + &mut self, + builtin: NativeSecretBinding, + id: &str, + ) -> anyhow::Result { + #[cfg(feature = "aws-secrets-manager")] + { + let NativeSecretBinding::SecretsManager(settings) = builtin; + let handle = self + .aws + .as_mut() + .ok_or_else(|| failure("secrets", id, "invalid preparation session"))? + .prepare_secrets(&settings, id) + .await + .map_err(|category| failure("secrets", id, &category.to_string()))?; + Ok(BoundSecretStore::new(handle, id.to_owned())) + } + #[cfg(not(feature = "aws-secrets-manager"))] + { + let _: (NativeSecretBinding, &mut NativeAwsSession) = (builtin, &mut self.aws); + Err(failure("secrets", id, "provider not compiled")) + } + } +} + +#[cfg(test)] +mod tests { + use super::{NativeKvSource, NativeSourcePlan, NativeStoreOverrides, NativeStoreSource}; + use crate::{ + native_bindings::{NativeBindingsLimits, NativeStoreBindings}, + run_options::AxumRunOptions, + }; + use edgezero_core::app::{StoreMetadata, StoresMetadata}; + use std::collections::BTreeMap; + + const CELLD_KV: &str = "version=1\n[kv.state]\nprovider='celld-kv'\nendpoint='http://127.0.0.1:8788'\nstore='state-fixture'\n[kv.state.access]\nmode='trusted-network'\n"; + const CELLD_CONFIG: &str = "version=1\n[config.app]\nprovider='celld-config'\nendpoint='http://127.0.0.1:8788'\ndefault_key='settings'\n[config.app.access]\nmode='trusted-network'\n[config.app.documents.settings]\nstore='config-fixture'\nkey='settings'\n"; + + #[test] + fn celld_variants_check_independent_capabilities_without_local_roots() { + for (input, metadata, compiled) in [ + ( + CELLD_KV, + StoresMetadata { + config: None, + kv: Some(StoreMetadata { + ids: &["state"], + default: "state", + }), + secrets: None, + }, + cfg!(feature = "celld-kv"), + ), + ( + CELLD_CONFIG, + StoresMetadata { + config: Some(StoreMetadata { + ids: &["app"], + default: "app", + }), + kv: None, + secrets: None, + }, + cfg!(feature = "celld-config"), + ), + ] { + let document = NativeStoreBindings::parse(input, NativeBindingsLimits::default()) + .expect("settings only"); + let options = AxumRunOptions::new("127.0.0.1:8787".parse().expect("address")) + .expect("options") + .with_store_bindings(document) + .expect("selected"); + let plan = + NativeSourcePlan::validate(metadata, &options, NativeStoreOverrides::default()); + if compiled { + let selected = plan.expect("no local roots required"); + let sessions = selected.sessions().expect("provider sessions"); + #[cfg(any(feature = "celld-config", feature = "celld-kv"))] + assert!(sessions.celld.is_some()); + #[cfg(any( + feature = "aws-appconfig-agent", + feature = "aws-secrets-manager", + feature = "aws-dynamodb" + ))] + assert!(sessions.aws.is_none()); + let _: super::NativePreparationSessions = sessions; + } else { + let error = plan.err().expect("selected feature absent"); + assert!(error.to_string().contains("provider not compiled")); + } + } + } + + #[test] + fn local_ready_and_deferred_select_no_provider_sessions() { + use crate::key_value_store::PersistentKvStore; + use edgezero_core::key_value_store::KvHandle; + use std::cell::Cell; + use std::rc::Rc; + use std::sync::Arc; + let root = tempfile::tempdir().expect("root"); + let handle = KvHandle::new(Arc::new( + PersistentKvStore::new(root.path().join("supplied.redb")).expect("caller-owned store"), + )); + let mut overrides = NativeStoreOverrides::default(); + overrides.insert_kv("ready", handle.clone()).expect("ready"); + let polls = Rc::new(Cell::new(0_usize)); + let observed = Rc::clone(&polls); + overrides + .prepare_kv("deferred", async move { + observed.set(1); + Ok(handle) + }) + .expect("deferred"); + let options = AxumRunOptions::new("127.0.0.1:8787".parse().expect("address")) + .expect("options") + .with_data_dir(root.path()) + .expect("root"); + let metadata = StoresMetadata { + config: None, + kv: Some(StoreMetadata { + ids: &["local", "ready", "deferred"], + default: "local", + }), + secrets: None, + }; + let plan = + NativeSourcePlan::validate(metadata, &options, overrides).expect("valid sources"); + assert_eq!(polls.get(), 0); + let sessions = plan.sessions().expect("no sessions"); + #[cfg(any(feature = "celld-config", feature = "celld-kv"))] + assert!(sessions.celld.is_none()); + #[cfg(any( + feature = "aws-appconfig-agent", + feature = "aws-secrets-manager", + feature = "aws-dynamodb" + ))] + assert!(sessions.aws.is_none()); + let _: super::NativePreparationSessions = sessions; + } + + #[cfg(all(feature = "aws-appconfig-agent", feature = "celld-kv"))] + #[test] + fn sessions_select_actual_aws_celld_and_mixed_variants() { + let aws = "version=1\n[config.app]\nprovider='aws-appconfig-agent'\nendpoint='http://127.0.0.1:2772'\ndefault_key='settings'\n[config.app.documents.settings]\napplication='example'\nenvironment='production'\nprofile='settings'\n"; + for mixed in [false, true] { + let input = if mixed { + format!("{aws}{}", CELLD_KV.replace("version=1\n", "")) + } else { + aws.to_owned() + }; + let options = AxumRunOptions::new("127.0.0.1:8787".parse().expect("address")) + .expect("options") + .with_store_bindings( + NativeStoreBindings::parse(&input, NativeBindingsLimits::default()) + .expect("bindings"), + ) + .expect("selection"); + let metadata = StoresMetadata { + config: Some(StoreMetadata { + ids: &["app"], + default: "app", + }), + kv: mixed.then_some(StoreMetadata { + ids: &["state"], + default: "state", + }), + secrets: None, + }; + let sessions = + NativeSourcePlan::validate(metadata, &options, NativeStoreOverrides::default()) + .expect("plan") + .sessions() + .expect("sessions"); + assert!(sessions.aws.is_some()); + assert_eq!(sessions.celld.is_some(), mixed); + } + } + + #[cfg(all(feature = "celld-config", feature = "celld-kv"))] + #[test] + fn typed_celld_sources_retain_distinct_explicit_tokens() { + use super::NativeConfigSource; + let input = format!("{CELLD_CONFIG}{}", CELLD_KV.replace("version=1\n", "")) + .replace("http://127.0.0.1:8788", "https://unreachable.invalid") + .replace( + "[config.app.access]\nmode='trusted-network'", + "[config.app.access]\nmode='authenticated'\ntoken_env='CONFIG_TOKEN'", + ) + .replace( + "[kv.state.access]\nmode='trusted-network'", + "[kv.state.access]\nmode='authenticated'\ntoken_env='KV_TOKEN'", + ); + let config_token = "config-token-012345678901234567890123456789"; + let kv_token = "kv-token-0123456789012345678901234567890123"; + let options = + AxumRunOptions::from_vars([("CONFIG_TOKEN", config_token), ("KV_TOKEN", kv_token)]) + .expect("explicit values") + .with_store_bindings( + NativeStoreBindings::parse(&input, NativeBindingsLimits::default()) + .expect("bindings"), + ) + .expect("selection"); + let metadata = StoresMetadata { + config: Some(StoreMetadata { + ids: &["app"], + default: "app", + }), + kv: Some(StoreMetadata { + ids: &["state"], + default: "state", + }), + secrets: None, + }; + let plan = NativeSourcePlan::validate(metadata, &options, NativeStoreOverrides::default()) + .expect("captured plan"); + let Some(NativeStoreSource::BuiltIn(NativeConfigSource::Celld { + token: Some(config), + .. + })) = plan.config.get("app") + else { + panic!("typed config token") + }; + let Some(NativeStoreSource::BuiltIn(NativeKvSource::Celld { + token: Some(kv), .. + })) = plan.kv.get("state") + else { + panic!("typed KV token") + }; + assert_eq!(config.expose(), config_token); + assert_eq!(kv.expose(), kv_token); + } + + #[test] + fn later_celld_token_or_capability_failure_never_polls_or_opens_local_stores() { + use std::cell::Cell; + use std::fs::read_dir; + use std::rc::Rc; + let root = tempfile::tempdir().expect("root"); + let input = CELLD_KV + .replace("http://127.0.0.1:8788", "https://unreachable.invalid") + .replace( + "mode='trusted-network'", + "mode='authenticated'\ntoken_env='ABSENT_TOKEN'", + ); + let options = AxumRunOptions::new("127.0.0.1:8787".parse().expect("address")) + .expect("options") + .with_config_dir(root.path()) + .expect("config root") + .with_data_dir(root.path()) + .expect("data root") + .with_store_bindings( + NativeStoreBindings::parse(&input, NativeBindingsLimits::default()) + .expect("bindings"), + ) + .expect("selection"); + let mut overrides = NativeStoreOverrides::default(); + let polls = Rc::new(Cell::new(0_usize)); + let observed = Rc::clone(&polls); + overrides + .prepare_config("deferred", async move { + observed.set(1); + Err(super::NativeStorePreparationError::Unavailable) + }) + .expect("non-Send future"); + let metadata = StoresMetadata { + config: Some(StoreMetadata { + ids: &["local", "deferred"], + default: "local", + }), + kv: Some(StoreMetadata { + ids: &["state", "local"], + default: "state", + }), + secrets: None, + }; + let error = NativeSourcePlan::validate(metadata, &options, overrides) + .err() + .expect("invalid selected source"); + assert_eq!(polls.get(), 0); + assert_eq!(read_dir(root.path()).expect("root").count(), 0); + assert_eq!(error.chain().count(), 1); + assert!(!format!("{error:?}").contains("unreachable.invalid")); + if cfg!(feature = "celld-kv") { + assert!(error.to_string().contains("celld token")); + } else { + assert!(error.to_string().contains("provider not compiled")); + } + } + + #[cfg(feature = "aws-dynamodb")] + #[test] + fn dynamodb_local_ready_and_non_send_deferred_sources_validate_without_polling() { + use crate::key_value_store::PersistentKvStore; + use edgezero_core::key_value_store::KvHandle; + use std::cell::Cell; + use std::fs::read_dir; + use std::rc::Rc; + use std::sync::Arc; + let directory = tempfile::tempdir().unwrap(); + // The caller owns this already-open handle; validation opens no other store. + let supplied = KvHandle::new(Arc::new( + PersistentKvStore::new(directory.path().join("supplied.redb")).unwrap(), + )); + let polls = Rc::new(Cell::new(0_usize)); + let observed = Rc::clone(&polls); + let deferred_handle = supplied.clone(); + let mut overrides = NativeStoreOverrides::default(); + overrides.insert_kv("supplied", supplied).unwrap(); + overrides + .prepare_kv("deferred", async move { + observed.set(1); + Ok(deferred_handle) + }) + .unwrap(); + let bindings = NativeStoreBindings::parse("version=1\n[kv.remote]\nprovider='aws-dynamodb'\nregion='us-east-1'\ntable='edgezero-kv'\nnamespace='scope'", NativeBindingsLimits::default()).unwrap(); + let options = AxumRunOptions::new("127.0.0.1:8787".parse().unwrap()) + .unwrap() + .with_data_dir(directory.path()) + .unwrap() + .with_store_bindings(bindings) + .unwrap(); + let metadata = StoresMetadata { + kv: Some(StoreMetadata { + ids: &["remote", "local", "supplied", "deferred"], + default: "remote", + }), + config: None, + secrets: None, + }; + let validated = NativeSourcePlan::validate(metadata, &options, overrides).unwrap(); + assert!(matches!( + validated.kv.get("remote"), + Some(NativeStoreSource::BuiltIn(NativeKvSource::DynamoDb(_))) + )); + assert!(matches!( + validated.kv.get("local"), + Some(NativeStoreSource::Local) + )); + assert!(matches!( + validated.kv.get("supplied"), + Some(NativeStoreSource::Ready(_)) + )); + assert!(matches!( + validated.kv.get("deferred"), + Some(NativeStoreSource::Prepare(_)) + )); + assert_eq!(polls.get(), 0); + assert_eq!(read_dir(directory.path()).unwrap().count(), 1); + } + + #[test] + fn dynamodb_only_needs_no_local_root_and_checks_actual_feature() { + let bindings = NativeStoreBindings::parse("version=1\n[kv.state]\nprovider='aws-dynamodb'\nregion='us-east-1'\ntable='edgezero-kv'\nnamespace='scope'", NativeBindingsLimits::default()).unwrap(); + let options = AxumRunOptions::new("127.0.0.1:8787".parse().unwrap()) + .unwrap() + .with_store_bindings(bindings) + .unwrap(); + let metadata = StoresMetadata { + kv: Some(StoreMetadata { + ids: &["state"], + default: "state", + }), + config: None, + secrets: None, + }; + let plan = NativeSourcePlan::validate(metadata, &options, NativeStoreOverrides::default()); + if cfg!(feature = "aws-dynamodb") { + let validated_plan = plan.unwrap(); + assert!(matches!( + validated_plan.kv.get("state"), + Some(NativeStoreSource::BuiltIn(NativeKvSource::DynamoDb(_))) + )); + } else { + assert!(plan.is_err()); + } + let empty = NativeStoreBindings::new(BTreeMap::new(), BTreeMap::new()); + assert!(empty.kv().is_empty()); + } +} diff --git a/crates/edgezero-adapter-axum/src/peer_networks.rs b/crates/edgezero-adapter-axum/src/peer_networks.rs new file mode 100644 index 000000000..dd2fec164 --- /dev/null +++ b/crates/edgezero-adapter-axum/src/peer_networks.rs @@ -0,0 +1,143 @@ +//! Shared bounded socket-peer networks for forwarding and protected Geo admission. +use crate::proxy::ProxyTrustConfigError; +use ipnet::{IpNet, Ipv4Net}; +use std::net::{IpAddr, Ipv6Addr}; +const MAX_BYTES: usize = 0x4000; +pub(crate) const MAX_NETWORKS: usize = 256; + +#[derive(Clone, Default, Eq, PartialEq)] +pub(crate) struct PeerNetworks(Vec); +impl PeerNetworks { + pub(crate) fn is_empty(&self) -> bool { + self.0.is_empty() + } + pub(crate) fn len(&self) -> usize { + self.0.len() + } + pub(crate) fn new(networks: I) -> Result + where + I: IntoIterator, + S: AsRef, + { + let mut normalized = Vec::new(); + let mut bytes = 0_usize; + for (index, entry) in networks.into_iter().enumerate() { + if index >= MAX_NETWORKS { + return Err(ProxyTrustConfigError); + } + let raw = entry.as_ref(); + bytes = bytes + .checked_add(raw.len()) + .and_then(|size| size.checked_add(usize::from(index != 0))) + .filter(|size| *size <= MAX_BYTES) + .ok_or(ProxyTrustConfigError)?; + if raw + .bytes() + .any(|byte| byte.is_ascii_control() && byte != b'\t') + { + return Err(ProxyTrustConfigError); + } + let literal = raw.trim_matches([' ', '\t']); + let parsed = literal + .parse::() + .or_else(|_error| literal.parse::().map(IpNet::from)) + .map_err(|_error| ProxyTrustConfigError)?; + let network = normalize_network(parsed)?; + if !normalized.contains(&network) { + normalized.push(network); + } + } + if covers_family(&normalized, false) || covers_family(&normalized, true) { + return Err(ProxyTrustConfigError); + } + Ok(Self(normalized)) + } + pub(crate) fn trusts(&self, address: IpAddr) -> bool { + let canonical = canonical_ip(address); + self.0.iter().any(|network| network.contains(&canonical)) + } +} +pub(crate) fn canonical_ip(address: IpAddr) -> IpAddr { + match address { + IpAddr::V6(v6) => v6.to_ipv4_mapped().map_or(IpAddr::V6(v6), IpAddr::V4), + IpAddr::V4(v4) => IpAddr::V4(v4), + } +} + +fn normalize_network(network: IpNet) -> Result { + let truncated = network.trunc(); + if truncated.prefix_len() == 0 { + return Err(ProxyTrustConfigError); + } + if let IpNet::V6(v6) = truncated { + let mapped_start = Ipv6Addr::from(0xffff_u128 << 32_u32); + if v6.prefix_len() < 96 && v6.contains(&mapped_start) { + // A shorter overlapping prefix contains every mapped IPv4 address. + return Err(ProxyTrustConfigError); + } + if let Some(v4) = v6.network().to_ipv4_mapped() { + let prefix = v6 + .prefix_len() + .checked_sub(96) + .ok_or(ProxyTrustConfigError)?; + if prefix == 0 { + return Err(ProxyTrustConfigError); + } + return Ipv4Net::new(v4, prefix) + .map(IpNet::V4) + .map_err(|_error| ProxyTrustConfigError); + } + } + Ok(truncated) +} + +#[expect( + clippy::arithmetic_side_effects, + reason = "the maximum endpoint returns before end + 1; IPv4 endpoints fit u128 and the IPv6 maximum is handled explicitly" +)] +fn covers_family(networks: &[IpNet], ipv6: bool) -> bool { + let mut intervals: Vec<(u128, u128)> = networks + .iter() + .filter_map(|network| match network { + IpNet::V4(v4) if !ipv6 => Some(( + u128::from(u32::from(v4.network())), + u128::from(u32::from(v4.broadcast())), + )), + IpNet::V6(v6) if ipv6 => Some((u128::from(v6.network()), u128::from(v6.broadcast()))), + IpNet::V4(_) | IpNet::V6(_) => None, + }) + .collect(); + intervals.sort_unstable(); + let maximum = if ipv6 { + u128::MAX + } else { + u128::from(u32::MAX) + }; + let mut next = 0_u128; + for (start, end) in intervals { + if start > next { + return false; + } + if end == maximum { + return true; + } + next = next.max(end + 1); + } + false +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn family_interval_merging_checks_maximum_endpoints() { + // Exercise interval merging's maximum endpoint without enumerating addresses. + let v6 = ["::/1".parse().unwrap(), "8000::/1".parse().unwrap()]; + assert!(covers_family(&v6, true)); + assert!(!covers_family(&["ffff::/16".parse().unwrap()], true)); + assert!(!covers_family( + &["0.0.0.0/1".parse().unwrap(), "192.0.0.0/2".parse().unwrap()], + false + )); + } +} diff --git a/crates/edgezero-adapter-axum/src/proxy.rs b/crates/edgezero-adapter-axum/src/proxy.rs new file mode 100644 index 000000000..b5a58d28f --- /dev/null +++ b/crates/edgezero-adapter-axum/src/proxy.rs @@ -0,0 +1,1784 @@ +//! Checked incoming-proxy assertions for the managed native socket boundary. + +#[cfg(test)] +use crate::peer_networks::MAX_NETWORKS; +use crate::peer_networks::{PeerNetworks, canonical_ip}; +use std::error::Error; +use std::fmt; +use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}; +use std::str::from_utf8; + +use edgezero_core::http::{Authority, HeaderMap, HeaderName, RequestParts}; +use edgezero_core::ingress::{CheckedEffectiveHost, CheckedHostSource}; + +const MAX_BYTES: usize = 0x4000; +const MAX_CHAIN_SLOTS: usize = 64; +const MAX_PARAMETER_SLOTS: usize = 16; +const X_HEADERS: [&str; 3] = ["x-forwarded-for", "x-forwarded-host", "x-forwarded-proto"]; + +/// The sole forwarding assertion family selected by an operator. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ForwardingHeaderFamily { + Forwarded, + XForwarded, +} + +/// Invalid trust configuration. Formatting never includes supplied values. +#[derive(Clone, Copy, Eq, PartialEq)] +pub struct ProxyTrustConfigError; + +impl fmt::Display for ProxyTrustConfigError { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("invalid_trusted_proxy_configuration") + } +} + +impl fmt::Debug for ProxyTrustConfigError { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(self, f) + } +} + +#[expect( + clippy::missing_trait_methods, + reason = "the source-free safe category uses Error defaults; remaining methods are deprecated or unstable" +)] +impl Error for ProxyTrustConfigError {} + +/// Immutable, bounded literal-network policy. Empty policies never enable trust. +#[derive(Clone, Eq, PartialEq)] +pub struct TrustedProxyPolicy { + family: Option, + networks: PeerNetworks, +} + +impl fmt::Debug for TrustedProxyPolicy { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("TrustedProxyPolicy") + .field("family", &self.family) + .field("network_count", &self.networks.len()) + .finish() + } +} + +impl Default for TrustedProxyPolicy { + #[inline] + fn default() -> Self { + Self::no_trust() + } +} + +impl TrustedProxyPolicy { + /// Parses captured standalone settings. Unknown supplied families always fail. + /// + /// # Errors + /// Rejects invalid settings and nonempty networks without a selected family. + #[inline] + pub fn from_values( + cidrs: Option<&str>, + family: Option<&str>, + ) -> Result { + let selected_family = family + .map(|value| match value.trim_matches([' ', '\t']) { + "forwarded" => Ok(ForwardingHeaderFamily::Forwarded), + "x-forwarded" => Ok(ForwardingHeaderFamily::XForwarded), + _ => Err(ProxyTrustConfigError), + }) + .transpose()?; + let raw = cidrs.unwrap_or(""); + if raw.len() > MAX_BYTES + || raw + .bytes() + .any(|byte| byte.is_ascii_control() && byte != b'\t') + { + return Err(ProxyTrustConfigError); + } + if raw.trim_matches([' ', '\t']).is_empty() { + return Ok(Self { + family: selected_family, + networks: PeerNetworks::default(), + }); + } + Self::new( + selected_family.ok_or(ProxyTrustConfigError)?, + raw.split(','), + ) + } + + /// Validates literal IPs/CIDRs without exposing the network-library type. + /// + /// # Errors + /// Rejects invalid literals, parser-budget excesses and whole-family trust. + #[inline] + pub fn new( + family: ForwardingHeaderFamily, + networks: I, + ) -> Result + where + I: IntoIterator, + S: AsRef, + { + Ok(Self { + family: Some(family), + networks: PeerNetworks::new(networks)?, + }) + } + + #[must_use] + #[inline] + pub fn no_trust() -> Self { + Self { + family: None, + networks: PeerNetworks::default(), + } + } + + fn trusts(&self, address: IpAddr) -> bool { + self.networks.trusts(address) + } +} + +/// A bounded assertion outcome, never a raw header or source error. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ForwardingResult { + Absent, + Accepted, + Malformed, + Partial, + UnresolvedChain, + UntrustedPeer, +} + +impl ForwardingResult { + #[must_use] + #[inline] + pub const fn as_str(self) -> &'static str { + match self { + Self::Absent => "absent", + Self::Accepted => "accepted", + Self::Partial => "partial", + Self::UntrustedPeer => "untrusted_peer", + Self::Malformed => "malformed", + Self::UnresolvedChain => "unresolved_chain", + } + } +} + +/// The checked public scheme, independent of the client-address chain. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum Scheme { + Http, + Https, +} + +impl Scheme { + #[must_use] + #[inline] + pub const fn as_str(self) -> &'static str { + match self { + Self::Http => "http", + Self::Https => "https", + } + } +} + +/// Checked ingress facts, not host authorization or authenticated visitor identity. +#[derive(Clone)] +pub struct CheckedProxyMetadata { + client_source: CheckedHostSource, + direct_peer: Option, + effective_client: Option, + effective_host: CheckedEffectiveHost, + effective_scheme: Scheme, + forwarding_result: ForwardingResult, + scheme_source: CheckedHostSource, +} + +impl fmt::Debug for CheckedProxyMetadata { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("CheckedProxyMetadata") + .field("forwarding_result", &self.forwarding_result) + .finish_non_exhaustive() + } +} + +impl CheckedProxyMetadata { + #[must_use] + #[inline] + pub fn client_source(&self) -> CheckedHostSource { + self.client_source + } + + #[must_use] + #[inline] + pub fn direct_peer(&self) -> Option { + self.direct_peer + } + + #[must_use] + #[inline] + pub fn effective_client(&self) -> Option { + self.effective_client + } + + #[must_use] + #[inline] + pub fn effective_host(&self) -> &CheckedEffectiveHost { + &self.effective_host + } + + #[must_use] + #[inline] + pub fn effective_scheme(&self) -> Scheme { + self.effective_scheme + } + + #[must_use] + #[inline] + pub fn forwarding_result(&self) -> ForwardingResult { + self.forwarding_result + } + + #[must_use] + #[inline] + pub fn scheme_source(&self) -> CheckedHostSource { + self.scheme_source + } +} + +#[derive(Default)] +struct ParsedAssertion { + host: Option, + nodes: Vec>, + scheme: Option, +} + +pub(crate) fn normalize( + policy: &TrustedProxyPolicy, + peer: Option, + parts: &RequestParts, +) -> CheckedProxyMetadata { + let direct_authority = match parts.uri.authority() { + Some(authority) => parse_authority(authority.as_str().as_bytes()).ok(), + None => parts + .headers + .get("host") + .and_then(|value| parse_authority(value.as_bytes()).ok()), + }; + let host_source = if direct_authority.is_some() { + CheckedHostSource::Direct + } else { + CheckedHostSource::Unavailable + }; + let mut metadata = CheckedProxyMetadata { + direct_peer: peer, + effective_client: peer.map(|socket| canonical_ip(socket.ip())), + effective_host: CheckedEffectiveHost::from_ingress(direct_authority, host_source), + effective_scheme: Scheme::Http, + client_source: if peer.is_some() { + CheckedHostSource::Direct + } else { + CheckedHostSource::Unavailable + }, + scheme_source: CheckedHostSource::Direct, + forwarding_result: ForwardingResult::Absent, + }; + let present = match policy.family { + Some(ForwardingHeaderFamily::Forwarded) => parts.headers.contains_key("forwarded"), + Some(ForwardingHeaderFamily::XForwarded) => X_HEADERS + .iter() + .any(|name| parts.headers.contains_key(*name)), + None => parts.headers.keys().any(is_forwarding_header), + }; + if !present { + return metadata; + } + if !peer.is_some_and(|socket| policy.trusts(socket.ip())) { + metadata.forwarding_result = ForwardingResult::UntrustedPeer; + return metadata; + } + let (parse_result, source) = match policy.family { + Some(ForwardingHeaderFamily::Forwarded) => ( + parse_forwarded(&parts.headers), + CheckedHostSource::TrustedForwarded, + ), + Some(ForwardingHeaderFamily::XForwarded) => ( + parse_x_forwarded(&parts.headers), + CheckedHostSource::TrustedXForwarded, + ), + None => return metadata, + }; + let Ok(parsed) = parse_result else { + metadata.forwarding_result = ForwardingResult::Malformed; + return metadata; + }; + let mut accepted_client = false; + let mut unresolved = false; + if !parsed.nodes.is_empty() { + unresolved = true; + for predecessor in parsed.nodes.iter().rev() { + let Some(address) = predecessor else { + break; + }; + if !policy.trusts(*address) { + metadata.effective_client = Some(*address); + metadata.client_source = source; + accepted_client = true; + unresolved = false; + break; + } + } + } + let accepted_host = parsed.host.is_some(); + let accepted_scheme = parsed.scheme.is_some(); + if let Some(host) = parsed.host { + metadata.effective_host = CheckedEffectiveHost::from_ingress(Some(host), source); + } + if let Some(scheme) = parsed.scheme { + metadata.effective_scheme = scheme; + metadata.scheme_source = source; + } + metadata.forwarding_result = if unresolved { + ForwardingResult::UnresolvedChain + } else if accepted_client && accepted_host && accepted_scheme { + ForwardingResult::Accepted + } else if accepted_client || accepted_host || accepted_scheme { + ForwardingResult::Partial + } else { + ForwardingResult::Absent + }; + metadata +} + +fn is_forwarding_header(name: &HeaderName) -> bool { + name.as_str() == "forwarded" || name.as_str().starts_with("x-forwarded-") +} + +/// Removes all raw forwarding assertions, including unselected extensions. +pub(crate) fn strip_forwarding_headers(headers: &mut HeaderMap) { + while let Some(name) = headers + .keys() + .find(|name| is_forwarding_header(name)) + .cloned() + { + headers.remove(name); + } +} + +// Input is preflighted before any parsing scratch or retained nodes are allocated. +fn preflight(headers: &HeaderMap, names: &[&str]) -> Result<(), ()> { + let mut bytes = 0_usize; + for name in names { + for (index, value) in headers.get_all(*name).iter().enumerate() { + bytes = bytes + .checked_add(value.as_bytes().len()) + .and_then(|size| size.checked_add(usize::from(index != 0))) + .filter(|size| *size <= MAX_BYTES) + .ok_or(())?; + } + } + Ok(()) +} + +#[expect( + clippy::indexing_slicing, + reason = "both endpoints are positions found in this slice, and first non-OWS cannot follow last non-OWS" +)] +fn trim_ows(value: &[u8]) -> &[u8] { + let Some(start) = value.iter().position(|byte| !matches!(byte, b' ' | b'\t')) else { + return &[]; + }; + let Some(end) = value.iter().rposition(|byte| !matches!(byte, b' ' | b'\t')) else { + return &[]; + }; + &value[start..=end] +} + +// Slots include empty members; repeated-line boundaries join rather than add slots. +#[expect( + clippy::arithmetic_side_effects, + clippy::indexing_slicing, + reason = "private callers cap slots at 64 or 16; delimiter indices come from the same preflighted slice and start advances at most to its length" +)] +fn split_slots<'input>( + value: &'input [u8], + delimiter: u8, + slots: &mut usize, + limit: usize, + quoting: bool, +) -> Result, ()> { + let mut result = Vec::new(); + let mut start = 0; + let mut quoted = false; + let mut escaped = false; + *slots += 1; + if *slots > limit { + return Err(()); + } + for (index, byte) in value.iter().copied().enumerate() { + if quoting && escaped { + escaped = false; + } else if quoting && quoted && byte == b'\\' { + escaped = true; + } else if quoting && byte == b'"' { + quoted = !quoted; + } else if !quoted && byte == delimiter { + *slots += 1; + if *slots > limit { + return Err(()); + } + result.push(&value[start..index]); + start = index + 1; + } else { + // An ordinary byte does not change delimiter/quoting state. + } + } + if quoted || escaped { + return Err(()); + } + result.push(&value[start..]); + Ok(result) +} + +fn token_byte(byte: u8) -> bool { + byte.is_ascii_alphanumeric() || b"!#$%&'*+-.^_`|~".contains(&byte) +} + +// Unknown extensions are validated without allocating or retaining their payload. +#[expect( + clippy::arithmetic_side_effects, + clippy::indexing_slicing, + reason = "the quoted branch checks length >= 2 before excluding the two quote bytes" +)] +fn validate_value(value: &[u8]) -> Result { + if value.first() != Some(&b'"') { + return if !value.is_empty() && value.iter().copied().all(token_byte) { + Ok(false) + } else { + Err(()) + }; + } + if value.len() < 2 || value.last() != Some(&b'"') { + return Err(()); + } + let mut escaped = false; + for byte in value[1..value.len() - 1].iter().copied() { + if escaped { + if !(byte == b'\t' || byte == b' ' || (0x21..=0x7e).contains(&byte) || byte >= 0x80) { + return Err(()); + } + escaped = false; + } else if byte == b'\\' { + escaped = true; + } else if !(byte == b'\t' + || byte == b' ' + || byte == 0x21 + || (0x23..=0x5b).contains(&byte) + || (0x5d..=0x7e).contains(&byte) + || byte >= 0x80) + { + return Err(()); + } else { + // RFC qdtext, including obs-text, requires no state change. + } + } + if escaped { Err(()) } else { Ok(true) } +} + +#[expect( + clippy::arithmetic_side_effects, + clippy::indexing_slicing, + reason = "only called after validate_value; quoted input has two validated delimiters and fits the assertion byte budget" +)] +fn decode_value(value: &[u8], quoted: bool) -> Vec { + if !quoted { + return value.to_vec(); + } + let mut decoded = Vec::with_capacity(value.len() - 2); + let mut escaped = false; + for byte in value[1..value.len() - 1].iter().copied() { + if !escaped && byte == b'\\' { + escaped = true; + } else { + decoded.push(byte); + escaped = false; + } + } + decoded +} + +#[expect( + clippy::arithmetic_side_effects, + clippy::indexing_slicing, + reason = "equals is a position found within the same nonempty parameter slice, so both name/value ranges and equals + 1 are bounded" +)] +fn parse_forwarded(headers: &HeaderMap) -> Result { + preflight(headers, &["forwarded"])?; + let mut parsed = ParsedAssertion::default(); + let mut slots = 0; + for field in headers.get_all("forwarded") { + for raw_element in split_slots(field.as_bytes(), b',', &mut slots, MAX_CHAIN_SLOTS, true)? { + let element = trim_ows(raw_element); + if element.is_empty() { + continue; + } + let mut parameter_slots = 0; + let mut names: Vec<&[u8]> = Vec::new(); + let mut node = None; + let mut host = None; + let mut scheme = None; + for raw_parameter in split_slots( + element, + b';', + &mut parameter_slots, + MAX_PARAMETER_SLOTS, + true, + )? { + let parameter = trim_ows(raw_parameter); + if parameter.is_empty() { + continue; + } + let equals = parameter.iter().position(|byte| *byte == b'=').ok_or(())?; + let name = ¶meter[..equals]; + if name.is_empty() + || !name.iter().copied().all(token_byte) + || names + .iter() + .any(|previous| previous.eq_ignore_ascii_case(name)) + { + return Err(()); + } + names.push(name); + let value = ¶meter[equals + 1..]; + let quoted = validate_value(value)?; + if name.eq_ignore_ascii_case(b"for") || name.eq_ignore_ascii_case(b"by") { + let decoded = decode_value(value, quoted); + let address = parse_node(&decoded, false)?; + if name.eq_ignore_ascii_case(b"for") { + node = address; + } + } else if name.eq_ignore_ascii_case(b"host") { + host = Some(parse_authority(&decode_value(value, quoted))?); + } else if name.eq_ignore_ascii_case(b"proto") { + scheme = Some(parse_scheme(&decode_value(value, quoted))?); + } else { + // Unknown payloads are syntactically checked, never retained. + } + } + if names.is_empty() { + continue; + } + parsed.nodes.push(node); + // A new substantive element replaces, rather than borrows, final-hop facts. + parsed.host = host; + parsed.scheme = scheme; + } + } + Ok(parsed) +} + +fn singleton<'headers>( + headers: &'headers HeaderMap, + name: &str, +) -> Result, ()> { + let mut values = headers.get_all(name).iter(); + let value = values.next().map(|field| trim_ows(field.as_bytes())); + if values.next().is_some() || value.is_some_and(|bytes| bytes.contains(&b',')) { + return Err(()); + } + Ok(value) +} + +fn parse_x_forwarded(headers: &HeaderMap) -> Result { + preflight(headers, &X_HEADERS)?; + let mut parsed = ParsedAssertion::default(); + let mut slots = 0; + for value in headers.get_all("x-forwarded-for") { + for node in split_slots(value.as_bytes(), b',', &mut slots, MAX_CHAIN_SLOTS, false)? { + // XFF has no RFC empty-list tolerance: never skip a predecessor slot. + parsed.nodes.push(parse_node(trim_ows(node), true)?); + } + } + parsed.host = singleton(headers, "x-forwarded-host")? + .map(parse_authority) + .transpose()?; + parsed.scheme = singleton(headers, "x-forwarded-proto")? + .map(parse_scheme) + .transpose()?; + Ok(parsed) +} + +fn parse_scheme(value: &[u8]) -> Result { + if value.eq_ignore_ascii_case(b"http") { + Ok(Scheme::Http) + } else if value.eq_ignore_ascii_case(b"https") { + Ok(Scheme::Https) + } else { + Err(()) + } +} + +fn valid_obfuscated(value: &str) -> bool { + value.starts_with('_') + && value.len() > 1 + && value + .as_bytes() + .iter() + .skip(1) + .all(|byte| byte.is_ascii_alphanumeric() || b"._-".contains(byte)) +} + +fn validate_node_port(port: &str) -> Result<(), ()> { + if valid_obfuscated(port) + || (!port.is_empty() + && port.bytes().all(|byte| byte.is_ascii_digit()) + && port.parse::().is_ok()) + { + Ok(()) + } else { + Err(()) + } +} + +#[expect( + clippy::arithmetic_side_effects, + clippy::string_slice, + reason = "input is checked ASCII before slicing; closing is the bracket position found in that same string and its successor is at most the string length" +)] +fn parse_node(bytes: &[u8], allow_bare_ipv6: bool) -> Result, ()> { + let value = from_utf8(bytes).map_err(|_error| ())?; + if value.is_empty() || !value.is_ascii() { + return Err(()); + } + if let Some(bracketed) = value.strip_prefix('[') { + let closing = bracketed.find(']').ok_or(())?; + let address = bracketed[..closing] + .parse::() + .map_err(|_error| ())?; + let remainder = &bracketed[closing + 1..]; + if !remainder.is_empty() { + validate_node_port(remainder.strip_prefix(':').ok_or(())?)?; + } + return Ok(Some(canonical_ip(IpAddr::V6(address)))); + } + if allow_bare_ipv6 && let Ok(address) = value.parse::() { + return Ok(Some(canonical_ip(IpAddr::V6(address)))); + } + let (node, optional_port) = value + .split_once(':') + .map_or((value, None), |(name, number)| (name, Some(number))); + if let Some(port) = optional_port { + validate_node_port(port)?; + } + if node.eq_ignore_ascii_case("unknown") || valid_obfuscated(node) { + return Ok(None); + } + node.parse::() + .map(|address| Some(IpAddr::V4(address))) + .map_err(|_error| ()) +} + +#[expect( + clippy::arithmetic_side_effects, + clippy::string_slice, + reason = "input is checked ASCII before slicing; closing is found in the same bracketed authority and closing + 1 cannot exceed its length" +)] +fn parse_authority(bytes: &[u8]) -> Result { + let value = from_utf8(bytes).map_err(|_error| ())?; + if value.is_empty() + || !value.is_ascii() + || value.bytes().any(|byte| { + byte.is_ascii_whitespace() || byte.is_ascii_control() || b"@/?#\\,".contains(&byte) + }) + { + return Err(()); + } + let authority = value.parse::().map_err(|_error| ())?; + if authority.host().is_empty() { + return Err(()); + } + let optional_port = if let Some(bracketed) = value.strip_prefix('[') { + let closing = bracketed.find(']').ok_or(())?; + bracketed[..closing] + .parse::() + .map_err(|_error| ())?; + let suffix = &bracketed[closing + 1..]; + if suffix.is_empty() { + None + } else { + Some(suffix.strip_prefix(':').ok_or(())?) + } + } else { + let mut components = value.split(':'); + let host = components.next().ok_or(())?; + if host.is_empty() || host.contains(['[', ']']) { + return Err(()); + } + let port = components.next(); + if components.next().is_some() { + return Err(()); + } + port + }; + if let Some(port) = optional_port + && (port.is_empty() + || !port.bytes().all(|byte| byte.is_ascii_digit()) + || port.parse::().is_err()) + { + return Err(()); + } + Ok(authority) +} + +#[cfg(test)] +mod tests { + use std::fmt::Write as _; + use std::iter::{empty, repeat, repeat_n}; + + use super::*; + use edgezero_core::body::Body; + use edgezero_core::http::{HeaderName, HeaderValue, request_builder}; + + const PEER: &str = "10.0.0.2:1234"; + const CLIENT: &str = "198.51.100.7"; + + fn policy(family: ForwardingHeaderFamily) -> TrustedProxyPolicy { + TrustedProxyPolicy::new(family, ["10.0.0.0/24"]).unwrap() + } + + fn parts(headers: &[(&str, &str)]) -> RequestParts { + let mut request = request_builder().uri("/").body(Body::empty()).unwrap(); + request + .headers_mut() + .insert("host", HeaderValue::from_static("direct.example:8080")); + for (name, value) in headers { + request.headers_mut().append( + name.parse::().unwrap(), + HeaderValue::from_str(value).unwrap(), + ); + } + request.into_parts().0 + } + + fn checked(family: ForwardingHeaderFamily, headers: &[(&str, &str)]) -> CheckedProxyMetadata { + normalize( + &policy(family), + Some(PEER.parse().unwrap()), + &parts(headers), + ) + } + + fn direct_fallback(metadata: &CheckedProxyMetadata, result: ForwardingResult) { + assert_eq!(metadata.forwarding_result(), result); + assert_eq!(metadata.direct_peer(), Some(PEER.parse().unwrap())); + assert_eq!( + metadata.effective_client(), + Some("10.0.0.2".parse().unwrap()) + ); + assert_eq!(metadata.client_source(), CheckedHostSource::Direct); + assert_eq!( + metadata.effective_host().authority(), + Some("direct.example:8080") + ); + assert_eq!( + metadata.effective_host().source(), + CheckedHostSource::Direct + ); + assert_eq!(metadata.effective_scheme(), Scheme::Http); + assert_eq!(metadata.scheme_source(), CheckedHostSource::Direct); + } + + fn assert_client(metadata: &CheckedProxyMetadata, client: &str) { + assert_eq!(metadata.effective_client(), Some(client.parse().unwrap())); + } + + #[test] + fn defaults_and_strict_configuration() { + let default_policy = TrustedProxyPolicy::default(); + assert_eq!(default_policy, TrustedProxyPolicy::no_trust()); + assert!(!default_policy.trusts("127.0.0.1".parse().unwrap())); + assert_eq!( + TrustedProxyPolicy::from_values(None, None).unwrap(), + default_policy + ); + for raw in ["", " ", "\t "] { + assert!( + TrustedProxyPolicy::from_values(Some(raw), None) + .unwrap() + .networks + .is_empty() + ); + } + for family in ["forwarded", "x-forwarded"] { + assert!( + TrustedProxyPolicy::from_values(None, Some(family)) + .unwrap() + .networks + .is_empty() + ); + } + assert!( + TrustedProxyPolicy::new(ForwardingHeaderFamily::Forwarded, empty::<&str>()) + .unwrap() + .networks + .is_empty() + ); + TrustedProxyPolicy::from_values(Some("10.0.0.1"), None).unwrap_err(); + for family in ["", "auto", "Forwarded", "forwarded\n", "sentinel-secret"] { + TrustedProxyPolicy::from_values(None, Some(family)).unwrap_err(); + } + for cidrs in [ + "all", + "private", + "localhost", + "example.com", + "10.0.0.1,", + ",10.0.0.1", + "10.0.0.1,,10.0.0.2", + "10.0.0.1/33", + "\n", + "10.0.0.1\r", + "10.0.0.1\0", + ] { + TrustedProxyPolicy::from_values(Some(cidrs), Some("forwarded")).expect_err(cidrs); + } + TrustedProxyPolicy::from_values(Some(" \t10.0.0.1 , 10.0.0.2/32\t"), Some("forwarded")) + .unwrap(); + TrustedProxyPolicy::new(ForwardingHeaderFamily::Forwarded, [""]).unwrap_err(); + } + + #[test] + fn configuration_errors_are_safe() { + let error = TrustedProxyPolicy::from_values(Some("sentinel-secret"), Some("forwarded")) + .unwrap_err(); + assert_eq!(error.to_string(), "invalid_trusted_proxy_configuration"); + assert_eq!(format!("{error:?}"), error.to_string()); + let policy = policy(ForwardingHeaderFamily::Forwarded); + assert!(!format!("{policy:?}").contains("10.0.0")); + } + + #[test] + fn configuration_entry_and_byte_bounds_precede_deduplication() { + let family = ForwardingHeaderFamily::Forwarded; + assert_eq!( + TrustedProxyPolicy::new(family, repeat_n("10.0.0.1", MAX_NETWORKS)) + .unwrap() + .networks + .len(), + 1 + ); + TrustedProxyPolicy::new(family, repeat("10.0.0.1")).unwrap_err(); + let exact = format!("{}127.0.0.1", " ".repeat(MAX_BYTES - "127.0.0.1".len())); + TrustedProxyPolicy::new(family, [&exact]).unwrap(); + TrustedProxyPolicy::from_values(Some(&exact), Some("forwarded")).unwrap(); + let over = format!(" {exact}"); + TrustedProxyPolicy::new(family, [&over]).unwrap_err(); + TrustedProxyPolicy::from_values(Some(&over), Some("forwarded")).unwrap_err(); + let second = "10.0.0.2"; + let first = format!( + "{}10.0.0.1", + " ".repeat(MAX_BYTES - 1 - second.len() - "10.0.0.1".len()) + ); + TrustedProxyPolicy::new(family, [first.as_str(), second]).unwrap(); + TrustedProxyPolicy::new(family, [format!(" {first}"), second.to_owned()]).unwrap_err(); + } + + #[test] + fn cidrs_truncate_deduplicate_and_project_mapped_ipv6() { + let policy = TrustedProxyPolicy::new( + ForwardingHeaderFamily::Forwarded, + [ + "192.0.2.19/24", + "192.0.2.0/24", + "::ffff:192.0.2.99/120", + "2001:db8::9/32", + ], + ) + .unwrap(); + assert_eq!(policy.networks.len(), 2); + for address in [ + "192.0.2.0", + "192.0.2.255", + "::ffff:192.0.2.7", + "2001:db8::ffff", + ] { + assert!(policy.trusts(address.parse().unwrap()), "{address}"); + } + for address in [ + "192.0.3.0", + "::ffff:192.0.3.1", + "2001:db9::", + "127.0.0.1", + "10.0.0.1", + ] { + assert!(!policy.trusts(address.parse().unwrap()), "{address}"); + } + let mapped = + TrustedProxyPolicy::new(ForwardingHeaderFamily::Forwarded, ["::ffff:192.0.2.7"]) + .unwrap(); + assert!(mapped.trusts("192.0.2.7".parse().unwrap())); + assert!(!mapped.trusts("192.0.2.8".parse().unwrap())); + } + + #[test] + fn universal_and_mapped_crossing_policies_are_rejected() { + for entries in [ + vec!["0.0.0.0/0"], + vec!["::/0"], + vec!["0.0.0.0/1", "128.0.0.0/1"], + vec!["0.0.0.0/2", "64.0.0.0/2", "128.0.0.0/1"], + vec!["::/1", "8000::/1"], + vec!["::ffff:0.0.0.0/96"], + vec!["::fffe:0:0/95"], + vec!["::/80"], + vec!["0.0.0.0/1", "::ffff:128.0.0.0/97"], + ] { + TrustedProxyPolicy::new(ForwardingHeaderFamily::Forwarded, entries).unwrap_err(); + } + TrustedProxyPolicy::new( + ForwardingHeaderFamily::Forwarded, + ["::fffd:0:0/96", "ffff::/16"], + ) + .unwrap(); + } + + #[test] + #[expect( + clippy::shadow_unrelated, + reason = "the same checked variable describes separate missing-peer and nonmatching-peer fixture observations" + )] + fn absent_disabled_untrusted_and_missing_peer_never_parse_assertions() { + let assertion = parts(&[ + ("forwarded", "invalid sentinel-secret"), + ("x-forwarded-host", "evil.example"), + ]); + for policy in [ + TrustedProxyPolicy::no_trust(), + policy(ForwardingHeaderFamily::Forwarded), + ] { + let peer: SocketAddr = "127.0.0.1:10".parse().unwrap(); + let checked = normalize(&policy, Some(peer), &assertion); + assert_eq!(checked.forwarding_result(), ForwardingResult::UntrustedPeer); + assert_eq!(checked.effective_client(), Some(peer.ip())); + assert_eq!( + checked.effective_host().authority(), + Some("direct.example:8080") + ); + let checked = normalize(&policy, None, &assertion); + assert_eq!(checked.effective_client(), None); + assert_eq!(checked.client_source(), CheckedHostSource::Unavailable); + assert_eq!(checked.forwarding_result(), ForwardingResult::UntrustedPeer); + } + direct_fallback( + &checked(ForwardingHeaderFamily::Forwarded, &[]), + ForwardingResult::Absent, + ); + let empty = + TrustedProxyPolicy::new(ForwardingHeaderFamily::Forwarded, empty::<&str>()).unwrap(); + assert_eq!( + normalize(&empty, Some(PEER.parse().unwrap()), &assertion).forwarding_result(), + ForwardingResult::UntrustedPeer + ); + } + + #[test] + #[expect( + clippy::shadow_unrelated, + reason = "paired metadata observations compare Forwarded and X-Forwarded family selection with the same assertions" + )] + fn family_selection_never_borrows_or_validates_other_family() { + let metadata = checked( + ForwardingHeaderFamily::Forwarded, + &[ + ("forwarded", "for=198.51.100.7"), + ("x-forwarded-host", "evil.example"), + ("x-forwarded-proto", "garbage"), + ], + ); + assert_client(&metadata, CLIENT); + assert_eq!( + metadata.effective_host().authority(), + Some("direct.example:8080") + ); + assert_eq!(metadata.effective_scheme(), Scheme::Http); + assert_eq!(metadata.forwarding_result(), ForwardingResult::Partial); + let metadata = checked( + ForwardingHeaderFamily::XForwarded, + &[ + ("forwarded", "malformed"), + ("x-forwarded-host", "public.example"), + ("x-forwarded-proto", "https"), + ], + ); + assert_eq!(metadata.forwarding_result(), ForwardingResult::Partial); + assert_client(&metadata, "10.0.0.2"); + assert_eq!( + metadata.effective_host().authority(), + Some("public.example") + ); + assert_eq!(metadata.effective_scheme(), Scheme::Https); + direct_fallback( + &checked( + ForwardingHeaderFamily::Forwarded, + &[("x-forwarded-for", CLIENT)], + ), + ForwardingResult::Absent, + ); + } + + #[test] + #[expect( + clippy::shadow_unrelated, + reason = "paired chain fixtures compare a final asserted host/scheme against final-hop omission without changing assertion names" + )] + fn repeated_forwarded_order_and_final_proxy_host_scheme() { + let metadata = checked( + ForwardingHeaderFamily::Forwarded, + &[ + ( + "forwarded", + "for=203.0.113.99;host=attacker.example;proto=http, for=198.51.100.7", + ), + ("forwarded", "for=10.0.0.1;HOST=public.example;PROTO=HTTPS"), + ], + ); + assert_client(&metadata, CLIENT); + assert_eq!(metadata.forwarding_result(), ForwardingResult::Accepted); + assert_eq!( + metadata.effective_host().authority(), + Some("public.example") + ); + assert_eq!( + metadata.effective_host().source(), + CheckedHostSource::TrustedForwarded + ); + assert_eq!(metadata.effective_scheme(), Scheme::Https); + assert_eq!( + metadata.scheme_source(), + CheckedHostSource::TrustedForwarded + ); + assert_eq!( + metadata.client_source(), + CheckedHostSource::TrustedForwarded + ); + let metadata = checked( + ForwardingHeaderFamily::Forwarded, + &[( + "forwarded", + "for=198.51.100.7;host=earlier.example;proto=https, for=10.0.0.1", + )], + ); + assert_eq!( + metadata.effective_host().authority(), + Some("direct.example:8080") + ); + assert_eq!(metadata.effective_scheme(), Scheme::Http); + } + + #[test] + fn forwarded_rfc_empty_slots_and_unknown_extensions() { + for value in [",,", "", " ; ; ", " , ; ; , "] { + direct_fallback( + &checked(ForwardingHeaderFamily::Forwarded, &[("forwarded", value)]), + ForwardingResult::Absent, + ); + } + let metadata = checked( + ForwardingHeaderFamily::Forwarded, + &[( + "forwarded", + " , ;for=198.51.100.7;;host=public.example;proto=https;ext=\"a,b;c\\\"d\\\\e\"; , ", + )], + ); + assert_eq!(metadata.forwarding_result(), ForwardingResult::Accepted); + assert_client(&metadata, CLIENT); + let mut parts = parts(&[]); + parts.headers.append( + "forwarded", + HeaderValue::from_bytes( + b"for=198.51.100.7;host=public.example;proto=https;ext=\"\x80\xff\\\xfe\"", + ) + .unwrap(), + ); + assert_eq!( + normalize( + &policy(ForwardingHeaderFamily::Forwarded), + Some(PEER.parse().unwrap()), + &parts + ) + .forwarding_result(), + ForwardingResult::Accepted + ); + } + + #[test] + fn quoted_values_unescape_before_recognized_validation() { + let metadata = checked( + ForwardingHeaderFamily::Forwarded, + &[( + "forwarded", + "for=\"198.51.100.7\";host=\"public\\.example:443\";proto=\"h\\ttps\"", + )], + ); + assert_eq!(metadata.forwarding_result(), ForwardingResult::Accepted); + assert_eq!( + metadata.effective_host().authority(), + Some("public.example:443") + ); + for value in [ + "for=\"198.51.100.7", + "for=198.51.100.7;ext=\"a\\\"", + "for=198.51.100.7;ext=\"a\"x", + "for=198.51.100.7;ext=a b", + "for =198.51.100.7", + "for= 198.51.100.7", + "for=198.51.100.7;ext=", + ] { + direct_fallback( + &checked(ForwardingHeaderFamily::Forwarded, &[("forwarded", value)]), + ForwardingResult::Malformed, + ); + } + } + + #[test] + #[expect( + clippy::shadow_unrelated, + reason = "each node table row runs identical metadata assertions against both family-specific results" + )] + fn recognized_nodes_and_ports_follow_family_grammar() { + for (value, expected) in [ + ("198.51.100.7", Some(CLIENT)), + ("198.51.100.7:65535", Some(CLIENT)), + ("198.51.100.7:0", Some(CLIENT)), + ("198.51.100.7:_masked", Some(CLIENT)), + ("[2001:db8::7]", Some("2001:db8::7")), + ("[2001:db8::7]:443", Some("2001:db8::7")), + ("[2001:db8::7]:_masked", Some("2001:db8::7")), + ("[::ffff:198.51.100.7]:80", Some(CLIENT)), + ("unknown", None), + ("UNKNOWN:80", None), + ("_hidden:_port", None), + ] { + let metadata = checked( + ForwardingHeaderFamily::Forwarded, + &[( + "forwarded", + &format!("for=\"{value}\";host=public.example;proto=https"), + )], + ); + let metadata_x = checked( + ForwardingHeaderFamily::XForwarded, + &[ + ("x-forwarded-for", value), + ("x-forwarded-host", "public.example"), + ("x-forwarded-proto", "https"), + ], + ); + for metadata in [&metadata, &metadata_x] { + if let Some(address) = expected { + assert_client(metadata, address); + assert_eq!( + metadata.forwarding_result(), + ForwardingResult::Accepted, + "{value}" + ); + } else { + assert_client(metadata, "10.0.0.2"); + assert_eq!( + metadata.forwarding_result(), + ForwardingResult::UnresolvedChain, + "{value}" + ); + assert_eq!(metadata.effective_scheme(), Scheme::Https); + } + } + } + let bare_v6 = checked( + ForwardingHeaderFamily::XForwarded, + &[("x-forwarded-for", "2001:db8::7")], + ); + assert_client(&bare_v6, "2001:db8::7"); + direct_fallback( + &checked( + ForwardingHeaderFamily::Forwarded, + &[("forwarded", "for=\"2001:db8::7\";proto=https")], + ), + ForwardingResult::Malformed, + ); + for value in [ + "", + "example.com", + "_", + "_bad!", + "unknown:", + "198.51.100.7:65536", + "198.51.100.7:-1", + "198.51.100.7:+1", + "198.51.100.7:_", + "[2001:db8::7]:65536", + "[fe80::1%eth0]", + "[2001:db8::7]extra", + "[2001:db8::7", + "[198.51.100.7]", + "010.0.0.1", + "[::1]::80", + ] { + parse_node(value.as_bytes(), true).expect_err(value); + parse_node(value.as_bytes(), false).expect_err(value); + } + } + + #[test] + fn whole_family_validation_includes_unselected_prefix_and_by_nodes() { + for value in [ + "for=garbage,for=198.51.100.7;host=public.example;proto=https", + "for=198.51.100.7;by=garbage;host=public.example;proto=https", + "for=198.51.100.7;FOR=198.51.100.8;host=public.example;proto=https", + "for=198.51.100.7;ext=a;EXT=b;host=public.example;proto=https", + "for=203.0.113.9;proto=ftp,for=198.51.100.7;host=public.example;proto=https", + "for=203.0.113.9;host=\"bad.example:65536\",for=198.51.100.7;host=public.example;proto=https", + "for=\"198.51.100.7:65536\";host=public.example;proto=https", + ] { + direct_fallback( + &checked(ForwardingHeaderFamily::Forwarded, &[("forwarded", value)]), + ForwardingResult::Malformed, + ); + } + direct_fallback( + &checked( + ForwardingHeaderFamily::XForwarded, + &[ + ("x-forwarded-for", "garbage,198.51.100.7"), + ("x-forwarded-host", "public.example"), + ("x-forwarded-proto", "https"), + ], + ), + ForwardingResult::Malformed, + ); + let valid = checked( + ForwardingHeaderFamily::Forwarded, + &[("forwarded", "for=198.51.100.7;by=\"[2001:db8::1]:_port\"")], + ); + assert_client(&valid, CLIENT); + } + + #[test] + fn xff_empty_members_invalidate_even_good_host_and_proto() { + for value in [ + "", + " ", + ",", + ",198.51.100.7", + "198.51.100.7,", + "198.51.100.7,,10.0.0.1", + "198.51.100.7, \t,10.0.0.1", + ] { + direct_fallback( + &checked( + ForwardingHeaderFamily::XForwarded, + &[ + ("x-forwarded-for", value), + ("x-forwarded-host", "public.example"), + ("x-forwarded-proto", "https"), + ], + ), + ForwardingResult::Malformed, + ); + } + direct_fallback( + &checked( + ForwardingHeaderFamily::XForwarded, + &[ + ("x-forwarded-for", CLIENT), + ("x-forwarded-for", ""), + ("x-forwarded-host", "public.example"), + ("x-forwarded-proto", "https"), + ], + ), + ForwardingResult::Malformed, + ); + let no_xff = checked( + ForwardingHeaderFamily::XForwarded, + &[ + ("x-forwarded-host", "public.example"), + ("x-forwarded-proto", "https"), + ], + ); + assert_eq!(no_xff.forwarding_result(), ForwardingResult::Partial); + assert_eq!(no_xff.effective_host().authority(), Some("public.example")); + assert_eq!(no_xff.effective_scheme(), Scheme::Https); + assert_client(&no_xff, "10.0.0.2"); + } + + #[test] + fn x_forwarded_host_and_proto_are_strict_singletons() { + for headers in [ + vec![ + ("x-forwarded-for", CLIENT), + ("x-forwarded-host", "public.example"), + ("x-forwarded-host", "public.example"), + ], + vec![ + ("x-forwarded-for", CLIENT), + ("x-forwarded-proto", "https"), + ("x-forwarded-proto", "https"), + ], + vec![ + ("x-forwarded-for", CLIENT), + ("x-forwarded-host", "public.example,evil.example"), + ], + vec![ + ("x-forwarded-for", CLIENT), + ("x-forwarded-proto", "https,http"), + ], + vec![("x-forwarded-for", CLIENT), ("x-forwarded-host", "")], + vec![("x-forwarded-for", CLIENT), ("x-forwarded-proto", "")], + ] { + direct_fallback( + &checked(ForwardingHeaderFamily::XForwarded, &headers), + ForwardingResult::Malformed, + ); + } + let accepted = checked( + ForwardingHeaderFamily::XForwarded, + &[ + ("x-forwarded-for", "198.51.100.7, 10.0.0.1"), + ("x-forwarded-host", " \tpublic.example:443 "), + ("x-forwarded-proto", " HTTPS "), + ], + ); + assert_eq!(accepted.forwarding_result(), ForwardingResult::Accepted); + assert_eq!( + accepted.effective_host().source(), + CheckedHostSource::TrustedXForwarded + ); + assert_eq!( + accepted.client_source(), + CheckedHostSource::TrustedXForwarded + ); + assert_eq!( + accepted.scheme_source(), + CheckedHostSource::TrustedXForwarded + ); + } + + #[test] + fn authorities_and_schemes_are_checked_before_acceptance() { + for host in [ + "public.example", + "public.example:0", + "public.example:65535", + "[2001:db8::1]", + "[::1]:443", + "192.0.2.1:80", + ] { + parse_authority(host.as_bytes()).expect(host); + } + for host in [ + "", + "user@public.example", + "public.example/path", + "public.example?q=secret", + "public.example#secret", + "public.example\\path", + " public.example", + "public.example ", + "public.example:", + "public.example:65536", + "public.example:-1", + "public.example:_port", + "public.example:443:80", + "[fe80::1%eth0]", + "[::1]:", + "[::1]extra", + ":443", + "a,b", + ] { + parse_authority(host.as_bytes()).expect_err(host); + let encoded_host = host.replace('\\', "\\\\").replace('"', "\\\""); + direct_fallback( + &checked( + ForwardingHeaderFamily::Forwarded, + &[( + "forwarded", + &format!("for=198.51.100.7;host=\"{encoded_host}\";proto=https"), + )], + ), + ForwardingResult::Malformed, + ); + } + for scheme in ["ftp", "https:", "wss", "", "https http"] { + direct_fallback( + &checked( + ForwardingHeaderFamily::Forwarded, + &[( + "forwarded", + &format!("for=198.51.100.7;host=public.example;proto=\"{scheme}\""), + )], + ), + ForwardingResult::Malformed, + ); + } + } + + #[test] + fn walk_stops_at_first_untrusted_boundary_and_never_skips_gaps() { + for (chain, expected, result) in [ + ( + "203.0.113.99,198.51.100.7,10.0.0.1", + CLIENT, + ForwardingResult::Partial, + ), + ( + "203.0.113.99,198.51.100.7", + CLIENT, + ForwardingResult::Partial, + ), + ( + "203.0.113.99,10.1.0.1", + "10.1.0.1", + ForwardingResult::Partial, + ), + ( + "198.51.100.7,unknown,10.0.0.1", + "10.0.0.2", + ForwardingResult::UnresolvedChain, + ), + ( + "198.51.100.7,_gap,10.0.0.1", + "10.0.0.2", + ForwardingResult::UnresolvedChain, + ), + ("10.0.0.1", "10.0.0.2", ForwardingResult::UnresolvedChain), + ( + "10.0.0.1,10.0.0.3", + "10.0.0.2", + ForwardingResult::UnresolvedChain, + ), + ("unknown,198.51.100.7", CLIENT, ForwardingResult::Partial), + ] { + let metadata = checked( + ForwardingHeaderFamily::XForwarded, + &[("x-forwarded-for", chain)], + ); + assert_client(&metadata, expected); + assert_eq!(metadata.forwarding_result(), result, "{chain}"); + } + let gap = checked( + ForwardingHeaderFamily::Forwarded, + &[( + "forwarded", + "for=198.51.100.7,host=public.example;proto=https", + )], + ); + assert_eq!(gap.forwarding_result(), ForwardingResult::UnresolvedChain); + assert_client(&gap, "10.0.0.2"); + assert_eq!(gap.effective_host().authority(), Some("public.example")); + assert_eq!(gap.effective_scheme(), Scheme::Https); + } + + #[test] + fn mapped_peer_and_asserted_client_match_equivalent_ipv4_trust() { + let peer: SocketAddr = "[::ffff:10.0.0.2]:1234".parse().unwrap(); + let parts = parts(&[( + "forwarded", + "for=\"[::ffff:198.51.100.7]:_p\",for=\"[::ffff:10.0.0.1]\";host=public.example;proto=https", + )]); + let metadata = normalize( + &policy(ForwardingHeaderFamily::Forwarded), + Some(peer), + &parts, + ); + assert_eq!(metadata.direct_peer(), Some(peer)); + assert_client(&metadata, CLIENT); + assert_eq!(metadata.forwarding_result(), ForwardingResult::Accepted); + } + + #[test] + fn forwarded_list_slot_bounds_are_exact_across_repeated_fields_and_empty_members() { + let for_node = "for=198.51.100.7"; + let exact = repeat_n(for_node, MAX_CHAIN_SLOTS) + .collect::>() + .join(","); + assert_eq!( + checked(ForwardingHeaderFamily::Forwarded, &[("forwarded", &exact)]) + .forwarding_result(), + ForwardingResult::Partial + ); + direct_fallback( + &checked( + ForwardingHeaderFamily::Forwarded, + &[("forwarded", &format!("{exact},{for_node}"))], + ), + ForwardingResult::Malformed, + ); + let two_nodes = format!("{for_node},{for_node}"); + let mut repeated = parts(&[]); + for _ in 0_usize..32 { + repeated + .headers + .append("forwarded", HeaderValue::from_str(&two_nodes).unwrap()); + } + assert_eq!( + normalize( + &policy(ForwardingHeaderFamily::Forwarded), + Some(PEER.parse().unwrap()), + &repeated + ) + .forwarding_result(), + ForwardingResult::Partial + ); + repeated + .headers + .append("forwarded", HeaderValue::from_static("")); + direct_fallback( + &normalize( + &policy(ForwardingHeaderFamily::Forwarded), + Some(PEER.parse().unwrap()), + &repeated, + ), + ForwardingResult::Malformed, + ); + let exact_empty_slots = format!("{}{for_node}", ",".repeat(63)); + assert_eq!( + checked( + ForwardingHeaderFamily::Forwarded, + &[("forwarded", &exact_empty_slots)] + ) + .forwarding_result(), + ForwardingResult::Partial + ); + direct_fallback( + &checked( + ForwardingHeaderFamily::Forwarded, + &[("forwarded", &format!(",{exact_empty_slots}"))], + ), + ForwardingResult::Malformed, + ); + } + + #[test] + fn xff_list_slot_bounds_are_exact_across_repeated_fields() { + let exact = repeat_n(CLIENT, MAX_CHAIN_SLOTS) + .collect::>() + .join(","); + assert_eq!( + checked( + ForwardingHeaderFamily::XForwarded, + &[("x-forwarded-for", &exact)] + ) + .forwarding_result(), + ForwardingResult::Partial + ); + direct_fallback( + &checked( + ForwardingHeaderFamily::XForwarded, + &[("x-forwarded-for", &format!("{exact},{CLIENT}"))], + ), + ForwardingResult::Malformed, + ); + let mut repeated = parts(&[]); + for _ in 0_usize..32 { + repeated.headers.append( + "x-forwarded-for", + HeaderValue::from_str(&format!("{CLIENT},{CLIENT}")).unwrap(), + ); + } + assert_eq!( + normalize( + &policy(ForwardingHeaderFamily::XForwarded), + Some(PEER.parse().unwrap()), + &repeated + ) + .forwarding_result(), + ForwardingResult::Partial + ); + repeated + .headers + .append("x-forwarded-for", HeaderValue::from_str(CLIENT).unwrap()); + direct_fallback( + &normalize( + &policy(ForwardingHeaderFamily::XForwarded), + Some(PEER.parse().unwrap()), + &repeated, + ), + ForwardingResult::Malformed, + ); + } + + #[test] + fn parameter_slot_bounds_include_empty_parameters() { + let exact = format!("for=198.51.100.7{}", ";".repeat(15)); + assert_eq!( + checked(ForwardingHeaderFamily::Forwarded, &[("forwarded", &exact)]) + .forwarding_result(), + ForwardingResult::Partial + ); + direct_fallback( + &checked( + ForwardingHeaderFamily::Forwarded, + &[("forwarded", &format!("{exact};"))], + ), + ForwardingResult::Malformed, + ); + let mut named = String::from("for=198.51.100.7"); + for index in 0_usize..15 { + write!(named, ";e{index}=value").unwrap(); + } + assert_eq!( + checked(ForwardingHeaderFamily::Forwarded, &[("forwarded", &named)]) + .forwarding_result(), + ForwardingResult::Partial + ); + direct_fallback( + &checked( + ForwardingHeaderFamily::Forwarded, + &[("forwarded", &format!("{named};e15=value"))], + ), + ForwardingResult::Malformed, + ); + } + + #[test] + fn assertion_byte_budget_is_exact_and_counts_repeated_delimiters() { + let prefix = "for=198.51.100.7;host=public.example;proto=https;ext=\""; + let exact = format!("{prefix}{}\"", "a".repeat(MAX_BYTES - prefix.len() - 1)); + assert_eq!( + checked(ForwardingHeaderFamily::Forwarded, &[("forwarded", &exact)]) + .forwarding_result(), + ForwardingResult::Accepted + ); + let over = exact.replacen("ext=\"", "ext=\"a", 1); + direct_fallback( + &checked(ForwardingHeaderFamily::Forwarded, &[("forwarded", &over)]), + ForwardingResult::Malformed, + ); + let second = "for=198.51.100.7;host=public.example;proto=https"; + let first = format!("ext=\"{}\"", "a".repeat(MAX_BYTES - second.len() - 1 - 6)); + assert_eq!(first.len() + second.len() + 1, MAX_BYTES); + assert_eq!( + checked( + ForwardingHeaderFamily::Forwarded, + &[("forwarded", &first), ("forwarded", second)] + ) + .forwarding_result(), + ForwardingResult::Accepted + ); + direct_fallback( + &checked( + ForwardingHeaderFamily::Forwarded, + &[("forwarded", &format!(" {first}")), ("forwarded", second)], + ), + ForwardingResult::Malformed, + ); + let padding = " ".repeat(MAX_BYTES - CLIENT.len()); + assert_eq!( + checked( + ForwardingHeaderFamily::XForwarded, + &[("x-forwarded-for", &format!("{padding}{CLIENT}"))] + ) + .forwarding_result(), + ForwardingResult::Partial + ); + direct_fallback( + &checked( + ForwardingHeaderFamily::XForwarded, + &[ + ("x-forwarded-for", &format!(" {padding}{CLIENT}")), + ("x-forwarded-host", "public.example"), + ], + ), + ForwardingResult::Malformed, + ); + } + + #[test] + fn stripping_removes_every_forwarding_extension_without_changing_direct_head() { + let mut parts = parts(&[ + ("Forwarded", "for=198.51.100.7"), + ("X-Forwarded-For", CLIENT), + ("x-forwarded-host", "public.example"), + ("x-forwarded-proto", "https"), + ("X-Forwarded-Secret", "sentinel-secret"), + ("x-request-id", "visitor-id"), + ]); + let original_uri = parts.uri.clone(); + strip_forwarding_headers(&mut parts.headers); + assert!(!parts.headers.keys().any(is_forwarding_header)); + assert_eq!(parts.headers["host"], "direct.example:8080"); + assert_eq!(parts.headers["x-request-id"], "visitor-id"); + assert_eq!(parts.uri, original_uri); + } + + #[test] + #[expect( + clippy::shadow_unrelated, + reason = "sequential direct-head fixtures assert metadata with authority present and then unavailable" + )] + fn direct_authority_uses_http_authority_without_inferring_transport_scheme() { + let mut parts = parts(&[]); + parts.uri = "https://uri.example:443/path?secret=value".parse().unwrap(); + let metadata = normalize(&TrustedProxyPolicy::no_trust(), None, &parts); + assert_eq!( + metadata.effective_host().authority(), + Some("uri.example:443") + ); + assert_eq!(metadata.effective_scheme(), Scheme::Http); + assert_eq!(metadata.scheme_source(), CheckedHostSource::Direct); + parts.uri = "/".parse().unwrap(); + parts.headers.remove("host"); + let metadata = normalize(&TrustedProxyPolicy::no_trust(), None, &parts); + assert_eq!(metadata.effective_host().authority(), None); + assert_eq!( + metadata.effective_host().source(), + CheckedHostSource::Unavailable + ); + parts + .headers + .insert("host", HeaderValue::from_static("bad.example:65536")); + assert_eq!( + normalize(&TrustedProxyPolicy::no_trust(), None, &parts) + .effective_host() + .authority(), + None + ); + } + + #[test] + fn debug_metadata_does_not_dump_private_ingress_values() { + let metadata = checked( + ForwardingHeaderFamily::XForwarded, + &[ + ("x-forwarded-for", CLIENT), + ("x-forwarded-host", "sentinel-secret.example"), + ("x-forwarded-proto", "https"), + ], + ); + let debug = format!("{metadata:?}"); + for private in [CLIENT, "sentinel-secret", "10.0.0.2"] { + assert!(!debug.contains(private)); + } + for result in [ + ForwardingResult::Absent, + ForwardingResult::Accepted, + ForwardingResult::Partial, + ForwardingResult::UntrustedPeer, + ForwardingResult::Malformed, + ForwardingResult::UnresolvedChain, + ] { + assert!( + result + .as_str() + .bytes() + .all(|byte| byte.is_ascii_lowercase() || byte == b'_') + ); + } + assert_eq!(Scheme::Http.as_str(), "http"); + assert_eq!(Scheme::Https.as_str(), "https"); + } +} diff --git a/crates/edgezero-adapter-axum/src/response.rs b/crates/edgezero-adapter-axum/src/response.rs index 8ba115953..44fe9f32f 100644 --- a/crates/edgezero-adapter-axum/src/response.rs +++ b/crates/edgezero-adapter-axum/src/response.rs @@ -29,6 +29,8 @@ use tokio::task::yield_now; use tokio::time::Instant as TokioInstant; use tokio::time::timeout; +use crate::diagnostics::SelectedStatus; + const INTERNAL_FALLBACK_BODY: &[u8] = b"internal server error"; const TIMEOUT_FALLBACK_BODY: &[u8] = b"response write deadline exceeded"; @@ -499,11 +501,8 @@ impl HttpBody for AxumEgressBody { } Poll::Ready(Some(Ok(Frame::data(bytes)))) } - Poll::Ready(Some(Err(error))) => { - log::warn!( - "response-egress Axum body source failed after commit: {}", - response_egress_source_error_category(&error) - ); + Poll::Ready(Some(Err(_error))) => { + // The native terminal owner records this once, including request-log opt-out. this.source = ResponseSource::Done; this.fail(ResponseEgressOutcome::SourceError); Poll::Ready(Some(Err(AxumBodyError(ResponseEgressOutcome::SourceError)))) @@ -527,9 +526,13 @@ impl HttpBody for AxumEgressBody { /// Converts one owned core envelope into the private body used by `EdgeZero`'s Hyper server. pub(crate) async fn prepare_egress_response( - egress: ResponseEgressEnvelope, + mut egress: ResponseEgressEnvelope, connection: &EgressConnection, + selected: Option<&SelectedStatus>, ) -> Response { + if let Some(slot) = selected { + slot.select(egress.response_mut().status()); + } let request_method = egress.request_method().clone(); match egress.begin() { Ok((prepared, policy, attempt, clock)) => { @@ -540,6 +543,7 @@ pub(crate) async fn prepare_egress_response( attempt, &clock, connection, + selected, ); } let validated = match validate_empty_stream( @@ -552,7 +556,14 @@ pub(crate) async fn prepare_egress_response( { Ok(validated) => validated, Err(cause) => { - return prepare_fallback(&request_method, cause, attempt, &clock, connection); + return prepare_fallback( + &request_method, + cause, + attempt, + &clock, + connection, + selected, + ); } }; match start_prepared( @@ -562,6 +573,7 @@ pub(crate) async fn prepare_egress_response( clock.clone(), EgressKind::Application, connection, + selected, ) { Ok(response) => response, Err(returned_attempt) => { @@ -576,13 +588,21 @@ pub(crate) async fn prepare_egress_response( *returned_attempt, &clock, connection, + selected, ) } } } Err(failure) => { let (attempt, clock, cause) = failure.into_parts(); - prepare_fallback(&request_method, cause, attempt, &clock, connection) + prepare_fallback( + &request_method, + cause, + attempt, + &clock, + connection, + selected, + ) } } } @@ -639,10 +659,14 @@ fn start_prepared( clock: MonotonicClock, kind: EgressKind, connection: &EgressConnection, + selected: Option<&SelectedStatus>, ) -> Result, Box> { let declared_length = prepared.declared_length(); let transmits_body = prepared.transmits_body(); let (parts, core_body) = prepared.into_response().into_parts(); + if let Some(slot) = selected { + slot.select(parts.status); + } match AxumEgressBody::owned( core_body, declared_length, @@ -664,11 +688,18 @@ fn prepare_fallback( mut attempt: ResponseEgressAttempt, clock: &MonotonicClock, connection: &EgressConnection, + selected: Option<&SelectedStatus>, ) -> Response { if !attempt.begin_fallback(cause) { + if let Some(slot) = selected { + slot.select(StatusCode::INTERNAL_SERVER_ERROR); + } attempt.terminate(ResponseEgressOutcome::ConversionError, clock.now()); return detached_fallback(request_method, ResponseEgressOutcome::ConversionError); } + if let Some(slot) = selected { + slot.select(fallback_parts(cause).0); + } let started_at = clock.now(); let Some(fallback_deadline) = started_at.checked_add(RESPONSE_EGRESS_FALLBACK_SAFETY_BUDGET) else { @@ -687,6 +718,7 @@ fn prepare_fallback( clock.clone(), EgressKind::Fallback, connection, + selected, ) { Ok(response) => response, Err(mut returned_attempt) => { @@ -739,7 +771,8 @@ mod tests { use edgezero_core::body::Body; use edgezero_core::http::{HeaderMap, Method, StatusCode, Version, response_builder}; use edgezero_core::response_egress::{ - ResponseEgressAttempt, ResponseEgressCompletion, ResponseEgressHead, + RESPONSE_EGRESS_FALLBACK_SAFETY_BUDGET, ResponseEgressAttempt, ResponseEgressBodyKind, + ResponseEgressCompletion, ResponseEgressFallbackDisposition, ResponseEgressHead, ResponseEgressObserver, ResponseEgressObserverHandle, ResponseEgressOutcome, ResponseEgressReport, }; @@ -752,10 +785,88 @@ mod tests { use http_body::Body as _; use super::{ - AxumEgressBody, EgressConnection, EgressKind, INTERNAL_FALLBACK_BODY, + AxumEgressBody, EgressConnection, EgressKind, INTERNAL_FALLBACK_BODY, SelectedStatus, TIMEOUT_FALLBACK_BODY, prepare_fallback, start_prepared, }; + use crate::diagnostics::{ + NativeDiagnosticsHandle, NativeDiagnosticsSnapshot, NativeRequestObserver, + NativeRequestOutcome, NativeRequestRecord, NativeSession, + }; + + #[derive(Default)] + struct NativeFallbackProbe { + completion_calls: Arc, + diagnostics: NativeDiagnosticsHandle, + native: RecordingNativeObserver, + observer: RecordingObserver, + } + + #[derive(Clone, Default)] + struct RecordingNativeObserver(Arc>>); + + impl NativeFallbackProbe { + fn attempt( + &self, + now: MonotonicInstant, + clock: MonotonicClock, + ) -> (ResponseEgressAttempt, SelectedStatus) { + let session = NativeSession::new( + self.diagnostics + .clone() + .with_request_observer(self.native.clone()), + false, + ) + .expect("native session"); + let guard = session + .request(&Method::GET, now, clock.clone(), None) + .expect("native request"); + let (native_completion, selected) = guard.completion(); + selected.select(StatusCode::OK); + let calls = Arc::clone(&self.completion_calls); + let app_completion = ResponseEgressCompletion::new(move |_report| { + calls.fetch_add(1, Ordering::SeqCst); + }); + let headers = HeaderMap::new(); + let head = ResponseEgressHead::new( + StatusCode::OK, + Version::HTTP_11, + &headers, + None, + now, + None, + ); + ( + ResponseEgressAttempt::new( + &head, + now, + app_completion.join(native_completion), + ResponseEgressObserverHandle::new(self.observer.clone()), + clock, + ), + selected, + ) + } + + fn record(&self) -> NativeRequestRecord { + assert_eq!(self.completion_calls.load(Ordering::SeqCst), 1); + assert_eq!(self.observer.0.lock().expect("app reports").len(), 1); + assert_eq!( + self.diagnostics.snapshot(), + NativeDiagnosticsSnapshot::default() + ); + let records = self.native.0.lock().expect("native records"); + assert_eq!(records.len(), 1); + records.first().expect("one native record").clone() + } + } + + impl NativeRequestObserver for RecordingNativeObserver { + fn observe(&self, record: &NativeRequestRecord) { + self.0.lock().expect("native records").push(record.clone()); + } + } + #[derive(Clone, Default)] struct RecordingObserver(Arc>>); @@ -923,6 +1034,130 @@ mod tests { assert!(connection.is_empty()); } + #[test] + fn native_duplicate_live_fallback_selects_detached_500_before_terminal_record() { + let probe = NativeFallbackProbe::default(); + let now = MonotonicInstant::now(); + let clock = MonotonicClock::new(move || now); + let (mut attempt, selected) = probe.attempt(now, clock.clone()); + assert!(attempt.begin_fallback(ResponseEgressOutcome::DeadlineExceeded)); + selected.select(StatusCode::GATEWAY_TIMEOUT); + assert_eq!(probe.diagnostics.snapshot().active_requests, 1); + let connection = EgressConnection::default(); + let response = prepare_fallback( + &Method::GET, + ResponseEgressOutcome::ConversionError, + attempt, + &clock, + &connection, + Some(&selected), + ); + assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR); + let record = probe.record(); + assert_eq!(record.status, Some(StatusCode::INTERNAL_SERVER_ERROR)); + assert_eq!( + record.outcome, + NativeRequestOutcome::Egress(ResponseEgressOutcome::DeadlineExceeded) + ); + assert_eq!(record.body_kind, Some(ResponseEgressBodyKind::Fallback)); + assert_eq!( + record.fallback_disposition, + Some(ResponseEgressFallbackDisposition::Aborted) + ); + assert_eq!(record.bytes_written, Some(0)); + drop(response); + assert_eq!(probe.record().request_id, record.request_id); + assert!(connection.is_empty()); + } + + #[test] + fn native_already_terminal_fallback_cannot_rewrite_the_prior_200_record() { + let probe = NativeFallbackProbe::default(); + let now = MonotonicInstant::now(); + let clock = MonotonicClock::new(move || now); + let (mut attempt, selected) = probe.attempt(now, clock.clone()); + assert!(attempt.begin_writing()); + assert!(attempt.complete(now)); + let original = probe.record(); + assert_eq!(original.status, Some(StatusCode::OK)); + let connection = EgressConnection::default(); + let response = prepare_fallback( + &Method::GET, + ResponseEgressOutcome::DeadlineExceeded, + attempt, + &clock, + &connection, + Some(&selected), + ); + assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR); + drop(response); + let retained = probe.record(); + assert_eq!(retained.request_id, original.request_id); + assert_eq!(retained.status, original.status); + assert_eq!(retained.duration, original.duration); + assert_eq!( + retained.outcome, + NativeRequestOutcome::Egress(ResponseEgressOutcome::Completed) + ); + assert_eq!( + retained.body_kind, + Some(ResponseEgressBodyKind::Application) + ); + assert_eq!(retained.fallback_disposition, None); + assert_eq!(retained.bytes_written, Some(0)); + assert!(connection.is_empty()); + } + + #[test] + fn native_failed_fallback_registration_retains_500_or_504_before_aborted_record() { + for (cause, status) in [ + ( + ResponseEgressOutcome::ConversionError, + StatusCode::INTERNAL_SERVER_ERROR, + ), + ( + ResponseEgressOutcome::DeadlineExceeded, + StatusCode::GATEWAY_TIMEOUT, + ), + ] { + let probe = NativeFallbackProbe::default(); + let now = MonotonicInstant::now(); + let expired = now + .checked_add(RESPONSE_EGRESS_FALLBACK_SAFETY_BUDGET) + .expect("fallback deadline"); + // Preparation samples now, registration sees expiry, then settlement keeps that instant. + let clock = scripted_clock(vec![now, expired, expired]); + let (attempt, selected) = probe.attempt(now, clock.clone()); + assert_eq!(probe.diagnostics.snapshot().active_requests, 1); + let connection = EgressConnection::default(); + let response = prepare_fallback( + &Method::GET, + cause, + attempt, + &clock, + &connection, + Some(&selected), + ); + assert_eq!(response.status(), status); + let record = probe.record(); + assert_eq!(record.status, Some(status)); + assert_eq!(record.outcome, NativeRequestOutcome::Egress(cause)); + assert_eq!( + record.duration, + Some(RESPONSE_EGRESS_FALLBACK_SAFETY_BUDGET) + ); + assert_eq!(record.body_kind, Some(ResponseEgressBodyKind::Fallback)); + assert_eq!( + record.fallback_disposition, + Some(ResponseEgressFallbackDisposition::Aborted) + ); + assert_eq!(record.bytes_written, Some(0)); + drop(response); + assert_eq!(probe.record().request_id, record.request_id); + assert!(connection.is_empty()); + } + } + #[tokio::test(flavor = "current_thread")] async fn response_egress_post_ready_deadline_precedence() { let observer = RecordingObserver::default(); @@ -998,6 +1233,7 @@ mod tests { attempt(&observer, started_at), &clock, &connection, + None, ); let status = if cause == ResponseEgressOutcome::DeadlineExceeded { StatusCode::GATEWAY_TIMEOUT @@ -1073,6 +1309,7 @@ mod tests { clock, EgressKind::Application, &connection, + None, ) else { panic!("registered response"); }; diff --git a/crates/edgezero-adapter-axum/src/run_options.rs b/crates/edgezero-adapter-axum/src/run_options.rs new file mode 100644 index 000000000..7ea2ce358 --- /dev/null +++ b/crates/edgezero-adapter-axum/src/run_options.rs @@ -0,0 +1,1507 @@ +use std::collections::{BTreeMap, BTreeSet, btree_map::Entry}; +use std::env::{var_os, vars_os}; +use std::ffi::OsString; +use std::net::SocketAddr; +use std::path::{Path, PathBuf}; +use std::str::FromStr as _; +use std::time::{Duration, Instant}; + +use crate::diagnostics::NativeDiagnosticsHandle; +use crate::geo::{NativeGeo, NativeGeoCapability, NativeGeoConfigError, NativeGeoSelection}; +use crate::proxy::TrustedProxyPolicy; +use edgezero_core::addr; +use edgezero_core::app::StoresMetadata; +use edgezero_core::env_config::EnvConfig; +use edgezero_store_aws::settings::{AgentAccessToken, AwsPreparationLimits}; +use edgezero_store_celld::settings::{CelldAccessToken, CelldPreparationLimits}; +use log::LevelFilter; + +use crate::native_bindings::{ + NativeBindingsLimits, NativeConfigBinding, NativeKvBinding, NativePreparationPolicySources, + NativeStoreBindings, NativeStoreBindingsInput, +}; + +/// Validated explicit production hosting options. Construction never selects a +/// policy from build profile, cwd or container detection. Explicit constructors +/// do not merge environment values; only `from_env` reads hosting environment. +#[derive(Clone)] +pub struct AxumRunOptions { + addr: SocketAddr, + aws_limits: Option, + bindings: NativeStoreBindingsInput, + bindings_limits: NativeBindingsLimits, + bootstrap_values: BTreeMap, + celld_limits: Option, + config_dir: Option, + data_dir: Option, + diagnostics: NativeDiagnosticsHandle, + env: EnvConfig, + geo: NativeGeoSelection, + grace: Duration, + level: LevelFilter, + non_unicode_overrides: Vec>, + request_records: bool, + trusted_proxy_policy: TrustedProxyPolicy, +} + +#[derive(Clone)] +enum BootstrapValue { + Ambiguous, + Value(String), +} + +#[expect( + clippy::arbitrary_source_item_ordering, + reason = "scalar validation precedes local-root validation; compatibility constructors remain grouped" +)] +impl AxumRunOptions { + pub(crate) fn addr(&self) -> SocketAddr { + self.addr + } + + pub(crate) fn bindings(&self) -> &NativeStoreBindingsInput { + &self.bindings + } + + pub(crate) fn bindings_limits(&self) -> NativeBindingsLimits { + self.bindings_limits + } + + pub(crate) fn aws_limits(&self) -> Option { + self.aws_limits + } + + pub(crate) fn celld_limits(&self) -> Option { + self.celld_limits + } + + pub(crate) fn preparation_policies(&self) -> NativePreparationPolicySources { + NativePreparationPolicySources { + aws: self.aws_limits(), + celld: self.celld_limits(), + } + } + + pub(crate) fn celld_token(&self, name: &str) -> anyhow::Result { + CelldAccessToken::new(self.token_value(name, "celld token")?) + .map_err(|_category| failure("settings", "celld token", "invalid configured value")) + } + + pub(crate) fn agent_token(&self, name: &str) -> anyhow::Result { + AgentAccessToken::new(self.token_value(name, "Agent token")?) + .map_err(|_category| failure("settings", "Agent token", "invalid configured value")) + } + + fn token_value(&self, name: &str, subject: &str) -> anyhow::Result { + let value = match self.bootstrap_values.get(name) { + Some(BootstrapValue::Value(value)) => value.clone(), + Some(BootstrapValue::Ambiguous) => { + return Err(failure("settings", subject, "duplicate configured value")); + } + None => { + return Err(failure("settings", subject, "configured value absent")); + } + }; + Ok(value) + } + + pub(crate) fn config_dir(&self) -> Option<&Path> { + self.config_dir.as_deref() + } + + pub(crate) fn data_dir(&self) -> Option<&Path> { + self.data_dir.as_deref() + } + + pub(crate) fn diagnostics(&self) -> NativeDiagnosticsHandle { + self.diagnostics.clone() + } + + pub(crate) fn env_config(&self) -> &EnvConfig { + &self.env + } + + /// Captures hosting settings and store overrides once. Secret values still + /// use the existing environment-backed secret backend. + /// + /// # Errors + /// Returns safe setting categories for supplied invalid hosting settings. + #[inline] + pub fn from_env() -> anyhow::Result { + let mut pairs = Vec::new(); + let mut non_unicode = Vec::new(); + for (os_key, os_value) in vars_os() { + let Some(key) = os_key.to_str() else { continue }; + let Some(rest) = strip_native_config_prefix(key) else { + continue; + }; + let path: Vec = rest.split("__").map(str::to_ascii_lowercase).collect(); + if let Some(value) = os_value.to_str() { + pairs.push((key.to_owned(), value.to_owned())); + } else if let Some(setting) = fixed_setting(&path) { + return Err(failure("settings", setting, "non-Unicode value")); + } else { + let segments: Vec<&str> = path.iter().map(String::as_str).collect(); + if matches!( + segments.as_slice(), + ["stores", "config" | "kv" | "secrets", _, "name" | "key"] + ) { + non_unicode.push(path); + } + } + } + let mut options = Self::from_vars(pairs)?; + options.non_unicode_overrides = non_unicode; + // Only this env-loading boundary may resolve named token variables. + // Explicit constructors and selection methods never call var_os. + options.capture_agent_tokens(|name| var_os(name))?; + Ok(options) + } + + fn capture_agent_tokens(&mut self, mut lookup: Lookup) -> anyhow::Result<()> + where + Lookup: FnMut(&str) -> Option, + { + // Capture shared names once; validate them against every selected token type. + let mut names = BTreeMap::::new(); + if let Some(document) = self.bindings.document() { + for binding in document.config().values() { + match binding { + NativeConfigBinding::AppConfigAgent(settings) => { + if let Some(name) = &settings.access_token_env { + names.entry(name.clone()).or_default().0 = true; + } + } + NativeConfigBinding::Celld(settings) => { + if let Some(name) = settings.access.token_env() { + names.entry(name.to_owned()).or_default().1 = true; + } + } + } + } + for binding in document.kv().values() { + if let NativeKvBinding::Celld(settings) = binding + && let Some(name) = settings.access.token_env() + { + names.entry(name.to_owned()).or_default().1 = true; + } + } + } + for (name, (aws, celld)) in names { + let subject = if celld { "celld token" } else { "Agent token" }; + let value = lookup(&name) + .and_then(|value| value.into_string().ok()) + .ok_or_else(|| { + failure( + "settings", + subject, + "configured value absent or non-Unicode", + ) + })?; + if aws { + AgentAccessToken::new(value.clone()).map_err(|_category| { + failure("settings", "Agent token", "invalid configured value") + })?; + } + if celld { + CelldAccessToken::new(value.clone()).map_err(|_category| { + failure("settings", "celld token", "invalid configured value") + })?; + } + self.bootstrap_values + .insert(name, BootstrapValue::Value(value)); + } + Ok(()) + } + + /// Constructs options using only supplied string pairs and omission defaults. + /// Declared-store overrides are validated against baked metadata at startup. + /// + /// # Errors + /// Returns safe setting categories for invalid bind, logging, roots or grace. + #[inline] + pub fn from_vars(vars: I) -> anyhow::Result + where + I: IntoIterator, + K: AsRef, + V: Into, + { + let pairs: Vec<(String, String)> = vars + .into_iter() + .map(|(key, value)| (key.as_ref().to_owned(), value.into())) + .collect(); + let mut seen = BTreeSet::new(); + for (key, _value) in &pairs { + if let Some(rest) = strip_native_config_prefix(key) { + let path: Vec = rest.split("__").map(str::to_ascii_lowercase).collect(); + let segments: Vec<&str> = path.iter().map(String::as_str).collect(); + if (fixed_setting(&path).is_some() + || matches!( + segments.as_slice(), + ["stores", "config" | "kv" | "secrets", _, "name" | "key"] + )) + && !seen.insert(path) + { + return Err(failure( + "settings", + "bootstrap", + "duplicate normalized setting", + )); + } + } + } + let env = EnvConfig::from_vars(pairs.iter().map(|(key, value)| (key, value.clone()))); + let host = env.adapter_host().map_or(Ok(addr::DEFAULT_HOST), |raw| { + raw.parse() + .map_err(|_error| failure("settings", "HOST", "invalid IP literal")) + })?; + let port = env.adapter_port().map_or(Ok(addr::DEFAULT_PORT), |raw| { + raw.parse::() + .ok() + .filter(|port| *port != 0) + .ok_or_else(|| failure("settings", "PORT", "invalid nonzero port")) + })?; + let mut options = Self::new(SocketAddr::new(host, port))?; + if let Some(level) = env.logging_level() { + options.level = LevelFilter::from_str(level) + .map_err(|_error| failure("settings", "LOGGING__LEVEL", "invalid level"))?; + } + options.grace = shutdown_grace(&env)?; + (options.trusted_proxy_policy, options.request_records) = native_settings(&env)?; + if let Some(root) = env.get(&["adapter", "config_dir"]) { + options = options.with_config_dir(root)?; + } + if let Some(root) = env.get(&["adapter", "data_dir"]) { + options = options.with_data_dir(root)?; + } + options.env = env; + for (key, value) in pairs { + match options.bootstrap_values.entry(key) { + Entry::Vacant(entry) => { + entry.insert(BootstrapValue::Value(value)); + } + Entry::Occupied(mut entry) => { + entry.insert(BootstrapValue::Ambiguous); + } + } + } + if let Some(path) = options.env.get(&["store_bindings_file"]).map(str::to_owned) { + options = options.with_store_bindings_file(path)?; + } + Ok(options) + } + + pub(crate) fn logging_level(&self) -> LevelFilter { + self.level + } + + /// Explicit bind with default grace/logging, no roots, and no store overrides. + /// + /// # Errors + /// Rejects port zero. Production omission defaults apply only to omitted values. + #[inline] + pub fn new(addr: SocketAddr) -> anyhow::Result { + if addr.port() == 0 { + return Err(failure("settings", "PORT", "invalid nonzero port")); + } + Ok(Self { + diagnostics: NativeDiagnosticsHandle::new(), + trusted_proxy_policy: TrustedProxyPolicy::no_trust(), + request_records: true, + geo: NativeGeoSelection::Off, + bindings: NativeStoreBindingsInput::LocalDefaults, + bindings_limits: NativeBindingsLimits::default(), + aws_limits: None, + bootstrap_values: BTreeMap::new(), + celld_limits: None, + addr, + config_dir: None, + data_dir: None, + env: EnvConfig::default(), + grace: Duration::from_secs(10), + level: LevelFilter::Info, + non_unicode_overrides: Vec::new(), + }) + } + + /// Select geographic resolution explicitly. Validation happens before runner resources. + #[must_use] + #[inline] + pub fn with_geo(mut self, selection: NativeGeoSelection) -> Self { + self.geo = selection; + self + } + + /// Validate the finalized selection and capture its startup capability. + /// + /// # Errors + /// Rejects unavailable implementations and invalid protected-source policies. + #[inline] + pub fn geo_capability(&self) -> Result { + self.geo.capability() + } + + pub(crate) fn native_geo(&self) -> Result { + NativeGeo::prepare(&self.geo) + } + + /// Shares native counters and the optional request observer. + #[must_use] + #[inline] + pub fn with_diagnostics(mut self, diagnostics: NativeDiagnosticsHandle) -> Self { + self.diagnostics = diagnostics; + self + } + + pub(crate) fn request_records(&self) -> bool { + self.request_records + } + pub(crate) fn trusted_proxy_policy(&self) -> TrustedProxyPolicy { + self.trusted_proxy_policy.clone() + } + + pub(crate) fn shutdown_grace(&self) -> Duration { + self.grace + } + + #[cfg(test)] + fn validate_stores(&self, metadata: StoresMetadata) -> anyhow::Result<()> { + self.validate_store_overlays(metadata)?; + self.validate_local_roots(metadata.config.is_some(), metadata.kv.is_some()) + } + + pub(crate) fn validate_store_overlays(&self, metadata: StoresMetadata) -> anyhow::Result<()> { + for (kind, declaration) in [ + ("config", metadata.config), + ("kv", metadata.kv), + ("secrets", metadata.secrets), + ] { + if let Some(meta) = declaration { + for id in meta.ids { + for field in ["name", "key"] { + if self + .non_unicode_overrides + .iter() + .any(|path| match path.as_slice() { + [_, selected_kind, selected_id, selected_field] => { + selected_kind == kind + && selected_id.eq_ignore_ascii_case(id) + && selected_field == field + } + _ => false, + }) + { + return Err(failure("settings", id, "non-Unicode store override")); + } + if self.env.store_setting(kind, id, field).is_some_and(|raw| { + raw.is_empty() + || raw.chars().all(char::is_whitespace) + || raw.chars().any(char::is_control) + }) { + return Err(failure("settings", id, "invalid store NAME/KEY override")); + } + } + } + } + } + Ok(()) + } + + pub(crate) fn validate_local_roots(&self, config: bool, kv: bool) -> anyhow::Result<()> { + for (required, root, label) in [ + (config, self.config_dir.as_deref(), "CONFIG_DIR"), + (kv, self.data_dir.as_deref(), "DATA_DIR"), + ] { + if required && root.is_none() { + return Err(failure("settings", label, "required for declared stores")); + } + if let Some(directory) = root { + validate_root(directory, label)?; + } + } + Ok(()) + } + + /// Selects typed bindings. A second explicit selection is an error. + /// + /// # Errors + /// Rejects duplicate selection and invalid settings/policy sources. + #[inline] + pub fn with_store_bindings(mut self, bindings: NativeStoreBindings) -> anyhow::Result { + if !matches!(self.bindings, NativeStoreBindingsInput::LocalDefaults) { + return Err(failure("settings", "bindings", "duplicate selection")); + } + bindings + .validate_with_policies(self.bindings_limits, self.preparation_policies()) + .map_err(|category| failure("settings", "bindings", &category.to_string()))?; + self.bindings = NativeStoreBindingsInput::Inline(bindings); + Ok(self) + } + + /// Captures one absolute bindings file once. Explicit selection does not read environment values. + /// + /// # Errors + /// Rejects duplicate/relative/missing/malformed/oversized selections. + #[inline] + pub fn with_store_bindings_file>( + mut self, + locator: Root, + ) -> anyhow::Result { + if !matches!(self.bindings, NativeStoreBindingsInput::LocalDefaults) { + return Err(failure("settings", "bindings", "duplicate selection")); + } + let path = locator.into(); + let bindings = NativeStoreBindings::read_with_policies( + &path, + self.bindings_limits, + self.preparation_policies(), + ) + .map_err(|category| failure("settings", "bindings", &category.to_string()))?; + self.bindings = NativeStoreBindingsInput::File { path, bindings }; + Ok(self) + } + + /// Supplies named bootstrap values explicitly, never merging ambient variables. + /// + /// # Errors + /// Rejects duplicate variable inputs rather than selecting precedence. + #[inline] + pub fn with_bootstrap_values(mut self, values: I) -> anyhow::Result + where + I: IntoIterator, + K: Into, + V: Into, + { + for (name, value) in values { + if self + .bootstrap_values + .insert(name.into(), BootstrapValue::Value(value.into())) + .is_some() + { + return Err(failure("settings", "bootstrap", "duplicate variable")); + } + } + Ok(self) + } + + /// Selects trusted file/ID bounds before selecting a document. + /// + /// # Errors + /// Rejects invalid bounds or policy changes after a file/inline selection. + #[inline] + pub fn with_native_bindings_limits( + mut self, + limits: NativeBindingsLimits, + ) -> anyhow::Result { + if !matches!(self.bindings, NativeStoreBindingsInput::LocalDefaults) { + return Err(failure( + "settings", + "bindings", + "limits must precede selection", + )); + } + limits + .validate() + .map_err(|category| failure("settings", "bindings", &category.to_string()))?; + self.bindings_limits = limits; + Ok(self) + } + + /// Selects one explicit remote preparation policy, not an override/merge of file policies. + /// + /// # Errors + /// Rejects duplicate policies, competing document policies and invalid limits. + #[inline] + pub fn with_aws_preparation_limits( + mut self, + limits: AwsPreparationLimits, + ) -> anyhow::Result { + if self.aws_limits.is_some() { + return Err(failure("settings", "AWS limits", "duplicate policy")); + } + limits + .validate() + .map_err(|category| failure("settings", "AWS limits", &category.to_string()))?; + if let Some(document) = self.bindings.document() { + document + .validate_with_policies( + self.bindings_limits, + NativePreparationPolicySources { + aws: Some(limits), + celld: self.celld_limits, + }, + ) + .map_err(|category| failure("settings", "AWS limits", &category.to_string()))?; + } + self.aws_limits = Some(limits); + Ok(self) + } + + /// Selects a celld policy independently of any AWS policy. + /// + /// # Errors + /// Rejects duplicate policies, same-provider document conflicts and invalid limits. + #[inline] + pub fn with_celld_preparation_limits( + mut self, + limits: CelldPreparationLimits, + ) -> anyhow::Result { + if self.celld_limits.is_some() { + return Err(failure("settings", "celld limits", "duplicate policy")); + } + limits + .validate() + .map_err(|category| failure("settings", "celld limits", &category.to_string()))?; + if let Some(document) = self.bindings.document() { + document + .validate_with_policies( + self.bindings_limits, + NativePreparationPolicySources { + aws: self.aws_limits, + celld: Some(limits), + }, + ) + .map_err(|category| failure("settings", "celld limits", &category.to_string()))?; + } + self.celld_limits = Some(limits); + Ok(self) + } + + /// Selects an existing absolute config root; never creates it. + /// + /// # Errors + /// Rejects relative, missing or non-directory roots. + #[inline] + pub fn with_config_dir>(mut self, root: Root) -> anyhow::Result { + let directory = root.into(); + validate_root(&directory, "CONFIG_DIR")?; + self.config_dir = Some(directory); + Ok(self) + } + + /// Selects an existing absolute data root. Startup also checks actual write access. + /// + /// # Errors + /// Rejects relative, missing or non-directory roots. + #[inline] + pub fn with_data_dir>(mut self, root: Root) -> anyhow::Result { + let directory = root.into(); + validate_root(&directory, "DATA_DIR")?; + self.data_dir = Some(directory); + Ok(self) + } + + #[inline] + #[must_use] + pub fn with_logging_level(mut self, level: LevelFilter) -> Self { + self.level = level; + self + } + + /// Enables only default native request logs; observers and accounting remain active. + #[inline] + #[must_use] + pub fn with_request_records(mut self, enabled: bool) -> Self { + self.request_records = enabled; + self + } + + /// Selects a positive, representable integer-second drain budget. + /// + /// # Errors + /// Rejects zero, fractional seconds and unusable native timer values. + #[inline] + pub fn with_shutdown_grace(mut self, grace: Duration) -> anyhow::Result { + validate_grace(grace)?; + self.grace = grace; + Ok(self) + } + + /// Installs an explicit immutable incoming-proxy policy without reading the environment. + #[inline] + #[must_use] + pub fn with_trusted_proxy_policy(mut self, policy: TrustedProxyPolicy) -> Self { + self.trusted_proxy_policy = policy; + self + } +} + +pub(crate) fn native_settings(env: &EnvConfig) -> anyhow::Result<(TrustedProxyPolicy, bool)> { + let policy = TrustedProxyPolicy::from_values( + env.get(&["adapter", "trusted_proxy_cidrs"]), + env.get(&["adapter", "forwarding_header_family"]), + ) + .map_err(|_error| { + failure( + "settings", + "TRUSTED_PROXY_POLICY", + "invalid literal networks or forwarding family", + ) + })?; + let records = match env.get(&["logging", "request_records"]) { + None | Some("true") => true, + Some("false") => false, + Some(_) => { + return Err(failure( + "settings", + "LOGGING__REQUEST_RECORDS", + "invalid boolean", + )); + } + }; + Ok((policy, records)) +} + +pub(crate) fn failure(stage: &str, subject: &str, category: &str) -> anyhow::Error { + // No source chain survives this boundary. Subject is a fixed setting name or + // baked logical id, never settings/config/secret contents or a provider string. + anyhow::anyhow!("startup {stage}: {subject}: {category}") +} + +fn fixed_setting(path: &[String]) -> Option<&'static str> { + let segments: Vec<&str> = path.iter().map(String::as_str).collect(); + match segments.as_slice() { + ["adapter", "host"] => Some("HOST"), + ["adapter", "port"] => Some("PORT"), + ["adapter", "config_dir"] => Some("CONFIG_DIR"), + ["adapter", "data_dir"] => Some("DATA_DIR"), + ["adapter", "shutdown_grace_seconds"] => Some("SHUTDOWN_GRACE_SECONDS"), + ["adapter", "trusted_proxy_cidrs"] => Some("TRUSTED_PROXY_CIDRS"), + ["adapter", "forwarding_header_family"] => Some("FORWARDING_HEADER_FAMILY"), + ["logging", "request_records"] => Some("LOGGING__REQUEST_RECORDS"), + ["logging", "level"] => Some("LOGGING__LEVEL"), + ["store_bindings_file"] => Some("STORE_BINDINGS_FILE"), + _ => None, + } +} + +pub(crate) fn shutdown_grace(env: &EnvConfig) -> anyhow::Result { + let Some(raw) = env.get(&["adapter", "shutdown_grace_seconds"]) else { + return Ok(Duration::from_secs(10)); + }; + let seconds = raw + .parse::() + .ok() + .filter(|_seconds| raw.bytes().all(|byte| byte.is_ascii_digit())) + .ok_or_else(|| { + failure( + "settings", + "SHUTDOWN_GRACE_SECONDS", + "invalid positive integer seconds", + ) + })?; + let grace = Duration::from_secs(seconds); + validate_grace(grace)?; + Ok(grace) +} + +/// Match the process environment's key identity, as `EnvConfig` does. +pub(crate) fn strip_native_config_prefix(key: &str) -> Option<&str> { + const PREFIX: &str = "EDGEZERO__"; + if cfg!(windows) { + let prefix = key.get(..PREFIX.len())?; + if !prefix.eq_ignore_ascii_case(PREFIX) { + return None; + } + key.get(PREFIX.len()..) + } else { + key.strip_prefix(PREFIX) + } +} + +fn validate_grace(grace: Duration) -> anyhow::Result<()> { + if grace.is_zero() || grace.subsec_nanos() != 0 || Instant::now().checked_add(grace).is_none() { + return Err(failure( + "settings", + "SHUTDOWN_GRACE_SECONDS", + "invalid or unusable positive integer seconds", + )); + } + Ok(()) +} + +fn validate_root(root: &Path, setting: &str) -> anyhow::Result<()> { + if !root.is_absolute() || !root.is_dir() { + return Err(failure( + "settings", + setting, + "requires an absolute existing directory", + )); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::fs; + + use super::*; + use edgezero_core::app::StoreMetadata; + + const BINDINGS: &str = "version = 1\n[config.app]\nprovider = 'aws-appconfig-agent'\ndefault_key = 'settings'\nendpoint = 'http://127.0.0.1:2772'\naccess_token_env = 'AGENT_TOKEN'\n[config.app.documents.settings]\napplication = 'example'\nenvironment = 'production'\nprofile = 'envelope'\n"; + + const CELLD: &str = "version=1\n[config.remote]\nprovider='celld-config'\nendpoint='https://unreachable.invalid'\ndefault_key='settings'\n[config.remote.access]\nmode='authenticated'\ntoken_env='CELL_TOKEN'\n[config.remote.documents.settings]\nstore='config-fixture'\nkey='settings'\n[kv.state]\nprovider='celld-kv'\nendpoint='https://unreachable.invalid'\nstore='state-fixture'\n[kv.state.access]\nmode='authenticated'\ntoken_env='CELL_TOKEN'\n"; + const CELL_TOKEN: &str = "celld-bootstrap-value-01234567890123456789"; + + #[test] + fn geo_selection_defaults_and_whole_replacement_preserve_native_option_bounds() { + use crate::geo::ProtectedCloudFrontAlb; + let options = AxumRunOptions::new("127.0.0.1:8787".parse().unwrap()).unwrap(); + let _thread_safe_clone: Box = Box::new(options.clone()); + assert!(matches!( + options.geo_capability(), + Ok(NativeGeoCapability::Off) + )); + let selected = options.with_geo(NativeGeoSelection::CloudFront { + policy: ProtectedCloudFrontAlb::new(["127.0.0.1/32"], "x-origin-verify-fixture") + .unwrap(), + }); + let replaced = selected.with_geo(NativeGeoSelection::Off); + assert!(matches!( + replaced.geo_capability(), + Ok(NativeGeoCapability::Off) + )); + } + + #[test] + fn native_settings_defaults_explicit_builders_and_strict_values() { + let defaults = AxumRunOptions::from_vars(Vec::<(String, String)>::new()).expect("defaults"); + assert!(defaults.request_records()); + assert_eq!( + defaults.trusted_proxy_policy(), + TrustedProxyPolicy::no_trust() + ); + let configured = AxumRunOptions::from_vars([ + ("EDGEZERO__ADAPTER__TRUSTED_PROXY_CIDRS", "127.0.0.2/32"), + ("EDGEZERO__ADAPTER__FORWARDING_HEADER_FAMILY", "forwarded"), + ("EDGEZERO__LOGGING__REQUEST_RECORDS", "false"), + ]) + .expect("configured"); + assert!(!configured.request_records()); + assert_ne!( + configured.trusted_proxy_policy(), + TrustedProxyPolicy::no_trust() + ); + let explicit = AxumRunOptions::new(defaults.addr()) + .expect("explicit") + .with_request_records(false) + .with_trusted_proxy_policy(configured.trusted_proxy_policy()); + assert!(!explicit.request_records()); + assert_eq!( + explicit.trusted_proxy_policy(), + configured.trusted_proxy_policy() + ); + assert!( + explicit + .env_config() + .get(&["adapter", "trusted_proxy_cidrs"]) + .is_none() + ); + for (key, value) in [ + ("EDGEZERO__ADAPTER__TRUSTED_PROXY_CIDRS", "127.0.0.2"), + ("EDGEZERO__ADAPTER__TRUSTED_PROXY_CIDRS", "sentinel-policy"), + ( + "EDGEZERO__ADAPTER__FORWARDING_HEADER_FAMILY", + "sentinel-family", + ), + ("EDGEZERO__LOGGING__REQUEST_RECORDS", "TRUE"), + ("EDGEZERO__LOGGING__REQUEST_RECORDS", "false "), + ("EDGEZERO__LOGGING__REQUEST_RECORDS", ""), + ] { + let error = AxumRunOptions::from_vars([(key, value)]) + .err() + .expect("invalid supplied setting"); + assert_eq!(error.chain().count(), 1); + assert!(!error.to_string().contains("sentinel")); + } + } + + fn explicit() -> AxumRunOptions { + AxumRunOptions::new(SocketAddr::from(([127, 0, 0, 1], 1234))).expect("explicit options") + } + + #[test] + fn celld_capture_unions_config_and_kv_names_and_validates_shared_aws_use() { + for (input, expected) in [ + (CELLD.to_owned(), 1), + ( + CELLD.replace( + "[kv.state.access]\nmode='authenticated'\ntoken_env='CELL_TOKEN'", + "[kv.state.access]\nmode='authenticated'\ntoken_env='OTHER_TOKEN'", + ), + 2, + ), + ( + format!( + "{BINDINGS}{}", + CELLD + .replace("version=1\n", "") + .replace("CELL_TOKEN", "AGENT_TOKEN") + ), + 1, + ), + ] { + let document = NativeStoreBindings::parse(&input, NativeBindingsLimits::default()) + .expect("bindings"); + let mut options = explicit().with_store_bindings(document).expect("inline"); + let mut calls = 0_usize; + options + .capture_agent_tokens(|_name| { + calls += 1; + Some(OsString::from(CELL_TOKEN)) + }) + .expect("captured union"); + assert_eq!(calls, expected); + let name = if input.contains("AGENT_TOKEN") { + "AGENT_TOKEN" + } else { + "CELL_TOKEN" + }; + assert_eq!( + options.celld_token(name).expect("celld token").expose(), + CELL_TOKEN + ); + if name == "AGENT_TOKEN" { + assert_eq!( + options.agent_token(name).expect("AWS token").expose(), + CELL_TOKEN + ); + } + } + let shared = format!( + "{BINDINGS}{}", + CELLD + .replace("version=1\n", "") + .replace("CELL_TOKEN", "AGENT_TOKEN") + ); + let document = + NativeStoreBindings::parse(&shared, NativeBindingsLimits::default()).expect("bindings"); + let mut options = explicit().with_store_bindings(document).expect("inline"); + // Valid AWS token, invalid celld token: shared names must not skip the stricter type. + let error = options + .capture_agent_tokens(|_name| Some(OsString::from("short"))) + .expect_err("celld minimum"); + assert_eq!(error.chain().count(), 1); + assert!(error.to_string().contains("celld token")); + } + + #[test] + fn celld_bootstrap_is_explicit_closed_and_trusted_network_looks_up_nothing() { + let document = + NativeStoreBindings::parse(CELLD, NativeBindingsLimits::default()).expect("bindings"); + let options = explicit().with_store_bindings(document).expect("inline"); + options + .celld_token("CELL_TOKEN") + .expect_err("explicit token required"); + for raw in [ + None, + Some(OsString::new()), + Some(OsString::from("short")), + Some(OsString::from("invalid token sentinel")), + ] { + let error = options + .clone() + .capture_agent_tokens(|_name| raw.clone()) + .expect_err("invalid token"); + assert_eq!(error.chain().count(), 1); + assert!(!format!("{error:?}").contains("sentinel")); + } + let supplied = options + .with_bootstrap_values([("CELL_TOKEN", CELL_TOKEN)]) + .expect("explicit token"); + assert_eq!( + supplied.celld_token("CELL_TOKEN").expect("token").expose(), + CELL_TOKEN + ); + assert!(!format!("{:?}", supplied.celld_token("CELL_TOKEN")).contains(CELL_TOKEN)); + let duplicate = + AxumRunOptions::from_vars([("CELL_TOKEN", CELL_TOKEN), ("CELL_TOKEN", CELL_TOKEN)]) + .expect("pairs"); + duplicate + .celld_token("CELL_TOKEN") + .expect_err("duplicate token values rejected"); + let trusted = CELLD + .replace("https://unreachable.invalid", "http://127.0.0.1:8788") + .replace( + "mode='authenticated'\ntoken_env='CELL_TOKEN'", + "mode='trusted-network'", + ); + let mut trusted_options = explicit() + .with_store_bindings( + NativeStoreBindings::parse(&trusted, NativeBindingsLimits::default()) + .expect("trusted bindings"), + ) + .expect("inline"); + trusted_options + .capture_agent_tokens(|_name| panic!("trusted network must not resolve tokens")) + .expect("no lookup"); + } + + #[cfg(unix)] + #[test] + fn celld_non_unicode_capture_fails_closed() { + use std::os::unix::ffi::OsStringExt as _; + let document = + NativeStoreBindings::parse(CELLD, NativeBindingsLimits::default()).expect("bindings"); + let mut options = explicit().with_store_bindings(document).expect("inline"); + assert!( + options + .capture_agent_tokens(|_name| Some(OsString::from_vec(vec![0xff]))) + .is_err() + ); + } + + #[test] + fn options_preserve_independent_policy_sources_in_both_orders() { + let aws = AwsPreparationLimits { + max_remote_values: 7, + ..Default::default() + }; + let celld = CelldPreparationLimits { + max_remote_values: 9, + ..Default::default() + }; + for options in [ + explicit() + .with_aws_preparation_limits(aws) + .expect("AWS") + .with_celld_preparation_limits(celld) + .expect("celld"), + explicit() + .with_celld_preparation_limits(celld) + .expect("celld") + .with_aws_preparation_limits(aws) + .expect("AWS"), + ] { + assert_eq!( + options.aws_limits().expect("AWS policy").max_remote_values, + 7 + ); + assert_eq!( + options + .celld_limits() + .expect("celld policy") + .max_remote_values, + 9 + ); + assert!( + options + .clone() + .with_celld_preparation_limits(celld) + .is_err() + ); + assert!(options.with_aws_preparation_limits(aws).is_err()); + } + for (table, explicit_aws) in [ + ("[limits.celld]\nmax_remote_values=9", true), + ("[limits.aws]\nmax_remote_values=7", false), + ] { + let document = NativeStoreBindings::parse( + &format!("{CELLD}\n{table}"), + NativeBindingsLimits::default(), + ) + .expect("file policy"); + let selected = explicit() + .with_store_bindings(document.clone()) + .expect("inline"); + if explicit_aws { + selected + .with_aws_preparation_limits(aws) + .expect("cross-provider policy"); + explicit() + .with_aws_preparation_limits(aws) + .expect("AWS") + .with_store_bindings(document) + .expect("cross-provider file"); + } else { + selected + .with_celld_preparation_limits(celld) + .expect("cross-provider policy"); + explicit() + .with_celld_preparation_limits(celld) + .expect("celld") + .with_store_bindings(document) + .expect("cross-provider file"); + } + } + } + + #[test] + fn native_prefix_matching_agrees_with_portable_env_config() { + for key in [ + "EDGEZERO__ADAPTER__PORT", + "edgezero__adapter__port", + "EdGeZeRo__adapter__port", + "EDGEZERO_ADAPTER__PORT", + "short", + "EDGEZERO\u{e9}__adapter__port", + ] { + let portable = EnvConfig::from_vars([(key, "9090")]); + assert_eq!( + strip_native_config_prefix(key).is_some(), + portable.adapter_port().is_some(), + "prefix identity for {key}" + ); + } + } + + #[test] + fn prefix_case_variants_capture_bindings_and_reject_duplicate_settings() { + let root = tempfile::tempdir().expect("root"); + let path = root.path().join("bindings.toml"); + fs::write(&path, BINDINGS).expect("bindings"); + for prefix in ["EDGEZERO__", "edgezero__", "EdGeZeRo__"] { + let options = AxumRunOptions::from_vars([( + format!("{prefix}store_bindings_file"), + path.to_str().expect("path"), + )]) + .expect("valid locator or unrecognized Unix prefix"); + assert_eq!( + options.bindings().document().is_some(), + prefix == "EDGEZERO__" || cfg!(windows) + ); + let duplicate = AxumRunOptions::from_vars([ + ("EDGEZERO__ADAPTER__PORT".to_owned(), "9090"), + (format!("{prefix}adapter__port"), "9091"), + ]); + if prefix == "EDGEZERO__" || cfg!(windows) { + let error = duplicate.err().expect("duplicate normalized setting"); + assert!(error.to_string().contains("duplicate normalized setting")); + } else { + assert_eq!(duplicate.expect("Unix prefix ignored").addr.port(), 9090); + } + } + } + + #[test] + fn file_selection_is_captured_once_and_conflicts_do_not_read_a_second_file() { + let root = tempfile::tempdir().expect("root"); + let path = root.path().join("bindings.toml"); + fs::write(&path, BINDINGS).expect("write bindings"); + let options = explicit() + .with_store_bindings_file(&path) + .expect("selected file"); + fs::remove_file(&path).expect("remove file after capture"); + assert_eq!( + options + .bindings() + .document() + .expect("captured") + .config() + .len(), + 1 + ); + let document = NativeStoreBindings::parse(BINDINGS, NativeBindingsLimits::default()) + .expect("typed bindings"); + assert!( + options + .clone() + .with_store_bindings(document) + .err() + .expect("duplicate") + .to_string() + .contains("duplicate") + ); + assert!( + options + .with_store_bindings_file(path) + .err() + .expect("duplicate file") + .to_string() + .contains("duplicate") + ); + assert!( + explicit() + .with_store_bindings_file("relative.toml") + .is_err() + ); + } + + #[test] + fn explicit_bootstrap_values_do_not_consult_ambient_variables() { + let document = NativeStoreBindings::parse(BINDINGS, NativeBindingsLimits::default()) + .expect("bindings"); + let unconfigured = explicit().with_store_bindings(document).expect("inline"); + let missing = unconfigured + .agent_token("AGENT_TOKEN") + .expect_err("missing token"); + assert_eq!(missing.chain().count(), 1); + let options = unconfigured + .with_bootstrap_values([("AGENT_TOKEN", "TOKEN_SENTINEL")]) + .expect("supplied token"); + assert_eq!( + options + .agent_token("AGENT_TOKEN") + .expect("captured") + .expose(), + "TOKEN_SENTINEL" + ); + assert!(!format!("{:?}", options.agent_token("AGENT_TOKEN")).contains("SENTINEL")); + assert!( + options + .with_bootstrap_values([("AGENT_TOKEN", "OTHER_SENTINEL")]) + .is_err() + ); + } + + #[test] + fn explicit_vars_capture_locator_and_named_values_without_reopening() { + let root = tempfile::tempdir().expect("root"); + let path = root.path().join("bindings.toml"); + fs::write(&path, BINDINGS).expect("write bindings"); + let options = AxumRunOptions::from_vars([ + ( + "EDGEZERO__STORE_BINDINGS_FILE", + path.to_str().expect("path"), + ), + ("AGENT_TOKEN", "TOKEN_SENTINEL"), + ]) + .expect("supplied variables"); + fs::remove_file(path).expect("remove"); + assert!(options.bindings().document().is_some()); + assert_eq!( + options.agent_token("AGENT_TOKEN").expect("token").expose(), + "TOKEN_SENTINEL" + ); + for (first, second) in [ + ( + "EDGEZERO__STORE_BINDINGS_FILE", + "EDGEZERO__store_bindings_file", + ), + ( + "EDGEZERO__STORES__CONFIG__APP__NAME", + "EDGEZERO__stores__config__app__name", + ), + ("EDGEZERO__ADAPTER__PORT", "EDGEZERO__adapter__port"), + ] { + let error = AxumRunOptions::from_vars([(first, "SENTINEL"), (second, "SENTINEL")]) + .err() + .expect("duplicate normalized path"); + assert!(error.to_string().contains("duplicate")); + assert!(!format!("{error:?}").contains("SENTINEL")); + } + } + + #[test] + fn duplicate_unrelated_vars_keep_existing_env_semantics_but_tokens_fail_closed() { + let options = AxumRunOptions::from_vars([ + ("EDGEZERO__UNRELATED", "first"), + ("EDGEZERO__UNRELATED", "second"), + ("AGENT_TOKEN", "FIRST_SENTINEL"), + ("AGENT_TOKEN", "SECOND_SENTINEL"), + ]) + .expect("unrelated env keys retain existing semantics"); + assert_eq!(options.env.get(&["unrelated"]), Some("second")); + let error = options + .agent_token("AGENT_TOKEN") + .expect_err("ambiguous token"); + assert!(error.to_string().contains("duplicate")); + assert!(!format!("{error:?}").contains("SENTINEL")); + } + + #[test] + fn named_token_capture_is_once_per_variable_and_rejects_missing_empty() { + let duplicate = format!( + "{BINDINGS}\n{}", + BINDINGS + .replace("version = 1\n", "") + .replace("config.app", "config.other") + ); + let document = NativeStoreBindings::parse(&duplicate, NativeBindingsLimits::default()) + .expect("shared token bindings"); + let options = explicit().with_store_bindings(document).expect("inline"); + let mut captured = options.clone(); + let mut calls = 0_usize; + captured + .capture_agent_tokens(|name| { + assert_eq!(name, "AGENT_TOKEN"); + calls += 1; + Some(OsString::from("TOKEN_SENTINEL")) + }) + .expect("capture"); + assert_eq!(calls, 1); + for value in [None, Some(OsString::new())] { + let error = options + .clone() + .capture_agent_tokens(|_name| value.clone()) + .expect_err("configured token invalid"); + assert!(!format!("{error:?}").contains("SENTINEL")); + } + } + + #[cfg(unix)] + #[test] + fn non_unicode_configured_token_fails_closed() { + use std::os::unix::ffi::OsStringExt as _; + let document = NativeStoreBindings::parse(BINDINGS, NativeBindingsLimits::default()) + .expect("bindings"); + let mut options = explicit().with_store_bindings(document).expect("inline"); + let error = options + .capture_agent_tokens(|_name| Some(OsString::from_vec(vec![0xff]))) + .expect_err("non-Unicode token"); + assert_eq!(error.chain().count(), 1); + assert!(error.to_string().contains("non-Unicode")); + } + + #[test] + fn trusted_caps_and_remote_policy_sources_are_explicit() { + let document = NativeStoreBindings::parse(BINDINGS, NativeBindingsLimits::default()) + .expect("bindings"); + let options = explicit().with_store_bindings(document).expect("inline"); + assert!( + options + .with_native_bindings_limits(NativeBindingsLimits::default()) + .is_err() + ); + let with_policy = NativeStoreBindings::parse( + &format!("{BINDINGS}\n[limits.aws]\nmax_snapshot_bytes = 16777216"), + NativeBindingsLimits::default(), + ) + .expect("policy"); + assert!( + explicit() + .with_aws_preparation_limits(AwsPreparationLimits::default()) + .expect("explicit policy") + .with_store_bindings(with_policy.clone()) + .is_err() + ); + assert!( + explicit() + .with_store_bindings(with_policy) + .expect("file policy") + .with_aws_preparation_limits(AwsPreparationLimits::default()) + .is_err() + ); + } + + #[test] + fn env_bootstrap_child() { + use std::env; + + let Some(expected) = env::var_os("EDGEZERO_NATIVE_BOOTSTRAP_TEST") else { + return; + }; + if expected == "celld" { + let options = AxumRunOptions::from_env().expect("celld env token capture"); + assert_eq!( + options + .celld_token("CELL_TOKEN") + .expect("captured celld token") + .expose(), + CELL_TOKEN + ); + let path = env::var_os("EDGEZERO__STORE_BINDINGS_FILE").expect("locator"); + fs::write( + PathBuf::from(path), + CELLD.replace("CELL_TOKEN", "OTHER_TOKEN"), + ) + .expect("change file after capture"); + assert_eq!( + options + .celld_token("CELL_TOKEN") + .expect("retained token") + .expose(), + CELL_TOKEN + ); + } else if expected == "valid" { + let options = AxumRunOptions::from_env().expect("env token capture"); + assert_eq!( + options + .agent_token("AGENT_TOKEN") + .expect("captured token") + .expose(), + "TOKEN_SENTINEL" + ); + let path = env::var_os("EDGEZERO__STORE_BINDINGS_FILE").expect("locator"); + fs::write( + PathBuf::from(path), + BINDINGS.replace("AGENT_TOKEN", "OTHER_SENTINEL"), + ) + .expect("change file after capture"); + let NativeConfigBinding::AppConfigAgent(settings) = options + .bindings() + .document() + .expect("captured document") + .config() + .get("app") + .expect("captured binding") + else { + panic!("expected AppConfig binding") + }; + assert_eq!(settings.access_token_env.as_deref(), Some("AGENT_TOKEN")); + assert_eq!( + options + .agent_token("AGENT_TOKEN") + .expect("retained token") + .expose(), + "TOKEN_SENTINEL" + ); + } else { + let error = AxumRunOptions::from_env() + .err() + .expect("invalid env bootstrap"); + assert_eq!(error.chain().count(), 1); + assert!(!format!("{error:?}").contains("SENTINEL")); + } + } + + #[cfg(unix)] + #[test] + fn real_env_entrypoint_captures_named_tokens_and_rejects_non_unicode_inputs() { + use std::env::current_exe; + use std::os::unix::ffi::OsStringExt as _; + use std::process::Command; + + let root = tempfile::tempdir().expect("root"); + let path = root.path().join("bindings.toml"); + for (token, valid) in [ + (Some(OsString::from("TOKEN_SENTINEL")), true), + (None, false), + (Some(OsString::new()), false), + (Some(OsString::from_vec(vec![0xff])), false), + ] { + fs::write(&path, BINDINGS).expect("fresh bindings"); + let mut command = Command::new(current_exe().expect("test binary")); + command + .env_clear() + .arg("--exact") + .arg("run_options::tests::env_bootstrap_child") + .env( + "EDGEZERO_NATIVE_BOOTSTRAP_TEST", + if valid { "valid" } else { "invalid" }, + ) + .env("EDGEZERO__STORE_BINDINGS_FILE", &path); + if let Some(value) = token { + command.env("AGENT_TOKEN", value); + } + let output = command.output().expect("isolated env fixture"); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + assert!(!String::from_utf8_lossy(&output.stdout).contains("SENTINEL")); + assert!(!String::from_utf8_lossy(&output.stderr).contains("SENTINEL")); + } + let output = Command::new(current_exe().expect("binary")) + .env_clear() + .arg("--exact") + .arg("run_options::tests::env_bootstrap_child") + .env("EDGEZERO_NATIVE_BOOTSTRAP_TEST", "invalid") + .env( + "EDGEZERO__STORE_BINDINGS_FILE", + OsString::from_vec(vec![0xff]), + ) + .output() + .expect("non-Unicode locator fixture"); + assert!(output.status.success()); + } + + #[cfg(unix)] + #[test] + fn real_env_celld_capture_rejects_invalid_selected_values_and_retains_snapshot() { + use std::env::current_exe; + use std::os::unix::ffi::OsStringExt as _; + use std::process::Command; + let root = tempfile::tempdir().expect("root"); + let path = root.path().join("bindings.toml"); + for (token, valid) in [ + (Some(OsString::from(CELL_TOKEN)), true), + (None, false), + (Some(OsString::new()), false), + (Some(OsString::from("short")), false), + (Some(OsString::from_vec(vec![0xff])), false), + ] { + fs::write(&path, CELLD).expect("bindings"); + let mut command = Command::new(current_exe().expect("test binary")); + command + .env_clear() + .arg("--exact") + .arg("run_options::tests::env_bootstrap_child") + .env( + "EDGEZERO_NATIVE_BOOTSTRAP_TEST", + if valid { "celld" } else { "invalid" }, + ) + .env("EDGEZERO__STORE_BINDINGS_FILE", &path); + if let Some(value) = token { + command.env("CELL_TOKEN", value); + } + let output = command.output().expect("isolated env child"); + assert!(output.status.success(), "child completed assertions"); + assert!(!String::from_utf8_lossy(&output.stdout).contains(CELL_TOKEN)); + assert!(!String::from_utf8_lossy(&output.stderr).contains(CELL_TOKEN)); + } + } + + #[test] + fn defaults_and_invalid_supplied_scalars_are_distinct() { + let options = AxumRunOptions::from_vars(Vec::<(String, String)>::new()).expect("defaults"); + assert_eq!( + options.addr, + SocketAddr::from((addr::DEFAULT_HOST, addr::DEFAULT_PORT)) + ); + assert_eq!(options.grace, Duration::from_secs(10)); + for (key, values) in [ + ( + "EDGEZERO__ADAPTER__HOST", + vec!["sentinel-host", "", "localhost"], + ), + ( + "EDGEZERO__ADAPTER__PORT", + vec!["0", "65536", "sentinel-port", ""], + ), + ("EDGEZERO__LOGGING__LEVEL", vec!["sentinel-level", ""]), + ( + "EDGEZERO__ADAPTER__SHUTDOWN_GRACE_SECONDS", + vec![ + "0", + "", + "-1", + "+1", + "sentinel-duration", + "18446744073709551615", + ], + ), + ] { + for value in values { + let error = AxumRunOptions::from_vars([(key, value)]) + .err() + .expect("invalid"); + assert!(error.chain().count() == 1); + assert!(!format!("{error:?}").contains("sentinel")); + } + } + } + + #[test] + fn selected_override_validation_precedes_normalization() { + let metadata = StoresMetadata { + config: Some(StoreMetadata { + ids: &["settings"], + default: "settings", + }), + ..StoresMetadata::default() + }; + let root = tempfile::tempdir().expect("root"); + for value in ["", " ", "sentinel\n"] { + let options = AxumRunOptions::from_vars([ + ( + "EDGEZERO__ADAPTER__CONFIG_DIR", + root.path().to_str().expect("path"), + ), + ("EDGEZERO__STORES__CONFIG__SETTINGS__KEY", value), + ("EDGEZERO__STORES__KV__UNDECLARED__NAME", "\n"), + ]) + .expect("native settings"); + let error = options + .validate_stores(metadata) + .expect_err("declared invalid override"); + assert!(!format!("{error:?}").contains("sentinel")); + } + let options = + AxumRunOptions::new(SocketAddr::from(([127, 0, 0, 1], 1234))).expect("explicit"); + assert!(options.env.adapter_host().is_none()); + assert_eq!(options.env.store_key("config", "settings"), "settings"); + assert!(options.validate_stores(metadata).is_err()); + } +} diff --git a/crates/edgezero-adapter-axum/src/service.rs b/crates/edgezero-adapter-axum/src/service.rs index 4f5925afa..2a170dc47 100644 --- a/crates/edgezero-adapter-axum/src/service.rs +++ b/crates/edgezero-adapter-axum/src/service.rs @@ -1,5 +1,6 @@ use std::future::Future; use std::net::SocketAddr; +use std::panic::catch_unwind; use std::pin::Pin; use std::sync::Arc; #[cfg(test)] @@ -8,15 +9,18 @@ use std::task::{Context, Poll}; use axum::body::Body as AxumBody; use axum::extract::connect_info::ConnectInfo; use axum::http::{Request, Response}; -use edgezero_core::app::App; +use edgezero_core::app::{App, FrameworkErrorDetail}; use edgezero_core::config_store::ConfigStoreHandle; use edgezero_core::error::EdgeError; -use edgezero_core::http::Method; +use edgezero_core::geo::GeoOutcome; +use edgezero_core::http::{Method, RequestParts}; use edgezero_core::ingress::{ IngressBeginOutcome, IngressDispatchOutcome, IngressFraming, IngressHeadAccounting, - IngressHeadParts, validate_normalized_ingress_parts, + IngressHeadParts, NormalizedIngressHeadSummary, PreparedIngress, + validate_normalized_ingress_parts_with_summary, }; use edgezero_core::key_value_store::KvHandle; +use edgezero_core::probe::{LifecyclePhase, LifecycleReader}; #[cfg(test)] use edgezero_core::router::RouterService; use edgezero_core::secret_store::SecretHandle; @@ -26,14 +30,25 @@ use edgezero_core::store_registry::{ use edgezero_core::time::MonotonicInstant; use hyper::body::Incoming; use hyper::service::Service as HyperService; +use tokio::sync::watch; #[cfg(test)] use tower::Service as TowerService; #[cfg(test)] use crate::connection::{ConnectionExit, serve_http1}; +use crate::context::AxumRequestContext; +#[cfg(test)] +use crate::diagnostics::NativeDiagnosticsHandle; +use crate::diagnostics::{ + ConnectionActivity, ConnectionGuard, IngressDisposition, NativeIngressMetadata, NativeSession, + RequestGuard, +}; +use crate::geo::NativeGeo; #[cfg(test)] use crate::ingress_config::AxumIngressConfig; +#[cfg(test)] use crate::outbound::AxumOutboundClient; +use crate::proxy::{TrustedProxyPolicy, normalize, strip_forwarding_headers}; use crate::request::into_core_request_parts; use crate::response::{AxumEgressBody, EgressConnection, prepare_egress_response}; @@ -45,14 +60,19 @@ pub(crate) struct AxumIngressAbort; /// Shared application state used to construct one private Hyper service per HTTP/1 connection. #[derive(Clone)] pub(crate) struct AxumServiceState { + activity: Option, app: Arc, config_registry: Option, config_store_handle: Option, + geo: NativeGeo, kv_handle: Option, kv_registry: Option, + native: NativeSession, outbound_transport: Option>, + phase: Option>, secret_handle: Option, secret_registry: Option, + trusted_proxy_policy: TrustedProxyPolicy, } #[expect( @@ -63,19 +83,62 @@ impl AxumServiceState { /// Creates a service that preserves all policies configured on `App`. #[must_use] #[inline] + #[cfg(test)] pub fn from_app(app: App) -> Self { + Self::with_transport( + app, + AxumOutboundClient::try_transport().expect("test transport"), + ) + } + + #[cfg(test)] + pub(crate) fn with_transport(app: App, transport: Arc) -> Self { + Self::with_native_transport( + app, + transport, + NativeSession::new(NativeDiagnosticsHandle::new(), true).expect("test native identity"), + TrustedProxyPolicy::no_trust(), + NativeGeo::default(), + ) + } + + pub(crate) fn with_native_transport( + app: App, + transport: Arc, + native: NativeSession, + policy: TrustedProxyPolicy, + geo: NativeGeo, + ) -> Self { Self { + activity: None, + geo, + native, + trusted_proxy_policy: policy, app: Arc::new(app), config_registry: None, config_store_handle: None, kv_handle: None, kv_registry: None, - outbound_transport: AxumOutboundClient::try_transport().ok(), + outbound_transport: Some(transport), + phase: None, secret_handle: None, secret_registry: None, } } + #[cfg(test)] + pub(crate) fn with_native( + mut self, + native: NativeSession, + policy: TrustedProxyPolicy, + geo: NativeGeo, + ) -> Self { + self.native = native; + self.trusted_proxy_policy = policy; + self.geo = geo; + self + } + #[must_use] #[inline] #[cfg(test)] @@ -103,6 +166,7 @@ impl AxumServiceState { /// directly. #[must_use] #[inline] + #[cfg(test)] pub fn with_config_store_handle(mut self, handle: ConfigStoreHandle) -> Self { self.config_store_handle = Some(handle); self @@ -120,6 +184,7 @@ impl AxumServiceState { /// directly. #[must_use] #[inline] + #[cfg(test)] pub fn with_kv_handle(mut self, handle: KvHandle) -> Self { self.kv_handle = Some(handle); self @@ -146,6 +211,7 @@ impl AxumServiceState { /// [`Self::with_secret_registry`] directly. #[must_use] #[inline] + #[cfg(test)] pub fn with_secret_handle(mut self, handle: SecretHandle) -> Self { self.secret_handle = Some(handle); self @@ -159,6 +225,15 @@ impl AxumServiceState { self } + pub(crate) fn with_phase(mut self, phase: watch::Receiver) -> Self { + self.phase = Some(phase); + self + } + + pub(crate) fn connection_activity(&self) -> (ConnectionActivity, ConnectionGuard) { + self.native.connection() + } + pub(crate) fn for_connection( &self, remote_addr: SocketAddr, @@ -180,58 +255,155 @@ impl AxumServiceState { ) { // Legacy single-handle constructors synthesize only the conventional // `default` registry entry; request extensions contain registries only. - let config_registry = self.config_registry.clone().or_else(|| { - self.config_store_handle.clone().map(|handle| { - ConfigRegistry::single_id( - "default".to_owned(), - ConfigStoreBinding { - handle, - default_key: "default".to_owned(), - }, - ) - }) - }); - let kv_registry = self.kv_registry.clone().or_else(|| { - self.kv_handle - .clone() - .map(|handle| KvRegistry::single_id("default".to_owned(), handle)) - }); - let secret_registry = self.secret_registry.clone().or_else(|| { - self.secret_handle.clone().map(|handle| { - SecretRegistry::single_id( - "default".to_owned(), - BoundSecretStore::new(handle, "default".to_owned()), - ) - }) - }); + let config_registry = self + .config_registry + .clone() + .or_else(|| self.config_store_handle.clone().map(legacy_config_registry)); + let kv_registry = self + .kv_registry + .clone() + .or_else(|| self.kv_handle.clone().map(legacy_kv_registry)); + let secret_registry = self + .secret_registry + .clone() + .or_else(|| self.secret_handle.clone().map(legacy_secret_registry)); (config_registry, kv_registry, secret_registry) } + fn admission_open(&self) -> bool { + self.phase + .as_ref() + .is_none_or(|phase| *phase.borrow() == LifecyclePhase::Ready) + } + + fn sanitize_received_head( + &self, + parts: &mut RequestParts, + ) -> Result { + // Original accounting precedes credential removal, including rejected ingress. + let received = + validate_normalized_ingress_parts_with_summary(parts, self.app.ingress_head_limits()); + self.geo.strip_credential(&mut parts.headers); + received + } + + async fn dispatch_prepared( + &self, + prepared: PreparedIngress, + parts: RequestParts, + native_body: AxumBody, + connection: &EgressConnection, + guard: RequestGuard, + geo: GeoOutcome, + ) -> Response { + let app = &self.app; + let read_deadline = prepared.read_deadline(); + let monotonic_clock = prepared.monotonic_clock(); + let Some(outbound_transport) = self.outbound_transport.clone() else { + let egress = app.admitted_error_egress( + prepared, + EdgeError::internal(anyhow::anyhow!( + "failed to initialize outbound HTTP transport" + )), + ); + return prepare_native_egress(egress, connection, guard).await; + }; + let mut core_request = match into_core_request_parts( + parts, + native_body, + Some((read_deadline, monotonic_clock)), + Some(outbound_transport), + ) { + Ok(converted) => converted, + Err(error) => { + let egress = app.admitted_error_egress( + prepared, + EdgeError::internal(anyhow::anyhow!( + "failed to convert inbound request: {error}" + )), + ); + return prepare_native_egress(egress, connection, guard).await; + } + }; + if let Some(phase) = self.phase.clone() { + core_request + .extensions_mut() + .insert(LifecycleReader::new(move || Some(*phase.borrow()))); + } + let (config_registry, kv_registry, secret_registry) = self.resolved_store_registries(); + if let Some(registry) = config_registry { + core_request.extensions_mut().insert(registry); + } + if let Some(registry) = kv_registry { + core_request.extensions_mut().insert(registry); + } + if let Some(registry) = secret_registry { + core_request.extensions_mut().insert(registry); + } + let egress = app + .dispatch_admitted_with_geo( + prepared, + core_request, + FrameworkErrorDetail::CategoryOnly, + geo, + ) + .await; + prepare_native_egress(egress, connection, guard).await + } + async fn dispatch_request( &self, mut request: Request, connection: &EgressConnection, remote_addr: Option, ) -> Result, AxumIngressAbort> { + let request_start = self.app.monotonic_now(); + let mut guard = self.native.request(request.method(), request_start, self.app.monotonic_clock(), self.activity.clone()) + .map_err(|_error| { + let _logged = catch_unwind(|| log::error!(target: "edgezero::native", "event=lifecycle reason=request_identity_exhausted")); + AxumIngressAbort + })?; + // Identity and lifetime start before the gate, without admitting/body polling. + if !self.admission_open() { + guard.finish_ingress(IngressDisposition::Draining); + return Err(AxumIngressAbort); + } + // Only the owning socket supplies native peer facts, including an absent peer. + request.extensions_mut().remove::(); if let Some(peer_addr) = remote_addr { request.extensions_mut().insert(ConnectInfo(peer_addr)); + } else { + request.extensions_mut().remove::>(); } - let request_start = self.app.monotonic_now(); let app = Arc::clone(&self.app); - let outbound_transport_handle = self.outbound_transport.clone(); - let (config_registry, kv_registry, secret_registry) = self.resolved_store_registries(); - let (parts, native_body) = request.into_parts(); + let (mut parts, native_body) = request.into_parts(); let request_method = parts.method.clone(); - if let Err(error) = validate_normalized_ingress_parts(&parts, app.ingress_head_limits()) { - return prepare_detached_ingress_error( - &app, - error, - request_method.clone(), - request_start, - connection, - ) - .await; - } + // Account for the original received head, but remove credentials before any error + // renderer or admission callback. Sanitization must also cover rejected ingress. + let received = match self.sanitize_received_head(&mut parts) { + Ok(summary) => summary, + Err(error) => { + return prepare_detached_ingress_error( + &app, + error, + request_method.clone(), + request_start, + connection, + guard, + ) + .await; + } + }; + let checked = normalize(&self.trusted_proxy_policy, remote_addr, &parts); + let reader_input = self.geo.reader_input(&checked); + guard.set_forwarding_result(checked.forwarding_result()); + strip_forwarding_headers(&mut parts.headers); + parts.extensions.insert(checked.effective_host().clone()); + parts.extensions.insert(NativeIngressMetadata::new( + guard.request_id(), + checked, + received, + )); let head_parts = IngressHeadParts::from_parts( &parts, IngressHeadAccounting::HostManaged, @@ -240,9 +412,13 @@ impl AxumServiceState { let prepared = match app.begin_ingress(head_parts, request_start) { Ok(IngressBeginOutcome::Admitted(prepared)) => prepared, Ok(IngressBeginOutcome::Refused(response)) => { - return Ok(prepare_egress_response(response, connection).await); + guard.refused(); + return Ok(prepare_native_egress(response, connection, guard).await); + } + Ok(IngressBeginOutcome::Aborted) => { + guard.finish_ingress(IngressDisposition::Aborted); + return Err(AxumIngressAbort); } - Ok(IngressBeginOutcome::Aborted) => return Err(AxumIngressAbort), Ok(_) => { return prepare_detached_ingress_error( &app, @@ -250,6 +426,7 @@ impl AxumServiceState { request_method, request_start, connection, + guard, ) .await; } @@ -260,51 +437,17 @@ impl AxumServiceState { request_method.clone(), request_start, connection, + guard, ) .await; } }; - let read_deadline = prepared.read_deadline(); - let monotonic_clock = prepared.monotonic_clock(); - let Some(outbound_transport) = outbound_transport_handle else { - let egress = app.admitted_error_egress( - prepared, - EdgeError::internal(anyhow::anyhow!( - "failed to initialize outbound HTTP transport" - )), - ); - return Ok(prepare_egress_response(egress, connection).await); - }; - let mut core_request = match into_core_request_parts( - parts, - native_body, - Some((read_deadline, monotonic_clock)), - Some(outbound_transport), - ) { - Ok(converted) => converted, - Err(error) => { - let egress = app.admitted_error_egress( - prepared, - EdgeError::internal(anyhow::anyhow!( - "failed to convert inbound request: {error}" - )), - ); - return Ok(prepare_egress_response(egress, connection).await); - } - }; - - if let Some(registry) = config_registry { - core_request.extensions_mut().insert(registry); - } - if let Some(registry) = kv_registry { - core_request.extensions_mut().insert(registry); - } - if let Some(registry) = secret_registry { - core_request.extensions_mut().insert(registry); - } - - let egress = app.dispatch_admitted(prepared, core_request).await; - Ok(prepare_egress_response(egress, connection).await) + let geo = self + .geo + .resolve(remote_addr, &parts.headers, reader_input.as_ref()); + Ok(self + .dispatch_prepared(prepared, parts, native_body, connection, guard, geo) + .await) } } @@ -315,6 +458,13 @@ pub(crate) struct AxumConnectionService { state: AxumServiceState, } +impl AxumConnectionService { + pub(crate) fn with_activity(mut self, activity: ConnectionActivity) -> Self { + self.state.activity = Some(activity); + self + } +} + impl HyperService> for AxumConnectionService { type Error = AxumIngressAbort; type Future = Pin>>>; @@ -354,21 +504,67 @@ impl TowerService> for AxumServiceState { } } +pub(crate) fn legacy_config_registry(handle: ConfigStoreHandle) -> ConfigRegistry { + ConfigRegistry::single_id( + "default".to_owned(), + ConfigStoreBinding { + handle, + default_key: "default".to_owned(), + }, + ) +} + +pub(crate) fn legacy_kv_registry(handle: KvHandle) -> KvRegistry { + KvRegistry::single_id("default".to_owned(), handle) +} + +pub(crate) fn legacy_secret_registry(handle: SecretHandle) -> SecretRegistry { + SecretRegistry::single_id( + "default".to_owned(), + BoundSecretStore::new(handle, "default".to_owned()), + ) +} + async fn prepare_detached_ingress_error( app: &App, error: EdgeError, request_method: Method, request_start: MonotonicInstant, connection: &EgressConnection, + mut guard: RequestGuard, ) -> Result, AxumIngressAbort> { - match app.detached_ingress_error_egress(error, request_method, request_start) { - IngressDispatchOutcome::Response(envelope) => { - Ok(prepare_egress_response(*envelope, connection).await) - } - IngressDispatchOutcome::Aborted | _ => Err(AxumIngressAbort), + guard.set_ingress_error_kind(error.kind()); + if let IngressDispatchOutcome::Response(envelope) = + app.detached_ingress_error_egress(error, request_method, request_start) + { + Ok(prepare_native_egress(*envelope, connection, guard).await) + } else { + guard.finish_ingress(IngressDisposition::Aborted); + Err(AxumIngressAbort) + } +} + +async fn prepare_native_egress( + envelope: edgezero_core::ResponseEgressEnvelope, + connection: &EgressConnection, + mut guard: RequestGuard, +) -> Response { + if let Some(failure) = envelope.failure_classification() { + guard.set_failure(Some(failure)); } + let (completion, selected) = guard.completion(); + prepare_egress_response( + envelope.with_completion(completion), + connection, + Some(&selected), + ) + .await } +#[cfg(test)] +#[path = "service_geo_reader_tests.rs"] +mod geo_reader_tests; + #[cfg(test)] mod tests { #![expect( @@ -377,7 +573,11 @@ mod tests { )] use super::*; + use crate::diagnostics::{ + NativeRequestFailure, NativeRequestObserver, NativeRequestOutcome, NativeRequestRecord, + }; use bytes::{Bytes, BytesMut}; + use edgezero_core::action; use edgezero_core::body::Body; use edgezero_core::config_store::{ BoundedStoreRead, ConfigStore, ConfigStoreError, ConfigStoreHandle, @@ -414,7 +614,295 @@ mod tests { use tokio::time::{sleep, timeout}; use tower::ServiceExt as _; - async fn to_bytes(mut body: AxumEgressBody, limit: usize) -> Result { + #[expect( + clippy::too_many_lines, + reason = "one counted-provider scenario verifies admission and terminal boundaries without resetting shared evidence" + )] + #[tokio::test(flavor = "current_thread")] + async fn geo_managed_resolution_runs_once_from_owned_ingress() { + use crate::geo::{NativeGeoSelection, ProtectedCloudFrontAlb}; + use edgezero_core::geo::{ + GeoError, GeoInput, GeoOutcome, GeoProvider, GeoProviderId, GeoResolution, GeoStatus, + GeoUnavailableReason, + }; + use futures_util::FutureExt as _; + use futures_util::stream::iter as result_stream; + use std::future::pending; + struct Counted(Arc>>); + impl GeoProvider for Counted { + fn resolve(&self, input: &GeoInput) -> GeoResolution { + self.0.lock().unwrap().push(input.clone()); + if matches!(input, GeoInput::Assertions { country, .. } if country.as_bytes().is_err()) + { + GeoResolution::Error(GeoError::InvalidAssertion) + } else { + GeoResolution::MissingMetadata + } + } + } + struct ObserveAndMutate; + #[async_trait::async_trait(?Send)] + impl Middleware for ObserveAndMutate { + async fn handle( + &self, + mut ctx: RequestContext, + next: Next<'_>, + ) -> Result { + assert!(!ctx.headers().contains_key("x-origin-verify-fixture")); + let owned = ctx.geo().clone(); + ctx.extensions_mut().insert(GeoOutcome::NoSelection); + ctx.headers_mut() + .insert("cloudfront-viewer-country", HeaderValue::from_static("DE")); + assert_eq!(ctx.geo(), &owned); + next.run(ctx).await + } + } + let calls = Arc::new(Mutex::new(Vec::new())); + let selection = NativeGeoSelection::Supplied { + policy: ProtectedCloudFrontAlb::new(["127.0.0.1/32"], "x-origin-verify-fixture") + .unwrap(), + id: GeoProviderId::new("counted").unwrap(), + provider: Arc::new(Counted(Arc::clone(&calls))), + }; + let geo = NativeGeo::prepare(&selection).unwrap(); + let admission_calls = Arc::new(AtomicUsize::new(0)); + let handler_calls = Arc::new(AtomicUsize::new(0)); + let observed_handler = Arc::clone(&handler_calls); + let router = RouterService::builder() + .middleware(ObserveAndMutate) + .get("/{mode}", move |mut ctx: RequestContext| { + let handler = Arc::clone(&observed_handler); + async move { + handler.fetch_add(1, Ordering::SeqCst); + assert!(!ctx.headers().contains_key("x-origin-verify-fixture")); + let owned = ctx.geo().clone(); + assert!(matches!(owned, GeoOutcome::Selected { .. })); + ctx.extensions_mut().remove::(); + assert_eq!(ctx.geo(), &owned); + if ctx.uri().path() == "/handler-cancel" { + pending::<()>().await; + } + let label = match ctx.geo() { + GeoOutcome::Selected { + status: GeoStatus::Unavailable(GeoUnavailableReason::MissingMetadata), + .. + } => "admitted", + GeoOutcome::Selected { + status: GeoStatus::Error(_), + .. + } => "error", + GeoOutcome::NoSelection | GeoOutcome::Selected { .. } => "untrusted", + }; + let body = if ctx.uri().path() == "/egress-fail" { + Body::from_stream(result_stream([Err(EdgeError::bad_request( + "fixture egress failure", + ))])) + } else { + Body::from(label) + }; + response_builder() + .body(body) + .map_err(|error| EdgeError::internal(anyhow::Error::new(error))) + } + }) + .build(); + let mut app = App::new(router); + app.set_ingress_head_limits( + IngressHeadLimits::default() + .with_max_request_header_bytes(128) + .unwrap(), + ); + let observed_admission = Arc::clone(&admission_calls); + app.set_ingress_admission_policy(move |head| { + observed_admission.fetch_add(1, Ordering::SeqCst); + assert!(!head.headers().contains_key("x-origin-verify-fixture")); + match head.target().path() { + "/refuse" => AdmissionDecision::Refuse { + completion: ResponseEgressCompletion::empty(), + response: response_builder() + .status(StatusCode::SERVICE_UNAVAILABLE) + .body(Body::empty()) + .unwrap(), + }, + "/abort" => AdmissionDecision::Abort, + _ => AdmissionDecision::Admit { + completion: ResponseEgressCompletion::empty(), + grant: IngressGrant::empty(), + read_deadline: head.read_deadline_after(Duration::from_secs(1)), + }, + } + }); + let mut service = AxumServiceState::from_app(app); + service.geo = geo; + let trusted: SocketAddr = "127.0.0.1:1234".parse().unwrap(); + for (peer, country, path, expected) in [ + (Some(trusted), Some("US"), "/good", "admitted"), + (None, Some("US"), "/forged", "untrusted"), + ( + Some("127.0.0.2:1234".parse().unwrap()), + Some("US"), + "/forged", + "untrusted", + ), + (Some(trusted), None, "/missing", "admitted"), + ] { + let mut request = Request::builder() + .uri(path) + .body(AxumBody::empty()) + .unwrap(); + request.extensions_mut().insert(ConnectInfo(trusted)); + request.extensions_mut().insert(AxumRequestContext { + remote_addr: Some(trusted), + }); + request.extensions_mut().insert(GeoOutcome::NoSelection); + request.headers_mut().append( + "x-origin-verify-fixture", + HeaderValue::from_static("secret-one"), + ); + request.headers_mut().append( + "x-origin-verify-fixture", + HeaderValue::from_static("secret-two"), + ); + if let Some(assertion) = country { + request.headers_mut().insert( + "cloudfront-viewer-country", + HeaderValue::from_static(assertion), + ); + } + let response = service + .dispatch_request(request, &EgressConnection::default(), peer) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(to_bytes(response.into_body(), 128).await.unwrap(), expected); + } + assert_eq!(calls.lock().unwrap().len(), 2); + assert!( + matches!(&calls.lock().unwrap()[0], GeoInput::Assertions { country, .. } if country.as_bytes().unwrap() == Some(b"US".as_slice())) + ); + assert!( + matches!(&calls.lock().unwrap()[1], GeoInput::Assertions { country, .. } if country.as_bytes().unwrap().is_none()) + ); + for path in ["/invalid", "/egress-drop", "/egress-fail"] { + let expected_calls = calls.lock().unwrap().len().checked_add(1).unwrap(); + let mut request = Request::builder() + .uri(path) + .body(AxumBody::empty()) + .unwrap(); + if path == "/invalid" { + request + .headers_mut() + .append("cloudfront-viewer-country", HeaderValue::from_static("US")); + request + .headers_mut() + .append("cloudfront-viewer-country", HeaderValue::from_static("US")); + } + let response = service + .dispatch_request(request, &EgressConnection::default(), Some(trusted)) + .await + .unwrap(); + assert_eq!( + response.status(), + StatusCode::OK, + "Geo errors remain outcomes, not HTTP errors" + ); + assert_eq!(calls.lock().unwrap().len(), expected_calls); + match path { + "/invalid" => { + assert_eq!(to_bytes(response.into_body(), 128).await.unwrap(), "error"); + } + "/egress-drop" => drop(response), + _ => { + to_bytes(response.into_body(), 128).await.unwrap_err(); + } + } + assert_eq!( + calls.lock().unwrap().len(), + expected_calls, + "terminal work does not resolve Geo again" + ); + } + assert!( + matches!(&calls.lock().unwrap()[2], GeoInput::Assertions { country, .. } if country.as_bytes().is_err()) + ); + for path in ["/refuse", "/abort"] { + let request = Request::builder() + .uri(path) + .body(AxumBody::empty()) + .unwrap(); + let outcome = service + .dispatch_request(request, &EgressConnection::default(), Some(trusted)) + .await; + if path == "/refuse" { + assert_eq!(outcome.unwrap().status(), StatusCode::SERVICE_UNAVAILABLE); + } else { + assert!(outcome.is_err()); + } + } + let before = admission_calls.load(Ordering::SeqCst); + let invalid = Request::builder() + .uri("/oversized") + .header("x-origin-verify-fixture", "x".repeat(256)) + .body(AxumBody::empty()) + .unwrap(); + let (mut parts, _) = invalid.into_parts(); + parts.headers.append( + "x-origin-verify-fixture", + HeaderValue::from_static("duplicate-secret"), + ); + service.sanitize_received_head(&mut parts).unwrap_err(); + assert!(!parts.headers.contains_key("x-origin-verify-fixture")); + let request = Request::builder() + .uri("/oversized") + .header("x-origin-verify-fixture", "x".repeat(256)) + .body(AxumBody::empty()) + .unwrap(); + let response = service + .dispatch_request(request, &EgressConnection::default(), Some(trusted)) + .await + .unwrap(); + assert_eq!( + response.status(), + StatusCode::REQUEST_HEADER_FIELDS_TOO_LARGE + ); + assert_eq!(admission_calls.load(Ordering::SeqCst), before); + drop(response); + let cancelled = Request::builder() + .uri("/handler-cancel") + .body(AxumBody::empty()) + .unwrap(); + assert!( + service + .dispatch_request(cancelled, &EgressConnection::default(), Some(trusted)) + .now_or_never() + .is_none() + ); + assert_eq!( + calls.lock().unwrap().len(), + 6, + "cancellation does not resolve again" + ); + let (phase_owner, reader) = watch::channel(LifecyclePhase::Draining); + service.phase = Some(reader); + let draining = Request::builder() + .uri("/draining") + .body(AxumBody::empty()) + .unwrap(); + assert!( + service + .dispatch_request(draining, &EgressConnection::default(), Some(trusted)) + .await + .is_err() + ); + drop(phase_owner); + assert_eq!(handler_calls.load(Ordering::SeqCst), 8); + assert_eq!(calls.lock().unwrap().len(), 6); + } + + pub(super) async fn to_bytes( + mut body: AxumEgressBody, + limit: usize, + ) -> Result { let mut buffered = BytesMut::new(); loop { let next_frame = poll_future(|context| Pin::new(&mut body).poll_frame(context)).await; @@ -765,6 +1253,17 @@ mod tests { #[tokio::test(flavor = "current_thread")] async fn normalized_ingress_error_decision_abort_closes_without_response() { + #[derive(Clone)] + struct NativeRecords(Arc>>); + impl NativeRequestObserver for NativeRecords { + #[inline] + fn observe(&self, record: &NativeRequestRecord) { + self.0.lock().expect("native records").push(record.clone()); + } + } + let records = Arc::new(Mutex::new(Vec::new())); + let diagnostics = NativeDiagnosticsHandle::new() + .with_request_observer(NativeRecords(Arc::clone(&records))); let listener = TcpListener::bind("127.0.0.1:0") .await .expect("bind listener"); @@ -781,7 +1280,8 @@ mod tests { observed_factory_calls.fetch_add(1, Ordering::SeqCst); DetachedResponseEgressDecision::Abort }); - let state = AxumServiceState::from_app(app); + let mut state = AxumServiceState::from_app(app); + state.native = NativeSession::new(diagnostics.clone(), false).expect("test native session"); let server = async move { let (stream, remote_addr) = listener.accept().await.expect("accept client"); @@ -816,6 +1316,23 @@ mod tests { } assert!(response.is_empty()); assert_eq!(factory_calls.load(Ordering::SeqCst), 1); + let captured = records.lock().expect("native records"); + assert_eq!(captured.len(), 1); + assert_eq!(captured[0].event, "request_ingress"); + assert_eq!( + captured[0].outcome, + NativeRequestOutcome::Ingress(IngressDisposition::Aborted) + ); + assert_eq!( + captured[0].failure, + Some(NativeRequestFailure::PropagatedError { + kind: "uri_too_long" + }) + ); + assert!(captured[0].status.is_none()); + assert!(captured[0].bytes_written.is_none()); + assert!(captured[0].body_kind.is_none()); + assert_eq!(diagnostics.snapshot().active_requests, 0); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] @@ -1535,6 +2052,76 @@ mod tests { assert_eq!(&*body, b"injected"); } + #[test] + fn explicit_registries_win_in_both_setter_orders() { + use crate::config_store::AxumConfigStore; + use crate::config_store_limits::ConfigStoreLimits; + use crate::key_value_store::PersistentKvStore; + use crate::secret_store::EnvSecretStore; + + let temp = tempfile::tempdir().expect("database directory"); + let config = ConfigStoreHandle::new(Arc::new( + AxumConfigStore::from_map([], ConfigStoreLimits::default()).expect("config store"), + )); + let kv = KvHandle::new(Arc::new( + PersistentKvStore::new(temp.path().join("registry.redb")).expect("KV store"), + )); + let secrets = SecretHandle::new(Arc::new(EnvSecretStore::new())); + let with_handles = |service: AxumServiceState| { + service + .with_config_store_handle(config.clone()) + .with_kv_handle(kv.clone()) + .with_secret_handle(secrets.clone()) + }; + let with_registries = |service: AxumServiceState| { + service + .with_config_registry(ConfigRegistry::single_id( + "application".to_owned(), + ConfigStoreBinding { + handle: config.clone(), + default_key: "settings".to_owned(), + }, + )) + .with_kv_registry(KvRegistry::single_id("sessions".to_owned(), kv.clone())) + .with_secret_registry(SecretRegistry::single_id( + "runtime".to_owned(), + BoundSecretStore::new(secrets.clone(), "vault".to_owned()), + )) + }; + for registry_first in [true, false] { + let unwired = AxumServiceState::new(RouterService::builder().build()); + let service = if registry_first { + with_handles(with_registries(unwired)) + } else { + with_registries(with_handles(unwired)) + }; + let (config_option, kv_option, secret_option) = service.resolved_store_registries(); + let config_registry = config_option.expect("config registry"); + let kv_registry = kv_option.expect("KV registry"); + let secret_registry = secret_option.expect("secret registry"); + assert_eq!(config_registry.default_id(), "application"); + assert_eq!( + config_registry + .default_ref() + .expect("config binding") + .default_key, + "settings" + ); + assert_eq!(kv_registry.default_id(), "sessions"); + assert_eq!(secret_registry.default_id(), "runtime"); + assert_eq!( + secret_registry + .default_ref() + .expect("secret binding") + .store_name(), + "vault" + ); + assert!(config_registry.named("default").is_none()); + assert!(kv_registry.named("default").is_none()); + assert!(secret_registry.named("default").is_none()); + } + } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn kv_registry_wins_over_bare_handle_when_both_wired() { // Documents the precedence rule baked into the dispatcher: @@ -1880,6 +2467,1463 @@ mod tests { assert_eq!(body_at(&service, "/lookup/missing").await, "present=false"); } + mod managed_native { + use std::future::pending; + use std::net::{IpAddr, SocketAddr}; + + use http_body::Body as _; + use serde_json::{Value, json}; + + use crate::context::AxumRequestContext; + use crate::diagnostics::{ + NativeDiagnosticsSnapshot, NativeRequestFailure, NativeRequestObserver, + NativeRequestOutcome, NativeRequestRecord, + }; + use crate::proxy::{ForwardingHeaderFamily, ForwardingResult, Scheme}; + use edgezero_core::extractor::ForwardedHost; + use edgezero_core::http::{Authority, Uri}; + use edgezero_core::ingress::{CheckedEffectiveHost, CheckedHostSource, IngressHead}; + + use super::*; + + #[derive(Clone, Default)] + struct NativeProbe { + handle: NativeDiagnosticsHandle, + records: Arc>>, + snapshots: Arc>>, + events: Arc>>, + panics: bool, + } + + impl NativeRequestObserver for NativeProbe { + fn observe(&self, record: &NativeRequestRecord) { + self.records + .lock() + .expect("native records") + .push(record.clone()); + self.snapshots + .lock() + .expect("native snapshots") + .push(self.handle.snapshot()); + self.events.lock().expect("hook events").push("native"); + assert!(!self.panics, "native observer fixture panic"); + } + } + + struct OrderedAppObserver { + events: Arc>>, + reports: Arc>>, + } + + impl ResponseEgressObserver for OrderedAppObserver { + fn complete(&self, report: &ResponseEgressReport) { + self.events.lock().expect("hook events").push("observer"); + self.reports + .lock() + .expect("app reports") + .push(report.clone()); + } + } + + #[derive(Clone)] + struct PendingCalls(Arc); + + struct MetadataMiddleware(Arc>>); + + #[async_trait::async_trait(?Send)] + impl Middleware for MetadataMiddleware { + async fn handle( + &self, + ctx: RequestContext, + next: Next<'_>, + ) -> Result { + *self.0.lock().expect("middleware facts") = Some(context_facts(&ctx)); + next.run(ctx).await + } + } + + fn observed_service( + app: App, + policy: TrustedProxyPolicy, + probe: &NativeProbe, + ) -> AxumServiceState { + let handle = probe.handle.clone().with_request_observer(probe.clone()); + AxumServiceState::with_transport( + app, + AxumOutboundClient::try_transport().expect("fixture transport"), + ) + .with_native( + NativeSession::new(handle, false).expect("fixture identity"), + policy, + NativeGeo::default(), + ) + } + + fn tracked_connection(service: &mut AxumServiceState) -> ConnectionGuard { + let (activity, guard) = service.connection_activity(); + service.activity = Some(activity); + guard + } + + fn source_name(source: CheckedHostSource) -> &'static str { + match source { + CheckedHostSource::Direct => "direct", + CheckedHostSource::TrustedForwarded => "trusted_forwarded", + CheckedHostSource::TrustedXForwarded => "trusted_x_forwarded", + CheckedHostSource::Unavailable => "unavailable", + _ => "unknown", + } + } + + fn metadata_facts( + metadata: &NativeIngressMetadata, + headers: &HeaderMap, + uri: &Uri, + ) -> Value { + let summary = metadata.received_normalized_head(); + json!({ + "request_id": metadata.request_id().to_string(), + "peer": metadata.direct_peer().map(|peer| peer.to_string()), + "client": metadata.effective_client().map(|client| client.to_string()), + "host": metadata.effective_host().authority(), + "scheme": metadata.effective_scheme().as_str(), + "client_source": source_name(metadata.client_source()), + "host_source": source_name(metadata.host_source()), + "scheme_source": source_name(metadata.scheme_source()), + "forwarding_result": metadata.forwarding_result().as_str(), + "host_header": headers.get("host").and_then(|host| host.to_str().ok()), + "uri": uri.to_string(), + "raw_forwarding_absent": headers.keys().all(|name| name.as_str() != "forwarded" && !name.as_str().starts_with("x-forwarded-")), + "normalized": {"target_bytes": summary.target_bytes(), "header_bytes": summary.header_bytes(), "header_count": summary.header_count()}, + }) + } + + fn context_facts(ctx: &RequestContext) -> Value { + metadata_facts( + ctx.extensions() + .get::() + .expect("managed metadata"), + ctx.headers(), + ctx.uri(), + ) + } + + #[action] + async fn metadata_handler( + ctx: RequestContext, + ForwardedHost(host): ForwardedHost, + ) -> Result { + let peer = ctx + .extensions() + .get::() + .and_then(|native| native.remote_addr); + Ok(CoreResponse::new(Body::from( + json!({ + "facts": context_facts(&ctx), "forwarded_host": host, + "direct_context_peer": peer.map(|address| address.to_string()), + }) + .to_string(), + ))) + } + + #[action] + async fn mutate_metadata_handler(mut ctx: RequestContext) -> Result { + let original = ctx + .extensions_mut() + .remove::() + .expect("original metadata"); + ctx.extensions_mut() + .insert(CheckedEffectiveHost::from_ingress( + Some( + "application-mutated.example" + .parse::() + .expect("authority"), + ), + CheckedHostSource::Direct, + )); + Ok(original.request_id().to_string()) + } + + #[action] + async fn ordinary_status_handler(ctx: RequestContext) -> Result { + let status = if ctx.uri().path().ends_with("/413") { + StatusCode::PAYLOAD_TOO_LARGE + } else { + StatusCode::SERVICE_UNAVAILABLE + }; + Ok(response_builder() + .status(status) + .body(Body::from("application status")) + .expect("status response")) + } + + #[action] + async fn success_handler() -> Result<&'static str, EdgeError> { + Ok("ok") + } + + #[action] + async fn internal_error_handler() -> Result<&'static str, EdgeError> { + Err(EdgeError::internal(anyhow::anyhow!( + "PRIVATE_SERVICE_ERROR_SENTINEL" + ))) + } + + #[action] + async fn upstream_error_handler() -> Result<&'static str, EdgeError> { + Err(EdgeError::bad_gateway("PRIVATE_UPSTREAM_SENTINEL")) + } + + #[action] + async fn terminal_stream_handler(ctx: RequestContext) -> Result { + let source_failure = ctx.uri().path().ends_with("/source"); + let mut sent_prefix = false; + let stream = poll_fn(move |_context| { + if !sent_prefix { + sent_prefix = true; + return Poll::Ready(Some(Ok::<_, io::Error>(Bytes::from_static( + b"terminal-prefix", + )))); + } + if source_failure { + Poll::Ready(Some(Err(io::Error::other("PRIVATE_SOURCE_SENTINEL")))) + } else { + Poll::Pending + } + }); + Ok(CoreResponse::new(Body::from_external_stream(stream))) + } + + #[action] + async fn bad_framing_handler() -> Result { + Ok(response_builder() + .header("content-length", "1") + .body(Body::from("mismatched")) + .expect("framing response")) + } + + #[action] + async fn handler_not_found() -> Result<&'static str, EdgeError> { + Err(EdgeError::not_found("APPLICATION_CHOSEN_SENTINEL")) + } + + #[action] + async fn read_body_handler(ctx: RequestContext) -> Result { + ctx.extensions() + .get::() + .expect("handler calls") + .0 + .fetch_add(1, Ordering::SeqCst); + Ok(ctx.body_bytes(1024).await?.len().to_string()) + } + + #[action] + async fn pending_handler(ctx: RequestContext) -> Result<&'static str, EdgeError> { + ctx.extensions() + .get::() + .expect("handler calls") + .0 + .fetch_add(1, Ordering::SeqCst); + pending().await + } + + fn assert_one_released(probe: &NativeProbe) -> NativeRequestRecord { + let records = probe.records.lock().expect("native records"); + assert_eq!(records.len(), 1); + assert_eq!( + probe.snapshots.lock().expect("native snapshots")[0].active_requests, + 0 + ); + assert_eq!(probe.handle.snapshot().active_requests, 0); + records[0].clone() + } + + #[tokio::test] + #[expect( + clippy::too_many_lines, + reason = "The seven-case trust matrix keeps forged inputs, one managed dispatch, and all consumer-witness assertions together so each boundary proof is visible" + )] + async fn owned_peer_checked_metadata_precedes_all_managed_consumers() { + let trusted: SocketAddr = "192.0.2.2:4000".parse().expect("trusted peer"); + let untrusted: SocketAddr = "192.0.2.250:5000".parse().expect("untrusted peer"); + for ( + family, + peer, + enabled, + forwarded, + expected_host, + expected_scheme, + expected_client, + expected_result, + expected_source, + ) in [ + ( + ForwardingHeaderFamily::Forwarded, + Some(trusted), + false, + "for=203.0.113.7;host=public.example;proto=https", + "direct.example", + Scheme::Http, + Some(trusted.ip()), + ForwardingResult::UntrustedPeer, + CheckedHostSource::Direct, + ), + ( + ForwardingHeaderFamily::Forwarded, + Some(trusted), + true, + "for=203.0.113.7;host=public.example;proto=https", + "public.example", + Scheme::Https, + Some("203.0.113.7".parse::().expect("client")), + ForwardingResult::Accepted, + CheckedHostSource::TrustedForwarded, + ), + ( + ForwardingHeaderFamily::Forwarded, + None, + true, + "for=203.0.113.7;host=public.example;proto=https", + "direct.example", + Scheme::Http, + None, + ForwardingResult::UntrustedPeer, + CheckedHostSource::Direct, + ), + ( + ForwardingHeaderFamily::Forwarded, + Some(untrusted), + true, + "for=203.0.113.7;host=public.example;proto=https", + "direct.example", + Scheme::Http, + Some(untrusted.ip()), + ForwardingResult::UntrustedPeer, + CheckedHostSource::Direct, + ), + ( + ForwardingHeaderFamily::XForwarded, + Some(trusted), + true, + "for=203.0.113.7;host=public.example;proto=https", + "x-public.example", + Scheme::Https, + Some("203.0.113.9".parse::().expect("client")), + ForwardingResult::Accepted, + CheckedHostSource::TrustedXForwarded, + ), + ( + ForwardingHeaderFamily::Forwarded, + Some(trusted), + true, + "for=bad-address;host=public.example;proto=https", + "direct.example", + Scheme::Http, + Some(trusted.ip()), + ForwardingResult::Malformed, + CheckedHostSource::Direct, + ), + ( + ForwardingHeaderFamily::Forwarded, + Some(trusted), + true, + "for=unknown;host=public.example;proto=https", + "public.example", + Scheme::Https, + Some(trusted.ip()), + ForwardingResult::UnresolvedChain, + CheckedHostSource::TrustedForwarded, + ), + ] { + let probe = NativeProbe::default(); + let admission = Arc::new(Mutex::new(None)); + let middleware = Arc::new(Mutex::new(None)); + let captured_admission = Arc::clone(&admission); + let admission_handle = probe.handle.clone(); + let router = RouterService::builder() + .get("/native/{id}", metadata_handler) + .middleware(MetadataMiddleware(Arc::clone(&middleware))) + .build(); + let mut app = App::new(router); + app.set_ingress_admission_policy(move |head| { + assert!(matches!( + head.head_accounting(), + IngressHeadAccounting::HostManaged + )); + assert!(matches!(head.framing(), IngressFraming::HostManaged)); + assert_eq!(admission_handle.snapshot().active_requests, 1); + let metadata = head + .extension::() + .expect("admission metadata"); + assert_eq!( + head.extension::() + .expect("checked host"), + metadata.effective_host() + ); + *captured_admission.lock().expect("admission facts") = + Some(metadata_facts(metadata, head.headers(), head.target())); + AdmissionDecision::Admit { + completion: ResponseEgressCompletion::empty(), + grant: IngressGrant::empty(), + read_deadline: head.read_deadline_after(Duration::from_secs(30)), + } + }); + let policy = if enabled { + TrustedProxyPolicy::new(family, ["192.0.2.2"]).expect("policy") + } else { + TrustedProxyPolicy::no_trust() + }; + let mut service = observed_service(app, policy, &probe); + let owned_connection = tracked_connection(&mut service); + let incoming = Request::builder() + .uri("https://direct.example/native/SECRET-PATH?token=SECRET-QUERY") + .header("host", "direct.example") + .header("forwarded", forwarded) + .header("x-forwarded-for", "203.0.113.9") + .header("x-forwarded-host", "x-public.example") + .header("x-forwarded-proto", "https") + .header("X-FoRwArDeD-Private-Secret", "SECRET-HEADER") + .header("x-request-id", "SECRET-VISITOR-ID") + .body(AxumBody::empty()) + .expect("request"); + let (mut parts, body) = incoming.into_parts(); + let received = validate_normalized_ingress_parts_with_summary( + &parts, + IngressHeadLimits::default(), + ) + .expect("original summary"); + let foreign = NativeSession::new(NativeDiagnosticsHandle::new(), false) + .expect("foreign session"); + let foreign_guard = foreign + .request( + &Method::GET, + MonotonicInstant::now(), + MonotonicClock::default(), + None, + ) + .expect("foreign request"); + let foreign_id = foreign_guard.request_id(); + let claimed = normalize( + &TrustedProxyPolicy::new(ForwardingHeaderFamily::Forwarded, ["192.0.2.2"]) + .expect("forged policy"), + Some(trusted), + &parts, + ); + parts + .extensions + .insert(NativeIngressMetadata::new(foreign_id, claimed, received)); + parts.extensions.insert(CheckedEffectiveHost::from_ingress( + Some("FORGED-HOST.example".parse().expect("forged authority")), + CheckedHostSource::TrustedForwarded, + )); + parts + .extensions + .insert(ConnectInfo(if peer == Some(trusted) { + untrusted + } else { + trusted + })); + parts.extensions.insert(AxumRequestContext { + remote_addr: Some(if peer == Some(trusted) { + untrusted + } else { + trusted + }), + }); + drop(foreign_guard); + let response = service + .dispatch_request( + Request::from_parts(parts, body), + &EgressConnection::default(), + peer, + ) + .await + .expect("managed response"); + assert!(!response.headers().contains_key("x-request-id")); + assert!(!response.headers().contains_key("request-id")); + assert_eq!(probe.handle.snapshot().active_requests, 1); + let result: Value = serde_json::from_slice( + &to_bytes(response.into_body(), 8192) + .await + .expect("metadata body"), + ) + .expect("metadata JSON"); + let facts = &result["facts"]; + assert_eq!( + facts, + admission + .lock() + .expect("admission facts") + .as_ref() + .expect("observed admission") + ); + assert_eq!( + facts, + middleware + .lock() + .expect("middleware facts") + .as_ref() + .expect("observed middleware") + ); + assert_eq!( + facts["peer"], + json!(peer.map(|address| address.to_string())) + ); + assert_eq!( + facts["client"], + json!(expected_client.map(|address| address.to_string())) + ); + assert_eq!(facts["host"], expected_host); + assert_eq!(facts["scheme"], expected_scheme.as_str()); + assert_eq!(facts["host_source"], source_name(expected_source)); + assert_eq!(facts["scheme_source"], source_name(expected_source)); + let client_source = if peer.is_none() { + CheckedHostSource::Unavailable + } else if expected_result == ForwardingResult::Accepted { + expected_source + } else { + CheckedHostSource::Direct + }; + assert_eq!(facts["client_source"], source_name(client_source)); + assert_eq!(facts["forwarding_result"], expected_result.as_str()); + assert_eq!(facts["host_header"], "direct.example"); + assert_eq!( + facts["uri"], + "https://direct.example/native/SECRET-PATH?token=SECRET-QUERY" + ); + assert_eq!(facts["raw_forwarding_absent"], true); + assert_eq!( + facts["normalized"], + json!({"target_bytes":received.target_bytes(), "header_bytes":received.header_bytes(), "header_count":received.header_count()}) + ); + assert_eq!(result["forwarded_host"], expected_host); + assert_eq!( + result["direct_context_peer"], + json!(peer.map(|address| address.to_string())) + ); + assert_ne!(facts["request_id"], foreign_id.to_string()); + assert_ne!(facts["request_id"], "SECRET-VISITOR-ID"); + let record = assert_one_released(&probe); + assert_eq!(facts["request_id"], record.request_id.to_string()); + assert_eq!(record.route, "/native/{id}"); + assert_eq!(record.forwarding_result, expected_result); + assert_eq!(record.status, Some(StatusCode::OK)); + assert_eq!( + record.level(), + if expected_result == ForwardingResult::Malformed { + log::Level::Warn + } else { + log::Level::Info + } + ); + assert_eq!(probe.handle.snapshot().idle_connections, 1); + drop(owned_connection); + assert_eq!( + probe.handle.snapshot(), + NativeDiagnosticsSnapshot::default() + ); + } + } + + #[tokio::test] + async fn original_forwarding_head_overages_reject_before_stripping_or_body_poll() { + for limits in [ + IngressHeadLimits::default() + .with_max_request_header_count(1) + .expect("field limit"), + IngressHeadLimits::default() + .with_max_request_header_bytes(32) + .expect("byte limit"), + ] { + let probe = NativeProbe::default(); + let body_polls = Arc::new(AtomicUsize::new(0)); + let source_drops = Arc::new(AtomicUsize::new(0)); + let source_guard = DropSignal(Arc::clone(&source_drops)); + let counted_polls = Arc::clone(&body_polls); + let body = AxumBody::from_stream(poll_fn(move |_context| { + let _retained = &source_guard; + counted_polls.fetch_add(1, Ordering::SeqCst); + Poll::>>::Pending + })); + let admission_calls = Arc::new(AtomicUsize::new(0)); + let counted_admission = Arc::clone(&admission_calls); + let app_reports = Arc::new(Mutex::new(Vec::new())); + let mut app = App::new( + RouterService::builder() + .post("/head", success_handler) + .build(), + ); + app.set_ingress_head_limits(limits); + app.set_response_egress_observer(RecordingEgressObserver(Arc::clone(&app_reports))); + app.set_ingress_admission_policy(move |_head| { + counted_admission.fetch_add(1, Ordering::SeqCst); + AdmissionDecision::Abort + }); + let service = observed_service(app, TrustedProxyPolicy::no_trust(), &probe); + let incoming = Request::builder() + .method("POST") + .uri("/head") + .header("host", "d.test") + .header( + "forwarded", + "for=203.0.113.8;host=PUBLIC-SECRET.example;proto=https", + ) + .header("x-forwarded-private", "SECRET-HEADER") + .body(body) + .expect("head request"); + let (mut parts, native_body) = incoming.into_parts(); + let saved = parts.headers.clone(); + strip_forwarding_headers(&mut parts.headers); + validate_normalized_ingress_parts_with_summary(&parts, limits) + .expect("stripped head alone would fit"); + parts.headers = saved; + let response = service + .dispatch_request( + Request::from_parts(parts, native_body), + &EgressConnection::default(), + None, + ) + .await + .expect("detached head response"); + assert_eq!( + response.status(), + StatusCode::REQUEST_HEADER_FIELDS_TOO_LARGE + ); + assert_eq!(body_polls.load(Ordering::SeqCst), 0); + assert_eq!(admission_calls.load(Ordering::SeqCst), 0); + assert_eq!(source_drops.load(Ordering::SeqCst), 1); + let rendered = to_bytes(response.into_body(), 1024) + .await + .expect("head error body"); + assert!(!String::from_utf8_lossy(&rendered).contains("SECRET")); + let record = assert_one_released(&probe); + assert_eq!( + record.failure, + Some(NativeRequestFailure::PropagatedError { + kind: "request_header_fields_too_large" + }) + ); + assert_eq!( + record.status, + Some(StatusCode::REQUEST_HEADER_FIELDS_TOO_LARGE) + ); + assert_eq!(record.forwarding_result, ForwardingResult::Absent); + assert!(app_reports.lock().expect("detached app reports").is_empty()); + } + } + + #[tokio::test] + async fn public_metadata_mutation_cannot_change_private_record_identity_or_add_response_id() + { + let probe = NativeProbe::default(); + let router = RouterService::builder() + .get("/mutate", mutate_metadata_handler) + .build(); + let service = + observed_service(App::new(router), TrustedProxyPolicy::no_trust(), &probe); + let request = Request::builder() + .uri("/mutate") + .header("x-request-id", "VISITOR-SENTINEL") + .body(AxumBody::empty()) + .expect("request"); + let response = service + .dispatch_request(request, &EgressConnection::default(), None) + .await + .expect("response"); + assert!(!response.headers().contains_key("x-request-id")); + assert!(!response.headers().contains_key("request-id")); + let identity = to_bytes(response.into_body(), 1024) + .await + .expect("identity body"); + let record = assert_one_released(&probe); + assert_eq!(identity.as_ref(), record.request_id.to_string().as_bytes()); + assert_eq!(identity.len(), 64); + assert_ne!(identity, "VISITOR-SENTINEL"); + } + + #[tokio::test] + #[expect( + clippy::too_many_lines, + reason = "The abort, drain and refusal matrix shares lifetime fixtures while checking their distinct no-response versus detached-completion contracts" + )] + async fn native_abort_drain_and_refusal_release_once_without_inventing_egress() { + for mode in ["abort", "drain", "refuse"] { + let probe = NativeProbe::default(); + let body_polls = Arc::new(AtomicUsize::new(0)); + let observed_polls = Arc::clone(&body_polls); + let source_drops = Arc::new(AtomicUsize::new(0)); + let source_guard = DropSignal(Arc::clone(&source_drops)); + let body = AxumBody::from_stream(poll_fn(move |_context| { + let _retained = &source_guard; + observed_polls.fetch_add(1, Ordering::SeqCst); + Poll::>>::Pending + })); + let app_reports = Arc::new(Mutex::new(Vec::new())); + let completion_calls = Arc::new(AtomicUsize::new(0)); + let counted_completion = Arc::clone(&completion_calls); + let admission_calls = Arc::new(AtomicUsize::new(0)); + let counted_admission = Arc::clone(&admission_calls); + let admission_handle = probe.handle.clone(); + let mut app = App::new( + RouterService::builder() + .post("/gate", success_handler) + .build(), + ); + app.set_response_egress_observer(RecordingEgressObserver(Arc::clone(&app_reports))); + app.set_ingress_admission_policy(move |_head| { + counted_admission.fetch_add(1, Ordering::SeqCst); + assert_eq!(admission_handle.snapshot().active_requests, 1); + if mode == "refuse" { + let completed = Arc::clone(&counted_completion); + AdmissionDecision::Refuse { + completion: ResponseEgressCompletion::new(move |_report| { + completed.fetch_add(1, Ordering::SeqCst); + }), + response: response_builder() + .status(StatusCode::SERVICE_UNAVAILABLE) + .body(Body::from("admission refusal")) + .expect("refusal"), + } + } else { + AdmissionDecision::Abort + } + }); + let mut service = observed_service(app, TrustedProxyPolicy::no_trust(), &probe); + let (_phase_sender, phase) = watch::channel(if mode == "drain" { + LifecyclePhase::Draining + } else { + LifecyclePhase::Ready + }); + service.phase = Some(phase); + let owned_connection = tracked_connection(&mut service); + let request = Request::builder() + .method("POST") + .uri("/gate") + .body(body) + .expect("request"); + let outcome = service + .dispatch_request(request, &EgressConnection::default(), None) + .await; + if mode == "refuse" { + let response = outcome.expect("refusal response"); + assert_eq!(probe.handle.snapshot().active_requests, 1); + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); + assert_eq!( + to_bytes(response.into_body(), 1024) + .await + .expect("refusal body"), + "admission refusal" + ); + let record = assert_one_released(&probe); + assert_eq!(record.status, Some(StatusCode::SERVICE_UNAVAILABLE)); + assert_eq!(record.failure, Some(NativeRequestFailure::AdmissionRefused)); + assert_eq!( + record.outcome, + NativeRequestOutcome::Egress(ResponseEgressOutcome::HostHandoff) + ); + assert_eq!(completion_calls.load(Ordering::SeqCst), 1); + // Refusal retains its completion, not the admitted-request observer. + assert!(app_reports.lock().expect("app reports").is_empty()); + } else { + let Err(_aborted) = outcome else { + panic!("no response expected on abort or drain"); + }; + let record = assert_one_released(&probe); + assert_eq!(record.event, "request_ingress"); + assert_eq!(record.status, None); + assert_eq!(record.bytes_written, None); + assert_eq!(record.body_kind, None); + assert_eq!( + record.outcome, + NativeRequestOutcome::Ingress(if mode == "drain" { + IngressDisposition::Draining + } else { + IngressDisposition::Aborted + }) + ); + assert_eq!(completion_calls.load(Ordering::SeqCst), 0); + assert!(app_reports.lock().expect("app reports").is_empty()); + } + assert_eq!( + admission_calls.load(Ordering::SeqCst), + usize::from(mode != "drain") + ); + assert_eq!(body_polls.load(Ordering::SeqCst), 0); + assert_eq!(source_drops.load(Ordering::SeqCst), 1); + assert_eq!(probe.handle.snapshot().idle_connections, 1); + drop(owned_connection); + assert_eq!( + probe.handle.snapshot(), + NativeDiagnosticsSnapshot::default() + ); + } + } + + #[tokio::test] + #[expect( + clippy::too_many_lines, + reason = "Three cancellation stages retain one fixture and its before-drop and after-drop assertions for native, body, grant and completion ownership" + )] + async fn cancelled_body_handler_and_fallback_futures_release_native_and_app_resources() { + for stage in ["body", "handler", "fallback"] { + let probe = NativeProbe::default(); + let body_polls = Arc::new(AtomicUsize::new(0)); + let observed_polls = Arc::clone(&body_polls); + let source_drops = Arc::new(AtomicUsize::new(0)); + let source_guard = DropSignal(Arc::clone(&source_drops)); + let body_handle = probe.handle.clone(); + let body = AxumBody::from_stream(poll_fn(move |_context| { + let _retained = &source_guard; + assert_eq!(body_handle.snapshot().active_requests, 1); + observed_polls.fetch_add(1, Ordering::SeqCst); + Poll::>>::Pending + })); + let handler_calls = Arc::new(AtomicUsize::new(0)); + let router = match stage { + "body" => RouterService::builder() + .post("/pending", read_body_handler) + .build(), + "handler" => RouterService::builder() + .post("/pending", pending_handler) + .build(), + _ => RouterService::builder().build(), + }; + let app_reports = Arc::new(Mutex::new(Vec::new())); + let completion_calls = Arc::new(AtomicUsize::new(0)); + let retained_drops = Arc::new(AtomicUsize::new(0)); + let grant_drops = Arc::new(AtomicUsize::new(0)); + let captured_completions = Arc::clone(&completion_calls); + let captured_retained = Arc::clone(&retained_drops); + let captured_grants = Arc::clone(&grant_drops); + let mut app = App::new(router); + app.set_response_egress_observer(RecordingEgressObserver(Arc::clone(&app_reports))); + app.set_ingress_admission_policy(move |head| { + let calls = Arc::clone(&captured_completions); + let retained = DropSignal(Arc::clone(&captured_retained)); + let completion = ResponseEgressCompletion::new(move |_report| { + let _resource = retained; + calls.fetch_add(1, Ordering::SeqCst); + }); + let grant = IngressGrant::new(DropSignal(Arc::clone(&captured_grants))); + if stage == "fallback" { + AdmissionDecision::ReadBodyBeforeFallback { + completion, + grant, + max_body_bytes: 1024, + read_deadline: head.read_deadline_after(Duration::from_secs(30)), + on_exceeded: BufferedIngressResponse::text(StatusCode::OK, "overflow"), + on_timeout: BufferedIngressResponse::text(StatusCode::OK, "timeout"), + } + } else { + AdmissionDecision::Admit { + completion, + grant, + read_deadline: head.read_deadline_after(Duration::from_secs(30)), + } + } + }); + let mut service = observed_service(app, TrustedProxyPolicy::no_trust(), &probe); + let owned_connection = tracked_connection(&mut service); + let mut request = Request::builder() + .method("POST") + .uri("/pending") + .body(body) + .expect("request"); + request + .extensions_mut() + .insert(PendingCalls(Arc::clone(&handler_calls))); + let connection = EgressConnection::default(); + let mut response = Box::pin(service.dispatch_request(request, &connection, None)); + assert!(matches!( + futures_util::poll!(response.as_mut()), + Poll::Pending + )); + assert_eq!( + probe.handle.snapshot(), + NativeDiagnosticsSnapshot { + active_requests: 1, + active_connections: 1, + idle_connections: 0 + } + ); + assert_eq!(source_drops.load(Ordering::SeqCst), 0); + assert_eq!(retained_drops.load(Ordering::SeqCst), 0); + assert_eq!(grant_drops.load(Ordering::SeqCst), 0); + assert_eq!( + handler_calls.load(Ordering::SeqCst), + usize::from(stage != "fallback") + ); + assert_eq!(body_polls.load(Ordering::SeqCst) > 0, stage != "handler"); + drop(response); + assert_eq!(source_drops.load(Ordering::SeqCst), 1); + assert_eq!(retained_drops.load(Ordering::SeqCst), 1); + assert_eq!(grant_drops.load(Ordering::SeqCst), 1); + assert_eq!(completion_calls.load(Ordering::SeqCst), 0); + assert!(app_reports.lock().expect("app reports").is_empty()); + let record = assert_one_released(&probe); + assert_eq!( + record.outcome, + NativeRequestOutcome::Ingress(IngressDisposition::Abandoned) + ); + assert_eq!(record.status, None); + assert_eq!(record.bytes_written, None); + assert_eq!(probe.handle.snapshot().idle_connections, 1); + drop(owned_connection); + assert_eq!( + probe.handle.snapshot(), + NativeDiagnosticsSnapshot::default() + ); + } + } + + #[tokio::test] + async fn prepared_response_drop_is_one_failing_egress_record_with_original_status() { + let probe = NativeProbe::default(); + let app_reports = Arc::new(Mutex::new(Vec::new())); + let mut app = App::new( + RouterService::builder() + .get("/drop", success_handler) + .build(), + ); + app.set_response_egress_observer(RecordingEgressObserver(Arc::clone(&app_reports))); + let mut service = observed_service(app, TrustedProxyPolicy::no_trust(), &probe); + let owned_connection = tracked_connection(&mut service); + let response = service + .dispatch_request( + Request::builder() + .uri("/drop") + .body(AxumBody::empty()) + .expect("request"), + &EgressConnection::default(), + None, + ) + .await + .expect("response"); + assert_eq!(probe.handle.snapshot().active_requests, 1); + drop(response); + let record = assert_one_released(&probe); + assert_eq!(record.event, "request_terminal"); + assert_eq!(record.status, Some(StatusCode::OK)); + assert_eq!( + record.outcome, + NativeRequestOutcome::Egress(ResponseEgressOutcome::TransportError) + ); + assert_eq!(record.level(), log::Level::Warn); + assert_eq!(app_reports.lock().expect("app reports").len(), 1); + drop(owned_connection); + assert_eq!( + probe.handle.snapshot(), + NativeDiagnosticsSnapshot::default() + ); + } + + #[tokio::test] + async fn managed_native_observer_panic_preserves_release_and_application_observer() { + for abort in [false, true] { + let probe = NativeProbe { + panics: true, + ..NativeProbe::default() + }; + let app_reports = Arc::new(Mutex::new(Vec::new())); + let mut app = App::new( + RouterService::builder() + .get("/observer", success_handler) + .build(), + ); + app.set_response_egress_observer(RecordingEgressObserver(Arc::clone(&app_reports))); + if abort { + app.set_ingress_admission_policy(|_head| AdmissionDecision::Abort); + } + let service = observed_service(app, TrustedProxyPolicy::no_trust(), &probe); + let request = Request::builder() + .uri("/observer") + .body(AxumBody::empty()) + .expect("request"); + let outcome = service + .dispatch_request(request, &EgressConnection::default(), None) + .await; + if abort { + let Err(_aborted) = outcome else { + panic!("abort produced a response"); + }; + assert_eq!(assert_one_released(&probe).status, None); + assert!(app_reports.lock().expect("app reports").is_empty()); + } else { + let response = outcome.expect("response despite observer panic"); + assert_eq!( + to_bytes(response.into_body(), 1024) + .await + .expect("body despite observer panic"), + "ok" + ); + assert_eq!(assert_one_released(&probe).status, Some(StatusCode::OK)); + assert_eq!(app_reports.lock().expect("app reports").len(), 1); + } + } + } + + #[tokio::test] + async fn configured_fallback_200_has_owner_failure_but_deliberate_413_and_503_do_not() { + for timed_out in [false, true] { + let probe = NativeProbe::default(); + let mut app = App::new(RouterService::builder().build()); + app.set_ingress_admission_policy(move |head| { + AdmissionDecision::ReadBodyBeforeFallback { + completion: ResponseEgressCompletion::empty(), + grant: IngressGrant::empty(), + max_body_bytes: 1, + read_deadline: if timed_out { + Deadline::at_instant(head.request_start()) + } else { + head.read_deadline_after(Duration::from_secs(30)) + }, + on_exceeded: BufferedIngressResponse::text( + StatusCode::OK, + "configured overflow", + ), + on_timeout: BufferedIngressResponse::text( + StatusCode::OK, + "configured timeout", + ), + } + }); + let service = observed_service(app, TrustedProxyPolicy::no_trust(), &probe); + let request = Request::builder() + .method("POST") + .uri("/missing") + .body(AxumBody::from("ab")) + .expect("fallback request"); + let response = service + .dispatch_request(request, &EgressConnection::default(), None) + .await + .expect("configured response"); + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + to_bytes(response.into_body(), 1024) + .await + .expect("configured body"), + if timed_out { + "configured timeout" + } else { + "configured overflow" + } + ); + let record = assert_one_released(&probe); + assert_eq!( + record.failure, + Some(if timed_out { + NativeRequestFailure::FallbackReadTimedOut + } else { + NativeRequestFailure::FallbackBodyExceeded + }) + ); + assert_eq!(record.status, Some(StatusCode::OK)); + assert_eq!(record.level(), log::Level::Warn); + } + for (path, status) in [ + ("/ordinary/413", StatusCode::PAYLOAD_TOO_LARGE), + ("/ordinary/503", StatusCode::SERVICE_UNAVAILABLE), + ] { + let probe = NativeProbe::default(); + let service = observed_service( + App::new( + RouterService::builder() + .get("/ordinary/{status}", ordinary_status_handler) + .build(), + ), + TrustedProxyPolicy::no_trust(), + &probe, + ); + let request = Request::builder() + .uri(path) + .body(AxumBody::empty()) + .expect("ordinary request"); + let response = service + .dispatch_request(request, &EgressConnection::default(), None) + .await + .expect("ordinary response"); + assert_eq!(response.status(), status); + to_bytes(response.into_body(), 1024) + .await + .expect("ordinary body"); + let record = assert_one_released(&probe); + assert_eq!(record.failure, None); + assert_eq!(record.status, Some(status)); + assert_eq!(record.level(), log::Level::Info); + } + } + + // #407 JSON-ceiling coverage is excluded from this narrower ingress reconciliation. + + #[tokio::test] + #[expect( + clippy::too_many_lines, + reason = "Framework routing, handler errors and custom rendering are paired negative controls that keep privacy and provenance assertions in one matrix" + )] + async fn native_routing_privacy_preserves_router_provenance_allow_and_application_control() + { + for (path, method, status, framework, kind, public_sentinel) in [ + ( + "/missing/FRAMEWORK_PATH_SENTINEL", + "GET", + StatusCode::NOT_FOUND, + true, + "not_found", + false, + ), + ( + "/known", + "PRIVATE-METHOD-SENTINEL", + StatusCode::METHOD_NOT_ALLOWED, + true, + "method_not_allowed", + false, + ), + ( + "/handler404", + "GET", + StatusCode::NOT_FOUND, + false, + "not_found", + true, + ), + ( + "/internal", + "GET", + StatusCode::INTERNAL_SERVER_ERROR, + false, + "internal", + false, + ), + ( + "/upstream", + "GET", + StatusCode::BAD_GATEWAY, + false, + "bad_gateway", + false, + ), + ] { + let probe = NativeProbe::default(); + let router = RouterService::builder() + .get("/known", success_handler) + .get("/handler404", handler_not_found) + .get("/internal", internal_error_handler) + .get("/upstream", upstream_error_handler) + .build(); + let service = + observed_service(App::new(router), TrustedProxyPolicy::no_trust(), &probe); + let request = Request::builder() + .method(method) + .uri(path) + .body(AxumBody::empty()) + .expect("error request"); + let response = service + .dispatch_request(request, &EgressConnection::default(), None) + .await + .expect("error response"); + assert_eq!(response.status(), status); + if status == StatusCode::METHOD_NOT_ALLOWED { + assert_eq!(response.headers().get("allow").expect("Allow"), "GET"); + } + let body = to_bytes(response.into_body(), 2048) + .await + .expect("error body"); + assert_eq!( + String::from_utf8_lossy(&body).contains("SENTINEL"), + public_sentinel + ); + let record = assert_one_released(&probe); + assert_eq!( + record.failure, + Some(if framework { + NativeRequestFailure::FrameworkRouting { kind } + } else { + NativeRequestFailure::PropagatedError { kind } + }) + ); + assert_eq!( + record.level(), + if framework { + log::Level::Info + } else { + log::Level::Warn + } + ); + assert_eq!(record.method, if method == "GET" { "GET" } else { "other" }); + } + let probe = NativeProbe::default(); + let mut app = App::new(RouterService::builder().build()); + app.set_error_response_renderer(|error| { + let chosen_body = error.to_string(); + assert!(chosen_body.contains("CUSTOM_ROUTE_SENTINEL")); + CoreResponse::new(Body::from(chosen_body)) + }); + let service = observed_service(app, TrustedProxyPolicy::no_trust(), &probe); + let request = Request::builder() + .uri("/CUSTOM_ROUTE_SENTINEL") + .body(AxumBody::empty()) + .expect("custom request"); + let response = service + .dispatch_request(request, &EgressConnection::default(), None) + .await + .expect("custom response"); + assert_eq!(response.status(), StatusCode::NOT_FOUND); + assert!( + String::from_utf8_lossy( + &to_bytes(response.into_body(), 1024) + .await + .expect("custom body") + ) + .contains("CUSTOM_ROUTE_SENTINEL") + ); + assert_eq!( + assert_one_released(&probe).failure, + Some(NativeRequestFailure::FrameworkRouting { kind: "not_found" }) + ); + } + + #[tokio::test] + async fn managed_source_and_deadline_failures_keep_selected_200_prefix_accounting_and_one_record() + { + for failure in [ + ResponseEgressOutcome::SourceError, + ResponseEgressOutcome::DeadlineExceeded, + ] { + let probe = NativeProbe::default(); + let app_reports = Arc::new(Mutex::new(Vec::new())); + let start = MonotonicInstant::now(); + let now = Arc::new(Mutex::new(start)); + let captured_clock = Arc::clone(&now); + let mut app = App::new( + RouterService::builder() + .get("/stream/{failure}", terminal_stream_handler) + .build(), + ); + app.set_monotonic_clock(MonotonicClock::new(move || { + *captured_clock.lock().expect("clock") + })); + app.set_response_egress_observer(RecordingEgressObserver(Arc::clone(&app_reports))); + app.set_response_egress_policy(|_head, egress_started_at| ResponseEgressPolicy { + write_deadline: Deadline::at_instant( + egress_started_at + .checked_add(Duration::from_secs(1)) + .expect("write deadline"), + ), + }); + let mut service = observed_service(app, TrustedProxyPolicy::no_trust(), &probe); + let owned_connection = tracked_connection(&mut service); + let path = if failure == ResponseEgressOutcome::SourceError { + "/stream/source" + } else { + "/stream/deadline" + }; + let request = Request::builder() + .uri(path) + .body(AxumBody::empty()) + .expect("stream request"); + let response = service + .dispatch_request(request, &EgressConnection::default(), None) + .await + .expect("stream response"); + assert_eq!(response.status(), StatusCode::OK); + let mut body = response.into_body(); + let prefix = poll_future(|context| Pin::new(&mut body).poll_frame(context)) + .await + .expect("prefix frame") + .expect("prefix data") + .into_data() + .expect("data frame"); + assert_eq!(prefix, "terminal-prefix"); + assert_eq!(probe.handle.snapshot().active_requests, 1); + assert!(probe.records.lock().expect("native records").is_empty()); + if failure == ResponseEgressOutcome::DeadlineExceeded { + *now.lock().expect("clock") = start + .checked_add(Duration::from_secs(1)) + .expect("expired instant"); + } + let terminal_error = poll_future(|context| Pin::new(&mut body).poll_frame(context)) + .await + .expect("terminal frame") + .expect_err("terminal failure"); + assert!(!terminal_error.to_string().contains("SENTINEL")); + // Hyper stops after a body error; an ended body need not be repolled. + assert!(body.is_end_stream()); + drop(body); + let record = assert_one_released(&probe); + assert_eq!(record.status, Some(StatusCode::OK)); + assert_eq!(record.outcome, NativeRequestOutcome::Egress(failure)); + assert_eq!(record.bytes_written, Some(15)); + assert_eq!(record.fallback_disposition, None); + assert_eq!(record.route, "/stream/{failure}"); + assert_eq!(record.level(), log::Level::Warn); + assert_eq!(app_reports.lock().expect("app reports").len(), 1); + assert_eq!(app_reports.lock().expect("app reports")[0].outcome, failure); + drop(owned_connection); + assert_eq!( + probe.handle.snapshot(), + NativeDiagnosticsSnapshot::default() + ); + } + } + + #[tokio::test] + async fn managed_precommit_framing_fallback_records_selected_500_and_original_cause() { + let probe = NativeProbe::default(); + let service = observed_service( + App::new( + RouterService::builder() + .get("/framing", bad_framing_handler) + .build(), + ), + TrustedProxyPolicy::no_trust(), + &probe, + ); + let request = Request::builder() + .uri("/framing") + .body(AxumBody::empty()) + .expect("request"); + let response = service + .dispatch_request(request, &EgressConnection::default(), None) + .await + .expect("fallback response"); + assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR); + let body = to_bytes(response.into_body(), 1024) + .await + .expect("fallback body"); + assert!(!String::from_utf8_lossy(&body).contains("mismatched")); + let record = assert_one_released(&probe); + assert_eq!(record.status, Some(StatusCode::INTERNAL_SERVER_ERROR)); + assert_eq!( + record.outcome, + NativeRequestOutcome::Egress(ResponseEgressOutcome::ConversionError) + ); + assert_eq!( + record.body_kind, + Some(edgezero_core::ResponseEgressBodyKind::Fallback) + ); + assert_eq!( + record.fallback_disposition, + Some(ResponseEgressFallbackDisposition::Completed) + ); + } + + #[tokio::test] + async fn managed_hooks_keep_app_native_observer_order_and_authoritative_total_duration() { + let probe = NativeProbe::default(); + let start = MonotonicInstant::now(); + let now = Arc::new(Mutex::new(start)); + let source = Arc::clone(&now); + let admission_now = Arc::clone(&now); + let completion_now = Arc::clone(&now); + let completion_events = Arc::clone(&probe.events); + let reports = Arc::new(Mutex::new(Vec::new())); + let mut app = App::new( + RouterService::builder() + .get("/timed", success_handler) + .build(), + ); + app.set_monotonic_clock(MonotonicClock::new(move || *source.lock().expect("clock"))); + app.set_ingress_admission_policy(move |head: &IngressHead| { + assert_eq!(head.request_start(), start); + *admission_now.lock().expect("clock") = start + .checked_add(Duration::from_secs(3)) + .expect("pre-egress instant"); + let events = Arc::clone(&completion_events); + let observed_clock = Arc::clone(&completion_now); + AdmissionDecision::Admit { + completion: ResponseEgressCompletion::new(move |_report| { + events.lock().expect("hook events").push("app"); + *observed_clock.lock().expect("clock") = start + .checked_add(Duration::from_secs(80)) + .expect("callback instant"); + }), + grant: IngressGrant::empty(), + read_deadline: head.read_deadline_after(Duration::from_secs(30)), + } + }); + app.set_response_egress_observer(OrderedAppObserver { + events: Arc::clone(&probe.events), + reports: Arc::clone(&reports), + }); + let mut service = observed_service(app, TrustedProxyPolicy::no_trust(), &probe); + let owned_connection = tracked_connection(&mut service); + let response = service + .dispatch_request( + Request::builder() + .uri("/timed") + .body(AxumBody::empty()) + .expect("request"), + &EgressConnection::default(), + None, + ) + .await + .expect("response"); + assert_eq!(probe.handle.snapshot().active_requests, 1); + *now.lock().expect("clock") = start + .checked_add(Duration::from_secs(8)) + .expect("terminal instant"); + to_bytes(response.into_body(), 1024) + .await + .expect("response body"); + let record = assert_one_released(&probe); + assert_eq!(record.duration, Some(Duration::from_secs(8))); + assert_eq!( + reports.lock().expect("app reports")[0].elapsed, + Duration::from_secs(5) + ); + assert_eq!( + probe.events.lock().expect("hook events").as_slice(), + ["app", "native", "observer"] + ); + assert_eq!(probe.handle.snapshot().idle_connections, 1); + drop(owned_connection); + assert_eq!( + probe.handle.snapshot(), + NativeDiagnosticsSnapshot::default() + ); + } + + #[tokio::test] + async fn managed_backwards_request_clock_is_safe_native_fault_without_fake_duration() { + let probe = NativeProbe::default(); + let now = MonotonicInstant::now(); + let request_start = now + .checked_add(Duration::from_secs(1)) + .expect("request start"); + let samples = Arc::new(AtomicUsize::new(0)); + let captured_samples = Arc::clone(&samples); + let mut app = App::new( + RouterService::builder() + .get("/backwards", success_handler) + .build(), + ); + app.set_monotonic_clock(MonotonicClock::new(move || { + if captured_samples.fetch_add(1, Ordering::SeqCst) == 0 { + request_start + } else { + now + } + })); + let service = observed_service(app, TrustedProxyPolicy::no_trust(), &probe); + let request = Request::builder() + .uri("/backwards") + .body(AxumBody::empty()) + .expect("request"); + let response = service + .dispatch_request(request, &EgressConnection::default(), None) + .await + .expect("response"); + to_bytes(response.into_body(), 1024) + .await + .expect("response body"); + let record = assert_one_released(&probe); + assert_eq!(record.status, Some(StatusCode::OK)); + assert_eq!(record.duration, None); + assert_eq!(record.level(), log::Level::Error); + } + } + /// Send a GET request through `service` and return the response body as a UTF-8 string. /// Lifted out of the registry-aware tests so each can stay flat (clippy /// `items_after_statements` rejects nested `async fn` definitions). diff --git a/crates/edgezero-adapter-axum/src/service_geo_reader_tests.rs b/crates/edgezero-adapter-axum/src/service_geo_reader_tests.rs new file mode 100644 index 000000000..262de44ed --- /dev/null +++ b/crates/edgezero-adapter-axum/src/service_geo_reader_tests.rs @@ -0,0 +1,185 @@ +//! Managed checked-IP dispatch retains the existing admission and terminal owner. + +use super::tests::to_bytes; +use super::*; +use crate::geo::{NativeGeoSelection, ReaderIpExpectation}; +use crate::proxy::ForwardingHeaderFamily; +use edgezero_core::Body; +use edgezero_core::context::RequestContext; +use edgezero_core::geo::{ + GeoInput, GeoIpSource, GeoProvider, GeoProviderId, GeoResolution, GeoStatus, + GeoUnavailableReason, +}; +use edgezero_core::http::Response as CoreResponse; +use edgezero_core::http::{HeaderValue, StatusCode, response_builder}; +use edgezero_core::ingress::{AdmissionDecision, IngressGrant}; +use edgezero_core::middleware::{Middleware, Next}; +use edgezero_core::response_egress::ResponseEgressCompletion; +use std::sync::Mutex; +use std::time::Duration; + +struct Counted(Arc>>); +impl GeoProvider for Counted { + fn resolve(&self, input: &GeoInput) -> GeoResolution { + self.0.lock().unwrap().push(input.clone()); + GeoResolution::NoMatch + } +} + +struct Mutate; +#[async_trait::async_trait(?Send)] +impl Middleware for Mutate { + async fn handle( + &self, + mut ctx: RequestContext, + next: Next<'_>, + ) -> Result { + let original = ctx.geo().clone(); + ctx.extensions_mut().insert(GeoOutcome::NoSelection); + ctx.extensions_mut().insert(GeoInput::ReaderIp { + address: "198.51.100.7".parse().unwrap(), + source: GeoIpSource::DirectPeer, + }); + ctx.headers_mut() + .insert("x-forwarded-for", HeaderValue::from_static("198.51.100.7")); + assert_eq!(ctx.geo(), &original); + next.run(ctx).await + } +} + +#[edgezero_core::action] +async fn observe(ctx: RequestContext) -> Result<&'static str, EdgeError> { + Ok(match ctx.geo() { + GeoOutcome::Selected { + status: GeoStatus::Unavailable(GeoUnavailableReason::NoMatch), + .. + } => "no_match", + GeoOutcome::Selected { + status: GeoStatus::Unavailable(GeoUnavailableReason::MissingReaderIp), + .. + } => "missing_reader", + GeoOutcome::NoSelection | GeoOutcome::Selected { .. } => "unexpected", + }) +} + +#[expect( + clippy::too_many_lines, + reason = "one counted service scenario proves admission and terminal call counts for both reader expectations" +)] +#[tokio::test(flavor = "current_thread")] +async fn geo_reader_managed_dispatch_resolves_once_after_admission() { + for expectation in [ + ReaderIpExpectation::PeerAllowed, + ReaderIpExpectation::ForwardedRequired, + ] { + let calls = Arc::new(Mutex::new(Vec::new())); + let geo = NativeGeo::prepare(&NativeGeoSelection::SuppliedReaderIp { + expectation, + id: GeoProviderId::new("counted-reader").unwrap(), + provider: Arc::new(Counted(Arc::clone(&calls))), + }) + .unwrap(); + let router = RouterService::builder() + .middleware(Mutate) + .get("/{mode}", observe) + .build(); + let mut app = App::new(router); + app.set_ingress_admission_policy(|head| match head.target().path() { + "/refuse" => AdmissionDecision::Refuse { + completion: ResponseEgressCompletion::empty(), + response: response_builder() + .status(StatusCode::SERVICE_UNAVAILABLE) + .body(Body::empty()) + .unwrap(), + }, + "/abort" => AdmissionDecision::Abort, + _ => AdmissionDecision::Admit { + completion: ResponseEgressCompletion::empty(), + grant: IngressGrant::empty(), + read_deadline: head.read_deadline_after(Duration::from_secs(1)), + }, + }); + let service = AxumServiceState::with_native_transport( + app, + AxumOutboundClient::try_transport().unwrap(), + NativeSession::new(NativeDiagnosticsHandle::new(), true).unwrap(), + TrustedProxyPolicy::new(ForwardingHeaderFamily::XForwarded, ["127.0.0.1/32"]).unwrap(), + geo, + ); + let peer: SocketAddr = "127.0.0.1:1234".parse().unwrap(); + for (actual_peer, forwarded, expected, invoked) in [ + (Some(peer), Some("203.0.113.9"), "no_match", true), + ( + Some(peer), + None, + if expectation == ReaderIpExpectation::PeerAllowed { + "no_match" + } else { + "missing_reader" + }, + expectation == ReaderIpExpectation::PeerAllowed, + ), + (None, Some("203.0.113.9"), "missing_reader", false), + ] { + let before = calls.lock().unwrap().len(); + let mut request = Request::builder() + .uri("/observe") + .body(AxumBody::empty()) + .unwrap(); + request.extensions_mut().insert(ConnectInfo(peer)); + request.extensions_mut().insert(GeoInput::ReaderIp { + address: "198.51.100.7".parse().unwrap(), + source: GeoIpSource::DirectPeer, + }); + if let Some(assertion) = forwarded { + request + .headers_mut() + .insert("x-forwarded-for", HeaderValue::from_static(assertion)); + } + let response = service + .dispatch_request(request, &EgressConnection::default(), actual_peer) + .await + .unwrap(); + assert_eq!(to_bytes(response.into_body(), 128).await.unwrap(), expected); + assert_eq!(calls.lock().unwrap().len(), before + usize::from(invoked)); + } + assert_eq!( + calls.lock().unwrap().first(), + Some(&GeoInput::ReaderIp { + address: "203.0.113.9".parse().unwrap(), + source: GeoIpSource::TrustedXForwarded + }) + ); + let before = calls.lock().unwrap().len(); + for path in ["/refuse", "/abort"] { + let request = Request::builder() + .uri(path) + .header("x-forwarded-for", "203.0.113.9") + .body(AxumBody::empty()) + .unwrap(); + let response = service + .dispatch_request(request, &EgressConnection::default(), Some(peer)) + .await; + if path == "/refuse" { + assert_eq!(response.unwrap().status(), StatusCode::SERVICE_UNAVAILABLE); + } else { + assert!(response.is_err()); + } + assert_eq!(calls.lock().unwrap().len(), before); + } + let (_phase, reader) = watch::channel(LifecyclePhase::Draining); + let draining = service.with_phase(reader); + let request = Request::builder() + .uri("/observe") + .header("x-forwarded-for", "203.0.113.9") + .body(AxumBody::empty()) + .unwrap(); + assert!( + draining + .dispatch_request(request, &EgressConnection::default(), Some(peer)) + .await + .is_err() + ); + assert_eq!(calls.lock().unwrap().len(), before); + } +} diff --git a/crates/edgezero-adapter-axum/src/templates/Cargo.toml.hbs b/crates/edgezero-adapter-axum/src/templates/Cargo.toml.hbs index 6823d9071..59728e4fa 100644 --- a/crates/edgezero-adapter-axum/src/templates/Cargo.toml.hbs +++ b/crates/edgezero-adapter-axum/src/templates/Cargo.toml.hbs @@ -24,3 +24,12 @@ log = { workspace = true } simple_logger = { workspace = true } tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } tracing = { workspace = true } + +[features] +default = [] +aws-appconfig-agent = ["edgezero-adapter-axum/aws-appconfig-agent"] +aws-secrets-manager = ["edgezero-adapter-axum/aws-secrets-manager"] +aws-stores = ["aws-appconfig-agent", "aws-secrets-manager"] +aws-dynamodb = ["edgezero-adapter-axum/aws-dynamodb"] +celld-config = ["edgezero-adapter-axum/celld-config"] +celld-kv = ["edgezero-adapter-axum/celld-kv"] diff --git a/crates/edgezero-adapter-axum/src/templates/src/main.rs.hbs b/crates/edgezero-adapter-axum/src/templates/src/main.rs.hbs index a73eb8769..19030546f 100644 --- a/crates/edgezero-adapter-axum/src/templates/src/main.rs.hbs +++ b/crates/edgezero-adapter-axum/src/templates/src/main.rs.hbs @@ -1,6 +1,6 @@ -use edgezero_adapter_axum::dev_server::run_app; +use edgezero_adapter_axum::dev_server::run_generated_app; fn main() -> anyhow::Result<()> { - run_app::<{{proj_core_mod}}::App>()?; + run_generated_app::<{{proj_core_mod}}::App>()?; Ok(()) } diff --git a/crates/edgezero-adapter-axum/tests/fixtures/geo_hosting.rs b/crates/edgezero-adapter-axum/tests/fixtures/geo_hosting.rs new file mode 100644 index 000000000..4877c44c6 --- /dev/null +++ b/crates/edgezero-adapter-axum/tests/fixtures/geo_hosting.rs @@ -0,0 +1,663 @@ +// Real-listener reader-IP scenarios. Included by production_hosting's tests so +// process ownership and root-free shutdown retain the existing fixture helpers. + +#[cfg(feature = "geo-maxmind")] +use std::path::{Path, PathBuf}; + +fn geo_reader_address(ipv6: bool) -> SocketAddr { + TcpListener::bind(if ipv6 { "[::1]:0" } else { "127.0.0.1:0" }) + .expect("reader listener reservation") + .local_addr() + .expect("reader listener address") +} + +fn geo_forwarding(family: &str, reader: &str) -> String { + match family { + "forwarded" if reader.contains(':') => format!("Forwarded: for=\"[{reader}]\"\r\n"), + "forwarded" => format!("Forwarded: for={reader}\r\n"), + "x-forwarded" => format!("X-Forwarded-For: {reader}\r\n"), + _ => panic!("unknown fixture forwarding family"), + } +} + +fn assert_reader_location(value: &Value, provider: &str, source: &str, reader_source: &str) { + assert_eq!(value["geo"]["status"], "located"); + assert_eq!(value["geo"]["provider"], provider); + assert_eq!(value["geo"]["country"], "US"); + assert_eq!(value["geo"]["subdivision"], "CA"); + assert_eq!(value["geo"]["source"], source); + assert_eq!( + value["geo"]["trust_basis"], + format!("CheckedReaderIp({reader_source})") + ); +} + +fn assert_geo_startup_rejected(server: &mut Server, cwd: &TempDir, category: &str) { + assert!(!server.wait().success(), "{}", server.log()); + let log = server.log(); + assert!(log.contains(category), "{log}"); + for forbidden in [ + "INITIALIZED", + "HOOK_ENTERED", + "ready", + "event=geo_database", + "PRIVATE_RECORD_SENTINEL", + ] { + assert!(!log.contains(forbidden), "{forbidden}: {log}"); + } + TcpStream::connect_timeout(&server.addr, Duration::from_millis(50)) + .expect_err("rejected preparation must not bind"); + assert_eq!(fs::read_dir(cwd.path()).expect("unrelated cwd").count(), 0); +} + +#[test] +#[cfg(not(feature = "geo-maxmind"))] +fn geo_hosting_rejects_unavailable_maxmind() { + let cwd = tempfile::tempdir().expect("unrelated cwd"); + let database = tempfile::tempdir().expect("separate database root"); + let path = database.path().join("PRIVATE_DATABASE_PATH.mmdb"); + let addr = address(); + let mut command = geo_command("geo-maxmind", &cwd, addr); + command.env("FIXTURE_GEO_DB", &path); + let mut server = Server::spawn(command, addr); + assert_geo_startup_rejected(&mut server, &cwd, "maxmind implementation unavailable"); + assert!(!server.log().contains(&path.display().to_string())); + assert!(!server.log().contains("PRIVATE_DATABASE_PATH")); + assert_eq!( + fs::read_dir(database.path()) + .expect("database root") + .count(), + 0 + ); +} + +#[test] +fn geo_hosting_supplied_reader_without_builtin() { + // This test deliberately has no built-in feature gate. The axum-only lane + // exercises the same programmatic composition as combined-feature builds. + for ipv6 in [false, true] { + let cwd = tempfile::tempdir().expect("unrelated cwd"); + let addr = geo_reader_address(ipv6); + let mut server = Server::spawn(geo_command("geo-supplied-reader", &cwd, addr), addr); + server.ready(); + let first = geo_response(&server, "CloudFront-Viewer-Country: GB\r\n"); + assert_reader_location(&first, "fixture-reader", "CheckedReaderIp", "DirectPeer"); + let second = geo_response(&server, ""); + assert_reader_location(&second, "fixture-reader", "CheckedReaderIp", "DirectPeer"); + assert_eq!( + second["resolutions"].as_u64(), + first["resolutions"].as_u64().map(|n| n + 1) + ); + finish_geo(&mut server, &cwd); + let log = server.log(); + let calls = log + .lines() + .filter(|line| line.starts_with("GEO_CALL ")) + .collect::>(); + let last = calls + .last() + .expect("admitted requests invoke supplied provider"); + let total: usize = last + .split_whitespace() + .nth(1) + .expect("count") + .parse() + .expect("numeric count"); + assert_eq!(calls.len(), total, "each lookup increments exactly once"); + for call in calls { + assert!(call.contains("[redacted]"), "{call}"); + assert!(!call.contains("127.0.0.1"), "{call}"); + assert!(!call.contains("::1"), "{call}"); + } + } +} + +#[expect(clippy::cognitive_complexity, reason = "one process-owned matrix checks both forwarding families, both socket families and terminal call counts")] +#[test] +fn geo_hosting_supplied_reader_checked_forwarding_and_call_counts() { + for family in ["forwarded", "x-forwarded"] { + for ipv6 in [false, true] { + let cwd = tempfile::tempdir().expect("unrelated cwd"); + let addr = geo_reader_address(ipv6); + let mut command = geo_command("geo-supplied-reader", &cwd, addr); + command + .env("FIXTURE_GEO_EXPECTATION", "forwarded-required") + .env("FIXTURE_GEO_PROXY_FAMILY", family); + let mut server = Server::spawn(command, addr); + server.ready(); + let absent = geo_response(&server, ""); + assert_eq!(absent["geo"]["status"], "missing_reader_ip"); + assert_eq!(absent["geo"]["provider"], "fixture-reader"); + assert_eq!(absent["resolutions"], 0_u64); + let source = if family == "forwarded" { + "TrustedForwarded" + } else { + "TrustedXForwarded" + }; + for (index, reader) in ["192.0.2.1", "2001:db8::1", "::ffff:192.0.2.1"] + .into_iter() + .enumerate() + { + // Missing forwarded host/scheme is intentionally a partial + // result. A checked client is sufficient for reader admission. + let value = geo_response(&server, &geo_forwarding(family, reader)); + assert_reader_location(&value, "fixture-reader", "CheckedReaderIp", source); + assert_eq!(value["resolutions"], index + 1); + } + let no_match = geo_response(&server, &geo_forwarding(family, "198.51.100.9")); + assert_eq!(no_match["geo"]["status"], "no_match"); + assert_eq!(no_match["geo"]["provider"], "fixture-reader"); + assert_eq!(no_match["resolutions"], 4_u64); + let malformed = if family == "forwarded" { + "Forwarded: for=invalid\r\n" + } else { + "X-Forwarded-For: invalid\r\n" + }; + let wrong_family = if family == "forwarded" { + "x-forwarded" + } else { + "forwarded" + }; + for headers in [ + malformed.to_owned(), + geo_forwarding(family, "127.0.0.1"), + geo_forwarding(wrong_family, "192.0.2.1"), + "CloudFront-Viewer-Country: US\r\nCloudFront-Viewer-Country-Region: CA\r\n" + .to_owned(), + ] { + let value = geo_response(&server, &headers); + assert_eq!(value["geo"]["status"], "missing_reader_ip"); + assert_eq!( + value["resolutions"], 4_u64, + "inadmissible readers never call provider" + ); + } + let rejected = raw_request( + addr, + &format!( + "GET /native-refuse HTTP/1.1\r\nHost: fixture\r\n{}Connection: close\r\n\r\n", + geo_forwarding(family, "192.0.2.1") + ), + ); + assert!(rejected.starts_with("HTTP/1.1 503"), "{rejected}"); + let mut aborted_socket = server.socket().expect("abort socket"); + write!( + aborted_socket, + "GET /native-abort HTTP/1.1\r\nHost: fixture\r\n{}Connection: close\r\n\r\n", + geo_forwarding(family, "192.0.2.1") + ) + .expect("abort request"); + let mut aborted = Vec::new(); + if let Err(error) = aborted_socket.read_to_end(&mut aborted) { + assert_eq!(error.kind(), io::ErrorKind::ConnectionReset); + } + assert!(aborted.is_empty(), "admission abort must not dispatch"); + let after = geo_response(&server, &geo_forwarding(family, "192.0.2.1")); + assert_eq!( + after["resolutions"], 5_u64, + "refused and aborted requests invoke no provider" + ); + server.wait_line("GEO_COMPLETED 5"); + finish_geo(&mut server, &cwd); + let log = server.log(); + let calls = log + .lines() + .filter(|line| line.starts_with("GEO_CALL ")) + .collect::>(); + assert_eq!(calls.len(), 5, "no finalization lookup"); + for call in calls { + assert!(call.contains("[redacted]"), "{call}"); + for reader in ["192.0.2.1", "2001:db8::1", "198.51.100.9"] { + assert!( + !call.contains(reader), + "reader input Debug must redact addresses: {call}" + ); + } + } + } + } +} + +#[test] +fn geo_hosting_reader_expectations_and_untrusted_forwarding() { + for family in ["forwarded", "x-forwarded"] { + for expectation in ["peer-allowed", "forwarded-required"] { + for untrusted in [false, true] { + let cwd = tempfile::tempdir().expect("unrelated cwd"); + let addr = address(); + let mut command = geo_command("geo-supplied-reader", &cwd, addr); + command + .env("FIXTURE_GEO_EXPECTATION", expectation) + .env("FIXTURE_GEO_PROXY_FAMILY", family); + if untrusted { + command.env("FIXTURE_GEO_PROXY_NETWORKS", "203.0.113.0/24"); + } + let mut server = Server::spawn(command, addr); + server.ready(); + let direct = geo_response(&server, ""); + if expectation == "peer-allowed" { + assert_reader_location( + &direct, + "fixture-reader", + "CheckedReaderIp", + "DirectPeer", + ); + } else { + assert_eq!(direct["geo"]["status"], "missing_reader_ip"); + assert_eq!(direct["resolutions"], 0_u64); + } + let forwarded = geo_response(&server, &geo_forwarding(family, "192.0.2.1")); + if untrusted { + if expectation == "peer-allowed" { + assert_reader_location( + &forwarded, + "fixture-reader", + "CheckedReaderIp", + "DirectPeer", + ); + } else { + assert_eq!(forwarded["geo"]["status"], "missing_reader_ip"); + assert_eq!(forwarded["resolutions"], 0_u64); + } + } else { + let source = if family == "forwarded" { + "TrustedForwarded" + } else { + "TrustedXForwarded" + }; + assert_reader_location(&forwarded, "fixture-reader", "CheckedReaderIp", source); + } + finish_geo(&mut server, &cwd); + } + } + } +} + +#[test] +fn geo_hosting_supplied_reader_drain_does_not_resolve_pipeline() { + let cwd = tempfile::tempdir().expect("unrelated cwd"); + let addr = address(); + let mut command = geo_command("geo-supplied-reader", &cwd, addr); + command + .env("FIXTURE_GEO_EXPECTATION", "forwarded-required") + .env("FIXTURE_GEO_PROXY_FAMILY", "forwarded"); + let mut server = Server::spawn(command, addr); + server.ready(); + let mut client = server.socket().expect("finite pipeline socket"); + client.write_all(b"GET /finite HTTP/1.1\r\nHost: fixture\r\nForwarded: for=192.0.2.1\r\n\r\nGET /geo HTTP/1.1\r\nHost: fixture\r\nForwarded: for=192.0.2.1\r\nConnection: close\r\n\r\n").expect("pipeline"); + server.wait_line("DISPATCH"); + assert!( + server + .request("/stop") + .expect("caller stop") + .ends_with("stopping") + ); + server.wait_line("DRAINING"); + let mut response = String::new(); + client + .read_to_string(&mut response) + .expect("drained finite response"); + assert!(response.contains("finite-complete"), "{response}"); + assert!( + !response.contains("immutable"), + "pipelined Geo must not dispatch" + ); + assert!(server.wait().success(), "{}", server.log()); + assert_eq!( + server + .log() + .lines() + .filter(|line| line.starts_with("GEO_CALL ")) + .count(), + 1 + ); + assert_eq!(fs::read_dir(cwd.path()).expect("unrelated cwd").count(), 0); +} + +#[cfg(feature = "geo-maxmind")] +fn geo_database_fixture(name: &str) -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/geoip") + .join(name) +} + +#[cfg(feature = "geo-maxmind")] +fn geo_maxmind_command(cwd: &TempDir, addr: SocketAddr, path: &Path) -> Command { + let mut command = geo_command("geo-maxmind", cwd, addr); + command.env("FIXTURE_GEO_DB", path); + command +} + +#[cfg(feature = "geo-maxmind")] +fn assert_geo_snapshot( + server: &Server, + fixture: &str, + database_type: &str, + epoch: u64, + private_path: &Path, +) { + let sums = fs::read_to_string(geo_database_fixture("SHA256SUMS")).expect("fixture checksums"); + let hash = sums + .lines() + .find_map(|line| { + let (hash, name) = line.split_once(" ")?; + (name == fixture).then_some(hash) + }) + .expect("snapshot checksum"); + let log = server.log(); + let snapshots = captured_json(&log, "FRAMEWORK_LOG ") + .into_iter() + .filter(|record| { + record["message"] + .as_str() + .is_some_and(|message| message.contains("event=geo_database")) + }) + .collect::>(); + assert_eq!(snapshots.len(), 1, "one loaded snapshot diagnostic: {log}"); + let message = snapshots[0]["message"].as_str().expect("snapshot message"); + assert!( + message.contains(&format!("database_type={database_type}")), + "{message}" + ); + assert!( + message.contains(&format!("build_epoch={epoch}")), + "{message}" + ); + assert!(message.contains(&format!("sha256={hash}")), "{message}"); + assert!(!message.contains("127.0.0.1")); + assert!(!message.contains("192.0.2")); + assert!( + !log.contains(&private_path.display().to_string()), + "database paths must stay redacted: {log}" + ); + assert!(!log.contains("PRIVATE_DATABASE_PATH"), "{log}"); +} + +#[test] +#[cfg(feature = "geo-maxmind")] +fn geo_hosting_maxmind_snapshot_and_restart() { + let cwd = tempfile::tempdir().expect("unrelated cwd"); + let database = tempfile::tempdir().expect("separate database root"); + let path = database.path().join("PRIVATE_DATABASE_PATH.mmdb"); + fs::copy(geo_database_fixture("city.mmdb"), &path).expect("initial City snapshot"); + fs::set_permissions(&path, fs::Permissions::from_mode(0o444)).expect("read-only database"); + fs::set_permissions(database.path(), fs::Permissions::from_mode(0o555)) + .expect("read-only database directory"); + let addr = address(); + let mut server = Server::spawn(geo_maxmind_command(&cwd, addr, &path), addr); + server.ready(); + let first = geo_response(&server, "CloudFront-Viewer-Country: GB\r\n"); + assert_reader_location(&first, "maxmind", "LocalDatabase", "DirectPeer"); + assert_geo_snapshot(&server, "city.mmdb", "GeoLite2-City", 1_793_750_400, &path); + assert_eq!( + fs::read_dir(database.path()).expect("no sidecars").count(), + 1 + ); + assert_eq!( + fs::metadata(&path) + .expect("source permissions") + .permissions() + .mode() + & 0o777, + 0o444 + ); + assert_eq!( + fs::read(&path).expect("unchanged source"), + fs::read(geo_database_fixture("city.mmdb")).expect("City bytes") + ); + fs::set_permissions(database.path(), fs::Permissions::from_mode(0o755)) + .expect("publish replacement"); + let staged = database.path().join("next.mmdb"); + fs::copy(geo_database_fixture("city-replacement.mmdb"), &staged) + .expect("stage complete replacement"); + fs::rename(&staged, &path).expect("publish replacement atomically"); + let changed = geo_response(&server, ""); + assert_eq!( + changed["geo"], first["geo"], + "active reader retains original bytes" + ); + fs::remove_file(&path).expect("delete loaded source"); + let deleted = geo_response(&server, ""); + assert_eq!( + deleted["geo"], first["geo"], + "active reader needs no filesystem access" + ); + assert_eq!( + fs::read_dir(database.path()) + .expect("no sidecars after deletion") + .count(), + 0 + ); + finish_geo(&mut server, &cwd); + assert_geo_snapshot(&server, "city.mmdb", "GeoLite2-City", 1_793_750_400, &path); + fs::copy(geo_database_fixture("city-replacement.mmdb"), &path) + .expect("publish restart snapshot"); + let restart_addr = address(); + let mut restarted = Server::spawn(geo_maxmind_command(&cwd, restart_addr, &path), restart_addr); + restarted.ready(); + let replacement = geo_response(&restarted, ""); + assert_eq!(replacement["geo"]["country"], "US"); + assert_eq!(replacement["geo"]["subdivision"], "NY"); + assert_eq!(replacement["geo"]["provider"], "maxmind"); + assert_geo_snapshot( + &restarted, + "city-replacement.mmdb", + "GeoLite2-City", + 1_793_836_800, + &path, + ); + finish_geo(&mut restarted, &cwd); + fs::write(&path, b"PRIVATE_RECORD_SENTINEL invalid replacement").expect("invalid replacement"); + let invalid_addr = address(); + let mut invalid = Server::spawn(geo_maxmind_command(&cwd, invalid_addr, &path), invalid_addr); + assert_geo_startup_rejected(&mut invalid, &cwd, "invalid database"); + assert!(!invalid.log().contains("PRIVATE_DATABASE_PATH")); + assert_eq!( + fs::read_dir(database.path()) + .expect("no replacement sidecars") + .count(), + 1 + ); +} + +#[test] +#[cfg(feature = "geo-maxmind")] +fn geo_hosting_maxmind_checked_inputs_and_record_outcomes() { + for family in ["forwarded", "x-forwarded"] { + for (fixture, reader, country, subdivision) in [ + ("city.mmdb", "192.0.2.1", "US", Some("CA")), + ("country.mmdb", "192.0.2.1", "US", None), + ("ipv6.mmdb", "2001:db8::1", "GB", Some("ENG")), + ] { + let cwd = tempfile::tempdir().expect("unrelated cwd"); + let database = tempfile::tempdir().expect("separate database root"); + let path = database.path().join("PRIVATE_DATABASE_PATH.mmdb"); + fs::copy(geo_database_fixture(fixture), &path).expect("reader database"); + let addr = address(); + let mut command = geo_maxmind_command(&cwd, addr, &path); + command + .env("FIXTURE_GEO_EXPECTATION", "forwarded-required") + .env("FIXTURE_GEO_PROXY_FAMILY", family); + let mut server = Server::spawn(command, addr); + server.ready(); + let absent = geo_response(&server, ""); + assert_eq!(absent["geo"]["status"], "missing_reader_ip"); + assert_eq!(absent["geo"]["provider"], "maxmind"); + let located = geo_response(&server, &geo_forwarding(family, reader)); + assert_eq!(located["geo"]["status"], "located"); + assert_eq!(located["geo"]["country"], country); + assert_eq!(located["geo"]["subdivision"].as_str(), subdivision); + assert_eq!(located["geo"]["source"], "LocalDatabase"); + let source = if family == "forwarded" { + "TrustedForwarded" + } else { + "TrustedXForwarded" + }; + assert_eq!( + located["geo"]["trust_basis"], + format!("CheckedReaderIp({source})") + ); + let no_match = geo_response(&server, &geo_forwarding(family, "198.51.100.9")); + assert_eq!(no_match["geo"]["status"], "no_match"); + assert_eq!(no_match["geo"]["provider"], "maxmind"); + if fixture == "city.mmdb" { + for (case_reader, expected, case_subdivision) in [ + ("192.0.2.2", "located", None), + ("192.0.2.3", "located", Some("01")), + ("192.0.2.4", "located", Some("ZZ")), + ("192.0.2.5", "located", None), + ("192.0.2.7", "no_match", None), + ("192.0.2.8", "invalid_assertion", None), + ("192.0.2.10", "invalid_assertion", None), + ("2001:db8::1", "no_match", None), + ("::ffff:192.0.2.1", "located", Some("CA")), + ] { + let value = geo_response(&server, &geo_forwarding(family, case_reader)); + assert_eq!(value["geo"]["status"], expected, "reader={case_reader}"); + assert_eq!(value["geo"]["subdivision"].as_str(), case_subdivision); + } + } + finish_geo(&mut server, &cwd); + } + } +} + +#[test] +#[cfg(feature = "geo-maxmind")] +fn geo_hosting_maxmind_reader_expectations_on_real_peers() { + for ipv6 in [false, true] { + for family in ["forwarded", "x-forwarded"] { + for expectation in ["peer-allowed", "forwarded-required"] { + let cwd = tempfile::tempdir().expect("unrelated cwd"); + let database = tempfile::tempdir().expect("separate database root"); + let path = database.path().join("PRIVATE_DATABASE_PATH.mmdb"); + fs::copy(geo_database_fixture("city.mmdb"), &path).expect("City database"); + let addr = geo_reader_address(ipv6); + let mut command = geo_maxmind_command(&cwd, addr, &path); + command + .env("FIXTURE_GEO_EXPECTATION", expectation) + .env("FIXTURE_GEO_PROXY_FAMILY", family); + let mut server = Server::spawn(command, addr); + server.ready(); + let direct = geo_response(&server, ""); + if expectation == "forwarded-required" { + assert_eq!(direct["geo"]["status"], "missing_reader_ip"); + } else if ipv6 { + assert_eq!( + direct["geo"]["status"], "no_match", + "IPv6 peer against IPv4-only database" + ); + } else { + assert_reader_location(&direct, "maxmind", "LocalDatabase", "DirectPeer"); + } + let forwarded = geo_response(&server, &geo_forwarding(family, "192.0.2.1")); + let source = if family == "forwarded" { + "TrustedForwarded" + } else { + "TrustedXForwarded" + }; + assert_reader_location(&forwarded, "maxmind", "LocalDatabase", source); + let malformed = geo_response( + &server, + if family == "forwarded" { + "Forwarded: for=invalid\r\n" + } else { + "X-Forwarded-For: invalid\r\n" + }, + ); + assert_eq!( + malformed["geo"], direct["geo"], + "malformed forwarding cannot invent a checked client" + ); + finish_geo(&mut server, &cwd); + } + } + } +} + +#[test] +#[cfg(feature = "geo-maxmind")] +fn geo_hosting_maxmind_preparation_failures_never_initialize_or_bind() { + for (fixture, category, byte_limit, visit_limit) in [ + ("missing", "database unavailable", None, None), + ("directory", "database unavailable", None, None), + ("empty", "invalid database", None, None), + ("invalid", "invalid database", None, None), + ("truncated", "invalid database", None, None), + ("unsupported.mmdb", "unsupported database", None, None), + ( + "bad-country-code.mmdb", + "incompatible database schema", + None, + None, + ), + ( + "bad-subdivision-code.mmdb", + "incompatible database schema", + None, + None, + ), + ("bad-offpath-pointer.mmdb", "invalid database", None, None), + ("bad-offpath-utf8.mmdb", "invalid database", None, None), + ("bad-metadata.mmdb", "invalid database", None, None), + ( + "expansion-limit.mmdb", + "preparation limit exceeded", + None, + None, + ), + ("city.mmdb", "preparation limit exceeded", Some(1_usize), None), + ("city.mmdb", "preparation limit exceeded", None, Some(1_usize)), + ( + "city.mmdb", + "invalid database preparation limits", + Some(0_usize), + None, + ), + ( + "city.mmdb", + "invalid database preparation limits", + None, + Some(0_usize), + ), + ] { + let cwd = tempfile::tempdir().expect("unrelated cwd"); + let database = tempfile::tempdir().expect("separate database root"); + let path = database.path().join("PRIVATE_DATABASE_PATH.mmdb"); + match fixture { + "missing" => {} + "directory" => fs::create_dir_all(&path).expect("nonregular database"), + "empty" => fs::write(&path, []).expect("empty database"), + "invalid" => fs::write(&path, b"PRIVATE_RECORD_SENTINEL").expect("invalid database"), + "truncated" => { + let mut bytes = fs::read(geo_database_fixture("city.mmdb")).expect("City bytes"); + bytes.truncate(bytes.len().div_ceil(2)); + fs::write(&path, bytes).expect("truncated database"); + } + name => { + fs::copy(geo_database_fixture(name), &path).expect("malformed or limited database"); + } + } + let addr = address(); + let mut command = geo_maxmind_command(&cwd, addr, &path); + if let Some(limit) = byte_limit { + command.env("FIXTURE_GEO_MAX_BYTES", limit.to_string()); + } + if let Some(limit) = visit_limit { + command.env("FIXTURE_GEO_MAX_NETWORKS", limit.to_string()); + } + let mut server = Server::spawn(command, addr); + assert_geo_startup_rejected(&mut server, &cwd, category); + let log = server.log(); + assert!(!log.contains(&path.display().to_string()), "{log}"); + assert!(!log.contains("PRIVATE_DATABASE_PATH"), "{log}"); + assert!(!log.contains("192.0.2.1"), "{log}"); + assert!(!log.contains("203.0.113.1"), "{log}"); + assert_eq!( + fs::read_dir(database.path()) + .expect("no preparation sidecars") + .count(), + usize::from(fixture != "missing") + ); + } +} diff --git a/crates/edgezero-adapter-axum/tests/fixtures/geoip/README.md b/crates/edgezero-adapter-axum/tests/fixtures/geoip/README.md new file mode 100644 index 000000000..fa4eaa5bd --- /dev/null +++ b/crates/edgezero-adapter-axum/tests/fixtures/geoip/README.md @@ -0,0 +1,26 @@ +# Synthetic GeoIP fixtures + +These files are original test data for EdgeZero, licensed under this repository's Apache-2.0 license. They contain no MaxMind, GeoLite, DB-IP or other production dataset content. Metadata names describe the schema under test and do not claim publisher authenticity. Addresses and locations are fabricated. Loopback US/CA and US/NY records exist solely for local hosting tests. + +Rebuild from the repository root with Node.js: + +```sh +node scripts/generate_geoip_fixtures.mjs +node scripts/generate_geoip_fixtures.mjs --check +``` + +The dependency-free generator writes deterministic MMDB version 2 files and `SHA256SUMS`. The fixed build epochs are synthetic. The default 24-bit tree, 28/32-bit variants, IPv4 and IPv6 records exercise reader format support, not data accuracy. No production download, account, credentials or network access is required. + +`city.mmdb` includes 127/8 as US/CA and a matrix at 192.0.2.1 through 192.0.2.20 covering case normalization, country-only records, numeric subdivision `01`, unrecognized `ZZ`, missing first code with a later code, empty arrays, empty records, unsupported/empty/malformed codes, subdivision without country, registered/represented-country non-substitution, duplicate-key first-value behavior and unrelated later-entry types. Other addresses have no record. `city-replacement.mmdb` changes loopback and 192.0.2.1 to US/NY and changes the epoch. `country.mmdb` deliberately gives subdivisions an incompatible geographic shape to prove Country ignores them. `ipv6.mmdb` contains 2001:db8::/32 as GB/ENG and ::c000:201/128 for IPv4 lookup 192.0.2.1 as US/CA. + +The `bad-*` files deliberately contain consumed type errors, off-path invalid pointers/UTF-8, nested non-string map keys, prohibited pointer chains, a tree cycle, a corrupt separator, or an unknown metadata pointer outside its section. `truncated.mmdb` is incomplete. `unsupported.mmdb` has an unsupported dataset type. `expansion-limit.mmdb` uses repeated shared arrays that pass memoized verification but exceed normal decoder expansion limits. `verification-limit.mmdb` and `metadata-verification-limit.mmdb` use overlapping valid string targets to exhaust the respective verifier work budgets. They are intentionally not usable databases. `all-values.mmdb` covers all scalar types and valid single pointers without imposing unrelated geographic schemas. + +## Validation boundary + +Preparation uses maxminddb 0.32.0 without mmap or unsafe string decoding. Its verifier checks metadata and reachable tree/data structure. An additional non-materializing Serde traversal follows all reachable data values through the normal decoder, requires string map keys and catches prohibited pointer chains even in unrelated fields. It uses the pinned reader's `deserialize_identifier` entry point recursively because that entry point checks pointer targets before falling back to normal value decoding. Direct `deserialize_any` would still accept pointer chains. This behavior is covered by regressions that first prove upstream verification accepts each malformed data fixture. Geographic paths are borrowed strings and are identical at startup and lookup. Geographic code semantics are intentionally checked only on lookup. + +Residual metadata limitation: `bad-metadata-key.mmdb` reproduces a known startup-acceptance failure, retained in the explicitly ignored `geo_maxmind_rejects_unused_metadata_nonstring_keys` regression. Running that test with `--ignored` currently fails. This blocks #438 acceptance. The pinned public API exposes typed metadata and `verify()`, but no raw metadata decode operation. Consequently unknown metadata fields retain the upstream verifier's acceptance of non-string nested map keys and pointer-to-pointer chains. Known metadata fields receive upstream typed decoding; malformed unknown metadata pointers and UTF-8 still fail verification. No claim of stricter full metadata conformance is made. Unreachable data bytes are outside the reachable-record contract. + +Normal decoding uses upstream finite value, payload and depth budgets per unique root. Shared graphs are expanded, not silently skipped. Network visits include no-data leaves and aliases before record-offset deduplication. A precheck reserves upstream's `3 * node_count + 1` tree allowance before verification. Bounds do not guarantee startup duration or a fixed RSS. No production-size GeoLite performance or memory measurement is established by these tiny fixtures. + +File acquisition is supported only on Unix, with read-only `O_NONBLOCK` open, descriptor regular-file validation and bounded actual bytes read. Other platforms reject selected MaxMind as an unavailable implementation before opening a source because equivalent special-file safety has not been proven. Nonblocking open does not impose a deadline on regular-file filesystem I/O. Snapshots retain owned bytes, not a source handle or mmap, and remain valid after source replacement/deletion. diff --git a/crates/edgezero-adapter-axum/tests/fixtures/geoip/SHA256SUMS b/crates/edgezero-adapter-axum/tests/fixtures/geoip/SHA256SUMS new file mode 100644 index 000000000..c6d8613a5 --- /dev/null +++ b/crates/edgezero-adapter-axum/tests/fixtures/geoip/SHA256SUMS @@ -0,0 +1,26 @@ +0b362c5d6db97a61b2509796331c5214238bc252fb676ec6a000f93a83d55684 city.mmdb +a9573925a4368b737a4f2d775234f8e5cd577af47de7803c0a1962dd2c3c2ae5 city-replacement.mmdb +89d9b81bbef523ca1a0f4c16659067264da6394d6bd292a9bd6ca409f90ae17f country.mmdb +7ada6d5baaf8a293981620732b9b6d15a889976f4fb45b9e13119574e2c48f49 ipv6.mmdb +bedb529c817853078dfeb76281974f0a7e40fb5bb2a2f9490da28005f11919b2 record-28.mmdb +8eadd10f167996b65ab582b7326ac6361a54c302351cab325a9f475e0dcd719e record-32.mmdb +53b973ef30fe31bb3c50bc8684f31d532197c3f333cfd774b31d63bd16486099 bad-country-parent.mmdb +e2d9f6a7774deafc59ff90fc951be37584bc9bfacbc23417fa56454f3b04dbe8 bad-country-code.mmdb +e08d35ffd9ee0f8a6a671f4306e6aaf2b64ca50a4e2d7d911c0a3be3da3a80ce bad-subdivisions-parent.mmdb +329ca8a0081bab2c813c29f8a22c5a2f00ec5d7ecf43b4eae83ed49a174e8ebc bad-first-subdivision.mmdb +fae49fa6bd8049e7403a978aa3fdc1702c366254e4eee54cf098a2eac60c6852 bad-subdivision-code.mmdb +b986cb11687beb3b82d3ff799fc2a2a7b761ecb294c11361a9792ca70b584fb4 bad-unrelated-key.mmdb +2fb92cad0e5a8b810f4cbcc22b9182b313b70206c1d3a582cd599b6a851ae6a9 bad-offpath-pointer.mmdb +0beee02808ad875b242ac3f5c17a9621871f31412353bb2bdf50f2a43263e73b bad-unrelated-chain.mmdb +7d0e396a96022e41648d1f906ff95ebced9ea0c2a3ea09401c12fcbc9c12b9e8 bad-root-chain.mmdb +a82449214ca752ccb8e54b1877c8b1bcd767cd6975f5fdb520eacd7f0c8834eb unsupported.mmdb +6d71d35b8b04c210de699dbc552d0c4808838c1713bc63fa3435672cec4bd32a bad-tree-cycle.mmdb +aa823a333b8eeb1a7234d0e98ce785ed95fdf004a5e4b72712c8d45694637d3f bad-separator.mmdb +8c7e06ca8cc3e797b8d649f169fd853411617f3d303c06b4bc2b62e6cd82de32 truncated.mmdb +8be95e3d68c462dcd2d52227205f99f0531e904179dcbbb1ff734f5a3a8b0217 bad-metadata.mmdb +7e0a2f9fb3a339c0103af51018c5158798909b759ddca8b5137a6c06bc95d433 bad-metadata-key.mmdb +18d7352606ad532e2bc3979ab3ab6bee0335cf62a01441e312e3a4bfc9705bc9 expansion-limit.mmdb +5f3816c54eb5a3e2886c2629317681e41330477069881206fed8348e76f2abe2 all-values.mmdb +d33136ae918dc67f67bbc4369015d0c13e7e9fd096afc986dac04cca28915b66 verification-limit.mmdb +1d531175e44bca0f0546c03b0fcdd9eb63ac7d53f4b17871dc91fd77fa77396c metadata-verification-limit.mmdb +3dffd178351e91a6a7d9238b211882a83e39c56ad468ae6382b740d7dcfbb749 bad-offpath-utf8.mmdb diff --git a/crates/edgezero-adapter-axum/tests/fixtures/geoip/all-values.mmdb b/crates/edgezero-adapter-axum/tests/fixtures/geoip/all-values.mmdb new file mode 100644 index 000000000..83d17d7c0 Binary files /dev/null and b/crates/edgezero-adapter-axum/tests/fixtures/geoip/all-values.mmdb differ diff --git a/crates/edgezero-adapter-axum/tests/fixtures/geoip/bad-country-code.mmdb b/crates/edgezero-adapter-axum/tests/fixtures/geoip/bad-country-code.mmdb new file mode 100644 index 000000000..5b438cc41 Binary files /dev/null and b/crates/edgezero-adapter-axum/tests/fixtures/geoip/bad-country-code.mmdb differ diff --git a/crates/edgezero-adapter-axum/tests/fixtures/geoip/bad-country-parent.mmdb b/crates/edgezero-adapter-axum/tests/fixtures/geoip/bad-country-parent.mmdb new file mode 100644 index 000000000..852d9b5da Binary files /dev/null and b/crates/edgezero-adapter-axum/tests/fixtures/geoip/bad-country-parent.mmdb differ diff --git a/crates/edgezero-adapter-axum/tests/fixtures/geoip/bad-first-subdivision.mmdb b/crates/edgezero-adapter-axum/tests/fixtures/geoip/bad-first-subdivision.mmdb new file mode 100644 index 000000000..4cf1295e3 Binary files /dev/null and b/crates/edgezero-adapter-axum/tests/fixtures/geoip/bad-first-subdivision.mmdb differ diff --git a/crates/edgezero-adapter-axum/tests/fixtures/geoip/bad-metadata-key.mmdb b/crates/edgezero-adapter-axum/tests/fixtures/geoip/bad-metadata-key.mmdb new file mode 100644 index 000000000..b49a73cb0 Binary files /dev/null and b/crates/edgezero-adapter-axum/tests/fixtures/geoip/bad-metadata-key.mmdb differ diff --git a/crates/edgezero-adapter-axum/tests/fixtures/geoip/bad-metadata.mmdb b/crates/edgezero-adapter-axum/tests/fixtures/geoip/bad-metadata.mmdb new file mode 100644 index 000000000..1bc490c36 Binary files /dev/null and b/crates/edgezero-adapter-axum/tests/fixtures/geoip/bad-metadata.mmdb differ diff --git a/crates/edgezero-adapter-axum/tests/fixtures/geoip/bad-offpath-pointer.mmdb b/crates/edgezero-adapter-axum/tests/fixtures/geoip/bad-offpath-pointer.mmdb new file mode 100644 index 000000000..c484ee379 Binary files /dev/null and b/crates/edgezero-adapter-axum/tests/fixtures/geoip/bad-offpath-pointer.mmdb differ diff --git a/crates/edgezero-adapter-axum/tests/fixtures/geoip/bad-offpath-utf8.mmdb b/crates/edgezero-adapter-axum/tests/fixtures/geoip/bad-offpath-utf8.mmdb new file mode 100644 index 000000000..ff263c67a Binary files /dev/null and b/crates/edgezero-adapter-axum/tests/fixtures/geoip/bad-offpath-utf8.mmdb differ diff --git a/crates/edgezero-adapter-axum/tests/fixtures/geoip/bad-root-chain.mmdb b/crates/edgezero-adapter-axum/tests/fixtures/geoip/bad-root-chain.mmdb new file mode 100644 index 000000000..b86be1c6f Binary files /dev/null and b/crates/edgezero-adapter-axum/tests/fixtures/geoip/bad-root-chain.mmdb differ diff --git a/crates/edgezero-adapter-axum/tests/fixtures/geoip/bad-separator.mmdb b/crates/edgezero-adapter-axum/tests/fixtures/geoip/bad-separator.mmdb new file mode 100644 index 000000000..f27f599c4 Binary files /dev/null and b/crates/edgezero-adapter-axum/tests/fixtures/geoip/bad-separator.mmdb differ diff --git a/crates/edgezero-adapter-axum/tests/fixtures/geoip/bad-subdivision-code.mmdb b/crates/edgezero-adapter-axum/tests/fixtures/geoip/bad-subdivision-code.mmdb new file mode 100644 index 000000000..a2ae91795 Binary files /dev/null and b/crates/edgezero-adapter-axum/tests/fixtures/geoip/bad-subdivision-code.mmdb differ diff --git a/crates/edgezero-adapter-axum/tests/fixtures/geoip/bad-subdivisions-parent.mmdb b/crates/edgezero-adapter-axum/tests/fixtures/geoip/bad-subdivisions-parent.mmdb new file mode 100644 index 000000000..87b311e07 Binary files /dev/null and b/crates/edgezero-adapter-axum/tests/fixtures/geoip/bad-subdivisions-parent.mmdb differ diff --git a/crates/edgezero-adapter-axum/tests/fixtures/geoip/bad-tree-cycle.mmdb b/crates/edgezero-adapter-axum/tests/fixtures/geoip/bad-tree-cycle.mmdb new file mode 100644 index 000000000..7d6e52b42 Binary files /dev/null and b/crates/edgezero-adapter-axum/tests/fixtures/geoip/bad-tree-cycle.mmdb differ diff --git a/crates/edgezero-adapter-axum/tests/fixtures/geoip/bad-unrelated-chain.mmdb b/crates/edgezero-adapter-axum/tests/fixtures/geoip/bad-unrelated-chain.mmdb new file mode 100644 index 000000000..4a3dc518b Binary files /dev/null and b/crates/edgezero-adapter-axum/tests/fixtures/geoip/bad-unrelated-chain.mmdb differ diff --git a/crates/edgezero-adapter-axum/tests/fixtures/geoip/bad-unrelated-key.mmdb b/crates/edgezero-adapter-axum/tests/fixtures/geoip/bad-unrelated-key.mmdb new file mode 100644 index 000000000..a494f2b66 Binary files /dev/null and b/crates/edgezero-adapter-axum/tests/fixtures/geoip/bad-unrelated-key.mmdb differ diff --git a/crates/edgezero-adapter-axum/tests/fixtures/geoip/city-replacement.mmdb b/crates/edgezero-adapter-axum/tests/fixtures/geoip/city-replacement.mmdb new file mode 100644 index 000000000..8421b7a8f Binary files /dev/null and b/crates/edgezero-adapter-axum/tests/fixtures/geoip/city-replacement.mmdb differ diff --git a/crates/edgezero-adapter-axum/tests/fixtures/geoip/city.mmdb b/crates/edgezero-adapter-axum/tests/fixtures/geoip/city.mmdb new file mode 100644 index 000000000..2e1065b7f Binary files /dev/null and b/crates/edgezero-adapter-axum/tests/fixtures/geoip/city.mmdb differ diff --git a/crates/edgezero-adapter-axum/tests/fixtures/geoip/country.mmdb b/crates/edgezero-adapter-axum/tests/fixtures/geoip/country.mmdb new file mode 100644 index 000000000..5b32057fc Binary files /dev/null and b/crates/edgezero-adapter-axum/tests/fixtures/geoip/country.mmdb differ diff --git a/crates/edgezero-adapter-axum/tests/fixtures/geoip/expansion-limit.mmdb b/crates/edgezero-adapter-axum/tests/fixtures/geoip/expansion-limit.mmdb new file mode 100644 index 000000000..4794ad2d3 Binary files /dev/null and b/crates/edgezero-adapter-axum/tests/fixtures/geoip/expansion-limit.mmdb differ diff --git a/crates/edgezero-adapter-axum/tests/fixtures/geoip/ipv6.mmdb b/crates/edgezero-adapter-axum/tests/fixtures/geoip/ipv6.mmdb new file mode 100644 index 000000000..59bae2960 Binary files /dev/null and b/crates/edgezero-adapter-axum/tests/fixtures/geoip/ipv6.mmdb differ diff --git a/crates/edgezero-adapter-axum/tests/fixtures/geoip/metadata-verification-limit.mmdb b/crates/edgezero-adapter-axum/tests/fixtures/geoip/metadata-verification-limit.mmdb new file mode 100644 index 000000000..f2cadfab5 Binary files /dev/null and b/crates/edgezero-adapter-axum/tests/fixtures/geoip/metadata-verification-limit.mmdb differ diff --git a/crates/edgezero-adapter-axum/tests/fixtures/geoip/record-28.mmdb b/crates/edgezero-adapter-axum/tests/fixtures/geoip/record-28.mmdb new file mode 100644 index 000000000..2a71780a5 Binary files /dev/null and b/crates/edgezero-adapter-axum/tests/fixtures/geoip/record-28.mmdb differ diff --git a/crates/edgezero-adapter-axum/tests/fixtures/geoip/record-32.mmdb b/crates/edgezero-adapter-axum/tests/fixtures/geoip/record-32.mmdb new file mode 100644 index 000000000..366413fce Binary files /dev/null and b/crates/edgezero-adapter-axum/tests/fixtures/geoip/record-32.mmdb differ diff --git a/crates/edgezero-adapter-axum/tests/fixtures/geoip/truncated.mmdb b/crates/edgezero-adapter-axum/tests/fixtures/geoip/truncated.mmdb new file mode 100644 index 000000000..4734c650f Binary files /dev/null and b/crates/edgezero-adapter-axum/tests/fixtures/geoip/truncated.mmdb differ diff --git a/crates/edgezero-adapter-axum/tests/fixtures/geoip/unsupported.mmdb b/crates/edgezero-adapter-axum/tests/fixtures/geoip/unsupported.mmdb new file mode 100644 index 000000000..6b2174730 Binary files /dev/null and b/crates/edgezero-adapter-axum/tests/fixtures/geoip/unsupported.mmdb differ diff --git a/crates/edgezero-adapter-axum/tests/fixtures/geoip/verification-limit.mmdb b/crates/edgezero-adapter-axum/tests/fixtures/geoip/verification-limit.mmdb new file mode 100644 index 000000000..c9e322e04 Binary files /dev/null and b/crates/edgezero-adapter-axum/tests/fixtures/geoip/verification-limit.mmdb differ diff --git a/crates/edgezero-adapter-axum/tests/fixtures/native_aws_host.rs b/crates/edgezero-adapter-axum/tests/fixtures/native_aws_host.rs new file mode 100644 index 000000000..178376e9c --- /dev/null +++ b/crates/edgezero-adapter-axum/tests/fixtures/native_aws_host.rs @@ -0,0 +1,109 @@ +//! Child executable used by `tests/native_aws_startup.rs`. + +use std::future; +use std::sync::Arc; + +use edgezero_adapter_axum::dev_server::{ + AxumRunOptions, LocalStartupFuture, PreparedStores, run_app_with_native_stores, +}; +use edgezero_adapter_axum::native_stores::NativeStoreOverrides; +use edgezero_core::app::{App, Hooks, StoreMetadata, StoresMetadata}; +use edgezero_core::context::RequestContext; +use edgezero_core::extractor::{AppConfig as TypedAppConfig, State}; +use edgezero_core::router::RouterService; +use edgezero_core::{EdgeError, action}; +use serde::Deserialize; +use validator::Validate; + +struct NativeAwsApp; + +#[derive(Deserialize, Validate, edgezero_core::AppConfig)] +struct FixtureConfig { + #[validate(length(min = 4_u64))] + greeting: String, + #[secret] + #[validate(length(min = 8_u64))] + token: String, +} + +#[derive(Clone)] +struct Snapshot { + greeting: String, + token: String, +} + +#[expect( + clippy::missing_trait_methods, + reason = "fixture retains unused Hooks defaults" +)] +impl Hooks for NativeAwsApp { + fn routes() -> RouterService { + RouterService::builder() + .get("/snapshot", snapshot) + .get("/local-secret", local_secret) + .build() + } + + fn stores() -> StoresMetadata { + StoresMetadata { + config: Some(StoreMetadata { + ids: &["app"], + default: "app", + }), + kv: None, + secrets: Some(StoreMetadata { + ids: &["vault", "local"], + default: "vault", + }), + } + } +} + +#[action] +async fn snapshot(State(snapshot): State>) -> Result { + Ok(format!("{}|{}", snapshot.greeting, snapshot.token)) +} + +#[action] +async fn local_secret(ctx: RequestContext) -> Result { + let local = ctx + .secret_store("local") + .ok_or_else(|| EdgeError::service_unavailable("local fixture store missing"))?; + Ok(local.require_str("LOCAL_FIXTURE_SECRET").await?) +} + +fn initialize<'startup>( + app: &'startup mut App, + stores: &'startup PreparedStores, +) -> LocalStartupFuture<'startup> { + Box::pin(async move { + let binding = stores + .config() + .and_then(|registry| registry.named_ref("app")) + .ok_or_else(|| EdgeError::service_unavailable("config fixture missing"))?; + let config = TypedAppConfig::::from_binding( + binding, + None, + stores.secrets(), + app.config_extraction_limits(), + app.monotonic_clock(), + ) + .await?; + app.insert_state(Arc::new(Snapshot { + greeting: config.greeting, + token: config.token, + })); + Ok(()) + }) +} + +fn main() -> anyhow::Result<()> { + let options = AxumRunOptions::from_env()?; + run_app_with_native_stores::( + options, + NativeStoreOverrides::default(), + future::pending::<()>(), + initialize, + )?; + Ok(()) +} diff --git a/crates/edgezero-adapter-axum/tests/fixtures/native_celld_host.rs b/crates/edgezero-adapter-axum/tests/fixtures/native_celld_host.rs new file mode 100644 index 000000000..e0b4f95e0 --- /dev/null +++ b/crates/edgezero-adapter-axum/tests/fixtures/native_celld_host.rs @@ -0,0 +1,130 @@ +//! Public-runner child for deterministic scoped HTTP startup tests, not a celld service. + +use std::env::var_os; +use std::fs::write; +use std::future; +use std::path::PathBuf; +use std::sync::Arc; +use std::time::Duration; + +use edgezero_adapter_axum::dev_server::{ + AxumRunOptions, LocalStartupFuture, PreparedStores, run_app_with_native_stores, +}; +use edgezero_adapter_axum::native_stores::{NativeStoreOverrides, NativeStorePreparationError}; +use edgezero_core::app::{App, Hooks, StoreMetadata, StoresMetadata}; +use edgezero_core::context::RequestContext; +use edgezero_core::extractor::{AppConfig as TypedAppConfig, State}; +use edgezero_core::router::RouterService; +use edgezero_core::{EdgeError, action}; +use serde::Deserialize; +use tokio::time::{sleep, timeout}; +use validator::Validate; + +struct NativeCelldApp; + +#[derive(Deserialize, Validate, edgezero_core::AppConfig)] +struct FixtureConfig { + #[validate(length(min = 4_u64))] + greeting: String, +} + +#[expect( + clippy::missing_trait_methods, + reason = "fixture retains unused Hooks defaults" +)] +impl Hooks for NativeCelldApp { + fn routes() -> RouterService { + RouterService::builder() + .get("/snapshot", snapshot) + .get("/kv", kv) + .build() + } + + fn stores() -> StoresMetadata { + StoresMetadata { + config: Some(StoreMetadata { + ids: &["app"], + default: "app", + }), + kv: Some(StoreMetadata { + ids: &["state"], + default: "state", + }), + secrets: None, + } + } +} + +#[action] +async fn snapshot(State(greeting): State>) -> Result { + Ok(greeting.to_string()) +} + +#[action] +async fn kv(ctx: RequestContext) -> Result { + let store = ctx + .kv_store("state") + .ok_or_else(|| EdgeError::service_unavailable("fixture KV missing"))?; + let bytes = store + .get_bytes("value") + .await? + .ok_or_else(|| EdgeError::service_unavailable("fixture value missing"))?; + String::from_utf8(bytes.to_vec()) + .map_err(|_error| EdgeError::internal(anyhow::anyhow!("fixture value malformed"))) +} + +fn initialize<'startup>( + app: &'startup mut App, + stores: &'startup PreparedStores, +) -> LocalStartupFuture<'startup> { + Box::pin(async move { + let binding = stores + .config() + .and_then(|registry| registry.named_ref("app")) + .ok_or_else(|| EdgeError::service_unavailable("fixture config missing"))?; + let config = TypedAppConfig::::from_binding( + binding, + None, + stores.secrets(), + app.config_extraction_limits(), + app.monotonic_clock(), + ) + .await?; + if let Some(marker) = var_os("FIXTURE_INITIALIZER_ENTERED") { + write(marker, b"entered").map_err(|_error| { + EdgeError::internal(anyhow::anyhow!("fixture marker unavailable")) + })?; + } + if let Some(path) = var_os("FIXTURE_INITIALIZER_GATE") { + let gate = PathBuf::from(path); + timeout(Duration::from_secs(8), async { + while !gate.is_file() { + sleep(Duration::from_millis(10)).await; + } + }) + .await + .map_err(|_error| { + EdgeError::internal(anyhow::anyhow!("fixture initializer deadline")) + })?; + } + app.insert_state(Arc::::from(config.greeting)); + Ok(()) + }) +} + +fn main() -> anyhow::Result<()> { + let mut overrides = NativeStoreOverrides::default(); + if let Some(marker) = var_os("FIXTURE_DEFERRED_MARKER") { + overrides.prepare_kv("state", async move { + write(marker, b"polled").map_err(|_error| NativeStorePreparationError::Unavailable)?; + Err(NativeStorePreparationError::Unavailable) + })?; + } + run_app_with_native_stores::( + AxumRunOptions::from_env()?, + overrides, + future::pending::<()>(), + initialize, + )?; + Ok(()) +} diff --git a/crates/edgezero-adapter-axum/tests/fixtures/native_proxy.rs b/crates/edgezero-adapter-axum/tests/fixtures/native_proxy.rs new file mode 100644 index 000000000..8d042cacd --- /dev/null +++ b/crates/edgezero-adapter-axum/tests/fixtures/native_proxy.rs @@ -0,0 +1,311 @@ +//! Test-only source-bound proxy and verified TLS ingress. No response is collected here. +#![expect( + clippy::arbitrary_source_item_ordering, + reason = "fixture ownership precedes bounded relay helpers" +)] + +use std::io::{self, ErrorKind}; +use std::net::{IpAddr, Ipv4Addr, SocketAddr, TcpListener as StdTcpListener}; +use std::sync::Arc; +use std::thread::{self, JoinHandle}; +use std::time::Duration; + +use rustls::pki_types::{CertificateDer, PrivateKeyDer, PrivatePkcs8KeyDer, ServerName}; +use rustls::{ClientConfig, RootCertStore, ServerConfig}; +use tokio::io::{ + AsyncRead, AsyncReadExt as _, AsyncWrite, AsyncWriteExt as _, copy_bidirectional_with_sizes, +}; +use tokio::net::{TcpListener, TcpSocket, TcpStream}; +use tokio::runtime::Builder; +use tokio::sync::oneshot; +use tokio::task::JoinSet; +use tokio::time::timeout; +use tokio_rustls::{TlsAcceptor, TlsConnector}; + +const MAX_HEAD: usize = 0x8000; +const RELAY_BUFFER: usize = 0x4000; + +#[derive(Clone, Copy)] +pub(crate) enum Assertions { + /// Keep deliberate malformed/gap inputs unchanged while preserving a real owned peer. + Preserve, + AppendForwarded { + host: &'static str, + scheme: &'static str, + }, + AppendXForwarded { + host: &'static str, + scheme: &'static str, + }, + ReplaceForwarded { + client: IpAddr, + host: &'static str, + scheme: &'static str, + }, +} + +pub(crate) struct LocalProxy { + addr: SocketAddr, + stop: Option>, + worker: Option>, +} + +impl LocalProxy { + #[must_use] + #[inline] + pub(crate) fn addr(&self) -> SocketAddr { + self.addr + } + + #[must_use] + #[inline] + pub(crate) fn start( + upstream: SocketAddr, + source: Ipv4Addr, + assertions: Assertions, + tls: bool, + ) -> Self { + let std_listener = StdTcpListener::bind((Ipv4Addr::LOCALHOST, 0)).expect("proxy listener"); + let addr = std_listener.local_addr().expect("proxy address"); + std_listener + .set_nonblocking(true) + .expect("proxy nonblocking"); + let (stop, mut stopped) = oneshot::channel(); + let worker = thread::spawn(move || { + let runtime = Builder::new_current_thread() + .enable_all() + .build() + .expect("proxy runtime"); + runtime.block_on(async move { + let listener = TcpListener::from_std(std_listener).expect("proxy adoption"); + let tls_acceptor = tls.then(|| TlsAcceptor::from(Arc::new(server_config()))); + let mut tasks = JoinSet::new(); + #[expect(clippy::integer_division_remainder_used, reason = "Tokio select macro branch arithmetic")] + loop { + tokio::select! { + () = async { let _closed = (&mut stopped).await; } => break, + _finished = tasks.join_next(), if !tasks.is_empty() => {}, + accepted = listener.accept(), if tasks.len() < 64 => { + let (socket, peer) = accepted.expect("proxy accept"); + let task_acceptor = tls_acceptor.clone(); + tasks.spawn(async move { + let operation = async { + if let Some(acceptor) = task_acceptor { + if let Ok(mut client) = acceptor.accept(socket).await { + let _relayed = relay(&mut client, peer, upstream, source, assertions).await; + } + } else { + let mut client = socket; + let _relayed = relay(&mut client, peer, upstream, source, assertions).await; + } + }; + let _bounded = timeout(Duration::from_secs(5), operation).await; + }); + } + } + } + tasks.abort_all(); + while tasks.join_next().await.is_some() {} + }); + }); + Self { + addr, + stop: Some(stop), + worker: Some(worker), + } + } +} + +impl Drop for LocalProxy { + fn drop(&mut self) { + if let Some(stop) = self.stop.take() { + let _sent = stop.send(()); + } + if let Some(worker) = self.worker.take() { + worker.join().expect("proxy worker"); + } + } +} + +fn invalid() -> io::Error { + io::Error::new(ErrorKind::InvalidData, "invalid bounded fixture head") +} + +async fn read_head( + client: &mut Client, +) -> io::Result<(String, Vec)> { + let mut received = Vec::with_capacity(1024); + let mut chunk = [0_u8; 512]; + loop { + if let Some(end) = received.windows(4).position(|bytes| bytes == b"\r\n\r\n") { + let body_at = end.checked_add(4).ok_or_else(invalid)?; + let head = String::from_utf8(received.get(..end).ok_or_else(invalid)?.to_vec()) + .map_err(|_error| invalid())?; + let body = received.get(body_at..).ok_or_else(invalid)?.to_vec(); + return Ok((head, body)); + } + let count = client.read(&mut chunk).await?; + if count == 0 { + return Err(io::Error::new( + ErrorKind::UnexpectedEof, + "fixture head incomplete", + )); + } + received + .len() + .checked_add(count) + .filter(|bytes| *bytes <= MAX_HEAD) + .ok_or_else(invalid)?; + received.extend_from_slice(chunk.get(..count).ok_or_else(invalid)?); + } +} + +fn rewrite(head: &str, peer: SocketAddr, assertions: Assertions) -> io::Result { + let mut lines = head.split("\r\n"); + let request = lines + .next() + .filter(|line| !line.is_empty()) + .ok_or_else(invalid)?; + let mut headers = Vec::new(); + for line in lines { + let (name, value) = line.split_once(':').ok_or_else(invalid)?; + headers.push((name.to_owned(), value.trim().to_owned())); + } + let predecessor = match peer.ip() { + IpAddr::V4(ip) => ip.to_string(), + IpAddr::V6(ip) => format!("\"[{ip}]\""), + }; + match assertions { + Assertions::Preserve => {} + Assertions::AppendForwarded { host, scheme } => { + let previous = headers + .iter() + .filter(|(name, _)| name.eq_ignore_ascii_case("forwarded")) + .map(|(_, value)| value.as_str()) + .collect::>() + .join(","); + headers.retain(|(name, _)| !name.eq_ignore_ascii_case("forwarded")); + let assertion = format!("for={predecessor};host=\"{host}\";proto={scheme}"); + headers.push(( + "Forwarded".into(), + if previous.is_empty() { + assertion + } else { + format!("{previous},{assertion}") + }, + )); + } + Assertions::AppendXForwarded { host, scheme } => { + let previous = headers + .iter() + .filter(|(name, _)| name.eq_ignore_ascii_case("x-forwarded-for")) + .map(|(_, value)| value.as_str()) + .collect::>() + .join(","); + headers.retain(|(name, _)| !name.to_ascii_lowercase().starts_with("x-forwarded-")); + let node = peer.ip().to_string(); + headers.push(( + "X-Forwarded-For".into(), + if previous.is_empty() { + node + } else { + format!("{previous},{node}") + }, + )); + headers.push(("X-Forwarded-Host".into(), host.into())); + headers.push(("X-Forwarded-Proto".into(), scheme.into())); + } + Assertions::ReplaceForwarded { + client, + host, + scheme, + } => { + headers.retain(|(name, _)| { + !name.eq_ignore_ascii_case("forwarded") + && !name.to_ascii_lowercase().starts_with("x-forwarded-") + }); + // The test gateway's client assertion is explicitly supplied by its trusted producer, + // never taken from the visitor's fields. This is not a provider-discovery mechanism. + let node = match client { + IpAddr::V4(ip) => ip.to_string(), + IpAddr::V6(ip) => format!("\"[{ip}]\""), + }; + headers.push(( + "Forwarded".into(), + format!("for={node};host=\"{host}\";proto={scheme}"), + )); + } + } + let mut rewritten = format!("{request}\r\n"); + for (name, value) in headers { + use std::fmt::Write as _; + write!(rewritten, "{name}: {value}\r\n").map_err(|_error| invalid())?; + } + rewritten.push_str("\r\n"); + if rewritten.len() > MAX_HEAD { + return Err(invalid()); + } + Ok(rewritten) +} + +async fn relay( + client: &mut Client, + peer: SocketAddr, + upstream: SocketAddr, + source: Ipv4Addr, + assertions: Assertions, +) -> io::Result<()> { + let (head, body_prefix) = read_head(client).await?; + let rewritten = rewrite(&head, peer, assertions)?; + let socket = TcpSocket::new_v4()?; + socket.bind(SocketAddr::new(IpAddr::V4(source), 0))?; + let mut origin = socket.connect(upstream).await?; + origin.write_all(rewritten.as_bytes()).await?; + origin.write_all(&body_prefix).await?; + // Finite, per-direction relay buffers. Never collect a response or body for proxying. + let _copied = + copy_bidirectional_with_sizes(client, &mut origin, RELAY_BUFFER, RELAY_BUFFER).await?; + Ok(()) +} + +fn server_config() -> ServerConfig { + let certificate = CertificateDer::from(include_bytes!("tls/server.der").to_vec()); + let key = PrivateKeyDer::Pkcs8(PrivatePkcs8KeyDer::from( + include_bytes!("tls/server-key.der").to_vec(), + )); + ServerConfig::builder() + .with_no_client_auth() + .with_single_cert(vec![certificate], key) + .expect("test certificate/key") +} + +#[inline] +pub(crate) fn verified_tls_request( + addr: SocketAddr, + request: &[u8], + server_name: &'static str, +) -> io::Result { + let runtime = Builder::new_current_thread().enable_all().build()?; + runtime.block_on(async move { + let mut roots = RootCertStore::empty(); + roots + .add(CertificateDer::from(include_bytes!("tls/ca.der").to_vec())) + .map_err(|_error| invalid())?; + let config = ClientConfig::builder() + .with_root_certificates(roots) + .with_no_client_auth(); + let connector = TlsConnector::from(Arc::new(config)); + let operation = async { + let socket = TcpStream::connect(addr).await?; + let name = ServerName::try_from(server_name).map_err(|_error| invalid())?; + let mut client = connector.connect(name, socket).await?; + client.write_all(request).await?; + let mut response = String::new(); + client.read_to_string(&mut response).await?; + Ok(response) + }; + timeout(Duration::from_secs(5), operation) + .await + .map_err(|_error| io::Error::new(ErrorKind::TimedOut, "TLS fixture timeout"))? + }) +} diff --git a/crates/edgezero-adapter-axum/tests/fixtures/production_host.rs b/crates/edgezero-adapter-axum/tests/fixtures/production_host.rs new file mode 100644 index 000000000..c71c9c50d --- /dev/null +++ b/crates/edgezero-adapter-axum/tests/fixtures/production_host.rs @@ -0,0 +1,1224 @@ +//! Test-only executable for real startup, signal, drain and storage acceptance. +#![expect(clippy::print_stdout, reason = "bounded subprocess IPC markers")] + +#[path = "../../../edgezero-core/tests/fixtures/production_handlers.rs"] +mod fixture; + +use std::cell::Cell; +use std::env; +use std::future; +use std::net::{IpAddr, SocketAddr}; +use std::panic::set_hook; +use std::rc::Rc; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex, PoisonError}; +use std::time::Duration; + +use async_trait::async_trait; +use bytes::Bytes; +use edgezero_adapter_axum::context::AxumRequestContext; +use edgezero_adapter_axum::dev_server::{ + AxumRunOptions, run_app, run_app_with_options_and_initializer, run_production_app, + run_production_app_with_initializer, +}; +use edgezero_adapter_axum::diagnostics::{ + IngressDisposition, NativeDiagnosticsHandle, NativeDiagnosticsSnapshot, NativeIngressMetadata, + NativeRequestFailure, NativeRequestObserver, NativeRequestOutcome, NativeRequestRecord, +}; +use edgezero_adapter_axum::geo::{ + MaxMindGeoConfig, NativeGeoCapability, NativeGeoInputContract, NativeGeoSelection, + ProtectedCloudFrontAlb, ReaderIpExpectation, +}; +use edgezero_adapter_axum::proxy::TrustedProxyPolicy; +use edgezero_core::app::{App, Hooks, StoreMetadata, StoresMetadata}; +use edgezero_core::context::RequestContext; +use edgezero_core::extractor::{AppConfig, ForwardedHost, FromRequest as _, Json, State}; +use edgezero_core::geo::{ + CountryCode, GeoError, GeoInput, GeoLocation, GeoOutcome, GeoProvider, GeoProviderId, + GeoResolution, GeoStatus, GeoUnavailableReason, SubdivisionCode, +}; +use edgezero_core::http::{HeaderMap, HeaderValue, Response, StatusCode, Uri}; +use edgezero_core::ingress::{ + AdmissionDecision, CheckedEffectiveHost, CheckedHostSource, IngressGrant, +}; +use edgezero_core::key_value_store::KvHandle; +use edgezero_core::middleware::{Middleware, Next}; +use edgezero_core::probe::{self, LifecyclePhase, LifecycleReader}; +use edgezero_core::response::response_with_body; +use edgezero_core::response_egress::{ + ResponseEgressBodyKind, ResponseEgressCompletion, ResponseEgressDeadline, + ResponseEgressFallbackDisposition, ResponseEgressObserver, ResponseEgressOutcome, + ResponseEgressReport, +}; +use edgezero_core::router::RouterService; +use edgezero_core::{BadGatewayReason, Body, EdgeError, MonotonicClock, MonotonicInstant, action}; +use futures::stream; +use log::{LevelFilter, Log, Metadata, Record}; +use serde_json::{Value, json}; +use tokio::signal; +#[cfg(unix)] +use tokio::signal::unix::{SignalKind, signal as unix_signal}; +use tokio::sync::oneshot; +use tokio::time::sleep; + +static ADMISSION_FACTS: Mutex> = Mutex::new(None); +static MIDDLEWARE_FACTS: Mutex> = Mutex::new(None); +static PANIC_COMPLETION: AtomicBool = AtomicBool::new(false); +static LOGGER_FAULT_REPORTED: AtomicBool = AtomicBool::new(false); +static NATIVE_LOGGER: NativeFixtureLogger = NativeFixtureLogger; +static GEO_RESOLUTIONS: AtomicUsize = AtomicUsize::new(0); + +#[derive(Clone, Default)] +struct NativeFailureInputs { + configuration: String, + secret: String, +} + +struct NativeFixtureLogger; + +impl Log for NativeFixtureLogger { + #[inline] + fn enabled(&self, metadata: &Metadata<'_>) -> bool { + metadata.target().starts_with("edgezero") + } + + #[inline] + fn flush(&self) {} + + #[inline] + #[expect( + clippy::panic, + reason = "regression intentionally panics during the core completion-fault log attempt to prove the joined native completion and app observer still run" + )] + fn log(&self, record: &Record<'_>) { + if !self.enabled(record.metadata()) { + return; + } + let message = record.args().to_string(); + println!( + "FRAMEWORK_LOG {}", + json!({"level": record.level().as_str(), "target": record.target(), "message": message}) + ); + if PANIC_COMPLETION.load(Ordering::Relaxed) + && message == "response-egress completion panicked after terminal transition" + && !LOGGER_FAULT_REPORTED.swap(true, Ordering::Relaxed) + { + println!("LOGGER_FAULT"); + panic!("fixture completion logger fault"); + } + } +} + +struct NativeFixtureObserver(NativeDiagnosticsHandle); + +impl NativeRequestObserver for NativeFixtureObserver { + #[inline] + fn observe(&self, record: &NativeRequestRecord) { + println!( + "NATIVE_OBSERVATION {}", + json!({ + "id": record.request_id.to_string(), + "event": record.event, + "method": record.method, + "route": record.route, + "status": record.status.map(|status| status.as_u16()), + "duration_us": record.duration.map(|duration| duration.as_micros()), + "outcome": native_outcome(record.outcome), + "failure": native_failure(record.failure), + "forwarding_result": record.forwarding_result.as_str(), + "bytes_written": record.bytes_written, + "body_kind": record.body_kind.map(|kind| match kind { + ResponseEgressBodyKind::Application => "application", + ResponseEgressBodyKind::Fallback => "fallback", + _ => "unknown", + }), + "fallback": record.fallback_disposition.map(|disposition| match disposition { + ResponseEgressFallbackDisposition::Aborted => "aborted", + ResponseEgressFallbackDisposition::Completed => "completed", + _ => "unknown", + }), + "level": record.level().as_str(), + "counts": snapshot_json(self.0.snapshot()), + }) + ); + } +} + +struct FixtureEgressObserver; + +impl ResponseEgressObserver for FixtureEgressObserver { + #[inline] + fn complete(&self, report: &ResponseEgressReport) { + println!("APP_OBSERVER {}", egress_outcome(report.outcome)); + } +} + +struct NativeFactsMiddleware; + +#[derive(Clone)] +struct GeoObserved(GeoOutcome); + +struct GeoMutationMiddleware(bool); + +struct FixtureGeoProvider(bool); + +struct FixtureReaderIpProvider; + +#[async_trait(?Send)] +impl Middleware for NativeFactsMiddleware { + #[inline] + async fn handle(&self, ctx: RequestContext, next: Next<'_>) -> Result { + if ctx.uri().path() == "/native-metadata" { + let metadata = ingress_metadata(&ctx)?; + *MIDDLEWARE_FACTS + .lock() + .unwrap_or_else(PoisonError::into_inner) = + Some(metadata_facts(metadata, ctx.headers(), ctx.uri())); + } + next.run(ctx).await + } +} + +struct FixtureApp; + +#[expect( + clippy::missing_trait_methods, + reason = "fixture retains unused Hooks defaults" +)] +impl Hooks for FixtureApp { + #[expect( + clippy::panic_in_result_fn, + reason = "test-only admission assertion proves credential removal before application callbacks" + )] + fn configure(app: &mut App) -> Result<(), EdgeError> { + println!("HOOK_ENTERED"); + let selected = mode(); + if selected.starts_with("diagnostics") { + configure_native_fixture(app, &selected); + } + if selected.starts_with("geo-") { + app.set_ingress_admission_policy(|head| { + assert!( + !head.headers().contains_key("x-origin-verify-fixture"), + "origin credential must be removed before admission" + ); + if head.target().path() == "/native-abort" { + return AdmissionDecision::Abort; + } + let completion = ResponseEgressCompletion::new(|_report| { + println!("GEO_COMPLETED {}", GEO_RESOLUTIONS.load(Ordering::SeqCst)); + }); + if head.target().path() == "/native-refuse" { + let mut response = Response::new(Body::text("fixture admission refused")); + *response.status_mut() = StatusCode::SERVICE_UNAVAILABLE; + return AdmissionDecision::Refuse { + completion, + response, + }; + } + AdmissionDecision::Admit { + completion, + grant: IngressGrant::empty(), + read_deadline: head.read_deadline_after(Duration::from_secs(30)), + } + }); + } + if mode() == "hook-error" { + return Err(EdgeError::internal(anyhow::anyhow!("HOOK_SENTINEL"))); + } + if env::var_os("FIXTURE_FREEZE_CLOCK").is_some() { + let frozen = MonotonicInstant::now(); + app.set_monotonic_clock(MonotonicClock::new(move || frozen)); + } + Ok(()) + } + + fn owns_logging() -> bool { + true + } + + fn routes() -> RouterService { + RouterService::builder() + .post("/json", buffered_json) + .get("/ready", probe::readiness) + .get("/live", probe::liveness) + .get("/state", fixture::state) + .get("/typed", fixture::typed) + .get("/finite", finite) + .get("/pending", pending) + .get("/stream", endless_stream) + .get("/flood", flood) + .get("/after", after) + .get("/write", write) + .get("/read", read) + .get("/alias", alias) + .get("/distinct", distinct) + .get("/snapshot", snapshot) + .get("/stop", stop) + .get("/geo", geo_observation) + .get("/native-metadata", native_metadata) + .get("/native-error", native_error) + .get("/native-upstream", native_upstream) + .get("/native-status503", native_status503) + .get("/native-handler404", native_handler404) + .get("/native-mutated", native_mutated) + .get("/native-source", native_source) + .get("/native-deadline", native_deadline) + .get("/native-expired", native_expired) + .get("/native-conversion", native_conversion) + .post("/native-json-number", native_json_number) + .get("/native-counts", native_counts) + .get("/native-refuse", native_status503) + .get("/native-abort", native_status503) + .middleware(NativeFactsMiddleware) + .middleware(GeoMutationMiddleware(false)) + .middleware(GeoMutationMiddleware(true)) + .build() + } + + fn stores() -> StoresMetadata { + if mode().starts_with("geo-") + || matches!( + mode().as_str(), + "bare" + | "bare-dev" + | "hook-error" + | "initializer-error" + | "starting" + | "embedding-bare" + | "diagnostics" + | "diagnostics-panic" + | "diagnostics-optout" + | "diagnostics-custom" + ) + { + return StoresMetadata::default(); + } + StoresMetadata { + config: Some(StoreMetadata { + ids: &["settings"], + default: "settings", + }), + kv: Some(StoreMetadata { + ids: &["first", "second", "third"], + default: "first", + }), + secrets: Some(StoreMetadata { + ids: &["vault"], + default: "vault", + }), + } + } +} + +struct WorkGuard(&'static str); + +impl Drop for WorkGuard { + fn drop(&mut self) { + println!("{}", self.0); + } +} + +struct StopState(Mutex>>); + +#[async_trait(?Send)] +#[expect( + clippy::expect_used, + reason = "fixture assertions require the first middleware observation and known-valid provider ID" +)] +impl Middleware for GeoMutationMiddleware { + async fn handle(&self, mut ctx: RequestContext, next: Next<'_>) -> Result { + if ctx.uri().path() != "/geo" { + return next.run(ctx).await; + } + assert!( + !ctx.headers().contains_key("x-origin-verify-fixture"), + "origin credential must be absent before middleware" + ); + let original = if self.0 { + ctx.extensions() + .get::() + .expect("first Geo middleware") + .0 + .clone() + } else { + ctx.geo().clone() + }; + assert_eq!( + ctx.geo(), + &original, + "middleware must share the owned outcome" + ); + if self.0 { + ctx.extensions_mut().clear(); + } else { + ctx.parts_mut() + .headers + .insert("cloudfront-viewer-country", HeaderValue::from_static("GB")); + ctx.parts_mut().headers.insert( + "cloudfront-viewer-country-region", + HeaderValue::from_static("LND"), + ); + } + ctx.extensions_mut().insert(GeoOutcome::Selected { + provider: GeoProviderId::new("forged-extension").expect("fixture provider ID"), + status: GeoStatus::Error(GeoError::ProviderFailure), + }); + ctx.extensions_mut().insert(GeoObserved(original.clone())); + assert_eq!(ctx.geo(), &original, "public mutation cannot replace Geo"); + next.run(ctx).await + } +} + +#[expect( + clippy::expect_used, + reason = "supplied fixture provider returns fixed validated country and subdivision values" +)] +impl GeoProvider for FixtureGeoProvider { + fn resolve(&self, _input: &GeoInput) -> GeoResolution { + GEO_RESOLUTIONS.fetch_add(1, Ordering::SeqCst); + if self.0 { + GeoResolution::Error(GeoError::ProviderFailure) + } else { + GeoResolution::Located(GeoLocation::new( + CountryCode::new("US").expect("fixture country"), + Some(SubdivisionCode::new("CA").expect("fixture subdivision")), + )) + } + } +} + +#[expect( + clippy::expect_used, + clippy::use_debug, + reason = "reader fixture returns fixed geographic codes and observes input Debug redaction" +)] +impl GeoProvider for FixtureReaderIpProvider { + fn resolve(&self, input: &GeoInput) -> GeoResolution { + let count = GEO_RESOLUTIONS + .fetch_add(1, Ordering::SeqCst) + .saturating_add(1); + println!("GEO_CALL {count} {input:?}"); + let GeoInput::ReaderIp { address, .. } = input else { + return GeoResolution::Error(GeoError::ProviderFailure); + }; + let located = match address { + IpAddr::V4(v4) => v4.is_loopback() || v4.octets() == [192, 0, 2, 1], + IpAddr::V6(v6) => { + v6.is_loopback() || v6.segments() == [0x2001, 0xdb8, 0, 0, 0, 0, 0, 1] + } + }; + if located { + GeoResolution::Located(GeoLocation::new( + CountryCode::new("US").expect("fixture country"), + Some(SubdivisionCode::new("CA").expect("fixture subdivision")), + )) + } else { + GeoResolution::NoMatch + } + } +} + +fn configure_native_fixture(app: &mut App, selected: &str) { + let panic_completion = selected == "diagnostics-panic"; + app.set_response_egress_observer(FixtureEgressObserver); + app.set_ingress_admission_policy(move |head| { + let metadata = head.extension::(); + if head.target().path() == "/native-metadata" + && let Some(checked) = metadata + { + *ADMISSION_FACTS + .lock() + .unwrap_or_else(PoisonError::into_inner) = + Some(metadata_facts(checked, head.headers(), head.target())); + } + if head.target().path() == "/native-abort" { + return AdmissionDecision::Abort; + } + let id = metadata.map_or_else( + || "unavailable".to_owned(), + |checked| checked.request_id().to_string(), + ); + let completion = ResponseEgressCompletion::new(move |_report| { + println!("APP_COMPLETION {id}"); + assert!(!panic_completion, "fixture application completion fault"); + }); + if head.target().path() == "/native-refuse" { + let mut response = Response::new(Body::text("fixture admission refused")); + *response.status_mut() = StatusCode::SERVICE_UNAVAILABLE; + return AdmissionDecision::Refuse { + completion, + response, + }; + } + AdmissionDecision::Admit { + completion, + grant: IngressGrant::empty(), + read_deadline: head.read_deadline_after(Duration::from_secs(30)), + } + }); + if selected == "diagnostics-custom" { + app.set_error_response_renderer(|error| { + let mut response = Response::new(Body::text(error.message())); + *response.status_mut() = error.status(); + response + }); + } +} + +fn metadata_facts(metadata: &NativeIngressMetadata, headers: &HeaderMap, uri: &Uri) -> Value { + json!({ + "request_id": metadata.request_id().to_string(), + "peer": metadata.direct_peer().map(|peer| peer.to_string()), + "client": metadata.effective_client().map(|client| client.to_string()), + "host": metadata.effective_host().authority(), + "scheme": metadata.effective_scheme().as_str(), + "client_source": host_source(metadata.client_source()), + "host_source": host_source(metadata.host_source()), + "scheme_source": host_source(metadata.scheme_source()), + "forwarding_result": metadata.forwarding_result().as_str(), + "host_header": headers.get("host").and_then(|value| value.to_str().ok()), + "uri": uri.to_string(), + "raw_forwarding_absent": !headers.keys().any(|name| name.as_str() == "forwarded" || name.as_str().starts_with("x-forwarded-")), + }) +} + +fn ingress_metadata(ctx: &RequestContext) -> Result<&NativeIngressMetadata, EdgeError> { + ctx.extensions() + .get::() + .ok_or_else(|| EdgeError::internal(anyhow::anyhow!("fixture ingress metadata missing"))) +} + +fn host_source(source: CheckedHostSource) -> &'static str { + match source { + CheckedHostSource::Direct => "direct", + CheckedHostSource::TrustedForwarded => "trusted_forwarded", + CheckedHostSource::TrustedXForwarded => "trusted_x_forwarded", + CheckedHostSource::Unavailable => "unavailable", + _ => "unknown", + } +} + +fn egress_outcome(outcome: ResponseEgressOutcome) -> &'static str { + match outcome { + ResponseEgressOutcome::ClientDisconnected => "client_disconnected", + ResponseEgressOutcome::Completed => "completed", + ResponseEgressOutcome::ConversionError => "conversion_error", + ResponseEgressOutcome::DeadlineExceeded => "deadline_exceeded", + ResponseEgressOutcome::HostHandoff => "host_handoff", + ResponseEgressOutcome::RequestCancelled => "request_cancelled", + ResponseEgressOutcome::SourceError => "source_error", + ResponseEgressOutcome::TransportError => "transport_error", + ResponseEgressOutcome::Unspecified => "unspecified", + _ => "unknown", + } +} + +fn native_outcome(outcome: NativeRequestOutcome) -> &'static str { + match outcome { + NativeRequestOutcome::Egress(egress) => egress_outcome(egress), + NativeRequestOutcome::Ingress(IngressDisposition::Aborted) => "aborted", + NativeRequestOutcome::Ingress(IngressDisposition::Draining) => "draining", + NativeRequestOutcome::Ingress(IngressDisposition::Abandoned) => "abandoned", + } +} + +fn native_failure(failure: Option) -> Option<&'static str> { + failure.map(|category| match category { + NativeRequestFailure::FrameworkRouting { kind } + | NativeRequestFailure::PropagatedError { kind } => kind, + NativeRequestFailure::FallbackBodyExceeded => "fallback_body_exceeded", + NativeRequestFailure::FallbackReadTimedOut => "fallback_read_timed_out", + NativeRequestFailure::AdmissionRefused => "admission_refused", + }) +} + +fn json_response(value: &Value) -> Result { + let mut response = response_with_body( + StatusCode::OK, + Body::json(value).map_err(EdgeError::internal)?, + )?; + response + .headers_mut() + .insert("content-type", HeaderValue::from_static("application/json")); + Ok(response) +} + +fn snapshot_json(snapshot: NativeDiagnosticsSnapshot) -> Value { + json!({ + "active_requests": snapshot.active_requests, + "active_connections": snapshot.active_connections, + "idle_connections": snapshot.idle_connections, + }) +} + +#[expect( + clippy::expect_used, + reason = "fixture-owned SIGTERM registration is fatal setup: without that owner this child cannot exercise graceful stop" +)] +#[expect( + clippy::integer_division_remainder_used, + reason = "tokio::select uses internal bounded remainder to randomize branch polling; the fixture itself introduces no integer division" +)] +async fn native_stop() { + #[cfg(unix)] + { + let mut terminate = unix_signal(SignalKind::terminate()).expect("fixture SIGTERM owner"); + tokio::select! { + _interrupted = signal::ctrl_c() => {}, + _terminated = terminate.recv() => {}, + } + } + #[cfg(not(unix))] + { + let _interrupted = signal::ctrl_c().await; + } +} + +fn run_native_fixture(selected: &str) -> anyhow::Result<()> { + PANIC_COMPLETION.store(selected == "diagnostics-panic", Ordering::Relaxed); + set_hook(Box::new(|_info| println!("FIXTURE_PANIC"))); + log::set_logger(&NATIVE_LOGGER)?; + log::set_max_level(LevelFilter::Trace); + println!("APP_LOGGER_INSTALLED"); + let base = NativeDiagnosticsHandle::new(); + // Capture a counter-only clone, not the observer-bearing handle, to avoid a cycle. + let observed = base + .clone() + .with_request_observer(NativeFixtureObserver(base)); + let options = AxumRunOptions::from_env()?.with_diagnostics(observed.clone()); + let state = observed.clone(); + let private_inputs = NativeFailureInputs { + configuration: env::var("PRIVATE_CONFIG_SENTINEL").unwrap_or_default(), + secret: env::var("fixture_token").unwrap_or_default(), + }; + let result = run_app_with_options_and_initializer::( + options, + native_stop(), + move |app, _stores| { + Box::pin(async move { + app.insert_state(state); + app.insert_state(private_inputs); + println!("INITIALIZED"); + Ok(()) + }) + }, + ); + println!("FINAL_SNAPSHOT {}", snapshot_json(observed.snapshot())); + result +} + +#[action] +async fn buffered_json(Json(value): Json) -> Result { + println!("JSON_HANDLER_ENTERED"); + Ok(value.len().to_string()) +} + +#[action] +async fn native_metadata( + ctx: RequestContext, + ForwardedHost(forwarded_host): ForwardedHost, +) -> Result { + let metadata = ingress_metadata(&ctx)?; + let summary = metadata.received_normalized_head(); + let admission = ADMISSION_FACTS + .lock() + .unwrap_or_else(PoisonError::into_inner) + .take() + .ok_or_else(|| EdgeError::internal(anyhow::anyhow!("fixture admission facts missing")))?; + let middleware = MIDDLEWARE_FACTS + .lock() + .unwrap_or_else(PoisonError::into_inner) + .take() + .ok_or_else(|| EdgeError::internal(anyhow::anyhow!("fixture middleware facts missing")))?; + json_response(&json!({ + "admission": admission, + "middleware": middleware, + "handler": metadata_facts(metadata, ctx.headers(), ctx.uri()), + "forwarded_host": forwarded_host, + "direct_context_peer": ctx.extensions().get::().and_then(|context| context.remote_addr).map(|peer| peer.to_string()), + "normalized": { + "target_bytes": summary.target_bytes(), + "header_bytes": summary.header_bytes(), + "header_count": summary.header_count(), + }, + })) +} + +#[action] +async fn native_error( + State(inputs): State, +) -> Result<&'static str, EdgeError> { + println!("PRIVATE_INPUTS_USED"); + Err(EdgeError::internal(anyhow::anyhow!( + "PRIVATE_ERROR_SENTINEL {} {}", + inputs.configuration, + inputs.secret, + ))) +} + +#[action] +async fn native_upstream( + State(inputs): State, +) -> Result<&'static str, EdgeError> { + println!("PRIVATE_INPUTS_USED"); + Err(EdgeError::bad_gateway_with_reason( + format!( + "PRIVATE_ORIGIN_SENTINEL {} {}", + inputs.configuration, inputs.secret + ), + BadGatewayReason::Transport, + )) +} + +#[action] +async fn native_status503() -> Result { + response_with_body( + StatusCode::SERVICE_UNAVAILABLE, + Body::text("application-selected 503"), + ) +} + +#[action] +async fn native_handler404() -> Result<&'static str, EdgeError> { + Err(EdgeError::not_found("HANDLER_404_SENTINEL")) +} + +#[action] +async fn native_mutated(mut ctx: RequestContext) -> Result { + let removed = ctx + .extensions_mut() + .remove::() + .ok_or_else(|| EdgeError::internal(anyhow::anyhow!("fixture ingress metadata missing")))?; + let removed_id = removed.request_id().to_string(); + ctx.extensions_mut() + .insert(CheckedEffectiveHost::from_ingress( + Some("replacement.example".parse().map_err(EdgeError::internal)?), + CheckedHostSource::Direct, + )); + let ForwardedHost(after_host) = ForwardedHost::from_request(&ctx).await?; + json_response(&json!({ + "removed_id": removed_id, + "metadata_absent": ctx.extensions().get::().is_none(), + "forwarded_host_after": after_host, + })) +} + +#[action] +async fn native_source() -> Result { + let guard = Rc::new(WorkGuard("SOURCE_DROPPED")); + Ok(Body::from_stream(async_stream::stream! { + let _retained = guard; + yield Ok(Bytes::from_static(b"source-prefix")); + sleep(Duration::from_millis(100)).await; + yield Err(EdgeError::internal(anyhow::anyhow!("SOURCE_SENTINEL"))); + })) +} + +#[action] +async fn native_deadline() -> Result { + let guard = Rc::new(WorkGuard("DEADLINE_DROPPED")); + let body = Body::from_stream(async_stream::stream! { + let _retained = guard; + yield Ok(Bytes::from_static(b"deadline-prefix")); + future::pending::<()>().await; + }); + let mut response = Response::new(body); + response + .extensions_mut() + .insert(ResponseEgressDeadline::after(Duration::from_millis(150))); + Ok(response) +} + +#[action] +async fn native_expired() -> Result { + let mut response = Response::new(Body::text("expired-private-body")); + response + .extensions_mut() + .insert(ResponseEgressDeadline::after(Duration::ZERO)); + Ok(response) +} + +#[action] +async fn native_conversion() -> Result { + let mut response = Response::new(Body::text("CONVERSION_PAYLOAD_SENTINEL")); + response + .headers_mut() + .insert("content-length", HeaderValue::from_static("1")); + Ok(response) +} + +#[action] +async fn native_json_number(Json(value): Json) -> Result { + Ok(value.to_string()) +} + +#[action] +async fn native_counts( + ctx: RequestContext, + State(handle): State, +) -> Result { + let snapshot = handle.snapshot(); + let phase = match reader(&ctx)?.phase() { + Some(LifecyclePhase::Starting) => "starting", + Some(LifecyclePhase::Ready) => "ready", + Some(LifecyclePhase::Draining) => "draining", + Some(LifecyclePhase::Stopped) => "stopped", + None => "unavailable", + }; + json_response(&json!({ + "active_requests": snapshot.active_requests, + "active_connections": snapshot.active_connections, + "idle_connections": snapshot.idle_connections, + "phase": phase, + })) +} + +fn geo_json(outcome: &GeoOutcome) -> Value { + match outcome { + GeoOutcome::NoSelection => json!({"status": "no_selection"}), + GeoOutcome::Selected { provider, status } => match status { + GeoStatus::Unavailable(reason) => { + json!({"provider": provider.as_str(), "status": match reason { + GeoUnavailableReason::MissingMetadata => "missing_metadata", + GeoUnavailableReason::MissingReaderIp => "missing_reader_ip", + GeoUnavailableReason::NoMatch => "no_match", + GeoUnavailableReason::UntrustedIngress => "untrusted_ingress", + }}) + } + GeoStatus::Error(error) => { + json!({"provider": provider.as_str(), "status": match error { + GeoError::InvalidAssertion => "invalid_assertion", + GeoError::ProviderFailure => "provider_failure", + }}) + } + GeoStatus::Located { + location, + source, + trust_basis, + } => json!({ + "provider": provider.as_str(), + "status": "located", + "country": location.country().as_str(), + "subdivision": location.subdivision().map(SubdivisionCode::as_str), + "source": format!("{source:?}"), + "trust_basis": format!("{trust_basis:?}"), + }), + }, + } +} + +#[action] +async fn geo_observation(mut ctx: RequestContext) -> Result { + let original = ctx + .extensions() + .get::() + .ok_or_else(|| { + EdgeError::internal(anyhow::anyhow!( + "fixture Geo middleware observation missing" + )) + })? + .0 + .clone(); + assert_eq!( + ctx.geo(), + &original, + "handler must share middleware outcome" + ); + ctx.extensions_mut().remove::(); + assert_eq!( + ctx.geo(), + &original, + "removing public outcome cannot affect Geo" + ); + let credential_absent = !ctx.headers().contains_key("x-origin-verify-fixture"); + let mutated_country = ctx + .headers() + .get("cloudfront-viewer-country") + .and_then(|value| value.to_str().ok()); + json_response(&json!({ + "geo": geo_json(ctx.geo()), + "credential_absent": credential_absent, + "immutable": ctx.geo() == &original, + "mutated_country": mutated_country, + "resolutions": GEO_RESOLUTIONS.load(Ordering::SeqCst), + })) +} + +fn fixture_geo_selection( + selected: &str, + expectation: ReaderIpExpectation, +) -> anyhow::Result { + let peer_network = if selected.ends_with("untrusted") { + "192.0.2.1/32" + } else { + "127.0.0.1/32" + }; + let policy = || { + ProtectedCloudFrontAlb::new( + if selected == "geo-invalid-policy" { + Vec::new() + } else { + vec![peer_network] + }, + "x-origin-verify-fixture", + ) + }; + Ok(if selected == "geo-off" { + NativeGeoSelection::Off + } else if selected == "geo-maxmind" { + let mut config = MaxMindGeoConfig::new( + env::var_os("FIXTURE_GEO_DB").unwrap_or_default(), + expectation, + ); + if let Ok(value) = env::var("FIXTURE_GEO_MAX_BYTES") { + config.limits.max_database_bytes = value.parse()?; + } + if let Ok(value) = env::var("FIXTURE_GEO_MAX_NETWORKS") { + config.limits.max_validation_networks = value.parse()?; + } + NativeGeoSelection::MaxMind { config } + } else if selected == "geo-supplied-reader" { + NativeGeoSelection::SuppliedReaderIp { + expectation, + id: GeoProviderId::new("fixture-reader")?, + provider: Arc::new(FixtureReaderIpProvider), + } + } else if selected.starts_with("geo-supplied") || selected == "geo-provider-failure" { + NativeGeoSelection::Supplied { + policy: policy()?, + id: GeoProviderId::new("fixture-local")?, + provider: Arc::new(FixtureGeoProvider(selected == "geo-provider-failure")), + } + } else { + NativeGeoSelection::CloudFront { policy: policy()? } + }) +} + +#[expect( + clippy::panic_in_result_fn, + clippy::expect_used, + clippy::use_debug, + reason = "test-only initializer asserts root-free startup, safe Debug output, capability capture and known-valid forged state" +)] +fn run_geo_fixture(selected: &str) -> anyhow::Result<()> { + log::set_logger(&NATIVE_LOGGER)?; + log::set_max_level(LevelFilter::Trace); + let address: SocketAddr = env::var("FIXTURE_BIND")?.parse()?; + let expectation = match env::var("FIXTURE_GEO_EXPECTATION").as_deref() { + Ok("forwarded-required") => ReaderIpExpectation::ForwardedRequired, + Ok("peer-allowed") | Err(_) => ReaderIpExpectation::PeerAllowed, + Ok(_) => anyhow::bail!("invalid fixture reader expectation"), + }; + let selection = fixture_geo_selection(selected, expectation)?; + println!("GEO_SELECTION {selection:?}"); + let mut options = AxumRunOptions::new(address)?.with_geo(selection); + let family = env::var("FIXTURE_GEO_PROXY_FAMILY").ok(); + let networks = env::var("FIXTURE_GEO_PROXY_NETWORKS") + .unwrap_or_else(|_| "127.0.0.1/32,::1/128".to_owned()); + if let Some(selected_family) = family { + options = options.with_trusted_proxy_policy(TrustedProxyPolicy::from_values( + Some(&networks), + Some(&selected_family), + )?); + } + // Capture from the same finalized options, without bypassing the runner's validation. + let capability = options.geo_capability(); + println!("GEO_CAPABILITY {capability:?}"); + let is_off = selected == "geo-off"; + let is_reader = matches!(selected, "geo-maxmind" | "geo-supplied-reader"); + let (sender, receiver) = oneshot::channel(); + run_app_with_options_and_initializer::( + options, + async move { + let _closed = receiver.await; + }, + move |app, stores| { + Box::pin(async move { + assert!( + stores.config().is_none(), + "Geo fixture declares no config stores" + ); + assert!(stores.kv().is_none(), "Geo fixture declares no KV stores"); + assert!( + stores.secrets().is_none(), + "Geo fixture declares no secret stores" + ); + let captured_capability = capability.map_err(EdgeError::internal)?; + assert_eq!( + matches!(captured_capability, NativeGeoCapability::Off), + is_off, + "initializer sees the finalized selection capability" + ); + if is_reader { + assert!( + matches!( + captured_capability, + NativeGeoCapability::Selected { + input: NativeGeoInputContract::ReaderIp { expectation: configured }, + .. + } if configured == expectation + ), + "initializer sees the selected reader input expectation" + ); + } + app.insert_state(GeoOutcome::Selected { + provider: GeoProviderId::new("forged-app-state").expect("fixture provider ID"), + status: GeoStatus::Error(GeoError::ProviderFailure), + }); + app.insert_state(Arc::new(StopState(Mutex::new(Some(sender))))); + println!("INITIALIZED"); + Ok(()) + }) + }, + ) +} + +fn mode() -> String { + env::var("FIXTURE_MODE").unwrap_or_else(|_| "initialized".to_owned()) +} + +fn reader(ctx: &RequestContext) -> Result { + ctx.extensions() + .get::() + .cloned() + .ok_or_else(|| EdgeError::service_unavailable("fixture reader missing")) +} + +#[action] +async fn after() -> Result<&'static str, EdgeError> { + println!("FORBIDDEN_DISPATCH"); + Ok("forbidden dispatch") +} + +#[action] +async fn finite(ctx: RequestContext) -> Result<&'static str, EdgeError> { + let observed = reader(&ctx)?; + let _guard = WorkGuard("FINITE_DROPPED"); + println!("DISPATCH"); + while observed.phase() == Some(LifecyclePhase::Ready) { + sleep(Duration::from_millis(5)).await; + } + println!("DRAINING"); + sleep(Duration::from_millis(80)).await; + Ok("finite-complete") +} + +#[action] +async fn pending(ctx: RequestContext) -> Result<&'static str, EdgeError> { + let observed = reader(&ctx)?; + let _guard = WorkGuard("WORK_DROPPED"); + println!("DISPATCH"); + let mut reported = false; + #[expect( + clippy::infinite_loop, + reason = "fixture proves process shutdown cancels a never-ending handler" + )] + loop { + if !reported && observed.phase() == Some(LifecyclePhase::Draining) { + println!("DRAINING"); + reported = true; + } + sleep(Duration::from_millis(5)).await; + } +} + +#[action] +async fn endless_stream(ctx: RequestContext) -> Result { + let observed = reader(&ctx)?; + let guard = Rc::new(WorkGuard("STREAM_DROPPED")); + println!("STREAM_READY"); + Ok(Body::from_stream(async_stream::stream! { + let _retained = guard; + yield Ok(Bytes::from_static(b"stream-prefix")); + let mut reported = false; + loop { + if !reported && observed.phase() == Some(LifecyclePhase::Draining) { + println!("DRAINING"); + reported = true; + } + sleep(Duration::from_millis(5)).await; + } + })) +} + +#[action] +async fn flood() -> Result { + static BLOCK: [u8; 0x0001_0000] = [b'x'; 0x0001_0000]; + let guard = Rc::new(WorkGuard("STREAM_DROPPED")); + println!("STREAM_READY"); + Ok(Body::from_stream(stream::repeat_with(move || { + let _retained = &guard; + Ok(Bytes::from_static(&BLOCK)) + }))) +} + +fn kv(ctx: &RequestContext, id: &str) -> Result { + ctx.kv_store(id) + .ok_or_else(|| EdgeError::service_unavailable("fixture KV missing")) +} + +#[action] +async fn write(ctx: RequestContext) -> Result<&'static str, EdgeError> { + kv(&ctx, "first")? + .put("persisted", &"durable-value") + .await?; + Ok("written") +} + +#[action] +async fn read(ctx: RequestContext) -> Result { + Ok(kv(&ctx, "first")? + .get::("persisted") + .await? + .unwrap_or_default()) +} + +#[action] +async fn alias(ctx: RequestContext) -> Result { + Ok(kv(&ctx, "second")? + .get::("persisted") + .await? + .unwrap_or_default()) +} + +#[action] +async fn distinct(ctx: RequestContext) -> Result { + Ok(kv(&ctx, "third")? + .get::("persisted") + .await? + .unwrap_or_default()) +} + +#[action] +async fn snapshot(ctx: RequestContext) -> Result { + let binding = ctx + .config_store_default_binding() + .ok_or_else(|| EdgeError::service_unavailable("fixture config missing"))?; + Ok(binding + .handle + .get(&binding.default_key) + .await? + .unwrap_or_default() + .as_str() + .to_owned()) +} + +#[action] +async fn stop(State(state): State>) -> Result<&'static str, EdgeError> { + if let Some(sender) = state + .0 + .lock() + .map_err(|_error| EdgeError::internal(anyhow::anyhow!("fixture stop lock poisoned")))? + .take() + { + let _sent = sender.send(()); + } + Ok("stopping") +} + +#[expect( + clippy::panic_in_result_fn, + reason = "test-only executable asserts initializer ownership and call counts" +)] +fn main() -> anyhow::Result<()> { + let selected = mode(); + if selected.starts_with("diagnostics") { + return run_native_fixture(&selected); + } + if selected.starts_with("geo-") { + return run_geo_fixture(&selected); + } + if matches!(selected.as_str(), "development" | "bare-dev") { + return run_app::(); + } + if matches!(selected.as_str(), "framework" | "bare" | "hook-error") { + return run_production_app::(); + } + if selected == "starting" { + return run_production_app_with_initializer::(|_app, _stores| { + Box::pin(async { + let _guard = WorkGuard("STARTUP_DROPPED"); + println!("INITIALIZING"); + future::pending::>().await + }) + }); + } + if selected == "initializer-error" { + return run_production_app_with_initializer::(|_app, _stores| { + Box::pin(async { Err(EdgeError::internal(anyhow::anyhow!("CALLBACK_SENTINEL"))) }) + }); + } + let calls = Rc::new(Cell::new(0_u32)); + let initialized = Rc::clone(&calls); + if selected.starts_with("embedding") { + let address: SocketAddr = env::var("FIXTURE_BIND")?.parse()?; + let mut options = AxumRunOptions::new(address)?; + if selected == "embedding" { + options = options + .with_config_dir(env::var("FIXTURE_CONFIG_ROOT")?)? + .with_data_dir(env::var("FIXTURE_DATA_ROOT")?)?; + } + let (sender, receiver) = oneshot::channel(); + let stop_capture = Rc::clone(&calls); + return run_app_with_options_and_initializer::( + options, + async move { + let _closed = receiver.await; + assert_eq!( + stop_capture.get(), + 1, + "initializer ran once before caller stop" + ); + }, + move |app, stores| { + Box::pin(async move { + if let Some(binding) = + stores.config().and_then(|registry| registry.default_ref()) + { + let config = AppConfig::::from_binding( + binding, + None, + stores.secrets(), + app.config_extraction_limits(), + app.monotonic_clock(), + ) + .await?; + app.insert_state(Arc::new(fixture::HostState { + greeting: config.greeting, + })); + } + app.insert_state(Arc::new(StopState(Mutex::new(Some(sender))))); + initialized.set(initialized.get().saturating_add(1)); + println!("INITIALIZED"); + Ok(()) + }) + }, + ); + } + run_production_app_with_initializer::(move |app, stores| { + Box::pin(async move { + let binding = stores + .config() + .and_then(|registry| registry.default_ref()) + .ok_or_else(|| { + EdgeError::service_unavailable("required fixture binding missing") + })?; + let config = AppConfig::::from_binding( + binding, + None, + stores.secrets(), + app.config_extraction_limits(), + app.monotonic_clock(), + ) + .await?; + app.insert_state(Arc::new(fixture::HostState { + greeting: config.greeting, + })); + initialized.set(initialized.get().saturating_add(1)); + assert_eq!(initialized.get(), 1, "initializer runs exactly once"); + println!("INITIALIZED"); + Ok(()) + }) + }) +} diff --git a/crates/edgezero-adapter-axum/tests/fixtures/tls/README.md b/crates/edgezero-adapter-axum/tests/fixtures/tls/README.md new file mode 100644 index 000000000..0851892ea --- /dev/null +++ b/crates/edgezero-adapter-axum/tests/fixtures/tls/README.md @@ -0,0 +1,11 @@ +# Local proxy TLS fixtures + +These public test assets are only for the local native-host proxy fixture. `server-key.der` is an intentionally published test key, not a credential. Never use it outside tests. + +- `ca.der` is the fixture CA certificate. +- `server.der` is the server certificate for `native-proxy.test`. +- `server-key.der` is its PKCS#8 private key. + +The server certificate is valid from October 6, 2026 to October 3, 2036. The test client trusts only this fixture CA and verifies the certificate and DNS name. A request for a different server name must fail verification. Renew the fixture assets before expiry; do not disable certificate or name checks to extend their lifetime. + +The fixture terminates TLS and relays HTTP to the managed native host. It does not add incoming TLS support to the Axum adapter. diff --git a/crates/edgezero-adapter-axum/tests/fixtures/tls/ca.der b/crates/edgezero-adapter-axum/tests/fixtures/tls/ca.der new file mode 100644 index 000000000..025acfc6c Binary files /dev/null and b/crates/edgezero-adapter-axum/tests/fixtures/tls/ca.der differ diff --git a/crates/edgezero-adapter-axum/tests/fixtures/tls/server-key.der b/crates/edgezero-adapter-axum/tests/fixtures/tls/server-key.der new file mode 100644 index 000000000..c2f0fb2d5 Binary files /dev/null and b/crates/edgezero-adapter-axum/tests/fixtures/tls/server-key.der differ diff --git a/crates/edgezero-adapter-axum/tests/fixtures/tls/server.der b/crates/edgezero-adapter-axum/tests/fixtures/tls/server.der new file mode 100644 index 000000000..3a7ef3ae8 Binary files /dev/null and b/crates/edgezero-adapter-axum/tests/fixtures/tls/server.der differ diff --git a/crates/edgezero-adapter-axum/tests/native_aws_startup.rs b/crates/edgezero-adapter-axum/tests/native_aws_startup.rs new file mode 100644 index 000000000..784d906c4 --- /dev/null +++ b/crates/edgezero-adapter-axum/tests/native_aws_startup.rs @@ -0,0 +1,700 @@ +//! Loopback-only production startup tests. +//! +//! These tests require `aws-stores` and `test-utils` and launch `fixtures/native_aws_host.rs`. +//! They exercise the public runner and real pinned SDK, not a provider seam. No AWS account, +//! DNS hostname, external socket, image, or real cloud credential is used. + +#[cfg(test)] +#[expect( + clippy::arbitrary_source_item_ordering, + reason = "loopback fixture helpers remain grouped before startup scenarios" +)] +mod tests { + use std::fmt::Write as _; + use std::fs; + use std::io::{ErrorKind, Read, Write as _}; + use std::net::{SocketAddr, TcpListener, TcpStream}; + use std::path::Path; + use std::process::{Child, Command, Stdio}; + use std::sync::mpsc; + use std::sync::{Arc, Mutex}; + use std::thread::{self, JoinHandle}; + use std::time::{Duration, Instant}; + + use edgezero_core::blob_envelope::BlobEnvelope; + use serde_json::{Value, json}; + use tempfile::TempDir; + + const WAIT: Duration = Duration::from_secs(12); + const ARN: &str = "arn:aws:secretsmanager:us-east-1:111122223333:secret:native-fixture-AbCdEf"; + const TOKEN_SENTINEL: &str = "native-fixture-secret-value"; + const LOCAL_SENTINEL: &str = "native-fixture-local-secret"; + const AGENT_TOKEN: &str = "native-fixture-agent-token"; + const FALLBACK_SENTINEL: &str = "local-fallback-must-not-be-used"; + + #[derive(Debug)] + struct Request { + headers: String, + body: String, + } + + struct Reply { + status: u16, + reason: &'static str, + body: String, + content_type: &'static str, + headers: &'static str, + } + + impl Reply { + fn ok(body: String, content_type: &'static str) -> Self { + Self { + status: 200, + reason: "OK", + body, + content_type, + headers: "", + } + } + } + + /// One finite server bound to an IPv4 loopback literal. `expected=false` watches briefly for an + /// unexpected request, which makes the Agent-failure/no-Secrets-call assertion observable. + struct FixtureServer { + endpoint: String, + request: mpsc::Receiver, + worker: Option>>, + } + + impl FixtureServer { + fn once(reply: Reply, expected: bool) -> Self { + let listener = TcpListener::bind("127.0.0.1:0").expect("bind literal loopback only"); + listener + .set_nonblocking(true) + .expect("nonblocking loopback listener"); + let address = listener.local_addr().expect("fixture address"); + let endpoint = format!("http://{address}"); + let (sender, request_receiver) = mpsc::channel(); + let worker = thread::spawn(move || { + let watch = if expected { + WAIT + } else { + Duration::from_millis(300) + }; + let deadline = Instant::now() + .checked_add(watch) + .expect("fixture deadline fits the monotonic clock"); + loop { + match listener.accept() { + Ok((mut stream, peer)) => { + assert!(peer.ip().is_loopback(), "fixture peer was not loopback"); + stream + .set_read_timeout(Some(Duration::from_secs(2))) + .expect("bounded request read"); + stream + .set_write_timeout(Some(Duration::from_secs(2))) + .expect("bounded response write"); + let request = read_request(&mut stream); + send_response(&mut stream, &reply); + let _sent = sender.send(Request { + headers: request.headers.clone(), + body: request.body.clone(), + }); + return Some(request); + } + Err(error) if error.kind() == ErrorKind::WouldBlock => { + if Instant::now() >= deadline { + assert!(!expected, "expected fixture request did not arrive"); + return None; + } + thread::sleep(Duration::from_millis(5)); + } + Err(error) => panic!("loopback fixture accept failed: {error}"), + } + } + }); + Self { + endpoint, + request: request_receiver, + worker: Some(worker), + } + } + + fn endpoint(&self) -> &str { + &self.endpoint + } + + fn take_request(&self) -> Request { + self.request + .recv_timeout(WAIT) + .expect("fixture captured one request") + } + + #[expect( + clippy::unwrap_in_result, + reason = "a broken fixture or panicked thread must fail the test, never become a normal absence" + )] + fn finish(mut self) -> Option { + self.worker + .take() + .expect("fixture worker") + .join() + .expect("fixture thread") + } + } + + fn read_request(stream: &mut TcpStream) -> Request { + let mut bytes = Vec::new(); + let mut scratch = [0_u8; 2048]; + let header_end = loop { + let count = stream.read(&mut scratch).expect("read request"); + assert_ne!(count, 0, "peer closed before request headers"); + let received = scratch + .get(..count) + .expect("read count does not exceed scratch buffer"); + bytes.extend_from_slice(received); + assert!(bytes.len() <= 64 * 1024, "bounded fixture request"); + if let Some(offset) = bytes.windows(4).position(|part| part == b"\r\n\r\n") { + break offset + .checked_add(4) + .expect("header delimiter offset fits request"); + } + }; + let header_bytes = bytes + .get(..header_end) + .expect("header terminator is within received request"); + let headers = String::from_utf8_lossy(header_bytes).into_owned(); + let content_length = headers + .lines() + .find_map(|line| { + let (_, value) = line + .split_once(':') + .filter(|(name, _)| name.eq_ignore_ascii_case("content-length"))?; + value.trim().parse::().ok() + }) + .unwrap_or(0); + let body_end = header_end + .checked_add(content_length) + .expect("bounded request body end fits usize"); + while bytes.len() < body_end { + let count = stream.read(&mut scratch).expect("read bounded body"); + assert_ne!(count, 0, "peer closed before request body"); + let received = scratch + .get(..count) + .expect("read count does not exceed scratch buffer"); + bytes.extend_from_slice(received); + assert!( + bytes.len() <= body_end, + "request body stays within declared length" + ); + } + let body_bytes = bytes + .get(header_end..body_end) + .expect("declared request body is within received bytes"); + Request { + headers: headers.to_ascii_lowercase(), + body: String::from_utf8_lossy(body_bytes).into_owned(), + } + } + + fn send_response(stream: &mut TcpStream, reply: &Reply) { + write!( + stream, + "HTTP/1.1 {} {}\r\nContent-Type: {}\r\nContent-Length: {}\r\n{}Connection: close\r\n\r\n", + reply.status, + reply.reason, + reply.content_type, + reply.body.len(), + reply.headers + ) + .expect("write response headers"); + stream + .write_all(reply.body.as_bytes()) + .expect("write response body"); + } + + struct ChildHost { + child: Child, + address: SocketAddr, + output: Arc>>, + readers: Vec>, + } + + impl ChildHost { + fn spawn(root: &TempDir, bindings: &Path, secrets_endpoint: &str) -> Self { + let home = root.path().join("home"); + let profiles = root.path().join("profiles"); + fs::create_dir_all(&home).expect("isolated HOME"); + fs::create_dir_all(&profiles).expect("isolated profile directory"); + let config = profiles.join("config"); + let credentials = profiles.join("credentials"); + fs::write(&config, "").expect("empty AWS config; no profiles"); + fs::write(&credentials, "").expect("empty AWS credentials; no profiles"); + let address = reserve_address(); + let mut child = Command::new(env!("CARGO_BIN_EXE_edgezero-native-aws-host")) + .env_clear() + .current_dir(root.path()) + .env("HOME", home) + .env("AWS_CONFIG_FILE", config) + .env("AWS_SHARED_CREDENTIALS_FILE", credentials) + .env("AWS_REGION", "us-east-1") + .env("AWS_EC2_METADATA_DISABLED", "true") + .env("AWS_ACCESS_KEY_ID", "DUMMY_LOOPBACK_ACCESS_KEY") + .env("AWS_SECRET_ACCESS_KEY", "DUMMY_LOOPBACK_SECRET_KEY") + .env("AWS_ENDPOINT_URL", secrets_endpoint) + .env("AWS_ENDPOINT_URL_SECRETS_MANAGER", secrets_endpoint) + .env("EDGEZERO__STORE_BINDINGS_FILE", bindings) + .env("EDGEZERO__ADAPTER__HOST", "127.0.0.1") + .env("EDGEZERO__ADAPTER__PORT", address.port().to_string()) + .env("EDGEZERO__LOGGING__LEVEL", "off") + .env("APPCONFIG_AGENT_TOKEN", AGENT_TOKEN) + .env("LOCAL_FIXTURE_SECRET", LOCAL_SENTINEL) + .env("token", FALLBACK_SENTINEL) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .expect("start production host fixture"); + let output = Arc::new(Mutex::new(Vec::new())); + let streams: [Box; 2] = [ + Box::new(child.stdout.take().expect("child stdout")), + Box::new(child.stderr.take().expect("child stderr")), + ]; + let readers = streams + .into_iter() + .map(|mut stream| { + let captured_output = Arc::clone(&output); + thread::spawn(move || { + let mut captured = Vec::new(); + stream + .read_to_end(&mut captured) + .expect("capture child output"); + captured_output + .lock() + .expect("captured output") + .extend(captured); + }) + }) + .collect(); + Self { + child, + address, + output, + readers, + } + } + + fn wait_http(&mut self, path: &str) -> String { + let deadline = Instant::now() + .checked_add(WAIT) + .expect("host readiness deadline fits the monotonic clock"); + loop { + if let Some(status) = self.child.try_wait().expect("child status") { + self.child.wait().expect("reap exited child"); + let output = self.collect_output(); + panic!("fixture exited before {path} (status {status}): {output}"); + } + if let Ok(mut stream) = + TcpStream::connect_timeout(&self.address, Duration::from_millis(40)) + { + stream + .set_read_timeout(Some(Duration::from_secs(2))) + .expect("bounded app response read"); + write!( + stream, + "GET {path} HTTP/1.1\r\nHost: 127.0.0.1\r\nConnection: close\r\n\r\n" + ) + .expect("write app request"); + let mut response = String::new(); + stream + .read_to_string(&mut response) + .expect("read complete app response"); + if response.starts_with("HTTP/1.1 200") { + return response; + } + } + assert!(Instant::now() < deadline, "host readiness deadline"); + thread::sleep(Duration::from_millis(10)); + } + } + + fn wait_failure(&mut self) -> String { + let deadline = Instant::now() + .checked_add(WAIT) + .expect("startup failure deadline fits the monotonic clock"); + loop { + if let Some(status) = self.child.try_wait().expect("child status") { + assert!(!status.success(), "expected startup failure"); + self.child.wait().expect("reap failed child"); + return self.collect_output(); + } + assert!(Instant::now() < deadline, "failed startup stayed alive"); + thread::sleep(Duration::from_millis(10)); + } + } + + fn stop(&mut self) { + if self.child.try_wait().expect("child status").is_none() { + self.child.kill().expect("stop loopback host"); + } + self.child.wait().expect("reap host"); + let _output = self.collect_output(); + } + + fn collect_output(&mut self) -> String { + for reader in self.readers.drain(..) { + reader.join().expect("child output reader"); + } + String::from_utf8_lossy(&self.output.lock().expect("captured output")).into_owned() + } + + fn assert_no_listener(&self) { + TcpStream::connect_timeout(&self.address, Duration::from_millis(50)) + .expect_err("startup failure must happen before listener acceptance"); + } + } + + impl Drop for ChildHost { + fn drop(&mut self) { + if self.child.try_wait().ok().flatten().is_none() { + let _killed = self.child.kill(); + } + let _status = self.child.wait(); + for reader in self.readers.drain(..) { + let _joined = reader.join(); + } + } + } + + fn reserve_address() -> SocketAddr { + let listener = TcpListener::bind("127.0.0.1:0").expect("reserve literal loopback port"); + listener.local_addr().expect("reserved address") + } + + fn response_body(response: &str) -> &str { + response + .split_once("\r\n\r\n") + .expect("HTTP response header terminator") + .1 + } + + fn envelope(token_reference: &str, greeting: &str) -> String { + serde_json::to_string(&BlobEnvelope::new( + json!({"greeting": greeting, "token": token_reference}), + "2026-10-07T12:00:00Z".to_owned(), + )) + .expect("serialize actual core BlobEnvelope") + } + + fn agent_reply(document: &str, status: u16) -> Reply { + if status == 200 { + Reply::ok(document.to_owned(), "application/json") + } else { + Reply { + status, + reason: "Service Unavailable", + body: "AGENT_BODY_SENTINEL".to_owned(), + content_type: "text/plain", + headers: "", + } + } + } + + fn secrets_reply(value: &str) -> Reply { + Reply::ok( + json!({ + "ARN": ARN, + "Name": "native-fixture", + "VersionId": "12345678901234567890123456789012", + "SecretString": value, + }) + .to_string(), + "application/x-amz-json-1.1", + ) + } + + fn denied_secrets_reply() -> Reply { + Reply { + status: 400, + reason: "Bad Request", + body: json!({ + "__type": "AccessDeniedException", + "message": "AWS_ERROR_SENTINEL ARN_SENTINEL CREDENTIAL_SENTINEL", + }) + .to_string(), + content_type: "application/x-amz-json-1.1", + headers: "", + } + } + + fn bindings_file(path: &Path, agent: &str, aws_limits: Option<(usize, usize, usize)>) { + let mut content = format!( + r#"version = 1 + +[config.app] +provider = "aws-appconfig-agent" +default_key = "settings" +endpoint = "{agent}" +access_token_env = "APPCONFIG_AGENT_TOKEN" + +[config.app.documents.settings] +application = "native-fixture" +environment = "test" +profile = "startup-proof" + +[secrets.vault] +provider = "aws-secrets-manager" +region = "us-east-1" + +[secrets.vault.entries.token] +secret_id = "{ARN}" +"# + ); + if let Some((snapshot, config, secret)) = aws_limits { + write!(content, "\n[limits.aws]\nmax_snapshot_bytes = {snapshot}\nmax_config_document_bytes = {config}\nmax_secret_value_bytes = {secret}\n").expect("format fixture limits"); + } + fs::write(path, content).expect("write explicit fictional bindings"); + } + + fn assert_safe_error(text: &str) { + for sentinel in [ + TOKEN_SENTINEL, + LOCAL_SENTINEL, + AGENT_TOKEN, + FALLBACK_SENTINEL, + "AGENT_BODY_SENTINEL", + "AWS_ERROR_SENTINEL", + "ARN_SENTINEL", + "CREDENTIAL_SENTINEL", + "DUMMY_LOOPBACK_ACCESS_KEY", + "DUMMY_LOOPBACK_SECRET_KEY", + ] { + assert!(!text.contains(sentinel), "startup output leaked {sentinel}"); + } + } + + fn request_body_is_serialized_get_secret(request: &Request) { + assert!( + request + .headers + .contains("x-amz-target: secretsmanager.getsecretvalue"), + "real SDK request target observed" + ); + assert!( + request + .headers + .contains("credential=dummy_loopback_access_key/") + ); + let body: Value = serde_json::from_str(&request.body).expect("SDK JSON request body"); + assert_eq!(body["SecretId"], ARN); + assert!(body.get("VersionStage").is_none()); + assert!(body.get("VersionId").is_none()); + } + + #[cfg(feature = "celld-config")] + #[test] + fn production_runner_composes_celld_config_aws_secret_and_local_secret() { + let root = tempfile::tempdir().expect("isolated mixed-provider process"); + let document = envelope("token", "celld-startup-greeting"); + assert!( + document.len() > 128, + "celld document exceeds the independent AWS payload budget" + ); + let config = FixtureServer::once( + Reply { + headers: "x-celld-store-version: 1\r\n", + ..Reply::ok(document, "application/octet-stream") + }, + true, + ); + let secrets = FixtureServer::once(secrets_reply(TOKEN_SENTINEL), true); + let path = root.path().join("mixed-native-stores.toml"); + fs::write( + &path, + format!( + r#"version = 1 +[config.app] +provider = "celld-config" +endpoint = "{}" +default_key = "settings" +[config.app.access] +mode = "trusted-network" +[config.app.documents.settings] +store = "app-config" +key = "settings" +[secrets.vault] +provider = "aws-secrets-manager" +region = "us-east-1" +[secrets.vault.entries.token] +secret_id = "{ARN}" +[limits.aws] +max_snapshot_bytes = 128 +max_config_document_bytes = 128 +max_secret_value_bytes = 128 +"#, + config.endpoint() + ), + ) + .expect("mixed binding variants"); + let mut child = ChildHost::spawn(&root, &path, secrets.endpoint()); + assert_eq!( + response_body(&child.wait_http("/snapshot")), + "celld-startup-greeting|native-fixture-secret-value" + ); + assert_eq!( + response_body(&child.wait_http("/local-secret")), + LOCAL_SENTINEL + ); + let request = config.take_request(); + assert!( + request + .headers + .starts_with("get /v1/stores/app-config/value?key=settings "), + "scoped config request" + ); + assert!( + !request.headers.contains("authorization:"), + "unrelated Agent token must not enter trusted celld transport" + ); + request_body_is_serialized_get_secret(&secrets.take_request()); + child.stop(); + assert!(config.finish().is_some()); + assert!(secrets.finish().is_some()); + } + + #[test] + fn production_runner_loads_envelope_and_real_sdk_secret_before_serving_mixed_stores() { + let root = tempfile::tempdir().expect("temporary process root"); + let document = envelope("token", "startup-greeting"); + let agent = FixtureServer::once(agent_reply(&document, 200), true); + let secrets = FixtureServer::once(secrets_reply(TOKEN_SENTINEL), true); + let bindings = root.path().join("native-stores.toml"); + bindings_file(&bindings, agent.endpoint(), None); + let mut child = ChildHost::spawn(&root, &bindings, secrets.endpoint()); + + // The app initializer loads the actual serialized BlobEnvelope through core AppConfig, + // resolves its #[secret] field from the SDK-backed snapshot, and inserts typed state. + let first = child.wait_http("/snapshot"); + assert_eq!( + response_body(&first), + "startup-greeting|native-fixture-secret-value" + ); + let second = child.wait_http("/snapshot"); + assert_eq!( + response_body(&second), + "startup-greeting|native-fixture-secret-value" + ); + let local = child.wait_http("/local-secret"); + assert_eq!(response_body(&local), "native-fixture-local-secret"); + + let agent_request = agent.take_request(); + assert!(agent_request.headers.starts_with( + "get /applications/native-fixture/environments/test/configurations/startup-proof " + )); + assert!( + agent_request + .headers + .contains(&format!("authorization: bearer {AGENT_TOKEN}")) + ); + assert_eq!(agent_request.body, ""); + assert!(agent_request.headers.contains("accept-encoding: identity")); + let secret_request = secrets.take_request(); + request_body_is_serialized_get_secret(&secret_request); + assert_eq!( + agent.finish().expect("Agent call").headers, + agent_request.headers + ); + assert_eq!( + secrets.finish().expect("SDK call").body, + secret_request.body + ); + + // Each fixture accepted exactly one startup GET. Both request-time reads still succeeded + // after their fixture listeners closed, so no request-time AWS call occurred. + child.stop(); + } + + #[test] + fn agent_startup_error_stops_before_secret_client_request_and_listener() { + let root = tempfile::tempdir().expect("temporary process root"); + let agent = FixtureServer::once(agent_reply("AGENT_BODY_SENTINEL", 503), true); + let secrets = FixtureServer::once(denied_secrets_reply(), false); + let bindings = root.path().join("native-stores.toml"); + bindings_file(&bindings, agent.endpoint(), None); + let mut child = ChildHost::spawn(&root, &bindings, secrets.endpoint()); + let output = child.wait_failure(); + assert_safe_error(&output); + child.assert_no_listener(); + assert!(agent.finish().is_some(), "Agent request observed"); + assert!( + secrets.finish().is_none(), + "no SDK request after Agent failure" + ); + } + + #[test] + fn explicit_secret_denial_fails_without_local_fallback_or_listener() { + let root = tempfile::tempdir().expect("temporary process root"); + let document = envelope("token", "startup-greeting"); + let agent = FixtureServer::once(agent_reply(&document, 200), true); + let secrets = FixtureServer::once(denied_secrets_reply(), true); + let bindings = root.path().join("native-stores.toml"); + bindings_file(&bindings, agent.endpoint(), None); + let mut child = ChildHost::spawn(&root, &bindings, secrets.endpoint()); + let output = child.wait_failure(); + assert_safe_error(&output); + child.assert_no_listener(); + assert!(agent.finish().is_some()); + let request = secrets.finish().expect("real SDK request captured"); + request_body_is_serialized_get_secret(&request); + } + + #[test] + fn nested_secret_miss_fails_initializer_before_listener() { + let root = tempfile::tempdir().expect("temporary process root"); + let document = envelope("unmapped-secret-key", "startup-greeting"); + let agent = FixtureServer::once(agent_reply(&document, 200), true); + let secrets = FixtureServer::once(secrets_reply(TOKEN_SENTINEL), true); + let bindings = root.path().join("native-stores.toml"); + bindings_file(&bindings, agent.endpoint(), None); + let mut child = ChildHost::spawn(&root, &bindings, secrets.endpoint()); + let output = child.wait_failure(); + assert_safe_error(&output); + child.assert_no_listener(); + assert!(agent.finish().is_some()); + assert!(secrets.finish().is_some()); + } + + #[test] + fn shared_payload_quota_spans_appconfig_and_secrets_before_readiness() { + let root = tempfile::tempdir().expect("temporary process root"); + let document = envelope("token", "startup-greeting"); + let secret = TOKEN_SENTINEL; + let aggregate_cap = document.len().max(secret.len()); + let combined_payload_bytes = document + .len() + .checked_add(secret.len()) + .expect("fixture payload lengths fit usize"); + assert!(aggregate_cap < combined_payload_bytes); + let agent = FixtureServer::once(agent_reply(&document, 200), true); + let secrets = FixtureServer::once(secrets_reply(secret), true); + let bindings = root.path().join("native-stores.toml"); + bindings_file( + &bindings, + agent.endpoint(), + Some((aggregate_cap, document.len(), secret.len())), + ); + let mut child = ChildHost::spawn(&root, &bindings, secrets.endpoint()); + let output = child.wait_failure(); + assert_safe_error(&output); + assert!( + output.contains("size limit"), + "aggregate quota category: {output}" + ); + child.assert_no_listener(); + assert!(agent.finish().is_some(), "config snapshot charged first"); + assert!( + secrets.finish().is_some(), + "secret response fetched before aggregate charge" + ); + } +} diff --git a/crates/edgezero-adapter-axum/tests/native_celld_startup.rs b/crates/edgezero-adapter-axum/tests/native_celld_startup.rs new file mode 100644 index 000000000..2db75cd60 --- /dev/null +++ b/crates/edgezero-adapter-axum/tests/native_celld_startup.rs @@ -0,0 +1,423 @@ +//! Deterministic loopback HTTP fixtures through the public native runner. +//! This is not real celld, TLS, fleet, persistence or generated-consumer acceptance. + +#![cfg(all(feature = "celld-config", feature = "celld-kv"))] +#![expect( + clippy::arbitrary_source_item_ordering, + reason = "fixture helpers precede startup scenarios" +)] + +#[cfg(test)] +mod tests { + use std::fs; + use std::io::{ErrorKind, Read, Write as _}; + use std::net::{SocketAddr, TcpListener, TcpStream}; + use std::path::Path; + use std::process::{Child, Command, Stdio}; + use std::sync::atomic::{AtomicBool, Ordering}; + use std::sync::{Arc, Mutex}; + use std::thread::{self, JoinHandle}; + use std::time::{Duration, Instant}; + + use edgezero_core::blob_envelope::BlobEnvelope; + use serde_json::json; + + const WAIT: Duration = Duration::from_secs(12); + + struct ScopedFixture { + address: SocketAddr, + config: Arc)>>, + gate: Arc, + requests: Arc>>, + stop: Arc, + worker: Option>, + } + + impl ScopedFixture { + fn new(config_status: u16, document: Vec, gated: bool) -> Self { + let listener = TcpListener::bind("127.0.0.1:0").expect("loopback fixture"); + listener.set_nonblocking(true).expect("nonblocking fixture"); + let address = listener.local_addr().expect("fixture address"); + let config = Arc::new(Mutex::new((config_status, document))); + let requests = Arc::new(Mutex::new(Vec::new())); + let gate = Arc::new(AtomicBool::new(!gated)); + let stop = Arc::new(AtomicBool::new(false)); + let worker = { + let worker_config = Arc::clone(&config); + let worker_requests = Arc::clone(&requests); + let worker_gate = Arc::clone(&gate); + let worker_stop = Arc::clone(&stop); + thread::spawn(move || { + let deadline = Instant::now().checked_add(WAIT).expect("fixture deadline"); + while !worker_stop.load(Ordering::Relaxed) && Instant::now() < deadline { + match listener.accept() { + Ok((mut stream, peer)) => { + assert!(peer.ip().is_loopback()); + stream + .set_read_timeout(Some(Duration::from_secs(2))) + .expect("bounded read"); + stream + .set_write_timeout(Some(Duration::from_secs(2))) + .expect("bounded write"); + let request = read_request(&mut stream); + assert!(!request.to_ascii_lowercase().contains("authorization:")); + let line = request.lines().next().expect("request line").to_owned(); + worker_requests + .lock() + .expect("request log") + .push(line.clone()); + let (status, body) = if line.starts_with( + "GET /v1/stores/config-fixture/value?key=settings ", + ) { + while !worker_gate.load(Ordering::Relaxed) + && !worker_stop.load(Ordering::Relaxed) + && Instant::now() < deadline + { + thread::sleep(Duration::from_millis(5)); + } + worker_config.lock().expect("document").clone() + } else { + assert!( + line.starts_with( + "GET /v1/stores/state-fixture/value?key=value " + ), + "unexpected scoped request" + ); + (200, b"live-value".to_vec()) + }; + let content_type = if status == 200 { + "application/octet-stream" + } else { + "application/json" + }; + // Failure tests may close early after checking Content-Length. + let _sent = write!(stream, "HTTP/1.1 {status} Fixture\r\nContent-Length: {}\r\nContent-Type: {content_type}\r\nx-celld-store-version: 1\r\nConnection: close\r\n\r\n", body.len()) + .and_then(|()| stream.write_all(&body)); + } + Err(error) if error.kind() == ErrorKind::WouldBlock => { + thread::sleep(Duration::from_millis(5)); + } + Err(error) => panic!("fixture accept failed: {error}"), + } + } + }) + }; + Self { + address, + config, + gate, + requests, + stop, + worker: Some(worker), + } + } + + fn endpoint(&self) -> String { + format!("http://{}", self.address) + } + + fn count(&self) -> usize { + self.requests.lock().expect("request log").len() + } + + fn wait_request(&self) { + wait_until(|| self.count() != 0, "scoped request"); + } + } + + impl Drop for ScopedFixture { + fn drop(&mut self) { + self.stop.store(true, Ordering::Relaxed); + self.gate.store(true, Ordering::Relaxed); + if let Some(worker) = self.worker.take() { + worker.join().expect("fixture thread"); + } + } + } + + fn read_request(stream: &mut TcpStream) -> String { + let mut bytes = Vec::new(); + let mut buffer = [0_u8; 1_024]; + while !bytes.windows(4).any(|part| part == b"\r\n\r\n") { + let count = stream.read(&mut buffer).expect("request read"); + assert_ne!(count, 0, "complete request headers"); + bytes.extend_from_slice(buffer.get(..count).expect("read fits buffer")); + assert!(bytes.len() <= 0x4000, "bounded request"); + } + String::from_utf8(bytes).expect("HTTP fixture request") + } + + struct Host { + child: Child, + address: SocketAddr, + readers: Vec>>, + } + + impl Host { + fn spawn(root: &Path, bindings: &Path, initializer_gate: bool, supply_state: bool) -> Self { + let listener = TcpListener::bind("127.0.0.1:0").expect("reserve app address"); + let address = listener.local_addr().expect("app address"); + drop(listener); + let mut command = Command::new(env!("CARGO_BIN_EXE_edgezero-native-celld-host")); + command + .env_clear() + .current_dir(root) + .env("AWS_EC2_METADATA_DISABLED", "true") + .env("EDGEZERO__STORE_BINDINGS_FILE", bindings) + .env("EDGEZERO__ADAPTER__HOST", "127.0.0.1") + .env("EDGEZERO__ADAPTER__PORT", address.port().to_string()) + .env("EDGEZERO__LOGGING__LEVEL", "off") + .env( + "FIXTURE_INITIALIZER_ENTERED", + root.join("initializer-entered"), + ) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + if initializer_gate { + command.env("FIXTURE_INITIALIZER_GATE", root.join("initializer-release")); + } + if supply_state { + command.env("FIXTURE_DEFERRED_MARKER", root.join("deferred-polled")); + } + let mut child = command.spawn().expect("public runner child"); + let streams: [Box; 2] = [ + Box::new(child.stdout.take().expect("stdout")), + Box::new(child.stderr.take().expect("stderr")), + ]; + let readers = streams + .into_iter() + .map(|mut stream| { + thread::spawn(move || { + let mut bytes = Vec::new(); + stream.read_to_end(&mut bytes).expect("child output"); + bytes + }) + }) + .collect(); + Self { + child, + address, + readers, + } + } + + fn no_listener(&self) { + TcpStream::connect_timeout(&self.address, Duration::from_millis(40)) + .expect_err("no listener before readiness"); + } + + fn get(&mut self, path: &str) -> String { + let deadline = Instant::now() + .checked_add(WAIT) + .expect("readiness deadline"); + loop { + assert!( + self.child.try_wait().expect("child status").is_none(), + "host exited before readiness" + ); + if let Ok(mut stream) = + TcpStream::connect_timeout(&self.address, Duration::from_millis(40)) + { + stream + .set_read_timeout(Some(Duration::from_secs(2))) + .expect("bounded app read"); + write!( + stream, + "GET {path} HTTP/1.1\r\nHost: 127.0.0.1\r\nConnection: close\r\n\r\n" + ) + .expect("app request"); + let mut response = String::new(); + stream.read_to_string(&mut response).expect("app response"); + assert!( + response.starts_with("HTTP/1.1 200"), + "successful fixture response" + ); + return response + .split_once("\r\n\r\n") + .expect("response headers") + .1 + .to_owned(); + } + assert!(Instant::now() < deadline, "host readiness timeout"); + thread::sleep(Duration::from_millis(10)); + } + } + + fn failure(&mut self) -> String { + let deadline = Instant::now().checked_add(WAIT).expect("failure deadline"); + loop { + if let Some(status) = self.child.try_wait().expect("child status") { + assert!(!status.success(), "expected startup failure"); + self.child.wait().expect("reap failed host"); + let mut bytes = Vec::new(); + for reader in self.readers.drain(..) { + bytes.extend(reader.join().expect("output reader")); + } + return String::from_utf8_lossy(&bytes).into_owned(); + } + assert!(Instant::now() < deadline, "startup failure timeout"); + thread::sleep(Duration::from_millis(10)); + } + } + } + + impl Drop for Host { + fn drop(&mut self) { + if self.child.try_wait().ok().flatten().is_none() { + let _killed = self.child.kill(); + } + let _reaped = self.child.wait(); + for reader in self.readers.drain(..) { + let _joined = reader.join(); + } + } + } + + fn wait_until bool>(ready: Ready, label: &str) { + let deadline = Instant::now().checked_add(WAIT).expect("fixture deadline"); + while !ready() { + assert!(Instant::now() < deadline, "timeout waiting for {label}"); + thread::sleep(Duration::from_millis(10)); + } + } + + fn envelope(greeting: &str) -> Vec { + serde_json::to_vec(&BlobEnvelope::new( + json!({"greeting": greeting}), + "2026-10-15T12:00:00Z".to_owned(), + )) + .expect("typed config envelope") + } + + fn bindings(endpoint: &str) -> String { + format!( + "version=1 +[config.app] +provider='celld-config' +endpoint='{endpoint}' +default_key='settings' +[config.app.access] +mode='trusted-network' +[config.app.documents.settings] +store='config-fixture' +key='settings' +[kv.state] +provider='celld-kv' +endpoint='{endpoint}' +store='state-fixture' +[kv.state.access] +mode='trusted-network' +" + ) + } + + #[test] + fn public_runner_gates_preparation_and_initializer_then_keeps_snapshot_and_live_kv() { + let root = tempfile::tempdir().expect("isolated child root"); + let fixture = ScopedFixture::new(200, envelope("first-greeting"), true); + let path = root.path().join("bindings.toml"); + fs::write(&path, bindings(&fixture.endpoint())).expect("bindings"); + let mut host = Host::spawn(root.path(), &path, true, false); + fixture.wait_request(); + host.no_listener(); + assert!(!root.path().join("initializer-entered").exists()); + // File settings were captured before the gated fetch; later edits cannot change sources/defaults. + fs::write(&path, "invalid later file contents").expect("replace captured file"); + fixture.gate.store(true, Ordering::Relaxed); + wait_until( + || root.path().join("initializer-entered").is_file(), + "initializer entry", + ); + host.no_listener(); + assert_eq!(fixture.count(), 1, "KV setup performs no remote probe"); + fs::write(root.path().join("initializer-release"), b"ready").expect("release initializer"); + assert_eq!(host.get("/snapshot"), "first-greeting"); + *fixture.config.lock().expect("edit remote document") = (200, envelope("second-greeting")); + assert_eq!(host.get("/snapshot"), "first-greeting"); + assert_eq!(fixture.count(), 1, "snapshot reads never fetch again"); + assert_eq!(host.get("/kv"), "live-value"); + assert_eq!(host.get("/kv"), "live-value"); + assert_eq!(fixture.count(), 3); + drop(host); + fs::write(&path, bindings(&fixture.endpoint())).expect("fresh process bindings"); + let mut restarted = Host::spawn(root.path(), &path, false, false); + assert_eq!(restarted.get("/snapshot"), "second-greeting"); + assert_eq!(fixture.count(), 4, "fresh preparation adopts remote edits"); + } + + #[test] + fn config_failures_never_initialize_listen_or_fall_back() { + for (status, document, policy, category) in [ + ( + 404, + json!({"error": "not_found"}).to_string().into_bytes(), + "", + "missing", + ), + (200, vec![0xff], "", "malformed"), + ( + 200, + envelope("document-too-large"), + "\n[limits.celld]\nmax_config_document_bytes=4\n", + "size limit", + ), + ] { + let root = tempfile::tempdir().expect("root"); + let fixture = ScopedFixture::new(status, document, false); + let path = root.path().join("bindings.toml"); + fs::write(&path, format!("{}{policy}", bindings(&fixture.endpoint()))) + .expect("bindings"); + let mut host = Host::spawn(root.path(), &path, false, false); + let error = host.failure(); + host.no_listener(); + assert!(!root.path().join("initializer-entered").exists()); + assert_eq!(fixture.count(), 1); + assert!(error.contains(category), "closed failure category"); + assert!(!error.contains("config-fixture")); + assert!(!error.contains(&fixture.endpoint())); + } + } + + #[test] + fn supplied_remote_source_conflict_never_polls_or_fetches() { + let root = tempfile::tempdir().expect("root"); + let fixture = ScopedFixture::new(200, envelope("unused-greeting"), false); + let path = root.path().join("bindings.toml"); + fs::write(&path, bindings(&fixture.endpoint())).expect("bindings"); + let mut host = Host::spawn(root.path(), &path, false, true); + let error = host.failure(); + host.no_listener(); + assert!(error.contains("duplicate")); + assert_eq!(fixture.count(), 0); + assert!(!root.path().join("initializer-entered").exists()); + assert!(!root.path().join("deferred-polled").exists()); + } + + #[test] + fn invalid_later_kv_credentials_fail_before_any_config_request() { + let root = tempfile::tempdir().expect("root"); + let fixture = ScopedFixture::new(200, envelope("unused-greeting"), false); + let path = root.path().join("bindings.toml"); + let input = bindings(&fixture.endpoint()) + .replace( + &format!( + "[kv.state]\nprovider='celld-kv'\nendpoint='{}'", + fixture.endpoint() + ), + "[kv.state]\nprovider='celld-kv'\nendpoint='https://unreachable.invalid'", + ) + .replace( + "[kv.state.access]\nmode='trusted-network'", + "[kv.state.access]\nmode='authenticated'\ntoken_env='ABSENT_TOKEN'", + ); + fs::write(&path, input).expect("bindings"); + let mut host = Host::spawn(root.path(), &path, false, false); + let error = host.failure(); + host.no_listener(); + assert!(error.contains("celld token")); + assert!(!root.path().join("initializer-entered").exists()); + assert_eq!(fixture.count(), 0); + assert!(!error.contains("ABSENT_TOKEN")); + assert!(!error.contains("unreachable.invalid")); + } +} diff --git a/crates/edgezero-adapter-axum/tests/production_hosting.rs b/crates/edgezero-adapter-axum/tests/production_hosting.rs new file mode 100644 index 000000000..78ad7cfed --- /dev/null +++ b/crates/edgezero-adapter-axum/tests/production_hosting.rs @@ -0,0 +1,2233 @@ +//! Real local processes, public entrypoints and ordinary main termination output. +#![cfg(all(feature = "axum", feature = "test-utils", unix))] + +#[cfg(test)] +#[path = "fixtures/native_proxy.rs"] +mod native_proxy; + +#[cfg(test)] +#[expect( + clippy::arbitrary_source_item_ordering, + reason = "subprocess fixtures group setup, process ownership and acceptance scenarios" +)] +mod tests { + use super::native_proxy::{Assertions, LocalProxy, verified_tls_request}; + use std::collections::HashSet; + use std::ffi::OsString; + use std::fs; + use std::io::{self, BufRead as _, BufReader, Read, Write as _}; + use std::net::{IpAddr, Ipv4Addr, SocketAddr, TcpListener, TcpStream}; + use std::os::unix::fs::{MetadataExt as _, PermissionsExt as _, symlink}; + use std::os::unix::process::ExitStatusExt as _; + use std::process::{Child, Command, ExitStatus, Stdio}; + use std::sync::{Arc, Mutex, mpsc}; + use std::thread::{self, JoinHandle}; + use std::time::{Duration, Instant}; + + use edgezero_core::blob_envelope::BlobEnvelope; + use serde_json::{Value, json}; + use tempfile::TempDir; + + const WAIT: Duration = Duration::from_secs(5); + + struct Roots { + config: TempDir, + cwd: TempDir, + data: TempDir, + } + + impl Roots { + fn new() -> Self { + let roots = Self { + config: tempfile::tempdir().expect("config root"), + cwd: tempfile::tempdir().expect("unrelated cwd"), + data: tempfile::tempdir().expect("data root"), + }; + roots.map(&json!({"settings": blob("configured-greeting", "fixture_token")})); + roots + } + + fn map(&self, value: &Value) { + fs::write( + self.config.path().join("local-config-settings.json"), + serde_json::to_vec(value).expect("JSON"), + ) + .expect("config fixture"); + } + } + + fn blob(greeting: &str, token: &str) -> String { + serde_json::to_string(&BlobEnvelope::new( + json!({"greeting": greeting, "token": token}), + "2026-10-05T00:00:00Z".to_owned(), + )) + .expect("envelope") + } + + fn address() -> SocketAddr { + TcpListener::bind("127.0.0.1:0") + .expect("port reservation") + .local_addr() + .expect("address") + } + + fn command(mode: &str, roots: &Roots, addr: SocketAddr) -> Command { + let mut command = Command::new(env!("CARGO_BIN_EXE_edgezero-hosting-fixture")); + command + .env_clear() + .current_dir(roots.cwd.path()) + .env("FIXTURE_MODE", mode) + .env("FIXTURE_BIND", addr.to_string()) + .env("FIXTURE_CONFIG_ROOT", roots.config.path()) + .env("FIXTURE_DATA_ROOT", roots.data.path()) + .env("EDGEZERO__ADAPTER__HOST", "127.0.0.1") + .env("EDGEZERO__ADAPTER__PORT", addr.port().to_string()) + .env("EDGEZERO__ADAPTER__SHUTDOWN_GRACE_SECONDS", "1") + .env("EDGEZERO__ADAPTER__CONFIG_DIR", roots.config.path()) + .env("EDGEZERO__ADAPTER__DATA_DIR", roots.data.path()) + .env("EDGEZERO__STORES__KV__FIRST__NAME", "shared-fixture") + .env("EDGEZERO__STORES__KV__SECOND__NAME", "shared-fixture") + .env("fixture_token", "resolved-fixture-token"); + command + } + + struct Server { + addr: SocketAddr, + child: Child, + events: mpsc::Receiver, + output: Arc>, + readers: Vec>, + } + + impl Server { + fn spawn(mut command: Command, addr: SocketAddr) -> Self { + let mut child = command + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .expect("fixture process"); + let output = Arc::new(Mutex::new(String::new())); + let (sender, events) = mpsc::channel(); + let stdout = child.stdout.take().expect("stdout"); + let stderr = child.stderr.take().expect("stderr"); + let mut readers = Vec::new(); + let streams: [Box; 2] = [Box::new(stdout), Box::new(stderr)]; + for stream in streams { + let captured = Arc::clone(&output); + let stream_sender = sender.clone(); + readers.push(thread::spawn(move || { + for result in BufReader::new(stream).lines() { + let Ok(line) = result else { break }; + let mut captured_output = captured.lock().expect("output"); + captured_output.push_str(&line); + captured_output.push('\n'); + drop(captured_output); + let _sent = stream_sender.send(line); + } + })); + } + Self { + addr, + child, + events, + output, + readers, + } + } + + fn log(&self) -> String { + self.output.lock().expect("output").clone() + } + + fn ready(&mut self) { + let deadline = Instant::now().checked_add(WAIT).expect("startup timer"); + loop { + assert!( + Instant::now() < deadline, + "startup deadline: {}", + self.log() + ); + assert!( + self.child.try_wait().expect("status").is_none(), + "early process failure: {}", + self.log() + ); + if let Ok(response) = self.request("/ready") + && response.starts_with("HTTP/1.1 200") + { + return; + } + thread::sleep(Duration::from_millis(5)); + } + } + + fn request(&self, path: &str) -> io::Result { + let mut socket = self.socket()?; + write!( + socket, + "GET {path} HTTP/1.1\r\nHost: fixture\r\nConnection: close\r\n\r\n" + )?; + let mut response = String::new(); + socket.read_to_string(&mut response)?; + Ok(response) + } + + fn signal(&self, name: &str) { + assert!( + Command::new("kill") + .arg(format!("-{name}")) + .arg(self.child.id().to_string()) + .status() + .expect("send signal") + .success() + ); + } + + fn socket(&self) -> io::Result { + let socket = TcpStream::connect_timeout(&self.addr, Duration::from_millis(50))?; + socket.set_read_timeout(Some(WAIT))?; + socket.set_write_timeout(Some(WAIT))?; + Ok(socket) + } + + fn wait(&mut self) -> ExitStatus { + let deadline = Instant::now().checked_add(WAIT).expect("exit timer"); + loop { + if let Some(status) = self.child.try_wait().expect("process status") { + for reader in self.readers.drain(..) { + reader.join().expect("output reader"); + } + return status; + } + assert!( + Instant::now() < deadline, + "process exit deadline: {}", + self.log() + ); + thread::sleep(Duration::from_millis(5)); + } + } + + fn wait_line(&self, expected: &str) { + let deadline = Instant::now().checked_add(WAIT).expect("event timer"); + loop { + let remaining = deadline.saturating_duration_since(Instant::now()); + let line = self.events.recv_timeout(remaining).unwrap_or_else(|error| { + panic!("waiting for {expected}: {error}; {}", self.log()) + }); + if line == expected { + return; + } + } + } + } + + impl Drop for Server { + fn drop(&mut self) { + let _killed = self.child.kill(); + let _status = self.child.wait(); + for reader in self.readers.drain(..) { + let _joined = reader.join(); + } + } + } + + fn failed(command: Command, addr: SocketAddr, category: &str) -> String { + let mut server = Server::spawn(command, addr); + assert!(!server.wait().success()); + let log = server.log(); + assert!(log.contains(category), "missing category {category}: {log}"); + assert!( + !log.contains("SENTINEL"), + "startup diagnostic leaked fixture contents: {log}" + ); + assert!(!log.contains("INITIALIZED")); + TcpStream::connect_timeout(&addr, Duration::from_millis(50)) + .expect_err("failed startup never accepts connections"); + log + } + + fn post_json(server: &Server, path: &str, body: &str) -> String { + let response = raw_request( + server.addr, + &format!( + "POST {path} HTTP/1.1\r\nHost: fixture\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ), + ); + let (head, payload) = response + .split_once("\r\n\r\n") + .expect("usable response head"); + let length = head + .lines() + .find_map(|line| { + let (key, value) = line.split_once(':')?; + key.eq_ignore_ascii_case("content-length") + .then(|| value.trim().parse::().expect("response length")) + }) + .expect("finite response framing"); + assert_eq!(payload.len(), length, "complete response: {response}"); + response + } + + fn captured_json(log: &str, marker: &str) -> Vec { + log.lines() + .filter_map(|line| line.strip_prefix(marker)) + .map(|value| serde_json::from_str(value).expect("fixture JSON marker")) + .collect() + } + + fn observations(server: &Server) -> Vec { + captured_json(&server.log(), "NATIVE_OBSERVATION ") + } + + fn framework_logs(server: &Server) -> Vec { + captured_json(&server.log(), "FRAMEWORK_LOG ") + } + + fn wait_for(server: &Server, predicate: impl Fn(&str) -> bool) { + let deadline = Instant::now().checked_add(WAIT).expect("observation timer"); + loop { + let log = server.log(); + if predicate(&log) { + return; + } + assert!(Instant::now() < deadline, "observation deadline: {log}"); + thread::sleep(Duration::from_millis(5)); + } + } + + fn route_record(server: &Server, route: &str) -> Value { + wait_for(server, |log| { + captured_json(log, "NATIVE_OBSERVATION ") + .iter() + .any(|record| record["route"] == route) + }); + let records = observations(server) + .into_iter() + .filter(|record| record["route"] == route) + .collect::>(); + assert_eq!(records.len(), 1, "one terminal owner: {}", server.log()); + records.into_iter().next().expect("route record") + } + + fn raw_request(addr: SocketAddr, request: &str) -> String { + let mut socket = TcpStream::connect_timeout(&addr, WAIT).expect("wire connection"); + socket.set_read_timeout(Some(WAIT)).expect("read timeout"); + socket.set_write_timeout(Some(WAIT)).expect("write timeout"); + socket.write_all(request.as_bytes()).expect("wire request"); + let mut response = String::new(); + socket.read_to_string(&mut response).expect("wire response"); + response + } + + fn response_json(response: &str) -> Value { + let (_, body) = response.split_once("\r\n\r\n").expect("response head"); + serde_json::from_str(body).expect("finite fixture JSON response") + } + + fn metadata_request(headers: &str) -> String { + format!( + "GET /native-metadata HTTP/1.1\r\nHost: direct.example:8080\r\n{headers}Connection: close\r\n\r\n" + ) + } + + fn assert_metadata(value: &Value, client: &str, host: &str, scheme: &str, result: &str) { + let facts = &value["handler"]; + assert_eq!(value["admission"], *facts, "admission view"); + assert_eq!(value["middleware"], *facts, "middleware view"); + assert_eq!(facts["client"], client); + assert_eq!(facts["host"], host); + assert_eq!(facts["scheme"], scheme); + assert_eq!(facts["forwarding_result"], result); + assert_eq!(facts["host_header"], "direct.example:8080"); + assert_eq!(facts["uri"], "/native-metadata"); + assert_eq!(facts["raw_forwarding_absent"], true); + assert_eq!(value["forwarded_host"], host); + assert_eq!(value["direct_context_peer"], facts["peer"]); + let id = facts["request_id"].as_str().expect("application ID"); + assert_eq!(id.len(), 64); + assert!(id.bytes().all(|byte| byte.is_ascii_hexdigit())); + } + + fn assert_no_diagnostic_headers(response: &str) { + let (head, _) = response.split_once("\r\n\r\n").expect("response head"); + for line in head.lines().skip(1) { + let (name, _) = line.split_once(':').expect("response header"); + let normalized_name = name.to_ascii_lowercase(); + assert!( + !normalized_name.contains("request-id") + && !normalized_name.starts_with("x-edgezero") + ); + assert!(!matches!( + normalized_name.as_str(), + "traceparent" | "tracestate" | "server-timing" + )); + } + } + + fn native_command( + mode: &str, + roots: &Roots, + addr: SocketAddr, + networks: &str, + family: &str, + ) -> Command { + let mut launch = command(mode, roots, addr); + launch + .env("EDGEZERO__ADAPTER__TRUSTED_PROXY_CIDRS", networks) + .env("EDGEZERO__ADAPTER__FORWARDING_HEADER_FAMILY", family); + launch + } + + fn assert_final_zero(server: &Server) { + let snapshots = captured_json(&server.log(), "FINAL_SNAPSHOT "); + assert_eq!( + snapshots, + vec![ + json!({"active_requests": 0_u64, "active_connections": 0_u64, "idle_connections": 0_u64}) + ] + ); + } + + fn assert_hook_order(server: &Server, id: &str) { + let log = server.log(); + let completion = log + .find(&format!("APP_COMPLETION {id}\n")) + .expect("application completion"); + let native_line = log + .lines() + .find(|line| line.starts_with("NATIVE_OBSERVATION ") && line.contains(id)) + .expect("native observation"); + let native = log.find(native_line).expect("native offset"); + let observer = log + .get(native..) + .expect("native suffix") + .find("APP_OBSERVER ") + .expect("application observer"); + assert!(completion < native); + assert!(observer > 0); + assert_eq!(log.matches(&format!("APP_COMPLETION {id}\n")).count(), 1); + assert_eq!( + observations(server) + .iter() + .filter(|record| record["id"] == id) + .count(), + 1 + ); + } + + fn assert_record_log(server: &Server, record: &Value) { + let id = record["id"].as_str().expect("record ID"); + let request_logs = framework_logs(server) + .into_iter() + .filter(|entry| { + let message = entry["message"].as_str().expect("framework message"); + (message.starts_with("request_terminal ") + || message.starts_with("request_ingress ")) + && message.contains(id) + }) + .collect::>(); + assert_eq!( + request_logs.len(), + 1, + "facade cardinality: {}", + server.log() + ); + let entry = request_logs + .into_iter() + .next() + .expect("request facade entry"); + assert_eq!(entry["level"], record["level"]); + let message = entry["message"].as_str().expect("safe request text"); + assert!(message.contains(&format!( + "method={}", + record["method"].as_str().expect("method") + ))); + assert!(message.contains("duration_us=")); + assert!(message.contains("bytes_written=") || record["bytes_written"].is_null()); + } + + #[test] + fn native_direct_boundary_strips_spoofs_and_accounts_the_original_head() { + let roots = Roots::new(); + let addr = address(); + let mut server = Server::spawn(command("diagnostics", &roots, addr), addr); + server.ready(); + let spoof = "Forwarded: for=198.51.100.9;host=FORWARDED_SENTINEL;proto=https\r\nX-Forwarded-For: 203.0.113.9\r\nX-Forwarded-Host: XFH_SENTINEL\r\nX-Forwarded-Proto: https\r\nX-Forwarded-Secret: HEADER_SENTINEL\r\nX-Request-ID: VISITOR_ID_SENTINEL\r\n"; + let request = metadata_request(spoof); + let response = raw_request(addr, &request); + assert_no_diagnostic_headers(&response); + let value = response_json(&response); + assert_metadata( + &value, + "127.0.0.1", + "direct.example:8080", + "http", + "untrusted_peer", + ); + for field in ["client_source", "host_source", "scheme_source"] { + assert_eq!(value["handler"][field], "direct"); + } + let peer: SocketAddr = value["handler"]["peer"] + .as_str() + .expect("owned peer") + .parse() + .expect("socket peer"); + assert_eq!(peer.ip(), IpAddr::V4(Ipv4Addr::LOCALHOST)); + let expected_bytes = request + .split("\r\n") + .skip(1) + .filter(|line| !line.is_empty()) + .map(|line| line.len().checked_add(2).expect("normalized line bytes")) + .try_fold(2_usize, usize::checked_add) + .expect("normalized head bytes"); + assert_eq!(value["normalized"]["header_bytes"], expected_bytes); + assert_eq!(value["normalized"]["header_count"], 8_u64); + assert_eq!( + value["normalized"]["target_bytes"], + "/native-metadata".len() + ); + server.signal("TERM"); + assert!(server.wait().success()); + assert_final_zero(&server); + let record = route_record(&server, "/native-metadata"); + assert_eq!(record["id"], value["handler"]["request_id"]); + assert_record_log(&server, &record); + for entry in framework_logs(&server) { + let message = entry["message"].as_str().expect("framework message"); + assert!(!message.contains("SENTINEL")); + assert!(!message.contains("127.0.0.1") && !message.contains("direct.example")); + } + } + + #[test] + fn native_source_bound_single_proxy_and_mapped_cidr_match_only_owned_peers() { + for networks in [ + "127.0.0.2/32", + "::ffff:127.0.0.2/128", + "::ffff:127.0.0.2/127", + "127.0.0.3/32", + ] { + let roots = Roots::new(); + let addr = address(); + let mut server = Server::spawn( + native_command("diagnostics", &roots, addr, networks, "forwarded"), + addr, + ); + server.ready(); + let proxy = LocalProxy::start( + addr, + Ipv4Addr::new(127, 0, 0, 2), + Assertions::AppendForwarded { + host: "public.example", + scheme: "https", + }, + false, + ); + let response = raw_request( + proxy.addr(), + &metadata_request( + "Forwarded: for=198.51.100.99;host=attacker.example;proto=http\r\nX-Forwarded-Host: conflicting.example\r\n", + ), + ); + let value = response_json(&response); + let trusted = networks != "127.0.0.3/32"; + assert_metadata( + &value, + if trusted { "127.0.0.1" } else { "127.0.0.2" }, + if trusted { + "public.example" + } else { + "direct.example:8080" + }, + if trusted { "https" } else { "http" }, + if trusted { + "accepted" + } else { + "untrusted_peer" + }, + ); + let peer: SocketAddr = value["handler"]["peer"] + .as_str() + .expect("proxy peer") + .parse() + .expect("peer address"); + assert_eq!(peer.ip(), IpAddr::V4(Ipv4Addr::new(127, 0, 0, 2))); + assert_no_diagnostic_headers(&response); + server.signal("TERM"); + assert!(server.wait().success()); + assert_final_zero(&server); + } + } + + #[test] + fn native_two_proxy_chain_stops_at_untrusted_hop_and_gateway_replaces_prefix() { + for (networks, boundary) in [ + ("127.0.0.2/32,127.0.0.3/32", "127.0.0.1"), + ("127.0.0.3/32", "127.0.0.2"), + ] { + let roots = Roots::new(); + let addr = address(); + let mut server = Server::spawn( + native_command("diagnostics", &roots, addr, networks, "forwarded"), + addr, + ); + server.ready(); + let ingress = LocalProxy::start( + addr, + Ipv4Addr::new(127, 0, 0, 3), + Assertions::AppendForwarded { + host: "final.example", + scheme: "https", + }, + false, + ); + let first = LocalProxy::start( + ingress.addr(), + Ipv4Addr::new(127, 0, 0, 2), + Assertions::AppendForwarded { + host: "earlier.example", + scheme: "http", + }, + false, + ); + let value = response_json(&raw_request( + first.addr(), + &metadata_request( + "Forwarded: for=198.51.100.99;host=attacker.example;proto=http\r\n", + ), + )); + assert_metadata(&value, boundary, "final.example", "https", "accepted"); + assert_eq!(value["handler"]["host_source"], "trusted_forwarded"); + assert!( + value["handler"]["peer"] + .as_str() + .expect("last proxy peer") + .starts_with("127.0.0.3:") + ); + server.signal("TERM"); + assert!(server.wait().success()); + assert_final_zero(&server); + } + let roots = Roots::new(); + let addr = address(); + let mut server = Server::spawn( + native_command("diagnostics", &roots, addr, "127.0.0.3/32", "forwarded"), + addr, + ); + server.ready(); + let gateway = LocalProxy::start( + addr, + Ipv4Addr::new(127, 0, 0, 3), + Assertions::ReplaceForwarded { + client: "203.0.113.7".parse().expect("verified gateway client"), + host: "gateway.example", + scheme: "https", + }, + false, + ); + let value = response_json(&raw_request( + gateway.addr(), + &metadata_request( + "Forwarded: for=198.51.100.99;host=attacker.example;proto=http\r\nX-Forwarded-Host: attacker.example\r\n", + ), + )); + assert_metadata( + &value, + "203.0.113.7", + "gateway.example", + "https", + "accepted", + ); + server.signal("TERM"); + assert!(server.wait().success()); + assert_final_zero(&server); + } + + fn assert_missing_unresolved_and_duplicate_family(addr: SocketAddr, family: &str) { + // This source-bound relay preserves deliberate bad inputs rather than sanitizing them. + let gateway = LocalProxy::start( + addr, + Ipv4Addr::new(127, 0, 0, 2), + Assertions::Preserve, + false, + ); + let absent = if family == "forwarded" { + "Forwarded: host=partial.example;proto=https\r\nX-Forwarded-For: 198.51.100.9\r\n" + } else { + "X-Forwarded-Host: partial.example\r\nX-Forwarded-Proto: https\r\nForwarded: for=198.51.100.9\r\n" + }; + let missing_client = response_json(&raw_request(gateway.addr(), &metadata_request(absent))); + assert_metadata( + &missing_client, + "127.0.0.2", + "partial.example", + "https", + if family == "forwarded" { + "unresolved_chain" + } else { + "partial" + }, + ); + for headers in [ + if family == "forwarded" { + "Forwarded: for=unknown;host=gap.example;proto=https\r\n" + } else { + "X-Forwarded-For: unknown\r\nX-Forwarded-Host: gap.example\r\nX-Forwarded-Proto: https\r\n" + }, + if family == "forwarded" { + "Forwarded: for=127.0.0.2;host=gap.example;proto=https\r\n" + } else { + "X-Forwarded-For: 127.0.0.2\r\nX-Forwarded-Host: gap.example\r\nX-Forwarded-Proto: https\r\n" + }, + ] { + let value = response_json(&raw_request(gateway.addr(), &metadata_request(headers))); + assert_metadata( + &value, + "127.0.0.2", + "gap.example", + "https", + "unresolved_chain", + ); + } + let duplicate = if family == "forwarded" { + "Forwarded: for=198.51.100.8;for=203.0.113.9;host=good.example;proto=https\r\n" + } else { + "X-Forwarded-For: 198.51.100.8\r\nX-Forwarded-Host: good.example\r\nX-Forwarded-Host: duplicate.example\r\nX-Forwarded-Proto: https\r\n" + }; + let malformed_duplicate = + response_json(&raw_request(gateway.addr(), &metadata_request(duplicate))); + assert_metadata( + &malformed_duplicate, + "127.0.0.2", + "direct.example:8080", + "http", + "malformed", + ); + } + + #[test] + fn native_selected_families_conflicts_malformed_sets_gaps_and_exhaustion() { + for family in ["forwarded", "x-forwarded"] { + let roots = Roots::new(); + let addr = address(); + let mut server = Server::spawn( + native_command("diagnostics", &roots, addr, "127.0.0.2/32", family), + addr, + ); + server.ready(); + let assertions = if family == "forwarded" { + Assertions::AppendForwarded { + host: "selected.example", + scheme: "https", + } + } else { + Assertions::AppendXForwarded { + host: "selected.example", + scheme: "https", + } + }; + let proxy = LocalProxy::start(addr, Ipv4Addr::new(127, 0, 0, 2), assertions, false); + for (prefix, expected_result, client, host, scheme) in [ + ( + "Forwarded: for=198.51.100.1;host=wrong.example;proto=http\r\nX-Forwarded-For: 203.0.113.1\r\nX-Forwarded-Host: wrong.example\r\nX-Forwarded-Proto: http\r\n", + "accepted", + "127.0.0.1", + "selected.example", + "https", + ), + ( + if family == "forwarded" { + "Forwarded: for=bad-address;host=good.example;proto=https\r\n" + } else { + "X-Forwarded-For: bad-address\r\n" + }, + "malformed", + "127.0.0.2", + "direct.example:8080", + "http", + ), + ( + if family == "forwarded" { + "Forwarded: for=unknown\r\n" + } else { + "X-Forwarded-For: unknown\r\n" + }, + "accepted", + "127.0.0.1", + "selected.example", + "https", + ), + ] { + let value = response_json(&raw_request(proxy.addr(), &metadata_request(prefix))); + assert_metadata(&value, client, host, scheme, expected_result); + } + assert_missing_unresolved_and_duplicate_family(addr, family); + server.signal("TERM"); + assert!(server.wait().success()); + assert_final_zero(&server); + let records = observations(&server) + .into_iter() + .filter(|record| record["route"] == "/native-metadata") + .collect::>(); + assert_eq!(records.len(), 7); + for record in records { + assert_record_log(&server, &record); + if record["forwarding_result"] == "malformed" { + assert_eq!(record["level"], "WARN"); + } + } + } + } + + #[test] + fn native_verified_tls_ingress_overwrites_public_assertions_and_rejects_wrong_name() { + let roots = Roots::new(); + let addr = address(); + let mut server = Server::spawn( + native_command("diagnostics", &roots, addr, "127.0.0.2/32", "forwarded"), + addr, + ); + server.ready(); + let proxy = LocalProxy::start( + addr, + Ipv4Addr::new(127, 0, 0, 2), + Assertions::AppendForwarded { + host: "native-proxy.test", + scheme: "https", + }, + true, + ); + let request = metadata_request( + "Forwarded: for=198.51.100.99;host=attacker.example;proto=http\r\nX-Forwarded-Host: attacker.example\r\nX-Forwarded-Proto: http\r\n", + ); + let response = verified_tls_request(proxy.addr(), request.as_bytes(), "native-proxy.test") + .expect("CA and hostname verified TLS"); + let value = response_json(&response); + assert_metadata( + &value, + "127.0.0.1", + "native-proxy.test", + "https", + "accepted", + ); + assert_eq!(value["handler"]["scheme_source"], "trusted_forwarded"); + assert_no_diagnostic_headers(&response); + let wrong_name = verified_tls_request(proxy.addr(), request.as_bytes(), "wrong-name.test") + .expect_err("wrong server name must fail verification"); + assert_eq!( + wrong_name.kind(), + io::ErrorKind::InvalidData, + "certificate verification failure, not a connection timeout" + ); + assert!( + format!("{wrong_name:?}").contains("NotValidForName"), + "hostname verification must be the rejecting boundary" + ); + server.signal("TERM"); + assert!(server.wait().success()); + assert_final_zero(&server); + assert_eq!( + observations(&server) + .iter() + .filter(|record| record["route"] == "/native-metadata") + .count(), + 1 + ); + } + + fn assert_native_error_record( + server: &Server, + path: &str, + status: u16, + failure: &str, + level: &str, + ) { + if path.starts_with("/PRIVATE_PATH") { + return; + } + let route = path.split('?').next().expect("registered route"); + let record = route_record(server, route); + assert_eq!(record["status"], status); + assert_eq!(record["level"], level); + if !failure.is_empty() { + assert_eq!(record["failure"], failure); + } + if path == "/native-status503" { + assert!(record["failure"].is_null(), "status alone is not rejection"); + } + if matches!(path, "/native-conversion" | "/native-expired") { + assert_eq!(record["body_kind"], "fallback"); + assert_eq!(record["fallback"], "completed"); + assert_ne!(record["outcome"], "completed"); + } + } + + fn assert_default_error_observations(server: &Server) { + let records = observations(server); + let extension = records + .iter() + .find(|record| record["method"] == "other") + .expect("extension method bounded category"); + assert_eq!(extension["status"], 405_u16); + assert_eq!(extension["level"], "INFO"); + for entry in framework_logs(server) { + assert!( + !entry["message"] + .as_str() + .expect("framework message") + .contains("SENTINEL"), + "framework log leak: {entry}" + ); + let message = entry["message"].as_str().expect("framework message"); + assert!( + !message.contains("CONFIG_VALUE_SENTINEL") + && !message.contains("SECRET_VALUE_SENTINEL"), + "environment-derived private inputs leaked through the facade: {entry}" + ); + } + for record in records { + assert_record_log(server, &record); + } + } + + #[test] + fn native_default_errors_and_logs_keep_sentinels_private() { + let roots = Roots::new(); + let addr = address(); + let mut launch = command("diagnostics", &roots, addr); + launch + .env("PRIVATE_CONFIG_SENTINEL", "CONFIG_VALUE_SENTINEL") + .env("fixture_token", "SECRET_VALUE_SENTINEL"); + let mut server = Server::spawn(launch, addr); + server.ready(); + for (path, status, failure, level) in [ + ( + "/PRIVATE_PATH_SENTINEL?token=QUERY_SENTINEL", + 404, + "not_found", + "INFO", + ), + ( + "/native-error?token=QUERY_SENTINEL", + 500, + "internal", + "WARN", + ), + ("/native-upstream", 502, "bad_gateway", "WARN"), + ("/native-conversion", 500, "", "WARN"), + ("/native-expired", 504, "", "WARN"), + ("/native-status503", 503, "", "INFO"), + ] { + let response = raw_request( + addr, + &format!( + "GET {path} HTTP/1.1\r\nHost: HOST_SENTINEL\r\nAuthorization: Bearer AUTH_SENTINEL\r\nCookie: COOKIE_SENTINEL\r\nX-Request-ID: ID_SENTINEL\r\nConnection: close\r\n\r\n" + ), + ); + assert!( + response.starts_with(&format!("HTTP/1.1 {status}")), + "{response}" + ); + assert!( + !response.contains("SENTINEL"), + "default wire leak: {response}" + ); + assert!( + !response.contains("CONFIG_VALUE_SENTINEL") + && !response.contains("SECRET_VALUE_SENTINEL"), + "environment-derived private inputs leaked through the wire: {response}" + ); + assert_no_diagnostic_headers(&response); + assert_native_error_record(&server, path, status, failure, level); + } + let extension_response = raw_request( + addr, + "METHOD_SENTINEL /native-status503?token=QUERY_SENTINEL HTTP/1.1\r\nHost: fixture\r\nConnection: close\r\n\r\n", + ); + assert!(extension_response.starts_with("HTTP/1.1 405")); + assert!(!extension_response.contains("SENTINEL")); + assert!(extension_response.to_ascii_lowercase().contains("allow:")); + let json_response = post_json(&server, "/native-json-number", "\"PAYLOAD_SENTINEL\""); + assert!(json_response.starts_with("HTTP/1.1 400"), "{json_response}"); + assert!(!json_response.contains("PAYLOAD_SENTINEL")); + assert_eq!( + response_json(&json_response)["error"]["message"], + "invalid JSON payload" + ); + let handler_error = server + .request("/native-handler404") + .expect("application error"); + assert!( + handler_error.contains("HANDLER_404_SENTINEL"), + "application error messages remain application policy" + ); + let handler_record = route_record(&server, "/native-handler404"); + assert_eq!(handler_record["level"], "WARN"); + server.signal("TERM"); + assert!(server.wait().success()); + assert_final_zero(&server); + assert_eq!( + server + .log() + .lines() + .filter(|line| *line == "PRIVATE_INPUTS_USED") + .count(), + 2, + "both error paths consumed the fixture's captured environment-derived configuration and secret inputs" + ); + assert_default_error_observations(&server); + } + + #[test] + fn native_custom_renderer_keeps_ownership_but_framework_json_parse_text_stays_fixed() { + let roots = Roots::new(); + let addr = address(); + let mut server = Server::spawn(command("diagnostics-custom", &roots, addr), addr); + server.ready(); + let custom = server + .request("/CUSTOM_ROUTE_SENTINEL") + .expect("custom route error"); + assert!(custom.starts_with("HTTP/1.1 404")); + assert!( + custom.contains("CUSTOM_ROUTE_SENTINEL"), + "custom renderer retains original routing detail" + ); + let parsed = post_json(&server, "/native-json-number", "\"PAYLOAD_SENTINEL\""); + assert!(parsed.starts_with("HTTP/1.1 400")); + assert!(parsed.ends_with("invalid JSON payload")); + assert!(!parsed.contains("PAYLOAD_SENTINEL")); + server.signal("TERM"); + assert!(server.wait().success()); + assert_final_zero(&server); + for entry in framework_logs(&server) { + assert!( + !entry["message"] + .as_str() + .expect("framework message") + .contains("SENTINEL") + ); + } + } + + #[test] + fn native_original_head_overage_cannot_be_hidden_by_raw_header_stripping() { + let roots = Roots::new(); + let addr = address(); + let mut launch = command("diagnostics", &roots, addr); + // Let the received head reach managed normalized validation. The newer + // raw-wire parser otherwise rejects this request before service dispatch. + launch.env("EDGEZERO__ADAPTER__INGRESS__MAX_RAW_HEAD_BYTES", "131072"); + let mut server = Server::spawn(launch, addr); + server.ready(); + let request = metadata_request(&format!( + "X-Forwarded-Secret: {}\r\n", + "HEAD_SENTINEL".repeat(6000) + )); + let response = raw_request(addr, &request); + assert!( + response.starts_with("HTTP/1.1 431"), + "original normalized head limit: {response}" + ); + assert!(!response.contains("SENTINEL")); + server.signal("TERM"); + assert!(server.wait().success()); + assert_final_zero(&server); + let rejections = observations(&server) + .into_iter() + .filter(|record| record["status"] == 431_u16) + .collect::>(); + assert_eq!(rejections.len(), 1); + let record = rejections + .into_iter() + .next() + .expect("head rejection record"); + assert_eq!(record["level"], "WARN"); + assert_eq!( + server.log().matches("APP_COMPLETION ").count(), + 1, + "head failure precedes app admission; only readiness was admitted" + ); + assert_record_log(&server, &record); + } + + #[test] + fn native_ids_remain_private_under_mutation_and_concurrent_hosting_requests() { + let roots = Roots::new(); + let addr = address(); + let other_addr = address(); + let mut server = Server::spawn(command("diagnostics", &roots, addr), addr); + let mut other = Server::spawn(command("diagnostics", &roots, other_addr), other_addr); + server.ready(); + other.ready(); + let response = raw_request( + addr, + "GET /native-mutated HTTP/1.1\r\nHost: fixture\r\nX-Request-ID: VISITOR_ID_SENTINEL\r\nConnection: close\r\n\r\n", + ); + assert_no_diagnostic_headers(&response); + let mutated = response_json(&response); + assert_eq!(mutated["metadata_absent"], true); + assert_eq!(mutated["forwarded_host_after"], "replacement.example"); + let authoritative = route_record(&server, "/native-mutated"); + assert_eq!(authoritative["id"], mutated["removed_id"]); + let mut clients = Vec::new(); + for target in [addr, other_addr] { + for _request in 0..4_u8 { + clients.push(thread::spawn(move || { + let concurrent_response = raw_request(target, "GET /native-status503 HTTP/1.1\r\nHost: fixture\r\nX-Request-ID: VISITOR_ID_SENTINEL\r\nConnection: close\r\n\r\n"); + assert!(concurrent_response.starts_with("HTTP/1.1 503")); + assert_no_diagnostic_headers(&concurrent_response); + })); + } + } + for client in clients { + client.join().expect("bounded concurrent wire client"); + } + for running in [&mut server, &mut other] { + running.signal("TERM"); + assert!(running.wait().success()); + assert_final_zero(running); + } + let mut ids = HashSet::new(); + for running in [&server, &other] { + let records = observations(running); + assert_eq!( + records + .iter() + .filter(|record| record["route"] == "/native-status503") + .count(), + 4 + ); + assert_eq!( + running.log().matches("APP_COMPLETION ").count(), + records.len() + ); + assert_eq!( + running.log().matches("APP_OBSERVER ").count(), + records.len() + ); + for record in records { + let id = record["id"].as_str().expect("generated ID"); + assert_eq!(id.len(), 64); + assert!(id.bytes().all(|byte| byte.is_ascii_hexdigit())); + assert!(ids.insert(id.to_owned()), "no process/request ID collision"); + assert!( + record["counts"]["active_requests"] + .as_u64() + .expect("request count") + <= 3, + "the observed request releases before its sink; only other concurrent requests remain" + ); + assert_record_log(running, &record); + } + assert!(!running.log().contains("VISITOR_ID_SENTINEL")); + } + } + + #[test] + fn native_logger_panic_preserves_order_cardinality_and_resource_release() { + let roots = Roots::new(); + let addr = address(); + let mut server = Server::spawn(command("diagnostics-panic", &roots, addr), addr); + server.ready(); + assert!( + server + .request("/native-status503") + .expect("surviving response") + .starts_with("HTTP/1.1 503") + ); + let selected_record = route_record(&server, "/native-status503"); + wait_for(&server, |log| log.matches("APP_OBSERVER ").count() >= 2); + assert_hook_order(&server, selected_record["id"].as_str().expect("record ID")); + assert_eq!(selected_record["counts"]["active_requests"], 0_u64); + server.signal("TERM"); + assert!(server.wait().success()); + assert_final_zero(&server); + let log = server.log(); + assert_eq!(log.matches("APP_LOGGER_INSTALLED").count(), 1); + assert_eq!( + log.matches("LOGGER_FAULT").count(), + 1, + "no recursive logger fault reporting" + ); + let records = observations(&server); + assert_eq!(log.matches("APP_COMPLETION ").count(), records.len()); + assert_eq!(log.matches("APP_OBSERVER ").count(), records.len()); + for record in records { + assert_record_log(&server, &record); + } + } + + fn wire_prefix(socket: &mut TcpStream, prefix: &[u8]) -> Vec { + let mut received = Vec::new(); + let deadline = Instant::now().checked_add(WAIT).expect("prefix timer"); + while !received.windows(prefix.len()).any(|bytes| bytes == prefix) { + assert!(Instant::now() < deadline, "wire prefix deadline"); + assert!(received.len() < 0x4000, "finite response head and prefix"); + let mut chunk = [0_u8; 512]; + let count = socket + .read(&mut chunk) + .expect("wire prefix before terminal failure"); + assert_ne!(count, 0, "connection closed before prefix: {received:?}"); + received.extend_from_slice(chunk.get(..count).expect("wire read length")); + } + assert!(received.starts_with(b"HTTP/1.1 200")); + assert!(received.windows(4).any(|bytes| bytes == b"\r\n\r\n")); + received + } + + #[test] + fn native_actual_wire_prefix_precedes_source_and_deadline_failure_without_second_response() { + for (path, prefix, outcome, dropped) in [ + ( + "/native-source", + b"source-prefix".as_slice(), + "source_error", + "SOURCE_DROPPED", + ), + ( + "/native-deadline", + b"deadline-prefix".as_slice(), + "deadline_exceeded", + "DEADLINE_DROPPED", + ), + ] { + let roots = Roots::new(); + let addr = address(); + let mut server = Server::spawn(command("diagnostics", &roots, addr), addr); + server.ready(); + let mut client = server.socket().expect("streaming client"); + write!( + client, + "GET {path} HTTP/1.1\r\nHost: fixture\r\nConnection: close\r\n\r\n" + ) + .expect("stream request"); + let mut wire = wire_prefix(&mut client, prefix); + let mut tail = Vec::new(); + let terminal = client.read_to_end(&mut tail); + if let Err(error) = terminal { + assert!( + matches!( + error.kind(), + io::ErrorKind::ConnectionReset | io::ErrorKind::UnexpectedEof + ), + "failure did not settle before timeout: {error}" + ); + } + wire.extend_from_slice(&tail); + let wire_text = String::from_utf8(wire).expect("ASCII fixture response"); + assert_eq!( + wire_text.matches("HTTP/1.1 ").count(), + 1, + "no replacement response after commitment" + ); + assert!(!wire_text.contains("SENTINEL")); + assert!( + !wire_text.ends_with("0\r\n\r\n"), + "stream failure must not look like completed chunked framing" + ); + let record = route_record(&server, path); + assert_eq!( + record["status"], 200_u16, + "selected status is not successful delivery" + ); + assert_eq!(record["outcome"], outcome); + assert_eq!(record["level"], "WARN"); + assert_eq!(record["bytes_written"], prefix.len()); + assert_eq!(record["body_kind"], "application"); + assert!(record["fallback"].is_null()); + assert_eq!(record["counts"]["active_requests"], 0_u64); + wait_for(&server, |log| { + log.contains(dropped) && log.matches("APP_OBSERVER ").count() >= 2 + }); + assert_hook_order(&server, record["id"].as_str().expect("record ID")); + assert_record_log(&server, &record); + server.signal("TERM"); + assert!(server.wait().success()); + assert_final_zero(&server); + assert_eq!(server.log().matches(dropped).count(), 1); + assert_eq!( + observations(&server) + .iter() + .filter(|observed| observed["route"] == path) + .count(), + 1 + ); + for entry in framework_logs(&server) { + assert!( + !entry["message"] + .as_str() + .expect("framework message") + .contains("SENTINEL") + ); + } + } + } + + #[test] + fn native_actual_wire_disconnect_keeps_selected_status_and_one_terminal_owner() { + let roots = Roots::new(); + let addr = address(); + let mut server = Server::spawn(command("diagnostics", &roots, addr), addr); + server.ready(); + let mut client = server.socket().expect("stream client"); + client + .write_all(b"GET /stream HTTP/1.1\r\nHost: fixture\r\n\r\n") + .expect("stream request"); + let wire = wire_prefix(&mut client, b"stream-prefix"); + assert_eq!( + String::from_utf8(wire) + .expect("wire head") + .matches("HTTP/1.1 ") + .count(), + 1 + ); + drop(client); + let record = route_record(&server, "/stream"); + assert_eq!(record["status"], 200_u16); + // The existing HTTP/1 connection owner reports this peer loss as a transport failure. + assert_eq!(record["outcome"], "transport_error"); + assert_eq!(record["level"], "WARN"); + assert_eq!(record["counts"]["active_requests"], 0_u64); + wait_for(&server, |log| { + log.contains("STREAM_DROPPED") && log.matches("APP_OBSERVER ").count() >= 2 + }); + assert_hook_order(&server, record["id"].as_str().expect("record ID")); + server.signal("TERM"); + assert!(server.wait().success()); + assert_final_zero(&server); + assert_eq!(server.log().matches("STREAM_DROPPED").count(), 1); + assert_record_log(&server, &record); + } + + #[test] + fn native_refusal_and_no_response_abort_keep_separate_observer_contracts() { + let roots = Roots::new(); + let addr = address(); + let mut server = Server::spawn(command("diagnostics", &roots, addr), addr); + server.ready(); + let response = server.request("/native-refuse").expect("refusal response"); + assert!(response.starts_with("HTTP/1.1 503")); + let mut client = server.socket().expect("abort connection"); + client + .write_all(b"GET /native-abort HTTP/1.1\r\nHost: fixture\r\nConnection: close\r\n\r\n") + .expect("abort request"); + let mut aborted_wire = Vec::new(); + let abort_read = client.read_to_end(&mut aborted_wire); + if let Err(error) = abort_read { + assert_eq!(error.kind(), io::ErrorKind::ConnectionReset); + } + assert!(aborted_wire.is_empty(), "no fabricated abort response"); + server.signal("TERM"); + assert!(server.wait().success()); + assert_final_zero(&server); + let records = observations(&server); + let refusal = records + .iter() + .filter(|record| record["failure"] == "admission_refused") + .collect::>(); + assert_eq!(refusal.len(), 1); + assert_eq!( + refusal.into_iter().next().expect("refusal record")["status"], + 503_u16 + ); + let abort_records = records + .iter() + .filter(|record| record["outcome"] == "aborted") + .collect::>(); + assert_eq!(abort_records.len(), 1); + let abort_record = abort_records + .into_iter() + .next() + .expect("no response ingress record"); + assert_eq!(abort_record["event"], "request_ingress"); + assert!(abort_record["status"].is_null() && abort_record["bytes_written"].is_null()); + assert_eq!( + server.log().matches("APP_COMPLETION ").count(), + 2, + "readiness plus supplied refusal completion only" + ); + assert_eq!( + server.log().matches("APP_OBSERVER ").count(), + 1, + "refusal is detached; abort has no egress observer" + ); + for record in records { + assert_record_log(&server, &record); + } + } + + fn native_counts(server: &Server, expected_requests: u64, expected_idle: u64) -> Value { + let deadline = Instant::now().checked_add(WAIT).expect("snapshot timer"); + loop { + let counts = response_json(&server.request("/native-counts").expect("counter query")); + if counts["active_requests"] == expected_requests + && counts["idle_connections"] == expected_idle + { + return counts; + } + assert!( + Instant::now() < deadline, + "snapshot did not converge: {counts}; {}", + server.log() + ); + thread::sleep(Duration::from_millis(5)); + } + } + + fn assert_shutdown_observations(server: &Server, path: &str, graceful: bool, optout: bool) { + let logs = framework_logs(server); + for phase in ["starting", "ready", "draining", "stopped"] { + assert!( + logs.iter().any(|entry| entry["message"] + .as_str() + .expect("lifecycle message") + .contains(&format!("event=lifecycle phase={phase}"))), + "missing lifecycle {phase}" + ); + } + if !graceful { + assert!(logs.iter().any(|entry| { + entry["message"] + .as_str() + .expect("fault category") + .contains("reason=grace_exhausted") + })); + } + let records = observations(server); + assert!(!records.is_empty(), "native observer retained"); + assert!(server.log().contains("APP_OBSERVER ") && server.log().contains("APP_COMPLETION ")); + if optout { + assert!( + !logs.iter().any(|entry| { + let message = entry["message"].as_str().expect("framework message"); + message.starts_with("request_terminal ") + || message.starts_with("request_ingress ") + }), + "request-record opt-out" + ); + } + let abandoned = records + .iter() + .filter(|record| record["outcome"] == "abandoned") + .collect::>(); + if matches!(path, "/pending" | "/json") { + assert_eq!(abandoned.len(), 1); + let record = abandoned + .into_iter() + .next() + .expect("pre-attempt abandonment"); + assert!(record["status"].is_null()); + assert_eq!(record["event"], "request_ingress"); + } + } + + #[test] + fn native_idle_and_held_work_snapshots_drain_to_zero_with_request_records_opted_out() { + for optout in [false, true] { + for (path, prefix, marker, graceful) in [ + ("/finite", None, "DISPATCH", true), + ("/pending", None, "DISPATCH", false), + ( + "/stream", + Some(b"stream-prefix".as_slice()), + "STREAM_READY", + false, + ), + ("/json", None, "", false), + ] { + let roots = Roots::new(); + let addr = address(); + let mut launch = command( + if optout { + "diagnostics-optout" + } else { + "diagnostics" + }, + &roots, + addr, + ); + if optout { + launch.env("EDGEZERO__LOGGING__REQUEST_RECORDS", "false"); + } + let mut server = Server::spawn(launch, addr); + server.ready(); + let mut idle = server.socket().expect("owned keepalive connection"); + idle.write_all(b"GET /ready HTTP/1.1\r\nHost: fixture\r\n\r\n") + .expect("keepalive request"); + let mut ready = Vec::new(); + while !ready.ends_with(b"ready") { + let mut buffer = [0_u8; 512]; + let count = idle.read(&mut buffer).expect("keepalive ready response"); + assert_ne!(count, 0); + ready.extend_from_slice(buffer.get(..count).expect("read length")); + } + let idle_counts = native_counts(&server, 1, 1); + assert_eq!( + idle_counts["active_connections"], 2_u64, + "idle connection plus count query" + ); + let mut held = server.socket().expect("held request"); + if path == "/json" { + held.write_all(b"POST /json HTTP/1.1\r\nHost: fixture\r\nContent-Type: application/json\r\nContent-Length: 64\r\n\r\n\"x").expect("partial body held open"); + } else { + write!(held, "GET {path} HTTP/1.1\r\nHost: fixture\r\n\r\n") + .expect("held request"); + server.wait_line(marker); + } + if let Some(expected_prefix) = prefix { + let _wire = wire_prefix(&mut held, expected_prefix); + } + let held_counts = native_counts(&server, 2, 1); + assert_eq!( + held_counts["active_connections"], 3_u64, + "held, idle and query connections" + ); + assert_eq!(held_counts["phase"], "ready"); + let started = Instant::now(); + server.signal("TERM"); + if graceful { + let mut response = String::new(); + held.read_to_string(&mut response) + .expect("graceful finite response"); + assert!( + response.starts_with("HTTP/1.1 200") + && response.contains("finite-complete") + ); + } + let status = server.wait(); + assert_eq!(status.success(), graceful, "{}", server.log()); + assert!(started.elapsed() < Duration::from_secs(4)); + if !graceful { + assert!(started.elapsed() >= Duration::from_millis(700)); + } + assert_final_zero(&server); + assert_eq!(idle.read(&mut [0_u8; 1]).expect("idle close"), 0); + assert_shutdown_observations(&server, path, graceful, optout); + } + } + } + + #[test] + fn native_invalid_proxy_and_record_settings_fail_before_ready_without_input_disclosure() { + let roots = Roots::new(); + for (networks, family, records, category) in [ + ( + Some("NETWORK_SENTINEL.invalid"), + Some("forwarded"), + None, + "TRUSTED_PROXY_POLICY", + ), + ( + Some("0.0.0.0/0"), + Some("forwarded"), + None, + "TRUSTED_PROXY_POLICY", + ), + ( + Some("::/0"), + Some("forwarded"), + None, + "TRUSTED_PROXY_POLICY", + ), + ( + Some("0.0.0.0/1,128.0.0.0/1"), + Some("forwarded"), + None, + "TRUSTED_PROXY_POLICY", + ), + (Some("127.0.0.2/32"), None, None, "TRUSTED_PROXY_POLICY"), + ( + Some("127.0.0.2/32"), + Some("FAMILY_SENTINEL"), + None, + "TRUSTED_PROXY_POLICY", + ), + ( + Some(""), + Some("FAMILY_SENTINEL"), + None, + "TRUSTED_PROXY_POLICY", + ), + (None, None, Some("RECORDS_SENTINEL"), "REQUEST_RECORDS"), + (None, None, Some("FALSE"), "REQUEST_RECORDS"), + ] { + let addr = address(); + let mut launch = command("diagnostics", &roots, addr); + if let Some(value) = networks { + launch.env("EDGEZERO__ADAPTER__TRUSTED_PROXY_CIDRS", value); + } + if let Some(value) = family { + launch.env("EDGEZERO__ADAPTER__FORWARDING_HEADER_FAMILY", value); + } + if let Some(value) = records { + launch.env("EDGEZERO__LOGGING__REQUEST_RECORDS", value); + } + let log = failed(launch, addr, category); + assert!( + !log.contains("HOOK_ENTERED"), + "invalid settings precede app construction" + ); + assert!(!log.contains("phase=ready")); + if let Some(value) = networks.filter(|text| !text.is_empty()) { + assert!(!log.contains(value), "network input leaked: {log}"); + } + if let Some(value) = records { + assert!(!log.contains(value), "record-setting input leaked: {log}"); + } + } + } + + #[test] + fn native_non_unicode_proxy_and_record_settings_fail_instead_of_defaulting() { + use std::os::unix::ffi::OsStringExt as _; + let roots = Roots::new(); + for (key, category) in [ + ( + "EDGEZERO__ADAPTER__TRUSTED_PROXY_CIDRS", + "TRUSTED_PROXY_CIDRS", + ), + ( + "EDGEZERO__ADAPTER__FORWARDING_HEADER_FAMILY", + "FORWARDING_HEADER_FAMILY", + ), + ("EDGEZERO__LOGGING__REQUEST_RECORDS", "REQUEST_RECORDS"), + ] { + let addr = address(); + let mut launch = command("diagnostics", &roots, addr); + let mut raw = b"NON_UNICODE_SENTINEL".to_vec(); + raw.push(0xff); + launch.env(key, OsString::from_vec(raw)); + let log = failed(launch, addr, category); + assert!( + log.contains("non-Unicode"), + "captured native setting was rejected: {log}" + ); + assert!(!log.contains("HOOK_ENTERED") && !log.contains("phase=ready")); + assert!( + !log.contains('\u{fffd}'), + "no lossy rendering of non-Unicode input" + ); + } + } + + include!("fixtures/geo_hosting.rs"); + + fn geo_command(mode: &str, cwd: &TempDir, addr: SocketAddr) -> Command { + let mut command = Command::new(env!("CARGO_BIN_EXE_edgezero-hosting-fixture")); + command + .env_clear() + .current_dir(cwd.path()) + .env("FIXTURE_MODE", mode) + .env("FIXTURE_BIND", addr.to_string()); + command + } + + fn geo_response(server: &Server, headers: &str) -> Value { + let response = raw_request( + server.addr, + &format!("GET /geo HTTP/1.1\r\nHost: fixture\r\n{headers}Connection: close\r\n\r\n"), + ); + assert!(response.starts_with("HTTP/1.1 200"), "{response}"); + assert!(!response.contains("ORIGIN_CREDENTIAL_SENTINEL")); + assert!(!response.contains("x-origin-verify-fixture")); + let value = response_json(&response); + assert_eq!(value["immutable"], true); + assert_eq!(value["credential_absent"], true); + assert_eq!(value["mutated_country"], "GB"); + value + } + + fn finish_geo(server: &mut Server, cwd: &TempDir) { + assert!( + server + .request("/stop") + .expect("Geo caller stop") + .ends_with("stopping") + ); + assert!(server.wait().success(), "{}", server.log()); + let log = server.log(); + assert_eq!(log.matches("INITIALIZED").count(), 1); + assert!(!log.contains("ORIGIN_CREDENTIAL_SENTINEL")); + assert!(!log.contains("x-origin-verify-fixture")); + assert_eq!( + fs::read_dir(cwd.path()).expect("unrelated cwd").count(), + 0, + "Geo-only startup must not create store roots or local databases" + ); + } + + #[test] + #[cfg(not(feature = "geo-cloudfront"))] + fn geo_hosting_rejects_unavailable_cloudfront() { + let cwd = tempfile::tempdir().expect("unrelated cwd"); + let addr = address(); + let mut server = Server::spawn(geo_command("geo-cloudfront", &cwd, addr), addr); + assert!(!server.wait().success()); + let log = server.log(); + assert!(!log.is_empty(), "bounded startup error must be reported"); + assert!(!log.contains("INITIALIZED")); + assert!(!log.contains("HOOK_ENTERED")); + assert!(!log.contains("ready")); + TcpStream::connect_timeout(&addr, Duration::from_millis(50)) + .expect_err("unavailable implementation never binds"); + assert_eq!(fs::read_dir(cwd.path()).expect("unrelated cwd").count(), 0); + } + + #[test] + #[cfg(not(feature = "geo-cloudfront"))] + fn geo_hosting_supplied_provider_without_builtin() { + let cwd = tempfile::tempdir().expect("unrelated cwd"); + let addr = address(); + let mut server = Server::spawn(geo_command("geo-supplied", &cwd, addr), addr); + server.ready(); + let headers = "CloudFront-Viewer-Country: us\r\nCloudFront-Viewer-Country-Region: ca\r\nx-origin-verify-fixture: ORIGIN_CREDENTIAL_SENTINEL\r\nx-origin-verify-fixture: SECOND_ORIGIN_CREDENTIAL_SENTINEL\r\n"; + let first = geo_response(&server, headers); + assert_eq!(first["geo"]["status"], "located"); + assert_eq!(first["geo"]["provider"], "fixture-local"); + assert_eq!(first["geo"]["country"], "US"); + assert_eq!(first["geo"]["subdivision"], "CA"); + assert_eq!(first["geo"]["source"], "CloudFront"); + assert_eq!(first["geo"]["trust_basis"], "DeclaredProtectedAlb"); + let second = geo_response(&server, ""); + assert_eq!(second["geo"]["status"], "located"); + assert_eq!( + second["resolutions"].as_u64().expect("count"), + first["resolutions"].as_u64().expect("count") + 1 + ); + finish_geo(&mut server, &cwd); + } + + #[test] + fn geo_hosting_default_off_untrusted_supplied_and_failure_stay_outcomes() { + for (mode, expected, provider) in [ + ("geo-off", "no_selection", None), + ( + "geo-supplied-untrusted", + "untrusted_ingress", + Some("fixture-local"), + ), + ( + "geo-provider-failure", + "provider_failure", + Some("fixture-local"), + ), + ] { + let cwd = tempfile::tempdir().expect("unrelated cwd"); + let addr = address(); + let mut server = Server::spawn(geo_command(mode, &cwd, addr), addr); + server.ready(); + let headers = if mode == "geo-off" { + "CloudFront-Viewer-Country: us\r\nCloudFront-Viewer-Country-Region: ca\r\n" + } else { + "CloudFront-Viewer-Country: us\r\nCloudFront-Viewer-Country-Region: ca\r\nx-origin-verify-fixture: ORIGIN_CREDENTIAL_SENTINEL\r\n" + }; + let value = geo_response(&server, headers); + assert_eq!(value["geo"]["status"], expected); + assert_eq!(value["geo"]["provider"].as_str(), provider); + if mode == "geo-provider-failure" { + assert!(value["resolutions"].as_u64().expect("provider invoked") >= 1); + } else { + assert_eq!(value["resolutions"], 0_u64); + } + finish_geo(&mut server, &cwd); + } + } + + #[test] + fn geo_hosting_invalid_policy_never_initializes_or_listens() { + let cwd = tempfile::tempdir().expect("unrelated cwd"); + let addr = address(); + let mut server = Server::spawn(geo_command("geo-invalid-policy", &cwd, addr), addr); + assert!(!server.wait().success()); + assert!(!server.log().contains("INITIALIZED")); + assert!(!server.log().contains("HOOK_ENTERED")); + assert!(!server.log().contains("x-origin-verify-fixture")); + TcpStream::connect_timeout(&addr, Duration::from_millis(50)) + .expect_err("invalid policy never binds"); + assert_eq!(fs::read_dir(cwd.path()).expect("unrelated cwd").count(), 0); + } + + #[test] + #[cfg(feature = "geo-cloudfront")] + fn geo_hosting_proves_real_peer_and_shared_outcome() { + let cwd = tempfile::tempdir().expect("unrelated cwd"); + let addr = address(); + let mut server = Server::spawn(geo_command("geo-cloudfront", &cwd, addr), addr); + server.ready(); + for (headers, expected, subdivision) in [ + ( + "CloudFront-Viewer-Country: us\r\nCloudFront-Viewer-Country-Region: ca\r\n", + "located", + Some("CA"), + ), + ("CloudFront-Viewer-Country: US\r\n", "located", None), + ( + "CloudFront-Viewer-Country: US\r\nCloudFront-Viewer-Country-Region: ZZ\r\n", + "located", + Some("ZZ"), + ), + ("", "missing_metadata", None), + ( + "CloudFront-Viewer-Country: US\r\nCloudFront-Viewer-Country: US\r\n", + "invalid_assertion", + None, + ), + ( + "CloudFront-Viewer-Country: XX\r\n", + "invalid_assertion", + None, + ), + ( + "CloudFront-Viewer-Country-Region: CA\r\n", + "invalid_assertion", + None, + ), + ( + "CloudFront-Viewer-Country: US,CA\r\n", + "invalid_assertion", + None, + ), + ] { + let value = geo_response( + &server, + &format!( + "{headers}x-origin-verify-fixture: ORIGIN_CREDENTIAL_SENTINEL\r\nx-origin-verify-fixture: SECOND_ORIGIN_CREDENTIAL_SENTINEL\r\n" + ), + ); + assert_eq!(value["geo"]["status"], expected); + assert_eq!(value["geo"]["provider"], "cloudfront"); + if expected == "located" { + assert_eq!(value["geo"]["country"], "US"); + assert_eq!(value["geo"]["subdivision"].as_str(), subdivision); + assert_eq!(value["geo"]["source"], "CloudFront"); + assert_eq!(value["geo"]["trust_basis"], "DeclaredProtectedAlb"); + } + } + finish_geo(&mut server, &cwd); + let untrusted_addr = address(); + let mut untrusted = Server::spawn( + geo_command("geo-cloudfront-untrusted", &cwd, untrusted_addr), + untrusted_addr, + ); + untrusted.ready(); + let value = geo_response( + &untrusted, + "CloudFront-Viewer-Country: US\r\nx-origin-verify-fixture: ORIGIN_CREDENTIAL_SENTINEL\r\n", + ); + assert_eq!(value["geo"]["status"], "untrusted_ingress"); + assert_eq!(value["geo"]["provider"], "cloudfront"); + finish_geo(&mut untrusted, &cwd); + } + + #[test] + fn both_signals_stop_idle_processes() { + for signal in ["INT", "TERM"] { + let roots = Roots::new(); + let addr = address(); + let mut server = Server::spawn(command("bare", &roots, addr), addr); + server.ready(); + server.signal(signal); + assert!(server.wait().success(), "{}", server.log()); + } + } + + #[test] + fn sigterm_drains_dispatched_response_closes_idle_and_gates_pipeline() { + let roots = Roots::new(); + let addr = address(); + let mut server = Server::spawn(command("bare", &roots, addr), addr); + server.ready(); + let mut idle = server.socket().expect("idle socket"); + idle.write_all(b"GET /ready HTTP/1.1\r\nHost: fixture\r\n\r\n") + .expect("idle request"); + let mut idle_response = Vec::new(); + while !idle_response.ends_with(b"ready") { + let mut buffer = [0_u8; 1024]; + let read = idle.read(&mut buffer).expect("idle response"); + assert_ne!(read, 0); + idle_response.extend_from_slice(buffer.get(..read).expect("read length")); + } + let mut client = server.socket().expect("finite socket"); + client.write_all(b"GET /finite HTTP/1.1\r\nHost: fixture\r\n\r\nGET /after HTTP/1.1\r\nHost: fixture\r\nConnection: close\r\n\r\n") + .expect("pipeline"); + server.wait_line("DISPATCH"); + server.signal("TERM"); + server.wait_line("DRAINING"); + let mut response = String::new(); + client + .read_to_string(&mut response) + .expect("complete finite response"); + assert!(response.contains("finite-complete")); + assert!(!response.contains("forbidden")); + assert_eq!(idle.read(&mut [0_u8; 1]).expect("idle close"), 0); + assert!(server.wait().success(), "{}", server.log()); + assert!(!server.log().contains("FORBIDDEN_DISPATCH")); + } + + #[test] + fn process_budget_forces_pending_handler_stream_and_zero_read_peer() { + for (path, marker, dropped) in [ + ("/pending", "DISPATCH", "WORK_DROPPED"), + ("/stream", "STREAM_READY", "STREAM_DROPPED"), + ("/flood", "STREAM_READY", "STREAM_DROPPED"), + ] { + let roots = Roots::new(); + let addr = address(); + let mut cmd = command("bare", &roots, addr); + cmd.env("FIXTURE_FREEZE_CLOCK", "1"); + let mut server = Server::spawn(cmd, addr); + server.ready(); + let mut client = server.socket().expect("pending socket"); + write!(client, "GET {path} HTTP/1.1\r\nHost: fixture\r\n\r\n").expect("request"); + server.wait_line(marker); + let started = Instant::now(); + server.signal("TERM"); + assert!(!server.wait().success()); + assert!(started.elapsed() >= Duration::from_millis(700)); + assert!(started.elapsed() < Duration::from_secs(4)); + assert!(server.log().contains("grace exhausted"), "{}", server.log()); + assert_eq!(server.log().matches(dropped).count(), 1); + } + } + + #[test] + fn second_signal_forces_before_budget_exhaustion() { + let roots = Roots::new(); + let addr = address(); + let mut server = Server::spawn(command("bare", &roots, addr), addr); + server.ready(); + let mut client = server.socket().expect("pending socket"); + client + .write_all(b"GET /pending HTTP/1.1\r\nHost: fixture\r\n\r\n") + .expect("request"); + server.wait_line("DISPATCH"); + server.signal("TERM"); + server.wait_line("DRAINING"); + let started = Instant::now(); + server.signal("INT"); + assert!(!server.wait().success()); + assert!(started.elapsed() < Duration::from_millis(700)); + assert!(server.log().contains("forced stop")); + assert_eq!(server.log().matches("WORK_DROPPED").count(), 1); + } + + #[test] + fn stop_during_startup_never_binds_or_becomes_ready() { + let roots = Roots::new(); + let addr = address(); + let mut server = Server::spawn(command("starting", &roots, addr), addr); + server.wait_line("INITIALIZING"); + TcpStream::connect_timeout(&addr, Duration::from_millis(50)) + .expect_err("initialization precedes binding"); + server.signal("TERM"); + assert!(server.wait().success()); + assert!(server.log().contains("STARTUP_DROPPED")); + } + + #[test] + fn invalid_settings_and_callback_sources_are_redacted_nonzero_failures() { + let roots = Roots::new(); + for (key, value, category) in [ + ("EDGEZERO__ADAPTER__HOST", "HOST_SENTINEL", "HOST"), + ("EDGEZERO__ADAPTER__PORT", "PORT_SENTINEL", "PORT"), + ("EDGEZERO__ADAPTER__PORT", "0", "PORT"), + ("EDGEZERO__LOGGING__LEVEL", "LOGGING_SENTINEL", "LEVEL"), + ( + "EDGEZERO__ADAPTER__SHUTDOWN_GRACE_SECONDS", + "0", + "SHUTDOWN_GRACE_SECONDS", + ), + ( + "EDGEZERO__ADAPTER__CONFIG_DIR", + "ROOT_SENTINEL", + "CONFIG_DIR", + ), + ( + "EDGEZERO__STORES__CONFIG__SETTINGS__KEY", + "KEY_SENTINEL\n", + "settings", + ), + ] { + let addr = address(); + let mut cmd = command("initialized", &roots, addr); + cmd.env(key, value); + failed(cmd, addr, category); + } + for (mode, category) in [ + ("hook-error", "application configuration"), + ("initializer-error", "initializer"), + ] { + let addr = address(); + failed(command(mode, &roots, addr), addr, category); + } + let addr = address(); + let mut cmd = command("initialized", &roots, addr); + cmd.env_remove("EDGEZERO__ADAPTER__CONFIG_DIR"); + failed(cmd, addr, "CONFIG_DIR"); + let missing = roots.cwd.path().join("missing-data-SENTINEL"); + let missing_addr = address(); + let mut missing_cmd = command("initialized", &roots, missing_addr); + missing_cmd.env("EDGEZERO__ADAPTER__DATA_DIR", &missing); + failed(missing_cmd, missing_addr, "DATA_DIR"); + assert!(!missing.exists()); + } + + #[test] + fn occupied_bind_is_a_real_redacted_process_failure() { + let roots = Roots::new(); + let occupied = TcpListener::bind("127.0.0.1:0").expect("occupied port"); + let addr = occupied.local_addr().expect("address"); + let mut server = Server::spawn(command("bare", &roots, addr), addr); + assert!(!server.wait().success()); + assert!(server.log().contains("bind")); + assert!(!server.log().contains("SENTINEL")); + } + + #[test] + fn required_files_and_typed_initializer_fail_before_acceptance() { + let roots = Roots::new(); + let file = roots.config.path().join("local-config-settings.json"); + fs::remove_file(&file).expect("missing file fixture"); + let addr = address(); + failed(command("initialized", &roots, addr), addr, "missing"); + fs::create_dir_all(&file).expect("non-regular fixture"); + let directory_addr = address(); + failed( + command("initialized", &roots, directory_addr), + directory_addr, + "regular file", + ); + fs::remove_dir(&file).expect("remove fixture directory"); + roots.map(&json!({"settings": {"CONFIG_SENTINEL": true}})); + let malformed_addr = address(); + failed( + command("initialized", &roots, malformed_addr), + malformed_addr, + "malformed", + ); + for (map, category) in [ + (json!({}), "MissingBlob"), + (json!({"settings": "BLOB_SENTINEL"}), "MalformedEnvelope"), + ( + json!({"settings": blob("x", "fixture_token")}), + "Validation", + ), + ( + json!({"settings": blob("valid-greeting", "MISSING_SECRET_SENTINEL")}), + "MissingSecret", + ), + ] { + roots.map(&map); + let typed_addr = address(); + failed( + command("initialized", &roots, typed_addr), + typed_addr, + category, + ); + } + let mut envelope: Value = + serde_json::from_str(&blob("valid-greeting", "fixture_token")).expect("envelope"); + envelope["data"]["greeting"] = json!("CHANGED_CONFIG_SENTINEL"); + roots.map(&json!({"settings": serde_json::to_string(&envelope).expect("envelope")})); + let integrity_addr = address(); + failed( + command("initialized", &roots, integrity_addr), + integrity_addr, + "IntegrityMismatch", + ); + } + + #[test] + fn convenience_checks_framework_without_guessing_required_schema() { + let roots = Roots::new(); + roots.map(&json!({"settings": "INVALID_TYPED_SENTINEL"})); + let addr = address(); + let mut server = Server::spawn(command("framework", &roots, addr), addr); + server.ready(); + assert!( + !server + .request("/typed") + .expect("typed failure") + .starts_with("HTTP/1.1 200") + ); + server.signal("TERM"); + assert!(server.wait().success()); + assert!(!server.log().contains("INITIALIZED")); + } + + #[test] + fn retained_state_key_aliases_snapshot_restart_and_closed_backup_restore() { + let roots = Roots::new(); + roots.map(&json!({"settings": blob("wrong-key-greeting", "fixture_token"), "selected": blob("selected-greeting", "fixture_token")})); + let addr = address(); + let mut cmd = command("initialized", &roots, addr); + cmd.env("EDGEZERO__STORES__CONFIG__SETTINGS__KEY", "selected"); + let mut server = Server::spawn(cmd, addr); + server.ready(); + for _request in 0..3_u8 { + assert!( + server + .request("/state") + .expect("state") + .ends_with("selected-greeting") + ); + assert!( + server + .request("/typed") + .expect("typed") + .ends_with("selected-greeting") + ); + } + assert!( + server + .request("/write") + .expect("write") + .ends_with("written") + ); + assert!( + server + .request("/alias") + .expect("alias") + .ends_with("durable-value") + ); + assert!( + !server + .request("/distinct") + .expect("distinct") + .contains("durable-value") + ); + roots.map(&json!({"settings": blob("restart-greeting", "fixture_token")})); + assert!( + server + .request("/typed") + .expect("snapshot") + .ends_with("selected-greeting") + ); + let second_addr = address(); + failed( + command("initialized", &roots, second_addr), + second_addr, + "locked", + ); + assert_eq!( + fs::read_dir(roots.data.path()).expect("databases").count(), + 2 + ); + server.signal("TERM"); + assert!(server.wait().success()); + assert_eq!(server.log().matches("INITIALIZED").count(), 1); + let mut restarted = Server::spawn(command("initialized", &roots, addr), addr); + restarted.ready(); + assert!( + restarted + .request("/state") + .expect("restarted config") + .ends_with("restart-greeting") + ); + assert!( + restarted + .request("/read") + .expect("persisted") + .ends_with("durable-value") + ); + restarted.signal("TERM"); + assert!(restarted.wait().success()); + let restore = tempfile::tempdir().expect("restore root"); + for result in fs::read_dir(roots.data.path()).expect("closed databases") { + let entry = result.expect("database entry"); + fs::copy(entry.path(), restore.path().join(entry.file_name())) + .expect("stopped-writer backup"); + } + let mut restored_command = command("initialized", &roots, addr); + restored_command.env("EDGEZERO__ADAPTER__DATA_DIR", restore.path()); + let mut restored = Server::spawn(restored_command, addr); + restored.ready(); + assert!( + restored + .request("/read") + .expect("restored value") + .ends_with("durable-value") + ); + restored.signal("TERM"); + assert!(restored.wait().success()); + } + + #[test] + fn explicit_embedding_ignores_ambient_settings_and_uses_local_rc_stop() { + let roots = Roots::new(); + let addr = address(); + let mut cmd = command("embedding", &roots, addr); + cmd.env("EDGEZERO__ADAPTER__HOST", "IGNORED_HOST_SENTINEL") + .env("EDGEZERO__ADAPTER__PORT", "0") + .env("EDGEZERO__ADAPTER__CONFIG_DIR", "IGNORED_ROOT_SENTINEL") + .env( + "EDGEZERO__STORES__CONFIG__SETTINGS__KEY", + "IGNORED_KEY_SENTINEL", + ); + let mut server = Server::spawn(cmd, addr); + server.ready(); + assert!( + server + .request("/state") + .expect("state") + .ends_with("configured-greeting") + ); + assert!( + server + .request("/stop") + .expect("caller stop") + .ends_with("stopping") + ); + assert!(server.wait().success(), "{}", server.log()); + assert_eq!(server.log().matches("INITIALIZED").count(), 1); + } + + #[test] + fn embedding_does_not_install_process_signal_handlers() { + let roots = Roots::new(); + let addr = address(); + let mut server = Server::spawn(command("embedding-bare", &roots, addr), addr); + server.ready(); + server.signal("TERM"); + assert_eq!(server.wait().signal(), Some(15_i32)); + } + + #[test] + fn projected_readonly_config_and_writable_data_preserve_uid_and_permissions() { + let roots = Roots::new(); + let file = roots.config.path().join("local-config-settings.json"); + let target = roots.config.path().join("projected-value"); + fs::rename(&file, &target).expect("projected target"); + symlink(&target, &file).expect("projected symlink"); + fs::set_permissions(&target, fs::Permissions::from_mode(0o444)).expect("read-only config"); + fs::set_permissions(roots.config.path(), fs::Permissions::from_mode(0o555)) + .expect("read-only root"); + let config_before = fs::metadata(roots.config.path()).expect("config metadata"); + let data_before = fs::metadata(roots.data.path()).expect("data metadata"); + let addr = address(); + let mut server = Server::spawn(command("initialized", &roots, addr), addr); + server.ready(); + assert!( + server + .request("/write") + .expect("write") + .ends_with("written") + ); + server.signal("TERM"); + assert!(server.wait().success()); + let config_after = fs::metadata(roots.config.path()).expect("config metadata"); + let data_after = fs::metadata(roots.data.path()).expect("data metadata"); + assert_eq!( + (config_before.uid(), config_before.mode()), + (config_after.uid(), config_after.mode()) + ); + assert_eq!( + (data_before.uid(), data_before.mode()), + (data_after.uid(), data_after.mode()) + ); + fs::set_permissions(roots.config.path(), fs::Permissions::from_mode(0o700)) + .expect("fixture cleanup"); + } +} diff --git a/crates/edgezero-adapter-cloudflare/tests/contract.rs b/crates/edgezero-adapter-cloudflare/tests/contract.rs index e957e0cb9..4da5dacfc 100644 --- a/crates/edgezero-adapter-cloudflare/tests/contract.rs +++ b/crates/edgezero-adapter-cloudflare/tests/contract.rs @@ -51,6 +51,7 @@ mod tests { }; use edgezero_core::context::RequestContext; use edgezero_core::error::EdgeError; + use edgezero_core::geo::{GeoError, GeoOutcome, GeoProviderId, GeoStatus}; use edgezero_core::http::{Method, Response, StatusCode, response_builder}; use edgezero_core::outbound::{OutboundHttpClient as _, OutboundRequest}; use edgezero_core::router::RouterService; @@ -128,8 +129,17 @@ mod tests { } } + fn forged_geo() -> GeoOutcome { + GeoOutcome::Selected { + provider: GeoProviderId::new("forged-portable").expect("fixture provider ID"), + status: GeoStatus::Error(GeoError::ProviderFailure), + } + } + fn build_test_app() -> App { async fn capture_uri(ctx: RequestContext) -> Result { + assert_eq!(ctx.geo(), &GeoOutcome::NoSelection); + assert_eq!(ctx.extensions().get::(), Some(&forged_geo())); let body = Body::text(ctx.uri().to_string()); let response = response_builder() .status(StatusCode::OK) @@ -204,7 +214,9 @@ mod tests { .get("/config-value", config_value) .build(); - App::new(router) + let mut app = App::new(router); + app.insert_state(forged_geo()); + app } fn cf_request(method: CfMethod, path: &str, body: Option<&[u8]>) -> CfRequest { @@ -215,6 +227,12 @@ mod tests { let headers = worker::Headers::new(); headers.set("host", "example.com").expect("host header"); + headers + .set("cloudfront-viewer-country", "US") + .expect("country header"); + headers + .set("cloudfront-viewer-country-region", "CA") + .expect("subdivision header"); headers.set("x-edgezero-test", "1").expect("custom header"); init.with_headers(headers); diff --git a/crates/edgezero-adapter-fastly/tests/contract.rs b/crates/edgezero-adapter-fastly/tests/contract.rs index 842aeee0d..ea74cd0e5 100644 --- a/crates/edgezero-adapter-fastly/tests/contract.rs +++ b/crates/edgezero-adapter-fastly/tests/contract.rs @@ -34,6 +34,7 @@ mod tests { }; use edgezero_core::context::RequestContext; use edgezero_core::error::EdgeError; + use edgezero_core::geo::{GeoError, GeoOutcome, GeoProviderId, GeoStatus}; use edgezero_core::http::{Method, Response, StatusCode, response_builder}; use edgezero_core::response_egress::ResponseEgressEnvelope; use edgezero_core::router::{RouteMetadata, RouterService}; @@ -108,8 +109,17 @@ mod tests { } } + fn forged_geo() -> GeoOutcome { + GeoOutcome::Selected { + provider: GeoProviderId::new("forged-portable").expect("fixture provider ID"), + status: GeoStatus::Error(GeoError::ProviderFailure), + } + } + fn build_test_app() -> App { async fn capture_uri(ctx: RequestContext) -> Result { + assert_eq!(ctx.geo(), &GeoOutcome::NoSelection); + assert_eq!(ctx.extensions().get::(), Some(&forged_geo())); let body = Body::text(ctx.uri().to_string()); let response = response_builder() .status(StatusCode::OK) @@ -167,7 +177,9 @@ mod tests { .get("/config", config_value) .build(); - App::new(router) + let mut app = App::new(router); + app.insert_state(forged_geo()); + app } fn fastly_request(method: FastlyMethod, path: &str, body: Option<&[u8]>) -> FastlyRequest { @@ -176,6 +188,8 @@ mod tests { let mut req = FastlyRequest::new(method, format!("http://example.com{path}")); req.set_header("host", "example.com"); req.set_header("x-edgezero-test", "1"); + req.set_header("cloudfront-viewer-country", "US"); + req.set_header("cloudfront-viewer-country-region", "CA"); if let Some(bytes) = body { req.set_body(bytes.to_vec()); } @@ -260,6 +274,8 @@ mod tests { let request_events = Arc::clone(&handler_events); async move { request_events.lock().expect("events").push("handler"); + assert_eq!(ctx.geo(), &GeoOutcome::NoSelection); + assert_eq!(ctx.extensions().get::(), Some(&forged_geo())); let value = ctx .extensions() .get::() @@ -281,6 +297,7 @@ mod tests { prepare_events.lock().expect("events").push("prepare"); raw.set_url("http://example.com/hook"); extensions.insert(HookValue("visible")); + extensions.insert(forged_geo()); }, move |response| { finalize_events.lock().expect("events").push("finalize"); diff --git a/crates/edgezero-adapter-spin/tests/contract.rs b/crates/edgezero-adapter-spin/tests/contract.rs index 067622eec..4dc7f4668 100644 --- a/crates/edgezero-adapter-spin/tests/contract.rs +++ b/crates/edgezero-adapter-spin/tests/contract.rs @@ -42,6 +42,7 @@ mod tests { }; use edgezero_core::context::RequestContext; use edgezero_core::error::EdgeError; + use edgezero_core::geo::{GeoError, GeoOutcome, GeoProviderId, GeoStatus}; use edgezero_core::http::{Response, StatusCode, request_builder, response_builder}; use edgezero_core::key_value_store::{KvError, KvHandle, KvPage, KvStore}; use edgezero_core::router::RouterService; @@ -213,8 +214,17 @@ mod tests { } } + fn forged_geo() -> GeoOutcome { + GeoOutcome::Selected { + provider: GeoProviderId::new("forged-portable").expect("fixture provider ID"), + status: GeoStatus::Error(GeoError::ProviderFailure), + } + } + fn build_test_app() -> App { async fn capture_uri(ctx: RequestContext) -> Result { + assert_eq!(ctx.geo(), &GeoOutcome::NoSelection); + assert_eq!(ctx.extensions().get::(), Some(&forged_geo())); let body = Body::text(ctx.uri().to_string()); let response = response_builder() .status(StatusCode::OK) @@ -316,7 +326,9 @@ mod tests { .get("/secret-value", secret_value) .build(); - App::new(router) + let mut app = App::new(router); + app.insert_state(forged_geo()); + app } #[test] @@ -583,6 +595,19 @@ mod tests { use super::*; + #[test] + fn geo_managed_dispatch_ignores_forged_extensions() { + let app = build_test_app(); + let response = block_on(dispatch_ingress_stream_for_test( + &app, + Method::GET, + "http://example.com/uri".parse().expect("URI"), + empty::>(), + )) + .expect("Spin admitted dispatch"); + assert_eq!(captured_body(response).as_ref(), b"http://example.com/uri"); + } + #[test] fn admission_abort() { let source_calls = Arc::new(AtomicUsize::new(0)); diff --git a/crates/edgezero-cli/Cargo.toml b/crates/edgezero-cli/Cargo.toml index 4fe54f60d..75c445f40 100644 --- a/crates/edgezero-cli/Cargo.toml +++ b/crates/edgezero-cli/Cargo.toml @@ -22,7 +22,7 @@ required-features = ["nested-app-config-check"] [dependencies] edgezero-core = { workspace = true } edgezero-adapter = { path = "../edgezero-adapter" } -edgezero-adapter-axum = { workspace = true, features = ["cli", "axum"], optional = true } +edgezero-adapter-axum = { workspace = true, features = ["cli", "native-bindings"], optional = true } edgezero-adapter-cloudflare = { workspace = true, features = ["cli"], optional = true } edgezero-adapter-fastly = { workspace = true, features = ["cli"], optional = true } edgezero-adapter-spin = { workspace = true, features = ["cli"], optional = true } @@ -48,7 +48,9 @@ toml = { workspace = true } [dev-dependencies] edgezero-core = { workspace = true, features = ["test-utils"] } +sha2 = { workspace = true } tempfile = { workspace = true } +url = { workspace = true } [features] default = [ @@ -58,6 +60,6 @@ default = [ "edgezero-adapter-cloudflare", "edgezero-adapter-spin", ] -cli = ["dep:clap", "dep:handlebars"] -demo-example = ["dep:app-demo-core", "edgezero-adapter-axum"] +cli = ["dep:clap", "dep:handlebars", "edgezero-adapter/cli"] +demo-example = ["dep:app-demo-core", "edgezero-adapter-axum", "edgezero-adapter-axum/axum"] nested-app-config-check = ["dep:proc-macro2", "dep:syn", "dep:walkdir"] diff --git a/crates/edgezero-cli/src/adapter.rs b/crates/edgezero-cli/src/adapter.rs index fd557d934..9fb9d9fe6 100644 --- a/crates/edgezero-cli/src/adapter.rs +++ b/crates/edgezero-cli/src/adapter.rs @@ -1,5 +1,7 @@ use edgezero_adapter::cli_support::validate_spin_outbound_host_contract; use edgezero_adapter::registry::{self as adapter_registry, AdapterAction}; +#[cfg(feature = "edgezero-adapter-axum")] +use edgezero_adapter_axum::native_build::{NativeCargoSelection, generated_axum_cargo_action}; use edgezero_core::manifest::{ CapabilitySupport, Manifest, ManifestContract, ManifestLoader, ResolvedEnvironment, }; @@ -117,6 +119,7 @@ pub fn execute( let root = loader.manifest().root().unwrap_or_else(|| Path::new(".")); let env = loader.manifest().environment_for(adapter_name); let adapter_bind = adapter_bind_from_manifest(loader.manifest(), adapter_name); + let build_features = adapter_build_features(loader.manifest(), adapter_name); return run_shell( command, root, @@ -125,6 +128,7 @@ pub fn execute( Some(env), adapter_bind, adapter_args, + &build_features, ); } @@ -177,6 +181,7 @@ pub fn execute_capture( let root = loader.manifest().root().unwrap_or_else(|| Path::new(".")); let env = loader.manifest().environment_for(adapter_name); let adapter_bind = adapter_bind_from_manifest(loader.manifest(), adapter_name); + let build_features = adapter_build_features(loader.manifest(), adapter_name); return run_shell_tee( command, root, @@ -185,6 +190,7 @@ pub fn execute_capture( Some(env), adapter_bind, adapter_args, + &build_features, ) .map(Some); } @@ -208,6 +214,9 @@ pub(crate) fn execute_capture_runtime( ) -> Result, String> { ensure_action_capabilities(runtime)?; if let ResolvedAdapterTarget::Shell(shell) = runtime.target() { + let build_features = runtime.manifest().map_or_else(Vec::new, |manifest| { + adapter_build_features(manifest, runtime.adapter_name()) + }); return run_shell_tee( shell.command(), shell.root(), @@ -216,6 +225,7 @@ pub(crate) fn execute_capture_runtime( Some(shell.environment().clone()), (shell.bind_host().map(str::to_owned), shell.bind_port()), adapter_args, + &build_features, ) .map(Some); } @@ -264,15 +274,21 @@ fn execute_after_gate(runtime: &ResolvedRuntime, adapter_args: &[String]) -> Res ResolvedAdapterTarget::Registered(_target) => { execute_registered_after_gate(runtime, adapter_args) } - ResolvedAdapterTarget::Shell(shell) => run_shell( - shell.command(), - shell.root(), - runtime.adapter_name(), - runtime.action(), - Some(shell.environment().clone()), - (shell.bind_host().map(str::to_owned), shell.bind_port()), - adapter_args, - ), + ResolvedAdapterTarget::Shell(shell) => { + let build_features = runtime.manifest().map_or_else(Vec::new, |manifest| { + adapter_build_features(manifest, runtime.adapter_name()) + }); + run_shell( + shell.command(), + shell.root(), + runtime.adapter_name(), + runtime.action(), + Some(shell.environment().clone()), + (shell.bind_host().map(str::to_owned), shell.bind_port()), + adapter_args, + &build_features, + ) + } } } @@ -302,13 +318,56 @@ fn execute_registered_after_gate( resolve_environment_defaults(&manifest.environment_for(runtime.adapter_name())) }); let selected = target.clone().with_environment_defaults(defaults); + let normalized_args = normalize_registered_axum_args(runtime, adapter_args)?; adapter.execute_target( AdapterAction::from(runtime.action()), &selected, - adapter_args, + &normalized_args, ) } +#[cfg(feature = "edgezero-adapter-axum")] +fn normalize_registered_axum_args( + runtime: &ResolvedRuntime, + adapter_args: &[String], +) -> Result, String> { + if !runtime.adapter_name().eq_ignore_ascii_case("axum") + || !matches!(runtime.action(), Action::Build | Action::Serve) + { + return Ok(adapter_args.to_vec()); + } + let features = runtime + .manifest() + .and_then(|manifest| { + manifest + .adapter_entry(runtime.adapter_name()) + .map(|(_name, adapter)| adapter.build.features.clone()) + }) + .unwrap_or_default(); + let selection = NativeCargoSelection::normalize_args(&features, adapter_args)?; + if matches!(runtime.action(), Action::Build) && !selection.run_args.is_empty() { + return Err("Axum build does not accept runtime arguments after `--`".to_owned()); + } + let mut normalized = selection.cargo_args(); + if matches!(runtime.action(), Action::Serve) && !selection.run_args.is_empty() { + normalized.push("--".to_owned()); + normalized.extend(selection.run_args); + } + Ok(normalized) +} + +#[cfg(not(feature = "edgezero-adapter-axum"))] +#[expect( + clippy::unnecessary_wraps, + reason = "retain the common fallible dispatch boundary without the optional native adapter" +)] +fn normalize_registered_axum_args( + _runtime: &ResolvedRuntime, + adapter_args: &[String], +) -> Result, String> { + Ok(adapter_args.to_vec()) +} + fn resolve_environment_defaults(environment: &ResolvedEnvironment) -> Vec<(String, String)> { let mut defaults = Vec::new(); for binding in &environment.variables { @@ -465,19 +524,26 @@ fn adapter_bind_from_manifest( /// command, with the manifest environment / bind hints applied. Shared /// by [`run_shell`] (inherited stdio) and [`run_shell_tee`] (piped + /// echoed stdio) so both dispatch paths apply identical env precedence. +fn adapter_build_features(manifest: &Manifest, adapter_name: &str) -> Vec { + manifest + .adapter_entry(adapter_name) + .map_or_else(Vec::new, |(_name, config)| config.build.features.clone()) +} + +#[expect( + clippy::too_many_arguments, + reason = "keeps command, action, environment, bind, passthrough, and selected features explicit at the shared shell boundary" +)] fn build_shell_command( command: &str, cwd: &Path, adapter_name: &str, + action: Action, environment: Option, adapter_bind: (Option, Option), adapter_args: &[String], + manifest_features: &[String], ) -> Result { - let full_command = if adapter_args.is_empty() { - command.to_owned() - } else { - format!("{} {}", command, shell_join(adapter_args)) - }; // Log only the manifest-defined `command`, never the trailing // `adapter_args` — passthrough args from `edgezero build/deploy // -- --token …` can carry deploy tokens, API keys, or other secrets that @@ -489,8 +555,17 @@ fn build_shell_command( cwd.display() ); - let mut cmd = Command::new("sh"); - cmd.arg("-c").arg(&full_command).current_dir(cwd); + let mut cmd = match native_axum_cargo_command( + adapter_name, + action, + command, + cwd, + adapter_args, + manifest_features, + )? { + Some(native_command) => native_command, + None => shell_command(command, adapter_args, cwd), + }; // Precedence (high to low) for `EDGEZERO__ADAPTER__HOST/PORT` on the // subprocess: @@ -523,6 +598,73 @@ fn build_shell_command( Ok(cmd) } +#[cfg(feature = "edgezero-adapter-axum")] +fn native_axum_cargo_command( + adapter_name: &str, + action: Action, + command: &str, + cwd: &Path, + adapter_args: &[String], + manifest_features: &[String], +) -> Result, String> { + if !adapter_name.eq_ignore_ascii_case("axum") + || !matches!(action, Action::Build | Action::Serve) + { + return Ok(None); + } + let Some((subcommand, package)) = generated_axum_cargo_action(command) else { + return Ok(None); + }; + let selection = NativeCargoSelection::normalize_args(manifest_features, adapter_args)?; + if subcommand == "build" && !selection.run_args.is_empty() { + return Err( + "generated `cargo build` does not accept runtime arguments after `--`".to_owned(), + ); + } + let mut cargo = Command::new("cargo"); + cargo + .arg(subcommand) + .arg("-p") + .arg(package) + .args(selection.cargo_args()) + .current_dir(cwd); + if subcommand == "run" && !selection.run_args.is_empty() { + cargo.arg("--").args(&selection.run_args); + } + Ok(Some(cargo)) +} + +#[cfg(not(feature = "edgezero-adapter-axum"))] +#[expect( + clippy::unnecessary_wraps, + reason = "retain the common fallible command boundary without the optional native adapter" +)] +fn native_axum_cargo_command( + _adapter_name: &str, + _action: Action, + _command: &str, + _cwd: &Path, + _adapter_args: &[String], + _manifest_features: &[String], +) -> Result, String> { + Ok(None) +} + +fn shell_command(command: &str, adapter_args: &[String], cwd: &Path) -> Command { + let full_command = if adapter_args.is_empty() { + command.to_owned() + } else { + format!("{} {}", command, shell_join(adapter_args)) + }; + let mut cmd = Command::new("sh"); + cmd.arg("-c").arg(full_command).current_dir(cwd); + cmd +} + +#[expect( + clippy::too_many_arguments, + reason = "keeps command, action, environment, bind, passthrough, and selected features explicit for inherited stdio" +)] fn run_shell( command: &str, cwd: &Path, @@ -531,14 +673,17 @@ fn run_shell( environment: Option, adapter_bind: (Option, Option), adapter_args: &[String], + manifest_features: &[String], ) -> Result<(), String> { let mut cmd = build_shell_command( command, cwd, adapter_name, + action, environment, adapter_bind, adapter_args, + manifest_features, )?; let status = cmd @@ -588,6 +733,10 @@ fn tee_stream(reader: R, mut writer: W) -> String { /// captured. Returns the captured `stdout + stderr` text so the caller /// can parse machine-readable lines (e.g. Fastly's activated /// `version=`) out of a command it does not otherwise control. +#[expect( + clippy::too_many_arguments, + reason = "keeps command, action, environment, bind, passthrough, and selected features explicit for captured stdio" +)] fn run_shell_tee( command: &str, cwd: &Path, @@ -596,14 +745,17 @@ fn run_shell_tee( environment: Option, adapter_bind: (Option, Option), adapter_args: &[String], + manifest_features: &[String], ) -> Result { let mut cmd = build_shell_command( command, cwd, adapter_name, + action, environment, adapter_bind, adapter_args, + manifest_features, )?; cmd.stdout(Stdio::piped()).stderr(Stdio::piped()); @@ -660,6 +812,8 @@ fn shell_join(args: &[String]) -> String { #[cfg(test)] mod tests { + #[cfg(feature = "edgezero-adapter-axum")] + use super::native_axum_cargo_command; use super::{ResolvedEnvironment, apply_environment, ensure_capabilities, execute_runtime}; use crate::adapter::Action; use crate::manifest_source::resolve_runtime_from; @@ -753,6 +907,39 @@ mod tests { ManifestLoader::load_from_str(&format!("[capabilities]\n{section}\n[adapters.gate]\n")) } + #[cfg(feature = "edgezero-adapter-axum")] + #[test] + fn generated_axum_cargo_intercept_is_build_and_serve_only() { + use std::path::Path; + + for action in [Action::Deploy, Action::AuthLogin] { + assert!( + native_axum_cargo_command( + "axum", + action, + "cargo build -p app", + Path::new("."), + &[], + &[], + ) + .unwrap() + .is_none() + ); + } + assert!( + native_axum_cargo_command( + "axum", + Action::Build, + "cargo build -p app", + Path::new("."), + &[], + &[], + ) + .unwrap() + .is_some() + ); + } + #[test] fn registered_dispatch_carries_only_selected_manifest_defaults_in_order() { const KEY: &str = "EDGEZERO_TEST_REGISTERED_INGRESS"; diff --git a/crates/edgezero-cli/src/args.rs b/crates/edgezero-cli/src/args.rs index cc86e79c3..eaa86060d 100644 --- a/crates/edgezero-cli/src/args.rs +++ b/crates/edgezero-cli/src/args.rs @@ -575,6 +575,9 @@ pub struct ConfigPushArgs { /// `[stores.config].ids` has length 1). #[arg(long)] pub store: Option, + /// Explicit native bindings file. Used only to reject unsupported AWS-bound publication. + #[arg(long)] + pub store_bindings: Option, /// Skip the inline diff prompt and write unconditionally. #[arg(long, short)] pub yes: bool, @@ -596,6 +599,7 @@ impl Default for ConfigPushArgs { runtime_config: None, staging: false, store: None, + store_bindings: None, yes: false, } } @@ -604,7 +608,14 @@ impl Default for ConfigPushArgs { /// Arguments for the `config validate` command. #[derive(clap::Args, Debug)] #[non_exhaustive] +#[expect( + clippy::struct_excessive_bools, + reason = "each bool is an independent Clap control; target toggles mirror Cargo feature flags" +)] pub struct ConfigValidateArgs { + /// Select bindings-only native validation; must be `axum`. + #[arg(long)] + pub adapter: Option, /// Path to the typed app-config file (default: `.toml` /// resolved from the manifest's `[app].name`, next to the manifest). #[arg(long)] @@ -614,13 +625,28 @@ pub struct ConfigValidateArgs { pub manifest: PathBuf, /// Skip the `__…__` env-var overlay when loading the /// typed app-config. The default loads the overlay so validation - /// sees the same values the runtime would. + /// sees the values the runtime would. #[arg(long)] pub no_env: bool, + /// Explicit absolute native bindings file for bindings-only validation. + #[arg(long)] + pub store_bindings: Option, /// Strict mode: additionally check capability-aware completeness /// for the declared adapter set and well-formed handler paths. #[arg(long)] pub strict: bool, + /// Native Cargo target control to normalize; provider availability remains unknown. + #[arg(long, requires = "store_bindings")] + pub target: Option, + /// Normalize the all-features target control without resolving provider availability. + #[arg(long)] + pub target_all_features: bool, + /// Additional native target feature controls; bindings validation remains structural. + #[arg(long, value_delimiter = ',')] + pub target_features: Vec, + /// Suppress target default features. + #[arg(long)] + pub target_no_default_features: bool, } impl Default for ConfigValidateArgs { @@ -628,10 +654,16 @@ impl Default for ConfigValidateArgs { #[inline] fn default() -> Self { Self { + adapter: None, app_config: None, manifest: default_manifest_path(), no_env: false, + store_bindings: None, strict: false, + target: None, + target_all_features: false, + target_features: Vec::new(), + target_no_default_features: false, } } } @@ -745,6 +777,7 @@ mod tests { assert_eq!(args.manifest, PathBuf::from("edgezero.toml")); assert!(args.adapter.is_empty()); assert!(args.app_config.is_none()); + assert!(args.store_bindings.is_none()); assert!(!args.dry_run); assert!(args.key.is_none()); assert!(!args.local); @@ -760,6 +793,12 @@ mod tests { let args = ConfigValidateArgs::default(); assert_eq!(args.manifest, PathBuf::from("edgezero.toml")); assert!(args.app_config.is_none()); + assert!(args.adapter.is_none()); + assert!(args.store_bindings.is_none()); + assert!(args.target.is_none()); + assert!(args.target_features.is_empty()); + assert!(!args.target_all_features); + assert!(!args.target_no_default_features); assert!(!args.no_env); assert!(!args.strict); } @@ -848,6 +887,39 @@ mod tests { assert!(!validate.strict); } + #[test] + fn config_validate_parses_native_bindings_target_controls() { + let args = Args::try_parse_from([ + "edgezero", + "config", + "validate", + "--adapter", + "axum", + "--store-bindings", + "/tmp/native.toml", + "--target", + "x86_64-unknown-linux-gnu", + "--target-features", + "aws-appconfig-agent,aws-secrets-manager", + "--target-no-default-features", + ]) + .expect("parse native bindings validation"); + let Command::Config(ConfigCmd::Validate(validate)) = args.cmd else { + panic!("expected config validate"); + }; + assert_eq!(validate.adapter.as_deref(), Some("axum")); + assert_eq!( + validate.store_bindings, + Some(PathBuf::from("/tmp/native.toml")) + ); + assert_eq!(validate.target.as_deref(), Some("x86_64-unknown-linux-gnu")); + assert_eq!( + validate.target_features, + ["aws-appconfig-agent", "aws-secrets-manager"] + ); + assert!(validate.target_no_default_features); + } + #[test] fn config_gc_parses_explicit_dry_run() { let args = Args::try_parse_from([ diff --git a/crates/edgezero-cli/src/config.rs b/crates/edgezero-cli/src/config.rs index 7a820269c..db69853ad 100644 --- a/crates/edgezero-cli/src/config.rs +++ b/crates/edgezero-cli/src/config.rs @@ -27,6 +27,13 @@ use crate::ensure_adapter_defined; use edgezero_adapter::registry::{ self as adapter_registry, ReadConfigEntry, ResolvedStoreId, TypedSecretEntry, }; +#[cfg(feature = "edgezero-adapter-axum")] +use edgezero_adapter_axum::native_bindings::{ + NativeBindingsLimits, NativeConfigBinding, NativeProviderCapabilities, NativeSourceSelection, + NativeStoreBindings, NativeStoreDeclarations, NativeSuppliedIds, +}; +#[cfg(feature = "edgezero-adapter-axum")] +use edgezero_adapter_axum::native_build::NativeCargoSelection; use edgezero_core::app_config::{ self, AppConfigError, AppConfigLoadOptions, AppConfigMeta, SecretField, SecretKind, SecretPathSegment, @@ -217,6 +224,10 @@ struct ResolvedTomlLeaf<'raw> { /// Returns a human-readable error string on any validation failure. #[inline] pub fn run_config_validate(args: &ConfigValidateArgs) -> Result<(), String> { + if args.store_bindings.is_some() || args.adapter.is_some() { + return run_native_bindings_validate(args); + } + reject_target_controls_without_bindings(args)?; let ctx = load_validation_context(args)?; run_shared_checks(&ctx, None)?; log::info!( @@ -236,6 +247,10 @@ pub fn run_config_validate_typed(args: &ConfigValidateArgs) -> Result<(), Str where C: DeserializeOwned + Validate + AppConfigMeta, { + if args.store_bindings.is_some() || args.adapter.is_some() { + return run_native_bindings_validate(args); + } + reject_target_controls_without_bindings(args)?; let ctx = load_validation_context(args)?; run_shared_checks(&ctx, None)?; @@ -443,6 +458,7 @@ pub fn run_config_push_typed(args: &ConfigPushArgs) -> Result<(), String> where C: DeserializeOwned + Serialize + Validate + AppConfigMeta, { + reject_remote_bound_config_push(args)?; // Pre-flight: load + validate. let ctx = load_push_context(args)?; run_shared_checks(&ctx.validation, Some(&args.adapter))?; @@ -606,6 +622,12 @@ where // adapter_typed_checks; no consent gate, no re-fetch). let validate_args = ConfigValidateArgs { app_config: args.app_config.clone(), + adapter: None, + store_bindings: None, + target: None, + target_features: Vec::new(), + target_all_features: false, + target_no_default_features: false, manifest: args.manifest.clone(), no_env: args.no_env, strict: false, @@ -1329,6 +1351,12 @@ fn load_push_context(args: &ConfigPushArgs) -> Result { // alongside the schema and selected-adapter shared checks. let validate_args = ConfigValidateArgs { app_config: args.app_config.clone(), + adapter: None, + store_bindings: None, + target: None, + target_features: Vec::new(), + target_all_features: false, + target_no_default_features: false, manifest: args.manifest.clone(), no_env: args.no_env, strict: true, @@ -1464,6 +1492,142 @@ fn generated_at_rfc3339() -> String { chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Secs, true) } +fn reject_target_controls_without_bindings(args: &ConfigValidateArgs) -> Result<(), String> { + if args.target.is_some() + || !args.target_features.is_empty() + || args.target_all_features + || args.target_no_default_features + { + return Err( + "native target controls require `--adapter axum --store-bindings `" + .to_owned(), + ); + } + Ok(()) +} + +#[cfg(feature = "edgezero-adapter-axum")] +fn load_validated_native_bindings( + manifest: &Manifest, + path: &Path, +) -> Result { + let limits = NativeBindingsLimits::default(); + let bindings = NativeStoreBindings::read_with_policy(path, limits, None) + .map_err(|_error| "invalid native store bindings".to_owned())?; + let declarations = NativeStoreDeclarations::try_from(&manifest.stores) + .map_err(|_error| "invalid store declarations for native bindings".to_owned())?; + NativeSourceSelection::validate( + declarations, + &bindings, + &NativeSuppliedIds::default(), + &EnvConfig::default(), + NativeProviderCapabilities { + appconfig_agent: true, + secrets_manager: true, + dynamodb: true, + celld_config: true, + celld_kv: true, + }, + limits, + None, + ) + .map_err(|_error| "native store bindings do not match manifest declarations".to_owned())?; + Ok(bindings) +} + +/// Validate only a caller-selected native bindings file. This path intentionally +/// does not load app TOML, ambient overlays, tokens, credentials, or deployment env. +#[cfg(feature = "edgezero-adapter-axum")] +fn run_native_bindings_validate(args: &ConfigValidateArgs) -> Result<(), String> { + let adapter = args + .adapter + .as_deref() + .ok_or_else(|| "bindings-only validation requires `--adapter axum`".to_owned())?; + if !adapter.eq_ignore_ascii_case("axum") { + return Err("bindings-only validation is supported only for `--adapter axum`".to_owned()); + } + let path = args.store_bindings.as_deref().ok_or_else(|| { + "bindings-only validation requires `--store-bindings `".to_owned() + })?; + if args.app_config.is_some() { + return Err("`--app-config` cannot be used with bindings-only validation".to_owned()); + } + let manifest = ManifestLoader::from_path(&args.manifest) + .map_err(|_error| "failed to load application manifest".to_owned())?; + load_validated_native_bindings(manifest.manifest(), path)?; + let mut target_args = Vec::new(); + for feature in &args.target_features { + target_args.extend(["--features".to_owned(), feature.clone()]); + } + if args.target_all_features { + target_args.push("--all-features".to_owned()); + } + if args.target_no_default_features { + target_args.push("--no-default-features".to_owned()); + } + if let Some(target) = &args.target { + target_args.extend(["--target".to_owned(), target.clone()]); + } + // Normalize and validate target controls, but this first version deliberately + // makes no provider-availability claim and runs no Cargo command. + let manifest_features = manifest + .manifest() + .adapter_entry("axum") + .map_or_else(Vec::new, |(_name, definition)| { + definition.build.features.clone() + }); + let selection = NativeCargoSelection::normalize_args(&manifest_features, &target_args)?; + log::info!( + "[edgezero] native store bindings structurally valid; target provider availability unknown (target={}, requested_features={}, all_features={}, no_default_features={})", + selection.target.as_deref().unwrap_or("host"), + selection.requested_features.len(), + selection.all_features, + selection.no_default_features, + ); + Ok(()) +} + +#[cfg(not(feature = "edgezero-adapter-axum"))] +fn run_native_bindings_validate(_args: &ConfigValidateArgs) -> Result<(), String> { + Err("bindings-only validation requires the CLI Axum native-bindings feature".to_owned()) +} + +#[cfg(feature = "edgezero-adapter-axum")] +fn reject_remote_bound_config_push(args: &ConfigPushArgs) -> Result<(), String> { + let Some(path) = args.store_bindings.as_deref() else { + return Ok(()); + }; + if !args.adapter.eq_ignore_ascii_case("axum") { + return Err("`--store-bindings` push selection requires `--adapter axum`".to_owned()); + } + if !path.is_absolute() { + return Err("`--store-bindings` must be an absolute path".to_owned()); + } + let manifest = ManifestLoader::from_path(&args.manifest) + .map_err(|_error| "failed to load application manifest".to_owned())?; + let logical = resolve_config_store_id(args.store.as_deref(), manifest.manifest())?; + let bindings = load_validated_native_bindings(manifest.manifest(), path)?; + let remote = match bindings.config().get(&logical) { + Some(NativeConfigBinding::AppConfigAgent(_)) => Some("AWS"), + Some(NativeConfigBinding::Celld(_)) => Some("celld"), + None => None, + }; + if let Some(provider) = remote { + return Err(format!( + "config push does not publish to {provider}-bound native config stores; update the selected binding through its deployment workflow" + )); + } + Ok(()) +} + +#[cfg(not(feature = "edgezero-adapter-axum"))] +fn reject_remote_bound_config_push(args: &ConfigPushArgs) -> Result<(), String> { + if args.store_bindings.is_some() { + return Err("`--store-bindings` requires the CLI Axum native-bindings feature".to_owned()); + } + Ok(()) +} + fn load_validation_context(args: &ConfigValidateArgs) -> Result { let manifest_loader = ManifestLoader::from_path(&args.manifest) .map_err(|err| format!("failed to load {}: {err}", args.manifest.display()))?; @@ -2012,6 +2176,8 @@ mod tests { use std::borrow::Cow; use std::fs; + use std::io::ErrorKind; + use std::net::TcpListener; #[cfg(unix)] use std::sync::Mutex; use tempfile::TempDir; @@ -2436,6 +2602,12 @@ source = "target/wasm32-wasip2/release/demo.wasm" fn args_for(manifest: &Path) -> ConfigValidateArgs { ConfigValidateArgs { app_config: None, + adapter: None, + store_bindings: None, + target: None, + target_features: Vec::new(), + target_all_features: false, + target_no_default_features: false, manifest: manifest.to_path_buf(), no_env: true, // tests don't want env leakage strict: false, @@ -2446,6 +2618,7 @@ source = "target/wasm32-wasip2/release/demo.wasm" ConfigPushArgs { adapter: adapter.to_owned(), app_config: None, + store_bindings: None, dry_run: false, key: None, local: false, @@ -2461,6 +2634,228 @@ source = "target/wasm32-wasip2/release/demo.wasm" // ---------- raw flow ---------- + #[cfg(feature = "edgezero-adapter-axum")] + const AWS_BINDINGS_FIXTURE: &str = r#"version = 1 +[config.remote] +provider = "aws-appconfig-agent" +default_key = "main" +endpoint = "http://127.0.0.1:2772" +[config.remote.documents.main] +application = "demo-app" +environment = "production" +profile = "envelope" +"#; + + #[cfg(feature = "edgezero-adapter-axum")] + #[test] + fn bindings_only_validation_does_not_require_app_toml() { + let manifest_text = r#" +[app] +name = "demo-app" +[adapters.axum.adapter] +crate = "crates/demo-axum" +[adapters.axum.commands] +build = "echo" +deploy = "echo" +serve = "echo" +[stores.config] +ids = ["remote"] +default = "remote" +"#; + let (dir, manifest, app_config) = setup_project(manifest_text, "not used"); + fs::remove_file(&app_config).expect("remove local config"); + let bindings_path = dir.path().join("bindings.toml"); + fs::write(&bindings_path, AWS_BINDINGS_FIXTURE).expect("write native bindings"); + let args = ConfigValidateArgs { + app_config: None, + adapter: Some("axum".to_owned()), + store_bindings: Some(bindings_path), + target: None, + target_features: Vec::new(), + target_all_features: false, + target_no_default_features: false, + manifest, + no_env: false, + strict: false, + }; + run_config_validate(&args).expect("structural validation does not load app TOML"); + } + + #[cfg(feature = "edgezero-adapter-axum")] + #[test] + fn push_guard_rejects_only_the_selected_aws_config_id() { + let manifest_text = PUSH_MANIFEST.replace( + "ids = [\"app_config\"]", + "ids = [\"app_config\", \"remote\"]\ndefault = \"app_config\"", + ); + let (dir, manifest, app_config) = setup_project(&manifest_text, VALID_APP_CONFIG); + fs::remove_file(app_config).expect("guard must not require local app TOML"); + let bindings_dir = tempfile::tempdir().expect("bindings directory"); + let bindings = bindings_dir.path().join("bindings.toml"); + fs::write( + &bindings, + AWS_BINDINGS_FIXTURE.replace("remote", "app_config"), + ) + .expect("write bindings"); + let mut args = push_args(&manifest, "axum"); + args.store_bindings = Some(bindings.clone()); + let error = reject_remote_bound_config_push(&args).expect_err("AWS-bound push is refused"); + assert!(error.contains("does not publish to AWS-bound")); + for (local, dry_run) in [(false, false), (true, false), (false, true), (true, true)] { + args.local = local; + args.dry_run = dry_run; + let push_error = run_config_push_typed::(&args) + .expect_err("public typed push refuses AWS before loading app TOML"); + assert!(push_error.contains("does not publish to AWS-bound")); + assert!( + !dir.path().join(".edgezero").exists(), + "refusal performs no local write" + ); + } + + fs::write(&bindings, AWS_BINDINGS_FIXTURE).expect("write unrelated AWS binding"); + reject_remote_bound_config_push(&args).expect("unrelated declared AWS binding is allowed"); + fs::write(&bindings, AWS_BINDINGS_FIXTURE.replace("remote", "unknown")) + .expect("write typo"); + reject_remote_bound_config_push(&args) + .expect_err("undeclared bindings are not a valid push selection"); + } + + #[cfg(feature = "edgezero-adapter-axum")] + #[test] + fn bindings_only_validation_and_push_share_structural_errors() { + let (dir, manifest, app_config) = setup_project(PUSH_MANIFEST, FIXTURE_APP_CONFIG); + fs::remove_file(app_config).expect("neither path needs app TOML for this rejection"); + let bindings = dir.path().join("bindings.toml"); + let mut validate = args_for(&manifest); + validate.adapter = Some("axum".to_owned()); + validate.store_bindings = Some(bindings.clone()); + let mut push = push_args(&manifest, "axum"); + push.store_bindings = Some(bindings.clone()); + for (input, expected) in [ + ("not valid TOML", "invalid native store bindings"), + ( + AWS_BINDINGS_FIXTURE, + "native store bindings do not match manifest declarations", + ), + ] { + fs::write(&bindings, input).expect("write invalid bindings"); + assert_eq!( + run_config_validate(&validate).expect_err("invalid bindings"), + expected + ); + assert_eq!( + run_config_push_typed::(&push).expect_err("invalid bindings"), + expected, + ); + assert!(!dir.path().join(".edgezero").exists()); + } + } + + #[cfg(feature = "edgezero-adapter-axum")] + const CELLD_BINDINGS_FIXTURE: &str = r#"version = 1 +[config.remote] +provider = "celld-config" +endpoint = "https://127.0.0.1:7780" +default_key = "main" +[config.remote.access] +mode = "authenticated" +token_env = "CELLD_TEST_TOKEN_MUST_NOT_BE_RESOLVED" +[config.remote.documents.main] +store = "checkout-config" +key = "application-envelope" +"#; + + #[cfg(feature = "edgezero-adapter-axum")] + #[test] + fn celld_bindings_only_validation_does_not_resolve_tokens_features_or_endpoints() { + let manifest_text = PUSH_MANIFEST.replace( + "ids = [\"app_config\"]", + "ids = [\"app_config\", \"remote\"]\ndefault = \"app_config\"", + ); + let (dir, manifest, app_config) = setup_project(&manifest_text, VALID_APP_CONFIG); + fs::remove_file(app_config).expect("structural validation needs no local app config"); + let listener = TcpListener::bind("127.0.0.1:0").expect("probe sentinel"); + listener + .set_nonblocking(true) + .expect("nonblocking sentinel"); + let endpoint = format!( + "https://{}", + listener.local_addr().expect("sentinel address") + ); + let bindings = dir.path().join("celld.toml"); + fs::write( + &bindings, + CELLD_BINDINGS_FIXTURE.replace("https://127.0.0.1:7780", &endpoint), + ) + .expect("write celld bindings"); + let mut args = args_for(&manifest); + args.adapter = Some("axum".to_owned()); + args.store_bindings = Some(bindings); + args.target_features = vec!["a-feature-that-does-not-exist".to_owned()]; + args.target_no_default_features = true; + run_config_validate(&args).expect("valid structure has unknown target availability"); + assert!(matches!(listener.accept(), Err(error) if error.kind() == ErrorKind::WouldBlock)); + assert!(!dir.path().join(".edgezero").exists()); + } + + #[cfg(feature = "edgezero-adapter-axum")] + #[test] + fn celld_push_refuses_only_selected_remote_config_without_local_side_effects() { + let manifest_text = PUSH_MANIFEST.replace( + "ids = [\"app_config\"]", + "ids = [\"app_config\", \"remote\"]\ndefault = \"app_config\"", + ); + let (dir, manifest, app_config) = setup_project(&manifest_text, VALID_APP_CONFIG); + fs::remove_file(app_config).expect("remote refusal precedes app TOML loading"); + let bindings = dir.path().join("celld.toml"); + fs::write( + &bindings, + CELLD_BINDINGS_FIXTURE.replace("config.remote", "config.app_config"), + ) + .expect("write selected celld binding"); + let mut args = push_args(&manifest, "axum"); + args.store_bindings = Some(bindings.clone()); + for (local, dry_run) in [(false, false), (true, false), (false, true), (true, true)] { + args.local = local; + args.dry_run = dry_run; + let error = run_config_push_typed::(&args) + .expect_err("remote publication unsupported"); + assert!(error.contains("does not publish to celld-bound")); + assert!(!error.contains("CELLD_TEST_TOKEN")); + assert!(!dir.path().join(".edgezero").exists()); + } + fs::write(&bindings, CELLD_BINDINGS_FIXTURE).expect("unrelated remote config"); + reject_remote_bound_config_push(&args) + .expect("local selected ID is not blocked by another remote ID"); + } + + #[cfg(feature = "edgezero-adapter-axum")] + #[test] + fn celld_validation_and_push_share_strict_structural_rejection() { + let (dir, manifest, app_config) = setup_project(PUSH_MANIFEST, FIXTURE_APP_CONFIG); + fs::remove_file(app_config).expect("structural errors need no app TOML"); + let bindings = dir.path().join("celld.toml"); + let mut validate = args_for(&manifest); + validate.adapter = Some("axum".to_owned()); + validate.store_bindings = Some(bindings.clone()); + let mut push = push_args(&manifest, "axum"); + push.store_bindings = Some(bindings.clone()); + for input in [ + CELLD_BINDINGS_FIXTURE.to_owned(), + CELLD_BINDINGS_FIXTURE.replace("celld-config", "celld-secret"), + CELLD_BINDINGS_FIXTURE.replace("default_key = \"main\"", "default_key = \"missing\""), + ] { + fs::write(&bindings, input).expect("invalid document"); + let validation = run_config_validate(&validate).expect_err("invalid bindings"); + assert_eq!( + run_config_push_typed::(&push).expect_err("invalid push bindings"), + validation + ); + assert!(!dir.path().join(".edgezero").exists()); + } + } + #[test] fn raw_validates_a_well_formed_project() { let _lock = manifest_guard().lock().expect("manifest guard"); @@ -3378,6 +3773,34 @@ default = "alpha" ); } + #[cfg(feature = "edgezero-adapter-axum")] + #[test] + fn axum_multi_secret_ids_pass_strict_validation_and_typed_push() { + let _lock = manifest_guard().lock().expect("manifest guard"); + let manifest_text = PUSH_MANIFEST.replace( + "ids = [\"default\"]", + "ids = [\"default\", \"extra\"]\ndefault = \"default\"", + ); + let (dir, manifest, _) = setup_project(&manifest_text, FIXTURE_APP_CONFIG); + let mut args = args_for(&manifest); + args.strict = true; + run_config_validate(&args).expect("Axum accepts multiple secret IDs"); + run_config_validate_typed::(&args) + .expect("typed Axum validation accepts multiple secret IDs"); + + let mut push = push_args(&manifest, "axum"); + push.yes = true; + run_config_push_typed::(&push) + .expect("Axum push accepts multiple secret IDs"); + let raw = fs::read_to_string(dir.path().join(".edgezero/local-config-app_config.json")) + .expect("push wrote the local snapshot"); + let entries: BTreeMap = serde_json::from_str(&raw).expect("snapshot map"); + let envelope: BlobEnvelope = + serde_json::from_str(&entries["app_config"]).expect("complete envelope"); + assert_eq!(envelope.data["greeting"], "hello"); + assert_eq!(envelope.data["api_token"], "demo_api_token"); + } + #[test] fn strict_handler_paths_rejects_malformed_handler() { let _lock = manifest_guard().lock().expect("manifest guard"); @@ -4308,12 +4731,11 @@ default = "one" "spin_manifest_version = 2\n[application]\nname = \"x\"\nversion = \"0\"\n[component.demo]\nsource = \"a.wasm\"\n", ) .expect("write spin.toml"); - // The selected adapter is Single-capable too, so its capability - // check must still reject before any push. - let err = run_config_push_typed::(&push_args(&manifest, "axum")) + // Only the selected adapter's capability check rejects the push. + let err = run_config_push_typed::(&push_args(&manifest, "spin")) .expect_err("Single-capable adapter with multi-id store must fail preflight"); assert!( - err.contains("axum") && err.contains("Single") && err.contains("secrets"), + err.contains("spin") && err.contains("Single") && err.contains("secrets"), "error must come from --strict capability check: {err}" ); } diff --git a/crates/edgezero-cli/src/generator.rs b/crates/edgezero-cli/src/generator.rs index 42070284c..d8c9195f5 100644 --- a/crates/edgezero-cli/src/generator.rs +++ b/crates/edgezero-cli/src/generator.rs @@ -1440,6 +1440,43 @@ mod tests { } fn assert_generated_axum_entrypoint(project_dir: &Path) { + let axum_cargo = + fs::read_to_string(project_dir.join("crates/demo-app-adapter-axum/Cargo.toml")) + .expect("read generated axum Cargo.toml"); + let parsed: toml::Value = + toml::from_str(&axum_cargo).expect("parse generated Axum Cargo.toml"); + let features = parsed + .get("features") + .and_then(toml::Value::as_table) + .expect("generated native features"); + for feature in [ + "aws-appconfig-agent", + "aws-secrets-manager", + "aws-dynamodb", + "celld-config", + "celld-kv", + ] { + assert_eq!( + features.get(feature), + Some(&toml::Value::Array(vec![toml::Value::String(format!( + "edgezero-adapter-axum/{feature}" + ))])), + "incorrect generated native forwarding for {feature}" + ); + } + assert_eq!( + features.get("default"), + Some(&toml::Value::Array(Vec::new())), + "native providers must be opt-in" + ); + assert_eq!( + features.get("aws-stores"), + Some(&toml::Value::Array(vec![ + toml::Value::String("aws-appconfig-agent".to_owned()), + toml::Value::String("aws-secrets-manager".to_owned()) + ])), + "AWS alias membership must not acquire other providers" + ); let axum_main = fs::read_to_string(project_dir.join("crates/demo-app-adapter-axum/src/main.rs")) .expect("read axum main.rs"); @@ -1447,7 +1484,7 @@ mod tests { !axum_main.contains("process::exit") && !axum_main.contains("EdgeZeroAxumService") && !axum_main.contains("into_axum_response") - && axum_main.contains("run_app::()"), + && axum_main.contains("run_generated_app::()"), "generated Axum entrypoint must use the connection-owning runner", ); } diff --git a/crates/edgezero-cli/src/provision.rs b/crates/edgezero-cli/src/provision.rs index 723576f9e..f8c49fc60 100644 --- a/crates/edgezero-cli/src/provision.rs +++ b/crates/edgezero-cli/src/provision.rs @@ -189,6 +189,27 @@ mod tests { .expect("axum dry-run also exits 0"); } + #[cfg(feature = "edgezero-adapter-axum")] + #[test] + fn axum_multi_secret_ids_pass_provision() { + let _lock = manifest_guard().lock().expect("manifest guard"); + let temp = TempDir::new().expect("temp dir"); + let manifest_path = temp.path().join("edgezero.toml"); + let manifest = PROVISION_MANIFEST.replace( + "ids = [\"default\"]", + "ids = [\"default\", \"extra\"]\ndefault = \"default\"", + ); + fs::write(&manifest_path, manifest).expect("write manifest"); + for dry_run in [true, false] { + run_provision(&ProvisionArgs { + adapter: "axum".to_owned(), + dry_run, + manifest: manifest_path.clone(), + }) + .expect("Axum provision accepts multiple secret IDs"); + } + } + #[test] fn run_provision_errors_on_unknown_adapter() { let _lock = manifest_guard().lock().expect("manifest guard"); diff --git a/crates/edgezero-cli/tests/celld_generated_proof.rs b/crates/edgezero-cli/tests/celld_generated_proof.rs new file mode 100644 index 000000000..a6e3f666d --- /dev/null +++ b/crates/edgezero-cli/tests/celld_generated_proof.rs @@ -0,0 +1,787 @@ +//! Explicit generated native consumer proofs for the scoped API embedded in celld. +//! Ordinary tests require no service. The ignored real-service proof requires an +//! explicit `CELLD_BIN`, curl and OpenSSL; missing tools are failures, not skips. + +#[cfg(test)] +#[cfg(unix)] +mod tests { + #![expect( + clippy::arbitrary_source_item_ordering, + reason = "keep fixture ownership, generated source and proof steps together" + )] + + use std::collections::BTreeMap; + use std::env; + use std::fs; + use std::io::{ErrorKind, Read as _, Write as _}; + use std::net::{SocketAddr, TcpListener, TcpStream}; + use std::path::{Path, PathBuf}; + use std::process::{Child, Command, Stdio}; + use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + use std::sync::{Arc, Mutex}; + use std::thread::{self, JoinHandle}; + use std::time::{Duration, Instant}; + + use edgezero_core::blob_envelope::BlobEnvelope; + use serde_json::json; + use sha2::{Digest as _, Sha256}; + use tempfile::TempDir; + use url::Url; + + const APP_TOKEN: &str = "fixture-app-token-not-a-production-credential-1234567890"; + const OPERATOR_TOKEN: &str = "fixture-operator-token-not-a-production-credential-123456"; + const WAIT: Duration = Duration::from_secs(30); + + fn deadline(duration: Duration) -> Instant { + Instant::now() + .checked_add(duration) + .expect("representable proof deadline") + } + + fn envelope(greeting: &str) -> Vec { + serde_json::to_vec(&BlobEnvelope::new( + json!({"greeting":greeting}), + "2026-10-15T12:00:00Z".to_owned(), + )) + .expect("whole configuration envelope") + } + + type Rows = BTreeMap<(String, String), Vec>; + + struct Fixture { + endpoint: String, + rows: Arc>, + requests: Arc, + config_reads: Arc, + stop: Arc, + worker: Option>, + } + + impl Fixture { + fn start() -> Self { + let listener = TcpListener::bind("127.0.0.1:0").expect("scoped fixture"); + listener.set_nonblocking(true).expect("nonblocking fixture"); + let endpoint = format!("http://{}", listener.local_addr().expect("fixture address")); + let rows = Arc::new(Mutex::new(BTreeMap::from([( + ("app-config".to_owned(), "settings".to_owned()), + envelope("first-config"), + )]))); + let requests = Arc::new(AtomicUsize::new(0)); + let config_reads = Arc::new(AtomicUsize::new(0)); + let stop = Arc::new(AtomicBool::new(false)); + let (worker_rows, worker_requests, worker_config, worker_stop) = ( + Arc::clone(&rows), + Arc::clone(&requests), + Arc::clone(&config_reads), + Arc::clone(&stop), + ); + let worker = thread::spawn(move || { + let end = deadline(Duration::from_mins(10)); + while !worker_stop.load(Ordering::Relaxed) && Instant::now() < end { + match listener.accept() { + Ok((mut stream, peer)) => { + assert!(peer.ip().is_loopback()); + stream + .set_read_timeout(Some(Duration::from_secs(3))) + .expect("fixture timeout"); + stream + .set_write_timeout(Some(Duration::from_secs(3))) + .expect("fixture timeout"); + worker_requests.fetch_add(1, Ordering::Relaxed); + let (method, target, body) = read_request(&mut stream); + let (status, payload, kind) = fixture_response( + &method, + &target, + &body, + &worker_rows, + &worker_config, + ); + write!(stream,"HTTP/1.1 {status}\r\nx-celld-store-version: 1\r\ncontent-type: {kind}\r\ncontent-length: {}\r\nconnection: close\r\n\r\n",payload.len()).expect("fixture headers"); + stream.write_all(&payload).expect("fixture body"); + } + Err(error) if error.kind() == ErrorKind::WouldBlock => { + thread::sleep(Duration::from_millis(5)); + } + Err(error) => panic!("scoped fixture accept: {error}"), + } + } + }); + Self { + endpoint, + rows, + requests, + config_reads, + stop, + worker: Some(worker), + } + } + } + impl Drop for Fixture { + fn drop(&mut self) { + self.stop.store(true, Ordering::Relaxed); + if let Some(worker) = self.worker.take() { + worker.join().expect("reap scoped fixture"); + } + } + } + + fn read_request(stream: &mut TcpStream) -> (String, String, Vec) { + let mut bytes = Vec::new(); + loop { + let mut byte = [0]; + stream.read_exact(&mut byte).expect("fixture request head"); + bytes.push(byte[0]); + assert!(bytes.len() <= 16 * 1024, "bounded fixture head"); + if bytes.ends_with(b"\r\n\r\n") { + break; + } + } + let head = String::from_utf8(bytes).expect("ASCII fixture head"); + let mut line = head + .lines() + .next() + .expect("request line") + .split_ascii_whitespace(); + let method = line.next().expect("method").to_owned(); + let target = line.next().expect("target").to_owned(); + let length = head + .lines() + .filter_map(|header| header.split_once(':')) + .find_map(|(key, value)| { + key.eq_ignore_ascii_case("content-length") + .then(|| value.trim().parse::().expect("length")) + }) + .unwrap_or(0); + assert!(length <= 1024 * 1024, "bounded fixture value"); + let mut body = vec![0; length]; + stream.read_exact(&mut body).expect("fixture request body"); + (method, target, body) + } + + fn fixture_response( + method: &str, + target: &str, + body: &[u8], + contents: &Mutex, + reads: &AtomicUsize, + ) -> (&'static str, Vec, &'static str) { + let url = Url::parse(&format!("http://fixture{target}")).expect("fixture URL"); + let parts = url.path().split('/').collect::>(); + let parameters = url.query_pairs().into_owned().collect::>(); + let store = parts.get(3).expect("store").to_string(); + let value_key = parameters.get("key").cloned().unwrap_or_default(); + let mut rows = contents.lock().expect("fixture rows"); + if parts.last() == Some(&"keys") { + let prefix = parameters.get("prefix").map_or("", String::as_str); + let after = parameters + .get("cursor") + .and_then(|continuation| continuation.strip_prefix("after:")) + .unwrap_or(""); + let limit = parameters + .get("limit") + .map_or(100, |value| value.parse::().expect("page limit")); + let available_keys = rows + .keys() + .filter(|(alias, key)| { + alias == &store && key.starts_with(prefix) && key.as_str() > after + }) + .map(|(_, key)| key.clone()) + .collect::>(); + let more = available_keys.len() > limit; + let keys = available_keys.into_iter().take(limit).collect::>(); + let cursor = + more.then(|| format!("after:{}", keys.last().expect("nonempty continuing page"))); + return ( + "200 OK", + serde_json::to_vec(&json!({"keys":keys,"cursor":cursor})).expect("page"), + "application/json", + ); + } + let pair = (store.clone(), value_key); + match method { + "GET" | "HEAD" => { + if store == "app-config" { + reads.fetch_add(1, Ordering::Relaxed); + } + match rows.get(&pair) { + Some(value) => ( + "200 OK", + if method == "HEAD" { + Vec::new() + } else { + value.clone() + }, + "application/octet-stream", + ), + None => ( + "404 Not Found", + if method == "HEAD" { + Vec::new() + } else { + serde_json::to_vec(&json!({"error":"not_found"})).expect("miss") + }, + "application/json", + ), + } + } + "PUT" => { + rows.insert(pair, body.to_vec()); + ("204 No Content", Vec::new(), "application/octet-stream") + } + "DELETE" => { + rows.remove(&pair); + ("204 No Content", Vec::new(), "application/octet-stream") + } + _ => panic!("unexpected scoped fixture method"), + } + } + + struct App { + child: Child, + address: SocketAddr, + log: PathBuf, + } + impl Drop for App { + fn drop(&mut self) { + let _killed = self.child.kill(); + let _reaped = self.child.wait(); + } + } + impl App { + fn start( + binary: &Path, + project: &Path, + bindings: &Path, + mode: &str, + tls: Option<(&str, &Path)>, + ) -> Self { + let listener = TcpListener::bind("127.0.0.1:0").expect("app address"); + let address = listener.local_addr().expect("app address"); + drop(listener); + let log = project.join(format!("app-{}-{mode}.log", address.port())); + let mut command = Command::new(binary); + command + .current_dir(project) + .env_clear() + .env("HOME", project.join("empty-home")) + .env("AWS_EC2_METADATA_DISABLED", "true") + .env("EDGEZERO__STORE_BINDINGS_FILE", bindings) + .env("EDGEZERO__ADAPTER__HOST", "127.0.0.1") + .env("EDGEZERO__ADAPTER__PORT", address.port().to_string()) + .env("CELLD_PROOF_MODE", mode) + .stdout(Stdio::null()) + .stderr(Stdio::from(fs::File::create(&log).expect("app log"))); + if let Some((token, ca)) = tls { + command + .env("CELLD_APP_TOKEN", token) + .env("SSL_CERT_FILE", ca); + } + Self { + child: command.spawn().expect("generated app"), + address, + log, + } + } + fn request(&mut self) -> String { + let end = deadline(WAIT); + loop { + if let Some(status) = self.child.try_wait().expect("app state") { + panic!( + "generated app exited {status}: {}", + fs::read_to_string(&self.log).unwrap_or_default() + ); + } + if let Ok(mut stream) = + TcpStream::connect_timeout(&self.address, Duration::from_millis(50)) + { + stream + .set_read_timeout(Some(Duration::from_secs(10))) + .expect("app response timeout"); + write!( + stream, + "GET /celld-proof HTTP/1.1\r\nHost: 127.0.0.1\r\nConnection: close\r\n\r\n" + ) + .expect("app request"); + let mut response = String::new(); + stream.read_to_string(&mut response).expect("app response"); + return response; + } + assert!(Instant::now() < end, "app startup deadline"); + thread::sleep(Duration::from_millis(10)); + } + } + fn rejected(&mut self) { + let end = deadline(WAIT); + loop { + if let Some(status) = self.child.try_wait().expect("rejected app state") { + assert!(!status.success(), "selected unavailable provider must fail"); + break; + } + assert!( + Instant::now() < end, + "invalid source unexpectedly stayed alive" + ); + thread::sleep(Duration::from_millis(10)); + } + TcpStream::connect_timeout(&self.address, Duration::from_millis(50)) + .expect_err("rejected startup has no listener"); + } + } + + fn generate(root: &Path) -> PathBuf { + let output = Command::new(env!("CARGO_BIN_EXE_edgezero")) + .args(["new", "celld-proof", "--dir"]) + .arg(root) + .output() + .expect("real CLI scaffold"); + assert!( + output.status.success(), + "scaffold: {}", + String::from_utf8_lossy(&output.stderr) + ); + let project = root.join("celld-proof"); + let manifest = project.join("edgezero.toml"); + let mut text = fs::read_to_string(&manifest).expect("generated manifest"); + text.push_str( + r#" +[[triggers.http]] +id = "celld-proof" +path = "/celld-proof" +methods = ["GET"] +handler = "celld_proof_core::handlers::celld_probe" +adapters = ["axum"] +[stores.config] +ids = ["app_config"] +[stores.kv] +ids = ["state"] +"#, + ); + fs::write(manifest, text).expect("probe declarations"); + fs::write( + project.join("crates/celld-proof-core/src/config.rs"), + " +use serde::Deserialize; +use validator::Validate; +#[derive(Debug, Deserialize, Validate, edgezero_core::AppConfig)] +#[serde(deny_unknown_fields)] +pub struct CelldProofConfig { + #[validate(length(min = 3_u64))] + pub greeting: String, +} +pub struct StartupState { + pub greeting: String, + pub mutated: std::sync::atomic::AtomicBool, +} +", + ) + .expect("typed config"); + let handlers = project.join("crates/celld-proof-core/src/handlers.rs"); + let mut source = fs::read_to_string(&handlers).expect("generated handlers"); + source.push_str(r#" +#[action] +pub async fn celld_probe( + edgezero_core::extractor::State(state): edgezero_core::extractor::State>, + kv: edgezero_core::extractor::Kv, +) -> Result, EdgeError> { + let store = kv.default().ok_or_else(|| EdgeError::service_unavailable("KV missing"))?; + if !state.mutated.swap(true, std::sync::atomic::Ordering::SeqCst) { + match std::env::var("CELLD_PROOF_MODE").unwrap_or_default().as_str() { + "put" => { + store.put_bytes("probe/binary", vec![0,255,1,128].into()).await?; + store.put_bytes("probe/empty", Vec::new().into()).await?; + store.put_bytes_with_ttl("probe/ttl", vec![7].into(),std::time::Duration::from_secs(60)).await?; + } + "delete" => { store.delete("probe/binary").await?; } + _ => {} + } + } + let binary = store.get_bytes("probe/binary").await?.map(|v|v.to_vec()); + let empty = store.get_bytes("probe/empty").await?.is_some_and(|v|v.is_empty()); + let exists = store.exists("probe/binary").await?; + let ttl = store.get_bytes("probe/ttl").await?.is_some(); + let page = store.list_keys_page("probe/",None,1).await?; + let next = store.list_keys_page("probe/",page.cursor.as_deref(),1).await?; + Ok(Text::new(format!("greeting={};binary={binary:?};empty={empty};exists={exists};ttl={ttl};first={};next={}",state.greeting,page.keys.join(","),next.keys.join(",")))) +} +"#); + fs::write(handlers, source).expect("portable proof handler"); + fs::write(project.join("crates/celld-proof-adapter-axum/src/main.rs"),r#" +use std::sync::{Arc,atomic::AtomicBool}; +use edgezero_adapter_axum::dev_server::run_production_app_with_initializer; +use edgezero_core::{EdgeError,extractor::AppConfig}; +use celld_proof_core::config::{CelldProofConfig,StartupState}; +fn main() -> anyhow::Result<()> { + run_production_app_with_initializer::(|app,stores| { + Box::pin(async move { + let binding=stores.config().and_then(|r|r.named_ref("app_config")) + .ok_or_else(|| EdgeError::service_unavailable("config binding missing"))?; + let config=AppConfig::::from_binding(binding,None,stores.secrets(), + app.config_extraction_limits(),app.monotonic_clock()).await?; + app.insert_state(Arc::new(StartupState{greeting:config.greeting.clone(),mutated:AtomicBool::new(false)})); + Ok(()) + }) + })?; + Ok(()) +} +"#).expect("production initializer"); + project + } + + fn bindings(project: &Path, endpoint: &str, authenticated: bool) -> PathBuf { + let path = project.join("celld-bindings.toml"); + let access = if authenticated { + "mode = \"authenticated\"\ntoken_env = \"CELLD_APP_TOKEN\"" + } else { + "mode = \"trusted-network\"" + }; + fs::write( + &path, + format!( + r#"version = 1 +[config.app_config] +provider = "celld-config" +endpoint = "{endpoint}" +default_key = "settings" +[config.app_config.access] +{access} +[config.app_config.documents.settings] +store = "app-config" +key = "settings" +[kv.state] +provider = "celld-kv" +endpoint = "{endpoint}" +store = "state" +[kv.state.access] +{access} +"# + ), + ) + .expect("native celld bindings"); + path + } + + fn build(project: &Path, features: Option<&str>) -> PathBuf { + let target = env::var_os("EDGEZERO_GENERATED_TEST_TARGET_DIR") + .map_or_else(|| project.join("target"), PathBuf::from); + let mut command = Command::new(env!("CARGO_BIN_EXE_edgezero")); + command + .current_dir(project) + .env("CARGO_TARGET_DIR", &target) + .args(["build", "--adapter", "axum", "--", "--offline"]); + if let Some(selected_features) = features { + command.args(["--features", selected_features]); + } + let output = command.output().expect("real CLI build"); + assert!( + output.status.success(), + "generated build: {}", + String::from_utf8_lossy(&output.stderr) + ); + target.join("debug/celld-proof-adapter-axum") + } + + #[test] + #[ignore = "builds actual generated native consumers; coordinator owns Cargo integration"] + fn generated_celld_default_off_kv_and_config_restart() { + let temp = TempDir::new().expect("project"); + let project = generate(temp.path()); + let fixture = Fixture::start(); + let binding = bindings(&project, &fixture.endpoint, false); + for features in [None, Some("celld-config"), Some("celld-kv")] { + let binary = build(&project, features); + let mut app = App::start(&binary, &project, &binding, "read", None); + app.rejected(); + assert_eq!( + fixture.requests.load(Ordering::Relaxed), + 0, + "all source validation must precede config I/O" + ); + } + let binary = build(&project, Some("celld-config,celld-kv")); + let mut first = App::start(&binary, &project, &binding, "put", None); + let response = first.request(); + assert!( + response.contains("200 OK") + && response.contains("greeting=first-config") + && response.contains("binary=Some([0, 255, 1, 128])") + && response.contains("empty=true") + && response.contains("exists=true") + && response.contains("first=probe/binary") + && response.contains("next=probe/empty"), + "generated live KV/config: {response}" + ); + assert_eq!(fixture.config_reads.load(Ordering::Relaxed), 1); + fixture.rows.lock().expect("fixture update").insert( + ("app-config".to_owned(), "settings".to_owned()), + envelope("second-config"), + ); + assert!(first.request().contains("greeting=first-config")); + assert_eq!( + fixture.config_reads.load(Ordering::Relaxed), + 1, + "requests use immutable startup snapshot" + ); + drop(first); + let mut second = App::start(&binary, &project, &binding, "read", None); + let restarted_response = second.request(); + assert!( + restarted_response.contains("greeting=second-config") + && restarted_response.contains("binary=Some([0, 255, 1, 128])"), + "restart adoption and persistence: {restarted_response}" + ); + drop(second); + let all_binary = build( + &project, + Some("celld-config,celld-kv,aws-stores,aws-dynamodb"), + ); + let mut coexistence = App::start(&all_binary, &project, &binding, "read", None); + let coexistence_response = coexistence.request(); + assert!( + coexistence_response.contains("greeting=second-config") + && coexistence_response.contains("empty=true"), + "compiled AWS providers do not require AWS inputs for celld-only bindings: {coexistence_response}" + ); + drop(coexistence); + let mut deleting = App::start(&binary, &project, &binding, "delete", None); + assert!(deleting.request().contains("binary=None")); + drop(deleting); + for value in [vec![255], Vec::new(), envelope("x")] { + fixture + .rows + .lock() + .expect("malformed config") + .insert(("app-config".to_owned(), "settings".to_owned()), value); + let mut failed = App::start(&binary, &project, &binding, "read", None); + failed.rejected(); + } + fixture + .rows + .lock() + .expect("missing config") + .remove(&("app-config".to_owned(), "settings".to_owned())); + let mut missing = App::start(&binary, &project, &binding, "read", None); + missing.rejected(); + assert!( + !project.join(".edgezero").exists(), + "remote-bound IDs create no local roots" + ); + } + + struct Celld { + child: Child, + endpoint: String, + cert: PathBuf, + } + impl Drop for Celld { + fn drop(&mut self) { + let _killed = self.child.kill(); + let _reaped = self.child.wait(); + } + } + impl Celld { + fn start(root: &Path) -> Self { + let binary = env::var_os("CELLD_BIN") + .map(PathBuf::from) + .expect("CELLD_BIN is required; this proof never silently skips"); + assert!( + binary.is_absolute() && binary.is_file(), + "explicit celld executable required" + ); + let cert = root.join("celld-cert.pem"); + let key = root.join("celld-key.pem"); + let certificate_status = Command::new("openssl") + .args([ + "req", + "-x509", + "-newkey", + "rsa:2048", + "-nodes", + "-days", + "1", + "-subj", + "/CN=celld-proof", + "-addext", + "subjectAltName=IP:127.0.0.1", + "-addext", + "basicConstraints=critical,CA:FALSE", + "-addext", + "extendedKeyUsage=serverAuth", + "-keyout", + ]) + .arg(&key) + .arg("-out") + .arg(&cert) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + .expect("explicit OpenSSL prerequisite"); + assert!(certificate_status.success()); + let policy = root.join("celld-policy.json"); + fs::write(&policy,serde_json::to_vec(&json!({ + "version":1_u8,"listen":"127.0.0.1:0","tls":{"cert":cert,"key":key}, + "stores":[{"alias":"app-config","namespace":"proof-config"},{"alias":"state","namespace":"proof-state"}], + "access":{"mode":"authenticated","tokens":[ + {"id":"app","sha256":format!("{:x}",Sha256::digest(APP_TOKEN)),"grants":[{"store":"app-config","operations":["read"]},{"store":"state","operations":["read","write","delete","list"]}]}, + {"id":"operator","sha256":format!("{:x}",Sha256::digest(OPERATOR_TOKEN)),"grants":[{"store":"app-config","operations":["read","write"]}]} + ]} + })).expect("operator policy")).expect("write operator policy"); + let log = root.join("celld.log"); + let output = fs::File::create(&log).expect("celld log"); + let child = Command::new(binary) + .current_dir(root) + .env_clear() + .env("CELLD_TEST_DATA_DIR", root.join("celld-data")) + .args([ + "--listen", + "127.0.0.1:0", + "--internal-listen", + "127.0.0.1:0", + "--store-api-config", + ]) + .arg(&policy) + .stdout(Stdio::from(output.try_clone().expect("log clone"))) + .stderr(Stdio::from(output)) + .spawn() + .expect("real celld process"); + let mut service = Self { + child, + endpoint: String::new(), + cert, + }; + let end = deadline(WAIT); + loop { + let text = fs::read_to_string(&log).expect("celld log"); + if let Some(address) = text.lines().find_map(|line| { + line.split_once("celld scoped stores listening on ") + .map(|(_, listen_address)| listen_address.trim()) + }) { + service.endpoint = format!("https://{address}"); + break; + } + if let Some(status) = service.child.try_wait().expect("celld state") { + panic!("celld startup {status}: {text}"); + } + assert!(Instant::now() < end, "real celld startup deadline"); + thread::sleep(Duration::from_millis(10)); + } + service + } + fn publish(&self, root: &Path, greeting: &str) { + let document = root.join("published-config.json"); + fs::write(&document, envelope(greeting)).expect("operator-owned document"); + let mut child = Command::new("curl") + .args([ + "--disable", + "--silent", + "--show-error", + "--noproxy", + "*", + "--max-time", + "10", + "--cacert", + ]) + .arg(&self.cert) + .args(["--request", "PUT", "--data-binary"]) + .arg(format!("@{}", document.display())) + .args([ + "--url", + &format!("{}/v1/stores/app-config/value?key=settings", self.endpoint), + "--config", + "-", + "--output", + ]) + .arg(root.join("publish-response")) + .args(["--write-out", "%{http_code}"]) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .expect("explicit curl operator prerequisite"); + child + .stdin + .take() + .expect("operator pipe") + .write_all( + format!("header = \"Authorization: Bearer {OPERATOR_TOKEN}\"\n").as_bytes(), + ) + .expect("operator credential on private stdin"); + let output = child.wait_with_output().expect("operator publish"); + assert!( + output.status.success(), + "operator publish: {}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!(String::from_utf8_lossy(&output.stdout), "204"); + } + } + + #[test] + #[ignore = "requires explicit modified CELLD_BIN, curl and OpenSSL; runs actual generated consumer over verified TLS"] + fn generated_native_consumer_uses_embedded_celld_over_tls() { + let temp = TempDir::new().expect("real-service fixture"); + let service = Celld::start(temp.path()); + service.publish(temp.path(), "first-config"); + let project = generate(temp.path()); + let binding = bindings(&project, &service.endpoint, true); + let binary = build(&project, Some("celld-config,celld-kv")); + let mut first = App::start( + &binary, + &project, + &binding, + "put", + Some((APP_TOKEN, &service.cert)), + ); + let response = first.request(); + assert!( + response.contains("200 OK") + && response.contains("greeting=first-config") + && response.contains("binary=Some([0, 255, 1, 128])") + && response.contains("ttl=true"), + "real service path: {response}" + ); + service.publish(temp.path(), "second-config"); + assert!(first.request().contains("greeting=first-config")); + thread::sleep(Duration::from_secs(61)); + let expired = first.request(); + assert!( + expired.contains("ttl=false") && expired.contains("greeting=first-config"), + "live elapsed service expiry with immutable configuration: {expired}" + ); + drop(first); + drop(service); + let restarted_service = Celld::start(temp.path()); + let restarted_binding = bindings(&project, &restarted_service.endpoint, true); + let mut second = App::start( + &binary, + &project, + &restarted_binding, + "read", + Some((APP_TOKEN, &restarted_service.cert)), + ); + let restarted_response = second.request(); + assert!( + restarted_response.contains("greeting=second-config") + && restarted_response.contains("binary=Some([0, 255, 1, 128])"), + "real service restart adoption: {restarted_response}" + ); + drop(second); + let mut rejected = App::start( + &binary, + &project, + &restarted_binding, + "read", + Some(( + "wrong-fixture-token-longer-than-32-bytes", + &restarted_service.cert, + )), + ); + rejected.rejected(); + let diagnostic = fs::read_to_string(&rejected.log).expect("closed error"); + assert!( + !diagnostic.contains(APP_TOKEN) + && !diagnostic.contains(OPERATOR_TOKEN) + && !diagnostic.contains(&restarted_service.endpoint) + ); + assert!(!project.join(".edgezero").exists()); + } +} diff --git a/crates/edgezero-cli/tests/dynamodb_generated_proof.rs b/crates/edgezero-cli/tests/dynamodb_generated_proof.rs new file mode 100644 index 000000000..50bc842c8 --- /dev/null +++ b/crates/edgezero-cli/tests/dynamodb_generated_proof.rs @@ -0,0 +1,592 @@ +//! Ignored end-to-end proof for the generated Axum `DynamoDB` KV consumer. +//! +//! This test runs explicitly in CI. To run it locally, use +//! `cargo test -p edgezero-cli --test dynamodb_generated_proof -- --ignored`. +//! The generated child uses the real generated `main`/`run_generated_app`; only its handler +//! and manifest are extended in the temporary project. SDK traffic uses a literal-loopback +//! fixture with dummy credentials and a cleared child environment. Serialize builds that share +//! `EDGEZERO_GENERATED_TEST_TARGET_DIR`. + +#[cfg(test)] +mod tests { + #[cfg(unix)] + mod generated_dynamodb_proof { + #![expect( + clippy::arbitrary_source_item_ordering, + reason = "group test fixture types and their protocol helpers by responsibility" + )] + + use std::collections::BTreeMap; + use std::env; + use std::fs; + use std::io::{ErrorKind, Read as _, Write as _}; + use std::net::{SocketAddr, TcpListener, TcpStream}; + use std::path::{Path, PathBuf}; + use std::process::{Child, Command, Stdio}; + use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + use std::sync::{Arc, Mutex}; + use std::thread::{self, JoinHandle}; + use std::time::{Duration, Instant}; + + use serde_json::{Value, json}; + use tempfile::TempDir; + + const WAIT: Duration = Duration::from_secs(25); + const TABLE: &str = "edgezero-kv"; + const NAMESPACE: &str = "fixture/state"; + + fn deadline_after(duration: Duration) -> Instant { + Instant::now() + .checked_add(duration) + .expect("proof deadline within supported range") + } + + /// Tiny `DynamoDB` JSON-protocol fixture. Persists rows across generated process restarts and + /// implements only the DescribeTable/GetItem/PutItem/DeleteItem/Query calls this proof sends. + struct DynamoFixture { + endpoint: String, + requests: Arc, + stop: Arc, + worker: Option>, + } + + impl DynamoFixture { + fn start() -> Self { + let listener = TcpListener::bind("127.0.0.1:0").expect("bind literal loopback"); + listener.set_nonblocking(true).expect("nonblocking fixture"); + let endpoint = format!("http://{}", listener.local_addr().expect("fixture addr")); + let rows = Arc::new(Mutex::new(BTreeMap::new())); + let shared_rows = Arc::clone(&rows); + let stop = Arc::new(AtomicBool::new(false)); + let worker_stop = Arc::clone(&stop); + let requests = Arc::new(AtomicUsize::new(0)); + let worker_requests = Arc::clone(&requests); + let worker = thread::spawn(move || { + let deadline = deadline_after(Duration::from_mins(3)); + while !worker_stop.load(Ordering::Relaxed) && Instant::now() < deadline { + match listener.accept() { + Ok((mut stream, peer)) => { + assert!(peer.ip().is_loopback(), "fixture accepts loopback only"); + stream + .set_read_timeout(Some(Duration::from_secs(4))) + .expect("read timeout"); + stream + .set_write_timeout(Some(Duration::from_secs(4))) + .expect("write timeout"); + worker_requests.fetch_add(1, Ordering::Relaxed); + let (target, body) = read_aws_request(&mut stream); + let response = dispatch(&target, &body, &shared_rows); + write_aws_response(&mut stream, &response); + } + Err(error) if error.kind() == ErrorKind::WouldBlock => { + thread::sleep(Duration::from_millis(5)); + } + Err(error) => panic!("DynamoDB fixture accept: {error}"), + } + } + }); + Self { + endpoint, + requests, + stop, + worker: Some(worker), + } + } + + fn count(&self) -> usize { + self.requests.load(Ordering::Relaxed) + } + } + + impl Drop for DynamoFixture { + fn drop(&mut self) { + self.stop.store(true, Ordering::Relaxed); + if let Some(worker) = self.worker.take() { + let _joined = worker.join(); + } + } + } + + fn read_aws_request(stream: &mut TcpStream) -> (String, Value) { + let mut data = Vec::new(); + let mut chunk = [0_u8; 4096]; + let header_end = loop { + let count = stream.read(&mut chunk).expect("read request"); + assert_ne!(count, 0, "request ended before headers"); + let read_bytes = chunk + .get(..count) + .expect("Read does not report more bytes than the supplied buffer"); + data.extend_from_slice(read_bytes); + if let Some(index) = data.windows(4).position(|window| window == b"\r\n\r\n") { + break index + .checked_add(4) + .expect("bounded header delimiter offset"); + } + assert!(data.len() < 64 * 1024, "bounded request headers"); + }; + let headers = String::from_utf8( + data.get(..header_end) + .expect("header boundary within request") + .to_vec(), + ) + .expect("HTTP headers"); + let content_length = headers + .lines() + .find_map(|line| { + let (name, value) = line.split_once(':')?; + name.eq_ignore_ascii_case("content-length") + .then(|| value.trim().parse::().expect("length")) + }) + .expect("content-length"); + while data.len().saturating_sub(header_end) < content_length { + let count = stream.read(&mut chunk).expect("read body"); + assert_ne!(count, 0, "request ended before body"); + let read_bytes = chunk + .get(..count) + .expect("Read does not report more bytes than the supplied buffer"); + data.extend_from_slice(read_bytes); + } + let target = headers + .lines() + .find_map(|line| { + let (name, value) = line.split_once(':')?; + name.eq_ignore_ascii_case("x-amz-target") + .then(|| value.trim().to_owned()) + }) + .expect("AWS target header"); + let body_end = header_end + .checked_add(content_length) + .expect("request body boundary does not overflow"); + let body_bytes = data + .get(header_end..body_end) + .expect("complete request body"); + let body = serde_json::from_slice(body_bytes).expect("AWS JSON body"); + (target, body) + } + + fn dispatch( + target: &str, + body: &Value, + rows: &Arc>>, + ) -> Value { + let operation = target.rsplit('.').next().unwrap_or(target); + match operation { + "DescribeTable" => json!({"Table": { + "TableName": TABLE, "TableStatus": "ACTIVE", + "KeySchema": [ + {"AttributeName":"pk", "KeyType":"HASH"}, + {"AttributeName":"sk", "KeyType":"RANGE"} + ], + "AttributeDefinitions": [ + {"AttributeName":"pk", "AttributeType":"S"}, + {"AttributeName":"sk", "AttributeType":"S"} + ], + "GlobalSecondaryIndexes": [], "LocalSecondaryIndexes": [] + }}), + "PutItem" => { + assert_eq!( + body.get("TableName").and_then(Value::as_str), + Some(TABLE), + "PutItem targets the fixture table" + ); + let item = body + .get("Item") + .cloned() + .expect("PutItem request includes Item"); + let key = required_string(string_attribute(&item, "sk"), "PutItem sort key") + .to_owned(); + rows.lock().expect("fixture rows").insert(key, item); + json!({}) + } + "GetItem" => { + assert_eq!( + body.get("TableName").and_then(Value::as_str), + Some(TABLE), + "GetItem targets the fixture table" + ); + let key = required_string( + body.get("Key").and_then(|key| string_attribute(key, "sk")), + "GetItem sort key", + ); + let item = rows.lock().expect("fixture rows").get(key).cloned(); + item.map_or_else(|| json!({}), |found_item| json!({"Item": found_item})) + } + "DeleteItem" => { + assert_eq!( + body.get("TableName").and_then(Value::as_str), + Some(TABLE), + "DeleteItem targets the fixture table" + ); + let key = required_string( + body.get("Key").and_then(|key| string_attribute(key, "sk")), + "DeleteItem sort key", + ); + rows.lock().expect("fixture rows").remove(key); + json!({}) + } + "Query" => { + assert_eq!( + body.get("TableName").and_then(Value::as_str), + Some(TABLE), + "Query targets the fixture table" + ); + let values = body + .get("ExpressionAttributeValues") + .expect("Query includes expression values"); + let pk = + required_string(string_attribute(values, ":pk"), "Query partition key"); + let prefix = string_attribute(values, ":prefix").unwrap_or(""); + let limit = body + .get("Limit") + .and_then(Value::as_u64) + .and_then(|value| usize::try_from(value).ok()) + .unwrap_or(1000); + let all = rows.lock().expect("fixture rows"); + let items = all + .values() + .filter(|item| { + string_attribute(item, "pk") == Some(pk) + && string_attribute(item, "sk") + .is_some_and(|key| key.starts_with(prefix)) + }) + .take(limit) + .cloned() + .collect::>(); + json!({"Items": items, "Count": items.len(), "ScannedCount": items.len()}) + } + unknown => panic!("unexpected DynamoDB operation {unknown}: {body}"), + } + } + + fn string_attribute<'attribute>( + item: &'attribute Value, + name: &str, + ) -> Option<&'attribute str> { + item.get(name)?.get("S")?.as_str() + } + + fn required_string<'attribute>( + value: Option<&'attribute str>, + field: &str, + ) -> &'attribute str { + value.expect(field) + } + + fn write_aws_response(stream: &mut TcpStream, body: &Value) { + let bytes = serde_json::to_vec(body).expect("serialize response"); + write!(stream, + "HTTP/1.1 200 OK\r\ncontent-type: application/x-amz-json-1.0\r\ncontent-length: {}\r\nconnection: close\r\n\r\n", + bytes.len()).expect("response headers"); + stream.write_all(&bytes).expect("response body"); + } + + struct GeneratedApp { + address: SocketAddr, + child: Child, + } + + impl GeneratedApp { + fn start( + executable: &Path, + project: &Path, + binding: &Path, + endpoint: &str, + mode: &str, + ) -> Self { + let listener = TcpListener::bind("127.0.0.1:0").expect("reserve app port"); + let address = listener.local_addr().expect("app address"); + drop(listener); + let mut command = Command::new(executable); + command + .current_dir(project) + .env_clear() + .env("HOME", project.join("empty-home")) + .env("AWS_ACCESS_KEY_ID", "dummy-access") + .env("AWS_SECRET_ACCESS_KEY", "dummy-secret") + .env("AWS_EC2_METADATA_DISABLED", "true") + .env("AWS_CONFIG_FILE", project.join("empty-aws-config")) + .env( + "AWS_SHARED_CREDENTIALS_FILE", + project.join("empty-aws-creds"), + ) + .env("AWS_ENDPOINT_URL", endpoint) + .env("EDGEZERO__STORE_BINDINGS_FILE", binding) + .env("EDGEZERO__DATA_DIR", project.join("must-not-create-data")) + .env("EDGEZERO__ADAPTER__HOST", "127.0.0.1") + .env("EDGEZERO__ADAPTER__PORT", address.port().to_string()) + .env("KV_PROOF_MODE", mode) + .stdout(Stdio::null()) + .stderr(Stdio::null()); + Self { + child: command.spawn().expect("spawn generated app"), + address, + } + } + + fn request(&mut self, path: &str, body: Option<&str>) -> String { + let deadline = deadline_after(WAIT); + loop { + if let Some(status) = self.child.try_wait().expect("child status") { + panic!("generated server exited before request {path}: {status}"); + } + if let Ok(mut stream) = + TcpStream::connect_timeout(&self.address, Duration::from_millis(50)) + { + stream + .set_read_timeout(Some(Duration::from_secs(3))) + .expect("response timeout"); + let (method, content_type, payload) = body + .map_or(("GET", "", ""), |value| { + ("POST", "Content-Type: application/json\r\n", value) + }); + write!(stream, "{method} {path} HTTP/1.1\r\nHost: 127.0.0.1\r\n{content_type}Content-Length: {}\r\nConnection: close\r\n\r\n{payload}", payload.len()).expect("send handler request"); + let mut response = String::new(); + stream + .read_to_string(&mut response) + .expect("read handler response"); + if !response.starts_with("HTTP/1.1 503") + && !response.starts_with("HTTP/1.1 500") + { + return response; + } + return response; + } + assert!( + Instant::now() < deadline, + "generated listener did not start" + ); + thread::sleep(Duration::from_millis(10)); + } + } + + fn stop(mut self) { + if self.child.try_wait().expect("child status").is_none() { + let _killed = self.child.kill(); + } + self.child.wait().expect("reap generated child"); + } + } + + impl Drop for GeneratedApp { + fn drop(&mut self) { + if self.child.try_wait().ok().flatten().is_none() { + let _killed = self.child.kill(); + let _reaped = self.child.wait(); + } + } + } + + fn install_consumer(project: &Path) { + let manifest = + fs::read_to_string(project.join("edgezero.toml")).expect("generated manifest"); + let entry = manifest + .lines() + .find_map(|line| line.strip_prefix("entry = ")) + .expect("generated core entry") + .trim_matches('"'); + let core_dir = project.join(entry); + let core_name = core_dir + .file_name() + .expect("core directory") + .to_string_lossy() + .replace('-', "_"); + let handlers = core_dir.join("src/handlers.rs"); + let mut handler_source = fs::read_to_string(&handlers).expect("generated handlers"); + handler_source.push_str(r#" +#[action] +pub async fn kv_probe(kv: edgezero_core::extractor::Kv) -> Result, EdgeError> { + let store = kv.default().ok_or_else(|| EdgeError::service_unavailable("default KV unavailable"))?; + let mode = std::env::var("KV_PROOF_MODE").unwrap_or_default(); + match mode.as_str() { + "put" => { + store.put("fixture/item", &"persisted-value").await?; + store.put_with_ttl("fixture/ttl", &"ttl-value", std::time::Duration::from_secs(60)).await?; + } + "ttl" => { store.put_with_ttl("fixture/ttl", &"ttl-value", std::time::Duration::from_secs(60)).await?; } + "delete" => { store.delete("fixture/item").await?; } + _ => {} + } + let value = store.get::("fixture/item").await?.unwrap_or_else(|| "missing".to_owned()); + let ttl = store.get::("fixture/ttl").await?.unwrap_or_else(|| "missing".to_owned()); + let page = store.list_keys_page("fixture/", None, 100).await?; + Ok(Text::new(format!("value={value};ttl={ttl};keys={}", page.keys.join(",")))) +} +"#); + fs::write(handlers, handler_source).expect("write temporary generated handler"); + + let manifest_path = project.join("edgezero.toml"); + let mut manifest_source = + fs::read_to_string(&manifest_path).expect("generated manifest"); + manifest_source.push_str( + r#" +[[triggers.http]] +id = "kv-proof" +path = "/kv-proof" +methods = ["GET", "POST"] +handler = "{core_name}::handlers::kv_probe" +adapters = ["axum"] + +[stores.kv] +ids = ["state"] +"#, + ); + let rendered_manifest = manifest_source.replace("{core_name}", &core_name); + fs::write(manifest_path, rendered_manifest) + .expect("write temporary generated manifest"); + } + + fn write_bindings(project: &Path) -> PathBuf { + let path = project.join("bindings.toml"); + let bindings = format!( + r#"version = 1 + +[kv.state] +provider = "aws-dynamodb" +region = "us-east-1" +table = "{TABLE}" +namespace = "{NAMESPACE}" +"# + ); + fs::write(&path, bindings).expect("write native TOML bindings fixture"); + path + } + + fn target_dir(project: &Path) -> PathBuf { + env::var_os("EDGEZERO_GENERATED_TEST_TARGET_DIR") + .map_or_else(|| project.join("target"), PathBuf::from) + } + + fn build_generated(project: &Path, target: &Path, enabled: bool) -> PathBuf { + let mut command = Command::new(env!("CARGO_BIN_EXE_edgezero")); + command.args(["build", "--adapter", "axum", "--", "--offline"]); + if enabled { + command.args(["--features", "aws-dynamodb"]); + } + let output = command + .current_dir(project) + .env("CARGO_TARGET_DIR", target) + .output() + .expect("real edgezero build"); + assert!( + output.status.success(), + "generated {}build should succeed: {}", + if enabled { "enabled " } else { "default-off " }, + String::from_utf8_lossy(&output.stderr) + ); + target.join("debug/dynamodb-proof-adapter-axum") + } + + fn prove_default_off( + project: &Path, + bindings: &Path, + fixture: &DynamoFixture, + binary: &Path, + ) { + // Reject the explicit binding before local data, SDK calls, or listener startup. + let mut rejected = + GeneratedApp::start(binary, project, bindings, &fixture.endpoint, "read"); + let exit_deadline = deadline_after(WAIT); + loop { + if rejected + .child + .try_wait() + .expect("default-off process state") + .is_some() + { + break; + } + assert!( + Instant::now() < exit_deadline, + "default-off selection unexpectedly listened" + ); + thread::sleep(Duration::from_millis(10)); + } + assert!( + !project.join("must-not-create-data").exists(), + "default-off selection must not create local DATA_DIR" + ); + assert!( + TcpStream::connect_timeout(&rejected.address, Duration::from_millis(50)).is_err(), + "default-off selection must not bind listener" + ); + assert_eq!( + fixture.count(), + 0, + "provider rejection must happen before SDK calls" + ); + rejected.stop(); + } + + fn prove_enabled_consumer( + project: &Path, + bindings: &Path, + fixture: &DynamoFixture, + binary: &Path, + ) { + let mut first = + GeneratedApp::start(binary, project, bindings, &fixture.endpoint, "put"); + let first_response = first.request("/kv-proof", None); + assert!( + first_response.contains("200 OK") && first_response.contains("persisted-value"), + "KV put/get: {first_response}" + ); + assert!( + first_response.contains("fixture/item"), + "KV query/list: {first_response}" + ); + assert!( + first_response.contains("ttl-value"), + "TTL write/read: {first_response}" + ); + first.stop(); + + // A fresh generated process reads fixture-retained data, not process-local state. + let mut second = + GeneratedApp::start(binary, project, bindings, &fixture.endpoint, "read"); + let second_response = second.request("/kv-proof", None); + assert!( + second_response.contains("persisted-value"), + "data survives generated process restart: {second_response}" + ); + second.stop(); + + let mut third = + GeneratedApp::start(binary, project, bindings, &fixture.endpoint, "delete"); + let third_response = third.request("/kv-proof", None); + assert!( + third_response.contains("value=missing") + && third_response.contains("ttl=ttl-value"), + "KV delete preserves other key: {third_response}" + ); + third.stop(); + } + + #[test] + #[ignore = "builds and launches a generated native app; coordinator owns Cargo integration"] + fn generated_cli_default_off_and_enabled_dynamodb_crud_list_ttl_restart() { + let temp = TempDir::new().expect("temp project"); + let output = Command::new(env!("CARGO_BIN_EXE_edgezero")) + .args(["new", "dynamodb-proof", "--dir"]) + .arg(temp.path()) + .output() + .expect("real edgezero new"); + assert!( + output.status.success(), + "edgezero new: {}", + String::from_utf8_lossy(&output.stderr) + ); + let project = temp.path().join("dynamodb-proof"); + install_consumer(&project); + let bindings = write_bindings(&project); + let fixture = DynamoFixture::start(); + let target = target_dir(&project); + + let default_binary = build_generated(&project, &target, false); + prove_default_off(&project, &bindings, &fixture, &default_binary); + + // The generated Axum alias opts into the actual provider feature. + let enabled_binary = build_generated(&project, &target, true); + prove_enabled_consumer(&project, &bindings, &fixture, &enabled_binary); + } + } +} diff --git a/crates/edgezero-cli/tests/native_aws_emulator_proof.rs b/crates/edgezero-cli/tests/native_aws_emulator_proof.rs new file mode 100644 index 000000000..eb8d5252b --- /dev/null +++ b/crates/edgezero-cli/tests/native_aws_emulator_proof.rs @@ -0,0 +1,1089 @@ +//! Independent local AWS proof. The runner supplies pinned `DynamoDB` Local, Moto and Agent +//! services. This test uses one real scaffold and one generated build with all three providers. +//! Run with `node scripts/run_native_aws_integration.mjs`. +//! No AWS SDK dependency is added to the CLI or to the temporary application. + +#[cfg(test)] +#[cfg(unix)] +mod tests { + #![expect( + clippy::arbitrary_source_item_ordering, + reason = "Group local-service proof helpers and source fixtures by responsibility." + )] + + use std::collections::BTreeSet; + use std::env; + use std::fmt::Write as _; + use std::fs; + use std::io::{self, Read as _, Write as _}; + use std::net::{Ipv4Addr, SocketAddr, TcpListener, TcpStream}; + use std::path::{Path, PathBuf}; + use std::process::{Child, Command, ExitStatus, Stdio}; + use std::str::from_utf8; + use std::thread; + use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + + use edgezero_core::blob_envelope::BlobEnvelope; + use serde_json::{Value, json}; + use tempfile::{NamedTempFile, TempDir}; + use toml::map::Map; + + const WAIT: Duration = Duration::from_secs(45); + const BUILD_WAIT: Duration = Duration::from_mins(15); + const RAW_A: &str = " opaque A\nnot JSON: caf\u{e9}\t\n"; + const RAW_B: &str = "\tupdated B\nnot JSON: e\u{301}\n"; + + struct Services { + dynamodb: String, + secrets: String, + agent: String, + directory: PathBuf, + node: String, + token: String, + } + + impl Services { + fn required() -> Self { + let required = + |name| env::var(name).unwrap_or_else(|_| panic!("required harness env {name}")); + let endpoint = |name| { + let value = required(name); + let address = value + .strip_prefix("http://") + .and_then(|authority| authority.parse::().ok()) + .expect("endpoint must be a literal http://127.0.0.1:"); + assert_eq!(address.ip(), Ipv4Addr::LOCALHOST); + assert_ne!(address.port(), 0); + assert_eq!(value, format!("http://{address}")); + value + }; + let directory = PathBuf::from(required("EDGEZERO_AWS_TEST_AGENT_DIRECTORY")); + assert!(directory.is_absolute() && directory.is_dir()); + let token = required("EDGEZERO_AWS_TEST_AGENT_TOKEN"); + assert_eq!(token, "local-agent-token", "only dummy Agent credentials"); + let node = required("EDGEZERO_AWS_TEST_NODE"); + assert!( + Path::new(&node).is_absolute(), + "runner supplies its Node executable" + ); + Self { + dynamodb: endpoint("EDGEZERO_AWS_TEST_DYNAMODB_ENDPOINT"), + secrets: endpoint("EDGEZERO_AWS_TEST_SECRETS_ENDPOINT"), + agent: endpoint("EDGEZERO_AWS_TEST_AGENT_ENDPOINT"), + directory, + node, + token, + } + } + + fn admin(&self, action: &str, input: &Value) -> Value { + let mut command = Command::new(&self.node); + command + .arg( + Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../scripts/native_aws_test_admin.mjs"), + ) + .arg(action) + .arg(input.to_string()) + .env("EDGEZERO_AWS_TEST_DYNAMODB_ENDPOINT", &self.dynamodb) + .env("EDGEZERO_AWS_TEST_SECRETS_ENDPOINT", &self.secrets); + let output = checked(command, WAIT, &format!("admin {action}")); + serde_json::from_slice(&output).expect("admin stdout is JSON") + } + + fn publish(&self, message: &str, raw: &str) -> String { + let typed = serde_json::to_string(&BlobEnvelope::new( + json!({"message": message, "nested": {"token": "current"}}), + "2026-10-07T12:00:00Z".to_owned(), + )) + .expect("serialize real typed envelope"); + for (name, bytes) in [ + ("probe:test:raw", raw), + ("probe:test:typed", typed.as_str()), + ] { + let pending = self.directory.join(format!(".{name}.pending")); + fs::write(&pending, bytes).expect("writable Agent mounted directory"); + fs::rename(pending, self.directory.join(name)) + .expect("atomic Agent file replacement"); + } + typed + } + + fn wait_agent(&self, profile: &str, expected: &str) { + let address = self + .agent + .strip_prefix("http://") + .expect("validated endpoint") + .parse() + .expect("literal Agent address"); + let deadline = deadline(WAIT); + loop { + if let Ok((status, body)) = http( + address, + "GET", + &format!("/applications/probe/environments/test/configurations/{profile}"), + &format!("Authorization: Bearer {}\r\n", self.token), + b"", + ) && status == 200 + && body == expected.as_bytes() + { + return; + } + assert!( + Instant::now() < deadline, + "Agent did not independently serve updated {profile}" + ); + thread::sleep(Duration::from_millis(100)); + } + } + } + + /// Output goes to files, so a full pipe cannot deadlock startup or a failed build. All children + /// inherit the runner's process group; the Node runner also cleans descendants on every exit. + struct Process { + child: Child, + stdout: NamedTempFile, + stderr: NamedTempFile, + } + + impl Process { + fn spawn(mut command: Command) -> Self { + let stdout = NamedTempFile::new().expect("stdout file"); + let stderr = NamedTempFile::new().expect("stderr file"); + command + .stdout(Stdio::from(stdout.reopen().expect("stdout handle"))) + .stderr(Stdio::from(stderr.reopen().expect("stderr handle"))); + Self { + child: command.spawn().expect("spawn bounded subprocess"), + stdout, + stderr, + } + } + + fn logs(&self) -> String { + format!( + "{}\n{}", + String::from_utf8_lossy(&fs::read(self.stdout.path()).expect("stdout")), + String::from_utf8_lossy(&fs::read(self.stderr.path()).expect("stderr")) + ) + } + + fn wait(&mut self, timeout: Duration) -> ExitStatus { + let until = deadline(timeout); + loop { + if let Some(status) = self.child.try_wait().expect("subprocess status") { + return status; + } + assert!( + Instant::now() < until, + "subprocess timed out: {}", + self.logs() + ); + thread::sleep(Duration::from_millis(20)); + } + } + } + + impl Drop for Process { + fn drop(&mut self) { + let _killed = self.child.kill(); + let _reaped = self.child.wait(); + } + } + + #[expect( + clippy::print_stdout, + reason = "The mandatory integration runner needs phase progress on slow generated builds and service failures." + )] + fn phase(message: &str) { + println!("{message}"); + } + + fn checked(command: Command, timeout: Duration, label: &str) -> Vec { + let mut process = Process::spawn(command); + assert!( + process.wait(timeout).success(), + "{label} failed: {}", + process.logs() + ); + fs::read(process.stdout.path()).expect("command stdout") + } + + fn deadline(timeout: Duration) -> Instant { + Instant::now() + .checked_add(timeout) + .expect("finite deadline") + } + + fn reserve_address() -> SocketAddr { + TcpListener::bind("127.0.0.1:0") + .expect("reserve loopback port") + .local_addr() + .expect("loopback address") + } + + fn http( + address: SocketAddr, + method: &str, + path: &str, + headers: &str, + body: &[u8], + ) -> io::Result<(u16, Vec)> { + let mut stream = TcpStream::connect_timeout(&address, Duration::from_millis(200))?; + stream.set_read_timeout(Some(Duration::from_secs(5)))?; + stream.set_write_timeout(Some(Duration::from_secs(5)))?; + write!( + stream, + "{method} {path} HTTP/1.1\r\nHost: 127.0.0.1\r\n{headers}Content-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + )?; + stream.write_all(body)?; + let mut response = Vec::new(); + stream.read_to_end(&mut response)?; + let boundary = response + .windows(4) + .position(|bytes| bytes == b"\r\n\r\n") + .ok_or_else(|| io::Error::other("missing HTTP headers end"))?; + let (head_bytes, header_tail) = response + .split_at_checked(boundary) + .ok_or_else(|| io::Error::other("HTTP header boundary"))?; + let head = from_utf8(head_bytes).map_err(io::Error::other)?; + let status = head + .split_whitespace() + .nth(1) + .ok_or_else(|| io::Error::other("missing HTTP status"))? + .parse::() + .map_err(io::Error::other)?; + let mut payload = header_tail + .strip_prefix(b"\r\n\r\n") + .ok_or_else(|| io::Error::other("HTTP response boundary"))? + .to_vec(); + if head + .lines() + .any(|line| line.eq_ignore_ascii_case("transfer-encoding: chunked")) + { + let mut decoded = Vec::new(); + let mut remaining = payload.as_slice(); + loop { + let end = remaining + .windows(2) + .position(|bytes| bytes == b"\r\n") + .ok_or_else(|| io::Error::other("missing chunk header"))?; + let (chunk_header, chunk_tail) = remaining + .split_at_checked(end) + .ok_or_else(|| io::Error::other("chunk header boundary"))?; + let size = usize::from_str_radix( + from_utf8(chunk_header) + .map_err(io::Error::other)? + .split(';') + .next() + .ok_or_else(|| io::Error::other("missing chunk size"))?, + 16, + ) + .map_err(io::Error::other)?; + if size == 0 { + break; + } + let chunk_payload = chunk_tail + .strip_prefix(b"\r\n") + .ok_or_else(|| io::Error::other("chunk header delimiter"))?; + let (data, body_tail) = chunk_payload + .split_at_checked(size) + .ok_or_else(|| io::Error::other("truncated HTTP chunk"))?; + decoded.extend_from_slice(data); + remaining = body_tail + .strip_prefix(b"\r\n") + .ok_or_else(|| io::Error::other("chunk body delimiter"))?; + } + payload = decoded; + } + Ok((status, payload)) + } + + struct App { + process: Process, + address: SocketAddr, + entered: PathBuf, + release: PathBuf, + _startup: TempDir, + } + + impl App { + fn spawn( + project: &Path, + binary: &Path, + bindings: &Path, + services: &Services, + token: &str, + ) -> Self { + let address = reserve_address(); + let startup = tempfile::tempdir_in(project).expect("unique startup gate directory"); + let entered = startup.path().join("entered"); + let release = startup.path().join("release"); + let mut command = Command::new(binary); + command.current_dir(project).env_clear() + .env("HOME", project.join("empty-home")) + .env("AWS_CONFIG_FILE", project.join("empty-aws-config")) + .env("AWS_SHARED_CREDENTIALS_FILE", project.join("empty-aws-credentials")) + .env("AWS_EC2_METADATA_DISABLED", "true") + .env("AWS_ACCESS_KEY_ID", "DUMMYLOCALACCESS123") + .env("AWS_SECRET_ACCESS_KEY", "DUMMYLOCALSECRET456") + .env("AWS_REGION", "us-east-1") + .env("AWS_ENDPOINT_URL", format!("http://{}", reserve_address())) + .env("AWS_ENDPOINT_URL_DYNAMODB", &services.dynamodb) + .env("AWS_ENDPOINT_URL_SECRETS_MANAGER", &services.secrets) + .env("AWS_IGNORE_CONFIGURED_ENDPOINT_URLS", "false") + .env("APPCONFIG_AGENT_TOKEN", token) + .env("EDGEZERO__STORE_BINDINGS_FILE", bindings) + .env("EDGEZERO__DATA_DIR", project.join("must-not-create-data")) + .env("EDGEZERO__ADAPTER__HOST", "127.0.0.1") + .env("EDGEZERO__ADAPTER__PORT", address.port().to_string()) + .env("EDGEZERO__LOGGING__LEVEL", "off") + .env("RUST_LOG", "off,aws_config=off,aws_smithy_runtime=off,aws_sdk_dynamodb=off,aws_sdk_secretsmanager=off,reqwest=off,hyper=off") + .env("PROOF_ENTERED", &entered).env("PROOF_RELEASE", &release); + Self { + process: Process::spawn(command), + address, + entered, + release, + _startup: startup, + } + } + + fn ready(&mut self, expected_secret: &str) { + let until = deadline(WAIT); + while !self.entered.is_file() { + assert!( + self.process + .child + .try_wait() + .expect("startup status") + .is_none(), + "startup failed before initializer: {}", + self.process.logs() + ); + self.assert_closed(); + assert!( + Instant::now() < until, + "initializer timeout: {}", + self.process.logs() + ); + thread::sleep(Duration::from_millis(10)); + } + assert_eq!( + fs::read_to_string(&self.entered).expect("resolved initializer marker"), + expected_secret, + "nested secret must resolve before readiness marker" + ); + self.assert_closed(); + fs::write(&self.release, b"release").expect("release initializer"); + while TcpStream::connect_timeout(&self.address, Duration::from_millis(30)).is_err() { + assert!( + self.process + .child + .try_wait() + .expect("startup status") + .is_none(), + "{}", + self.process.logs() + ); + assert!( + Instant::now() < until, + "listener timeout: {}", + self.process.logs() + ); + thread::sleep(Duration::from_millis(10)); + } + } + + fn assert_closed(&self) { + assert!( + TcpStream::connect_timeout(&self.address, Duration::from_millis(30)).is_err(), + "application listened before successful initialization" + ); + } + + fn rejects(mut self) { + let until = deadline(WAIT); + loop { + self.assert_closed(); + assert!( + !self.entered.exists(), + "invalid provider reached typed readiness marker" + ); + if let Some(status) = self + .process + .child + .try_wait() + .expect("negative startup status") + { + assert!(!status.success(), "invalid startup exited successfully"); + self.assert_closed(); + return; + } + assert!( + Instant::now() < until, + "invalid startup did not fail closed: {}", + self.process.logs() + ); + thread::sleep(Duration::from_millis(10)); + } + } + + fn call(&mut self, request: &Value) -> Value { + assert!( + self.process + .child + .try_wait() + .expect("running app status") + .is_none(), + "{}", + self.process.logs() + ); + let bytes = serde_json::to_vec(request).expect("probe request JSON"); + let (status, response) = http( + self.address, + "POST", + "/proof", + "Content-Type: application/json\r\n", + &bytes, + ) + .expect("bounded application HTTP request"); + assert_eq!( + status, + 200, + "probe {request}: {}", + String::from_utf8_lossy(&response) + ); + serde_json::from_slice(&response).expect("probe JSON response") + } + } + + fn install_consumer(project: &Path) { + let core = project.join("crates/native-aws-probe-core"); + let mut manifest: toml::Value = + toml::from_str(&fs::read_to_string(project.join("edgezero.toml")).expect("manifest")) + .expect("scaffold TOML"); + let root = manifest.as_table_mut().expect("manifest root table"); + root.insert( + "stores".to_owned(), + toml::Value::try_from(json!({ + "config": {"ids": ["app_config"], "default": "app_config"}, + "secrets": {"ids": ["vault"], "default": "vault"}, + "kv": {"ids": ["primary", "secondary"], "default": "primary"} + })) + .expect("store declaration TOML"), + ); + root.get_mut("triggers") + .and_then(toml::Value::as_table_mut) + .expect("scaffold triggers") + .insert( + "http".to_owned(), + toml::Value::try_from(json!([{ + "id": "proof", "path": "/proof", "methods": ["POST"], + "handler": "native_aws_probe_core::handlers::proof", "adapters": ["axum"] + }])) + .expect("probe trigger TOML"), + ); + root.get_mut("adapters") + .and_then(toml::Value::as_table_mut) + .and_then(|adapters| adapters.get_mut("axum")) + .and_then(toml::Value::as_table_mut) + .expect("scaffold Axum adapter") + .entry("build") + .or_insert_with(|| toml::Value::Table(Map::new())) + .as_table_mut() + .expect("Axum build settings") + .insert( + "features".to_owned(), + toml::Value::try_from(vec![ + "aws-appconfig-agent", + "aws-secrets-manager", + "aws-dynamodb", + ]) + .expect("all three native provider features"), + ); + fs::write( + project.join("edgezero.toml"), + toml::to_string(&manifest).expect("manifest TOML"), + ) + .expect("write temporary store declarations"); + let core_manifest = core.join("Cargo.toml"); + let mut dependencies: toml::Value = + toml::from_str(&fs::read_to_string(&core_manifest).expect("core Cargo manifest")) + .expect("core TOML"); + dependencies["dependencies"] + .as_table_mut() + .expect("core dependencies") + .insert("serde_json".to_owned(), toml::Value::String("1".to_owned())); + fs::write( + core_manifest, + toml::to_string(&dependencies).expect("core TOML"), + ) + .expect("temporary core JSON dependency"); + fs::write( + core.join("src/lib.rs"), + "pub mod config;\nmod handlers;\nedgezero_core::app!(\"../../edgezero.toml\");\n", + ) + .expect("minimal temporary generated core"); + fs::write(core.join("src/config.rs"), CONFIG_SOURCE).expect("typed nested config"); + fs::write(core.join("src/handlers.rs"), HANDLER_SOURCE).expect("public store API consumer"); + fs::write( + project.join("native-aws-probe.toml"), + "message = \"local validation only\"\n[nested]\ntoken = \"current\"\n", + ) + .expect("local build validation config"); + fs::write( + project.join("crates/native-aws-probe-adapter-axum/src/main.rs"), + MAIN_SOURCE, + ) + .expect("public production initializer"); + fs::create_dir_all(project.join("empty-home")).expect("isolated HOME"); + fs::write(project.join("empty-aws-config"), "").expect("empty config"); + fs::write(project.join("empty-aws-credentials"), "").expect("empty credentials"); + } + + const CONFIG_SOURCE: &str = " +use serde::{Deserialize, Serialize}; +use validator::Validate; +#[derive(Debug, Deserialize, Serialize, Validate, edgezero_core::AppConfig)] +#[serde(deny_unknown_fields)] +pub struct NativeAwsProbeConfig { + pub message: String, + #[app_config(nested)] + #[validate(nested)] + pub nested: Nested, +} +#[derive(Debug, Deserialize, Serialize, Validate, edgezero_core::AppConfig)] +#[serde(deny_unknown_fields)] +pub struct Nested { + #[secret] + #[validate(length(min = 8_u64))] + pub token: String, +} +pub struct StartupState { + pub message: String, + pub token: String, +} +"; + + const MAIN_SOURCE: &str = r#" +use std::sync::Arc; +use std::time::{Duration, Instant}; +use edgezero_adapter_axum::dev_server::run_production_app_with_initializer; +use edgezero_core::extractor::AppConfig; +use edgezero_core::EdgeError; +use native_aws_probe_core::config::{NativeAwsProbeConfig, StartupState}; +fn main() -> anyhow::Result<()> { + run_production_app_with_initializer::(|app, stores| { + Box::pin(async move { + let binding = stores.config().and_then(|registry| registry.named_ref("app_config")) + .ok_or_else(|| EdgeError::service_unavailable("missing typed config binding"))?; + let config = AppConfig::::from_binding( + binding, None, stores.secrets(), app.config_extraction_limits(), app.monotonic_clock(), + ).await?; + let entered = std::env::var("PROOF_ENTERED").map_err(EdgeError::internal)?; + std::fs::write(entered, &config.nested.token).map_err(EdgeError::internal)?; + let release = std::path::PathBuf::from(std::env::var("PROOF_RELEASE").map_err(EdgeError::internal)?); + let deadline = Instant::now() + Duration::from_secs(45); + while !release.is_file() { + if Instant::now() >= deadline { + return Err(EdgeError::service_unavailable("initializer gate timed out")); + } + std::thread::sleep(Duration::from_millis(10)); + } + app.insert_state(Arc::new(StartupState { message: config.message, token: config.nested.token })); + Ok(()) + }) + })?; + Ok(()) +} +"#; + + const HANDLER_SOURCE: &str = r#" +use std::sync::Arc; +use bytes::Bytes; +use edgezero_core::{action, EdgeError}; +use edgezero_core::extractor::{Config, Json as RequestJson, Kv, Secrets, State}; +use edgezero_core::body::Body; +use edgezero_core::http::{Response, StatusCode, response_builder}; +use serde_json::{Value, json}; +use crate::config::StartupState; +#[action] +pub async fn proof( + State(state): State>, config: Config, secrets: Secrets, + kv: Kv, RequestJson(input): RequestJson, +) -> Result { + let op = input["op"].as_str().unwrap_or(""); + if op == "snapshot" { + let config = config.named("app_config").ok_or_else(|| EdgeError::service_unavailable("missing config"))?; + let raw = config.get("raw").await?.ok_or_else(|| EdgeError::service_unavailable("missing raw"))?; + let secrets = secrets.named("vault").ok_or_else(|| EdgeError::service_unavailable("missing secrets"))?; + let mut values = serde_json::Map::new(); + for name in ["current", "pinned", "candidate", "version", "json", "binary"] { + values.insert(name.to_owned(), json!(secrets.require_bytes(name).await?.to_vec())); + } + return json_response(json!({"message": state.message, "token": state.token, + "raw": raw.as_bytes(), "secrets": values})); + } + let namespace = input["namespace"].as_str().unwrap_or("primary"); + let store = kv.named(namespace).ok_or_else(|| EdgeError::service_unavailable("missing KV"))?; + let key = input["key"].as_str().unwrap_or(""); + let result = match op { + "put" | "ttl" => { + let value: Vec = serde_json::from_value(input["value"].clone()).map_err(EdgeError::internal)?; + if op == "ttl" { + store.put_bytes_with_ttl(key, Bytes::from(value), std::time::Duration::from_secs(120)).await?; + } else { store.put_bytes(key, Bytes::from(value)).await?; } + json!(true) + } + "large_put" => { store.put_bytes(key, Bytes::from(vec![input["byte"].as_u64().unwrap_or(0) as u8; 256 * 1024])).await?; json!(true) } + "get" => json!(store.get_bytes(key).await?.map(|value| value.to_vec())), + "exists" => json!(store.exists(key).await?), + "delete" => { store.delete(key).await?; json!(true) } + "list" => { + let page = store.list_keys_page(input["prefix"].as_str().unwrap_or(""), input["cursor"].as_str(), + input["limit"].as_u64().unwrap_or(100) as usize).await?; + json!({"keys": page.keys, "cursor": page.cursor}) + } + _ => return Err(EdgeError::bad_request("unknown proof operation")), + }; + json_response(result) +} +fn json_response(value: Value) -> Result { + response_builder().status(StatusCode::OK).header("content-type", "application/json") + .body(Body::json(&value).map_err(EdgeError::internal)?) + .map_err(EdgeError::internal) +} +"#; + + fn write_bindings( + project: &Path, + services: &Services, + seed: &Value, + table: &str, + profile: &str, + name: &str, + ) -> PathBuf { + let mut source = format!( + r#"version = 1 +[config.app_config] +provider = "aws-appconfig-agent" +default_key = "settings" +endpoint = "{}" +access_token_env = "APPCONFIG_AGENT_TOKEN" +[config.app_config.documents.settings] +application = "probe" +environment = "test" +profile = "{profile}" +[config.app_config.documents.raw] +application = "probe" +environment = "test" +profile = "raw" +[secrets.vault] +provider = "aws-secrets-manager" +region = "us-east-1" +"#, + services.agent + ); + for (key, secret, selector) in [ + ("current", "text", String::new()), + ("pinned", "text", "version_stage = \"PINNED\"\n".to_owned()), + ( + "candidate", + "text", + "version_stage = \"CANDIDATE\"\n".to_owned(), + ), + ( + "version", + "text", + format!( + "version_id = {}\n", + toml_string( + seed["secrets"]["text"]["version_a"] + .as_str() + .expect("text version A") + ) + ), + ), + ("json", "json", String::new()), + ("binary", "binary", String::new()), + ] { + write!( + source, + "[secrets.vault.entries.{key}]\nsecret_id = {}\n{selector}", + toml_string( + seed["secrets"][secret]["arn"] + .as_str() + .expect("seed secret ARN") + ) + ) + .expect("write secret binding to String"); + } + for namespace in ["primary", "secondary"] { + write!(source, "[kv.{namespace}]\nprovider = \"aws-dynamodb\"\nregion = \"us-east-1\"\ntable = {}\nnamespace = \"{namespace}\"\n", toml_string(table)).expect("write KV binding to String"); + } + let path = project.join(format!("{name}.toml")); + fs::write(&path, source).expect("emulator bindings"); + path + } + + fn toml_string(value: &str) -> String { + toml::Value::String(value.to_owned()).to_string() + } + + fn snapshot(app: &mut App, message: &str, raw: &str, current: &str) { + assert_eq!( + app.call(&json!({"op": "snapshot"})), + json!({ + "message": message, "token": current, "raw": raw.as_bytes(), "secrets": { + "current": current.as_bytes(), "pinned": current.as_bytes(), + "candidate": b"secret-B".as_slice(), "version": b"secret-A".as_slice(), "json": b" { \"answer\": 42 }\n".as_slice(), + "binary": [0_u8, 255_u8, 10_u8, 34_u8] + } + }), + "all config and secret bytes must remain exact startup snapshots" + ); + } + + fn put(app: &mut App, namespace: &str, key: &str, value: &[u8]) { + assert_eq!( + app.call(&json!({"op":"put", "namespace":namespace, "key":key, "value":value})), + true + ); + } + + fn get(app: &mut App, namespace: &str, key: &str) -> Value { + app.call(&json!({"op":"get", "namespace":namespace, "key":key})) + } + + fn list(app: &mut App, namespace: &str, prefix: &str, cursor: &Value, limit: u32) -> Value { + app.call(&json!({"op":"list", "namespace":namespace, "prefix":prefix, "cursor":cursor, "limit":limit})) + } + + fn traverse(app: &mut App, namespace: &str, prefix: &str, limit: u32) -> Vec { + let mut cursor = Value::Null; + let mut keys = Vec::new(); + let mut cursors = BTreeSet::new(); + for _ in 0_u32..100_u32 { + let page = list(app, namespace, prefix, &cursor, limit); + for value in page["keys"].as_array().expect("list keys") { + let key = value.as_str().expect("string key").to_owned(); + assert!(key.starts_with(prefix)); + assert!( + keys.last().is_none_or(|previous| previous < &key), + "sorted pages without duplicates" + ); + keys.push(key); + } + cursor = page["cursor"].clone(); + if cursor.is_null() { + return keys; + } + assert!( + cursors.insert(cursor.as_str().expect("cursor string").to_owned()), + "cursor must advance" + ); + } + panic!("pagination did not terminate"); + } + + fn prove_kv_crud_and_namespaces(app: &mut App) { + phase("AWS emulator proof: raw KV CRUD, namespaces, Unicode and cursors"); + let raw = [0, 255, 10, 34, 128]; + assert_eq!(get(app, "primary", "crud/missing"), Value::Null); + assert_eq!( + app.call(&json!({"op":"exists", "key":"crud/missing"})), + false + ); + put(app, "primary", "crud/raw", &raw); + assert_eq!(get(app, "primary", "crud/raw"), json!(raw)); + assert_eq!(app.call(&json!({"op":"exists", "key":"crud/raw"})), true); + put(app, "primary", "crud/raw", b"overwritten"); + assert_eq!( + get(app, "primary", "crud/raw"), + json!(b"overwritten".as_slice()) + ); + put(app, "primary", "crud/empty", b""); + assert_eq!(get(app, "primary", "crud/empty"), json!([])); + assert_eq!(app.call(&json!({"op":"exists", "key":"crud/empty"})), true); + for key in ["crud/raw", "crud/empty", "crud/missing"] { + assert_eq!(app.call(&json!({"op":"delete", "key":key})), true); + assert_eq!(get(app, "primary", key), Value::Null); + assert_eq!(app.call(&json!({"op":"exists", "key":key})), false); + } + put(app, "primary", "isolation/same", b"primary"); + put(app, "secondary", "isolation/same", b"secondary"); + put(app, "primary", "isolation/only-primary", b"p"); + put(app, "secondary", "isolation/only-secondary", b"s"); + assert_eq!( + get(app, "primary", "isolation/same"), + json!(b"primary".as_slice()) + ); + assert_eq!( + get(app, "secondary", "isolation/same"), + json!(b"secondary".as_slice()) + ); + assert_eq!(get(app, "primary", "isolation/only-secondary"), Value::Null); + assert_eq!(get(app, "secondary", "isolation/only-primary"), Value::Null); + assert_eq!( + traverse(app, "primary", "isolation/", 1), + ["isolation/only-primary", "isolation/same"] + ); + assert_eq!( + traverse(app, "secondary", "isolation/", 1), + ["isolation/only-secondary", "isolation/same"] + ); + put(app, "primary", "unicode/\u{e9}", b"NFC"); + put(app, "primary", "unicode/e\u{301}", b"NFD"); + assert_eq!( + get(app, "primary", "unicode/\u{e9}"), + json!(b"NFC".as_slice()) + ); + assert_eq!( + get(app, "primary", "unicode/e\u{301}"), + json!(b"NFD".as_slice()) + ); + assert_eq!( + traverse(app, "primary", "unicode/", 1), + ["unicode/e\u{301}", "unicode/\u{e9}"] + ); + let expected = [ + "prefix/01", + "prefix/02", + "prefix/03", + "prefix/04", + "prefix/05", + ]; + for key in expected.into_iter().rev() { + put(app, "primary", key, b"v"); + } + put(app, "primary", "prefix-outside", b"excluded"); + assert_eq!(traverse(app, "primary", "prefix/", 2), expected); + } + + fn prove_kv_large_pages(app: &mut App) { + phase("AWS emulator proof: DynamoDB 1 MiB page boundary with keys-only projection"); + let large_keys = (0_u8..6_u8) + .map(|index| format!("large/{index}")) + .collect::>(); + for (index, key) in large_keys.iter().enumerate() { + assert_eq!( + app.call(&json!({"op":"large_put", "key":key, "byte":index})), + true + ); + } + let first = list(app, "primary", "large/", &Value::Null, 100); + let count = first["keys"].as_array().expect("large keys").len(); + assert!( + count > 0 && count < large_keys.len(), + "service byte cap must produce a short first page: {first}" + ); + assert!( + first["cursor"].is_string(), + "short service page retains continuation" + ); + assert_eq!(traverse(app, "primary", "large/", 100), large_keys); + for (index, key) in large_keys.iter().enumerate() { + let byte = u8::try_from(index).expect("six fixture values"); + assert_eq!( + get(app, "primary", key), + json!(vec![byte; 256 * 1024]), + "large value exact bytes" + ); + } + } + + fn prove_kv_expiry(app: &mut App, services: &Services) { + phase( + "AWS emulator proof: expiry filtering does not delete; renewal writes both TTL fields", + ); + let expired = list(app, "primary", "expired/", &Value::Null, 1); + assert_eq!(expired["keys"], json!([])); + assert!( + expired["cursor"].is_string(), + "empty filtered page must keep service cursor" + ); + let alive = list(app, "primary", "expired/", &expired["cursor"], 1); + assert_eq!(alive["keys"], json!(["expired/1"])); + assert_eq!(traverse(app, "primary", "expired/", 1), ["expired/1"]); + assert_eq!(get(app, "primary", "expired/0"), Value::Null); + assert_eq!(app.call(&json!({"op":"exists", "key":"expired/0"})), false); + let stored = services.admin("item", &json!({"namespace":"primary", "key":"expired/0"})); + assert_eq!( + stored["present"], true, + "expired read must not physically delete" + ); + assert_eq!(stored["expires_at_ms"], 1_u64); + assert_eq!(stored["ttl_epoch_s"], 1_u64); + assert_eq!(get(app, "primary", "expired/1"), json!(b"live".as_slice())); + assert_eq!(get(app, "primary", "renew/item"), Value::Null); + let before = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("epoch") + .as_millis(); + assert_eq!( + app.call(&json!({"op":"ttl", "key":"renew/item", "value":b"renewed".as_slice()})), + true + ); + assert_eq!( + get(app, "primary", "renew/item"), + json!(b"renewed".as_slice()) + ); + let renewed = services.admin("item", &json!({"namespace":"primary", "key":"renew/item"})); + assert_eq!(renewed["present"], true); + assert_eq!(renewed["value"], json!(b"renewed".as_slice())); + let expiration_ms = u128::from(renewed["expires_at_ms"].as_u64().expect("millisecond TTL")); + let ttl = u128::from( + renewed["ttl_epoch_s"] + .as_u64() + .expect("service TTL seconds"), + ); + assert!( + expiration_ms >= before.checked_add(60_000).expect("bounded epoch threshold"), + "renewed TTL is at least 60 seconds" + ); + assert_eq!( + ttl, + expiration_ms.div_ceil(1000), + "TTL fields describe the same expiry" + ); + put(app, "primary", "renew/item", b"permanent"); + let permanent = services.admin("item", &json!({"namespace":"primary", "key":"renew/item"})); + assert_eq!( + permanent, + json!({"present":true, "value":b"permanent".as_slice(), "expires_at_ms":null, "ttl_epoch_s":null}), + "normal overwrite must clear both expiration fields" + ); + } + + #[test] + #[expect( + clippy::too_many_lines, + reason = "Keep the single-build, startup and restart lifecycle together as one end-to-end proof." + )] + #[ignore = "requires pinned local services; run node scripts/run_native_aws_integration.mjs"] + fn generated_native_aws_emulator_contracts() { + let services = Services::required(); + phase("AWS emulator proof: seed independent services and scaffold real project"); + let seed = services.admin("seed", &json!({})); + let typed_a = services.publish("message-A", RAW_A); + services.wait_agent("raw", RAW_A); + services.wait_agent("typed", &typed_a); + let temp = TempDir::new().expect("temporary generated project"); + let mut new = Command::new(env!("CARGO_BIN_EXE_edgezero")); + new.args(["new", "native-aws-probe", "--dir"]) + .arg(temp.path()); + checked(new, WAIT, "real edgezero new"); + let project = temp.path().join("native-aws-probe"); + install_consumer(&project); + let target = env::var_os("EDGEZERO_GENERATED_TEST_TARGET_DIR").map_or_else( + || project.join("target"), + |value| { + let path = PathBuf::from(value); + if path.is_absolute() { + path + } else { + project.join(path) + } + }, + ); + phase("AWS emulator proof: one generated all-provider build"); + let mut build = Command::new(env!("CARGO_BIN_EXE_edgezero")); + build + .args(["build", "--adapter", "axum", "--", "--offline"]) + .current_dir(&project) + .env("CARGO_TARGET_DIR", &target); + checked(build, BUILD_WAIT, "single combined generated build"); + let binary = target.join("debug/native-aws-probe-adapter-axum"); + assert!(binary.is_file(), "generated executable"); + let table = seed["table"].as_str().expect("seed table"); + let bindings = write_bindings(&project, &services, &seed, table, "typed", "valid-bindings"); + phase("AWS emulator proof: enabled invalid resources fail before listener"); + for field in ["missing_table", "wrong_table", "indexed_table"] { + let bad = write_bindings( + &project, + &services, + &seed, + seed[field].as_str().expect("negative table"), + "typed", + field, + ); + App::spawn(&project, &binary, &bad, &services, &services.token).rejects(); + } + App::spawn( + &project, + &binary, + &bindings, + &services, + "wrong-local-agent-token", + ) + .rejects(); + let missing = write_bindings( + &project, + &services, + &seed, + table, + "missing-profile", + "missing-profile", + ); + App::spawn(&project, &binary, &missing, &services, &services.token).rejects(); + + phase("AWS emulator proof: combined typed initializer, exact config and secret snapshots"); + let mut first = App::spawn(&project, &binary, &bindings, &services, &services.token); + first.ready("secret-A"); + snapshot(&mut first, "message-A", RAW_A, "secret-A"); + prove_kv_crud_and_namespaces(&mut first); + prove_kv_large_pages(&mut first); + prove_kv_expiry(&mut first, &services); + phase( + "AWS emulator proof: rotate services, independently observe Agent B, retain running A", + ); + services.admin("rotate", &json!({})); + let typed_b = services.publish("message-B", RAW_B); + services.wait_agent("raw", RAW_B); + services.wait_agent("typed", &typed_b); + snapshot(&mut first, "message-A", RAW_A, "secret-A"); + drop(first); + + phase("AWS emulator proof: restart sees B, pinned version stays A, KV persists"); + let mut second = App::spawn(&project, &binary, &bindings, &services, &services.token); + second.ready("secret-B"); + snapshot(&mut second, "message-B", RAW_B, "secret-B"); + assert_eq!( + get(&mut second, "primary", "renew/item"), + json!(b"permanent".as_slice()) + ); + assert_eq!( + get(&mut second, "primary", "isolation/same"), + json!(b"primary".as_slice()) + ); + assert_eq!( + get(&mut second, "secondary", "isolation/same"), + json!(b"secondary".as_slice()) + ); + assert_eq!( + second.call(&json!({"op":"delete", "key":"renew/item"})), + true + ); + assert_eq!(get(&mut second, "primary", "renew/item"), Value::Null); + assert_eq!( + get(&mut second, "primary", "isolation/same"), + json!(b"primary".as_slice()) + ); + assert_eq!( + get(&mut second, "primary", "expired/1"), + json!(b"live".as_slice()) + ); + phase("AWS emulator proof: deletion preserves loaded B but fresh startup fails closed"); + services.admin("delete_secret", &json!({})); + snapshot(&mut second, "message-B", RAW_B, "secret-B"); + drop(second); + App::spawn(&project, &binary, &bindings, &services, &services.token).rejects(); + assert!( + !project.join("must-not-create-data").exists(), + "explicit AWS stores must not create local DATA_DIR" + ); + phase("AWS emulator proof: all independent local-service contracts passed"); + } +} diff --git a/crates/edgezero-cli/tests/native_store_generated_proof.rs b/crates/edgezero-cli/tests/native_store_generated_proof.rs new file mode 100644 index 000000000..d98bf8b5d --- /dev/null +++ b/crates/edgezero-cli/tests/native_store_generated_proof.rs @@ -0,0 +1,941 @@ +//! Ignored end-to-end proof for generated native AWS stores. +//! +//! It invokes the real `edgezero new` and `edgezero build --adapter axum -- --offline` commands, +//! builds the generated Axum target with default-off, Agent-only, Secrets-only and combined +//! manifest feature selections, and launches its binary. +//! +//! The untouched generated main calls `run_generated_app`, proving explicit AWS bindings fail +//! closed when services were not compiled. The success main is replaced only in the temporary +//! generated project with an application-owned public production initializer. That initializer +//! loads the real serialized envelope and resolves its nested `#[secret]` reference before +//! inserting typed state. The test holds both provider responses and the initializer on gates, +//! then confirms the generated server does not listen until they complete. +//! +//! All service traffic uses short-lived literal `127.0.0.1` fixtures. The generated child starts +//! with `env_clear`, empty profile files and HOME, dummy credentials, metadata lookup disabled, +//! and both `AWS_ENDPOINT_URL_SECRETS_MANAGER` and the global endpoint override pointing to the +//! Secrets Manager fixture. No AWS account or real credentials, DNS, external socket, image or +//! IAM action is involved. This ignored test runs explicitly in CI. To run it locally, use +//! `cargo test -p edgezero-cli --test native_store_generated_proof -- --ignored`. +//! `EDGEZERO_GENERATED_TEST_TARGET_DIR` can select a shared target directory; otherwise builds +//! use the generated project's `target/`. Serialize builds that share a target directory. + +#[cfg(test)] +#[cfg(unix)] +mod tests { + + use std::env::var_os; + use std::fs; + use std::io::{ErrorKind, Read as _, Write as _}; + use std::net::{SocketAddr, TcpListener, TcpStream}; + use std::path::{Path, PathBuf}; + use std::process::{Child, Command, Output, Stdio}; + use std::sync::mpsc::{self, Receiver, Sender}; + use std::thread::{self, JoinHandle}; + use std::time::{Duration, Instant}; + + use edgezero_core::blob_envelope::BlobEnvelope; + use serde_json::{Value, json}; + use tempfile::TempDir; + use toml::Value as TomlValue; + + const WAIT: Duration = Duration::from_secs(20); + const SERVICE_GRACE: Duration = Duration::from_millis(400); + const SECRET_ARN: &str = + "arn:aws:secretsmanager:us-east-1:111122223333:secret:native-probe-signing-AbCdEf"; + const APP_TOKEN: &str = "generated-proof-secret-value"; + const AGENT_TOKEN: &str = "generated-proof-agent-token"; + const FALLBACK_SENTINEL: &str = "must-not-fall-back-to-environment"; + + #[derive(Debug)] + struct HttpRequest { + body: String, + headers: String, + } + + struct HttpReply { + body: String, + content_type: &'static str, + reason: &'static str, + status: u16, + } + + impl HttpReply { + fn ok(body: String, content_type: &'static str) -> Self { + Self { + body, + content_type, + reason: "OK", + status: 200, + } + } + } + + /// A gated one-request HTTP/1 fixture. It holds the first response until the test releases it, + /// then watches for extra calls while the generated app serves requests from its snapshots. + struct GatedService { + endpoint: String, + finish_tx: Sender<()>, + reply_tx: Sender, + request_rx: Receiver, + worker: Option>>, + } + + impl GatedService { + fn bind() -> Self { + let listener = TcpListener::bind("127.0.0.1:0").expect("bind literal loopback"); + listener + .set_nonblocking(true) + .expect("nonblocking fixture listener"); + let endpoint = format!("http://{}", listener.local_addr().expect("fixture address")); + let (request_tx, request_rx) = mpsc::channel(); + let (reply_tx, reply_rx) = mpsc::channel(); + let (finish_tx, finish_rx) = mpsc::channel(); + let worker = thread::spawn(move || { + let (mut startup_stream, _) = accept_before(&listener, deadline_after(WAIT)); + startup_stream + .set_read_timeout(Some(Duration::from_secs(3))) + .expect("bounded fixture request read"); + startup_stream + .set_write_timeout(Some(Duration::from_secs(3))) + .expect("bounded fixture response write"); + let startup_request = read_request(&mut startup_stream); + request_tx + .send(startup_request) + .expect("parent receives fixture request"); + let reply = reply_rx + .recv_timeout(WAIT) + .expect("test releases fixture response"); + write_reply(&mut startup_stream, &reply); + finish_rx + .recv_timeout(WAIT) + .expect("parent completes snapshot assertions"); + + let mut extra = Vec::new(); + let deadline = deadline_after(SERVICE_GRACE); + while Instant::now() < deadline { + match listener.accept() { + Ok((mut extra_stream, peer)) => { + assert!(peer.ip().is_loopback(), "only literal loopback peers"); + extra_stream + .set_read_timeout(Some(Duration::from_secs(2))) + .expect("bounded extra request read"); + let extra_request = read_request(&mut extra_stream); + let _sent = request_tx.send(HttpRequest { + body: extra_request.body.clone(), + headers: extra_request.headers.clone(), + }); + extra.push(extra_request); + write_reply( + &mut extra_stream, + &HttpReply { + body: "REQUEST_TIME_PROVIDER_ACCESS".to_owned(), + content_type: "text/plain", + reason: "Unexpected request", + status: 500, + }, + ); + } + Err(error) if error.kind() == ErrorKind::WouldBlock => { + thread::sleep(Duration::from_millis(5)); + } + Err(error) => panic!("fixture accept failed: {error}"), + } + } + extra + }); + Self { + endpoint, + finish_tx, + reply_tx, + request_rx, + worker: Some(worker), + } + } + + fn endpoint(&self) -> &str { + &self.endpoint + } + + fn finish(mut self) -> Vec { + self.finish_tx + .send(()) + .expect("fixture waits until handler assertions finish"); + self.worker + .take() + .expect("fixture worker") + .join() + .expect("fixture thread") + } + + fn reply(&self, reply: HttpReply) { + self.reply_tx.send(reply).expect("service awaits response"); + } + + fn request(&self) -> HttpRequest { + self.request_rx + .recv_timeout(WAIT) + .expect("startup request reaches loopback service") + } + } + + struct RunningHost { + address: SocketAddr, + child: Child, + stderr: Option>>, + stdout: Option>>, + } + + impl RunningHost { + fn http(&mut self, path: &str) -> String { + let deadline = deadline_after(WAIT); + loop { + if let Some(status) = self.child.try_wait().expect("generated process status") { + panic!("generated server exited before {path} ({status})"); + } + if let Ok(mut stream) = + TcpStream::connect_timeout(&self.address, Duration::from_millis(40)) + { + stream + .set_read_timeout(Some(Duration::from_secs(2))) + .expect("bounded generated response"); + write!( + stream, + "GET {path} HTTP/1.1\r\nHost: 127.0.0.1\r\nConnection: close\r\n\r\n" + ) + .expect("send generated handler request"); + let mut response = String::new(); + stream + .read_to_string(&mut response) + .expect("read generated response"); + if response.starts_with("HTTP/1.1 200") { + return response; + } + } + assert!( + Instant::now() < deadline, + "generated listener startup timeout" + ); + thread::sleep(Duration::from_millis(10)); + } + } + + fn spawn(mut command: Command, address: SocketAddr) -> Self { + command.stdout(Stdio::piped()).stderr(Stdio::piped()); + let mut child = command.spawn().expect("spawn generated server"); + let stdout_pipe = child.stdout.take().expect("capture generated stdout"); + let stderr_pipe = child.stderr.take().expect("capture generated stderr"); + let stdout_reader = thread::spawn(move || { + let mut output = Vec::new(); + let mut pipe = stdout_pipe; + pipe.read_to_end(&mut output) + .expect("read generated stdout"); + output + }); + let stderr_reader = thread::spawn(move || { + let mut output = Vec::new(); + let mut pipe = stderr_pipe; + pipe.read_to_end(&mut output) + .expect("read generated stderr"); + output + }); + Self { + address, + child, + stderr: Some(stderr_reader), + stdout: Some(stdout_reader), + } + } + + fn stop(mut self) -> Output { + if self + .child + .try_wait() + .expect("generated process state") + .is_none() + { + let _killed = self.child.kill(); + } + let status = self.child.wait().expect("reap generated app"); + let stdout = self + .stdout + .take() + .expect("stdout reader") + .join() + .expect("stdout thread"); + let stderr = self + .stderr + .take() + .expect("stderr reader") + .join() + .expect("stderr thread"); + Output { + status, + stderr, + stdout, + } + } + } + + impl Drop for RunningHost { + fn drop(&mut self) { + if self.child.try_wait().ok().flatten().is_none() { + let _killed = self.child.kill(); + } + let _status = self.child.wait(); + if let Some(stderr) = self.stderr.take() { + let _output = stderr.join(); + } + if let Some(stdout) = self.stdout.take() { + let _output = stdout.join(); + } + } + } + + fn deadline_after(duration: Duration) -> Instant { + Instant::now() + .checked_add(duration) + .expect("deadline within representable range") + } + + fn accept_before(listener: &TcpListener, deadline: Instant) -> (TcpStream, SocketAddr) { + loop { + match listener.accept() { + Ok((stream, peer)) => { + assert!(peer.ip().is_loopback(), "fixture peer must be loopback"); + return (stream, peer); + } + Err(error) if error.kind() == ErrorKind::WouldBlock => { + assert!(Instant::now() < deadline, "fixture request deadline"); + thread::sleep(Duration::from_millis(5)); + } + Err(error) => panic!("fixture accept failed: {error}"), + } + } + } + + fn read_request(stream: &mut TcpStream) -> HttpRequest { + let mut bytes = Vec::new(); + let mut chunk = [0_u8; 2048]; + let header_end = loop { + let read = stream.read(&mut chunk).expect("read request headers"); + assert_ne!(read, 0, "peer closed before request headers"); + bytes.extend_from_slice(&chunk[..read]); + assert!(bytes.len() <= 64 * 1024, "fixture request header bound"); + if let Some(index) = bytes.windows(4).position(|part| part == b"\r\n\r\n") { + break index.checked_add(4).expect("header end offset"); + } + }; + let headers = String::from_utf8_lossy(&bytes[..header_end]).into_owned(); + let content_length = headers + .lines() + .find_map(|line| { + let (_, value) = line + .split_once(':') + .filter(|(name, _)| name.eq_ignore_ascii_case("content-length"))?; + value.trim().parse::().ok() + }) + .unwrap_or(0); + assert!(content_length <= 64 * 1024, "fixture request body bound"); + let body_end = header_end + .checked_add(content_length) + .expect("request body end offset"); + while bytes.len() < body_end { + let read = stream.read(&mut chunk).expect("read request body"); + assert_ne!(read, 0, "peer closed before request body"); + bytes.extend_from_slice(&chunk[..read]); + assert!(bytes.len() <= body_end, "bounded request body"); + } + HttpRequest { + body: String::from_utf8_lossy(&bytes[header_end..body_end]).into_owned(), + headers: headers.to_ascii_lowercase(), + } + } + + fn write_reply(stream: &mut TcpStream, reply: &HttpReply) { + write!( + stream, + "HTTP/1.1 {} {}\r\nContent-Type: {}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + reply.status, + reply.reason, + reply.content_type, + reply.body.len() + ) + .expect("write fixture response headers"); + stream + .write_all(reply.body.as_bytes()) + .expect("write fixture response body"); + } + + fn toml_table(entries: impl IntoIterator) -> TomlValue { + TomlValue::Table(entries.into_iter().collect()) + } + + fn toml_strings(values: &[&str]) -> TomlValue { + TomlValue::Array( + values + .iter() + .map(|value| TomlValue::String((*value).to_owned())) + .collect(), + ) + } + + fn install_store_declarations(project: &Path) { + let manifest_path = project.join("edgezero.toml"); + let source = fs::read_to_string(&manifest_path).expect("scaffold manifest"); + let mut manifest: TomlValue = toml::from_str(&source).expect("parse scaffold TOML"); + let root = manifest.as_table_mut().expect("manifest root table"); + let stores = toml_table([ + ( + "config".to_owned(), + toml_table([ + ("ids".to_owned(), toml_strings(&["app_config"])), + ( + "default".to_owned(), + TomlValue::String("app_config".to_owned()), + ), + ]), + ), + ( + "secrets".to_owned(), + toml_table([ + ("ids".to_owned(), toml_strings(&["signing"])), + ( + "default".to_owned(), + TomlValue::String("signing".to_owned()), + ), + ]), + ), + ]); + root.insert("stores".to_owned(), stores); + fs::write( + &manifest_path, + toml::to_string(&manifest).expect("serialize manifest"), + ) + .expect("write portable store IDs"); + } + + fn set_axum_features(project: &Path, features: &[&str]) { + let manifest_path = project.join("edgezero.toml"); + let source = fs::read_to_string(&manifest_path).expect("generated manifest"); + let mut manifest: TomlValue = toml::from_str(&source).expect("parse generated manifest"); + let axum_build = manifest + .as_table_mut() + .and_then(|root| root.get_mut("adapters")) + .and_then(TomlValue::as_table_mut) + .and_then(|adapters| adapters.get_mut("axum")) + .and_then(TomlValue::as_table_mut) + .and_then(|axum| axum.get_mut("build")) + .and_then(TomlValue::as_table_mut) + .expect("generated Axum build settings"); + axum_build.insert("features".to_owned(), toml_strings(features)); + fs::write( + &manifest_path, + toml::to_string(&manifest).expect("serialize manifest"), + ) + .expect("write selected Axum build features"); + } + + fn prepare_typed_generated_app(project: &Path) { + let core = project.join("crates/native-probe-core/src"); + fs::write( + core.join("config.rs"), + concat!( + "use serde::Deserialize;\n", + "use validator::Validate;\n\n", + "#[derive(Debug, Deserialize, Validate, edgezero_core::AppConfig)]\n", + "#[serde(deny_unknown_fields)]\n", + "pub struct NativeProbeConfig {\n", + " #[validate(length(min = 4_u64))]\n", + " pub greeting: String,\n", + " #[secret]\n", + " #[validate(length(min = 8_u64))]\n", + " pub api_token: String,\n", + "}\n\n", + "#[derive(Clone)]\n", + "pub struct StartupState {\n", + " pub greeting: String,\n", + " pub api_token: String,\n", + "}\n", + ), + ) + .expect("write typed app schema"); + + let handlers_path = core.join("handlers.rs"); + let mut handlers = fs::read_to_string(&handlers_path).expect("generated handlers"); + handlers = handlers.replace( + "use edgezero_core::extractor::{Headers, Json, Path};", + "use edgezero_core::extractor::{Headers, Json, Path, State};\nuse std::sync::Arc;", + ); + let start = handlers + .find("#[action]\npub async fn root()") + .expect("generated root handler"); + let handler_tail = handlers.get(start..).expect("root handler UTF-8 boundary"); + let relative_end = handler_tail + .find("\n}\n") + .expect("generated root handler end"); + let handler_end = start + .checked_add(relative_end) + .expect("generated root handler end offset"); + let close = handler_end + .checked_add(3) + .expect("generated root handler closing offset"); + handlers.replace_range( + start..close, + r#"#[action] + pub async fn root(State(state): State>) -> Text { + Text::new(format!("{}|{}", state.greeting, state.api_token)) + } + "#, + ); + fs::write(handlers_path, handlers).expect("wire typed response state"); + fs::write( + project.join("native-probe.toml"), + "greeting = \"local-typed-config\"\napi_token = \"signing_key\"\n", + ) + .expect("write local typed validation fixture"); + } + + fn install_startup_initializer(project: &Path) { + fs::write( + project.join("crates/native-probe-adapter-axum/src/main.rs"), + r#"use std::fs; + use std::path::PathBuf; + use std::sync::Arc; + use std::thread; + use std::time::{Duration, Instant}; + + use edgezero_adapter_axum::dev_server::run_production_app_with_initializer; + use edgezero_core::extractor::AppConfig; + use edgezero_core::EdgeError; + use native_probe_core::config::{NativeProbeConfig, StartupState}; + + fn main() -> anyhow::Result<()> { + run_production_app_with_initializer::(|app, stores| { + Box::pin(async move { + let binding = stores + .config() + .and_then(|registry| registry.named_ref("app_config")) + .ok_or_else(|| EdgeError::service_unavailable("config binding missing"))?; + let config = AppConfig::::from_binding( + binding, + None, + stores.secrets(), + app.config_extraction_limits(), + app.monotonic_clock(), + ) + .await?; + if let (Ok(entered), Ok(release)) = ( + std::env::var("NATIVE_PROBE_INITIALIZER_ENTERED"), + std::env::var("NATIVE_PROBE_INITIALIZER_RELEASE"), + ) { + fs::write(entered, b"typed config is valid") + .map_err(EdgeError::internal)?; + let release = PathBuf::from(release); + let deadline = Instant::now().checked_add(Duration::from_secs(10)) + .ok_or_else(|| EdgeError::service_unavailable("initializer deadline unavailable"))?; + while !release.is_file() { + if Instant::now() >= deadline { + return Err(EdgeError::internal(anyhow::anyhow!( + "test initializer gate timed out" + ))); + } + thread::sleep(Duration::from_millis(5)); + } + } + app.insert_state(Arc::new(StartupState { + greeting: config.greeting, + api_token: config.api_token, + })); + Ok(()) + }) + })?; + Ok(()) + } + "#, + ) + .expect("install app-owned typed startup check"); + } + + fn bindings_file(path: &Path, agent_endpoint: &str) { + fs::write( + path, + format!( + r#"version = 1 + + [config.app_config] + provider = "aws-appconfig-agent" + default_key = "settings" + endpoint = "{agent_endpoint}" + access_token_env = "APPCONFIG_AGENT_TOKEN" + + [config.app_config.documents.settings] + application = "native-probe" + environment = "test" + profile = "startup-proof" + + [secrets.signing] + provider = "aws-secrets-manager" + region = "us-east-1" + + [secrets.signing.entries.signing_key] + secret_id = "{SECRET_ARN}" + "# + ), + ) + .expect("write absolute fictional bindings file"); + } + + fn serialized_envelope() -> String { + serde_json::to_string(&BlobEnvelope::new( + json!({ + "greeting": "generated-startup-greeting", + "api_token": "signing_key", + }), + "2026-10-07T12:00:00Z".to_owned(), + )) + .expect("serialize actual core envelope") + } + + fn secret_reply() -> HttpReply { + HttpReply::ok( + json!({ + "ARN": SECRET_ARN, + "Name": "native-probe-signing", + "VersionId": "12345678901234567890123456789012", + "SecretString": APP_TOKEN, + }) + .to_string(), + "application/x-amz-json-1.1", + ) + } + + fn run_disabled_child(mut command: Command, address: SocketAddr) -> Output { + command.stdout(Stdio::piped()).stderr(Stdio::piped()); + let mut child = command.spawn().expect("spawn default-off generated app"); + let deadline = deadline_after(WAIT); + loop { + if child + .try_wait() + .expect("default-off child status") + .is_some() + { + return child + .wait_with_output() + .expect("capture disabled-provider error"); + } + if TcpStream::connect_timeout(&address, Duration::from_millis(25)).is_ok() { + let _killed = child.kill(); + let _output = child.wait_with_output(); + panic!("default-off generated app accepted traffic for an AWS binding"); + } + if Instant::now() >= deadline { + let _killed = child.kill(); + let _output = child.wait_with_output(); + panic!("disabled provider did not fail closed"); + } + thread::sleep(Duration::from_millis(10)); + } + } + + fn assert_no_listener(address: SocketAddr) { + TcpStream::connect_timeout(&address, Duration::from_millis(60)) + .expect_err("startup must not accept before provider and app checks finish"); + } + + fn reserve_app_address() -> SocketAddr { + let listener = TcpListener::bind("127.0.0.1:0").expect("reserve loopback app port"); + listener.local_addr().expect("app address") + } + + fn empty_endpoint() -> String { + let listener = TcpListener::bind("127.0.0.1:0").expect("reserve local closed endpoint"); + let address = listener.local_addr().expect("closed loopback endpoint"); + drop(listener); + format!("http://{address}") + } + + fn profiles(root: &Path) -> (PathBuf, PathBuf, PathBuf) { + let home = root.join("empty-home"); + let aws = root.join("empty-aws-profiles"); + fs::create_dir_all(&home).expect("empty HOME"); + fs::create_dir_all(&aws).expect("profile directory"); + let config = aws.join("config"); + let credentials = aws.join("credentials"); + fs::write(&config, "").expect("no AWS config profiles"); + fs::write(&credentials, "").expect("no AWS credential profiles"); + (home, config, credentials) + } + + fn child_command( + binary: &Path, + root: &Path, + bindings: &Path, + app_address: SocketAddr, + secrets_endpoint: &str, + initializer_gate: Option<(&Path, &Path)>, + ) -> Command { + let (home, aws_config, aws_credentials) = profiles(root); + let mut command = Command::new(binary); + command + .env_clear() + .current_dir(root) + .env("HOME", home) + .env("AWS_CONFIG_FILE", aws_config) + .env("AWS_SHARED_CREDENTIALS_FILE", aws_credentials) + .env("AWS_REGION", "us-east-1") + .env("AWS_EC2_METADATA_DISABLED", "true") + .env("AWS_ACCESS_KEY_ID", "DUMMY_GENERATED_ACCESS_KEY") + .env("AWS_SECRET_ACCESS_KEY", "DUMMY_GENERATED_SECRET_KEY") + .env("AWS_ENDPOINT_URL", secrets_endpoint) + .env("AWS_ENDPOINT_URL_SECRETS_MANAGER", secrets_endpoint) + .env("AWS_IGNORE_CONFIGURED_ENDPOINT_URLS", "false") + .env("EDGEZERO__STORE_BINDINGS_FILE", bindings) + .env("EDGEZERO__ADAPTER__HOST", "127.0.0.1") + .env("EDGEZERO__ADAPTER__PORT", app_address.port().to_string()) + .env("EDGEZERO__LOGGING__LEVEL", "off") + .env("APPCONFIG_AGENT_TOKEN", AGENT_TOKEN) + .env("signing_key", FALLBACK_SENTINEL); + if let Some((entered, release)) = initializer_gate { + command + .env("NATIVE_PROBE_INITIALIZER_ENTERED", entered) + .env("NATIVE_PROBE_INITIALIZER_RELEASE", release); + } + command + } + + fn assert_no_sensitive_output(output: &Output) { + let text = format!( + "{}{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + for sentinel in [ + APP_TOKEN, + AGENT_TOKEN, + FALLBACK_SENTINEL, + "DUMMY_GENERATED_ACCESS_KEY", + "DUMMY_GENERATED_SECRET_KEY", + "AGENT_BODY_SENTINEL", + "SECRET_BODY_SENTINEL", + ] { + assert!(!text.contains(sentinel), "child output leaked {sentinel}"); + } + } + + fn assert_cargo_command(project: &Path, features: &[&str]) { + set_axum_features(project, features); + let output = Command::new(env!("CARGO_BIN_EXE_edgezero")) + .args(["build", "--adapter", "axum", "--", "--offline"]) + .current_dir(project) + .env("CARGO_TARGET_DIR", target_directory(project)) + .output() + .expect("run root CLI native build"); + assert!( + output.status.success(), + "edgezero build with selected manifest features {features:?} failed:\n{}\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + } + + fn target_directory(project: &Path) -> PathBuf { + match var_os("EDGEZERO_GENERATED_TEST_TARGET_DIR") { + Some(target_override) => { + let target_path = PathBuf::from(target_override); + if target_path.is_absolute() { + target_path + } else { + project.join(target_path) + } + } + None => project.join("target"), + } + } + + fn generated_binary(project: &Path) -> PathBuf { + target_directory(project).join("debug/native-probe-adapter-axum") + } + + fn wait_for_file(path: &Path) { + let deadline = deadline_after(WAIT); + while !path.is_file() { + assert!( + Instant::now() < deadline, + "generated initializer gate timeout" + ); + thread::sleep(Duration::from_millis(5)); + } + } + + fn http_body(response: &str) -> &str { + response + .split_once("\r\n\r\n") + .expect("HTTP response boundary") + .1 + } + + fn prepare_workspace(temp: &TempDir) -> PathBuf { + let scaffold = Command::new(env!("CARGO_BIN_EXE_edgezero")) + .args(["new", "native-probe", "--dir"]) + .arg(temp.path()) + .output() + .expect("run real edgezero new command"); + assert!( + scaffold.status.success(), + "edgezero new failed:\n{}\n{}", + String::from_utf8_lossy(&scaffold.stdout), + String::from_utf8_lossy(&scaffold.stderr) + ); + let project = temp.path().join("native-probe"); + assert!( + project + .join("crates/native-probe-adapter-axum/src/main.rs") + .is_file() + ); + install_store_declarations(&project); + prepare_typed_generated_app(&project); + project + } + + #[test] + #[ignore = "runs the real scaffold and five offline generated native builds; coordinator-owned Cargo check"] + #[expect( + clippy::too_many_lines, + reason = "Keep this generated CLI lifecycle as one readable end-to-end proof." + )] + fn generated_cli_build_features_and_native_aws_startup_are_proven_together() { + let temp = tempfile::tempdir().expect("generated project root"); + let project = prepare_workspace(&temp); + let bindings = temp.path().join("native-stores.toml"); + let no_service_endpoint = empty_endpoint(); + bindings_file(&bindings, &no_service_endpoint); + + // Default generated Cargo features stay empty. With an explicit AWS binding selected, + // run_generated_app must use strict startup and reject the disabled provider. Port 1 is + // just a closed loopback port, so a broken feature guard still cannot reach AWS. + assert_cargo_command(&project, &[]); + let disabled_address = reserve_app_address(); + let disabled = run_disabled_child( + child_command( + &generated_binary(&project), + temp.path(), + &bindings, + disabled_address, + &no_service_endpoint, + None, + ), + disabled_address, + ); + assert!( + !disabled.status.success(), + "disabled provider must fail startup" + ); + assert_no_listener(disabled_address); + let disabled_stderr = String::from_utf8_lossy(&disabled.stderr); + assert!( + disabled_stderr.contains("provider not compiled"), + "disabled provider must report a closed capability error: {disabled_stderr}" + ); + assert_no_sensitive_output(&disabled); + + // The build command under test reads `[adapters.axum.build].features`, not features on + // this CLI binary or arbitrary dependency unification in the root workspace. + assert_cargo_command(&project, &["aws-appconfig-agent"]); + assert_cargo_command(&project, &["aws-secrets-manager"]); + assert_cargo_command(&project, &["aws-appconfig-agent", "aws-secrets-manager"]); + + // This app-owned check is deliberately in the generated consumer. The generated default + // main remains run_generated_app; replace it only for the real typed-startup scenario. + install_startup_initializer(&project); + assert_cargo_command(&project, &["aws-appconfig-agent", "aws-secrets-manager"]); + + let agent = GatedService::bind(); + let secrets = GatedService::bind(); + let document = serialized_envelope(); + let agent_endpoint = agent.endpoint().to_owned(); + let secrets_endpoint = secrets.endpoint().to_owned(); + bindings_file(&bindings, &agent_endpoint); + let app_address = reserve_app_address(); + let initializer_entered = temp.path().join("initializer-entered"); + let initializer_release = temp.path().join("initializer-release"); + let mut host = RunningHost::spawn( + child_command( + &generated_binary(&project), + temp.path(), + &bindings, + app_address, + &secrets_endpoint, + Some((&initializer_entered, &initializer_release)), + ), + app_address, + ); + + // Hold each provider response. Neither listener may accept while provider setup or the + // typed app initializer is still pending. + let agent_request = agent.request(); + assert!(agent_request.headers.starts_with( + "get /applications/native-probe/environments/test/configurations/startup-proof " + )); + assert!( + agent_request + .headers + .contains(&format!("authorization: bearer {AGENT_TOKEN}")) + ); + assert!(agent_request.headers.contains("accept-encoding: identity")); + assert_no_listener(app_address); + agent.reply(HttpReply::ok(document, "application/json")); + + let secret_request = secrets.request(); + assert!( + secret_request + .headers + .contains("x-amz-target: secretsmanager.getsecretvalue") + ); + assert!( + secret_request + .headers + .contains("credential=dummy_generated_access_key/") + ); + let secret_input: Value = + serde_json::from_str(&secret_request.body).expect("actual SDK request JSON"); + assert_eq!(secret_input["SecretId"], SECRET_ARN); + assert!(secret_input.get("VersionStage").is_none()); + assert!(secret_input.get("VersionId").is_none()); + assert_no_listener(app_address); + secrets.reply(secret_reply()); + + // The generated initializer has now parsed and validated the actual envelope and resolved + // the nested signing_key value. It pauses before returning; the generated app must still + // keep its listener closed until the application-owned check completes. + wait_for_file(&initializer_entered); + assert_no_listener(app_address); + fs::write(&initializer_release, "continue").expect("release generated initializer"); + + // AppConfig::from_binding validated the envelope, resolved `api_token = "signing_key"` + // from the AWS secret snapshot, and inserted the typed state before the listener appeared. + let first = host.http("/"); + assert_eq!( + http_body(&first), + "generated-startup-greeting|generated-proof-secret-value" + ); + let second = host.http("/"); + assert_eq!( + http_body(&second), + "generated-startup-greeting|generated-proof-secret-value" + ); + + // Both loopback providers received one startup request. Request-time reads came entirely + // from the process snapshots; an accidental remote read would be captured as an extra. + assert!( + agent.finish().is_empty(), + "no Agent request after preparation" + ); + assert!( + secrets.finish().is_empty(), + "no Secrets Manager request after preparation" + ); + let output = host.stop(); + assert_no_sensitive_output(&output); + } +} diff --git a/crates/edgezero-core/src/app.rs b/crates/edgezero-core/src/app.rs index 74c86b998..bb013ac03 100644 --- a/crates/edgezero-core/src/app.rs +++ b/crates/edgezero-core/src/app.rs @@ -1,8 +1,10 @@ +use std::panic::catch_unwind; use std::sync::Arc; use crate::body::Body; use crate::config_store::ConfigExtractionLimits; use crate::error::EdgeError; +use crate::geo::GeoOutcome; use crate::http::header::CONTENT_TYPE; use crate::http::{HeaderValue, Method, Request, Response, StatusCode}; use crate::ingress::{ @@ -12,15 +14,14 @@ use crate::ingress::{ }; use crate::manifest::BakedManifest; use crate::platform::PlatformMetadata; -use crate::response::IntoResponse as _; use crate::response_egress::{ DEFAULT_RESPONSE_WRITE_BUDGET, DetachedResponseEgressDecision, DetachedResponseEgressDecisionFactory, DetachedResponseEgressHead, ResponseEgressCompletion, - ResponseEgressDeadline, ResponseEgressEnvelope, ResponseEgressHead, ResponseEgressObserver, - ResponseEgressObserverHandle, ResponseEgressPolicy, ResponseEgressPolicyCallback, - ResponseEgressRequestMetadata, default_response_egress_policy, + ResponseEgressDeadline, ResponseEgressEnvelope, ResponseEgressFailureClassification, + ResponseEgressHead, ResponseEgressObserver, ResponseEgressObserverHandle, ResponseEgressPolicy, + ResponseEgressPolicyCallback, ResponseEgressRequestMetadata, default_response_egress_policy, }; -use crate::router::{RouteMetadata, RouterService}; +use crate::router::{RouteErrorKind, RouteMetadata, RouterService}; use crate::time::{MonotonicClock, MonotonicInstant}; /// Canonical adapter name for the Axum adapter. @@ -32,7 +33,18 @@ const DEFAULT_APP_NAME: &str = "EdgeZero App"; pub const FASTLY_ADAPTER: &str = "fastly"; /// Canonical adapter name for the Spin adapter. pub const SPIN_ADAPTER: &str = "spin"; -type ErrorResponseRenderer = Arc Response + Send + Sync + 'static>; +enum ErrorResponseRenderer { + Custom(Arc Response + Send + Sync + 'static>), + Default, +} + +/// Presentation of framework-generated routing errors on a managed adapter path. +#[doc(hidden)] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum FrameworkErrorDetail { + CategoryOnly, + Default, +} /// Lightweight container around a `RouterService` that can be extended via hook implementations. pub struct App { @@ -92,6 +104,7 @@ impl App { let request_start = prepared.request_start(); let route = prepared.route_metadata().cloned(); let (_resolved, _admitted, completion) = prepared.into_parts(); + let failure = ResponseEgressFailureClassification::error(error.kind()); self.response_egress_envelope( self.render_error_response(error), request_method, @@ -99,6 +112,7 @@ impl App { route, completion, ) + .with_failure(Some(failure)) } /// Resolves and admits a normalized request head before an adapter transfers native body @@ -188,6 +202,7 @@ impl App { request_method: Method, request_start: MonotonicInstant, ) -> IngressDispatchOutcome { + let failure = ResponseEgressFailureClassification::error(error.kind()); let head = DetachedResponseEgressHead::new( &request_method, request_start, @@ -204,12 +219,15 @@ impl App { response .extensions_mut() .insert(ResponseEgressDeadline::at(deadline)); - IngressDispatchOutcome::Response(Box::new(self.detached_response_egress( - response, - request_method, - request_start, - completion, - ))) + IngressDispatchOutcome::Response(Box::new( + self.detached_response_egress( + response, + request_method, + request_start, + completion, + ) + .with_failure(Some(failure)), + )) } } } @@ -237,20 +255,73 @@ impl App { &self, prepared: PreparedIngress, request: Request, + ) -> ResponseEgressEnvelope { + self.dispatch_admitted_with_framework_error_detail( + prepared, + request, + FrameworkErrorDetail::Default, + ) + .await + } + + /// Dispatches with native default routing-error presentation, preserving custom renderers. + #[doc(hidden)] + #[inline] + pub async fn dispatch_admitted_with_framework_error_detail( + &self, + prepared: PreparedIngress, + request: Request, + detail: FrameworkErrorDetail, + ) -> ResponseEgressEnvelope { + self.dispatch_admitted_with_geo(prepared, request, detail, GeoOutcome::NoSelection) + .await + } + + /// Dispatches with an adapter-owned outcome, independent of public request extensions. + #[doc(hidden)] + #[inline] + pub async fn dispatch_admitted_with_geo( + &self, + prepared: PreparedIngress, + request: Request, + detail: FrameworkErrorDetail, + geo: GeoOutcome, ) -> ResponseEgressEnvelope { let request_method = request.method().clone(); let request_start = prepared.request_start(); let route = prepared.route_metadata().cloned(); let (resolved, admitted, completion) = prepared.into_parts(); - let response = match self + let (result, route_error, mut failure) = self .router - .dispatch_resolved(resolved, request, admitted) + .dispatch_resolved_outcome( + resolved, + request, + admitted.with_request_context(geo, detail == FrameworkErrorDetail::CategoryOnly), + ) .await - { + .into_parts(); + let response = match result { Ok(response) => response, - Err(error) => self.render_error_response(error), + Err(error) => { + failure = Some(if route_error.is_some() { + ResponseEgressFailureClassification::routing(error.kind()) + } else { + ResponseEgressFailureClassification::error(error.kind()) + }); + let fixed = if detail == FrameworkErrorDetail::CategoryOnly { + match route_error { + Some(RouteErrorKind::NotFound) => Some("route not found"), + Some(RouteErrorKind::MethodNotAllowed) => Some("method not allowed"), + None => None, + } + } else { + None + }; + self.render_error_response_with_message(error, fixed) + } }; self.response_egress_envelope(response, request_method, request_start, route, completion) + .with_failure(failure) } /// Resolves, admits, and dispatches one normalized inbound request into owned egress. @@ -289,6 +360,19 @@ impl App { self.ingress_head_limits } + /// Registers typed handler state during startup, before ingress admission or serving. + /// + /// Same-type insertion replaces previous state for this App only. Shared routers + /// fork on write, preserving routes and peer Apps. Previously resolved or admitted + /// dispatch tokens are invalid for this App after insertion. This is not live reload. + #[inline] + pub fn insert_state(&mut self, value: T) + where + T: Clone + Send + Sync + 'static, + { + self.router.insert_state(value); + } + /// Consume the app and return the contained router service. #[must_use] #[inline] @@ -332,12 +416,25 @@ impl App { } fn render_error_response(&self, error: EdgeError) -> Response { + self.render_error_response_with_message(error, None) + } + + fn render_error_response_with_message( + &self, + error: EdgeError, + fixed: Option<&'static str>, + ) -> Response { let status = error.status(); let required_headers = match error.required_response_headers() { Ok(headers) => headers, Err(header_error) => return render_default_error_response(header_error), }; - let mut response = (self.error_response_renderer)(error); + let mut response = match &self.error_response_renderer { + ErrorResponseRenderer::Default => { + render_default_error_response_with_message(error, fixed) + } + ErrorResponseRenderer::Custom(renderer) => renderer(error), + }; *response.status_mut() = status; response.headers_mut().extend(required_headers); response @@ -419,7 +516,7 @@ impl App { where Renderer: Fn(EdgeError) -> Response + Send + Sync + 'static, { - self.error_response_renderer = Arc::new(renderer); + self.error_response_renderer = ErrorResponseRenderer::Custom(Arc::new(renderer)); } /// Installs the synchronous body-blind ingress admission callback. @@ -501,7 +598,7 @@ impl App { deadline: head.write_deadline_after(DEFAULT_RESPONSE_WRITE_BUDGET), } }), - error_response_renderer: Arc::new(render_default_error_response), + error_response_renderer: ErrorResponseRenderer::Default, ingress_head_limits: IngressHeadLimits::default(), ingress_policy: default_admission_policy(), monotonic_clock: MonotonicClock::default(), @@ -612,10 +709,18 @@ pub trait Hooks { } fn render_default_error_response(error: EdgeError) -> Response { - match error.into_response() { + render_default_error_response_with_message(error, None) +} + +fn render_default_error_response_with_message( + error: EdgeError, + fixed: Option<&'static str>, +) -> Response { + match error.into_response_with_message(fixed) { Ok(response) => response, - Err(render_error) => { - log::error!("failed to render typed ingress error response: {render_error}"); + Err(_render_error) => { + let _logged = + catch_unwind(|| log::error!("failed to render typed ingress error response")); // Keep the last-resort response independent of JSON serialization. let mut response = Response::new(Body::from( r#"{"error":{"kind":"internal","message":"internal server error","status":500}}"#, @@ -1981,6 +2086,264 @@ mod tests { ); } + #[test] + fn geo_owned_outcome_survives_public_request_mutation() { + use crate::geo::{GeoOutcome, GeoProviderId, GeoStatus, GeoUnavailableReason}; + use crate::params::PathParams; + fn selected() -> GeoOutcome { + GeoOutcome::Selected { + provider: GeoProviderId::new("owned").unwrap(), + status: GeoStatus::Unavailable(GeoUnavailableReason::MissingMetadata), + } + } + struct MutatePublic; + #[async_trait(?Send)] + impl Middleware for MutatePublic { + async fn handle( + &self, + mut ctx: RequestContext, + next: Next<'_>, + ) -> Result { + assert_eq!(ctx.geo(), &selected()); + ctx.extensions_mut().insert(GeoOutcome::NoSelection); + ctx.headers_mut() + .insert("cloudfront-viewer-country", HeaderValue::from_static("DE")); + assert_eq!(ctx.geo(), &selected()); + next.run(ctx).await + } + } + let router = RouterService::builder() + .middleware(MutatePublic) + .post("/geo", |mut ctx: RequestContext| async move { + assert_eq!(ctx.geo(), &selected()); + ctx.extensions_mut().remove::(); + ctx.parts_mut().extensions.insert(GeoOutcome::NoSelection); + assert_eq!(ctx.geo(), &selected()); + let error = ctx + .json_within::(64) + .await + .expect_err("wrong JSON type"); + assert_eq!(error.message(), "invalid JSON payload"); + ctx.extensions_mut().insert(selected()); + let reconstructed = + RequestContext::new(ctx.into_request().unwrap(), PathParams::default()); + assert_eq!(reconstructed.geo(), &GeoOutcome::NoSelection); + Err::<&str, _>(error) + }) + .build(); + let mut app = App::new(router); + app.set_error_response_renderer(|error| { + assert_eq!(error.message(), "invalid JSON payload"); + Response::new(Body::from("private parse failure")) + }); + let mut request = request_builder() + .method(Method::POST) + .uri("/geo") + .header("content-type", "application/json") + .body(Body::from("\"PAYLOAD_SENTINEL\"")) + .unwrap(); + request.extensions_mut().insert(GeoOutcome::NoSelection); + let head = IngressHeadParts::from_request( + &request, + IngressHeadAccounting::HostManaged, + IngressFraming::HostManaged, + ); + let IngressBeginOutcome::Admitted(prepared) = + app.begin_ingress(head, app.monotonic_now()).unwrap() + else { + panic!("admission") + }; + let response = complete_envelope(block_on(app.dispatch_admitted_with_geo( + prepared, + request, + FrameworkErrorDetail::CategoryOnly, + selected(), + ))); + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + assert_eq!( + response.body().as_bytes().unwrap(), + b"private parse failure" + ); + } + + #[test] + fn geo_legacy_dispatch_ignores_forged_extensions() { + use crate::geo::{GeoError, GeoOutcome, GeoProviderId, GeoStatus}; + use crate::params::PathParams; + let forged = GeoOutcome::Selected { + provider: GeoProviderId::new("forged").unwrap(), + status: GeoStatus::Error(GeoError::ProviderFailure), + }; + let mut unmanaged = request_builder().body(Body::empty()).unwrap(); + unmanaged.extensions_mut().insert(forged.clone()); + assert_eq!( + RequestContext::new(unmanaged, PathParams::default()).geo(), + &GeoOutcome::NoSelection + ); + let legacy = App::new( + RouterService::builder() + .get("/legacy", |ctx: RequestContext| async move { + assert_eq!(ctx.geo(), &GeoOutcome::NoSelection); + Ok::<_, EdgeError>("legacy") + }) + .build(), + ); + let mut legacy_request = request_builder() + .uri("/legacy") + .body(Body::empty()) + .unwrap(); + legacy_request.extensions_mut().insert(forged.clone()); + let legacy_head = IngressHeadParts::from_request( + &legacy_request, + IngressHeadAccounting::HostManaged, + IngressFraming::HostManaged, + ); + let IngressBeginOutcome::Admitted(legacy_prepared) = legacy + .begin_ingress(legacy_head, legacy.monotonic_now()) + .unwrap() + else { + panic!("admission") + }; + let legacy_response = complete_envelope(block_on( + legacy.dispatch_admitted(legacy_prepared, legacy_request), + )); + assert_eq!(legacy_response.status(), StatusCode::OK); + } + + fn managed_dispatch( + app: &App, + method: Method, + path: &str, + detail: FrameworkErrorDetail, + ) -> ResponseEgressEnvelope { + let request = request_builder() + .method(method) + .uri(path) + .body(Body::empty()) + .expect("request"); + let head = IngressHeadParts::from_request( + &request, + IngressHeadAccounting::HostManaged, + IngressFraming::HostManaged, + ); + let IngressBeginOutcome::Admitted(prepared) = app + .begin_ingress(head, app.monotonic_now()) + .expect("admission") + else { + panic!("admission"); + }; + block_on(app.dispatch_admitted_with_framework_error_detail(prepared, request, detail)) + } + + #[test] + fn native_default_routing_errors_do_not_reflect_path_or_custom_method() { + let app = App::new( + RouterService::builder() + .get("/known", |_ctx: RequestContext| async { + Ok::<_, EdgeError>("ok") + }) + .build(), + ); + for (method, path, status, fixed) in [ + ( + Method::GET, + "/private-sentinel", + StatusCode::NOT_FOUND, + "route not found", + ), + ( + Method::from_bytes(b"PRIVATE-SENTINEL").expect("method"), + "/known", + StatusCode::METHOD_NOT_ALLOWED, + "method not allowed", + ), + ] { + let envelope = managed_dispatch(&app, method, path, FrameworkErrorDetail::CategoryOnly); + let failure = envelope.failure_classification().expect("routing category"); + assert!(failure.is_framework_route_error()); + let response = complete_dispatch(IngressDispatchOutcome::Response(Box::new(envelope))); + assert_eq!(response.status(), status); + if status == StatusCode::METHOD_NOT_ALLOWED { + assert_eq!(response.headers()["allow"], "GET"); + } + let payload: serde_json::Value = + serde_json::from_slice(response.body().as_bytes().expect("body")).expect("json"); + assert_eq!(payload["error"]["message"], fixed); + assert!( + !str::from_utf8(response.body().as_bytes().expect("body")) + .expect("utf8") + .contains("sentinel") + ); + } + let legacy = managed_dispatch( + &app, + Method::GET, + "/private-sentinel", + FrameworkErrorDetail::Default, + ); + let response = complete_dispatch(IngressDispatchOutcome::Response(Box::new(legacy))); + assert!( + str::from_utf8(response.body().as_bytes().expect("body")) + .expect("utf8") + .contains("private-sentinel") + ); + } + + #[test] + fn native_routing_detail_preserves_custom_renderer_input_and_handler_errors() { + let mut app = App::new( + RouterService::builder() + .get("/handler", |_ctx: RequestContext| async { + Err::<&'static str, _>(EdgeError::not_found("handler-sentinel")) + }) + .build(), + ); + let handler = managed_dispatch( + &app, + Method::GET, + "/handler", + FrameworkErrorDetail::CategoryOnly, + ); + assert!( + !handler + .failure_classification() + .expect("handler error") + .is_framework_route_error() + ); + let handler_response = + complete_dispatch(IngressDispatchOutcome::Response(Box::new(handler))); + assert!( + str::from_utf8(handler_response.body().as_bytes().expect("body")) + .expect("utf8") + .contains("handler-sentinel") + ); + let seen = Arc::new(Mutex::new(String::new())); + let renderer_seen = Arc::clone(&seen); + app.set_error_response_renderer(move |error| { + *renderer_seen.lock().expect("seen") = error.to_string(); + Response::new(Body::from("application-owned")) + }); + let envelope = managed_dispatch( + &app, + Method::GET, + "/route-sentinel", + FrameworkErrorDetail::CategoryOnly, + ); + assert!( + envelope + .failure_classification() + .expect("routing error") + .is_framework_route_error() + ); + let response = complete_dispatch(IngressDispatchOutcome::Response(Box::new(envelope))); + assert_eq!(response.status(), StatusCode::NOT_FOUND); + assert_eq!( + response.body().as_bytes().expect("body"), + b"application-owned" + ); + assert!(seen.lock().expect("seen").contains("route-sentinel")); + } + #[test] fn custom_error_response_renderer_handles_admitted_dispatch_errors() { let router = RouterService::builder() @@ -2570,6 +2933,60 @@ mod tests { assert_eq!(complete_envelope(response).status(), StatusCode::OK); } + #[test] + fn startup_state_forks_router_and_invalidates_old_admission() { + use crate::extractor::State; + + #[crate::action] + async fn state(State(value): State>) -> Result { + Ok(value.to_string()) + } + + let router = RouterService::builder() + .get("/state/{id}", state) + .with_state(Arc::::from("builder")) + .build(); + let peer = App::new(router.clone()); + let mut app = App::new(router); + let head = || { + IngressHeadParts::new( + Method::GET, + "/state/42".parse().expect("URI"), + Version::HTTP_11, + HeaderMap::new(), + ) + }; + let IngressBeginOutcome::Admitted(old) = app + .begin_ingress(head(), MonotonicInstant::now()) + .expect("admission") + else { + panic!("expected admission") + }; + let request = || { + request_builder() + .uri("/state/42") + .body(Body::empty()) + .expect("request") + }; + app.insert_state(Arc::::from("first")); + app.insert_state(Arc::::from("startup")); + assert_eq!(app.router().routes(), peer.router().routes()); + let rejected = complete_envelope(block_on(app.dispatch_admitted(old, request()))); + assert_eq!(rejected.status(), StatusCode::INTERNAL_SERVER_ERROR); + for _ in 0..2_u8 { + let IngressBeginOutcome::Admitted(new) = app + .begin_ingress(head(), MonotonicInstant::now()) + .expect("admission") + else { + panic!("expected admission") + }; + let response = complete_envelope(block_on(app.dispatch_admitted(new, request()))); + assert_eq!(response.body().as_bytes().expect("body"), b"startup"); + } + let response = block_on(peer.router().oneshot(request())).expect("peer response"); + assert_eq!(response.body().as_bytes().expect("body"), b"builder"); + } + #[test] fn default_hooks_do_not_own_logging() { assert!(!DefaultHooks::owns_logging()); diff --git a/crates/edgezero-core/src/context.rs b/crates/edgezero-core/src/context.rs index 76f84a544..50cb0d9e1 100644 --- a/crates/edgezero-core/src/context.rs +++ b/crates/edgezero-core/src/context.rs @@ -5,6 +5,7 @@ use crate::config_store::ConfigExtractionLimits; use crate::error::{ BadGatewayReason, BudgetSource, EdgeError, ResponseLimitReason, StoreExtractionReason, }; +use crate::geo::GeoOutcome; use crate::http::{Extensions, HeaderMap, Method, Request, RequestParts, Uri, Version}; use crate::ingress::{AdmittedIngress, IngressGrant}; use crate::outbound::HttpClient; @@ -161,7 +162,9 @@ impl StoredError { /// Request context exposed to handlers and middleware. pub struct RequestContext { body: RefCell, + category_only_json_errors: bool, config_extraction_limits: ConfigExtractionLimits, + geo: GeoOutcome, ingress_grant: RefCell>, monotonic_clock: MonotonicClock, parts: RequestParts, @@ -349,6 +352,13 @@ impl RequestContext { .map_err(|err| EdgeError::bad_request(format!("invalid form payload: {err}"))) } + /// Returns this request's immutable geographic outcome. Public extensions do not replace it. + #[must_use] + #[inline] + pub fn geo(&self) -> &GeoOutcome { + &self.geo + } + #[must_use] #[inline] pub fn headers(&self) -> &HeaderMap { @@ -396,8 +406,13 @@ impl RequestContext { T: DeserializeOwned, { let bytes = self.body_bytes(max).await?; - serde_json::from_slice(bytes.as_ref()) - .map_err(|err| EdgeError::bad_request(format!("invalid JSON payload: {err}"))) + serde_json::from_slice(bytes.as_ref()).map_err(|err| { + if self.category_only_json_errors { + EdgeError::bad_request("invalid JSON payload") + } else { + EdgeError::bad_request(format!("invalid JSON payload: {err}")) + } + }) } /// Resolve the [`BoundKvStore`] for `id`. Strict lookup: when a @@ -437,6 +452,8 @@ impl RequestContext { pub fn new(request: Request, params: PathParams) -> Self { let (parts, body) = request.into_parts(); Self { + geo: GeoOutcome::NoSelection, + category_only_json_errors: false, body: RefCell::new(BodyState::Initial(body)), config_extraction_limits: ConfigExtractionLimits::default(), ingress_grant: RefCell::new(None), @@ -453,12 +470,15 @@ impl RequestContext { request: Request, params: PathParams, route_metadata: RouteMetadata, - ingress: AdmittedIngress, + mut ingress: AdmittedIngress, ) -> Self { + let (geo, category_only_json_errors) = ingress.take_request_context(); let (request_start, read_deadline, grant, config_extraction_limits, monotonic_clock) = ingress.into_parts(); let (parts, body) = request.into_parts(); Self { + geo, + category_only_json_errors, body: RefCell::new(BodyState::Initial(body)), config_extraction_limits, ingress_grant: RefCell::new(Some(grant)), diff --git a/crates/edgezero-core/src/error.rs b/crates/edgezero-core/src/error.rs index 66ead0991..9e7d6e1b9 100644 --- a/crates/edgezero-core/src/error.rs +++ b/crates/edgezero-core/src/error.rs @@ -286,6 +286,66 @@ impl EdgeError { } } + pub(crate) fn into_response_with_message( + self, + fixed_message: Option<&'static str>, + ) -> Result { + let required_headers = self.required_response_headers()?; + let kind = self.kind(); + // `ConfigOutOfDate { field_path: String::new(), .. }` (the missing-blob + // path) must OMIT the `field_path` JSON key entirely, not emit + // `"field_path": ""`. Per spec 6.3.1. + let field_path_opt: Option<&str> = match &self { + EdgeError::ConfigOutOfDate { field_path, .. } if !field_path.is_empty() => { + Some(field_path.as_str()) + } + EdgeError::StoreExtraction { + field_path: Some(field_path), + .. + } => Some(field_path.as_str()), + EdgeError::BadGateway { .. } + | EdgeError::BadRequest { .. } + | EdgeError::ConfigOutOfDate { .. } + | EdgeError::GatewayTimeout { .. } + | EdgeError::Internal { .. } + | EdgeError::MethodNotAllowed { .. } + | EdgeError::NotFound { .. } + | EdgeError::NotImplemented { .. } + | EdgeError::RequestHeaderFieldsTooLarge { .. } + | EdgeError::RequestTimeout { .. } + | EdgeError::ResponseTooLarge { .. } + | EdgeError::ServiceUnavailable { .. } + | EdgeError::StoreExtraction { .. } + | EdgeError::UriTooLong { .. } + | EdgeError::Validation { .. } => None, + }; + let status = self.status(); + let message = fixed_message.map_or_else(|| self.wire_message(), str::to_owned); + + let mut payload = json!({ + "error": { + "status": status.as_u16(), + "kind": kind, + "message": message, + }, + }); + if let Some(field_path) = field_path_opt + && let Some(error_obj) = payload + .get_mut("error") + .and_then(serde_json::Value::as_object_mut) + { + error_obj.insert("field_path".into(), json!(field_path)); + } + + let body = json_or_text(&payload); + let mut response = response_with_body(status, body)?; + response + .headers_mut() + .insert(CONTENT_TYPE, HeaderValue::from_static("application/json")); + response.headers_mut().extend(required_headers); + Ok(response) + } + /// Returns the stable public category without provider or application diagnostics. #[must_use] #[inline] @@ -556,60 +616,7 @@ impl From for EdgeError { impl IntoResponse for EdgeError { #[inline] fn into_response(self) -> Result { - let required_headers = self.required_response_headers()?; - let kind = self.kind(); - // `ConfigOutOfDate { field_path: String::new(), .. }` (the missing-blob - // path) must OMIT the `field_path` JSON key entirely, not emit - // `"field_path": ""`. Per spec 6.3.1. - let field_path_opt: Option<&str> = match &self { - EdgeError::ConfigOutOfDate { field_path, .. } if !field_path.is_empty() => { - Some(field_path.as_str()) - } - EdgeError::StoreExtraction { - field_path: Some(field_path), - .. - } => Some(field_path.as_str()), - EdgeError::BadGateway { .. } - | EdgeError::BadRequest { .. } - | EdgeError::ConfigOutOfDate { .. } - | EdgeError::GatewayTimeout { .. } - | EdgeError::Internal { .. } - | EdgeError::MethodNotAllowed { .. } - | EdgeError::NotFound { .. } - | EdgeError::NotImplemented { .. } - | EdgeError::RequestHeaderFieldsTooLarge { .. } - | EdgeError::RequestTimeout { .. } - | EdgeError::ResponseTooLarge { .. } - | EdgeError::ServiceUnavailable { .. } - | EdgeError::StoreExtraction { .. } - | EdgeError::UriTooLong { .. } - | EdgeError::Validation { .. } => None, - }; - let status = self.status(); - let message = self.wire_message(); - - let mut payload = json!({ - "error": { - "status": status.as_u16(), - "kind": kind, - "message": message, - }, - }); - if let Some(field_path) = field_path_opt - && let Some(error_obj) = payload - .get_mut("error") - .and_then(serde_json::Value::as_object_mut) - { - error_obj.insert("field_path".into(), json!(field_path)); - } - - let body = json_or_text(&payload); - let mut response = response_with_body(status, body)?; - response - .headers_mut() - .insert(CONTENT_TYPE, HeaderValue::from_static("application/json")); - response.headers_mut().extend(required_headers); - Ok(response) + self.into_response_with_message(None) } } diff --git a/crates/edgezero-core/src/extractor.rs b/crates/edgezero-core/src/extractor.rs index 0d5de5bcd..40c5353c1 100644 --- a/crates/edgezero-core/src/extractor.rs +++ b/crates/edgezero-core/src/extractor.rs @@ -17,6 +17,7 @@ use crate::config_store::{ConfigExtractionLimits, ConfigStoreError, ConfigStoreH use crate::context::{DEFAULT_INBOUND_FORM_BYTES, DEFAULT_INBOUND_JSON_BYTES, RequestContext}; use crate::error::{EdgeError, StoreExtractionReason}; use crate::http::HeaderMap; +use crate::ingress::CheckedEffectiveHost; use crate::secret_store::SecretError; use crate::store_registry::{ BoundConfigStore, BoundKvStore, BoundSecretStore, ConfigRegistry, ConfigStoreBinding, @@ -247,6 +248,10 @@ pub struct ForwardedHost(pub String); impl FromRequest for ForwardedHost { #[inline] async fn from_request(ctx: &RequestContext) -> Result { + if let Some(checked) = ctx.extensions().get::() { + return Ok(Self(checked.authority().unwrap_or("localhost").to_owned())); + } + // Legacy unmanaged paths retain their existing raw-header behavior. let headers = ctx.headers(); let host = headers .get("x-forwarded-host") @@ -844,10 +849,15 @@ where None, ) })?; - let key = binding.default_key.clone(); - extract_from_handle::(ctx, &binding.handle, &key) - .await - .map(AppConfig) + Self::from_binding( + binding, + None, + ctx.extensions().get::(), + ctx.config_extraction_limits(), + ctx.monotonic_clock(), + ) + .await + .map(AppConfig) } } @@ -855,6 +865,33 @@ impl AppConfig where C: DeserializeOwned + AppConfigMeta + Validate + Send + 'static, { + /// Loads one application-selected binding without a request or route dispatch. + /// + /// Uses the same limits, envelope checks, secret resolution, validation and + /// error categories as request extraction. `None` selects the binding's default + /// key. This does not cache config or infer required application schemas. + /// + /// # Errors + /// Returns a typed store-extraction error for backend, budget, envelope, + /// secret or schema failures. Native startup must classify errors before logging. + #[inline] + pub async fn from_binding( + binding: &ConfigStoreBinding, + key: Option<&str>, + secrets: Option<&SecretRegistry>, + limits: ConfigExtractionLimits, + clock: MonotonicClock, + ) -> Result { + extract_from_handle::( + &binding.handle, + key.unwrap_or(&binding.default_key), + secrets, + limits, + clock, + ) + .await + } + /// Read the typed config from a NON-default config store. /// `key = None` falls back to that store's `binding.default_key`. /// Returns the inner `C` directly per spec 6.2.1. @@ -874,8 +911,14 @@ where None, ) })?; - let resolved_key = key.unwrap_or(&binding.default_key).to_owned(); - extract_from_handle::(ctx, &binding.handle, &resolved_key).await + Self::from_binding( + binding, + key, + ctx.extensions().get::(), + ctx.config_extraction_limits(), + ctx.monotonic_clock(), + ) + .await } /// Read the typed config from the default store under an @@ -896,7 +939,14 @@ where None, ) })?; - extract_from_handle::(ctx, &binding.handle, key).await + Self::from_binding( + binding, + Some(key), + ctx.extensions().get::(), + ctx.config_extraction_limits(), + ctx.monotonic_clock(), + ) + .await } } @@ -1201,15 +1251,16 @@ fn future_format_reason(raw: &str) -> Option { /// Returns [`EdgeError::StoreExtraction`] with the stable reason corresponding /// to store, envelope, secret, or schema failure. async fn extract_from_handle( - ctx: &RequestContext, handle: &ConfigStoreHandle, key: &str, + secrets: Option<&SecretRegistry>, + limits: ConfigExtractionLimits, + clock: MonotonicClock, ) -> Result where C: DeserializeOwned + AppConfigMeta + Validate + Send + 'static, { - let mut budget = - ConfigExtractionBudget::start(ctx.config_extraction_limits(), ctx.monotonic_clock())?; + let mut budget = ConfigExtractionBudget::start(limits, clock)?; let read = handle .get_bounded( key, @@ -1274,7 +1325,7 @@ where let mut data = parsed?; drop(raw); - let resolved = secret_walk::(ctx, &mut budget, &mut data).await; + let resolved = secret_walk::(secrets, &mut budget, &mut data).await; budget.check_deadline()?; resolved?; @@ -1335,7 +1386,7 @@ fn map_config_store_error(err: &ConfigStoreError) -> EdgeError { /// /// `StoreRef` fields are skipped — their value is a store id, not a key. async fn secret_walk( - ctx: &RequestContext, + secrets: Option<&SecretRegistry>, budget: &mut ConfigExtractionBudget, data: &mut serde_json::Value, ) -> Result<(), EdgeError> @@ -1349,7 +1400,7 @@ where if matches!(field.kind, SecretKind::StoreRef) { continue; } - resolve_secret_field(ctx, budget, data, &field, &field.path, String::new()).await?; + resolve_secret_field(secrets, budget, data, &field, &field.path, String::new()).await?; } Ok(()) } @@ -1358,7 +1409,7 @@ where /// secret leaf(s). `rendered` is the dotted path so far (with concrete `[n]` /// indices) for error hints. fn resolve_secret_field<'walk>( - ctx: &'walk RequestContext, + secrets: Option<&'walk SecretRegistry>, budget: &'walk mut ConfigExtractionBudget, node: &'walk mut serde_json::Value, field: &'walk SecretField, @@ -1369,7 +1420,7 @@ fn resolve_secret_field<'walk>( match remaining.split_first() { // Leaf reached: `node` is the PARENT object; the last field is the key. Some((SecretPathSegment::Field(name), [])) => { - resolve_leaf(ctx, budget, node, field, name.as_ref(), &rendered).await + resolve_leaf(secrets, budget, node, field, name.as_ref(), &rendered).await } Some((SecretPathSegment::OptionalField(name), [])) => { if node.as_object().is_some_and(|parent| { @@ -1380,7 +1431,7 @@ fn resolve_secret_field<'walk>( }) { return Ok(()); } - resolve_leaf(ctx, budget, node, field, name.as_ref(), &rendered).await + resolve_leaf(secrets, budget, node, field, name.as_ref(), &rendered).await } // Required intermediates still reject stale blobs. Optional // intermediates are represented explicitly below rather than by the @@ -1394,7 +1445,8 @@ fn resolve_secret_field<'walk>( Some(next_rendered), )), Some(child) => { - resolve_secret_field(ctx, budget, child, field, rest, next_rendered).await + resolve_secret_field(secrets, budget, child, field, rest, next_rendered) + .await } } } @@ -1410,7 +1462,8 @@ fn resolve_secret_field<'walk>( match parent.get_mut(name.as_ref()) { None | Some(serde_json::Value::Null) => Ok(()), Some(child) => { - resolve_secret_field(ctx, budget, child, field, rest, next_rendered).await + resolve_secret_field(secrets, budget, child, field, rest, next_rendered) + .await } } } @@ -1426,7 +1479,7 @@ fn resolve_secret_field<'walk>( }; for (idx, item) in items.iter_mut().enumerate() { let indexed = format!("{rendered}[{idx}]"); - resolve_secret_field(ctx, budget, item, field, rest, indexed).await?; + resolve_secret_field(secrets, budget, item, field, rest, indexed).await?; } Ok(()) } @@ -1447,7 +1500,7 @@ fn join_field(prefix: &str, name: &str) -> String { /// secret field name; `store_ref_field` (for `KeyInNamedStore`) is a sibling /// within `parent`. async fn resolve_leaf( - ctx: &RequestContext, + secrets: Option<&SecretRegistry>, budget: &mut ConfigExtractionBudget, parent: &mut serde_json::Value, field: &SecretField, @@ -1486,7 +1539,7 @@ async fn resolve_leaf( let (bound, resolved_store_id) = match field.kind { SecretKind::KeyInDefault => { - let bound = ctx.secret_store_default().ok_or_else(|| { + let bound = secrets.and_then(SecretRegistry::default).ok_or_else(|| { store_extraction_error( StoreExtractionReason::MissingRegistry, format!( @@ -1514,19 +1567,21 @@ async fn resolve_leaf( ) })? .to_owned(); - let bound = ctx.secret_store(&store_id_str).ok_or_else(|| { - // `store_id_str` is the blob's store_ref VALUE — config data that - // may be sensitive — and this message reaches the HTTP body. Name - // the field, not the stored id. - store_extraction_error( - StoreExtractionReason::UnknownStore, - format!( - "secret field `{leaf_path}` names a store_ref that is not declared in \ + let bound = secrets + .and_then(|registry| registry.named(&store_id_str)) + .ok_or_else(|| { + // `store_id_str` is the blob's store_ref VALUE — config data that + // may be sensitive — and this message reaches the HTTP body. Name + // the field, not the stored id. + store_extraction_error( + StoreExtractionReason::UnknownStore, + format!( + "secret field `{leaf_path}` names a store_ref that is not declared in \ [stores.secrets] (id redacted)" - ), - Some(leaf_path.clone()), - ) - })?; + ), + Some(leaf_path.clone()), + ) + })?; (bound, store_id_str) } }; @@ -2274,6 +2329,56 @@ mod tests { } // ForwardedHost extractor tests + #[test] + fn forwarded_host_checked_value_precedes_both_raw_headers() { + use crate::ingress::CheckedHostSource; + let mut request = request_builder() + .uri("/test") + .body(Body::empty()) + .expect("request"); + request + .headers_mut() + .insert("host", HeaderValue::from_static("direct.example")); + request.headers_mut().insert( + "x-forwarded-host", + HeaderValue::from_static("spoofed.example"), + ); + request + .extensions_mut() + .insert(CheckedEffectiveHost::from_ingress( + Some("verified.example:8443".parse().expect("authority")), + CheckedHostSource::TrustedForwarded, + )); + let ctx = RequestContext::new(request, PathParams::default()); + let host = block_on(ForwardedHost::from_request(&ctx)).expect("host"); + assert_eq!(host.0, "verified.example:8443"); + } + + #[test] + fn forwarded_host_checked_unavailable_never_falls_through_to_raw_headers() { + use crate::ingress::CheckedHostSource; + let mut request = request_builder() + .uri("/test") + .body(Body::empty()) + .expect("request"); + request + .headers_mut() + .insert("host", HeaderValue::from_static("direct.example")); + request.headers_mut().insert( + "x-forwarded-host", + HeaderValue::from_static("spoofed.example"), + ); + request + .extensions_mut() + .insert(CheckedEffectiveHost::from_ingress( + None, + CheckedHostSource::Unavailable, + )); + let ctx = RequestContext::new(request, PathParams::default()); + let host = block_on(ForwardedHost::from_request(&ctx)).expect("host"); + assert_eq!(host.0, "localhost"); + } + #[test] fn forwarded_host_extractor_uses_x_forwarded_host_first() { let mut request = request_builder() @@ -2785,7 +2890,7 @@ mod tests { let mut budget = ConfigExtractionBudget::start(ConfigExtractionLimits::default(), ctx.monotonic_clock()) .expect("test budget"); - secret_walk::(ctx, &mut budget, data).await + secret_walk::(ctx.extensions().get::(), &mut budget, data).await } #[test] diff --git a/crates/edgezero-core/src/geo.rs b/crates/edgezero-core/src/geo.rs new file mode 100644 index 000000000..a475e7c80 --- /dev/null +++ b/crates/edgezero-core/src/geo.rs @@ -0,0 +1,659 @@ +//! Portable geographic assertions and synchronous, local-only providers. +//! +//! Geography is an estimate, not proof of residence, identity or consent. +//! Country-only results and unrecognized subdivision assertions remain partial. +//! Applications own jurisdiction recognition and permission policy. +//! +//! ``` +//! use edgezero_core::geo::{CountryCode, GeoLocation, SubdivisionCode}; +//! +//! let country_only = GeoLocation::new(CountryCode::new("us")?, None); +//! assert_eq!(country_only.country().as_str(), "US"); +//! assert!(country_only.subdivision().is_none()); +//! +//! // Syntax-valid does not mean an assigned or recognized US subdivision. +//! let assertion = SubdivisionCode::new("zz")?; +//! let location = GeoLocation::new(CountryCode::new("us")?, Some(assertion)); +//! assert_eq!(location.subdivision().map(SubdivisionCode::as_str), Some("ZZ")); +//! # Ok::<(), edgezero_core::geo::GeoValueError>(()) +//! ``` + +#![expect( + clippy::self_named_module_files, + reason = "repo convention uses geo.rs with submodules under geo/, not mod.rs" +)] + +mod country_codes; + +use std::fmt; +use std::net::IpAddr; + +/// A bounded validation category. Rejected source values are never retained. +#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] +pub enum GeoValueError { + #[error("invalid country code")] + InvalidCountry, + #[error("invalid geographic provider ID")] + InvalidProviderId, + #[error("invalid subdivision code")] + InvalidSubdivision, + #[error("reserved geographic provider ID")] + ReservedProviderId, +} + +/// A public diagnostic label, never a credential. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct GeoProviderId(String); + +impl GeoProviderId { + /// Return the validated public label. + #[inline] + #[must_use] + pub fn as_str(&self) -> &str { + &self.0 + } + + /// Identity reserved for the built-in `CloudFront` provider. + /// + /// Adapters must reject this ID for application-supplied selections. + #[inline] + #[must_use] + pub fn cloudfront() -> Self { + Self("cloudfront".to_owned()) + } + + /// Identity reserved for the built-in local `MaxMind` database provider. + #[inline] + #[must_use] + pub fn maxmind() -> Self { + Self("maxmind".to_owned()) + } + + /// Validate an application provider label matching `[a-z][a-z0-9._-]{0,63}`. + /// + /// # Errors + /// Returns a bounded category for invalid labels or a reserved built-in ID. + #[inline] + pub fn new(value: &str) -> Result { + if value.is_empty() + || value.len() > 64 + || !value.as_bytes().first().is_some_and(u8::is_ascii_lowercase) + || !value.bytes().all(|byte| { + byte.is_ascii_lowercase() || byte.is_ascii_digit() || b"._-".contains(&byte) + }) + { + return Err(GeoValueError::InvalidProviderId); + } + if matches!(value, "cloudfront" | "maxmind") { + return Err(GeoValueError::ReservedProviderId); + } + Ok(Self(value.to_owned())) + } +} + +/// An officially assigned ISO 3166-1 alpha-2 code from the pinned snapshot. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct CountryCode(String); + +impl CountryCode { + /// Return the normalized code. + #[inline] + #[must_use] + pub fn as_str(&self) -> &str { + &self.0 + } + + /// Validate two ASCII letters and normalize them to uppercase. + /// + /// # Errors + /// Rejects unassigned, reserved and unsupported codes without alias substitution. + #[inline] + pub fn new(value: &str) -> Result { + if value.len() != 2 || !value.bytes().all(|byte| byte.is_ascii_alphabetic()) { + return Err(GeoValueError::InvalidCountry); + } + let code = value.to_ascii_uppercase(); + if country_codes::ASSIGNED + .binary_search(&code.as_str()) + .is_err() + { + return Err(GeoValueError::InvalidCountry); + } + Ok(Self(code)) + } +} + +/// A syntax-checked ISO 3166-2 component assertion, without a country prefix. +/// +/// This does not verify membership in a country's subdivision catalogue. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct SubdivisionCode(String); + +impl SubdivisionCode { + /// Return the normalized component. + #[inline] + #[must_use] + pub fn as_str(&self) -> &str { + &self.0 + } + + /// Validate one to three ASCII alphanumerics, preserving leading zeroes. + /// + /// # Errors + /// Rejects malformed components. No trimming, decoding or prefix removal occurs. + #[inline] + pub fn new(value: &str) -> Result { + if !(1..=3).contains(&value.len()) + || !value.bytes().all(|byte| byte.is_ascii_alphanumeric()) + { + return Err(GeoValueError::InvalidSubdivision); + } + Ok(Self(value.to_ascii_uppercase())) + } +} + +/// A country estimate with an optional syntax-checked subdivision assertion. +/// +/// A subdivision-only location cannot be constructed: +/// ```compile_fail +/// use edgezero_core::geo::{GeoLocation, SubdivisionCode}; +/// let subdivision = SubdivisionCode::new("CA").unwrap(); +/// let location = GeoLocation::new(None, Some(subdivision)); +/// ``` +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct GeoLocation { + country: CountryCode, + subdivision: Option, +} + +impl GeoLocation { + /// Return the country estimate. + #[inline] + #[must_use] + pub fn country(&self) -> &CountryCode { + &self.country + } + + /// Build a location from validated codes. Country is always required. + #[inline] + #[must_use] + pub fn new(country: CountryCode, subdivision: Option) -> Self { + Self { + country, + subdivision, + } + } + + /// Return the asserted subdivision, if present. Absence remains unknown. + #[inline] + #[must_use] + pub fn subdivision(&self) -> Option<&SubdivisionCode> { + self.subdivision.as_ref() + } +} + +/// Bounded request-resolution errors, with no raw error chain or source data. +#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] +pub enum GeoError { + #[error("invalid geographic assertion")] + InvalidAssertion, + #[error("geographic provider failure")] + ProviderFailure, +} + +/// An absent, bounded or rejected source assertion. +/// +/// Ingress owns duplicate detection and must use [`Self::invalid`] for duplicates, +/// including identical duplicates. A bounded value is not yet a validated code. +#[derive(Clone, PartialEq, Eq)] +pub struct GeoAssertion { + state: AssertionState, +} + +#[derive(Clone, PartialEq, Eq)] +enum AssertionState { + Absent, + Invalid, + Value { bytes: [u8; N], len: usize }, +} + +impl GeoAssertion { + /// Record a missing assertion. + #[inline] + #[must_use] + pub const fn absent() -> Self { + Self { + state: AssertionState::Absent, + } + } + + /// Inspect the bounded assertion, preserving missing versus rejected input. + /// + /// # Errors + /// Returns [`GeoError::InvalidAssertion`] for an ingress-rejected assertion. + #[inline] + pub fn as_bytes(&self) -> Result, GeoError> { + match &self.state { + AssertionState::Absent => Ok(None), + AssertionState::Value { bytes, len } => Ok(Some(&bytes[..*len])), + AssertionState::Invalid => Err(GeoError::InvalidAssertion), + } + } + + /// Copy at most `N` bytes. Oversize values become invalid before copying. + /// + /// Empty, malformed and non-UTF-8 bounded values remain values for the + /// provider to reject; they never become absence or truncated valid input. + #[inline] + #[must_use] + pub fn from_bytes(value: &[u8]) -> Self { + if value.len() > N { + return Self::invalid(); + } + let mut bytes = [0; N]; + bytes[..value.len()].copy_from_slice(value); + Self { + state: AssertionState::Value { + bytes, + len: value.len(), + }, + } + } + + /// Record a rejected assertion without retaining its bytes. + #[inline] + #[must_use] + pub const fn invalid() -> Self { + Self { + state: AssertionState::Invalid, + } + } +} + +impl fmt::Debug for GeoAssertion { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(match self.state { + AssertionState::Absent => "GeoAssertion(Absent)", + AssertionState::Value { .. } => "GeoAssertion(Value([redacted]))", + AssertionState::Invalid => "GeoAssertion(Invalid)", + }) + } +} + +/// Checked ingress provenance for an admitted reader address, not identity proof. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GeoIpSource { + DirectPeer, + TrustedForwarded, + TrustedXForwarded, +} + +/// Source-admitted provider input. Reader addresses are redacted in diagnostics. +#[derive(Clone, PartialEq, Eq)] +pub enum GeoInput { + Assertions { + country: GeoAssertion<2>, + subdivision: GeoAssertion<3>, + }, + ReaderIp { + address: IpAddr, + source: GeoIpSource, + }, +} + +impl fmt::Debug for GeoInput { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Assertions { + country, + subdivision, + } => f + .debug_struct("Assertions") + .field("country", country) + .field("subdivision", subdivision) + .finish(), + Self::ReaderIp { source, .. } => f + .debug_struct("ReaderIp") + .field("address", &"[redacted]") + .field("source", source) + .finish(), + } + } +} + +/// Provider-owned resolution, without framework selection or provenance choices. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum GeoResolution { + Error(GeoError), + Located(GeoLocation), + MissingMetadata, + NoMatch, +} + +/// A synchronous provider performing only bounded local computation. +/// +/// Implementations must not perform network, filesystem or other blocking I/O, +/// launch background work or resolve geography again during response finalization. +/// They return owned results without futures. The portable trait intentionally +/// imposes no `Send` or `Sync` bound; native composition adds those bounds. +/// +/// Application Rust is trusted code. This narrow input prevents accidental +/// coupling, but does not sandbox a malicious implementation. +pub trait GeoProvider { + /// Resolve admitted input. Unsupported input cases return `ProviderFailure`. + fn resolve(&self, input: &GeoInput) -> GeoResolution; +} + +/// Why a selected provider could not supply geography. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GeoUnavailableReason { + MissingMetadata, + MissingReaderIp, + NoMatch, + UntrustedIngress, +} + +/// The source of a located assertion, not a distribution identity attestation. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GeoSource { + CheckedReaderIp, + CloudFront, + LocalDatabase, +} + +/// The declared infrastructure protection used for source admission. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GeoTrustBasis { + /// An address admitted through the native checked ingress boundary. + CheckedReaderIp(GeoIpSource), + /// Operator-declared credential-gated, network-isolated ALB path. + /// This does not prove the external listener or network configuration. + DeclaredProtectedAlb, +} + +/// Framework-owned status for a selected provider. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum GeoStatus { + Error(GeoError), + Located { + location: GeoLocation, + source: GeoSource, + trust_basis: GeoTrustBasis, + }, + Unavailable(GeoUnavailableReason), +} + +/// One owned request outcome. It neither sets HTTP status nor grants permission. +/// +/// Every selected status retains the deployment-selected provider identity, +/// including source rejection before a provider call. No fallback is implied. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum GeoOutcome { + NoSelection, + Selected { + provider: GeoProviderId, + status: GeoStatus, + }, +} + +#[cfg(test)] +mod tests { + use super::*; + use std::cell::Cell; + use std::rc::Rc; + use std::str; + + #[test] + fn geo_country_codes_normalize_and_reject_unsupported() { + assert_eq!(CountryCode::new("us").unwrap().as_str(), "US"); + assert_eq!(CountryCode::new("gB").unwrap().as_str(), "GB"); + assert_eq!(CountryCode::new("NA").unwrap().as_str(), "NA"); + for value in [ + "", "U", "USA", " US", "US ", "U S", "\u{e9}", "U1", "US,CA", "\0U", + ] { + assert_eq!(CountryCode::new(value), Err(GeoValueError::InvalidCountry)); + } + // Reserved, transitional and user-assigned codes are not aliases. + for value in [ + "XX", "ZZ", "UK", "XK", "AA", "QM", "QZ", "XA", "XZ", "EU", "SU", "AN", "TP", + ] { + assert_eq!(CountryCode::new(value), Err(GeoValueError::InvalidCountry)); + } + for (value, expected) in [ + ("ca", "CA"), + ("1", "1"), + ("01", "01"), + ("001", "001"), + ("a1b", "A1B"), + ("ZZ", "ZZ"), + ] { + assert_eq!(SubdivisionCode::new(value).unwrap().as_str(), expected); + } + for value in [ + "", "ABCD", "US-CA", " CA", "CA ", "A B", "\u{e9}", "A_", "1,2", "\0", + ] { + assert_eq!( + SubdivisionCode::new(value), + Err(GeoValueError::InvalidSubdivision) + ); + } + let country = CountryCode::new("US").unwrap(); + let unknown = GeoLocation::new(country.clone(), Some(SubdivisionCode::new("ZZ").unwrap())); + assert_eq!(unknown.country(), &country); + assert_eq!(unknown.subdivision().unwrap().as_str(), "ZZ"); + assert!(GeoLocation::new(country, None).subdivision().is_none()); + } + + #[test] + fn country_snapshot_is_ordered_unique_and_complete() { + use sha2::{Digest as _, Sha256}; + + assert_eq!(country_codes::ASSIGNED.len(), 249); + assert!( + country_codes::ASSIGNED + .windows(2) + .all(|pair| matches!(pair, [first, second] if first < second)) + ); + let mut snapshot = String::new(); + for code in country_codes::ASSIGNED { + assert_eq!(code.len(), 2); + assert!(code.bytes().all(|byte| byte.is_ascii_uppercase())); + assert_eq!(CountryCode::new(code).unwrap().as_str(), *code); + snapshot.push_str(code); + snapshot.push('\n'); + } + assert_eq!( + format!("{:x}", Sha256::digest(snapshot.as_bytes())), + country_codes::SNAPSHOT_SHA256 + ); + let mut accepted = 0; + for first in b'A'..=b'Z' { + for second in b'A'..=b'Z' { + let bytes = [first, second]; + let value = str::from_utf8(&bytes).unwrap(); + let valid = country_codes::ASSIGNED.contains(&value); + assert_eq!(CountryCode::new(value).is_ok(), valid); + assert_eq!(CountryCode::new(&value.to_ascii_lowercase()).is_ok(), valid); + accepted += usize::from(valid); + } + } + assert_eq!(accepted, 249); + } + + #[test] + fn provider_ids_are_bounded_public_labels_with_a_reserved_builtin() { + for value in [ + "a", + "local", + "test.provider-1_v2", + &format!("a{}", "9".repeat(63)), + ] { + assert_eq!(GeoProviderId::new(value).unwrap().as_str(), value); + } + for value in [ + "", + "A", + "1a", + "_a", + "a b", + "\u{e9}", + "a/b", + "a\n", + &"a".repeat(65), + ] { + assert_eq!( + GeoProviderId::new(value), + Err(GeoValueError::InvalidProviderId) + ); + } + assert_eq!( + GeoProviderId::new("cloudfront"), + Err(GeoValueError::ReservedProviderId) + ); + assert_eq!(GeoProviderId::cloudfront().as_str(), "cloudfront"); + } + + #[test] + fn assertions_are_bounded_owned_and_redacted() { + assert_eq!(GeoAssertion::<2>::absent().as_bytes(), Ok(None)); + assert_eq!( + GeoAssertion::<2>::invalid().as_bytes(), + Err(GeoError::InvalidAssertion) + ); + for bytes in [b"".as_slice(), b"U", b"US", &[0xff, 0xfe]] { + let assertion = GeoAssertion::<2>::from_bytes(bytes); + assert_eq!(assertion.as_bytes(), Ok(Some(bytes))); + assert_eq!(format!("{assertion:?}"), "GeoAssertion(Value([redacted]))"); + } + assert_eq!( + GeoAssertion::<2>::from_bytes(b"USA").as_bytes(), + Err(GeoError::InvalidAssertion) + ); + assert_eq!( + GeoAssertion::<3>::from_bytes(b"123").as_bytes(), + Ok(Some(b"123".as_slice())) + ); + assert_eq!( + GeoAssertion::<3>::from_bytes(b"1234").as_bytes(), + Err(GeoError::InvalidAssertion) + ); + assert_eq!( + GeoAssertion::<0>::from_bytes(b"X").as_bytes(), + Err(GeoError::InvalidAssertion) + ); + let mut source = *b"us"; + let assertion = GeoAssertion::<2>::from_bytes(&source); + source.fill(b'X'); + assert_eq!(assertion.as_bytes(), Ok(Some(b"us".as_slice()))); + let input = GeoInput::Assertions { + country: assertion, + subdivision: GeoAssertion::from_bytes(b"ca"), + }; + let debug = format!("{input:?}"); + assert!(!debug.contains("us")); + assert!(!debug.contains("ca")); + assert!(debug.contains("[redacted]")); + } + + #[test] + fn validated_values_and_outcomes_have_equality_and_safe_diagnostics() { + let location = GeoLocation::new( + CountryCode::new("us").unwrap(), + Some(SubdivisionCode::new("ca").unwrap()), + ); + assert_eq!( + location, + GeoLocation::new( + CountryCode::new("US").unwrap(), + Some(SubdivisionCode::new("CA").unwrap()) + ) + ); + let outcome = GeoOutcome::Selected { + provider: GeoProviderId::cloudfront(), + status: GeoStatus::Located { + location, + source: GeoSource::CloudFront, + trust_basis: GeoTrustBasis::DeclaredProtectedAlb, + }, + }; + assert_eq!(outcome.clone(), outcome); + let debug = format!("{outcome:?}"); + assert!(debug.contains("US")); + assert!(debug.contains("CA")); + assert!(debug.contains("cloudfront")); + assert_ne!(GeoOutcome::NoSelection, outcome); + for reason in [ + GeoUnavailableReason::MissingMetadata, + GeoUnavailableReason::UntrustedIngress, + ] { + let selected = GeoOutcome::Selected { + provider: GeoProviderId::new("custom").unwrap(), + status: GeoStatus::Unavailable(reason), + }; + assert!(format!("{selected:?}").contains("custom")); + } + assert_eq!( + GeoError::InvalidAssertion.to_string(), + "invalid geographic assertion" + ); + assert_eq!( + GeoError::ProviderFailure.to_string(), + "geographic provider failure" + ); + assert_eq!( + GeoValueError::InvalidCountry.to_string(), + "invalid country code" + ); + } + + #[test] + fn geo_reader_input_redacts_addresses_and_reserves_maxmind() { + assert_eq!( + GeoProviderId::new("maxmind"), + Err(GeoValueError::ReservedProviderId) + ); + assert_eq!(GeoProviderId::maxmind().as_str(), "maxmind"); + for source in [ + GeoIpSource::DirectPeer, + GeoIpSource::TrustedForwarded, + GeoIpSource::TrustedXForwarded, + ] { + for address in ["192.0.2.7", "2001:db8::7"] { + let input = GeoInput::ReaderIp { + address: address.parse().unwrap(), + source, + }; + assert_eq!(input, input.clone()); + let debug = format!("{input:?}"); + assert!(debug.contains("[redacted]")); + assert!(!debug.contains(address)); + } + } + assert_ne!(GeoResolution::NoMatch, GeoResolution::MissingMetadata); + assert_ne!( + GeoUnavailableReason::NoMatch, + GeoUnavailableReason::MissingReaderIp + ); + } + + #[test] + fn portable_provider_is_synchronous_and_need_not_be_send_or_sync() { + struct LocalProvider(Rc>); + impl GeoProvider for LocalProvider { + fn resolve(&self, _input: &GeoInput) -> GeoResolution { + self.0.set(self.0.get() + 1); + GeoResolution::MissingMetadata + } + } + let calls = Rc::new(Cell::new(0)); + let provider: Box = Box::new(LocalProvider(Rc::clone(&calls))); + let input = GeoInput::Assertions { + country: GeoAssertion::absent(), + subdivision: GeoAssertion::absent(), + }; + assert_eq!(provider.resolve(&input), GeoResolution::MissingMetadata); + assert_eq!(calls.get(), 1); + } +} diff --git a/crates/edgezero-core/src/geo/country_codes.rs b/crates/edgezero-core/src/geo/country_codes.rs new file mode 100644 index 000000000..6fe08e4a5 --- /dev/null +++ b/crates/edgezero-core/src/geo/country_codes.rs @@ -0,0 +1,42 @@ +//! Assigned ISO 3166-1 alpha-2 country codes, captured 2026-10-22. +//! +//! Authority: ISO 3166 Maintenance Agency, +//! and . +//! Imported from datasets/country-list revision +//! ca393dfa16db21ce07be77902c1c129d63c86d9f, data.csv: +//! +//! Source CSV SHA-256: 3187a6850a6d2d8f03c099ac6eb2477d38fe7d2a85a3a5c1394eaf12e6e03d76. +//! +//! Only the 249 code elements are used, sorted by ASCII code. No country names, +//! software, or original arrangement are reproduced. The upstream list derives +//! from pycountry and Debian iso-codes; its maintainers dedicate their own work +//! under ODC-PDDL-1.0. See that revision's README.md and ATTRIBUTION.txt. +//! ISO permits free-of-charge use of the codes: +//! . +//! This table does not reproduce the ISO standard and is not endorsed by ISO. +//! +//! Updates require an explicit source revision, checksum and exclusion review. +//! The snapshot checksum covers sorted codes, each followed by one LF byte. + +#[cfg(test)] +pub(super) const SNAPSHOT_SHA256: &str = + "801ef127f0b3e6b4e971c239c9b8475caedb65c17573d84ca1b57eed72523a0e"; + +pub(super) const ASSIGNED: &[&str] = &[ + "AD", "AE", "AF", "AG", "AI", "AL", "AM", "AO", "AQ", "AR", "AS", "AT", "AU", "AW", "AX", "AZ", + "BA", "BB", "BD", "BE", "BF", "BG", "BH", "BI", "BJ", "BL", "BM", "BN", "BO", "BQ", "BR", "BS", + "BT", "BV", "BW", "BY", "BZ", "CA", "CC", "CD", "CF", "CG", "CH", "CI", "CK", "CL", "CM", "CN", + "CO", "CR", "CU", "CV", "CW", "CX", "CY", "CZ", "DE", "DJ", "DK", "DM", "DO", "DZ", "EC", "EE", + "EG", "EH", "ER", "ES", "ET", "FI", "FJ", "FK", "FM", "FO", "FR", "GA", "GB", "GD", "GE", "GF", + "GG", "GH", "GI", "GL", "GM", "GN", "GP", "GQ", "GR", "GS", "GT", "GU", "GW", "GY", "HK", "HM", + "HN", "HR", "HT", "HU", "ID", "IE", "IL", "IM", "IN", "IO", "IQ", "IR", "IS", "IT", "JE", "JM", + "JO", "JP", "KE", "KG", "KH", "KI", "KM", "KN", "KP", "KR", "KW", "KY", "KZ", "LA", "LB", "LC", + "LI", "LK", "LR", "LS", "LT", "LU", "LV", "LY", "MA", "MC", "MD", "ME", "MF", "MG", "MH", "MK", + "ML", "MM", "MN", "MO", "MP", "MQ", "MR", "MS", "MT", "MU", "MV", "MW", "MX", "MY", "MZ", "NA", + "NC", "NE", "NF", "NG", "NI", "NL", "NO", "NP", "NR", "NU", "NZ", "OM", "PA", "PE", "PF", "PG", + "PH", "PK", "PL", "PM", "PN", "PR", "PS", "PT", "PW", "PY", "QA", "RE", "RO", "RS", "RU", "RW", + "SA", "SB", "SC", "SD", "SE", "SG", "SH", "SI", "SJ", "SK", "SL", "SM", "SN", "SO", "SR", "SS", + "ST", "SV", "SX", "SY", "SZ", "TC", "TD", "TF", "TG", "TH", "TJ", "TK", "TL", "TM", "TN", "TO", + "TR", "TT", "TV", "TW", "TZ", "UA", "UG", "UM", "US", "UY", "UZ", "VA", "VC", "VE", "VG", "VI", + "VN", "VU", "WF", "WS", "YE", "YT", "ZA", "ZM", "ZW", +]; diff --git a/crates/edgezero-core/src/ingress.rs b/crates/edgezero-core/src/ingress.rs index 855825966..0cc6eee15 100644 --- a/crates/edgezero-core/src/ingress.rs +++ b/crates/edgezero-core/src/ingress.rs @@ -1,5 +1,6 @@ use std::any::Any; use std::fmt; +use std::mem; use std::sync::Arc; use std::time::Duration; @@ -8,9 +9,10 @@ use bytes::Bytes; use crate::body::Body; use crate::config_store::ConfigExtractionLimits; use crate::error::EdgeError; +use crate::geo::GeoOutcome; use crate::http::{ - Extensions, HeaderMap, HeaderValue, Method, Request, RequestParts, Response, StatusCode, Uri, - Version, + Authority, Extensions, HeaderMap, HeaderValue, Method, Request, RequestParts, Response, + StatusCode, Uri, Version, header::{CONTENT_LENGTH, CONTENT_TYPE, TRANSFER_ENCODING}, }; use crate::response_egress::{ResponseEgressCompletion, ResponseEgressEnvelope}; @@ -249,11 +251,13 @@ pub enum IngressDispatchOutcome { /// Proof that the application selected one request disposition with a finite read deadline. pub struct AdmittedIngress { + category_only_json_errors: bool, config_extraction_limits: ConfigExtractionLimits, dispatch_disposition: IngressDispatchDisposition, grant: IngressGrant, monotonic_clock: MonotonicClock, read_deadline: Deadline, + request_geo: GeoOutcome, request_start: MonotonicInstant, } @@ -321,10 +325,93 @@ impl AdmittedIngress { self.request_start } + pub(crate) fn take_request_context(&mut self) -> (GeoOutcome, bool) { + ( + mem::replace(&mut self.request_geo, GeoOutcome::NoSelection), + self.category_only_json_errors, + ) + } + pub(crate) fn with_config_extraction_limits(mut self, limits: ConfigExtractionLimits) -> Self { self.config_extraction_limits = limits; self } + + pub(crate) fn with_request_context( + mut self, + geo: GeoOutcome, + category_only_json_errors: bool, + ) -> Self { + self.request_geo = geo; + self.category_only_json_errors = category_only_json_errors; + self + } +} + +/// Provenance of one independently checked effective proxy field. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[non_exhaustive] +pub enum CheckedHostSource { + Direct, + TrustedForwarded, + TrustedXForwarded, + Unavailable, +} + +/// Adapter-checked authority for portable host consumers. +/// This witness does not itself authorize forwarding assertions. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct CheckedEffectiveHost { + authority: Option, + source: CheckedHostSource, +} + +impl CheckedEffectiveHost { + #[must_use] + #[inline] + pub fn authority(&self) -> Option<&str> { + self.authority.as_ref().map(Authority::as_str) + } + + /// Constructs a witness only after adapter-owned trust and syntax checks. + #[doc(hidden)] + #[must_use] + #[inline] + pub const fn from_ingress(authority: Option, source: CheckedHostSource) -> Self { + Self { authority, source } + } + + #[must_use] + #[inline] + pub const fn source(&self) -> CheckedHostSource { + self.source + } +} + +/// Received normalized-head sizes; never proof of raw-wire accounting. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct NormalizedIngressHeadSummary { + header_bytes: u64, + header_count: u64, + target_bytes: u64, +} + +impl NormalizedIngressHeadSummary { + #[must_use] + #[inline] + pub const fn header_bytes(self) -> u64 { + self.header_bytes + } + #[must_use] + #[inline] + pub const fn header_count(self) -> u64 { + self.header_count + } + #[must_use] + #[inline] + pub const fn target_bytes(self) -> u64 { + self.target_bytes + } } /// Whether request-head accounting was enforced before normalized request construction. @@ -520,6 +607,7 @@ impl IngressHeadParts { #[inline] pub fn validate_normalized(&self, limits: IngressHeadLimits) -> Result<(), EdgeError> { validate_normalized_head(&self.target, self.version, &self.headers, limits) + .map(|_summary| ()) } #[must_use] @@ -803,6 +891,8 @@ pub(crate) fn apply_admission_policy( Ok(IngressAdmissionOutcome::Admitted { completion, ingress: AdmittedIngress { + request_geo: GeoOutcome::NoSelection, + category_only_json_errors: false, config_extraction_limits: ConfigExtractionLimits::default(), dispatch_disposition, grant, @@ -834,6 +924,19 @@ pub fn validate_normalized_ingress_parts( parts: &RequestParts, limits: IngressHeadLimits, ) -> Result<(), EdgeError> { + validate_normalized_ingress_parts_with_summary(parts, limits).map(|_summary| ()) +} + +/// Validates and accounts for the received head in the same bounded pass. +/// Call before stripping assertions; returned sizes remain normalized, not raw-wire. +/// +/// # Errors +/// Returns the same head/framing errors as [`validate_normalized_ingress_parts`]. +#[inline] +pub fn validate_normalized_ingress_parts_with_summary( + parts: &RequestParts, + limits: IngressHeadLimits, +) -> Result { validate_normalized_head(&parts.uri, parts.version, &parts.headers, limits) } @@ -842,7 +945,7 @@ fn validate_normalized_head( version: Version, headers: &HeaderMap, limits: IngressHeadLimits, -) -> Result<(), EdgeError> { +) -> Result { let target_bytes = u64::try_from(target.to_string().len()).map_err(|_length_error| { EdgeError::uri_too_long("normalized request target is too large") })?; @@ -886,7 +989,12 @@ fn validate_normalized_head( } } - validate_normalized_framing(version, headers) + validate_normalized_framing(version, headers)?; + Ok(NormalizedIngressHeadSummary { + header_bytes, + header_count, + target_bytes, + }) } fn validate_normalized_framing(version: Version, headers: &HeaderMap) -> Result<(), EdgeError> { diff --git a/crates/edgezero-core/src/lib.rs b/crates/edgezero-core/src/lib.rs index cc1fa9e83..e9662b48e 100644 --- a/crates/edgezero-core/src/lib.rs +++ b/crates/edgezero-core/src/lib.rs @@ -25,6 +25,7 @@ pub mod context; pub mod env_config; pub mod error; pub mod extractor; +pub mod geo; pub mod handler; pub mod http; pub mod ingress; @@ -36,6 +37,7 @@ pub mod middleware; pub mod outbound; pub mod params; pub mod platform; +pub mod probe; pub mod responder; pub mod response; pub mod response_egress; diff --git a/crates/edgezero-core/src/probe.rs b/crates/edgezero-core/src/probe.rs new file mode 100644 index 000000000..7f8ec783b --- /dev/null +++ b/crates/edgezero-core/src/probe.rs @@ -0,0 +1,163 @@ +//! App-owned probe routes with read-only live lifecycle observation. +//! +//! Adapters may publish a reader in request extensions. Absence does not imply +//! health or lifecycle support. Applications choose paths and middleware; these +//! helpers register no routes. Do not override framework probe/store extensions +//! through application state or middleware. Native dispatch uses a private gate. + +use std::fmt; +use std::sync::Arc; + +use crate::action; +use crate::body::Body; +use crate::context::RequestContext; +use crate::error::EdgeError; +use crate::http::{Response, StatusCode, response_builder}; + +/// Observed native runner phase. Once draining starts, a run never becomes ready again. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum LifecyclePhase { + Draining, + Ready, + Starting, + Stopped, +} + +/// Cloneable read-only observation, sampled when a handler executes. +/// +/// Readiness means the selected entrypoint's startup contract succeeded, not that +/// undisclosed application requirements or ongoing dependencies were checked. +#[derive(Clone)] +pub struct LifecycleReader { + read: Arc Option + Send + Sync>, +} + +impl LifecycleReader { + #[inline] + #[must_use] + pub fn new(read: Read) -> Self + where + Read: Fn() -> Option + Send + Sync + 'static, + { + Self { + read: Arc::new(read), + } + } + + #[inline] + #[must_use] + pub fn phase(&self) -> Option { + (self.read)() + } +} + +impl fmt::Debug for LifecycleReader { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("LifecycleReader").finish_non_exhaustive() + } +} + +/// Returns 200 while starting, ready or draining, otherwise 503. +/// +/// This table applies only if the handler executes. Before acceptance and after +/// listener withdrawal, supervisors must also handle refusal, reset or EOF. +/// +/// # Errors +/// Returns an internal error only if the static response cannot be constructed. +#[action] +pub async fn liveness(ctx: RequestContext) -> Result { + match phase(&ctx) { + Some(LifecyclePhase::Starting | LifecyclePhase::Ready | LifecyclePhase::Draining) => { + response(StatusCode::OK, "live") + } + Some(LifecyclePhase::Stopped) => response(StatusCode::SERVICE_UNAVAILABLE, "stopped"), + None => response(StatusCode::SERVICE_UNAVAILABLE, "probe state unavailable"), + } +} + +fn phase(ctx: &RequestContext) -> Option { + ctx.extensions() + .get::() + .and_then(LifecycleReader::phase) +} + +/// Returns 200 only in Ready, otherwise 503. Does not poll dependencies. +/// +/// # Errors +/// Returns an internal error only if the static response cannot be constructed. +#[action] +pub async fn readiness(ctx: RequestContext) -> Result { + match phase(&ctx) { + Some(LifecyclePhase::Ready) => response(StatusCode::OK, "ready"), + Some(_) => response(StatusCode::SERVICE_UNAVAILABLE, "not ready"), + None => response(StatusCode::SERVICE_UNAVAILABLE, "probe state unavailable"), + } +} + +fn response(status: StatusCode, text: &'static str) -> Result { + response_builder() + .status(status) + .header("content-type", "text/plain; charset=utf-8") + .header("cache-control", "no-store") + .body(Body::from(text)) + .map_err(EdgeError::internal) +} + +#[cfg(test)] +mod tests { + use std::sync::Mutex; + + use futures::executor::block_on; + + use super::*; + use crate::http::request_builder; + use crate::router::RouterService; + + #[test] + fn ordinary_routes_observe_live_phase_and_missing_state() { + #[action] + async fn health() -> Result<&'static str, EdgeError> { + Ok("app health") + } + let current = Arc::new(Mutex::new(None)); + let observed = Arc::clone(¤t); + let reader = LifecycleReader::new(move || *observed.lock().expect("phase")); + let router = RouterService::builder() + .get("/app-ready", readiness) + .get("/app-live", liveness) + .get("/health", health) + .build(); + for (value, ready, live) in [ + (None, 503, 503), + (Some(LifecyclePhase::Starting), 503, 200), + (Some(LifecyclePhase::Ready), 200, 200), + (Some(LifecyclePhase::Draining), 503, 200), + (Some(LifecyclePhase::Stopped), 503, 503), + ] { + *current.lock().expect("phase") = value; + for (path, expected) in [("/app-ready", ready), ("/app-live", live)] { + let mut request = request_builder() + .uri(path) + .body(Body::empty()) + .expect("request"); + request.extensions_mut().insert(reader.clone()); + let result = block_on(router.oneshot(request)).expect("probe"); + assert_eq!(result.status().as_u16(), expected); + assert_eq!(result.headers()["cache-control"], "no-store"); + assert_eq!( + result.headers()["content-type"], + "text/plain; charset=utf-8" + ); + } + } + for (path, expected) in [("/app-ready", 503), ("/health", 200), ("/ready", 404)] { + let request = request_builder() + .uri(path) + .body(Body::empty()) + .expect("request"); + let result = block_on(router.oneshot(request)).expect("response"); + assert_eq!(result.status().as_u16(), expected); + } + } +} diff --git a/crates/edgezero-core/src/response_egress.rs b/crates/edgezero-core/src/response_egress.rs index f7857b722..5bbf8e2ae 100644 --- a/crates/edgezero-core/src/response_egress.rs +++ b/crates/edgezero-core/src/response_egress.rs @@ -300,7 +300,68 @@ pub struct ResponseEgressReport { pub route: Option, } -type ResponseEgressCompletionCallback = Box; +type ResponseEgressCompletionCallback = + Box; + +/// Bounded reason retained independently of an error renderer's response. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[non_exhaustive] +pub enum ResponseEgressFailureKind { + FallbackBodyExceeded, + FallbackReadTimedOut, + PropagatedError, +} + +/// Framework-owned failure classification with no diagnostic source or input text. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct ResponseEgressFailureClassification { + error_kind: Option<&'static str>, + framework_route_error: bool, + kind: ResponseEgressFailureKind, +} + +impl ResponseEgressFailureClassification { + pub(crate) const fn error(kind: &'static str) -> Self { + Self { + kind: ResponseEgressFailureKind::PropagatedError, + error_kind: Some(kind), + framework_route_error: false, + } + } + + #[must_use] + #[inline] + pub const fn error_kind(self) -> Option<&'static str> { + self.error_kind + } + + pub(crate) const fn fallback(kind: ResponseEgressFailureKind) -> Self { + Self { + kind, + error_kind: None, + framework_route_error: false, + } + } + + #[must_use] + #[inline] + pub const fn is_framework_route_error(self) -> bool { + self.framework_route_error + } + + #[must_use] + #[inline] + pub const fn kind(self) -> ResponseEgressFailureKind { + self.kind + } + + pub(crate) const fn routing(kind: &'static str) -> Self { + Self { + framework_route_error: true, + ..Self::error(kind) + } + } +} /// Non-clone owner of one response-scoped terminal callback. /// @@ -425,13 +486,18 @@ struct ResponseEgressResourceState { } impl ResponseEgressCompletion { + #[cfg(test)] fn complete(&mut self, report: &ResponseEgressReport) { + self.complete_at(report, report.request_start); + } + + fn complete_at(&mut self, report: &ResponseEgressReport, observed_at: MonotonicInstant) { let Some(callback) = self.callback.take() else { return; }; - let result = catch_unwind(AssertUnwindSafe(|| callback(report))); + let result = catch_unwind(AssertUnwindSafe(|| callback(report, observed_at))); if result.is_err() { - log::error!("response-egress completion panicked after terminal transition"); + log_callback_fault("response-egress completion panicked after terminal transition"); } } @@ -455,9 +521,9 @@ impl ResponseEgressCompletion { #[must_use] #[inline] pub fn join(mut self, mut other: Self) -> Self { - Self::new(move |report| { - self.complete(report); - other.complete(report); + Self::new_with_terminal_time(move |report, observed_at| { + self.complete_at(report, observed_at); + other.complete_at(report, observed_at); }) } @@ -489,6 +555,18 @@ impl ResponseEgressCompletion { pub fn new(complete: Complete) -> Self where Complete: FnOnce(&ResponseEgressReport) + Send + 'static, + { + Self::new_with_terminal_time(move |report, _observed_at| complete(report)) + } + + /// Retains a callback receiving the existing terminal transition's instant. + /// The instant is captured before any application callback or logger runs. + #[doc(hidden)] + #[must_use] + #[inline] + pub fn new_with_terminal_time(complete: Complete) -> Self + where + Complete: FnOnce(&ResponseEgressReport, MonotonicInstant) + Send + 'static, { Self { callback: Some(Box::new(complete)), @@ -527,6 +605,7 @@ pub struct ResponseEgressObserverHandle { pub struct ResponseEgressEnvelope { clock: MonotonicClock, completion: ResponseEgressCompletion, + failure_classification: Option, observer: ResponseEgressObserverHandle, policy: ResponseEgressPolicyCallback, request_method: Method, @@ -616,7 +695,7 @@ impl ResponseEgressEnvelope { let Ok(selected_policy) = catch_unwind(AssertUnwindSafe(|| (self.policy)(&head, egress_started_at))) else { - log::error!("response-egress policy panicked before conversion"); + log_callback_fault("response-egress policy panicked before conversion"); return Err(Box::new(ResponseEgressBeginFailure { attempt, cause: ResponseEgressOutcome::ConversionError, @@ -644,7 +723,7 @@ impl ResponseEgressEnvelope { )), }; let Ok(prepared) = prepare_response_egress(&self.request_method, self.response) else { - log::error!("response-egress framing validation failed before commit"); + log_callback_fault("response-egress framing validation failed before commit"); return Err(Box::new(ResponseEgressBeginFailure { attempt, cause: ResponseEgressOutcome::ConversionError, @@ -671,6 +750,13 @@ impl ResponseEgressEnvelope { ) } + /// Returns the framework-owned category captured before rendering. + #[must_use] + #[inline] + pub const fn failure_classification(&self) -> Option { + self.failure_classification + } + pub(crate) fn new( response: Response, metadata: ResponseEgressRequestMetadata, @@ -680,6 +766,7 @@ impl ResponseEgressEnvelope { clock: MonotonicClock, ) -> Self { Self { + failure_classification: None, clock, completion, observer, @@ -705,13 +792,30 @@ impl ResponseEgressEnvelope { pub fn response_mut(&mut self) -> &mut Response { &mut self.response } + + /// Composes adapter diagnostics without replacing application ownership. + #[doc(hidden)] + #[must_use] + #[inline] + pub fn with_completion(mut self, completion: ResponseEgressCompletion) -> Self { + self.completion = self.completion.join(completion); + self + } + + pub(crate) fn with_failure( + mut self, + failure: Option, + ) -> Self { + self.failure_classification = failure; + self + } } impl ResponseEgressObserverHandle { fn complete(&self, report: &ResponseEgressReport) { let result = catch_unwind(AssertUnwindSafe(|| self.observer.complete(report))); if result.is_err() { - log::error!("response-egress observer panicked after terminal transition"); + log_callback_fault("response-egress observer panicked after terminal transition"); } } @@ -927,7 +1031,7 @@ impl ResponseEgressAttempt { if let Some(elapsed) = observed_at.checked_duration_since(self.egress_started_at) { elapsed } else { - log::error!("response-egress monotonic clock moved backwards"); + log_callback_fault("response-egress monotonic clock moved backwards"); outcome = ResponseEgressOutcome::Unspecified; Duration::ZERO }; @@ -942,7 +1046,7 @@ impl ResponseEgressAttempt { }; self.state = AttemptState::Terminal(report); if let AttemptState::Terminal(terminal_report) = &self.state { - self.completion.complete(terminal_report); + self.completion.complete_at(terminal_report, observed_at); self.observer.complete(terminal_report); true } else { @@ -987,6 +1091,12 @@ pub fn default_response_egress_policy( } } +fn log_callback_fault(category: &'static str) { + // A logger panic must not escape the owner's independent unwind boundary. + // Never retry through a logger that has itself failed. + let _result = catch_unwind(AssertUnwindSafe(|| log::error!("{category}"))); +} + #[cfg(test)] mod tests { use std::cell::Cell; @@ -1787,6 +1897,158 @@ mod tests { assert_eq!(reports[0].outcome, ResponseEgressOutcome::TransportError); } + #[test] + #[expect( + clippy::too_many_lines, + reason = "one isolated facade fixture exercises all three owner fault boundaries" + )] + fn core_fault_reporting_with_panicking_facade_is_isolated() { + use log::{Level, LevelFilter, Log, Metadata, Record, set_logger, set_max_level}; + use std::env::{current_exe, var_os}; + use std::process::Command; + + struct PanickingLogger; + static LOGGER: PanickingLogger = PanickingLogger; + static CALLS: AtomicUsize = AtomicUsize::new(0); + impl Log for PanickingLogger { + fn enabled(&self, metadata: &Metadata<'_>) -> bool { + metadata.level() == Level::Error + } + fn flush(&self) {} + fn log(&self, record: &Record<'_>) { + if self.enabled(record.metadata()) { + CALLS.fetch_add(1, Ordering::SeqCst); + panic!("fixture logger fault"); + } + } + } + const CHILD: &str = "EDGEZERO_TEST_CORE_FAULT_LOGGER"; + if var_os(CHILD).is_none() { + let output = Command::new(current_exe().expect("test executable")) + .args(["--exact", "response_egress::tests::core_fault_reporting_with_panicking_facade_is_isolated", "--nocapture"]) + .env(CHILD, "1").output().expect("isolated child"); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + return; + } + set_logger(&LOGGER).expect("single child logger"); + set_max_level(LevelFilter::Error); + let started = MonotonicInstant::now(); + let terminal = started + .checked_add(Duration::from_millis(42)) + .expect("instant"); + let after_callback = terminal + .checked_add(Duration::from_secs(3)) + .expect("instant"); + let current = Arc::new(Mutex::new(terminal)); + let clock_current = Arc::clone(¤t); + let clock = MonotonicClock::new(move || *clock_current.lock().expect("clock")); + let events = Arc::new(Mutex::new(Vec::new())); + let app_events = Arc::clone(&events); + let native_events = Arc::clone(&events); + let observed = Arc::new(Mutex::new(None)); + let native_observed = Arc::clone(&observed); + let callback_current = Arc::clone(¤t); + let completion = ResponseEgressCompletion::new(move |_| { + app_events.lock().expect("events").push("app"); + *callback_current.lock().expect("clock") = after_callback; + panic!("application completion fault"); + }) + .join(ResponseEgressCompletion::new_with_terminal_time( + move |_, at| { + native_events.lock().expect("events").push("native"); + *native_observed.lock().expect("observed") = Some(at); + }, + )); + let headers = HeaderMap::new(); + let head = ResponseEgressHead::new( + StatusCode::OK, + Version::HTTP_11, + &headers, + None, + started, + None, + ); + let mut attempt = ResponseEgressAttempt::new( + &head, + started, + completion, + ResponseEgressObserverHandle::new(OrderedObserver(Arc::clone(&events))), + clock.clone(), + ); + assert!(attempt.terminate(ResponseEgressOutcome::Completed, terminal)); + assert!(!attempt.terminate(ResponseEgressOutcome::SourceError, after_callback)); + assert_eq!( + *events.lock().expect("events"), + vec!["app", "native", "observer"] + ); + assert_eq!(*observed.lock().expect("observed"), Some(terminal)); + assert_eq!(clock.now(), after_callback); + assert_eq!(CALLS.load(Ordering::SeqCst), 1); + + let mut observer_attempt = ResponseEgressAttempt::new( + &head, + started, + ResponseEgressCompletion::empty(), + ResponseEgressObserverHandle::new(PanickingObserver), + clock.clone(), + ); + assert!(observer_attempt.terminate(ResponseEgressOutcome::Completed, terminal)); + assert!(!observer_attempt.terminate(ResponseEgressOutcome::TransportError, after_callback)); + assert_eq!(CALLS.load(Ordering::SeqCst), 2); + + let policy_calls = Arc::new(Mutex::new(Vec::new())); + let callback_calls = Arc::clone(&policy_calls); + let envelope = ResponseEgressEnvelope::new( + Response::new(Body::empty()), + ResponseEgressRequestMetadata::new(Method::GET, started, None), + ResponseEgressCompletion::new_with_terminal_time(move |_, at| { + callback_calls.lock().expect("calls").push(at); + }), + Arc::new(|_, _| panic!("application policy fault")), + ResponseEgressObserverHandle::default(), + clock, + ); + let failure = envelope.begin().err().expect("bounded policy failure"); + assert!(policy_calls.lock().expect("calls").is_empty()); + let (mut policy_attempt, _clock, cause) = failure.into_parts(); + assert_eq!(cause, ResponseEgressOutcome::ConversionError); + assert!(policy_attempt.terminate(cause, after_callback)); + assert_eq!(*policy_calls.lock().expect("calls"), vec![after_callback]); + assert_eq!(CALLS.load(Ordering::SeqCst), 3); + } + + #[test] + fn timed_nested_completion_preserves_terminal_instant_and_order() { + let events = Arc::new(Mutex::new(Vec::<(u8, MonotonicInstant)>::new())); + let started = MonotonicInstant::now(); + let terminal = started + .checked_add(Duration::from_millis(42)) + .expect("instant"); + let first = Arc::clone(&events); + let second = Arc::clone(&events); + let third = Arc::clone(&events); + let mut completion = ResponseEgressCompletion::new(move |_| { + first.lock().expect("events").push((1, terminal)); + }) + .join(ResponseEgressCompletion::new_with_terminal_time( + move |_, at| second.lock().expect("events").push((2, at)), + )) + .join(ResponseEgressCompletion::new_with_terminal_time( + move |_, at| third.lock().expect("events").push((3, at)), + )); + let report = completed_report(started); + completion.complete_at(&report, terminal); + completion.complete_at(&report, started); + assert_eq!( + *events.lock().expect("events"), + vec![(1, terminal), (2, terminal), (3, terminal)] + ); + } + #[test] fn zero_byte_completion_has_zero_accounting() { let completed_observer = RecordingObserver::default(); diff --git a/crates/edgezero-core/src/router.rs b/crates/edgezero-core/src/router.rs index 08d92e024..748a9edb4 100644 --- a/crates/edgezero-core/src/router.rs +++ b/crates/edgezero-core/src/router.rs @@ -14,6 +14,30 @@ use crate::introspection::{ManifestJson, RouteTable}; use crate::middleware::{BoxMiddleware, Middleware, Next}; use crate::params::PathParams; use crate::response::IntoResponse as _; +use crate::response_egress::{ResponseEgressFailureClassification, ResponseEgressFailureKind}; + +pub(crate) enum RouteErrorKind { + MethodNotAllowed, + NotFound, +} + +pub(crate) struct DispatchOutcome { + failure: Option, + response: Result, + route_error: Option, +} + +impl DispatchOutcome { + pub(crate) fn into_parts( + self, + ) -> ( + Result, + Option, + Option, + ) { + (self.response, self.route_error, self.failure) + } +} struct RouteEntry { handler: BoxHandler, @@ -345,6 +369,7 @@ impl RouterBuilder { } } +#[derive(Clone)] struct RouterInner { manifest_json: Option>, middlewares: Vec, @@ -487,45 +512,82 @@ impl RouterService { request: Request, ingress: AdmittedIngress, ) -> Result { - if !Arc::ptr_eq(&self.inner, &resolved.owner) { - return Err(EdgeError::internal(anyhow::anyhow!( - "resolved dispatch token belongs to another router" - ))); - } - if request.method() != resolved.method || request.uri().path() != resolved.path { - return Err(EdgeError::internal(anyhow::anyhow!( - "request method or path changed after ingress route resolution" - ))); - } + self.dispatch_resolved_outcome(resolved, request, ingress) + .await + .response + } - match resolved.target { - ResolvedTarget::Found(entry, params) => { - if ingress.is_fallback() { - return Err(EdgeError::internal(anyhow::anyhow!( - "fallback body policy cannot dispatch a matched route" - ))); - } - self.inner - .dispatch_found(request, &entry, params, Some(ingress)) - .await + pub(crate) async fn dispatch_resolved_outcome( + &self, + resolved: ResolvedDispatch, + request: Request, + ingress: AdmittedIngress, + ) -> DispatchOutcome { + let mut route_error = None; + let mut failure = None; + let response = async { + if !Arc::ptr_eq(&self.inner, &resolved.owner) { + return Err(EdgeError::internal(anyhow::anyhow!( + "resolved dispatch token belongs to another router" + ))); } - ResolvedTarget::MethodNotAllowed(allowed) => { - if let Some(response) = fallback_terminal_response(request, ingress).await? { - return Ok(response); - } - let methods = allowed - .iter() - .map(|metadata| metadata.method().clone()) - .collect::>(); - Err(EdgeError::method_not_allowed(&resolved.method, &methods)) + if request.method() != resolved.method || request.uri().path() != resolved.path { + return Err(EdgeError::internal(anyhow::anyhow!( + "request method or path changed after ingress route resolution" + ))); } - ResolvedTarget::NotFound => { - if let Some(response) = fallback_terminal_response(request, ingress).await? { - return Ok(response); + match resolved.target { + ResolvedTarget::Found(entry, params) => { + if ingress.is_fallback() { + return Err(EdgeError::internal(anyhow::anyhow!( + "fallback body policy cannot dispatch a matched route" + ))); + } + self.inner + .dispatch_found(request, &entry, params, Some(ingress)) + .await + } + ResolvedTarget::MethodNotAllowed(allowed) => { + if let Some((response, category)) = + fallback_terminal_response(request, ingress).await? + { + failure = Some(category); + return Ok(response); + } + let methods = allowed + .iter() + .map(|metadata| metadata.method().clone()) + .collect::>(); + route_error = Some(RouteErrorKind::MethodNotAllowed); + Err(EdgeError::method_not_allowed(&resolved.method, &methods)) + } + ResolvedTarget::NotFound => { + if let Some((response, category)) = + fallback_terminal_response(request, ingress).await? + { + failure = Some(category); + return Ok(response); + } + route_error = Some(RouteErrorKind::NotFound); + Err(EdgeError::not_found(resolved.path)) } - Err(EdgeError::not_found(resolved.path)) } } + .await; + DispatchOutcome { + failure, + response, + route_error, + } + } + + pub(crate) fn insert_state(&mut self, value: T) + where + T: Clone + Send + Sync + 'static, + { + Arc::make_mut(&mut self.inner) + .state_extensions + .insert(value); } fn new( @@ -595,7 +657,7 @@ impl RouterService { async fn fallback_terminal_response( request: Request, ingress: AdmittedIngress, -) -> Result, EdgeError> { +) -> Result, EdgeError> { let Some(fallback) = ingress.into_fallback() else { return Ok(None); }; @@ -612,8 +674,18 @@ async fn fallback_terminal_response( match outcome? { FallbackDrainOutcome::Complete => Ok(None), - FallbackDrainOutcome::Exceeded => Ok(Some(on_exceeded.into_response())), - FallbackDrainOutcome::TimedOut => Ok(Some(on_timeout.into_response())), + FallbackDrainOutcome::Exceeded => Ok(Some(( + on_exceeded.into_response(), + ResponseEgressFailureClassification::fallback( + ResponseEgressFailureKind::FallbackBodyExceeded, + ), + ))), + FallbackDrainOutcome::TimedOut => Ok(Some(( + on_timeout.into_response(), + ResponseEgressFailureClassification::fallback( + ResponseEgressFailureKind::FallbackReadTimedOut, + ), + ))), } } diff --git a/crates/edgezero-core/tests/fixtures/production_handlers.rs b/crates/edgezero-core/tests/fixtures/production_handlers.rs new file mode 100644 index 000000000..5a3a1e323 --- /dev/null +++ b/crates/edgezero-core/tests/fixtures/production_handlers.rs @@ -0,0 +1,32 @@ +//! Shared native/WASM hosting fixture. No native runtime dependency belongs here. + +use std::sync::Arc; + +use edgezero_core::EdgeError; +use edgezero_core::action; +use edgezero_core::extractor::{AppConfig, State}; +use serde::Deserialize; +use validator::Validate; + +#[derive(Deserialize, Validate, edgezero_core::AppConfig)] +pub struct FixtureConfig { + #[validate(length(min = 4_u64))] + pub greeting: String, + #[secret] + #[validate(length(min = 8_u64))] + pub token: String, +} + +pub struct HostState { + pub greeting: String, +} + +#[action] +pub async fn state(State(state): State>) -> Result { + Ok(state.greeting.clone()) +} + +#[action] +pub async fn typed(AppConfig(config): AppConfig) -> Result { + Ok(config.greeting) +} diff --git a/crates/edgezero-core/tests/production_handlers.rs b/crates/edgezero-core/tests/production_handlers.rs new file mode 100644 index 000000000..afa95fe74 --- /dev/null +++ b/crates/edgezero-core/tests/production_handlers.rs @@ -0,0 +1,41 @@ +//! Compile this same handler fixture for native and every supported WASM target. + +#[path = "fixtures/production_handlers.rs"] +mod fixture; + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use edgezero_core::Body; + use edgezero_core::http::{StatusCode, request_builder}; + use edgezero_core::probe; + use edgezero_core::router::RouterService; + use futures::executor::block_on; + + use super::fixture; + + #[test] + fn shared_state_config_and_probe_actions_use_portable_routes() { + let router = RouterService::builder() + .get("/state", fixture::state) + .get("/typed", fixture::typed) + .get("/ready", probe::readiness) + .with_state(Arc::new(fixture::HostState { + greeting: "retained".to_owned(), + })) + .build(); + for (path, status) in [ + ("/state", StatusCode::OK), + ("/ready", StatusCode::SERVICE_UNAVAILABLE), + ("/typed", StatusCode::INTERNAL_SERVER_ERROR), + ] { + let request = request_builder() + .uri(path) + .body(Body::empty()) + .expect("request"); + let response = block_on(router.oneshot(request)).expect("response"); + assert_eq!(response.status(), status); + } + } +} diff --git a/crates/edgezero-store-aws/Cargo.toml b/crates/edgezero-store-aws/Cargo.toml new file mode 100644 index 000000000..a73ba4180 --- /dev/null +++ b/crates/edgezero-store-aws/Cargo.toml @@ -0,0 +1,43 @@ +[package] +name = "edgezero-store-aws" +description = "Optional native AWS configuration and secret snapshots for EdgeZero" +authors = { workspace = true } +edition = { workspace = true } +license = { workspace = true } +publish = { workspace = true } +version = { workspace = true } + +[lints] +workspace = true + +[features] +default = [] +snapshot-stores = ["dep:edgezero-core", "dep:bytes", "dep:async-trait", "dep:tokio"] +appconfig-agent = ["snapshot-stores", "dep:reqwest", "dep:futures-util"] +secrets-manager = ["snapshot-stores", "dep:aws-config", "dep:aws-sdk-secretsmanager"] +dynamodb-kv = ["snapshot-stores", "dep:aws-config", "dep:aws-sdk-dynamodb", "dep:serde_json"] + +[dependencies] +percent-encoding = { workspace = true } +serde = { workspace = true } +thiserror = { workspace = true } +url = { workspace = true } + +[target.'cfg(not(target_arch = "wasm32"))'.dependencies] +async-trait = { workspace = true, optional = true } +aws-config = { workspace = true, optional = true } +aws-sdk-secretsmanager = { workspace = true, optional = true } +aws-sdk-dynamodb = { workspace = true, optional = true } +serde_json = { workspace = true, optional = true } +bytes = { workspace = true, optional = true } +edgezero-core = { workspace = true, optional = true } +futures-util = { workspace = true, optional = true } +reqwest = { workspace = true, optional = true, features = ["stream"] } +tokio = { workspace = true, optional = true } + +[dev-dependencies] +base64 = { workspace = true } +futures = { workspace = true } +serde_json = { workspace = true } +tempfile = { workspace = true } +toml = { workspace = true } diff --git a/crates/edgezero-store-aws/src/appconfig_tests.rs b/crates/edgezero-store-aws/src/appconfig_tests.rs new file mode 100644 index 000000000..22452b99e --- /dev/null +++ b/crates/edgezero-store-aws/src/appconfig_tests.rs @@ -0,0 +1,166 @@ +//! Literal-loopback HTTP tests for startup-only `AppConfig` snapshots. + +use std::collections::BTreeMap; +use std::time::Duration; + +use edgezero_core::time::MonotonicClock; +use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; +use tokio::net::TcpListener; +use tokio::task::JoinHandle; +use tokio::time::timeout; + +use crate::settings::{ + AgentAccessToken, AgentDocument, AppConfigAgentSettings, AwsPreparationLimits, +}; +use crate::{AwsPreparation, PreparationError}; + +async fn fixture(headers: &str, body: Vec) -> (AppConfigAgentSettings, JoinHandle) { + let listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("literal loopback"); + let endpoint = format!("http://{}", listener.local_addr().expect("address")); + let response_headers = headers.to_owned(); + let server = tokio::spawn(async move { + let (mut socket, _) = timeout(Duration::from_secs(2), listener.accept()) + .await + .expect("finite accept") + .expect("accept"); + let mut request = Vec::new(); + while !request.windows(4).any(|window| window == b"\r\n\r\n") { + let mut chunk = [0_u8; 1024]; + let count = socket.read(&mut chunk).await.expect("request"); + assert_ne!(count, 0, "request did not close before its headers"); + request.extend_from_slice(chunk.get(..count).expect("read buffer bound")); + } + socket + .write_all( + format!( + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n{response_headers}\r\n", + body.len() + ) + .as_bytes(), + ) + .await + .expect("response head"); + let _written = socket.write_all(&body).await; + String::from_utf8(request).expect("HTTP request") + }); + let settings = AppConfigAgentSettings { + endpoint, + access_token_env: None, + default_key: "main".to_owned(), + documents: BTreeMap::from([( + "main".to_owned(), + AgentDocument { + application: "app".to_owned(), + environment: "test".to_owned(), + profile: "freeform".to_owned(), + }, + )]), + }; + (settings, server) +} + +#[tokio::test] +async fn raw_empty_and_quoted_utf8_are_preserved_and_successes_are_deduplicated() { + for body in ["", "raw free-form\n", "\"quoted\""] { + let (mut settings, server) = fixture("", body.as_bytes().to_vec()).await; + settings + .documents + .insert("alias".to_owned(), settings.documents["main"].clone()); + let mut session = + AwsPreparation::new(MonotonicClock::default(), AwsPreparationLimits::default()) + .expect("session"); + let handle = session + .prepare_appconfig(&settings, None) + .await + .expect("snapshot"); + let _request = server.await.expect("server"); + for key in ["main", "alias"] { + let value = handle + .get(key) + .await + .expect("lookup") + .expect("mapped value"); + assert_eq!(value.as_ref(), body); + } + // The listener is gone. A second store preparation must reuse the successful target. + let cached = session + .prepare_appconfig(&settings, None) + .await + .expect("cached snapshot"); + assert_eq!( + cached + .get("main") + .await + .expect("read") + .expect("value") + .as_ref(), + body + ); + assert_eq!(session.stats().mappings, 4_u64); + assert_eq!( + session.stats().unique_retained_payload_bytes, + u64::try_from(body.len()).expect("length") + ); + assert!(session.stats().metadata_bytes > 0); + } +} + +#[tokio::test] +async fn bearer_header_and_identity_encoding_are_explicit() { + let (mut settings, server) = fixture("", b"opaque".to_vec()).await; + settings.access_token_env = Some("DUMMY_AGENT_TOKEN".to_owned()); + let token = AgentAccessToken::new("dummy-bearer-token".to_owned()).expect("token"); + let mut session = + AwsPreparation::new(MonotonicClock::default(), AwsPreparationLimits::default()) + .expect("session"); + let _handle = session + .prepare_appconfig(&settings, Some(&token)) + .await + .expect("snapshot"); + let request = server.await.expect("server").to_ascii_lowercase(); + assert!(request.contains("authorization: bearer dummy-bearer-token\r\n")); + assert!(request.contains("accept-encoding: identity\r\n")); +} + +#[tokio::test] +async fn nonidentity_encoding_invalid_utf8_and_payload_caps_fail_closed() { + for (headers, body, cap, expected) in [ + ( + "Content-Encoding: gzip\r\n", + b"raw".to_vec(), + 4_u64, + PreparationError::Malformed, + ), + ( + "Content-Encoding: identity\r\nContent-Encoding: identity\r\n", + b"raw".to_vec(), + 4_u64, + PreparationError::Malformed, + ), + ( + "Content-Encoding: identity, identity\r\n", + b"raw".to_vec(), + 4_u64, + PreparationError::Malformed, + ), + ("", vec![0xff_u8], 4_u64, PreparationError::Malformed), + ("", b"five!".to_vec(), 4_u64, PreparationError::SizeLimit), + ] { + let (settings, server) = fixture(headers, body).await; + let limits = AwsPreparationLimits { + max_config_document_bytes: cap, + ..AwsPreparationLimits::default() + }; + let mut session = AwsPreparation::new(MonotonicClock::default(), limits).expect("session"); + let error = session + .prepare_appconfig(&settings, None) + .await + .map(|_handle| ()) + .expect_err("closed failure"); + assert_eq!(error, expected); + assert_eq!(session.stats().unique_retained_payload_bytes, 0); + let _request = server.await.expect("server"); + } +} diff --git a/crates/edgezero-store-aws/src/dynamodb_kv.rs b/crates/edgezero-store-aws/src/dynamodb_kv.rs new file mode 100644 index 000000000..ecfe8d0bd --- /dev/null +++ b/crates/edgezero-store-aws/src/dynamodb_kv.rs @@ -0,0 +1,970 @@ +//! Live native `DynamoDB` KV operations. SDK models never cross the portable boundary. + +use std::collections::HashMap; +use std::future::Future; +use std::sync::Arc; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +use async_trait::async_trait; +use aws_config::BehaviorVersion; +use aws_sdk_dynamodb::Client; +use aws_sdk_dynamodb::config::timeout::TimeoutConfig; +use aws_sdk_dynamodb::config::{Builder, Region, retry::RetryConfig}; +use aws_sdk_dynamodb::error::SdkError; +use aws_sdk_dynamodb::primitives::Blob; +use aws_sdk_dynamodb::types::{AttributeValue, TableDescription}; +use bytes::Bytes; +use edgezero_core::key_value_store::{KvError, KvHandle, KvPage, KvStore}; +use edgezero_core::time::{Deadline, MonotonicClock}; +use serde::{Deserialize, Serialize}; +use tokio::time::timeout; + +use crate::PreparationError; +use crate::preparation::AwsPreparation; +use crate::settings::{DynamoDbKvLimits, DynamoDbKvSettings}; + +const MAX_CURSOR_BYTES: usize = 0x4000; +const MAX_ITEM_BYTES: usize = 409_600; +const MAX_KEY_BYTES: usize = 512; +const MAX_PAGE_KEYS: usize = 1_000; + +#[derive(Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +struct Cursor { + key: String, + namespace: String, + prefix: String, + region: String, + table: String, + version: u8, +} + +struct DynamoDbStore { + client: Client, + clock: MonotonicClock, + limits: DynamoDbKvLimits, + namespace: String, + region: String, + table: String, +} + +#[expect( + clippy::multiple_inherent_impl, + reason = "keep feature-gated DynamoDB transport separate from other AWS services" +)] +#[expect( + clippy::arbitrary_source_item_ordering, + reason = "keep the supported preparation entry point ahead of its private shared setup path" +)] +impl AwsPreparation { + /// Prepare a live namespace-bound KV handle after bounded table validation. + /// + /// No automatic retries are made. A failed write may already have committed. + /// + /// # Errors + /// Returns a closed error for invalid bindings, deadline expiry, or provider failure. + #[inline] + pub async fn prepare_dynamodb( + &mut self, + settings: &DynamoDbKvSettings, + ) -> Result { + self.validate_dynamodb_mapping(settings)?; + let key = (settings.region.clone(), settings.limits); + let client = if let Some(client) = self.dynamodb_clients.get(&key) { + client.clone() + } else { + let budget = self.dynamodb_budget(settings)?; + let result = timeout(budget, async { + let shared = aws_config::defaults(BehaviorVersion::v2026_01_12()) + .region(Region::new(settings.region.clone())) + .load() + .await; + self.check_deadline()?; + Ok::(Client::from_conf( + aws_sdk_dynamodb::Config::from(&shared) + .to_builder() + .region(Region::new(settings.region.clone())) + .retry_config(RetryConfig::standard().with_max_attempts(1)) + .timeout_config(timeout_policy( + settings.limits, + Duration::from_millis(settings.limits.operation_timeout_ms), + )) + .build(), + )) + }) + .await; + self.check_deadline()?; + let client = result.map_err(|_elapsed| PreparationError::Deadline)??; + self.dynamodb_clients.insert(key, client.clone()); + client + }; + self.finish_dynamodb_preparation(settings, client).await + } + + fn dynamodb_budget(&self, settings: &DynamoDbKvSettings) -> Result { + Ok(self + .operation_budget()? + .min(Duration::from_millis(settings.limits.operation_timeout_ms))) + } + + async fn finish_dynamodb_preparation( + &self, + settings: &DynamoDbKvSettings, + client: Client, + ) -> Result { + let budget = self.dynamodb_budget(settings)?; + let result = timeout( + budget, + client + .describe_table() + .table_name(&settings.table) + .customize() + .config_override( + aws_sdk_dynamodb::Config::builder() + .retry_config(RetryConfig::standard().with_max_attempts(1)) + .timeout_config(timeout_policy(settings.limits, budget)), + ) + .send(), + ) + .await; + self.check_deadline()?; + let output = result + .map_err(|_elapsed| PreparationError::Deadline)? + .map_err(|error| { + if matches!(&error, SdkError::TimeoutError(_)) { + return PreparationError::Deadline; + } + classify_preparation_error( + error + .as_service_error() + .and_then(|service| service.meta().code()), + error + .raw_response() + .map(|response| response.status().as_u16()), + ) + })?; + let validation = output + .table + .as_ref() + .ok_or(PreparationError::Malformed) + .and_then(|table| validate_table(table, &settings.table)); + self.check_deadline()?; + validation?; + let handle = KvHandle::new(Arc::new(DynamoDbStore::new( + settings, + client, + self.clock.clone(), + ))); + self.check_deadline()?; + Ok(handle) + } + + #[cfg(test)] + async fn prepare_dynamodb_with_client( + &mut self, + settings: &DynamoDbKvSettings, + client: Client, + ) -> Result { + self.validate_dynamodb_mapping(settings)?; + self.finish_dynamodb_preparation(settings, client).await + } + + fn validate_dynamodb_mapping( + &mut self, + settings: &DynamoDbKvSettings, + ) -> Result<(), PreparationError> { + settings.validate()?; + self.check_deadline()?; + let metadata = settings + .region + .len() + .checked_add(settings.table.len()) + .ok_or(PreparationError::SizeLimit)?; + self.charge_mapping(0, settings.namespace.len(), metadata) + } +} + +impl DynamoDbStore { + async fn bounded( + &self, + deadline: Deadline, + operation: Operation, + ) -> Result + where + Operation: Future>, + { + let remaining = deadline + .remaining_at(self.clock.now()) + .filter(|remaining| !remaining.is_zero()) + .ok_or(KvError::Unavailable)?; + let result = timeout(remaining, operation).await; + self.check_runtime_deadline(deadline)?; + result.map_err(|_elapsed| KvError::Unavailable)? + } + + fn check_runtime_deadline(&self, deadline: Deadline) -> Result<(), KvError> { + if deadline.is_expired_at(self.clock.now()) { + return Err(KvError::Unavailable); + } + Ok(()) + } + + fn decode_cursor(&self, prefix: &str, encoded: &str) -> Result { + decode_cursor(encoded, prefix, &self.region, &self.table, &self.namespace) + } + + fn encode_cursor(&self, prefix: &str, key: String) -> Result { + let cursor = Cursor { + key, + namespace: self.namespace.clone(), + prefix: prefix.to_owned(), + region: self.region.clone(), + table: self.table.clone(), + version: 1, + }; + let encoded = serde_json::to_string(&cursor).map_err(|_error| KvError::Unavailable)?; + if encoded.len() > MAX_CURSOR_BYTES { + return Err(KvError::Unavailable); + } + Ok(encoded) + } + + fn key_attributes(&self, key: &str) -> HashMap { + HashMap::from([ + ("pk".to_owned(), AttributeValue::S(self.namespace.clone())), + ("sk".to_owned(), AttributeValue::S(key.to_owned())), + ]) + } + + fn new(settings: &DynamoDbKvSettings, client: Client, clock: MonotonicClock) -> Self { + Self { + client, + clock, + limits: settings.limits, + namespace: settings.namespace.clone(), + region: settings.region.clone(), + table: settings.table.clone(), + } + } + + fn operation_policy(&self, deadline: Deadline) -> Result { + let budget = deadline + .remaining_at(self.clock.now()) + .filter(|remaining| !remaining.is_zero()) + .ok_or(KvError::Unavailable)?; + Ok(aws_sdk_dynamodb::Config::builder() + .retry_config(RetryConfig::standard().with_max_attempts(1)) + .timeout_config(timeout_policy(self.limits, budget))) + } + + async fn put(&self, key: &str, value: Bytes, ttl: Option) -> Result<(), KvError> { + let deadline = self.runtime_deadline()?; + if !valid_key(key) || value.len() > self.limits.max_value_bytes { + return Err(invalid_input()); + } + let expiry = ttl + .map(|duration| { + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_err(|_error| KvError::Unavailable)?; + expiry_from_duration(now, duration) + }) + .transpose()?; + validate_item_size(self.namespace.len(), key.len(), value.len(), expiry)?; + let mut item = self.key_attributes(key); + // The payload cap and complete item accounting are checked before this copy. + item.insert("v".to_owned(), AttributeValue::B(Blob::new(value.to_vec()))); + if let Some(expires_at) = expiry { + item.insert( + "expires_at_ms".to_owned(), + AttributeValue::N(expires_at.to_string()), + ); + item.insert( + "ttl_epoch_s".to_owned(), + AttributeValue::N(expires_at.div_ceil(1_000).to_string()), + ); + } + self.bounded(deadline, async { + self.client + .put_item() + .table_name(&self.table) + .set_item(Some(item)) + .customize() + .config_override(self.operation_policy(deadline)?) + .send() + .await + .map_err(|_error| KvError::Unavailable)?; + Ok(()) + }) + .await + } + + fn runtime_deadline(&self) -> Result { + self.clock + .now() + .checked_add(Duration::from_millis(self.limits.operation_timeout_ms)) + .map(Deadline::at_instant) + .ok_or(KvError::Unavailable) + } +} + +#[async_trait(?Send)] +impl KvStore for DynamoDbStore { + #[inline] + async fn delete(&self, key: &str) -> Result<(), KvError> { + let deadline = self.runtime_deadline()?; + if !valid_key(key) { + return Err(invalid_input()); + } + self.bounded(deadline, async { + self.client + .delete_item() + .table_name(&self.table) + .set_key(Some(self.key_attributes(key))) + .customize() + .config_override(self.operation_policy(deadline)?) + .send() + .await + .map_err(|_error| KvError::Unavailable)?; + Ok(()) + }) + .await + } + + #[inline] + async fn exists(&self, key: &str) -> Result { + Ok(self.get_bytes(key).await?.is_some()) + } + + #[inline] + async fn get_bytes(&self, key: &str) -> Result, KvError> { + let deadline = self.runtime_deadline()?; + if !valid_key(key) { + return Err(invalid_input()); + } + let result = self + .bounded(deadline, async { + let output = self + .client + .get_item() + .table_name(&self.table) + .set_key(Some(self.key_attributes(key))) + .consistent_read(true) + .customize() + .config_override(self.operation_policy(deadline)?) + .send() + .await + .map_err(|_error| KvError::Unavailable)?; + let Some(mut item) = output.item else { + return Ok(None); + }; + if row_key(&item, &self.namespace)? != key { + return Err(KvError::Unavailable); + } + let expiry = read_expiry(&item)?; + let value = match item.remove("v") { + Some(AttributeValue::B(value)) + if value.as_ref().len() <= self.limits.max_value_bytes => + { + Bytes::from(value.into_inner()) + } + _ => return Err(KvError::Unavailable), + }; + Ok(Some((value, expiry))) + }) + .await?; + let now = epoch_ms()?; + let value = result.and_then(|(value, expiry)| (!is_expired(expiry, now)).then_some(value)); + self.check_runtime_deadline(deadline)?; + Ok(value) + } + + #[inline] + async fn list_keys_page( + &self, + prefix: &str, + cursor: Option<&str>, + limit: usize, + ) -> Result { + let deadline = self.runtime_deadline()?; + if prefix.len() > MAX_KEY_BYTES + || prefix.chars().any(char::is_control) + || limit == 0 + || limit > MAX_PAGE_KEYS + { + return Err(invalid_input()); + } + let after = cursor + .map(|encoded| self.decode_cursor(prefix, encoded)) + .transpose()?; + let query_limit = i32::try_from(limit).map_err(|_error| invalid_input())?; + self.bounded(deadline, async { + let mut request = self + .client + .query() + .table_name(&self.table) + .consistent_read(true) + .scan_index_forward(true) + .limit(query_limit) + .key_condition_expression(if prefix.is_empty() { + "#pk = :pk" + } else { + "#pk = :pk AND begins_with(#sk, :prefix)" + }) + .projection_expression("#pk, #sk, #expires, #ttl") + .expression_attribute_names("#pk", "pk") + .expression_attribute_names("#sk", "sk") + .expression_attribute_names("#expires", "expires_at_ms") + .expression_attribute_names("#ttl", "ttl_epoch_s") + .expression_attribute_values(":pk", AttributeValue::S(self.namespace.clone())); + if !prefix.is_empty() { + request = request + .expression_attribute_values(":prefix", AttributeValue::S(prefix.to_owned())); + } + if let Some(key) = &after { + request = request.set_exclusive_start_key(Some(self.key_attributes(key))); + } + let output = request + .customize() + .config_override(self.operation_policy(deadline)?) + .send() + .await + .map_err(|_error| KvError::Unavailable)?; + let items = output.items.unwrap_or_default(); + if items.len() > limit { + return Err(KvError::Unavailable); + } + let mut rows = Vec::with_capacity(items.len()); + let mut previous = after.clone(); + for item in &items { + let key = row_key(item, &self.namespace)?; + if !key.starts_with(prefix) || previous.as_deref().is_some_and(|last| key <= last) { + return Err(KvError::Unavailable); + } + previous = Some(key.to_owned()); + rows.push((key.to_owned(), read_expiry(item)?)); + } + let next = match output.last_evaluated_key { + None => None, + Some(key) if key.is_empty() => None, + Some(key) => { + if key.len() != 2 { + return Err(KvError::Unavailable); + } + let next_key = row_key(&key, &self.namespace)?; + if !next_key.starts_with(prefix) + || after.as_deref().is_some_and(|last| next_key <= last) + || previous.as_deref().is_some_and(|last| next_key < last) + { + return Err(KvError::Unavailable); + } + Some(self.encode_cursor(prefix, next_key.to_owned())?) + } + }; + // Expiry is checked after validation and cursor encoding, not at request start. + let completed = epoch_ms()?; + let keys = rows + .into_iter() + .filter_map(|(key, expiry)| (!is_expired(expiry, completed)).then_some(key)) + .collect(); + Ok(KvPage { cursor: next, keys }) + }) + .await + } + + #[inline] + async fn put_bytes(&self, key: &str, value: Bytes) -> Result<(), KvError> { + self.put(key, value, None).await + } + + #[inline] + async fn put_bytes_with_ttl( + &self, + key: &str, + value: Bytes, + ttl: Duration, + ) -> Result<(), KvError> { + self.put(key, value, Some(ttl)).await + } +} + +fn classify_preparation_error(code: Option<&str>, status: Option) -> PreparationError { + match code { + Some("AccessDeniedException" | "AccessDenied") => PreparationError::AccessDenied, + Some("ExpiredTokenException" | "InvalidClientTokenId" | "UnrecognizedClientException") => { + PreparationError::Credentials + } + Some("ResourceNotFoundException") => PreparationError::Missing, + Some( + "ThrottlingException" + | "ProvisionedThroughputExceededException" + | "RequestLimitExceeded", + ) => PreparationError::Throttled, + Some("ValidationException") => PreparationError::InvalidResource, + _ => match status { + Some(401 | 403) => PreparationError::AccessDenied, + Some(429) => PreparationError::Throttled, + _ => PreparationError::Unavailable, + }, + } +} + +fn decode_cursor( + encoded: &str, + prefix: &str, + region: &str, + table: &str, + namespace: &str, +) -> Result { + if encoded.len() > MAX_CURSOR_BYTES { + return Err(invalid_input()); + } + let cursor: Cursor = serde_json::from_str(encoded).map_err(|_error| invalid_input())?; + if cursor.version != 1 + || cursor.region != region + || cursor.table != table + || cursor.namespace != namespace + || cursor.prefix != prefix + || !valid_key(&cursor.key) + || !cursor.key.starts_with(prefix) + { + return Err(invalid_input()); + } + Ok(cursor.key) +} + +fn epoch_ms() -> Result { + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_err(|_error| KvError::Unavailable)?; + u64::try_from(now.as_millis()).map_err(|_error| KvError::Unavailable) +} + +fn expiry_from_duration(now: Duration, ttl: Duration) -> Result { + if ttl.is_zero() { + return Err(invalid_input()); + } + let expires = now.checked_add(ttl).ok_or_else(invalid_input)?; + let rounded = expires + .as_millis() + .checked_add(u128::from( + !expires.subsec_nanos().is_multiple_of(1_000_000), + )) + .ok_or_else(invalid_input)?; + u64::try_from(rounded).map_err(|_error| invalid_input()) +} + +fn invalid_input() -> KvError { + KvError::Validation("invalid DynamoDB KV input".to_owned()) +} + +fn is_expired(expiry: Option, now: u64) -> bool { + expiry.is_some_and(|expires| now >= expires) +} + +fn read_expiry(item: &HashMap) -> Result, KvError> { + match (item.get("expires_at_ms"), item.get("ttl_epoch_s")) { + (None, None) => Ok(None), + (Some(AttributeValue::N(expiry)), Some(AttributeValue::N(ttl))) => { + let expires = read_unsigned(expiry)?; + if read_unsigned(ttl)? != expires.div_ceil(1_000) { + return Err(KvError::Unavailable); + } + Ok(Some(expires)) + } + _ => Err(KvError::Unavailable), + } +} + +fn read_unsigned(value: &str) -> Result { + if value.is_empty() || value.len() > 20 || !value.bytes().all(|byte| byte.is_ascii_digit()) { + return Err(KvError::Unavailable); + } + let number = value + .parse::() + .map_err(|_error| KvError::Unavailable)?; + if number.to_string() != value { + return Err(KvError::Unavailable); + } + Ok(number) +} + +fn row_key<'row>( + item: &'row HashMap, + namespace: &str, +) -> Result<&'row str, KvError> { + match (item.get("pk"), item.get("sk")) { + (Some(AttributeValue::S(partition)), Some(AttributeValue::S(key))) + if partition == namespace && valid_key(key) => + { + Ok(key) + } + _ => Err(KvError::Unavailable), + } +} + +fn timeout_policy(limits: DynamoDbKvLimits, budget: Duration) -> TimeoutConfig { + TimeoutConfig::builder() + .connect_timeout(budget.min(Duration::from_millis(limits.connect_timeout_ms))) + .operation_attempt_timeout(budget.min(Duration::from_millis(limits.attempt_timeout_ms))) + .operation_timeout(budget) + .build() +} + +fn valid_key(key: &str) -> bool { + !key.is_empty() + && key.len() <= MAX_KEY_BYTES + && key != "." + && key != ".." + && !key.chars().any(char::is_control) +} + +fn validate_item_size( + namespace: usize, + key: usize, + value: usize, + expiry: Option, +) -> Result<(), KvError> { + let expiry_bytes = expiry + .map(|expires| { + "expires_at_ms" + .len() + .checked_add(expires.to_string().len()) + .and_then(|size| size.checked_add("ttl_epoch_s".len())) + .and_then(|size| size.checked_add(expires.div_ceil(1_000).to_string().len())) + .ok_or_else(invalid_input) + }) + .transpose()? + .unwrap_or_default(); + // Decimal text overestimates DynamoDB numeric storage and keeps accounting conservative. + let size = namespace + .checked_add(key) + .and_then(|size| size.checked_add(value)) + .and_then(|size| size.checked_add("pk".len())) + .and_then(|size| size.checked_add("sk".len())) + .and_then(|size| size.checked_add("v".len())) + .and_then(|size| size.checked_add(expiry_bytes)) + .ok_or_else(invalid_input)?; + if namespace > MAX_KEY_BYTES || key > MAX_KEY_BYTES || size > MAX_ITEM_BYTES { + return Err(invalid_input()); + } + Ok(()) +} + +fn validate_table(table: &TableDescription, expected_name: &str) -> Result<(), PreparationError> { + if table.table_name.as_deref() != Some(expected_name) { + return Err(PreparationError::InvalidResource); + } + if table + .table_status + .as_ref() + .is_none_or(|status| status.as_str() != "ACTIVE") + || !table.global_secondary_indexes().is_empty() + || !table.local_secondary_indexes().is_empty() + || table + .vector_indexes + .as_ref() + .is_some_and(|indexes| !indexes.is_empty()) + || !table.replicas().is_empty() + || table.global_table_version.is_some() + { + return Err(PreparationError::InvalidResource); + } + let keys = table.key_schema(); + let attributes = table.attribute_definitions(); + if keys.len() != 2 + || attributes.len() != 2 + || !keys + .iter() + .any(|key| key.attribute_name() == "pk" && key.key_type().as_str() == "HASH") + || !keys + .iter() + .any(|key| key.attribute_name() == "sk" && key.key_type().as_str() == "RANGE") + || !attributes.iter().any(|attribute| { + attribute.attribute_name() == "pk" && attribute.attribute_type().as_str() == "S" + }) + || !attributes.iter().any(|attribute| { + attribute.attribute_name() == "sk" && attribute.attribute_type().as_str() == "S" + }) + { + return Err(PreparationError::InvalidResource); + } + Ok(()) +} + +#[cfg(test)] +#[path = "dynamodb_tests.rs"] +mod sdk_tests; + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + use std::time::Duration; + + use aws_sdk_dynamodb::types::{ + AttributeDefinition, AttributeValue, KeySchemaElement, KeyType, ScalarAttributeType, + TableDescription, TableStatus, + }; + use edgezero_core::key_value_store::KvError; + + use crate::PreparationError; + + use super::{ + Cursor, MAX_CURSOR_BYTES, decode_cursor, expiry_from_duration, is_expired, read_expiry, + read_unsigned, row_key, valid_key, validate_item_size, validate_table, + }; + + fn active_table_description() -> TableDescription { + TableDescription::builder() + .table_name("expected-table") + .table_status(TableStatus::Active) + .key_schema( + KeySchemaElement::builder() + .attribute_name("pk") + .key_type(KeyType::Hash) + .build() + .unwrap(), + ) + .key_schema( + KeySchemaElement::builder() + .attribute_name("sk") + .key_type(KeyType::Range) + .build() + .unwrap(), + ) + .attribute_definitions( + AttributeDefinition::builder() + .attribute_name("pk") + .attribute_type(ScalarAttributeType::S) + .build() + .unwrap(), + ) + .attribute_definitions( + AttributeDefinition::builder() + .attribute_name("sk") + .attribute_type(ScalarAttributeType::S) + .build() + .unwrap(), + ) + .build() + } + + #[test] + fn cursor_codec_rejects_extra_fields_and_versions_are_explicit() { + let cursor = Cursor { + key: "a/b".to_owned(), + namespace: "ns".to_owned(), + prefix: "a/".to_owned(), + region: "us-east-1".to_owned(), + table: "table".to_owned(), + version: 1, + }; + let encoded = serde_json::to_string(&cursor).unwrap(); + let decoded: Cursor = serde_json::from_str(&encoded).unwrap(); + assert_eq!(decoded.key, "a/b"); + assert_eq!(decoded.version, 1); + let mut object = serde_json::to_value(&cursor).unwrap(); + object + .as_object_mut() + .unwrap() + .insert("unknown".to_owned(), serde_json::json!(1_i32)); + assert!(serde_json::from_value::(object).is_err()); + } + + #[test] + fn cursor_validation_rejects_every_scope_and_invalid_continuations() { + let cursor = Cursor { + key: "a/b".to_owned(), + namespace: "ns".to_owned(), + prefix: "a/".to_owned(), + region: "us-east-1".to_owned(), + table: "expected-table".to_owned(), + version: 1, + }; + let valid_encoded = serde_json::to_string(&cursor).unwrap(); + assert_eq!( + decode_cursor(&valid_encoded, "a/", "us-east-1", "expected-table", "ns").unwrap(), + "a/b" + ); + for field in ["namespace", "prefix", "region", "table"] { + let mut object = serde_json::to_value(&cursor).unwrap(); + object + .as_object_mut() + .unwrap() + .insert(field.to_owned(), serde_json::json!("other")); + let scope_encoded = serde_json::to_string(&object).unwrap(); + assert!(matches!( + decode_cursor(&scope_encoded, "a/", "us-east-1", "expected-table", "ns") + .unwrap_err(), + KvError::Validation(_) + )); + } + for key in ["", ".", "..", "other/key", "a/\n"] { + let mut object = serde_json::to_value(&cursor).unwrap(); + object + .as_object_mut() + .unwrap() + .insert("key".to_owned(), serde_json::json!(key)); + let key_encoded = serde_json::to_string(&object).unwrap(); + assert!(matches!( + decode_cursor(&key_encoded, "a/", "us-east-1", "expected-table", "ns").unwrap_err(), + KvError::Validation(_) + )); + } + let mut object = serde_json::to_value(&cursor).unwrap(); + object + .as_object_mut() + .unwrap() + .insert("version".to_owned(), serde_json::json!(2_u8)); + let version_encoded = serde_json::to_string(&object).unwrap(); + assert!(matches!( + decode_cursor(&version_encoded, "a/", "us-east-1", "expected-table", "ns").unwrap_err(), + KvError::Validation(_) + )); + object.as_object_mut().unwrap().remove("namespace"); + let missing_scope_encoded = serde_json::to_string(&object).unwrap(); + decode_cursor( + &missing_scope_encoded, + "a/", + "us-east-1", + "expected-table", + "ns", + ) + .unwrap_err(); + let oversized = "x".repeat(MAX_CURSOR_BYTES.checked_add(1).unwrap()); + decode_cursor(&oversized, "a/", "us-east-1", "expected-table", "ns").unwrap_err(); + } + + #[test] + fn expiry_metadata_rejects_wrong_types_and_unbounded_numbers() { + let valid = HashMap::from([ + ( + "expires_at_ms".to_owned(), + AttributeValue::N("1001".to_owned()), + ), + ("ttl_epoch_s".to_owned(), AttributeValue::N("2".to_owned())), + ]); + for attribute in ["expires_at_ms", "ttl_epoch_s"] { + for invalid in [ + AttributeValue::S("2".to_owned()), + AttributeValue::Bool(false), + AttributeValue::Null(true), + AttributeValue::N("-1".to_owned()), + AttributeValue::N("NaN".to_owned()), + AttributeValue::N("100000000000000000000".to_owned()), + ] { + let mut row = valid.clone(); + row.insert(attribute.to_owned(), invalid); + assert!(matches!( + read_expiry(&row).unwrap_err(), + KvError::Unavailable + )); + } + let mut row = valid.clone(); + row.remove(attribute); + assert!(matches!( + read_expiry(&row).unwrap_err(), + KvError::Unavailable + )); + } + let mut inconsistent = valid; + inconsistent.insert("ttl_epoch_s".to_owned(), AttributeValue::N("1".to_owned())); + assert!(matches!( + read_expiry(&inconsistent).unwrap_err(), + KvError::Unavailable + )); + let maximum = HashMap::from([ + ( + "expires_at_ms".to_owned(), + AttributeValue::N(u64::MAX.to_string()), + ), + ( + "ttl_epoch_s".to_owned(), + AttributeValue::N(u64::MAX.div_ceil(1_000).to_string()), + ), + ]); + assert_eq!(read_expiry(&maximum).unwrap(), Some(u64::MAX)); + } + + #[test] + fn expiry_pairs_are_checked_and_deadline_is_inclusive() { + let mut row = HashMap::from([ + ( + "expires_at_ms".to_owned(), + AttributeValue::N("1001".to_owned()), + ), + ("ttl_epoch_s".to_owned(), AttributeValue::N("2".to_owned())), + ]); + assert_eq!(read_expiry(&row).unwrap(), Some(1_001)); + assert!(!is_expired(Some(1_001), 1_000)); + assert!(is_expired(Some(1_001), 1_001)); + row.remove("ttl_epoch_s"); + assert!(matches!(read_expiry(&row), Err(KvError::Unavailable))); + row.clear(); + assert_eq!(read_expiry(&row).unwrap(), None); + } + + #[test] + fn expiry_rounding_never_shortens_ttl() { + assert_eq!( + expiry_from_duration(Duration::from_nanos(1), Duration::from_millis(1)).unwrap(), + 2 + ); + assert_eq!( + expiry_from_duration(Duration::from_secs(1), Duration::from_millis(1)).unwrap(), + 1_001 + ); + expiry_from_duration(Duration::ZERO, Duration::ZERO).unwrap_err(); + expiry_from_duration(Duration::MAX, Duration::from_millis(1)).unwrap_err(); + } + + #[test] + fn integer_metadata_is_bounded_and_canonical() { + for value in ["", "-1", "+1", "01", "1.0", "1e3", "18446744073709551616"] { + read_unsigned(value).unwrap_err(); + } + assert_eq!(read_unsigned("18446744073709551615").unwrap(), u64::MAX); + } + + #[test] + fn item_size_accounting_handles_boundaries_and_overflow() { + validate_item_size(512, 512, 393_216, Some(u64::MAX)).unwrap(); + validate_item_size(512, 512, 409_600, None).unwrap_err(); + validate_item_size(512, 512, usize::MAX, None).unwrap_err(); + } + + #[test] + fn keys_preserve_unicode_and_reject_invalid_rows() { + let composed = "\u{00e9}"; + let decomposed = "e\u{0301}"; + assert_ne!(composed, decomposed); + assert!(valid_key(composed)); + assert!(valid_key(decomposed)); + assert!(valid_key(&"x".repeat(512))); + assert!(!valid_key(&"x".repeat(513))); + for key in ["", ".", "..", "a\n"] { + assert!(!valid_key(key)); + } + let row = HashMap::from([ + ("pk".to_owned(), AttributeValue::S("ns".to_owned())), + ("sk".to_owned(), AttributeValue::S(decomposed.to_owned())), + ]); + assert_eq!(row_key(&row, "ns").unwrap(), decomposed); + row_key(&row, "other").unwrap_err(); + } + + #[test] + fn table_name_must_be_present_and_match_the_selected_resource() { + let mut table = active_table_description(); + validate_table(&table, "expected-table").unwrap(); + assert_eq!( + validate_table(&table, "wrong-table").unwrap_err(), + PreparationError::InvalidResource + ); + table.table_name = None; + assert_eq!( + validate_table(&table, "expected-table").unwrap_err(), + PreparationError::InvalidResource + ); + } +} diff --git a/crates/edgezero-store-aws/src/dynamodb_tests.rs b/crates/edgezero-store-aws/src/dynamodb_tests.rs new file mode 100644 index 000000000..d0d575e6c --- /dev/null +++ b/crates/edgezero-store-aws/src/dynamodb_tests.rs @@ -0,0 +1,935 @@ +//! Loopback SDK and portable-contract tests for the real `DynamoDbStore`. +//! +//! These fixtures exercise the actual AWS SDK serializer/deserializer and the +//! actual `DynamoDbStore`; the in-process service is only the HTTP peer. No +//! ambient AWS config is consulted by `loopback_client`. + +#![expect( + clippy::arbitrary_source_item_ordering, + reason = "fixture, protocol, runtime wrapper and contract cases are grouped" +)] + +use super::*; +use crate::preparation::AwsPreparation; +use crate::settings::{AwsPreparationLimits, DynamoDbKvLimits, DynamoDbKvSettings}; +use edgezero_core::time::MonotonicClock; +use std::collections::{BTreeMap, HashMap, VecDeque}; +use std::io::{ErrorKind, Read as _, Write as _}; +use std::net::{SocketAddr, TcpListener, TcpStream}; +use std::str; +use std::sync::{ + Arc, Mutex, + atomic::{AtomicBool, AtomicUsize, Ordering}, +}; +use std::thread::{self, JoinHandle}; +use std::time::{Duration, Instant}; + +use async_trait::async_trait; +use aws_config::BehaviorVersion; +use aws_sdk_dynamodb::Client; +use aws_sdk_dynamodb::config::Credentials; +use aws_sdk_dynamodb::config::timeout::TimeoutConfig; +use aws_sdk_dynamodb::config::{Region, retry::RetryConfig}; +use aws_sdk_dynamodb::operation::get_item::GetItemOutput; +use aws_sdk_dynamodb::primitives::Blob; +use aws_sdk_dynamodb::types::AttributeValue; +use base64::Engine as _; +use base64::engine::general_purpose::STANDARD; +use bytes::Bytes; +use edgezero_core::key_value_store::{KvError, KvHandle, KvPage, KvStore}; +use serde_json::{Value, json}; +use tokio::runtime::{Builder as RuntimeBuilder, Runtime}; + +const TABLE: &str = "fixture-kv"; +const NAMESPACE: &str = "sdk-tests"; +const REGION: &str = "us-east-1"; +const MAX_HEADER_BYTES: usize = 0x1_0000; +const MAX_REQUEST_BODY_BYTES: usize = 0x20_0000; +const TEST_KEY: &str = "nfc/\u{e9}"; +const TEST_VALUE: &[u8] = &[0, 0xff, b'\n', b'\'']; + +#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)] +struct RowKey { + pk: String, + sk: String, +} + +#[derive(Clone, Debug)] +struct Row { + expires_at_ms: Option, + ttl_epoch_s: Option, + value: Vec, +} + +#[derive(Default)] +struct FixtureState { + commit_before_error: bool, + describe_response: Option, + fail_target: Option<(String, u16, Value)>, + query_pages: VecDeque, + requests: Vec<(String, Value)>, + rows: BTreeMap, + stall_target: Option, +} + +/// Minimal `DynamoDB` JSON 1.0 endpoint. `Drop` stops accept and joins the thread; +/// every socket read has a deadline, and each response closes the connection. +struct DynamoFixture { + accepted: Arc, + addr: SocketAddr, + state: Arc>, + stopped: Arc, + worker: Option>, +} + +impl DynamoFixture { + fn commit_before_error(&self) { + self.state.lock().unwrap().commit_before_error = true; + } + + fn endpoint(&self) -> String { + format!("http://{}", self.addr) + } + + fn fail_target(&self, target: &str, status: u16, response: Value) { + self.state.lock().unwrap().fail_target = Some((target.to_owned(), status, response)); + } + + fn has_row(&self, key: &str) -> bool { + self.state.lock().unwrap().rows.contains_key(&RowKey { + pk: NAMESPACE.into(), + sk: key.into(), + }) + } + + fn push_query_page(&self, response: Value) { + self.state.lock().unwrap().query_pages.push_back(response); + } + + fn request_count(&self) -> usize { + self.state.lock().unwrap().requests.len() + } + + fn requests(&self) -> Vec<(String, Value)> { + self.state.lock().unwrap().requests.clone() + } + + fn seed(&self, key: &str, row: Row) { + self.state.lock().unwrap().rows.insert( + RowKey { + pk: NAMESPACE.into(), + sk: key.into(), + }, + row, + ); + } + + fn set_describe_response(&self, response: Value) { + self.state.lock().unwrap().describe_response = Some(response); + } + + fn stall_target(&self, target: &str) { + self.state.lock().unwrap().stall_target = Some(target.to_owned()); + } + + fn start() -> Self { + let listener = TcpListener::bind(("127.0.0.1", 0)).expect("bind literal loopback"); + listener.set_nonblocking(true).expect("nonblocking accept"); + let addr = listener.local_addr().unwrap(); + let state = Arc::new(Mutex::new(FixtureState::default())); + let stopped = Arc::new(AtomicBool::new(false)); + let accepted = Arc::new(AtomicUsize::new(0)); + let worker_state = Arc::clone(&state); + let worker_stopped = Arc::clone(&stopped); + let worker_accepted = Arc::clone(&accepted); + let worker = thread::spawn(move || { + while !worker_stopped.load(Ordering::Acquire) { + match listener.accept() { + Ok((mut stream, _)) => { + worker_accepted.fetch_add(1, Ordering::Relaxed); + stream + .set_read_timeout(Some(Duration::from_secs(3))) + .expect("set read timeout"); + stream + .set_write_timeout(Some(Duration::from_secs(3))) + .expect("set write timeout"); + serve_one(&mut stream, &worker_state); + } + Err(error) if error.kind() == ErrorKind::WouldBlock => { + thread::sleep(Duration::from_millis(2)); + } + Err(_) if worker_stopped.load(Ordering::Acquire) => break, + Err(error) => panic!("fixture accept failed: {error}"), + } + } + }); + Self { + accepted, + addr, + state, + stopped, + worker: Some(worker), + } + } +} + +impl Drop for DynamoFixture { + fn drop(&mut self) { + self.stopped.store(true, Ordering::Release); + if let Some(worker) = self.worker.take() { + worker.join().expect("fixture thread joined"); + } + debug_assert!( + self.request_count() + .checked_add(1) + .is_some_and(|requests| self.accepted.load(Ordering::Relaxed) <= requests) + ); + } +} + +fn serve_one(stream: &mut TcpStream, state: &Arc>) { + let Some((target, body)) = read_request(stream) else { + return; + }; + let parsed: Value = serde_json::from_slice(&body).expect("SDK emits valid JSON"); + let mut fixture = state.lock().unwrap(); + fixture.requests.push((target.clone(), parsed.clone())); + if fixture.stall_target.as_deref() == Some(&target) { + // Client operation timeout must bound the wait; fixture's finite upper + // bound also prevents a broken client from hanging test teardown. + drop(fixture); + thread::sleep(Duration::from_secs(3)); + return; + } + let (status, response) = if let Some((failed, status, response)) = fixture.fail_target.clone() { + if failed == target { + if fixture.commit_before_error + && matches!(target.rsplit('.').next(), Some("PutItem" | "DeleteItem")) + { + drop(respond(&mut fixture, &target, &parsed)); + } + (status, response) + } else { + respond(&mut fixture, &target, &parsed) + } + } else { + respond(&mut fixture, &target, &parsed) + }; + drop(fixture); + let encoded = serde_json::to_vec(&response).unwrap(); + let status_text = if status == 200 { "OK" } else { "Bad Request" }; + let head = format!( + "HTTP/1.1 {status} {status_text}\r\ncontent-type: application/x-amz-json-1.0\r\ncontent-length: {}\r\nconnection: close\r\n\r\n", + encoded.len() + ); + // Timed-out clients may close the socket before the finite fixture response. + stream + .write_all(head.as_bytes()) + .and_then(|()| stream.write_all(&encoded)) + .unwrap_or_default(); +} + +fn read_request(stream: &mut TcpStream) -> Option<(String, Vec)> { + let mut data = Vec::with_capacity(4096); + let mut chunk = [0_u8; 2048]; + let header_end = loop { + let count = stream.read(&mut chunk).ok()?; + if count == 0 { + return None; + } + data.extend_from_slice(&chunk[..count]); + if let Some(end) = data.windows(4).position(|window| window == b"\r\n\r\n") { + break end.checked_add(4)?; + } + assert!(data.len() < MAX_HEADER_BYTES, "oversized request headers"); + }; + let headers = str::from_utf8(data.get(..header_end)?).ok()?; + let target = headers.lines().find_map(|line| { + line.split_once(':') + .filter(|(name, _)| name.eq_ignore_ascii_case("x-amz-target")) + .map(|(_, value)| value.trim().to_owned()) + })?; + let length = headers.lines().find_map(|line| { + let (name, value) = line.split_once(':')?; + if name.eq_ignore_ascii_case("content-length") { + value.trim().parse::().ok() + } else { + None + } + })?; + assert!( + length < MAX_REQUEST_BODY_BYTES, + "fixture request body too large" + ); + let body_end = header_end.checked_add(length)?; + while data.len() < body_end { + let count = stream.read(&mut chunk).ok()?; + if count == 0 { + return None; + } + data.extend_from_slice(&chunk[..count]); + } + Some((target, data.get(header_end..body_end)?.to_vec())) +} + +fn respond(state: &mut FixtureState, target: &str, body: &Value) -> (u16, Value) { + match target.rsplit('.').next().unwrap_or_default() { + "DescribeTable" if state.describe_response.is_some() => { + (200, state.describe_response.take().unwrap()) + } + "DescribeTable" => (200, active_table()), + "PutItem" => { + let item = &body["Item"]; + let key = RowKey { + pk: item["pk"]["S"].as_str().unwrap().into(), + sk: item["sk"]["S"].as_str().unwrap().into(), + }; + assert_eq!( + key.pk, NAMESPACE, + "wire partition must be explicit namespace" + ); + let value = item["v"]["B"].as_str().expect("Binary value on wire"); + let bytes = decode_base64_known(value); + let expires_at_ms = item + .get("expires_at_ms") + .and_then(|attribute| attribute["N"].as_str()) + .and_then(|number| number.parse().ok()); + let ttl_epoch_s = item + .get("ttl_epoch_s") + .and_then(|attribute| attribute["N"].as_str()) + .and_then(|number| number.parse().ok()); + state.rows.insert( + key, + Row { + value: bytes, + expires_at_ms, + ttl_epoch_s, + }, + ); + (200, json!({})) + } + "GetItem" => { + let key = wire_key(body); + let mut response = json!({}); + if let Some(row) = state.rows.get(&key) { + response["Item"] = row_to_item(&key, row); + } + (200, response) + } + "DeleteItem" => { + state.rows.remove(&wire_key(body)); + (200, json!({})) + } + "Query" => { + if let Some(page) = state.query_pages.pop_front() { + return (200, page); + } + let partition = body["ExpressionAttributeValues"][":pk"]["S"] + .as_str() + .unwrap(); + assert_eq!(partition, NAMESPACE, "query is namespace scoped"); + let prefix = body["ExpressionAttributeValues"] + .get(":prefix") + .and_then(|attribute| attribute["S"].as_str()) + .unwrap_or(""); + let after = body + .get("ExclusiveStartKey") + .and_then(|key| key["sk"]["S"].as_str()) + .unwrap_or(""); + let limit = usize::try_from(body["Limit"].as_u64().unwrap_or(100)).unwrap_or(100); + let matching: Vec<_> = state + .rows + .iter() + .filter(|(key, _)| { + key.pk == partition && key.sk.starts_with(prefix) && key.sk.as_str() > after + }) + .collect(); + let selected: Vec<_> = matching.iter().take(limit).copied().collect(); + let rows: Vec = selected + .iter() + .map(|(key, row)| row_to_item(key, row)) + .collect(); + let mut response = json!({"Items": rows}); + if matching.len() > limit { + let (last_key, _) = selected + .last() + .expect("positive query limit has selected row"); + response["LastEvaluatedKey"] = + json!({"pk":{"S":last_key.pk}, "sk":{"S":last_key.sk}}); + } + (200, response) + } + other => panic!("unexpected DynamoDB SDK operation target: {other}"), + } +} + +fn wire_key(body: &Value) -> RowKey { + RowKey { + pk: body["Key"]["pk"]["S"].as_str().unwrap().into(), + sk: body["Key"]["sk"]["S"].as_str().unwrap().into(), + } +} + +fn row_to_item(key: &RowKey, row: &Row) -> Value { + let mut item = + json!({"pk":{"S":key.pk}, "sk":{"S":key.sk}, "v":{"B":encode_base64(&row.value)}}); + if let Some(value) = row.expires_at_ms { + item["expires_at_ms"] = json!({"N":value.to_string()}); + } + if let Some(value) = row.ttl_epoch_s { + item["ttl_epoch_s"] = json!({"N":value.to_string()}); + } + item +} + +fn encode_base64(bytes: &[u8]) -> String { + STANDARD.encode(bytes) +} + +fn decode_base64_known(input: &str) -> Vec { + STANDARD + .decode(input) + .expect("SDK Binary uses standard base64") +} + +fn loopback_client(runtime: &Runtime, endpoint: &str, limits: DynamoDbKvLimits) -> Client { + // Region and static credentials are explicitly injected before config load; + // the service endpoint is explicit in the DynamoDB config. There is no + // profile, web-identity, IMDS, environment-credential, or endpoint lookup. + let shared = runtime.block_on( + aws_config::defaults(BehaviorVersion::v2026_01_12()) + .region(Region::new(REGION)) + .credentials_provider(Credentials::new( + "fixture-access-key", + "fixture-secret-key", + None, + None, + "sdk-tests", + )) + .load(), + ); + let budget = Duration::from_millis(limits.operation_timeout_ms); + let config = aws_sdk_dynamodb::Config::from(&shared) + .to_builder() + .region(Region::new(REGION)) + .endpoint_url(endpoint) + .retry_config(RetryConfig::standard().with_max_attempts(1)) + .timeout_config( + TimeoutConfig::builder() + .connect_timeout(budget.min(Duration::from_millis(limits.connect_timeout_ms))) + .operation_attempt_timeout( + budget.min(Duration::from_millis(limits.attempt_timeout_ms)), + ) + .operation_timeout(budget) + .build(), + ) + .build(); + Client::from_conf(config) +} + +fn active_table() -> Value { + json!({"Table": { + "TableName": TABLE, "TableStatus": "ACTIVE", + "KeySchema": [ + {"AttributeName":"pk", "KeyType":"HASH"}, + {"AttributeName":"sk", "KeyType":"RANGE"} + ], + "AttributeDefinitions": [ + {"AttributeName":"pk", "AttributeType":"S"}, + {"AttributeName":"sk", "AttributeType":"S"} + ], "GlobalSecondaryIndexes": [], "LocalSecondaryIndexes": [], "Replicas": [] + }}) +} + +/// Runtime is created before constructing the actual provider and before +/// invoking the futures-executor contract macro. The wrapper is only an +/// executor bridge; every operation still reaches `DynamoDbStore` + the real SDK. +struct RuntimeStore { + fixture: DynamoFixture, + runtime: Runtime, + store: DynamoDbStore, +} + +impl RuntimeStore { + fn new() -> Self { + let fixture = DynamoFixture::start(); + // Establish the runtime before provider construction/factory use. + let runtime = RuntimeBuilder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let limits = DynamoDbKvLimits::default(); + let client = loopback_client(&runtime, &fixture.endpoint(), limits); + let store = DynamoDbStore::new(&settings(), client, MonotonicClock::default()); + Self { + fixture, + runtime, + store, + } + } +} + +fn settings() -> DynamoDbKvSettings { + DynamoDbKvSettings { + region: REGION.into(), + table: TABLE.into(), + namespace: NAMESPACE.into(), + limits: DynamoDbKvLimits::default(), + } +} + +#[async_trait(?Send)] +impl KvStore for RuntimeStore { + async fn delete(&self, key: &str) -> Result<(), KvError> { + self.runtime.block_on(self.store.delete(key)) + } + async fn exists(&self, key: &str) -> Result { + self.runtime.block_on(self.store.exists(key)) + } + async fn get_bytes(&self, key: &str) -> Result, KvError> { + self.runtime.block_on(self.store.get_bytes(key)) + } + async fn list_keys_page( + &self, + prefix: &str, + cursor: Option<&str>, + limit: usize, + ) -> Result { + self.runtime + .block_on(self.store.list_keys_page(prefix, cursor, limit)) + } + async fn put_bytes(&self, key: &str, value: Bytes) -> Result<(), KvError> { + self.runtime.block_on(self.store.put_bytes(key, value)) + } + async fn put_bytes_with_ttl( + &self, + key: &str, + value: Bytes, + ttl: Duration, + ) -> Result<(), KvError> { + self.runtime + .block_on(self.store.put_bytes_with_ttl(key, value, ttl)) + } +} + +edgezero_core::key_value_store_contract_tests!(dynamodb_real_store_contract, RuntimeStore::new()); + +#[test] +fn sdk_wire_put_preserves_raw_binary_and_explicit_namespace() { + let store = RuntimeStore::new(); + store + .runtime + .block_on( + store + .store + .put_bytes(TEST_KEY, Bytes::from_static(TEST_VALUE)), + ) + .unwrap(); + let requests = store.fixture.requests(); + let (target, body) = requests + .iter() + .find(|(target, _)| target.ends_with("PutItem")) + .unwrap(); + assert!(target.starts_with("DynamoDB_20120810.")); + assert_eq!(body["Item"]["pk"]["S"], NAMESPACE); + assert_eq!(body["Item"]["sk"]["S"], TEST_KEY); + assert_eq!( + decode_base64_known(body["Item"]["v"]["B"].as_str().unwrap()), + TEST_VALUE + ); +} + +#[test] +fn nfc_and_nfd_are_distinct_wire_sort_keys() { + let store = RuntimeStore::new(); + store.runtime.block_on(async { + store + .store + .put_bytes("\u{e9}", Bytes::from_static(b"nfc")) + .await + .unwrap(); + store + .store + .put_bytes("e\u{301}", Bytes::from_static(b"nfd")) + .await + .unwrap(); + assert_eq!( + store.store.get_bytes("\u{e9}").await.unwrap(), + Some(Bytes::from_static(b"nfc")) + ); + assert_eq!( + store.store.get_bytes("e\u{301}").await.unwrap(), + Some(Bytes::from_static(b"nfd")) + ); + }); + let sort_keys: Vec<_> = store + .fixture + .requests() + .iter() + .filter(|(target, _)| target.ends_with("PutItem")) + .map(|(_, body)| body["Item"]["sk"]["S"].as_str().unwrap().to_owned()) + .collect(); + assert_eq!(sort_keys, ["\u{e9}", "e\u{301}"]); +} + +#[test] +fn malformed_and_cross_scope_cursors_fail_before_http() { + let store = RuntimeStore::new(); + let before = store.fixture.request_count(); + let malformed = store + .runtime + .block_on(store.store.list_keys_page("", Some("not-a-cursor"), 2)); + assert!(matches!(malformed, Err(KvError::Validation(_)))); + assert_eq!(store.fixture.request_count(), before); + let cursor = store + .store + .encode_cursor("p/", "p/after".to_owned()) + .unwrap(); + let mut other_settings = settings(); + other_settings.namespace = "another-namespace".to_owned(); + let other_client = loopback_client( + &store.runtime, + &store.fixture.endpoint(), + other_settings.limits, + ); + let other = DynamoDbStore::new(&other_settings, other_client, MonotonicClock::default()); + let cross_scope = store + .runtime + .block_on(other.list_keys_page("p/", Some(&cursor), 2)); + assert!(matches!(cross_scope, Err(KvError::Validation(_)))); + assert_eq!(store.fixture.request_count(), before); +} + +#[test] +fn empty_expired_query_page_with_last_evaluated_key_is_not_filled() { + let store = RuntimeStore::new(); + let continuation = json!({"pk":{"S":NAMESPACE}, "sk":{"S":"p/expired"}}); + store.fixture.push_query_page(json!({ + "Items": [{ + "pk":{"S":NAMESPACE}, "sk":{"S":"p/expired"}, + "expires_at_ms":{"N":"1"}, "ttl_epoch_s":{"N":"1"} + }], + "LastEvaluatedKey": continuation + })); + store.fixture.push_query_page(json!({ + "Items": [{"pk":{"S":NAMESPACE}, "sk":{"S":"p/live"}, "v":{"B":"bGl2ZQ=="}}] + })); + let first = store + .runtime + .block_on(store.store.list_keys_page("p/", None, 1)) + .unwrap(); + assert!(first.keys.is_empty()); + assert!(first.cursor.is_some()); + assert_eq!( + store + .fixture + .requests() + .iter() + .filter(|(target, _)| target.ends_with("Query")) + .count(), + 1, + "provider must not fill an empty expired page with another Query" + ); + let second = store + .runtime + .block_on(store.store.list_keys_page("p/", first.cursor.as_deref(), 1)) + .unwrap(); + assert_eq!(second.keys, ["p/live"]); + let queries: Vec<_> = store + .fixture + .requests() + .into_iter() + .filter(|(target, _)| target.ends_with("Query")) + .collect(); + assert!(queries[1].1.get("ExclusiveStartKey").is_some()); + assert_eq!(queries.len(), 2); +} + +fn run_prepare( + fixture: &DynamoFixture, + settings: &DynamoDbKvSettings, +) -> Result { + let runtime = RuntimeBuilder::new_current_thread() + .enable_all() + .build() + .map_err(|_error| crate::PreparationError::Unavailable)?; + let client = loopback_client(&runtime, &fixture.endpoint(), settings.limits); + let mut preparation = + AwsPreparation::new(MonotonicClock::default(), AwsPreparationLimits::default())?; + runtime.block_on(preparation.prepare_dynamodb_with_client(settings, client)) +} + +#[test] +fn preparation_describes_and_accepts_only_the_supported_active_schema() { + let fixture = DynamoFixture::start(); + let result = run_prepare(&fixture, &settings()); + result.unwrap(); + let requests = fixture.requests(); + assert_eq!(requests.len(), 1); + assert!(requests[0].0.ends_with("DescribeTable")); + assert_eq!(requests[0].1["TableName"], TABLE); +} + +#[test] +fn preparation_rejects_missing_denied_and_unsupported_tables_closed() { + let cases = [ + ( + 400, + json!({"__type":"com.amazonaws.dynamodb.v20120810#ResourceNotFoundException", "message":"fixture-secret-marker"}), + ), + ( + 400, + json!({"__type":"com.amazonaws.dynamodb.v20120810#AccessDeniedException", "message":"fixture-secret-marker"}), + ), + ]; + for (status, response) in cases { + let fixture = DynamoFixture::start(); + fixture.fail_target("DynamoDB_20120810.DescribeTable", status, response); + let result = run_prepare(&fixture, &settings()); + let failure = result.unwrap_err(); + assert_eq!(fixture.request_count(), 1); + assert!(!format!("{failure:?}").contains("fixture-secret-marker")); + } + + let fixture = DynamoFixture::start(); + let mut wrong_schema = active_table(); + wrong_schema["Table"]["KeySchema"][0]["AttributeName"] = json!("wrong-pk"); + fixture.set_describe_response(wrong_schema); + let result = run_prepare(&fixture, &settings()); + assert_eq!(result.err(), Some(crate::PreparationError::InvalidResource)); + assert_eq!(fixture.request_count(), 1); +} + +#[test] +fn preparation_rejects_indexes_and_global_table_metadata() { + let indexed = [ + ("GlobalSecondaryIndexes", json!([{"IndexName":"gsi"}])), + ("LocalSecondaryIndexes", json!([{"IndexName":"lsi"}])), + ("Replicas", json!([{"RegionName":"us-west-2"}])), + ]; + for (field, value) in indexed { + let fixture = DynamoFixture::start(); + let mut table = active_table(); + table["Table"][field] = value; + fixture.set_describe_response(table); + let result = run_prepare(&fixture, &settings()); + assert_eq!( + result.err(), + Some(crate::PreparationError::InvalidResource), + "unexpectedly accepted {field}" + ); + assert_eq!(fixture.request_count(), 1); + } +} + +#[test] +fn provider_enforces_value_and_key_boundaries_before_io() { + let store = RuntimeStore::new(); + let accepted = Bytes::from(vec![0x5a; store.store.limits.max_value_bytes]); + store + .runtime + .block_on(store.store.put_bytes("boundary", accepted)) + .unwrap(); + assert_eq!( + store + .runtime + .block_on(store.store.get_bytes("boundary")) + .unwrap(), + Some(Bytes::from(vec![0x5a; store.store.limits.max_value_bytes])) + ); + let after_valid = store.fixture.request_count(); + assert_eq!(after_valid, 2); + let too_large_size = store + .store + .limits + .max_value_bytes + .checked_add(1) + .expect("test cap has headroom"); + let too_large = Bytes::from(vec![0x5a; too_large_size]); + let oversized_result = store + .runtime + .block_on(store.store.put_bytes("too-large", too_large)); + assert!(matches!(oversized_result, Err(KvError::Validation(_)))); + assert_eq!(store.fixture.request_count(), after_valid); + + store + .runtime + .block_on(store.store.put_bytes("empty-binary", Bytes::new())) + .unwrap(); + assert_eq!( + store + .runtime + .block_on(store.store.get_bytes("empty-binary")) + .unwrap(), + Some(Bytes::new()) + ); + + let key_512 = "k".repeat(512); + store + .runtime + .block_on(store.store.put_bytes(&key_512, Bytes::new())) + .unwrap(); + let before_invalid_key = store.fixture.request_count(); + let invalid_key_result = store + .runtime + .block_on(store.store.get_bytes(&"k".repeat(513))); + assert!(matches!(invalid_key_result, Err(KvError::Validation(_)))); + assert_eq!(store.fixture.request_count(), before_invalid_key); +} + +#[test] +fn malformed_stored_expiry_fails_closed() { + let store = RuntimeStore::new(); + store.fixture.seed( + "malformed", + Row { + expires_at_ms: Some(1_000), + ttl_epoch_s: Some(9), + value: b"secret-row-value".to_vec(), + }, + ); + let result = store.runtime.block_on(store.store.get_bytes("malformed")); + assert!(matches!(&result, Err(KvError::Unavailable))); + assert_eq!(store.fixture.request_count(), 1); + assert!(!format!("{result:?}").contains("secret-row-value")); +} + +#[test] +fn ttl_renewal_survives_and_plain_put_clears_expiry_fields() { + let store = RuntimeStore::new(); + store + .runtime + .block_on(store.store.put_bytes_with_ttl( + "renew", + Bytes::from_static(b"old"), + Duration::from_millis(1), + )) + .unwrap(); + store + .runtime + .block_on(store.store.put_bytes_with_ttl( + "renew", + Bytes::from_static(b"renewed"), + Duration::from_secs(2), + )) + .unwrap(); + thread::sleep(Duration::from_millis(25)); + assert_eq!( + store + .runtime + .block_on(store.store.get_bytes("renew")) + .unwrap(), + Some(Bytes::from_static(b"renewed")) + ); + store + .runtime + .block_on( + store + .store + .put_bytes("renew", Bytes::from_static(b"permanent")), + ) + .unwrap(); + let puts: Vec<_> = store + .fixture + .requests() + .into_iter() + .filter(|(target, _)| target.ends_with("PutItem")) + .collect(); + assert_eq!(puts.len(), 3); + assert!(puts[0].1["Item"].get("expires_at_ms").is_some()); + assert!(puts[0].1["Item"].get("ttl_epoch_s").is_some()); + assert!(puts[2].1["Item"].get("expires_at_ms").is_none()); + assert!(puts[2].1["Item"].get("ttl_epoch_s").is_none()); +} + +#[test] +fn write_failure_is_one_attempt_and_reports_no_secret_material() { + let store = RuntimeStore::new(); + let marker = "private-fixture-value-7fbc"; + store.fixture.commit_before_error(); + store.fixture.fail_target( + "DynamoDB_20120810.PutItem", + 500, + json!({"__type":"InternalServerError", "message":"fixture-secret-marker"}), + ); + let result = store.runtime.block_on( + store + .store + .put_bytes("private-fixture-key", Bytes::from(marker)), + ); + assert!(matches!(&result, Err(KvError::Unavailable))); + let requests = store.fixture.requests(); + assert_eq!( + requests + .iter() + .filter(|(target, _)| target.ends_with("PutItem")) + .count(), + 1 + ); + assert!( + store.fixture.has_row("private-fixture-key"), + "fixture committed before returning 500" + ); + let diagnostic = format!("{result:?}"); + for secret in [ + marker, + "private-fixture-key", + "fixture-secret-key", + "fixture-secret-marker", + ] { + assert!(!diagnostic.contains(secret)); + } +} + +#[test] +fn ambiguous_delete_can_remove_row_before_unavailable_response() { + let store = RuntimeStore::new(); + store.fixture.seed( + "ambiguous-delete", + Row { + expires_at_ms: None, + ttl_epoch_s: None, + value: b"before".to_vec(), + }, + ); + store.fixture.commit_before_error(); + store.fixture.fail_target( + "DynamoDB_20120810.DeleteItem", + 500, + json!({"__type":"InternalServerError", "message":"fixture"}), + ); + let result = store + .runtime + .block_on(store.store.delete("ambiguous-delete")); + assert!(matches!(result, Err(KvError::Unavailable))); + assert_eq!(store.fixture.request_count(), 1); + assert!(!store.fixture.has_row("ambiguous-delete")); +} + +#[test] +fn operation_stall_is_bounded() { + let store = RuntimeStore::new(); + store.fixture.stall_target("DynamoDB_20120810.GetItem"); + let start = Instant::now(); + let result = store.runtime.block_on(store.store.get_bytes("stall")); + assert!(matches!(result, Err(KvError::Unavailable))); + assert!( + start.elapsed() < Duration::from_secs(4), + "operation timeout did not bound stall" + ); + assert_eq!(store.fixture.request_count(), 1); +} + +#[test] +fn sdk_get_output_debug_contains_key_and_binary_hex() { + let key = "debug-key-sentinel-91a2"; + let binary = AttributeValue::B(Blob::new(vec![0xde, 0xad, 0xb3, 0x3f])); + let item = HashMap::from([ + ("sk".to_owned(), AttributeValue::S(key.to_owned())), + ("v".to_owned(), binary), + ]); + let output = GetItemOutput::builder().set_item(Some(item)).build(); + let diagnostic = format!("{output:?}"); + assert!(diagnostic.contains(key)); + assert!(diagnostic.contains("deadb33f")); +} diff --git a/crates/edgezero-store-aws/src/lib.rs b/crates/edgezero-store-aws/src/lib.rs new file mode 100644 index 000000000..160e6b397 --- /dev/null +++ b/crates/edgezero-store-aws/src/lib.rs @@ -0,0 +1,87 @@ +//! Native AWS store preparation, independent of the HTTP adapter. +//! +//! Settings remain available without either service feature. Service failures +//! use closed categories; no SDK, HTTP, settings or payload source chain survives. + +#![cfg_attr( + all( + not(target_arch = "wasm32"), + any( + feature = "appconfig-agent", + feature = "secrets-manager", + feature = "dynamodb-kv" + ) + ), + expect( + clippy::pub_use, + reason = "keep the provider's supported preparation API at the crate root" + ) +)] + +pub mod settings; + +#[cfg(test)] +#[cfg(not(target_arch = "wasm32"))] +#[cfg(feature = "appconfig-agent")] +mod appconfig_tests; + +#[cfg(all(not(target_arch = "wasm32"), feature = "dynamodb-kv"))] +mod dynamodb_kv; +#[cfg(all( + not(target_arch = "wasm32"), + any( + feature = "appconfig-agent", + feature = "secrets-manager", + feature = "dynamodb-kv" + ) +))] +mod preparation; +#[cfg(all(not(target_arch = "wasm32"), feature = "secrets-manager"))] +mod secrets_manager; +#[cfg(test)] +#[cfg(all(not(target_arch = "wasm32"), feature = "secrets-manager"))] +mod snapshot_tests; +#[cfg(all( + not(target_arch = "wasm32"), + any( + feature = "appconfig-agent", + feature = "secrets-manager", + feature = "dynamodb-kv" + ) +))] +pub use preparation::{AwsPreparation, PreparationStats}; + +/// Sanitized preparation failures. Never attach provider response details. +#[derive(Clone, Copy, Debug, Eq, PartialEq, thiserror::Error)] +pub enum PreparationError { + #[error("access denied")] + AccessDenied, + #[error("credential failure")] + Credentials, + #[error("preparation deadline")] + Deadline, + #[error("decryption failure")] + Decryption, + #[error("invalid binding")] + InvalidBinding, + #[error("invalid agent endpoint")] + InvalidEndpoint, + #[error("invalid preparation limits")] + InvalidLimits, + #[error("invalid resource reference")] + InvalidResource, + #[error("invalid version selector")] + InvalidSelector, + #[error("malformed value")] + Malformed, + #[error("missing resource or version")] + Missing, + #[error("size limit")] + SizeLimit, + #[error("throttled")] + Throttled, + #[error("provider unavailable")] + Unavailable, + #[error("provider not compiled")] + UnsupportedProvider, +} diff --git a/crates/edgezero-store-aws/src/preparation.rs b/crates/edgezero-store-aws/src/preparation.rs new file mode 100644 index 000000000..6c2facf61 --- /dev/null +++ b/crates/edgezero-store-aws/src/preparation.rs @@ -0,0 +1,752 @@ +//! Native service preparation session and immutable `AppConfig` snapshots. +//! +//! This module is enabled only when at least one native AWS store service is +//! selected. The session's counters and caches are private to one startup. + +#[cfg(any(feature = "appconfig-agent", feature = "secrets-manager"))] +use std::collections::BTreeMap; +use std::collections::HashMap; +use std::fmt; +#[cfg(feature = "appconfig-agent")] +use std::sync::Arc; +use std::time::Duration; + +#[cfg(any(feature = "appconfig-agent", feature = "secrets-manager"))] +use edgezero_core::config_store::BoundedStoreRead; +#[cfg(feature = "appconfig-agent")] +use edgezero_core::config_store::ConfigStoreHandle; +#[cfg(feature = "appconfig-agent")] +use edgezero_core::config_store::{ + ConfigStore, ConfigStoreError, ConfigValue, finish_bounded_config_read, +}; +#[cfg(test)] +#[cfg(feature = "appconfig-agent")] +use edgezero_core::config_store_contract_tests; +#[cfg(feature = "secrets-manager")] +use edgezero_core::secret_store::{SecretError, SecretStore, finish_bounded_secret_read}; +use edgezero_core::time::{Deadline, MonotonicClock}; +#[cfg(feature = "appconfig-agent")] +use futures_util::StreamExt as _; +#[cfg(feature = "appconfig-agent")] +use reqwest::header::{ACCEPT_ENCODING, AUTHORIZATION, CONTENT_ENCODING, HeaderValue}; +#[cfg(feature = "appconfig-agent")] +use reqwest::{Client, ClientBuilder, redirect::Policy}; +#[cfg(feature = "appconfig-agent")] +use tokio::time::timeout; + +use crate::PreparationError; +use crate::settings::AwsPreparationLimits; +#[cfg(feature = "dynamodb-kv")] +use crate::settings::DynamoDbKvLimits; +#[cfg(feature = "appconfig-agent")] +use crate::settings::{AgentAccessToken, AppConfigAgentSettings}; +#[cfg(feature = "dynamodb-kv")] +use aws_sdk_dynamodb::Client as DynamoDbClient; + +/// Aggregate startup preparation. Never share this object across startup runs. +#[expect( + clippy::field_scoped_visibility_modifiers, + reason = "the session shares selected state with its feature-gated Secrets Manager sibling module" +)] +pub struct AwsPreparation { + accounting: Accounting, + #[cfg(feature = "appconfig-agent")] + appconfig_cache: HashMap, + #[cfg(feature = "appconfig-agent")] + appconfig_client: Option, + pub(crate) clock: MonotonicClock, + pub(crate) deadline: Deadline, + #[cfg(feature = "dynamodb-kv")] + pub(crate) dynamodb_clients: HashMap<(String, DynamoDbKvLimits), DynamoDbClient>, + pub(crate) limits: AwsPreparationLimits, + #[cfg(feature = "secrets-manager")] + pub(crate) secret_cache: HashMap, + #[cfg(feature = "secrets-manager")] + pub(crate) secret_clients: HashMap, +} + +impl fmt::Debug for AwsPreparation { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("AwsPreparation") + .field("stats", &self.stats()) + .finish_non_exhaustive() + } +} + +#[expect( + clippy::arbitrary_source_item_ordering, + reason = "keep startup entry points and feature-gated service operations grouped ahead of private helpers" +)] +impl AwsPreparation { + /// Start the aggregate deadline before creating any service client. + /// + /// A timeout bounds observation of async work, not arbitrary blocking provider + /// work or subprocesses: dropping the future is not a hard wall-clock guarantee. + /// + /// # Errors + /// + /// Returns [`PreparationError::InvalidLimits`] when limits are invalid or the + /// aggregate deadline cannot be represented. + #[inline] + pub fn new( + clock: MonotonicClock, + limits: AwsPreparationLimits, + ) -> Result { + limits.validate()?; + let duration = Duration::from_millis(limits.preparation_timeout_ms); + let deadline = clock + .now() + .checked_add(duration) + .map(Deadline::at_instant) + .ok_or(PreparationError::InvalidLimits)?; + Ok(Self { + clock, + deadline, + limits, + accounting: Accounting::default(), + #[cfg(feature = "dynamodb-kv")] + dynamodb_clients: HashMap::new(), + #[cfg(feature = "appconfig-agent")] + appconfig_client: None, + #[cfg(feature = "appconfig-agent")] + appconfig_cache: HashMap::new(), + #[cfg(feature = "secrets-manager")] + secret_cache: HashMap::new(), + #[cfg(feature = "secrets-manager")] + secret_clients: HashMap::new(), + }) + } + + /// Fetch and freeze all mapped `AppConfig` Agent documents into a core handle. + /// + /// # Errors + /// + /// Returns a sanitized [`PreparationError`] when settings, authorization, the + /// deadline, a response, or a configured size limit prevents preparation. + #[inline] + #[cfg(feature = "appconfig-agent")] + pub async fn prepare_appconfig( + &mut self, + settings: &AppConfigAgentSettings, + token: Option<&AgentAccessToken>, + ) -> Result { + settings.validate()?; + if settings.access_token_env.is_some() != token.is_some() { + return Err(PreparationError::InvalidBinding); + } + let mut targets = Vec::new(); + for (key, document) in &settings.documents { + self.check_deadline()?; + let url = settings.document_url(document)?; + let auth = token.map_or_else( + || AuthorizationIdentity::NoBearer { + env_name: settings.access_token_env.clone(), + }, + |value| AuthorizationIdentity::Bearer { + env_name: settings.access_token_env.clone(), + token: value.expose().to_owned(), + }, + ); + let target = AppConfigTargetKey { + url: url.as_str().to_owned(), + auth, + }; + let auth_metadata = match &target.auth { + AuthorizationIdentity::NoBearer { env_name } => { + env_name.as_ref().map_or(0, String::len) + } + AuthorizationIdentity::Bearer { + env_name, + token: bearer_token, + } => env_name + .as_ref() + .map_or(Some(0), |name| Some(name.len())) + .and_then(|bytes| bytes.checked_add(bearer_token.len())) + .ok_or(PreparationError::SizeLimit)?, + }; + let target_metadata = target + .url + .len() + .checked_add(auth_metadata) + .ok_or(PreparationError::SizeLimit)?; + self.accounting + .charge_mapping(&self.limits, key.len(), 0, target_metadata)?; + targets.push((key, url, target)); + } + let mut entries = BTreeMap::new(); + for (key, url, target) in targets { + self.check_deadline()?; + let value = if let Some(value) = self.appconfig_cache.get(&target) { + value.clone() + } else { + let fetched = self.fetch_agent_document(url, token).await?; + self.accounting + .charge_retained(&self.limits, fetched.as_ref().len())?; + self.appconfig_cache.insert(target, fetched.clone()); + fetched + }; + entries.insert(key.clone(), value); + } + self.check_deadline()?; + Ok(ConfigStoreHandle::new(Arc::new(ConfigSnapshot { entries }))) + } + + /// Safe counters only; no identifiers, lengths of individual values, or auth data. + #[must_use] + #[inline] + pub fn stats(&self) -> PreparationStats { + PreparationStats { + mappings: self.accounting.mapping_count, + unique_retained_payload_bytes: self.accounting.unique_retained_payload_bytes, + metadata_bytes: self.accounting.metadata_bytes, + peak_transient_staging_bytes: self.accounting.peak_transient_staging_bytes, + } + } + + pub(crate) fn check_deadline(&self) -> Result { + self.deadline + .remaining_at(self.clock.now()) + .filter(|remaining| !remaining.is_zero()) + .ok_or(PreparationError::Deadline) + } + + /// Current operation allowance, clipped to the shared deadline. + pub(crate) fn operation_budget(&self) -> Result { + Ok(self + .check_deadline()? + .min(Duration::from_millis(self.limits.operation_timeout_ms))) + } + + /// Charge one logical mapping before cache lookup/deduplication. + #[cfg(any(feature = "secrets-manager", feature = "dynamodb-kv"))] + pub(crate) fn charge_mapping( + &mut self, + key_bytes: usize, + namespace_bytes: usize, + target_metadata_bytes: usize, + ) -> Result<(), PreparationError> { + self.accounting.charge_mapping( + &self.limits, + key_bytes, + namespace_bytes, + target_metadata_bytes, + ) + } + + /// Retain one successful unique payload. Call only after all value caps pass. + #[cfg(feature = "secrets-manager")] + pub(crate) fn charge_retained(&mut self, bytes: usize) -> Result<(), PreparationError> { + self.accounting.charge_retained(&self.limits, bytes) + } + + #[cfg(feature = "appconfig-agent")] + async fn fetch_agent_document( + &mut self, + url: url::Url, + token: Option<&AgentAccessToken>, + ) -> Result { + self.check_deadline()?; + let client = if let Some(client) = &self.appconfig_client { + client.clone() + } else { + let client = ClientBuilder::new() + .no_proxy() + .connection_verbose(false) + .redirect(Policy::none()) + .no_gzip() + .no_brotli() + .no_deflate() + .no_zstd() + .connect_timeout( + self.operation_budget()? + .min(Duration::from_millis(self.limits.connect_timeout_ms)), + ) + .build() + .map_err(|_error| PreparationError::Unavailable)?; + self.check_deadline()?; + self.appconfig_client = Some(client.clone()); + client + }; + let budget = self + .operation_budget()? + .min(Duration::from_millis(self.limits.attempt_timeout_ms)); + let mut request = client + .get(url) + .header(ACCEPT_ENCODING, "identity") + .timeout(budget); + if let Some(access_token) = token { + let mut value = HeaderValue::from_str(&format!("Bearer {}", access_token.expose())) + .map_err(|_error| PreparationError::InvalidBinding)?; + value.set_sensitive(true); + request = request.header(AUTHORIZATION, value); + } + let remaining_payload = self + .limits + .max_snapshot_bytes + .checked_sub(self.accounting.unique_retained_payload_bytes) + .ok_or(PreparationError::SizeLimit)?; + let cap = usize::try_from(self.limits.max_config_document_bytes.min(remaining_payload)) + .map_err(|_error| PreparationError::InvalidLimits)?; + let fetched = timeout(budget, async { + let response = request.send().await.map_err(map_http_error)?; + if !response.status().is_success() { + return Err(match response.status().as_u16() { + 401 | 403 => PreparationError::AccessDenied, + 404 => PreparationError::Missing, + 429 => PreparationError::Throttled, + _ => PreparationError::Unavailable, + }); + } + if response.headers().get_all(CONTENT_ENCODING).iter().count() > 1 + || response + .headers() + .get_all(CONTENT_ENCODING) + .iter() + .any(|value| { + value.to_str().map_or(true, |encoding| { + !encoding.trim().eq_ignore_ascii_case("identity") + }) + }) + { + return Err(PreparationError::Malformed); + } + if response + .content_length() + .is_some_and(|length| length > u64::try_from(cap).unwrap_or(u64::MAX)) + { + return Err(PreparationError::SizeLimit); + } + let mut stream = response.bytes_stream(); + let mut body = Vec::with_capacity(cap.min(16 * 1024)); + while let Some(chunk_result) = stream.next().await { + let chunk_bytes = chunk_result.map_err(map_http_error)?; + if chunk_bytes.len() > cap.saturating_sub(body.len()) { + return Err(PreparationError::SizeLimit); + } + self.check_deadline()?; + self.accounting.observe_staging( + body.len() + .checked_add(chunk_bytes.len()) + .ok_or(PreparationError::SizeLimit)?, + )?; + body.extend_from_slice(&chunk_bytes); + } + String::from_utf8(body).map_err(|_error| PreparationError::Malformed) + }) + .await + .map_err(|_elapsed| PreparationError::Deadline)?; + self.check_deadline()?; + let text = fetched?; + // Preserve every UTF-8 byte, including empty/non-JSON free-form documents. + self.accounting.observe_staging(text.len())?; + let value = ConfigValue::from(text); + self.check_deadline()?; + Ok(value) + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct PreparationStats { + pub mappings: u64, + pub metadata_bytes: u64, + pub peak_transient_staging_bytes: u64, + pub unique_retained_payload_bytes: u64, +} + +#[derive(Default)] +struct Accounting { + mapping_count: u64, + metadata_bytes: u64, + peak_transient_staging_bytes: u64, + unique_retained_payload_bytes: u64, +} + +impl Accounting { + fn charge_mapping( + &mut self, + limits: &AwsPreparationLimits, + key_bytes: usize, + namespace_bytes: usize, + target_metadata_bytes: usize, + ) -> Result<(), PreparationError> { + let mapping_count = self + .mapping_count + .checked_add(1) + .filter(|count| *count <= limits.max_remote_values) + .ok_or(PreparationError::SizeLimit)?; + let metadata = key_bytes + .checked_add(namespace_bytes) + .and_then(|value| value.checked_add(target_metadata_bytes)) + .and_then(|value| u64::try_from(value).ok()) + .and_then(|value| self.metadata_bytes.checked_add(value)) + .ok_or(PreparationError::SizeLimit)?; + // Metadata has no independent configured ceiling. Checked arithmetic keeps + // the diagnostic statistic representable without pretending it is an RSS cap. + self.mapping_count = mapping_count; + self.metadata_bytes = metadata; + Ok(()) + } + + #[cfg(any(feature = "appconfig-agent", feature = "secrets-manager"))] + fn charge_retained( + &mut self, + limits: &AwsPreparationLimits, + bytes: usize, + ) -> Result<(), PreparationError> { + let byte_count = u64::try_from(bytes).map_err(|_error| PreparationError::SizeLimit)?; + let total = self + .unique_retained_payload_bytes + .checked_add(byte_count) + .filter(|total| *total <= limits.max_snapshot_bytes) + .ok_or(PreparationError::SizeLimit)?; + self.unique_retained_payload_bytes = total; + Ok(()) + } + + #[cfg(feature = "appconfig-agent")] + fn observe_staging(&mut self, bytes: usize) -> Result<(), PreparationError> { + let byte_count = u64::try_from(bytes).map_err(|_error| PreparationError::SizeLimit)?; + self.peak_transient_staging_bytes = self.peak_transient_staging_bytes.max(byte_count); + Ok(()) + } +} + +#[cfg(feature = "appconfig-agent")] +#[derive(Clone, Eq, Hash, PartialEq)] +struct AppConfigTargetKey { + auth: AuthorizationIdentity, + url: String, +} + +#[cfg(feature = "appconfig-agent")] +#[derive(Clone, Eq, Hash, PartialEq)] +enum AuthorizationIdentity { + Bearer { + env_name: Option, + token: String, + }, + NoBearer { + env_name: Option, + }, +} + +#[cfg(feature = "appconfig-agent")] +impl fmt::Debug for AuthorizationIdentity { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("AuthorizationIdentity([redacted])") + } +} + +#[cfg(feature = "appconfig-agent")] +impl fmt::Debug for AppConfigTargetKey { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("AppConfigTargetKey([redacted])") + } +} + +#[cfg(feature = "secrets-manager")] +#[derive(Clone, Eq, Hash, PartialEq)] +#[expect( + clippy::field_scoped_visibility_modifiers, + reason = "the client cache key is shared only with the feature-gated Secrets Manager sibling module" +)] +pub(crate) struct SecretClientKey { + pub(crate) policy: String, + pub(crate) region: String, +} + +#[cfg(feature = "secrets-manager")] +#[expect( + clippy::field_scoped_visibility_modifiers, + reason = "the SDK client is shared only with the feature-gated Secrets Manager sibling module" +)] +pub(crate) struct SecretClient { + pub(crate) client: aws_sdk_secretsmanager::Client, +} + +#[cfg(feature = "secrets-manager")] +#[derive(Clone, Eq, Hash, PartialEq)] +#[expect( + clippy::field_scoped_visibility_modifiers, + reason = "the target cache key is shared only with the feature-gated Secrets Manager sibling module" +)] +pub(crate) struct SecretTargetKey { + pub(crate) client_policy: String, + pub(crate) region: String, + pub(crate) secret_id: String, + /// Exact private selector identity; never include in diagnostics. + pub(crate) selector: String, +} + +#[cfg(feature = "secrets-manager")] +impl fmt::Debug for SecretTargetKey { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("SecretTargetKey([redacted])") + } +} + +#[cfg(feature = "appconfig-agent")] +struct ConfigSnapshot { + entries: BTreeMap, +} + +#[cfg(feature = "appconfig-agent")] +impl fmt::Debug for ConfigSnapshot { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ConfigSnapshot") + .field("entry_count", &self.entries.len()) + .finish() + } +} + +#[cfg(feature = "appconfig-agent")] +#[async_trait::async_trait(?Send)] +impl ConfigStore for ConfigSnapshot { + async fn get(&self, key: &str) -> Result, ConfigStoreError> { + Ok(self.entries.get(key).cloned()) + } + + async fn get_bounded( + &self, + key: &str, + clock: &MonotonicClock, + deadline: Deadline, + max_backend_bytes: u64, + max_value_bytes: u64, + ) -> Result, ConfigStoreError> { + if deadline.is_expired_at(clock.now()) { + return Err(ConfigStoreError::DeadlineExceeded); + } + let result = finish_bounded_config_read( + Ok(self.entries.get(key).cloned()), + clock, + deadline, + max_backend_bytes, + max_value_bytes, + ); + if deadline.is_expired_at(clock.now()) { + return Err(ConfigStoreError::DeadlineExceeded); + } + result + } +} + +#[cfg(feature = "secrets-manager")] +#[expect( + clippy::field_scoped_visibility_modifiers, + reason = "the immutable snapshot is constructed by the feature-gated Secrets Manager sibling module" +)] +pub(crate) struct SecretSnapshot { + pub(crate) entries: BTreeMap, + pub(crate) namespace: String, +} + +#[cfg(feature = "secrets-manager")] +impl fmt::Debug for SecretSnapshot { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("SecretSnapshot") + .field("entry_count", &self.entries.len()) + .finish_non_exhaustive() + } +} + +#[cfg(feature = "secrets-manager")] +#[async_trait::async_trait(?Send)] +impl SecretStore for SecretSnapshot { + async fn get_bytes( + &self, + store_name: &str, + key: &str, + ) -> Result, SecretError> { + Ok((store_name == self.namespace) + .then(|| self.entries.get(key).cloned()) + .flatten()) + } + + async fn get_bytes_bounded( + &self, + store_name: &str, + key: &str, + clock: &MonotonicClock, + deadline: Deadline, + max_backend_bytes: u64, + max_value_bytes: u64, + ) -> Result, SecretError> { + if deadline.is_expired_at(clock.now()) { + return Err(SecretError::DeadlineExceeded); + } + let value = (store_name == self.namespace) + .then(|| self.entries.get(key).cloned()) + .flatten(); + let result = finish_bounded_secret_read( + Ok(value), + clock, + deadline, + max_backend_bytes, + max_value_bytes, + ); + if deadline.is_expired_at(clock.now()) { + return Err(SecretError::DeadlineExceeded); + } + result + } +} + +#[cfg(feature = "appconfig-agent")] +fn map_http_error(_error: reqwest::Error) -> PreparationError { + PreparationError::Unavailable +} + +#[cfg(test)] +#[cfg(feature = "appconfig-agent")] +mod tests { + use super::*; + use edgezero_core::time::MonotonicInstant; + use futures::executor::block_on; + use std::sync::Mutex; + + fn agent_settings(endpoint: &str) -> AppConfigAgentSettings { + use crate::settings::AgentDocument; + AppConfigAgentSettings { + default_key: "main".to_owned(), + endpoint: endpoint.to_owned(), + access_token_env: Some("EDGE_TOKEN".to_owned()), + documents: BTreeMap::from([( + "main".to_owned(), + AgentDocument { + application: "app".to_owned(), + environment: "test".to_owned(), + profile: "profile".to_owned(), + }, + )]), + } + } + + fn test_clock() -> MonotonicClock { + MonotonicClock::default() + } + + #[test] + fn absolute_deadline_is_checked_and_snapshot_contract_is_ordinary() { + let clock = test_clock(); + let _session = AwsPreparation::new(clock, AwsPreparationLimits::default()) + .expect("default limits produce a valid preparation session"); + let accounting = Accounting::default(); + assert_eq!(accounting.mapping_count, 0); + } + + #[test] + fn accounting_charges_aliases_before_dedup_and_checks_overflow() { + let limits = AwsPreparationLimits { + max_remote_values: 1, + ..AwsPreparationLimits::default() + }; + let mut accounting = Accounting::default(); + accounting + .charge_mapping(&limits, 3, 0, 4) + .expect("first mapping is within the configured limit"); + assert_eq!( + accounting.charge_mapping(&limits, 3, 0, 4), + Err(PreparationError::SizeLimit) + ); + accounting.unique_retained_payload_bytes = u64::MAX; + assert_eq!( + accounting.charge_retained(&limits, 1), + Err(PreparationError::SizeLimit) + ); + } + + #[tokio::test] + async fn appconfig_real_loopback_fetch_and_exact_body_snapshot() { + use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; + use tokio::net::TcpListener; + + let listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("bind loopback"); + let address = listener.local_addr().expect("local addr"); + let server = tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.expect("accept"); + let mut request_bytes = vec![0_u8; 4096]; + let count = socket.read(&mut request_bytes).await.expect("read request"); + let request_text = String::from_utf8_lossy(&request_bytes[..count]); + assert!( + request_text + .to_ascii_lowercase() + .contains("accept-encoding: identity") + ); + assert!( + request_text.contains("/applications/app/environments/test/configurations/profile") + ); + let response = "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: 12\r\nConnection: close\r\n\r\n{\"x\":\"\u{fc}\"} "; + socket + .write_all(response.as_bytes()) + .await + .expect("write response"); + }); + let mut settings = agent_settings(&format!("http://{address}")); + settings.access_token_env = None; + let mut session = + AwsPreparation::new(test_clock(), AwsPreparationLimits::default()).expect("session"); + let handle = session + .prepare_appconfig(&settings, None) + .await + .expect("prepare"); + let value = handle + .get("main") + .await + .expect("read snapshot") + .expect("entry"); + assert_eq!(value.as_ref(), "{\"x\":\"\u{fc}\"} "); + server.await.expect("server"); + } + + #[test] + fn token_debug_is_redacted_even_when_target_identity_is_debugged() { + let token = AgentAccessToken::new("very-secret-token".to_owned()).expect("token"); + assert!(!format!("{token:?}").contains("very-secret-token")); + let key = AppConfigTargetKey { + url: "http://127.0.0.1".to_owned(), + auth: AuthorizationIdentity::Bearer { + env_name: Some("EDGE_TOKEN".to_owned()), + token: token.expose().to_owned(), + }, + }; + assert!(!format!("{key:?}").contains("very-secret-token")); + } + + #[test] + fn bounded_snapshot_deadline_wins_over_size_error_at_final_check() { + let start = MonotonicInstant::now(); + let now = Arc::new(Mutex::new(start)); + let clock_now = Arc::clone(&now); + let clock = MonotonicClock::new(move || { + let mut instant = clock_now.lock().expect("clock mutex"); + let sampled = *instant; + *instant = instant + .checked_add(Duration::from_millis(1)) + .expect("test instant advances"); + sampled + }); + let deadline = Deadline::at_instant( + start + .checked_add(Duration::from_millis(2)) + .expect("deadline"), + ); + let snapshot = ConfigSnapshot { + entries: BTreeMap::from([("large".to_owned(), ConfigValue::from("payload"))]), + }; + let result = block_on(snapshot.get_bounded("large", &clock, deadline, 1, 1)); + assert!(matches!(result, Err(ConfigStoreError::DeadlineExceeded))); + } + + config_store_contract_tests!(appconfig_snapshot_contract, { + ConfigSnapshot { + entries: BTreeMap::from([ + ("contract.key.a".to_owned(), ConfigValue::from("value_a")), + ("contract.key.b".to_owned(), ConfigValue::from("value_b")), + ]), + } + }); +} diff --git a/crates/edgezero-store-aws/src/secrets_manager.rs b/crates/edgezero-store-aws/src/secrets_manager.rs new file mode 100644 index 000000000..b215bc10c --- /dev/null +++ b/crates/edgezero-store-aws/src/secrets_manager.rs @@ -0,0 +1,809 @@ +//! Pinned AWS SDK transport and immutable Secrets Manager snapshots. +//! +//! This module is intentionally private to the native provider crate. Public +//! callers receive only a core `SecretHandle`; neither the SDK client nor the +//! remote resource map survives preparation. + +use std::collections::BTreeMap; +use std::sync::Arc; +use std::time::Duration; + +use aws_config::BehaviorVersion; +use aws_sdk_secretsmanager::Client; +use aws_sdk_secretsmanager::config::timeout::TimeoutConfig; +use aws_sdk_secretsmanager::config::{Region, retry::RetryConfig}; +use aws_sdk_secretsmanager::error::SdkError; +#[cfg(test)] +use aws_sdk_secretsmanager::primitives::Blob; +use bytes::Bytes; +use edgezero_core::secret_store::SecretHandle; +use tokio::time::timeout; + +use crate::PreparationError; +use crate::preparation::{ + AwsPreparation, SecretClient, SecretClientKey, SecretSnapshot, SecretTargetKey, +}; +use crate::settings::{SecretTarget, SecretVersion, SecretsManagerSettings}; + +const MAX_NAMESPACE_BYTES: usize = 512; +const DEFAULT_CLIENT_POLICY: &str = "fixed-default"; + +#[expect( + clippy::multiple_inherent_impl, + reason = "keep the feature-gated Secrets Manager transport in its provider-specific module" +)] +impl AwsPreparation { + /// Load every explicitly mapped secret and return an immutable namespace-bound snapshot. + /// + /// The aggregate budget begins in `AwsPreparation::new`, before credential or SDK + /// initialization. This method adds no outer retry loop; the pinned SDK owns at most + /// three total attempts for each `GetSecretValue` operation. + /// + /// # Errors + /// + /// Returns a sanitized [`PreparationError`] if a binding is invalid, the + /// aggregate deadline expires, the provider fails, or a configured size limit is exceeded. + #[inline] + #[expect( + clippy::too_many_lines, + reason = "the validate-fetch-check-charge-publish sequence is easier to audit as one operation" + )] + pub async fn prepare_secrets( + &mut self, + settings: &SecretsManagerSettings, + namespace: &str, + ) -> Result { + // Validate all portable handle names before constructing a client or doing IO. + if !valid_secret_name(namespace) || namespace.len() > MAX_NAMESPACE_BYTES { + return Err(PreparationError::InvalidBinding); + } + settings.validate()?; + if settings + .entries + .keys() + .any(|key| !valid_secret_name(key) || key.len() > MAX_NAMESPACE_BYTES) + { + return Err(PreparationError::InvalidBinding); + } + + self.check_deadline()?; + for (key, target) in &settings.entries { + self.charge_mapping(key.len(), namespace.len(), target_metadata_bytes(target)?)?; + } + let region = settings.region.clone(); + let client_key = SecretClientKey { + region: region.clone(), + policy: DEFAULT_CLIENT_POLICY.to_owned(), + }; + if !self.secret_clients.contains_key(&client_key) { + let remaining = self.operation_budget()?; + let connect = remaining.min(Duration::from_millis(self.limits.connect_timeout_ms)); + let attempt = remaining.min(Duration::from_millis(self.limits.attempt_timeout_ms)); + let client = timeout(remaining, async { + let shared = aws_config::defaults(BehaviorVersion::v2026_01_12()) + .region(Region::new(region.clone())) + .load() + .await; + self.check_deadline()?; + let config = Client::from_conf( + aws_sdk_secretsmanager::Config::from(&shared) + .to_builder() + .region(Region::new(region.clone())) + .retry_config(RetryConfig::standard().with_max_attempts(3)) + .timeout_config( + TimeoutConfig::builder() + .connect_timeout(connect) + .operation_timeout(remaining) + .operation_attempt_timeout(attempt) + .build(), + ) + .build(), + ); + self.check_deadline()?; + Ok::(config) + }) + .await + .map_err(|_elapsed| PreparationError::Deadline)??; + self.check_deadline()?; + self.secret_clients + .insert(client_key.clone(), SecretClient { client }); + } + + let client = self + .secret_clients + .get(&client_key) + .map(|entry| entry.client.clone()) + .ok_or(PreparationError::Unavailable)?; + let mut entries = BTreeMap::new(); + for (key, target) in &settings.entries { + self.check_deadline()?; + // Charge every logical mapping before target-cache lookup: aliases do not + // bypass the session's configured mapping bound. + let target_key = SecretTargetKey { + region: region.clone(), + client_policy: DEFAULT_CLIENT_POLICY.to_owned(), + secret_id: target.secret_id().to_owned(), + selector: target_selector_identity(target), + }; + let value = if let Some(value) = self.secret_cache.get(&target_key) { + value.clone() + } else { + let operation_budget = self.operation_budget()?; + let attempt_budget = + operation_budget.min(Duration::from_millis(self.limits.attempt_timeout_ms)); + let connect_budget = + operation_budget.min(Duration::from_millis(self.limits.connect_timeout_ms)); + let mut request = client.get_secret_value().secret_id(target.secret_id()); + match target.version() { + SecretVersion::Current => { + // Omission has the service-defined AWSCURRENT meaning. + } + SecretVersion::Stage(stage) => request = request.version_stage(stage), + SecretVersion::VersionId(version) => request = request.version_id(version), + } + let per_operation = TimeoutConfig::builder() + .connect_timeout(connect_budget) + .operation_timeout(operation_budget) + .operation_attempt_timeout(attempt_budget) + .build(); + let result = timeout( + operation_budget, + request + .customize() + .config_override( + aws_sdk_secretsmanager::Config::builder().timeout_config(per_operation), + ) + .send(), + ) + .await + .map_err(|_elapsed| PreparationError::Deadline)?; + self.check_deadline()?; + let output = result.map_err(|error| { + if matches!(&error, SdkError::TimeoutError(_)) { + return PreparationError::Deadline; + } + let service = error.as_service_error().map(|service| { + ( + service.is_decryption_failure(), + service.is_resource_not_found_exception(), + service.is_invalid_request_exception(), + service.is_invalid_parameter_exception(), + service.meta().code().unwrap_or_default().to_owned(), + ) + }); + let status = error + .raw_response() + .map(|response| response.status().as_u16()); + classify_sdk_error(service, status) + })?; + let cap = usize::try_from(self.limits.max_secret_value_bytes) + .map_err(|_error| PreparationError::InvalidLimits)?; + let size = match (output.secret_string(), output.secret_binary()) { + (Some(value), None) => value.len(), + (None, Some(value)) => value.as_ref().len(), + (None, None) | (Some(_), Some(_)) => return Err(PreparationError::Malformed), + }; + self.check_deadline()?; + if size > cap { + return Err(PreparationError::SizeLimit); + } + // SDK response parsing/materialization happens before this check. This + // is a retained-value cap, not an SDK RSS or transfer-allocation bound. + self.check_deadline()?; + self.charge_retained(size)?; + let value = match (output.secret_string, output.secret_binary) { + (Some(value), None) => Bytes::from(value.into_bytes()), + (None, Some(value)) => Bytes::from(value.into_inner()), + (None, None) | (Some(_), Some(_)) => return Err(PreparationError::Malformed), + }; + self.check_deadline()?; + self.secret_cache.insert(target_key, value.clone()); + value + }; + entries.insert(key.clone(), value); + } + + self.check_deadline()?; + Ok(SecretHandle::new(Arc::new(SecretSnapshot { + namespace: namespace.to_owned(), + entries, + }))) + } +} + +fn valid_secret_name(value: &str) -> bool { + !value.is_empty() && value.len() <= MAX_NAMESPACE_BYTES && !value.chars().any(char::is_control) +} + +fn target_selector_identity(target: &SecretTarget) -> String { + match target.version() { + SecretVersion::Current => "stage:AWSCURRENT".to_owned(), + SecretVersion::Stage(value) => format!("stage:{value}"), + SecretVersion::VersionId(value) => format!("version:{value}"), + } +} + +fn target_metadata_bytes(target: &SecretTarget) -> Result { + let selector_bytes = match target.version() { + SecretVersion::Current => "AWSCURRENT".len(), + SecretVersion::Stage(value) | SecretVersion::VersionId(value) => value.len(), + }; + target + .secret_id() + .len() + .checked_add(selector_bytes) + .ok_or(PreparationError::SizeLimit) +} + +#[cfg(test)] +fn materialize_payload( + secret_string: Option<&str>, + secret_binary: Option<&Blob>, +) -> Result { + match (secret_string, secret_binary) { + (Some(value), None) => Ok(Bytes::copy_from_slice(value.as_bytes())), + (None, Some(value)) => Ok(Bytes::copy_from_slice(value.as_ref())), + // Empty strings/blobs are Some(empty) and remain valid. Neither and both + // fields are malformed service responses; never choose a precedence. + (None, None) | (Some(_), Some(_)) => Err(PreparationError::Malformed), + } +} + +fn classify_sdk_error( + service: Option<(bool, bool, bool, bool, String)>, + status: Option, +) -> PreparationError { + if let Some((decryption, not_found, invalid_request, invalid_parameter, code)) = service { + if decryption { + return PreparationError::Decryption; + } + if not_found || invalid_request { + return PreparationError::Missing; + } + if invalid_parameter { + return PreparationError::InvalidResource; + } + match code.as_str() { + "AccessDeniedException" | "AccessDenied" => { + return PreparationError::AccessDenied; + } + "ExpiredTokenException" | "InvalidClientTokenId" | "UnrecognizedClientException" => { + return PreparationError::Credentials; + } + "ThrottlingException" | "Throttling" | "TooManyRequestsException" => { + return PreparationError::Throttled; + } + _ => {} + } + } + if status == Some(403) { + return PreparationError::AccessDenied; + } + if status == Some(429) { + return PreparationError::Throttled; + } + // SDK error/source chains are deliberately consumed and discarded here. + // Credential-process failures are not consistently exposed as a stable + // typed operation error by this SDK version; without an unredacted chain + // they conservatively collapse to the closed unavailable category. + PreparationError::Unavailable +} + +#[cfg(test)] +#[expect( + clippy::absolute_paths, + reason = "explicit std and crate paths keep the isolated child-process fixture auditable" +)] +mod tests { + use super::materialize_payload; + use crate::PreparationError; + use aws_sdk_secretsmanager::operation::get_secret_value::GetSecretValueOutput; + use aws_sdk_secretsmanager::primitives::Blob; + use bytes::Bytes; + use std::io::{Read as _, Write as _}; + use std::net::{TcpListener, TcpStream}; + use std::process::{Command, Stdio}; + use std::sync::atomic::{AtomicU64, Ordering}; + use std::thread; + use std::time::{Duration, Instant}; + + const TEST_ARN: &str = "arn:aws:secretsmanager:us-east-1:111122223333:secret:fixture-AbCdEf"; + static NEXT_FIXTURE: AtomicU64 = AtomicU64::new(0); + + #[derive(Clone)] + enum FixtureResponse { + Decryption, + Delay(Duration), + Denied, + InternalError, + Json(String), + Missing, + Oversize, + Success, + } + + fn assert_sdk_fixture_scope() { + let endpoint = + std::env::var("AWS_ENDPOINT_URL_SECRETS_MANAGER").expect("fixture endpoint required"); + let endpoint_url = url::Url::parse(&endpoint).expect("fixture URL"); + assert!( + endpoint_url.scheme() == "http" + && endpoint_url.host_str() == Some("127.0.0.1") + && endpoint_url.port().is_some(), + "SDK child accepts only the literal-loopback fixture" + ); + assert!( + std::env::var("AWS_ACCESS_KEY_ID").is_ok_and(|value| value.starts_with("DUMMY")), + "SDK child requires dummy credentials" + ); + } + + fn respond(mut stream: TcpStream, response: FixtureResponse) -> (String, String) { + stream + .set_read_timeout(Some(Duration::from_secs(3))) + .expect("request read timeout"); + let mut request = Vec::new(); + let mut chunk = [0_u8; 4096]; + let header_end = loop { + let read = stream.read(&mut chunk).expect("read request headers"); + assert_ne!(read, 0, "SDK closed before request headers"); + request.extend_from_slice(&chunk[..read]); + if let Some(end) = request.windows(4).position(|bytes| bytes == b"\r\n\r\n") { + break end + .checked_add(4) + .expect("header terminator offset fits request"); + } + assert!(request.len() < 64 * 1024, "bounded fixture request headers"); + }; + let headers = String::from_utf8_lossy(&request[..header_end]).to_ascii_lowercase(); + let content_length = headers + .lines() + .find_map(|line| line.strip_prefix("content-length:").map(str::trim)) + .and_then(|value| value.parse::().ok()) + .expect("SDK content length"); + let body_end = header_end + .checked_add(content_length) + .expect("bounded request body end fits usize"); + while request.len() < body_end { + let read = stream.read(&mut chunk).expect("read request body"); + assert_ne!(read, 0, "SDK closed before request body"); + request.extend_from_slice(&chunk[..read]); + } + let body = String::from_utf8_lossy(&request[header_end..body_end]).into_owned(); + assert!(headers.contains("x-amz-target: secretsmanager.getsecretvalue")); + assert!( + body.contains(TEST_ARN), + "request must contain only the fictional fixture ARN" + ); + if let FixtureResponse::Delay(delay) = response { + thread::sleep(delay); + let payload = b"{\"SecretString\":\"VALUE_SENTINEL\"}"; + let _header_write = write!( + stream, + "HTTP/1.1 200 OK\r\nContent-Type: application/x-amz-json-1.1\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + payload.len() + ); + let _body_write = stream.write_all(payload); + return (headers, body); + } + let (status, code, payload) = match response { + FixtureResponse::Decryption => (400_u16, "Bad Request", r#"{"__type":"DecryptionFailure","message":"VALUE_SENTINEL"}"#.to_owned()), + FixtureResponse::Denied => (400_u16, "Bad Request", r#"{"__type":"AccessDeniedException","message":"ARN_SENTINEL CREDENTIAL_SENTINEL"}"#.to_owned()), + FixtureResponse::Delay(_) => return (headers, body), + FixtureResponse::InternalError => (500_u16, "Internal Server Error", r#"{"__type":"InternalServiceError","message":"SDK_ERROR_SENTINEL"}"#.to_owned()), + FixtureResponse::Json(payload) => (200_u16, "OK", payload), + FixtureResponse::Missing => (400_u16, "Bad Request", r#"{"__type":"ResourceNotFoundException","message":"ARN_SENTINEL"}"#.to_owned()), + FixtureResponse::Oversize => (200_u16, "OK", format!(r#"{{"ARN":"{TEST_ARN}","Name":"fixture","VersionId":"12345678901234567890123456789012","SecretString":"{}"}}"#, "x".repeat(0x0001_0001))), + FixtureResponse::Success => (200_u16, "OK", format!(r#"{{"ARN":"{TEST_ARN}","Name":"fixture","VersionId":"12345678901234567890123456789012","SecretString":"VALUE_SENTINEL"}}"#)), + }; + let response_body = payload.as_bytes(); + write!(stream, "HTTP/1.1 {status} {code}\r\nContent-Type: application/x-amz-json-1.1\r\nx-amzn-RequestId: fixture-request-id\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", response_body.len()).expect("write response headers"); + stream + .write_all(response_body) + .expect("write response body"); + (headers, body) + } + + fn loopback_fixture( + responses: Vec, + ) -> (String, thread::JoinHandle>) { + let listener = TcpListener::bind("127.0.0.1:0").expect("bind literal loopback only"); + listener + .set_nonblocking(true) + .expect("nonblocking listener"); + let address = listener.local_addr().expect("loopback address"); + let endpoint = format!("http://{address}"); + let worker = thread::spawn(move || { + let deadline = Instant::now() + .checked_add(Duration::from_secs(12)) + .expect("fixture deadline fits the monotonic clock"); + let mut bodies = Vec::new(); + for response in responses { + let stream = loop { + match listener.accept() { + Ok((stream, peer)) => { + assert!(peer.ip().is_loopback(), "fixture peer must be loopback"); + break stream; + } + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { + assert!(Instant::now() < deadline, "SDK did not contact fixture"); + thread::sleep(Duration::from_millis(5)); + } + Err(error) => panic!("fixture accept failed: {error}"), + } + }; + stream + .set_write_timeout(Some(Duration::from_secs(3))) + .expect("response write timeout"); + let (_, body) = respond(stream, response); + bodies.push(body); + } + bodies + }); + (endpoint, worker) + } + + fn isolated_sdk_child(endpoint: &str, scenario: &str) -> std::process::Output { + let directory = std::env::temp_dir().join(format!( + "edgezero-secrets-fixture-{}-{}", + std::process::id(), + NEXT_FIXTURE.fetch_add(1, Ordering::Relaxed), + )); + std::fs::create_dir_all(&directory).expect("create isolated profile directory"); + let config = directory.join("config"); + let credentials = directory.join("credentials"); + std::fs::write(&config, "[profile fixture]\nregion = us-east-1\n") + .expect("write isolated config"); + std::fs::write( + &credentials, + "[fixture]\naws_access_key_id = DUMMY_ACCESS_KEY\naws_secret_access_key = DUMMY_SECRET_KEY\n", + ) + .expect("write dummy credentials"); + let output = Command::new(std::env::current_exe().expect("test binary")) + .args([ + "--exact", + "secrets_manager::tests::sdk_child_entry", + "--nocapture", + ]) + .env_clear() + .env("HOME", &directory) + .env("AWS_CONFIG_FILE", config) + .env("AWS_SHARED_CREDENTIALS_FILE", credentials) + .env("AWS_PROFILE", "fixture") + .env("AWS_ACCESS_KEY_ID", "DUMMY_ACCESS_KEY") + .env("AWS_SECRET_ACCESS_KEY", "DUMMY_SECRET_KEY") + .env("AWS_REGION", "us-east-1") + .env("AWS_EC2_METADATA_DISABLED", "true") + .env("AWS_ENDPOINT_URL_SECRETS_MANAGER", endpoint) + .env("AWS_RETRY_MODE", "standard") + .env("AWS_MAX_ATTEMPTS", "3") + .env("AWS_IGNORE_CONFIGURED_ENDPOINT_URLS", "false") + .env("EDGEZERO_SECRETS_SCENARIO", scenario) + .stdin(Stdio::null()) + .output() + .expect("run isolated SDK child"); + std::fs::remove_dir_all(directory).expect("remove isolated profile directory"); + output + } + + #[test] + fn sdk_blob_bytes_are_preserved_without_a_second_base64_decode() { + let binary = Blob::new(b"YWJjZA==".to_vec()); + assert_eq!( + materialize_payload(None, Some(&binary)), + Ok(Bytes::from_static(b"YWJjZA==")) + ); + assert_eq!(materialize_payload(Some(""), None), Ok(Bytes::new())); + assert_eq!( + materialize_payload(None, Some(&Blob::new(Vec::new()))), + Ok(Bytes::new()) + ); + assert_eq!( + materialize_payload(None, None), + Err(PreparationError::Malformed) + ); + assert_eq!( + materialize_payload(Some("x"), Some(&binary)), + Err(PreparationError::Malformed) + ); + } + + #[test] + fn real_sdk_uses_child_only_service_endpoint_and_capped_retries() { + let (endpoint, fixture) = loopback_fixture(vec![ + FixtureResponse::InternalError, + FixtureResponse::InternalError, + FixtureResponse::Success, + ]); + let child = isolated_sdk_child(&endpoint, "retry"); + let requests = fixture.join().expect("fixture thread"); + assert!( + child.status.success(), + "isolated SDK child failed: {}", + String::from_utf8_lossy(&child.stderr) + ); + assert_eq!( + requests.len(), + 3, + "standard SDK performs at most three total attempts" + ); + assert!(requests.iter().all(|body| body.contains(TEST_ARN))); + } + + #[test] + fn real_sdk_serializes_exact_current_stage_and_version_selectors() { + for (scenario, selector) in [ + ("current", ""), + ("stage", r#""VersionStage":"BLUE""#), + ( + "version", + r#""VersionId":"12345678901234567890123456789012""#, + ), + ] { + let (endpoint, fixture) = loopback_fixture(vec![FixtureResponse::Success]); + let child = isolated_sdk_child(&endpoint, scenario); + let requests = fixture.join().expect("fixture thread"); + assert!( + child.status.success(), + "isolated SDK child failed for {scenario}: {}", + String::from_utf8_lossy(&child.stderr) + ); + assert_eq!(requests.len(), 1); + assert!(requests[0].contains(TEST_ARN)); + if !selector.is_empty() { + assert!( + requests[0].contains(selector), + "selector serialized exactly" + ); + } + if scenario == "current" { + assert!( + !requests[0].contains("VersionStage") && !requests[0].contains("VersionId") + ); + } + } + } + + #[test] + fn real_sdk_service_failures_are_closed_and_non_retryable() { + for (response, scenario, category) in [ + (FixtureResponse::Denied, "denied", "access denied"), + ( + FixtureResponse::Decryption, + "decryption", + "decryption failure", + ), + ( + FixtureResponse::Missing, + "missing", + "missing resource or version", + ), + ] { + let (endpoint, fixture) = loopback_fixture(vec![response]); + let child = isolated_sdk_child(&endpoint, scenario); + let requests = fixture.join().expect("fixture thread"); + assert!( + child.status.success(), + "isolated child rejected expected category: {}", + String::from_utf8_lossy(&child.stderr) + ); + assert_eq!( + requests.len(), + 1, + "non-retryable service failures use one SDK attempt" + ); + let output = format!( + "{}{}", + String::from_utf8_lossy(&child.stdout), + String::from_utf8_lossy(&child.stderr) + ); + assert!( + output.contains(category), + "child reports only its closed category" + ); + for sentinel in [ + "ARN_SENTINEL", + "VALUE_SENTINEL", + "CREDENTIAL_SENTINEL", + "SDK_ERROR_SENTINEL", + ] { + assert!( + !output.contains(sentinel), + "public child output excludes source details" + ); + } + } + } + + #[test] + fn real_sdk_binary_empty_and_exactly_one_payload_fields() { + for (scenario, payload) in [ + ( + "binary", + serde_json::json!({"SecretBinary": "WW1sdVlYSjU="}), + ), + ("empty", serde_json::json!({"SecretString": ""})), + ( + "both", + serde_json::json!({"SecretString": "opaque", "SecretBinary": ""}), + ), + ("neither", serde_json::json!({})), + ( + "dedup", + serde_json::json!({"SecretString": "VALUE_SENTINEL"}), + ), + ] { + let (endpoint, fixture) = + loopback_fixture(vec![FixtureResponse::Json(payload.to_string())]); + let child = isolated_sdk_child(&endpoint, scenario); + let requests = fixture.join().expect("fixture thread"); + assert!( + child.status.success(), + "payload child failed: {}", + String::from_utf8_lossy(&child.stderr) + ); + assert_eq!( + requests.len(), + 1, + "exactly one service request despite aliases/repeated snapshot reads" + ); + } + } + + #[test] + fn real_sdk_payload_cap_rejects_materialized_oversize_value() { + let (endpoint, fixture) = loopback_fixture(vec![FixtureResponse::Oversize]); + let child = isolated_sdk_child(&endpoint, "oversize"); + let requests = fixture.join().expect("fixture thread"); + assert!( + child.status.success(), + "oversize child failed: {}", + String::from_utf8_lossy(&child.stderr) + ); + assert_eq!(requests.len(), 1); + let output = format!( + "{}{}", + String::from_utf8_lossy(&child.stdout), + String::from_utf8_lossy(&child.stderr) + ); + assert!(output.contains("size limit")); + assert!( + !output.contains(&"x".repeat(128)), + "payload is not included in public output" + ); + } + + #[test] + fn real_sdk_attempt_timeout_is_observed_without_an_outer_retry_loop() { + let (endpoint, fixture) = + loopback_fixture(vec![FixtureResponse::Delay(Duration::from_millis(350))]); + let child = isolated_sdk_child(&endpoint, "timeout"); + let requests = fixture.join().expect("fixture thread"); + assert!( + child.status.success(), + "timeout child failed: {}", + String::from_utf8_lossy(&child.stderr) + ); + assert_eq!( + requests.len(), + 1, + "SDK timeout behavior is inside the one provider operation" + ); + } + + /// Invoked only by an env-cleared child process launched by the parent fixture. + #[expect( + clippy::err_expect, + reason = "the opaque SecretHandle success value intentionally has no Debug API" + )] + #[expect( + clippy::print_stdout, + reason = "the child reports its closed error category to the parent fixture" + )] + #[tokio::test] + async fn sdk_child_entry() { + let Ok(scenario) = std::env::var("EDGEZERO_SECRETS_SCENARIO") else { + return; + }; + assert_sdk_fixture_scope(); + let version = match scenario.as_str() { + "stage" => crate::settings::SecretVersion::Stage("BLUE".to_owned()), + "version" => crate::settings::SecretVersion::VersionId( + "12345678901234567890123456789012".to_owned(), + ), + _ => crate::settings::SecretVersion::Current, + }; + let target = crate::settings::SecretTarget::new(TEST_ARN.to_owned(), version) + .expect("valid fictional ARN"); + let settings = crate::settings::SecretsManagerSettings { + region: "us-east-1".to_owned(), + entries: std::collections::BTreeMap::from([("active".to_owned(), target)]), + }; + let limits = if scenario == "timeout" { + crate::settings::AwsPreparationLimits { + connect_timeout_ms: 50, + attempt_timeout_ms: 100, + operation_timeout_ms: 500, + preparation_timeout_ms: 2_000, + ..crate::settings::AwsPreparationLimits::default() + } + } else { + crate::settings::AwsPreparationLimits::default() + }; + let mut session = crate::preparation::AwsPreparation::new( + edgezero_core::time::MonotonicClock::default(), + limits, + ) + .expect("valid session"); + let result = session.prepare_secrets(&settings, "runtime").await; + match scenario.as_str() { + "retry" | "current" | "stage" | "version" | "dedup" => { + let handle = result.expect("fixture-backed mapped secret prepares"); + let bytes = handle + .get_bytes("runtime", "active") + .await + .expect("snapshot read") + .expect("mapped value"); + assert_eq!(bytes.as_ref(), b"VALUE_SENTINEL"); + if scenario == "dedup" { + let cached = session + .prepare_secrets(&settings, "second") + .await + .expect("success cache shared across bound namespaces"); + assert_eq!( + cached + .get_bytes("second", "active") + .await + .expect("read") + .expect("mapped") + .as_ref(), + b"VALUE_SENTINEL" + ); + assert_eq!(session.stats().mappings, 2_u64); + assert_eq!(session.stats().unique_retained_payload_bytes, 14_u64); + } + } + "binary" | "empty" => { + let handle = result.expect("payload snapshot"); + let bytes = handle + .get_bytes("runtime", "active") + .await + .expect("snapshot") + .expect("mapped empty value is not a miss"); + let expected: &[u8] = if scenario == "binary" { + b"YmluYXJ5" + } else { + b"" + }; + assert_eq!(bytes.as_ref(), expected); + } + "both" | "neither" => { + assert_eq!( + result.err().expect("exactly one payload field required"), + crate::PreparationError::Malformed + ); + } + "denied" | "decryption" | "missing" | "timeout" | "oversize" => { + let expected = match scenario.as_str() { + "denied" => crate::PreparationError::AccessDenied, + "decryption" => crate::PreparationError::Decryption, + "missing" => crate::PreparationError::Missing, + "timeout" => crate::PreparationError::Deadline, + "oversize" => crate::PreparationError::SizeLimit, + _ => panic!("unknown failure scenario"), + }; + let error = result.err().expect("failed preparation must not publish"); + assert_eq!(error, expected); + println!("closed-category: {error}"); + } + _ => panic!("unknown child scenario"), + } + } + + #[test] + fn response_builder_exposes_the_pinned_sdk_fields_used_by_transport() { + let output = GetSecretValueOutput::builder().secret_string("").build(); + assert_eq!(output.secret_string(), Some("")); + assert!(output.secret_binary().is_none()); + } +} diff --git a/crates/edgezero-store-aws/src/settings.rs b/crates/edgezero-store-aws/src/settings.rs new file mode 100644 index 000000000..162902228 --- /dev/null +++ b/crates/edgezero-store-aws/src/settings.rs @@ -0,0 +1,998 @@ +//! SDK-free typed settings and validation. Resource values never enter diagnostics. + +#![expect( + clippy::arbitrary_source_item_ordering, + reason = "settings, validators and their tests are grouped by provider" +)] + +use std::collections::BTreeMap; +use std::fmt; +use std::net::IpAddr; +use std::time::{Duration, Instant}; + +use percent_encoding::percent_decode_str; +use serde::de::Error as _; +use serde::{Deserialize, Deserializer}; +use url::Url; + +use crate::PreparationError; + +// Portable SecretHandle limits names to 512 UTF-8 bytes. Keep SDK-free settings +// independent of core; service implementations must use the same core bound. +const MAX_SECRET_KEY_BYTES: usize = 512; + +/// One configuration key maps to one complete free-form Agent document. +#[derive(Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct AgentDocument { + pub application: String, + pub environment: String, + pub profile: String, +} + +impl fmt::Debug for AgentDocument { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("AgentDocument").finish_non_exhaustive() + } +} + +/// `AppConfig` Agent settings. The token field names a bootstrap variable, not a value. +#[derive(Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct AppConfigAgentSettings { + pub default_key: String, + pub endpoint: String, + pub access_token_env: Option, + pub documents: BTreeMap, +} + +impl fmt::Debug for AppConfigAgentSettings { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("AppConfigAgentSettings") + .finish_non_exhaustive() + } +} + +impl AppConfigAgentSettings { + /// Builds an exact Agent request URL using checked path segments. + /// + /// # Errors + /// Rejects ambiguous/non-loopback endpoints and unsafe document identifiers. + #[inline] + pub fn document_url(&self, document: &AgentDocument) -> Result { + let mut url = agent_endpoint(&self.endpoint)?; + let expected = [ + "applications", + document.application.as_str(), + "environments", + document.environment.as_str(), + "configurations", + document.profile.as_str(), + ]; + for segment in [ + &document.application, + &document.environment, + &document.profile, + ] { + if !valid_agent_segment(segment) { + return Err(PreparationError::InvalidResource); + } + } + url.path_segments_mut() + .map_err(|_error| PreparationError::InvalidEndpoint)? + .clear() + .extend(expected); + let reparsed = + Url::parse(url.as_str()).map_err(|_error| PreparationError::InvalidEndpoint)?; + if !reparsed.path_segments().is_some_and(|segments| { + segments + .map(|segment| percent_decode_str(segment).decode_utf8()) + .eq(expected.into_iter().map(|segment| Ok(segment.into()))) + }) { + return Err(PreparationError::InvalidResource); + } + Ok(reparsed) + } + + /// Validates every mapping without HTTP or bootstrap environment access. + /// + /// # Errors + /// Returns a closed category for invalid endpoint, mapping, default or token name. + #[inline] + pub fn validate(&self) -> Result<(), PreparationError> { + agent_endpoint(&self.endpoint)?; + if self.documents.is_empty() + || !self.documents.contains_key(&self.default_key) + || self.documents.keys().any(|key| !valid_lookup_key(key)) + || self + .access_token_env + .as_deref() + .is_some_and(|name| !valid_variable_name(name)) + { + return Err(PreparationError::InvalidBinding); + } + for document in self.documents.values() { + self.document_url(document)?; + } + Ok(()) + } +} + +/// A bootstrap token. Debug hides both the token and its length. +#[derive(Clone)] +pub struct AgentAccessToken(String); + +impl AgentAccessToken { + /// Captures an explicit nonempty bearer token without reading the environment. + /// + /// # Errors + /// Rejects missing/empty tokens and bytes that cannot be sent as a bearer value. + #[inline] + pub fn new(value: String) -> Result { + if value.is_empty() || !value.bytes().all(|byte| byte.is_ascii_graphic()) { + return Err(PreparationError::InvalidBinding); + } + Ok(Self(value)) + } + + /// Exposes the token only to the provider transport. + #[inline] + #[must_use] + pub fn expose(&self) -> &str { + &self.0 + } +} + +impl fmt::Debug for AgentAccessToken { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("AgentAccessToken([redacted])") + } +} + +/// Independent policy for live `DynamoDB` KV operations, not snapshot retention. +#[derive(Clone, Copy, Debug, Deserialize, Eq, Hash, PartialEq)] +#[serde(default, deny_unknown_fields)] +pub struct DynamoDbKvLimits { + pub max_value_bytes: usize, + pub operation_timeout_ms: u64, + pub connect_timeout_ms: u64, + pub attempt_timeout_ms: u64, +} + +impl Default for DynamoDbKvLimits { + #[inline] + fn default() -> Self { + Self { + max_value_bytes: 393_216, + operation_timeout_ms: 5_000, + connect_timeout_ms: 1_000, + attempt_timeout_ms: 2_000, + } + } +} + +impl DynamoDbKvLimits { + /// Checks payload bounds and representable, ordered operation budgets. + /// + /// # Errors + /// Returns a closed category for invalid policy. + #[inline] + pub fn validate(&self) -> Result<(), PreparationError> { + if self.max_value_bytes == 0 + || self.max_value_bytes > 393_216 + || self.connect_timeout_ms == 0 + || self.connect_timeout_ms > self.attempt_timeout_ms + || self.attempt_timeout_ms > self.operation_timeout_ms + || Instant::now() + .checked_add(Duration::from_millis(self.operation_timeout_ms)) + .is_none() + { + return Err(PreparationError::InvalidLimits); + } + Ok(()) + } +} + +/// Explicit physical `DynamoDB` address. Debug never exposes resource names. +#[derive(Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct DynamoDbKvSettings { + pub region: String, + pub table: String, + pub namespace: String, + #[serde(default)] + pub limits: DynamoDbKvLimits, +} + +impl fmt::Debug for DynamoDbKvSettings { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("DynamoDbKvSettings").finish_non_exhaustive() + } +} + +impl DynamoDbKvSettings { + /// Validates an address without credential discovery or service I/O. + /// + /// # Errors + /// Rejects malformed region/table/namespace and invalid limits. + #[inline] + pub fn validate(&self) -> Result<(), PreparationError> { + self.limits.validate()?; + let partition = if self.region.starts_with("cn-") { + "aws-cn" + } else if self.region.starts_with("us-gov-") { + "aws-us-gov" + } else { + "aws" + }; + if self.region.len() > 64 + || !valid_region(partition, &self.region) + || !(3..=255).contains(&self.table.len()) + || !self + .table + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'-' | b'.')) + || self.namespace.is_empty() + || self.namespace.len() > 512 + || self.namespace.chars().any(char::is_control) + { + return Err(PreparationError::InvalidResource); + } + Ok(()) + } +} + +/// Exactly one version selector. Omission means AWSCURRENT at startup. +/// API lengths count characters; non-control Unicode and embedded spaces retain +/// their exact meaning. Validation does not assert that a version/stage exists. +#[derive(Clone, Eq, Ord, PartialEq, PartialOrd)] +pub enum SecretVersion { + Current, + Stage(String), + VersionId(String), +} + +impl SecretVersion { + fn validate(&self) -> Result<(), PreparationError> { + let (value, minimum, maximum) = match self { + Self::Current => return Ok(()), + Self::Stage(value) => (value, 1, 256), + Self::VersionId(value) => (value, 32, 64), + }; + let length = value.chars().count(); + if !(minimum..=maximum).contains(&length) + || value.chars().all(char::is_whitespace) + || value.chars().any(char::is_control) + { + return Err(PreparationError::InvalidSelector); + } + Ok(()) + } +} + +impl fmt::Debug for SecretVersion { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("SecretVersion([redacted])") + } +} + +/// A complete ARN and a validated selector. Construction never discovers credentials. +#[derive(Clone, Eq, Ord, PartialEq, PartialOrd)] +pub struct SecretTarget { + secret_id: String, + version: SecretVersion, +} + +impl SecretTarget { + /// Validates a complete ARN and mutually exclusive typed selector. + /// + /// # Errors + /// Rejects incomplete/unsupported resources and invalid selector lengths. + #[inline] + pub fn new(secret_id: String, version: SecretVersion) -> Result { + secret_region(&secret_id)?; + version.validate()?; + let selected = match version { + SecretVersion::Stage(stage) if stage == "AWSCURRENT" => SecretVersion::Current, + other @ (SecretVersion::Current + | SecretVersion::Stage(_) + | SecretVersion::VersionId(_)) => other, + }; + Ok(Self { + secret_id, + version: selected, + }) + } + + /// Complete resource identifier for the service request, never diagnostics. + #[inline] + #[must_use] + pub fn secret_id(&self) -> &str { + &self.secret_id + } + + /// Exact version selection for the service request. + #[inline] + #[must_use] + pub fn version(&self) -> &SecretVersion { + &self.version + } +} + +impl fmt::Debug for SecretTarget { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("SecretTarget").finish_non_exhaustive() + } +} + +impl<'de> Deserialize<'de> for SecretTarget { + #[inline] + fn deserialize>(deserializer: D) -> Result { + #[derive(Deserialize)] + #[serde(deny_unknown_fields)] + struct Input { + secret_id: String, + version_id: Option, + version_stage: Option, + } + let input = Input::deserialize(deserializer)?; + let version = match (input.version_id, input.version_stage) { + (None, None) => SecretVersion::Current, + (None, Some(stage)) => SecretVersion::Stage(stage), + (Some(id), None) => SecretVersion::VersionId(id), + (Some(_), Some(_)) => { + return Err(D::Error::custom(PreparationError::InvalidSelector)); + } + }; + Self::new(input.secret_id, version).map_err(D::Error::custom) + } + + #[inline] + fn deserialize_in_place>( + deserializer: D, + place: &mut Self, + ) -> Result<(), D::Error> { + *place = Self::deserialize(deserializer)?; + Ok(()) + } +} + +/// Explicit region and finite secret key mappings. Namespaces are supplied by the host. +#[derive(Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SecretsManagerSettings { + pub region: String, + pub entries: BTreeMap, +} + +impl SecretsManagerSettings { + /// Validates explicit region, keys and every ARN's region/partition relationship. + /// + /// # Errors + /// Rejects empty mappings, invalid keys, missing regions and mismatched ARNs. + #[inline] + pub fn validate(&self) -> Result<(), PreparationError> { + // Match the portable SecretHandle byte limit without adding core to the + // settings-only graph. Provider code must also enforce core's bound. + if self.entries.is_empty() + || self + .entries + .keys() + .any(|key| !valid_lookup_key(key) || key.len() > MAX_SECRET_KEY_BYTES) + { + return Err(PreparationError::InvalidBinding); + } + for target in self.entries.values() { + if secret_region(target.secret_id())? != self.region { + return Err(PreparationError::InvalidResource); + } + } + Ok(()) + } +} + +impl fmt::Debug for SecretsManagerSettings { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("SecretsManagerSettings") + .finish_non_exhaustive() + } +} + +/// Remote preparation policy for one explicitly shared session, not a process/RSS cap. +/// Omitted fields use documented defaults from this single policy source. +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq)] +#[serde(default, deny_unknown_fields)] +pub struct AwsPreparationLimits { + pub max_remote_values: u64, + pub max_config_document_bytes: u64, + pub max_secret_value_bytes: u64, + pub max_snapshot_bytes: u64, + pub preparation_timeout_ms: u64, + pub operation_timeout_ms: u64, + pub connect_timeout_ms: u64, + pub attempt_timeout_ms: u64, +} + +impl Default for AwsPreparationLimits { + #[inline] + fn default() -> Self { + Self { + max_remote_values: 1_024, + max_config_document_bytes: 1_024 * 1_024, + max_secret_value_bytes: 64 * 1_024, + max_snapshot_bytes: 8 * 1_024 * 1_024, + preparation_timeout_ms: 30_000, + operation_timeout_ms: 5_000, + connect_timeout_ms: 1_000, + attempt_timeout_ms: 2_000, + } + } +} + +impl AwsPreparationLimits { + /// Validates finite representable bounds and consistency, without hardcoding the defaults as maxima. + /// + /// # Errors + /// Rejects zero, unusable timer/counter values and inconsistent limits. + #[inline] + pub fn validate(&self) -> Result<(), PreparationError> { + for value in [ + self.max_remote_values, + self.max_config_document_bytes, + self.max_secret_value_bytes, + self.max_snapshot_bytes, + ] { + if value == 0 || usize::try_from(value).is_err() { + return Err(PreparationError::InvalidLimits); + } + } + for millis in [ + self.preparation_timeout_ms, + self.operation_timeout_ms, + self.connect_timeout_ms, + self.attempt_timeout_ms, + ] { + if millis == 0 + || Instant::now() + .checked_add(Duration::from_millis(millis)) + .is_none() + { + return Err(PreparationError::InvalidLimits); + } + } + if self.max_config_document_bytes > self.max_snapshot_bytes + || self.max_secret_value_bytes > self.max_snapshot_bytes + || self.connect_timeout_ms > self.attempt_timeout_ms + || self.attempt_timeout_ms > self.operation_timeout_ms + || self.operation_timeout_ms > self.preparation_timeout_ms + { + return Err(PreparationError::InvalidLimits); + } + Ok(()) + } +} + +fn valid_lookup_key(key: &str) -> bool { + !key.is_empty() && !key.chars().all(char::is_whitespace) && !key.chars().any(char::is_control) +} + +fn valid_variable_name(name: &str) -> bool { + let mut bytes = name.bytes(); + bytes + .next() + .is_some_and(|byte| byte.is_ascii_alphabetic() || byte == b'_') + && bytes.all(|byte| byte.is_ascii_alphanumeric() || byte == b'_') +} + +fn valid_agent_segment(segment: &str) -> bool { + !segment.is_empty() + && segment.len() <= 128 + && !matches!(segment, "." | "..") + && !segment.chars().any(|character| { + character.is_control() || matches!(character, '/' | '\\' | '%' | '?' | '#') + }) +} + +fn agent_endpoint(endpoint: &str) -> Result { + // Check the original authority too: URL parsing otherwise accepts integer/octal + // IPv4, backslashes and dot paths while changing their spelling/meaning. + let rest = endpoint + .strip_prefix("http://") + .ok_or(PreparationError::InvalidEndpoint)?; + let authority = rest.strip_suffix('/').unwrap_or(rest); + if authority.contains(['/', '\\', '@', '?', '#', '%']) + || endpoint + .bytes() + .any(|byte| byte.is_ascii_whitespace() || byte.is_ascii_control()) + { + return Err(PreparationError::InvalidEndpoint); + } + let host = if let Some(ipv6) = authority.strip_prefix('[') { + ipv6.split_once(']').map(|(host, _port)| host) + } else { + Some( + authority + .split_once(':') + .map_or(authority, |(host, _port)| host), + ) + } + .ok_or(PreparationError::InvalidEndpoint)?; + let literal = host + .parse::() + .map_err(|_error| PreparationError::InvalidEndpoint)?; + let url = Url::parse(endpoint).map_err(|_error| PreparationError::InvalidEndpoint)?; + if !literal.is_loopback() + || url.scheme() != "http" + || !url.username().is_empty() + || url.password().is_some() + || url.query().is_some() + || url.fragment().is_some() + || url.path() != "/" + || url.port_or_known_default().is_none_or(|port| port == 0) + || url.host_str().is_none() + { + return Err(PreparationError::InvalidEndpoint); + } + Ok(url) +} + +fn secret_region(arn: &str) -> Result<&str, PreparationError> { + if arn.len() > 2_048 { + return Err(PreparationError::InvalidResource); + } + let parts: Vec<&str> = arn.split(':').collect(); + let [prefix, partition, service, region, account, resource, name] = parts.as_slice() else { + return Err(PreparationError::InvalidResource); + }; + let Some((secret_name, suffix)) = name.rsplit_once('-') else { + return Err(PreparationError::InvalidResource); + }; + if *prefix != "arn" + || *service != "secretsmanager" + || *resource != "secret" + || account.len() != 12 + || !account.bytes().all(|byte| byte.is_ascii_digit()) + || secret_name.is_empty() + || secret_name.len() > 512 + || !secret_name.bytes().all(|byte| { + byte.is_ascii_alphanumeric() + || matches!(byte, b'/' | b'_' | b'+' | b'=' | b'.' | b'@' | b'-') + }) + || suffix.len() != 6 + || !suffix.bytes().all(|byte| byte.is_ascii_alphanumeric()) + || arn.len() > 2_048 + || !valid_region(partition, region) + { + return Err(PreparationError::InvalidResource); + } + Ok(region) +} + +fn valid_region(partition: &str, region: &str) -> bool { + let Some((family, number)) = region.rsplit_once('-') else { + return false; + }; + if number.is_empty() + || !number.bytes().all(|byte| byte.is_ascii_digit()) + || number.starts_with('0') + || !family + .split('-') + .all(|part| !part.is_empty() && part.bytes().all(|byte| byte.is_ascii_lowercase())) + { + return false; + } + // Supported partition families, not an assertion that an arbitrary matching + // region is currently served. Isolated/sovereign partitions are not v1 support. + match partition { + "aws-cn" => family.starts_with("cn-"), + "aws-us-gov" => family.starts_with("us-gov-"), + "aws" => { + [ + "af-", "ap-", "ca-", "eu-", "il-", "me-", "mx-", "sa-", "us-", + ] + .iter() + .any(|prefix| family.starts_with(prefix)) + && !family.starts_with("us-gov-") + && !family.starts_with("us-iso") + } + _ => false, + } +} + +#[cfg(test)] +mod tests { + #![expect( + clippy::panic_in_result_fn, + reason = "test functions combine checked setup with behavioral assertions" + )] + #![expect( + clippy::assertions_on_result_states, + reason = "malformed input matrix observes parser acceptance without unwrapping" + )] + + #[test] + fn dynamodb_settings_are_strict_bounded_and_redacted() { + let settings: super::DynamoDbKvSettings = toml::from_str( + "region='us-east-1'\ntable='edgezero-kv'\nnamespace='private-namespace'", + ) + .unwrap(); + settings.validate().unwrap(); + assert!(!format!("{settings:?}").contains("private-namespace")); + assert!(toml::from_str::( + "region='us-east-1'\ntable='edgezero-kv'\nnamespace='x'\nendpoint='http://example.com'" + ).is_err()); + let mut invalid = settings; + invalid.limits.max_value_bytes = 393_217; + assert_eq!( + invalid.validate(), + Err(super::PreparationError::InvalidLimits) + ); + invalid.limits = super::DynamoDbKvLimits::default(); + invalid.namespace = "\n".to_owned(); + assert_eq!( + invalid.validate(), + Err(super::PreparationError::InvalidResource) + ); + } + + use std::borrow::Cow; + use std::collections::BTreeMap; + + use percent_encoding::percent_decode_str; + use url::Url; + + use super::{ + AgentAccessToken, AgentDocument, AppConfigAgentSettings, AwsPreparationLimits, + SecretTarget, SecretVersion, SecretsManagerSettings, + }; + use crate::PreparationError; + + fn agent(endpoint: &str) -> AppConfigAgentSettings { + AppConfigAgentSettings { + default_key: "settings".to_owned(), + endpoint: endpoint.to_owned(), + access_token_env: None, + documents: BTreeMap::from([( + "settings".to_owned(), + AgentDocument { + application: "example-app".to_owned(), + environment: "production".to_owned(), + profile: "application-envelope".to_owned(), + }, + )]), + } + } + + #[test] + fn literal_loopback_and_exact_request_path() -> Result<(), PreparationError> { + for endpoint in [ + "http://127.0.0.1:2772", + "http://127.1.2.3:2772/", + "http://[::1]:2772", + ] { + let settings = agent(endpoint); + settings.validate()?; + let document = settings + .documents + .get("settings") + .ok_or(PreparationError::InvalidBinding)?; + assert_eq!( + settings.document_url(document)?.path(), + "/applications/example-app/environments/production/configurations/application-envelope" + ); + } + let mut settings = agent("http://127.0.0.1:2772"); + if let Some(document) = settings.documents.get_mut("settings") { + document.profile = "Profile name \u{fc}".to_owned(); + } + let document = settings + .documents + .get("settings") + .ok_or(PreparationError::InvalidBinding)?; + assert_eq!( + settings.document_url(document)?.path(), + "/applications/example-app/environments/production/configurations/Profile%20name%20%C3%BC" + ); + Ok(()) + } + + #[test] + fn endpoints_and_segments_fail_without_echoing_inputs() { + for endpoint in [ + "http://localhost:2772", + "https://127.0.0.1", + "http://192.0.2.1", + "http://user:sentinel@127.0.0.1", + "http://127.0.0.1?sentinel", + "http://127.0.0.1#sentinel", + "http://127.0.0.1/x/..", + "http://127.0.0.1/%2e", + "http://2130706433", + "http://0177.0.0.1", + "http://127.0.0.1\\sentinel", + "http://[::1%25sentinel]", + "http://127.0.0.1:0", + "http://127.0.0.1:65536", + " http://127.0.0.1", + "http://127.0.0.1//", + ] { + let settings = agent(endpoint); + assert!(settings.validate().is_err(), "accepted unsafe endpoint"); + assert!(!format!("{settings:?}").contains("sentinel")); + } + for profile in ["", ".", "..", "a/b", "a\\b", "a%2Fb", "a?b", "a#b", "a\nb"] { + let mut settings = agent("http://127.0.0.1:2772"); + if let Some(document) = settings.documents.get_mut("settings") { + document.profile = profile.to_owned(); + } + assert!(settings.validate().is_err()); + } + } + + #[test] + fn defaults_tokens_and_mapping_keys_are_strict() { + let mut settings = agent("http://127.0.0.1:2772"); + settings.default_key = "Settings".to_owned(); + assert_eq!(settings.validate(), Err(PreparationError::InvalidBinding)); + settings.default_key = "settings".to_owned(); + settings.access_token_env = Some("TOKEN=sentinel".to_owned()); + assert_eq!(settings.validate(), Err(PreparationError::InvalidBinding)); + assert!(!format!("{settings:?}").contains("sentinel")); + for value in ["", "\n", "a b", "\u{fc}"] { + assert!(AgentAccessToken::new(value.to_owned()).is_err()); + } + let token = AgentAccessToken::new("sentinel-secret".to_owned()); + assert!(token.is_ok()); + assert!(!format!("{token:?}").contains("sentinel")); + } + + #[test] + fn complete_arns_regions_and_cross_account_references() -> Result<(), PreparationError> { + for (partition, region) in [ + ("aws", "us-east-1"), + ("aws-cn", "cn-north-1"), + ("aws-us-gov", "us-gov-west-1"), + ] { + let target = SecretTarget::new( + format!( + "arn:{partition}:secretsmanager:{region}:999988887777:secret:example/key-AbCdEf" + ), + SecretVersion::Current, + )?; + let mut settings = SecretsManagerSettings { + region: region.to_owned(), + entries: BTreeMap::from([("active".to_owned(), target)]), + }; + settings.validate()?; + settings.region = "us-west-2".to_owned(); + assert_eq!(settings.validate(), Err(PreparationError::InvalidResource)); + } + for arn in [ + "name", + "arn:aws:secretsmanager:us-east-1:111122223333:secret:sentinel", + "arn:aws:s3:us-east-1:111122223333:secret:sentinel-AbCdEf", + "arn:aws-cn:secretsmanager:us-east-1:111122223333:secret:sentinel-AbCdEf", + "arn:aws:secretsmanager:us-gov-west-1:111122223333:secret:sentinel-AbCdEf", + "arn:aws:secretsmanager:us-east-0:111122223333:secret:sentinel-AbCdEf", + "arn:aws:secretsmanager::111122223333:secret:sentinel-AbCdEf", + "arn:aws:secretsmanager:us-east-1:123:secret:sentinel-AbCdEf", + "arn:aws:secretsmanager:us-east-1:111122223333:other:sentinel-AbCdEf", + "arn:aws:secretsmanager:us-east-1:111122223333:secret:sentinel*-AbCdEf", + "arn:aws:secretsmanager:us-east-1:111122223333:secret:sentinel-AbCdEf:extra", + ] { + let error = SecretTarget::new(arn.to_owned(), SecretVersion::Current).err(); + assert_eq!(error, Some(PreparationError::InvalidResource)); + assert!(!format!("{error:?}").contains("sentinel")); + } + Ok(()) + } + + #[test] + fn selectors_are_exclusive_and_length_checked() { + let arn = "arn:aws:secretsmanager:us-east-1:111122223333:secret:sentinel-AbCdEf"; + for version in [ + SecretVersion::Stage(String::new()), + SecretVersion::Stage("x".repeat(257)), + SecretVersion::VersionId("x".repeat(31)), + SecretVersion::VersionId("x".repeat(65)), + SecretVersion::Stage("sentinel\n".to_owned()), + ] { + assert_eq!( + SecretTarget::new(arn.to_owned(), version).err(), + Some(PreparationError::InvalidSelector) + ); + } + for selector in [ + "", + "version_stage = 'AWSCURRENT'", + "version_id = '12345678901234567890123456789012'", + ] { + assert!( + toml::from_str::(&format!("secret_id = '{arn}'\n{selector}")).is_ok() + ); + } + for suffix in [ + "version_stage = 'AWSCURRENT'\nversion_id = '12345678901234567890123456789012'", + "unknown = 'sentinel'", + ] { + assert!( + toml::from_str::(&format!("secret_id = '{arn}'\n{suffix}")).is_err() + ); + } + } + + #[test] + fn secret_keys_fit_portable_handle_and_current_selectors_normalize() + -> Result<(), PreparationError> { + let arn = "arn:aws:secretsmanager:us-east-1:111122223333:secret:example-AbCdEf"; + let current = SecretTarget::new(arn.to_owned(), SecretVersion::Current)?; + let explicit = SecretTarget::new( + arn.to_owned(), + SecretVersion::Stage("AWSCURRENT".to_owned()), + )?; + assert_eq!(current, explicit); + let mut settings = SecretsManagerSettings { + region: "us-east-1".to_owned(), + entries: BTreeMap::from([("\u{fc}".repeat(256), current)]), + }; + settings.validate()?; + let target = settings + .entries + .values() + .next() + .cloned() + .ok_or(PreparationError::InvalidBinding)?; + settings.entries = BTreeMap::from([("\u{fc}".repeat(257), target)]); + assert_eq!(settings.validate(), Err(PreparationError::InvalidBinding)); + Ok(()) + } + + #[test] + fn allowed_segments_round_trip_after_serialization_and_decoding() -> Result<(), PreparationError> + { + let mut settings = agent("http://127.0.0.1:2772"); + settings.documents.insert( + "settings".to_owned(), + AgentDocument { + application: "App +@=".to_owned(), + environment: "Env:;name".to_owned(), + profile: "Profile \u{fc}".to_owned(), + }, + ); + let target = settings + .documents + .get("settings") + .ok_or(PreparationError::InvalidBinding)?; + let serialized = settings.document_url(target)?.to_string(); + let reparsed = + Url::parse(&serialized).map_err(|_error| PreparationError::InvalidEndpoint)?; + let decoded: Vec = reparsed + .path_segments() + .ok_or(PreparationError::InvalidEndpoint)? + .map(|segment| { + percent_decode_str(segment) + .decode_utf8() + .map(Cow::into_owned) + .map_err(|_error| PreparationError::InvalidResource) + }) + .collect::>()?; + assert_eq!( + decoded, + [ + "applications", + "App +@=", + "environments", + "Env:;name", + "configurations", + "Profile \u{fc}" + ] + ); + Ok(()) + } + + #[test] + fn selector_boundaries_preserve_unicode_and_embedded_spaces() -> Result<(), PreparationError> { + let arn = "arn:aws:secretsmanager:us-east-1:111122223333:secret:example-AbCdEf"; + for selector in [ + SecretVersion::Stage("x".to_owned()), + SecretVersion::Stage("\u{fc}".repeat(256)), + SecretVersion::Stage("a b".to_owned()), + SecretVersion::VersionId("\u{fc}".repeat(32)), + SecretVersion::VersionId("x".repeat(64)), + ] { + let target = SecretTarget::new(arn.to_owned(), selector.clone())?; + assert_eq!(target.version(), &selector); + } + assert_eq!( + SecretTarget::new(arn.to_owned(), SecretVersion::Stage(" ".repeat(256))).err(), + Some(PreparationError::InvalidSelector) + ); + Ok(()) + } + + #[test] + fn arn_size_name_and_region_shape_boundaries() -> Result<(), PreparationError> { + let valid = format!( + "arn:aws:secretsmanager:us-east-1:111122223333:secret:{}-AbCdEf", + "x".repeat(512) + ); + SecretTarget::new(valid, SecretVersion::Current)?; + let oversized_name = format!( + "arn:aws:secretsmanager:us-east-1:111122223333:secret:{}-AbCdEf", + "x".repeat(513) + ); + assert_eq!( + SecretTarget::new(oversized_name, SecretVersion::Current).err(), + Some(PreparationError::InvalidResource) + ); + // Family shape does not promise an existing/served region. + SecretTarget::new( + "arn:aws:secretsmanager:us-fictional-99:111122223333:secret:x-AbCdEf".to_owned(), + SecretVersion::Current, + )?; + let base = "arn:aws:secretsmanager::111122223333:secret:x-AbCdEf"; + let maximum_region = format!("us-{}-1", "a".repeat(2_048 - base.len() - 5)); + let maximum = + format!("arn:aws:secretsmanager:{maximum_region}:111122223333:secret:x-AbCdEf"); + assert_eq!(maximum.len(), 2_048); + SecretTarget::new(maximum.clone(), SecretVersion::Current)?; + assert_eq!( + SecretTarget::new(format!("{maximum}x"), SecretVersion::Current).err(), + Some(PreparationError::InvalidResource) + ); + for region in [ + "zz-east-1", + "us--east-1", + "us-East-1", + "us-east-01", + "us-east-x", + ] { + assert_eq!( + SecretTarget::new( + format!("arn:aws:secretsmanager:{region}:111122223333:secret:x-AbCdEf"), + SecretVersion::Current + ) + .err(), + Some(PreparationError::InvalidResource) + ); + } + Ok(()) + } + + #[test] + fn configurable_limits_defaults_consistency_and_unknown_fields() -> Result<(), PreparationError> + { + let defaults = AwsPreparationLimits::default(); + defaults.validate()?; + assert_eq!(defaults.max_snapshot_bytes, 8 * 1_024 * 1_024); + let raised: AwsPreparationLimits = toml::from_str("max_snapshot_bytes = 16777216") + .map_err(|_error| PreparationError::InvalidLimits)?; + raised.validate()?; + assert_eq!( + raised.max_config_document_bytes, + defaults.max_config_document_bytes + ); + for input in [ + "max_snapshot_bytes = 0", + "max_snapshot_bytes = 1", + "max_remote_values = -1", + "operation_timeout_ms = 0", + "connect_timeout_ms = 2001", + "attempt_timeout_ms = 5001", + "operation_timeout_ms = 30001", + "unknown = 1", + "max_remote_values = 18446744073709551616", + ] { + let parsed = toml::from_str::(input); + assert!(parsed.is_err() || parsed.is_ok_and(|limits| limits.validate().is_err())); + } + Ok(()) + } +} diff --git a/crates/edgezero-store-aws/src/snapshot_tests.rs b/crates/edgezero-store-aws/src/snapshot_tests.rs new file mode 100644 index 000000000..912e18be8 --- /dev/null +++ b/crates/edgezero-store-aws/src/snapshot_tests.rs @@ -0,0 +1,80 @@ +//! Private snapshot read contracts, with no service client or credentials. + +use std::collections::BTreeMap; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use bytes::Bytes; +use edgezero_core::secret_store::{SecretError, SecretStore as _}; +use edgezero_core::secret_store_contract_tests; +use edgezero_core::time::{Deadline, MonotonicClock, MonotonicInstant}; +use futures::executor::block_on; + +use crate::preparation::SecretSnapshot; + +fn snapshot() -> SecretSnapshot { + SecretSnapshot { + namespace: "mystore".to_owned(), + entries: BTreeMap::from([ + ( + "contract_key".to_owned(), + Bytes::from_static(b"contract_value"), + ), + ( + "contract_key_2".to_owned(), + Bytes::from_static(b"another_value"), + ), + ]), + } +} + +#[test] +fn final_deadline_check_precedes_size_error() { + let start = MonotonicInstant::now(); + let sampled = Arc::new(Mutex::new(start)); + let clock = MonotonicClock::new(move || { + let mut now = sampled.lock().expect("clock mutex"); + let result = *now; + *now = now + .checked_add(Duration::from_millis(1)) + .expect("clock advance"); + result + }); + let deadline = Deadline::at_instant( + start + .checked_add(Duration::from_millis(2)) + .expect("deadline"), + ); + let result = + block_on(snapshot().get_bytes_bounded("mystore", "contract_key", &clock, deadline, 1, 1)); + assert!(matches!(result, Err(SecretError::DeadlineExceeded))); +} + +#[test] +fn namespace_miss_and_backend_and_value_caps_are_enforced() { + let clock = MonotonicClock::default(); + let deadline = Deadline::at_instant( + clock + .now() + .checked_add(Duration::from_secs(1)) + .expect("deadline"), + ); + let provider = snapshot(); + let miss = + block_on(provider.get_bytes_bounded("other", "contract_key", &clock, deadline, 0, 0)) + .expect("namespace miss"); + assert!(miss.value.is_none()); + for (backend_cap, value_cap) in [(1_u64, 100_u64), (100_u64, 1_u64)] { + let result = block_on(provider.get_bytes_bounded( + "mystore", + "contract_key", + &clock, + deadline, + backend_cap, + value_cap, + )); + assert!(matches!(result, Err(SecretError::ValueTooLarge))); + } +} + +secret_store_contract_tests!(secret_snapshot_contract, snapshot()); diff --git a/crates/edgezero-store-celld/Cargo.toml b/crates/edgezero-store-celld/Cargo.toml new file mode 100644 index 000000000..434249ea7 --- /dev/null +++ b/crates/edgezero-store-celld/Cargo.toml @@ -0,0 +1,39 @@ +[package] +name = "edgezero-store-celld" +description = "Optional native scoped celld configuration snapshots and live KV for EdgeZero" +authors = { workspace = true } +edition = { workspace = true } +license = { workspace = true } +publish = { workspace = true } +version = { workspace = true } + +[lints] +workspace = true + +[features] +default = [] +config = ["_transport"] +kv = ["_transport", "dep:base64", "dep:sha2"] +_transport = ["dep:edgezero-core", "dep:bytes", "dep:async-trait", "dep:reqwest", "dep:tokio", "dep:futures-util", "dep:serde_json"] + +[dependencies] +serde = { workspace = true } +thiserror = { workspace = true } +url = { workspace = true } + +[target.'cfg(not(target_arch = "wasm32"))'.dependencies] +async-trait = { workspace = true, optional = true } +base64 = { workspace = true, optional = true } +bytes = { workspace = true, optional = true } +edgezero-core = { workspace = true, optional = true } +futures-util = { workspace = true, optional = true } +reqwest = { workspace = true, optional = true, features = ["stream"] } +serde_json = { workspace = true, optional = true } +sha2 = { workspace = true, optional = true } +tokio = { workspace = true, optional = true } + +[dev-dependencies] +serde_json = { workspace = true } + +[target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies] +tokio-rustls = { workspace = true } diff --git a/crates/edgezero-store-celld/src/client.rs b/crates/edgezero-store-celld/src/client.rs new file mode 100644 index 000000000..2f5563ec8 --- /dev/null +++ b/crates/edgezero-store-celld/src/client.rs @@ -0,0 +1,852 @@ +//! One bounded HTTP exchange. No discovery, redirects, proxies or mutation replay. + +#![expect( + clippy::arbitrary_source_item_ordering, + reason = "keep transport construction, request lifecycle and response validators together" +)] + +use std::fmt; +use std::time::Duration; + +use bytes::Bytes; +use edgezero_core::time::{Deadline, MonotonicClock}; +use futures_util::StreamExt as _; +#[cfg(feature = "kv")] +use reqwest::header::CONTENT_LENGTH; +use reqwest::header::{ + ACCEPT_ENCODING, AUTHORIZATION, CONTENT_ENCODING, CONTENT_TYPE, HeaderMap, HeaderValue, +}; +use reqwest::redirect::Policy; +use reqwest::retry::never; +use reqwest::{Client, Method, StatusCode}; +use serde::Deserialize; +use tokio::time::timeout; +use url::Url; + +use crate::PreparationError; +use crate::settings::{CelldAccessToken, CelldPreparationLimits}; + +const ERROR_CAP: usize = 1_024; +const VERSION_HEADER: &str = "x-celld-store-version"; + +pub(crate) struct ScopedClient { + authorization: Option, + client: Client, + clock: MonotonicClock, + endpoint: Url, + limits: CelldPreparationLimits, +} + +impl fmt::Debug for ScopedClient { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ScopedClient").finish_non_exhaustive() + } +} + +#[derive(Clone, Copy)] +pub(crate) enum ResponseKind { + #[cfg(feature = "kv")] + Exists, + #[cfg(feature = "kv")] + Mutation, + #[cfg(feature = "kv")] + Page, + #[cfg(feature = "config")] + Document, + #[cfg(feature = "kv")] + Value, +} + +impl ScopedClient { + pub(crate) fn new( + endpoint: Url, + token: Option<&CelldAccessToken>, + clock: MonotonicClock, + limits: CelldPreparationLimits, + ) -> Result { + Self::build(endpoint, token, clock, limits, None) + } + + // A root-only seam keeps tests on the production transport policy. Production + // always passes None; no arbitrary client or insecure TLS switch is exposed. + fn build( + endpoint: Url, + token: Option<&CelldAccessToken>, + clock: MonotonicClock, + limits: CelldPreparationLimits, + root: Option, + ) -> Result { + let mut builder = Client::builder() + .no_proxy() + .redirect(Policy::none()) + .retry(never()) + .no_gzip() + .no_brotli() + .no_deflate() + .no_zstd() + .connection_verbose(false) + .http1_only() + .connect_timeout(Duration::from_millis(limits.connect_timeout_ms)) + .timeout(Duration::from_millis(limits.operation_timeout_ms)); + if let Some(certificate) = root { + builder = builder.tls_certs_only([certificate]); + } + let authorization = token + .map(|credential| { + let mut header = HeaderValue::from_str(&format!("Bearer {}", credential.expose())) + .map_err(|_error| PreparationError::Credentials)?; + header.set_sensitive(true); + Ok(header) + }) + .transpose()?; + Ok(Self { + authorization, + client: builder + .build() + .map_err(|_error| PreparationError::Unavailable)?, + clock, + endpoint, + limits, + }) + } + + pub(crate) fn deadline(&self) -> Result { + self.clock + .now() + .checked_add(Duration::from_millis(self.limits.operation_timeout_ms)) + .map(Deadline::at_instant) + .ok_or(PreparationError::Deadline) + } + + pub(crate) fn check(&self, deadline: Deadline) -> Result { + deadline + .remaining_at(self.clock.now()) + .ok_or(PreparationError::Deadline) + } + + pub(crate) fn url(&self, store: &str, resource: &str) -> Result { + let mut url = self.endpoint.clone(); + url.path_segments_mut() + .map_err(|_error| PreparationError::InvalidEndpoint)? + .clear() + .extend(["v1", "stores", store, resource]); + Ok(url) + } + + /// Covers sending, streaming and decoding with one real timer and injected + /// monotonic checks. None means a validated miss, never a protocol failure. + pub(crate) async fn execute( + &self, + method: Method, + url: Url, + body: Option, + cap: usize, + deadline: Deadline, + #[cfg_attr( + not(feature = "kv"), + expect( + unused_variables, + reason = "config-only builds have one request kind and no optional misses" + ) + )] + kind: ResponseKind, + ) -> Result>, PreparationError> { + let budget = self + .check(deadline)? + .min(Duration::from_millis(self.limits.operation_timeout_ms)); + let mut request = self + .client + .request(method.clone(), url) + .header(ACCEPT_ENCODING, "identity") + .timeout(budget); + if let Some(authorization) = &self.authorization { + request = request.header(AUTHORIZATION, authorization.clone()); + } + if let Some(payload) = body { + request = request + .header(CONTENT_TYPE, "application/octet-stream") + .body(payload); + } + let result = timeout(budget, async { + let response = request.send().await.map_err(http_error)?; + let headers = response.headers(); + validate_headers(headers)?; + let status = response.status(); + #[cfg(feature = "kv")] + let success = status + == if matches!(kind, ResponseKind::Mutation) { + StatusCode::NO_CONTENT + } else { + StatusCode::OK + }; + #[cfg(not(feature = "kv"))] + let success = status == StatusCode::OK; + #[cfg(feature = "kv")] + if matches!(kind, ResponseKind::Mutation) + && success + && headers + .get(CONTENT_LENGTH) + .is_some_and(|length| length.as_bytes() != b"0") + { + return Err(PreparationError::Malformed); + } + let missing = status == StatusCode::NOT_FOUND; + // The wire HEAD contract has no error body, including non-404 errors. + if method == Method::HEAD { + return if success { + Ok(Some(Vec::new())) + } else if missing { + Ok(None) + } else { + Err(status_error(status) + .map_or(PreparationError::Malformed, |(_, error)| error)) + }; + } + #[cfg(feature = "kv")] + let expected_type = if !success || matches!(kind, ResponseKind::Page) { + "application/json" + } else { + "application/octet-stream" + }; + #[cfg(not(feature = "kv"))] + let expected_type = if success { + "application/octet-stream" + } else { + "application/json" + }; + validate_content_type(headers, expected_type)?; + let response_cap = if success { cap } else { ERROR_CAP }; + let bytes = collect(response, response_cap, &self.clock, deadline).await?; + if success { + return Ok(Some(bytes)); + } + let (expected, error) = status_error(status).ok_or(PreparationError::Malformed)?; + let parsed: ErrorBody = + serde_json::from_slice(&bytes).map_err(|_error| PreparationError::Malformed)?; + if parsed.error != expected { + return Err(PreparationError::Malformed); + } + #[cfg(feature = "kv")] + if missing && matches!(kind, ResponseKind::Value) { + return Ok(None); + } + Err(error) + }) + .await + .map_err(|_elapsed| PreparationError::Deadline); + self.check(deadline)?; + result? + } +} + +fn validate_headers(headers: &HeaderMap) -> Result<(), PreparationError> { + let mut versions = headers.get_all(VERSION_HEADER).iter(); + if versions.next().is_none_or(|value| value.as_bytes() != b"1") || versions.next().is_some() { + return Err(PreparationError::UnsupportedVersion); + } + let mut encodings = headers.get_all(CONTENT_ENCODING).iter(); + if encodings + .next() + .is_some_and(|value| !value.as_bytes().eq_ignore_ascii_case(b"identity")) + || encodings.next().is_some() + { + return Err(PreparationError::Malformed); + } + Ok(()) +} + +fn validate_content_type(headers: &HeaderMap, expected: &str) -> Result<(), PreparationError> { + let mut types = headers.get_all(CONTENT_TYPE).iter(); + if types + .next() + .is_none_or(|value| value.as_bytes() != expected.as_bytes()) + || types.next().is_some() + { + return Err(PreparationError::Malformed); + } + Ok(()) +} + +async fn collect( + response: reqwest::Response, + cap: usize, + clock: &MonotonicClock, + deadline: Deadline, +) -> Result, PreparationError> { + let cap_u64 = u64::try_from(cap).map_err(|_error| PreparationError::SizeLimit)?; + if response + .content_length() + .is_some_and(|length| length > cap_u64) + { + return Err(PreparationError::SizeLimit); + } + let mut bytes = Vec::with_capacity(cap.min(0x4000)); + let mut stream = response.bytes_stream(); + while let Some(next_chunk) = stream.next().await { + let chunk = next_chunk.map_err(http_error)?; + if deadline.is_expired_at(clock.now()) { + return Err(PreparationError::Deadline); + } + if chunk.len() > cap.saturating_sub(bytes.len()) { + return Err(PreparationError::SizeLimit); + } + bytes.extend_from_slice(&chunk); + } + Ok(bytes) +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct ErrorBody { + error: String, +} + +#[expect( + clippy::needless_pass_by_value, + reason = "consume transport errors at the redaction boundary so no source chain escapes" +)] +fn http_error(error: reqwest::Error) -> PreparationError { + if error.is_timeout() { + PreparationError::Deadline + } else { + PreparationError::Unavailable + } +} + +fn status_error(status: StatusCode) -> Option<(&'static str, PreparationError)> { + Some(match status.as_u16() { + 400 => ("invalid_input", PreparationError::InvalidResource), + 401 => ("unauthenticated", PreparationError::Unauthenticated), + 403 => ("forbidden", PreparationError::Forbidden), + 404 => ("not_found", PreparationError::Missing), + 413 => ("limit_exceeded", PreparationError::SizeLimit), + 426 => ("unsupported_version", PreparationError::UnsupportedVersion), + 503 => ("unavailable", PreparationError::Unavailable), + 504 => ("deadline", PreparationError::Deadline), + _ => return None, + }) +} + +#[cfg(test)] +#[cfg(any(feature = "config", feature = "kv"))] +mod tests { + use std::env::{current_exe, var}; + use std::io::ErrorKind; + use std::net::TcpListener; + use std::process::Command; + + use edgezero_core::time::MonotonicClock; + use reqwest::Method; + use url::Url; + + use super::{ResponseKind, ScopedClient}; + use crate::PreparationError; + use crate::settings::{CelldAccessToken, CelldPreparationLimits}; + use crate::test_support::{Fixture, Reply}; + + #[tokio::test(flavor = "current_thread")] + async fn native_tls_requires_trust_matching_identity_and_valid_dates() { + for (certificate, trusted, accepted) in [ + ( + include_bytes!("fixtures/tls-valid.der").as_slice(), + true, + true, + ), + ( + include_bytes!("fixtures/tls-valid.der").as_slice(), + false, + false, + ), + ( + include_bytes!("fixtures/tls-wrong.der").as_slice(), + true, + false, + ), + ( + include_bytes!("fixtures/tls-expired.der").as_slice(), + true, + false, + ), + ] { + let fixture = Fixture::tls(certificate, vec![]).await; + fixture.seed("config", "key", b"TLS").await; + let token = + CelldAccessToken::new("test-tls-bearer-credential-at-least-32-bytes".to_owned()) + .unwrap(); + let root = trusted.then(|| { + reqwest::Certificate::from_der(include_bytes!("fixtures/tls-ca.der")).unwrap() + }); + let client = ScopedClient::build( + Url::parse(fixture.endpoint()).unwrap(), + Some(&token), + MonotonicClock::default(), + CelldPreparationLimits::default(), + root, + ) + .unwrap(); + let mut url = client.url("config", "value").unwrap(); + url.query_pairs_mut().append_pair("key", "key"); + let result = client + .execute( + Method::GET, + url, + None, + 16, + client.deadline().unwrap(), + kind(), + ) + .await; + if accepted { + assert_eq!(result.unwrap(), Some(b"TLS".to_vec())); + assert_eq!( + fixture.requests().await[0].authorization.as_deref(), + Some(format!("Bearer {}", token.expose()).as_str()) + ); + } else { + assert_eq!(result.unwrap_err(), PreparationError::Unavailable); + assert_eq!( + fixture.count().await, + 0, + "TLS failure must never fall back to plaintext" + ); + } + } + } + + #[tokio::test(flavor = "current_thread")] + async fn native_tls_keeps_distinct_credentials_on_reused_connections() { + let fixture = Fixture::tls(include_bytes!("fixtures/tls-valid.der"), vec![]).await; + fixture.seed("config", "key", b"TLS").await; + let mut clients = Vec::new(); + for token_text in [ + "first-tls-credential-at-least-32-bytes", + "second-tls-credential-at-least-32-bytes", + ] { + let token = CelldAccessToken::new(token_text.to_owned()).unwrap(); + clients.push( + ScopedClient::build( + Url::parse(fixture.endpoint()).unwrap(), + Some(&token), + MonotonicClock::default(), + CelldPreparationLimits::default(), + Some( + reqwest::Certificate::from_der(include_bytes!("fixtures/tls-ca.der")) + .unwrap(), + ), + ) + .unwrap(), + ); + } + for index in [0, 1, 0] { + let client = &clients[index]; + let mut url = client.url("config", "value").unwrap(); + url.query_pairs_mut().append_pair("key", "key"); + assert_eq!( + client + .execute( + Method::GET, + url, + None, + 16, + client.deadline().unwrap(), + kind() + ) + .await + .unwrap(), + Some(b"TLS".to_vec()) + ); + } + let requests = fixture.requests().await; + assert_eq!(requests[0].connection, requests[2].connection); + assert_ne!(requests[0].connection, requests[1].connection); + assert_eq!(requests[0].authorization, requests[2].authorization); + assert_ne!(requests[0].authorization, requests[1].authorization); + } + + #[test] + fn ambient_proxy_is_not_used_by_the_production_client() { + let sentinel = TcpListener::bind("127.0.0.1:0").unwrap(); + sentinel.set_nonblocking(true).unwrap(); + let proxy = format!("http://{}", sentinel.local_addr().unwrap()); + let mut command = + Command::new(current_exe().expect("locate proxy fixture test executable")); + command + .env_clear() + .env("CELLD_PROXY_FIXTURE_CHILD", "selected"); + for variable in [ + "HTTP_PROXY", + "HTTPS_PROXY", + "ALL_PROXY", + "http_proxy", + "https_proxy", + "all_proxy", + ] { + command.env(variable, &proxy); + } + let output = command + .env("NO_PROXY", "") + .env("no_proxy", "") + .args([ + "--exact", + "client::tests::ambient_proxy_child", + "--nocapture", + ]) + .output() + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + assert!(String::from_utf8_lossy(&output.stdout).contains("CELLD_DIRECT_TRANSPORT_PROVEN")); + assert!(matches!(sentinel.accept(), Err(error) if error.kind() == ErrorKind::WouldBlock)); + } + + #[tokio::test(flavor = "current_thread")] + #[expect( + clippy::print_stdout, + reason = "the parent requires an explicit child execution marker" + )] + async fn ambient_proxy_child() { + if var("CELLD_PROXY_FIXTURE_CHILD").as_deref() != Ok("selected") { + return; + } + let fixture = Fixture::new(vec![]).await; + fixture.seed("config", "key", b"direct").await; + let client = ScopedClient::new( + Url::parse(fixture.endpoint()).unwrap(), + None, + MonotonicClock::default(), + CelldPreparationLimits::default(), + ) + .unwrap(); + let mut url = client.url("config", "value").unwrap(); + url.query_pairs_mut().append_pair("key", "key"); + assert_eq!( + client + .execute( + Method::GET, + url, + None, + 16, + client.deadline().unwrap(), + kind() + ) + .await + .unwrap(), + Some(b"direct".to_vec()) + ); + assert_eq!(fixture.count().await, 1); + println!("CELLD_DIRECT_TRANSPORT_PROVEN"); + } + + #[tokio::test(flavor = "current_thread")] + async fn oversized_response_headers_fail_at_the_http_parser_bound() { + let header = format!( + "x-celld-store-version: 1\r\nx-excess: {}\r\n", + "x".repeat(512 * 1024) + ); + let fixture = Fixture::new(vec![Reply::custom( + 200, + "application/octet-stream", + &header, + b"x", + )]) + .await; + let client = ScopedClient::new( + Url::parse(fixture.endpoint()).expect("fixture origin"), + None, + MonotonicClock::default(), + CelldPreparationLimits::default(), + ) + .expect("bounded client"); + let url = client.url("config", "value").expect("value URL"); + assert_eq!( + client + .execute( + Method::GET, + url, + None, + 16, + client.deadline().expect("deadline"), + kind() + ) + .await + .expect_err("oversized headers"), + PreparationError::Unavailable + ); + assert_eq!(fixture.count().await, 1); + } + + fn kind() -> ResponseKind { + #[cfg(feature = "kv")] + { + ResponseKind::Value + } + #[cfg(not(feature = "kv"))] + { + ResponseKind::Document + } + } + + async fn read(fixture: &Fixture, cap: usize) -> Result>, PreparationError> { + let client = ScopedClient::new( + Url::parse(fixture.endpoint()).expect("parse read fixture endpoint"), + None, + MonotonicClock::default(), + CelldPreparationLimits::default(), + ) + .expect("construct read fixture client"); + let mut url = client + .url("state", "value") + .expect("construct read fixture value URL"); + url.query_pairs_mut().append_pair("key", "k"); + client + .execute( + Method::GET, + url, + None, + cap, + client.deadline().expect("construct read fixture deadline"), + kind(), + ) + .await + } + + #[tokio::test(flavor = "current_thread")] + async fn raw_binary_empty_and_encoded_keys_use_real_transport() { + let fixture = Fixture::new(vec![]).await; + let key = " /+?%&#=\u{e9}\u{65e5}"; + fixture.seed("state", key, &[0, 255, 1]).await; + let client = ScopedClient::new( + Url::parse(fixture.endpoint()).unwrap(), + None, + MonotonicClock::default(), + CelldPreparationLimits::default(), + ) + .unwrap(); + let mut url = client.url("state", "value").unwrap(); + url.query_pairs_mut().append_pair("key", key); + let bytes = client + .execute( + Method::GET, + url.clone(), + None, + 3, + client.deadline().unwrap(), + kind(), + ) + .await + .unwrap() + .unwrap(); + assert_eq!(bytes, [0, 255, 1]); + fixture.seed("state", key, b"").await; + assert!( + client + .execute( + Method::GET, + url, + None, + 0, + client.deadline().unwrap(), + kind() + ) + .await + .unwrap() + .unwrap() + .is_empty() + ); + assert_eq!(fixture.value("state", key).await.unwrap(), b""); + let requests = fixture.requests().await; + assert_eq!(requests.len(), 2); + assert!( + requests + .iter() + .all(|request| request.authorization.is_none() + && request.encoding.as_deref() == Some("identity") + && request.body.is_empty() + && request.method == "GET") + ); + assert_eq!(requests[0].connection, requests[1].connection); + assert!(requests[0].target.contains("%2B")); + } + + #[tokio::test(flavor = "current_thread")] + async fn every_response_requires_exactly_one_version_and_identity_encoding() { + for headers in [ + "", + "x-celld-store-version: 2\r\n", + "x-celld-store-version: 1\r\nx-celld-store-version: 1\r\n", + "x-celld-store-version: 1, 1\r\n", + ] { + let fixture = Fixture::new(vec![Reply::custom( + 200, + "application/octet-stream", + headers, + b"v", + )]) + .await; + assert_eq!( + read(&fixture, 100).await, + Err(PreparationError::UnsupportedVersion) + ); + } + for headers in [ + "Content-Encoding: gzip\r\n", + "Content-Encoding: identity\r\nContent-Encoding: identity\r\n", + ] { + let fixture = Fixture::new(vec![Reply::custom( + 200, + "application/octet-stream", + &format!("x-celld-store-version: 1\r\n{headers}"), + b"v", + )]) + .await; + assert_eq!(read(&fixture, 100).await, Err(PreparationError::Malformed)); + } + } + + #[tokio::test(flavor = "current_thread")] + async fn exact_error_status_categories_and_bounded_strict_json() { + for (status, category, error) in [ + (400, "invalid_input", PreparationError::InvalidResource), + (401, "unauthenticated", PreparationError::Unauthenticated), + (403, "forbidden", PreparationError::Forbidden), + (413, "limit_exceeded", PreparationError::SizeLimit), + ( + 426, + "unsupported_version", + PreparationError::UnsupportedVersion, + ), + (503, "unavailable", PreparationError::Unavailable), + (504, "deadline", PreparationError::Deadline), + ] { + let bytes = serde_json::to_vec(&serde_json::json!({"error": category})).unwrap(); + let fixture = + Fixture::new(vec![Reply::response(status, "application/json", &bytes)]).await; + assert_eq!(read(&fixture, 100).await, Err(error)); + } + for bytes in [ + br#"{"error":"forbidden"}"#.as_slice(), + br#"{"error":"unavailable","extra":1}"#, + br#"{"error":"unavailable","error":"unavailable"}"#, + b"not json", + ] { + let fixture = Fixture::new(vec![Reply::response(503, "application/json", bytes)]).await; + assert_eq!(read(&fixture, 100).await, Err(PreparationError::Malformed)); + } + let fixture = Fixture::new(vec![Reply::response( + 503, + "application/json", + &vec![b'x'; 1_025], + )]) + .await; + assert_eq!(read(&fixture, 100).await, Err(PreparationError::SizeLimit)); + } + + #[tokio::test(flavor = "current_thread")] + async fn misses_are_distinct_from_empty_and_bad_statuses() { + let missing_fixture = Fixture::new(vec![]).await; + #[cfg(feature = "kv")] + assert_eq!(read(&missing_fixture, 1).await.unwrap(), None); + #[cfg(not(feature = "kv"))] + assert_eq!( + read(&missing_fixture, 1).await, + Err(PreparationError::Missing) + ); + for status in [201, 204, 206, 302, 500] { + let fixture = Fixture::new(vec![Reply::response( + status, + "application/octet-stream", + b"", + )]) + .await; + assert_eq!(read(&fixture, 10).await, Err(PreparationError::Malformed)); + } + } + + #[tokio::test(flavor = "current_thread")] + async fn declared_and_chunked_values_are_capped_before_unbounded_collection() { + let declared = Reply::Bytes(b"HTTP/1.1 200 Fixture\r\nx-celld-store-version: 1\r\nContent-Type: application/octet-stream\r\nContent-Length: 9999999\r\n\r\n".to_vec()); + let chunked = Reply::Bytes(b"HTTP/1.1 200 Fixture\r\nx-celld-store-version: 1\r\nContent-Type: application/octet-stream\r\nTransfer-Encoding: chunked\r\n\r\n3\r\nabc\r\n3\r\ndef\r\n0\r\n\r\n".to_vec()); + for reply in [declared, chunked] { + let fixture = Fixture::new(vec![reply]).await; + assert_eq!(read(&fixture, 5).await, Err(PreparationError::SizeLimit)); + } + } + + #[tokio::test(flavor = "current_thread")] + async fn redirects_never_reach_another_destination() { + let destination = Fixture::new(vec![]).await; + let source = Fixture::new(vec![Reply::custom( + 302, + "application/json", + &format!( + "x-celld-store-version: 1\r\nLocation: {}/v1/stores/state/value?key=k\r\n", + destination.endpoint() + ), + b"{}", + )]) + .await; + assert_eq!(read(&source, 100).await, Err(PreparationError::Malformed)); + assert_eq!(destination.count().await, 0); + } + + #[tokio::test(flavor = "current_thread")] + async fn slow_headers_and_body_share_a_finite_operation_deadline() { + for reply in [Reply::Stall, Reply::SlowBody] { + let fixture = Fixture::new(vec![reply]).await; + let limits = CelldPreparationLimits { + operation_timeout_ms: 40, + connect_timeout_ms: 10, + ..CelldPreparationLimits::default() + }; + let client = ScopedClient::new( + Url::parse(fixture.endpoint()).unwrap(), + None, + MonotonicClock::default(), + limits, + ) + .unwrap(); + let mut url = client.url("state", "value").unwrap(); + url.query_pairs_mut().append_pair("key", "k"); + assert_eq!( + client + .execute( + Method::GET, + url, + None, + 100, + client.deadline().unwrap(), + kind() + ) + .await, + Err(PreparationError::Deadline) + ); + } + } + + #[test] + fn sensitive_header_and_owned_debug_never_expose_tokens_or_resources() { + let token = CelldAccessToken::new("secret-sentinel".repeat(3)).unwrap(); + let client = ScopedClient::new( + Url::parse("https://resource-sentinel.invalid").unwrap(), + Some(&token), + MonotonicClock::default(), + CelldPreparationLimits::default(), + ) + .unwrap(); + assert!(client.authorization.as_ref().unwrap().is_sensitive()); + let debug = format!("{client:?} {token:?}"); + assert!(!debug.contains("secret-sentinel")); + assert!(!debug.contains("resource-sentinel")); + for error in [ + PreparationError::Credentials, + PreparationError::Unauthenticated, + PreparationError::Forbidden, + PreparationError::Unavailable, + ] { + assert!(!format!("{error} {error:?}").contains("sentinel")); + } + } +} diff --git a/crates/edgezero-store-celld/src/config.rs b/crates/edgezero-store-celld/src/config.rs new file mode 100644 index 000000000..78ecdbd28 --- /dev/null +++ b/crates/edgezero-store-celld/src/config.rs @@ -0,0 +1,140 @@ +//! Immutable config: there is deliberately no client in a published snapshot. + +use std::collections::BTreeMap; + +use edgezero_core::config_store::{ + BoundedStoreRead, ConfigStore, ConfigStoreError, ConfigValue, finish_bounded_config_read, +}; +use edgezero_core::time::{Deadline, MonotonicClock}; + +pub(crate) struct ConfigSnapshot { + entries: BTreeMap, +} + +impl ConfigSnapshot { + pub(crate) const fn new(entries: BTreeMap) -> Self { + Self { entries } + } +} + +#[async_trait::async_trait(?Send)] +impl ConfigStore for ConfigSnapshot { + async fn get(&self, key: &str) -> Result, ConfigStoreError> { + Ok(self.entries.get(key).cloned()) + } + + async fn get_bounded( + &self, + key: &str, + clock: &MonotonicClock, + deadline: Deadline, + max_backend_bytes: u64, + max_value_bytes: u64, + ) -> Result, ConfigStoreError> { + if deadline.is_expired_at(clock.now()) { + return Err(ConfigStoreError::DeadlineExceeded); + } + let result = finish_bounded_config_read( + Ok(self.entries.get(key).cloned()), + clock, + deadline, + max_backend_bytes, + max_value_bytes, + ); + // Check even if the helper returned a size failure: deadline wins. + if deadline.is_expired_at(clock.now()) { + return Err(ConfigStoreError::DeadlineExceeded); + } + result + } +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + use std::sync::Arc; + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::time::Duration; + + use edgezero_core::config_store::{ConfigStore as _, ConfigStoreError, ConfigValue}; + use edgezero_core::time::{Deadline, MonotonicClock, MonotonicInstant}; + + use super::ConfigSnapshot; + + #[tokio::test(flavor = "current_thread")] + async fn bounded_reads_count_exact_utf8_and_independent_caps() { + let snapshot = ConfigSnapshot::new(BTreeMap::from([( + "k".to_owned(), + ConfigValue::from("\u{e9}"), + )])); + let clock = MonotonicClock::default(); + for (backend, value) in [(1, 2), (2, 1)] { + assert!(matches!( + snapshot + .get_bounded( + "k", + &clock, + clock.deadline_after(Duration::from_secs(1)), + backend, + value + ) + .await, + Err(ConfigStoreError::ValueTooLarge) + )); + } + let result = snapshot + .get_bounded( + "k", + &clock, + clock.deadline_after(Duration::from_secs(1)), + 2, + 2, + ) + .await + .unwrap(); + assert_eq!(result.backend_bytes, 2); + assert_eq!(result.value.unwrap().as_str(), "\u{e9}"); + assert_eq!( + snapshot + .get_bounded( + "missing", + &clock, + clock.deadline_after(Duration::from_secs(1)), + 0, + 0 + ) + .await + .unwrap() + .backend_bytes, + 0 + ); + } + + #[tokio::test(flavor = "current_thread")] + async fn deadline_wins_even_after_size_failure() { + let snapshot = ConfigSnapshot::new(BTreeMap::from([( + "k".to_owned(), + ConfigValue::from("bytes"), + )])); + let now = MonotonicInstant::now(); + let later = now.checked_add(Duration::from_secs(2)).unwrap(); + let ticks = Arc::new(AtomicUsize::new(0)); + let clock = MonotonicClock::new(move || { + if ticks.fetch_add(1, Ordering::SeqCst) < 2 { + now + } else { + later + } + }); + let result = snapshot + .get_bounded( + "k", + &clock, + Deadline::at_instant(now.checked_add(Duration::from_secs(1)).unwrap()), + 0, + 0, + ) + .await; + assert!(matches!(result, Err(ConfigStoreError::DeadlineExceeded))); + } +} diff --git a/crates/edgezero-store-celld/src/fixtures/README.md b/crates/edgezero-store-celld/src/fixtures/README.md new file mode 100644 index 000000000..cc46fcec8 --- /dev/null +++ b/crates/edgezero-store-celld/src/fixtures/README.md @@ -0,0 +1,16 @@ +# Disposable TLS fixtures + +These DER certificates and the PKCS#8 key are public, test-only material. Never use +this key or trust this root outside the loopback test fixture. + +- `tls-valid.der`: SAN `127.0.0.1`, valid from 2020 through 2040. +- `tls-wrong.der`: SAN `127.0.0.2`, valid from 2020 through 2040. +- `tls-expired.der`: SAN `127.0.0.1`, expired January 2020. +- `tls-ca.der`: disposable signing root; its private key was deleted. +- `tls-key.der`: shared disposable P-256 leaf key. + +Generated locally with OpenSSL. Runtime tests use the normal rustls server and +production reqwest policy; their private seam changes only the trusted root. +No OpenSSL executable, system trust changes or external service is needed to run +these tests. Replace the valid leaves before 2040 if these tests are maintained +that long. diff --git a/crates/edgezero-store-celld/src/fixtures/tls-ca.der b/crates/edgezero-store-celld/src/fixtures/tls-ca.der new file mode 100644 index 000000000..c61b8057b Binary files /dev/null and b/crates/edgezero-store-celld/src/fixtures/tls-ca.der differ diff --git a/crates/edgezero-store-celld/src/fixtures/tls-expired.der b/crates/edgezero-store-celld/src/fixtures/tls-expired.der new file mode 100644 index 000000000..ca523182c Binary files /dev/null and b/crates/edgezero-store-celld/src/fixtures/tls-expired.der differ diff --git a/crates/edgezero-store-celld/src/fixtures/tls-key.der b/crates/edgezero-store-celld/src/fixtures/tls-key.der new file mode 100644 index 000000000..f176d61ab Binary files /dev/null and b/crates/edgezero-store-celld/src/fixtures/tls-key.der differ diff --git a/crates/edgezero-store-celld/src/fixtures/tls-valid.der b/crates/edgezero-store-celld/src/fixtures/tls-valid.der new file mode 100644 index 000000000..55dd3ca15 Binary files /dev/null and b/crates/edgezero-store-celld/src/fixtures/tls-valid.der differ diff --git a/crates/edgezero-store-celld/src/fixtures/tls-wrong.der b/crates/edgezero-store-celld/src/fixtures/tls-wrong.der new file mode 100644 index 000000000..65165c41c Binary files /dev/null and b/crates/edgezero-store-celld/src/fixtures/tls-wrong.der differ diff --git a/crates/edgezero-store-celld/src/kv.rs b/crates/edgezero-store-celld/src/kv.rs new file mode 100644 index 000000000..41e845335 --- /dev/null +++ b/crates/edgezero-store-celld/src/kv.rs @@ -0,0 +1,707 @@ +//! Live byte-oriented KV. Each method performs at most one bounded exchange. + +#![expect( + clippy::arbitrary_source_item_ordering, + reason = "group the live store and its operations before private wire models and cursor helpers" +)] + +use std::sync::Arc; +use std::time::Duration; + +use base64::Engine as _; +use base64::engine::general_purpose::URL_SAFE_NO_PAD; +use bytes::Bytes; +use edgezero_core::key_value_store::{KvError, KvPage, KvStore}; +use reqwest::Method; +use serde::{Deserialize, Serialize}; +use sha2::{Digest as _, Sha256}; +use url::Url; + +use crate::PreparationError; +use crate::client::{ResponseKind, ScopedClient}; +use crate::settings::{CelldPreparationLimits, valid_key}; + +const MAX_PROVIDER_CURSOR: usize = 8_192; +const MAX_SERVICE_CURSOR: usize = 2_048; + +pub(crate) struct CelldKvStore { + client: Arc, + endpoint: Url, + limits: CelldPreparationLimits, + store: String, +} + +impl CelldKvStore { + pub(crate) const fn new( + client: Arc, + endpoint: Url, + store: String, + limits: CelldPreparationLimits, + ) -> Self { + Self { + client, + endpoint, + limits, + store, + } + } + + fn value_url(&self, key: &str) -> Result { + if !valid_key(key, false) { + return Err(PreparationError::InvalidResource); + } + let mut url = self.client.url(&self.store, "value")?; + url.query_pairs_mut().append_pair("key", key); + Ok(url) + } + + #[expect( + clippy::big_endian_bytes, + reason = "cursor fingerprints frame each field with a canonical big-endian u64 length" + )] + fn fingerprint(&self, prefix: &str) -> Result { + let mut digest = Sha256::new(); + for part in [ + "celld-store-v1", + self.endpoint.as_str(), + &self.store, + prefix, + ] { + let size = u64::try_from(part.len()).map_err(|_error| PreparationError::SizeLimit)?; + digest.update(size.to_be_bytes()); + digest.update(part.as_bytes()); + } + Ok(URL_SAFE_NO_PAD.encode(digest.finalize())) + } + + async fn put( + &self, + key: &str, + value: Bytes, + ttl: Option, + ) -> Result<(), PreparationError> { + let deadline = self.client.deadline()?; + let mut url = self.value_url(key)?; + if u64::try_from(value.len()).map_err(|_error| PreparationError::SizeLimit)? + > self.limits.max_value_bytes + { + return Err(PreparationError::SizeLimit); + } + if let Some(duration) = ttl { + url.query_pairs_mut() + .append_pair("ttl_seconds", &ttl_seconds(duration)?.to_string()); + } + self.client + .execute( + Method::PUT, + url, + Some(value), + 0, + deadline, + ResponseKind::Mutation, + ) + .await?; + Ok(()) + } + + async fn page( + &self, + prefix: &str, + cursor: Option<&str>, + limit: usize, + ) -> Result { + let deadline = self.client.deadline()?; + if !valid_key(prefix, true) || !(1..=1_000).contains(&limit) { + return Err(PreparationError::InvalidResource); + } + let fingerprint = self.fingerprint(prefix)?; + let prior = cursor + .map(|encoded| decode_cursor(encoded, &fingerprint, prefix)) + .transpose()?; + let mut url = self.client.url(&self.store, "keys")?; + { + let mut query = url.query_pairs_mut(); + query + .append_pair("prefix", prefix) + .append_pair("limit", &limit.to_string()); + if let Some(previous) = &prior { + query.append_pair("cursor", &previous.service); + } + } + let cap = usize::try_from(self.limits.max_page_bytes) + .map_err(|_error| PreparationError::InvalidLimits)?; + let body = self + .client + .execute(Method::GET, url, None, cap, deadline, ResponseKind::Page) + .await? + .ok_or(PreparationError::Malformed)?; + let result = (|| { + let page: WirePage = + serde_json::from_slice(&body).map_err(|_error| PreparationError::Malformed)?; + if page.keys.len() > limit { + return Err(PreparationError::SizeLimit); + } + let mut last = prior.as_ref().and_then(|previous| previous.last.as_deref()); + for key in &page.keys { + if !valid_key(key, false) + || !key.starts_with(prefix) + || last.is_some_and(|previous_key| key.as_str() <= previous_key) + { + return Err(PreparationError::Malformed); + } + last = Some(key); + } + let next = if let Some(service) = page.cursor { + if service.is_empty() + || service.len() > MAX_SERVICE_CURSOR + || prior + .as_ref() + .is_some_and(|previous| service == previous.service) + { + return Err(PreparationError::Malformed); + } + let envelope = Cursor { + version: 1, + fingerprint, + last: last.map(str::to_owned), + service, + }; + let bytes = + serde_json::to_vec(&envelope).map_err(|_error| PreparationError::Malformed)?; + let encoded = URL_SAFE_NO_PAD.encode(bytes); + if encoded.len() > MAX_PROVIDER_CURSOR { + return Err(PreparationError::SizeLimit); + } + Some(encoded) + } else { + None + }; + Ok(KvPage { + keys: page.keys, + cursor: next, + }) + })(); + // Includes parsing/validation time and gives deadline precedence. + self.client.check(deadline)?; + result + } +} + +#[async_trait::async_trait(?Send)] +impl KvStore for CelldKvStore { + async fn delete(&self, key: &str) -> Result<(), KvError> { + let deadline = self.client.deadline().map_err(kv_error)?; + let url = self.value_url(key).map_err(kv_error)?; + self.client + .execute( + Method::DELETE, + url, + None, + 0, + deadline, + ResponseKind::Mutation, + ) + .await + .map_err(kv_error)?; + Ok(()) + } + + async fn exists(&self, key: &str) -> Result { + let deadline = self.client.deadline().map_err(kv_error)?; + let url = self.value_url(key).map_err(kv_error)?; + self.client + .execute(Method::HEAD, url, None, 0, deadline, ResponseKind::Exists) + .await + .map(|body| body.is_some()) + .map_err(kv_error) + } + + async fn get_bytes(&self, key: &str) -> Result, KvError> { + let deadline = self.client.deadline().map_err(kv_error)?; + let url = self.value_url(key).map_err(kv_error)?; + let cap = usize::try_from(self.limits.max_value_bytes) + .map_err(|_error| kv_error(PreparationError::InvalidLimits))?; + self.client + .execute(Method::GET, url, None, cap, deadline, ResponseKind::Value) + .await + .map(|body| body.map(Bytes::from)) + .map_err(kv_error) + } + + async fn list_keys_page( + &self, + prefix: &str, + cursor: Option<&str>, + limit: usize, + ) -> Result { + self.page(prefix, cursor, limit).await.map_err(kv_error) + } + + async fn put_bytes(&self, key: &str, value: Bytes) -> Result<(), KvError> { + self.put(key, value, None).await.map_err(kv_error) + } + + async fn put_bytes_with_ttl( + &self, + key: &str, + value: Bytes, + ttl: Duration, + ) -> Result<(), KvError> { + self.put(key, value, Some(ttl)).await.map_err(kv_error) + } +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct WirePage { + keys: Vec, + #[serde(deserialize_with = "optional_string")] + cursor: Option, +} + +fn optional_string<'de, D: serde::Deserializer<'de>>( + deserializer: D, +) -> Result, D::Error> { + Option::::deserialize(deserializer) +} + +#[derive(Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +struct Cursor { + version: u8, + fingerprint: String, + last: Option, + service: String, +} + +fn decode_cursor(text: &str, fingerprint: &str, prefix: &str) -> Result { + if text.is_empty() || text.len() > MAX_PROVIDER_CURSOR { + return Err(PreparationError::InvalidResource); + } + let bytes = URL_SAFE_NO_PAD + .decode(text) + .map_err(|_error| PreparationError::InvalidResource)?; + let cursor: Cursor = + serde_json::from_slice(&bytes).map_err(|_error| PreparationError::InvalidResource)?; + if cursor.version != 1 + || cursor.fingerprint != fingerprint + || cursor.service.is_empty() + || cursor.service.len() > MAX_SERVICE_CURSOR + || cursor + .last + .as_deref() + .is_some_and(|last| !valid_key(last, false) || !last.starts_with(prefix)) + { + return Err(PreparationError::InvalidResource); + } + Ok(cursor) +} + +fn ttl_seconds(ttl: Duration) -> Result { + // Validate the direct trait input BEFORE rounding. A sub-minimum or + // maximum-plus-one-nanosecond duration must not become a valid wire TTL. + if ttl < Duration::from_mins(1) || ttl > Duration::from_hours(8_760) { + return Err(PreparationError::InvalidResource); + } + ttl.as_secs() + .checked_add(u64::from(ttl.subsec_nanos() != 0)) + .ok_or(PreparationError::InvalidResource) +} + +fn kv_error(error: PreparationError) -> KvError { + match error { + PreparationError::InvalidBinding + | PreparationError::InvalidEndpoint + | PreparationError::InvalidLimits + | PreparationError::InvalidResource => KvError::Validation(error.to_string()), + PreparationError::SizeLimit => KvError::LimitExceeded { + message: "celld size limit".to_owned(), + }, + // Core has no typed auth/protocol/deadline variants. Do not smuggle + // remote JSON or transport source chains into its generic error variants. + PreparationError::Credentials + | PreparationError::Unauthenticated + | PreparationError::Forbidden + | PreparationError::Missing + | PreparationError::Malformed + | PreparationError::UnsupportedVersion + | PreparationError::Unavailable + | PreparationError::Deadline => KvError::Unavailable, + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + use std::sync::atomic::{AtomicU64, Ordering}; + use std::time::Duration; + + use bytes::Bytes; + use edgezero_core::key_value_store::KvStore as _; + use edgezero_core::time::{MonotonicClock, MonotonicInstant}; + use url::Url; + + use super::{CelldKvStore, ttl_seconds}; + use crate::CelldPreparation; + use crate::client::ScopedClient; + use crate::settings::{CelldAccess, CelldKvSettings, CelldPreparationLimits}; + use crate::test_support::{Fixture, Reply}; + + fn store(fixture: &Fixture, alias: &str) -> CelldKvStore { + let endpoint = Url::parse(fixture.endpoint()).unwrap(); + let limits = CelldPreparationLimits::default(); + let client = Arc::new( + ScopedClient::new(endpoint.clone(), None, MonotonicClock::default(), limits).unwrap(), + ); + CelldKvStore::new(client, endpoint, alias.to_owned(), limits) + } + + #[tokio::test(flavor = "current_thread")] + async fn public_setup_is_probe_free_and_runtime_deadlines_are_fresh() { + let fixture = Fixture::new(vec![]).await; + let now = MonotonicInstant::now(); + let elapsed = Arc::new(AtomicU64::new(0)); + let ticks = Arc::clone(&elapsed); + let clock = MonotonicClock::new(move || { + now.checked_add(Duration::from_millis(ticks.load(Ordering::SeqCst))) + .unwrap() + }); + let settings = CelldKvSettings { + endpoint: fixture.endpoint().to_owned(), + access: CelldAccess::TrustedNetwork { + allow_remote_http: false, + }, + store: "state".to_owned(), + }; + let mut preparation = + CelldPreparation::new(clock, CelldPreparationLimits::default()).unwrap(); + let kv = preparation.prepare_kv(&settings, None).await.unwrap(); + assert_eq!(fixture.count().await, 0); + assert_eq!(preparation.stats().mappings, 1); + assert_eq!(preparation.stats().unique_retained_payload_bytes, 0); + elapsed.store(60_000, Ordering::SeqCst); + kv.put_bytes("k", Bytes::from_static(&[0, 255, 7])) + .await + .unwrap(); + assert_eq!( + kv.get_bytes("k").await.unwrap().unwrap(), + Bytes::from_static(&[0, 255, 7]) + ); + assert!(kv.exists("k").await.unwrap()); + kv.put_bytes("k", Bytes::new()).await.unwrap(); + assert_eq!(kv.get_bytes("k").await.unwrap(), Some(Bytes::new())); + kv.delete("k").await.unwrap(); + kv.delete("k").await.unwrap(); + assert!(!kv.exists("k").await.unwrap()); + assert_eq!(kv.get_bytes("k").await.unwrap(), None); + assert_eq!(preparation.stats().unique_retained_payload_bytes, 0); + } + + #[tokio::test(flavor = "current_thread")] + async fn aliases_isolate_and_explicit_sharing_preserves_exact_unicode_keys() { + let fixture = Fixture::new(vec![]).await; + let first = store(&fixture, "first"); + let shared = store(&fixture, "first"); + let second = store(&fixture, "second"); + for key in ["\u{e9}", "e\u{301}", &"\u{e9}".repeat(256)] { + first + .put_bytes(key, Bytes::copy_from_slice(key.as_bytes())) + .await + .unwrap(); + assert_eq!( + shared.get_bytes(key).await.unwrap().unwrap().as_ref(), + key.as_bytes() + ); + assert_eq!(second.get_bytes(key).await.unwrap(), None); + } + let count = fixture.count().await; + first + .get_bytes(&"x".repeat(513)) + .await + .expect_err("overlong key must be rejected before transport"); + first + .get_bytes("control\n") + .await + .expect_err("control character in key must be rejected before transport"); + assert_eq!(fixture.count().await, count); + } + + #[tokio::test(flavor = "current_thread")] + async fn pagination_is_one_page_and_cursors_bind_origin_store_prefix_and_progress() { + let fixture = Fixture::new(vec![]).await; + for key in ["app/a", "app/b", "app/c", "other"] { + fixture.seed("state", key, b"v").await; + } + let kv = store(&fixture, "state"); + let page = kv.list_keys_page("app/", None, 1).await.unwrap(); + assert_eq!(page.keys, ["app/a"]); + let cursor = page.cursor.unwrap(); + let next = kv.list_keys_page("app/", Some(&cursor), 1).await.unwrap(); + assert_eq!(next.keys, ["app/b"]); + assert_eq!(fixture.count().await, 2); + let other_fixture = Fixture::new(vec![]).await; + store(&other_fixture, "state") + .list_keys_page("app/", Some(&cursor), 1) + .await + .expect_err("cursor from another origin must be rejected"); + assert_eq!(other_fixture.count().await, 0); + store(&fixture, "other") + .list_keys_page("app/", Some(&cursor), 1) + .await + .expect_err("cursor from another store must be rejected"); + kv.list_keys_page("different", Some(&cursor), 1) + .await + .expect_err("cursor from another prefix must be rejected"); + kv.list_keys_page("app/", Some(&"x".repeat(8_193)), 1) + .await + .expect_err("overlong provider cursor must be rejected"); + assert_eq!(fixture.count().await, 2); + let last = kv + .list_keys_page("app/", next.cursor.as_deref(), 1) + .await + .unwrap(); + assert_eq!(last.keys, ["app/c"]); + assert_eq!(last.cursor, None); + } + + #[tokio::test(flavor = "current_thread")] + async fn empty_changed_continuation_is_returned_once_and_retains_last_key() { + let fixture = Fixture::new(vec![ + Reply::response( + 200, + "application/json", + br#"{"keys":["p/a"],"cursor":"one"}"#, + ), + Reply::response(200, "application/json", br#"{"keys":[],"cursor":"two"}"#), + Reply::response( + 200, + "application/json", + br#"{"keys":["p/a"],"cursor":null}"#, + ), + ]) + .await; + let kv = store(&fixture, "state"); + let first = kv.list_keys_page("p/", None, 10).await.unwrap(); + let empty = kv + .list_keys_page("p/", first.cursor.as_deref(), 10) + .await + .unwrap(); + assert!(empty.keys.is_empty()); + assert!(empty.cursor.is_some()); + assert_eq!(fixture.count().await, 2); + kv.list_keys_page("p/", empty.cursor.as_deref(), 10) + .await + .expect_err("continuation must not repeat the previous page's last key"); + assert_eq!(fixture.count().await, 3); + } + + #[tokio::test(flavor = "current_thread")] + async fn pages_reject_unknown_duplicate_missing_malformed_and_unordered_fields() { + for bytes in [ + br#"{"keys":["z"],"cursor":null}"#.as_slice(), + br#"{"keys":["p/b","p/a"],"cursor":null}"#, + br#"{"keys":["p/a","p/a"],"cursor":null}"#, + br#"{"keys":[],"cursor":""}"#, + br#"{"keys":[],"cursor":null,"extra":1}"#, + br#"{"keys":[],"keys":[],"cursor":null}"#, + br#"{"keys":[]}"#, + b"not json", + ] { + let fixture = Fixture::new(vec![Reply::response(200, "application/json", bytes)]).await; + store(&fixture, "state") + .list_keys_page("p/", None, 100) + .await + .expect_err("malformed or unordered page must be rejected"); + } + let too_large_cursor = + serde_json::to_vec(&serde_json::json!({"keys":[], "cursor":"x".repeat(2_049)})) + .unwrap(); + let fixture = Fixture::new(vec![Reply::response( + 200, + "application/json", + &too_large_cursor, + )]) + .await; + store(&fixture, "state") + .list_keys_page("", None, 1_000) + .await + .expect_err("overlong service cursor must be rejected"); + let repeated_cursor_fixture = Fixture::new(vec![ + Reply::response(200, "application/json", br#"{"keys":[],"cursor":"same"}"#), + Reply::response(200, "application/json", br#"{"keys":[],"cursor":"same"}"#), + ]) + .await; + let kv = store(&repeated_cursor_fixture, "state"); + let first = kv.list_keys_page("", None, 1).await.unwrap(); + kv.list_keys_page("", first.cursor.as_deref(), 1) + .await + .expect_err("unchanged service cursor must be rejected"); + } + + #[tokio::test(flavor = "current_thread")] + async fn ttl_wire_rounding_renewal_and_plain_reset_are_explicit() { + let fixture = Fixture::new(vec![]).await; + let kv = store(&fixture, "state"); + kv.put_bytes_with_ttl("k", Bytes::new(), Duration::from_millis(1)) + .await + .expect_err("sub-minimum TTL must be rejected before transport"); + assert_eq!(fixture.count().await, 0); + kv.put_bytes_with_ttl( + "k", + Bytes::new(), + Duration::from_mins(1) + Duration::from_nanos(1), + ) + .await + .unwrap(); + kv.put_bytes_with_ttl("k", Bytes::new(), Duration::from_mins(2)) + .await + .unwrap(); + kv.put_bytes("k", Bytes::new()).await.unwrap(); + let requests = fixture.requests().await; + assert!(requests[0].target.contains("ttl_seconds=61")); + assert!(requests[1].target.contains("ttl_seconds=120")); + assert!(!requests[2].target.contains("ttl_seconds")); + // These are wire assertions, not proof of real service elapsed expiry. + } + + #[tokio::test(flavor = "current_thread")] + async fn applied_then_lost_put_and_delete_are_not_replayed_fresh_or_reused() { + for reused in [false, true] { + for delete in [false, true] { + let replies = if reused { + vec![Reply::response(404, "application/json", b""), Reply::Drop] + } else { + vec![Reply::Drop] + }; + let fixture = Fixture::new(replies).await; + fixture.seed("state", "mut", b"original").await; + let kv = store(&fixture, "state"); + if reused { + assert!(!kv.exists("warm").await.unwrap()); + } + let result = if delete { + kv.delete("mut").await + } else { + kv.put_bytes("mut", Bytes::from_static(b"applied")).await + }; + result.expect_err("lost mutation reply must return an error without replay"); + assert_eq!( + fixture.value("state", "mut").await, + if delete { + None + } else { + Some(b"applied".to_vec()) + } + ); + assert_eq!(fixture.count().await, if reused { 2 } else { 1 }); + let requests = fixture.requests().await; + if reused { + assert_eq!(requests[0].connection, requests[1].connection); + } + } + } + } + + #[tokio::test(flavor = "current_thread")] + async fn timed_out_write_may_have_applied_and_is_not_replayed() { + let fixture = Fixture::new(vec![Reply::Stall]).await; + let limits = CelldPreparationLimits { + operation_timeout_ms: 40, + connect_timeout_ms: 10, + ..CelldPreparationLimits::default() + }; + let endpoint = Url::parse(fixture.endpoint()).unwrap(); + let client = Arc::new( + ScopedClient::new(endpoint.clone(), None, MonotonicClock::default(), limits).unwrap(), + ); + let kv = CelldKvStore::new(client, endpoint, "state".to_owned(), limits); + kv.put_bytes("mut", Bytes::from_static(b"applied")) + .await + .expect_err("stalled mutation reply must reach the operation deadline"); + assert_eq!(fixture.value("state", "mut").await.unwrap(), b"applied"); + assert_eq!(fixture.count().await, 1); + } + + #[tokio::test(flavor = "current_thread")] + async fn provider_caps_and_direct_trait_input_checks_precede_requests() { + let fixture = Fixture::new(vec![]).await; + let endpoint = Url::parse(fixture.endpoint()).unwrap(); + let limits = CelldPreparationLimits { + max_value_bytes: 2, + max_config_document_bytes: 2, + max_page_bytes: 4, + ..CelldPreparationLimits::default() + }; + let client = Arc::new( + ScopedClient::new(endpoint.clone(), None, MonotonicClock::default(), limits).unwrap(), + ); + let kv = CelldKvStore::new(client, endpoint, "state".to_owned(), limits); + kv.put_bytes("k", Bytes::from_static(b"123")) + .await + .expect_err("value above the provider cap must be rejected before transport"); + kv.get_bytes("") + .await + .expect_err("empty value key must be rejected before transport"); + kv.list_keys_page("", None, 0) + .await + .expect_err("zero page size must be rejected before transport"); + kv.list_keys_page("", None, 1_001) + .await + .expect_err("page size above the service maximum must be rejected before transport"); + assert_eq!(fixture.count().await, 0); + fixture.seed("state", "k", b"123").await; + kv.get_bytes("k") + .await + .expect_err("response value above the provider cap must be rejected"); + kv.list_keys_page("", None, 1) + .await + .expect_err("response page above the provider cap must be rejected"); + assert_eq!(fixture.count().await, 2); + } + + #[tokio::test(flavor = "current_thread")] + async fn head_and_mutations_require_version_and_their_exact_success_status() { + for reply in [ + Reply::custom(404, "application/json", "", b""), + Reply::response(204, "application/octet-stream", b""), + ] { + let fixture = Fixture::new(vec![reply]).await; + store(&fixture, "state") + .exists("k") + .await + .expect_err("HEAD with invalid version or status must be rejected"); + } + for reply in [ + Reply::custom(204, "application/octet-stream", "", b""), + Reply::response(200, "application/octet-stream", b""), + Reply::response(201, "application/octet-stream", b""), + ] { + let fixture = Fixture::new(vec![reply]).await; + store(&fixture, "state") + .delete("k") + .await + .expect_err("mutation with invalid version or status must be rejected"); + } + } + + #[test] + fn ttl_validates_then_rounds_up_without_shortening() { + assert_eq!(ttl_seconds(Duration::from_mins(1)).unwrap(), 60); + assert_eq!( + ttl_seconds(Duration::from_mins(1) + Duration::from_nanos(1)).unwrap(), + 61 + ); + assert_eq!( + ttl_seconds(Duration::from_hours(8_760)).unwrap(), + 31_536_000 + ); + for ttl in [ + Duration::ZERO, + Duration::from_millis(59_999), + Duration::from_hours(8_760) + Duration::from_nanos(1), + Duration::MAX, + ] { + ttl_seconds(ttl).expect_err("out-of-range TTL must be rejected before rounding"); + } + } +} diff --git a/crates/edgezero-store-celld/src/lib.rs b/crates/edgezero-store-celld/src/lib.rs new file mode 100644 index 000000000..5a5bbce7b --- /dev/null +++ b/crates/edgezero-store-celld/src/lib.rs @@ -0,0 +1,63 @@ +//! Scoped native celld stores. Settings are available without runtime features. +//! +//! Config is an immutable startup snapshot; KV is live. Owned errors retain no +//! endpoint, credential, resource, remote body or underlying transport error. + +#![cfg_attr( + all(not(target_arch = "wasm32"), feature = "_transport"), + expect( + clippy::pub_use, + reason = "the supported preparation API is re-exported at the crate root; this lint is module-scoped" + ) +)] + +pub mod settings; + +#[cfg(all(not(target_arch = "wasm32"), feature = "_transport"))] +mod client; +#[cfg(all(not(target_arch = "wasm32"), feature = "config"))] +mod config; +#[cfg(all(not(target_arch = "wasm32"), feature = "kv"))] +mod kv; +#[cfg(all(not(target_arch = "wasm32"), feature = "_transport"))] +mod preparation; +#[cfg(test)] +#[cfg(all(not(target_arch = "wasm32"), feature = "_transport"))] +mod test_support; +#[cfg(all(not(target_arch = "wasm32"), feature = "_transport"))] +pub use preparation::{CelldPreparation, PreparationStats}; + +/// Closed failure categories, never transport error chains or server text. +#[derive(Clone, Copy, Debug, Eq, PartialEq, thiserror::Error)] +#[expect( + clippy::arbitrary_source_item_ordering, + reason = "keep binding validation, service failures and transport failures grouped in the agreed closed API" +)] +pub enum PreparationError { + #[error("invalid binding")] + InvalidBinding, + #[error("invalid endpoint")] + InvalidEndpoint, + #[error("invalid preparation limits")] + InvalidLimits, + #[error("invalid resource")] + InvalidResource, + #[error("credential failure")] + Credentials, + #[error("unauthenticated")] + Unauthenticated, + #[error("forbidden")] + Forbidden, + #[error("missing resource")] + Missing, + #[error("size limit")] + SizeLimit, + #[error("malformed response")] + Malformed, + #[error("unsupported version")] + UnsupportedVersion, + #[error("provider unavailable")] + Unavailable, + #[error("deadline exceeded")] + Deadline, +} diff --git a/crates/edgezero-store-celld/src/preparation.rs b/crates/edgezero-store-celld/src/preparation.rs new file mode 100644 index 000000000..c1ba95a80 --- /dev/null +++ b/crates/edgezero-store-celld/src/preparation.rs @@ -0,0 +1,715 @@ +//! One startup's aggregate clock, accounting and authorization-scoped caches. + +#![expect( + clippy::arbitrary_source_item_ordering, + reason = "group public session operations before private accounting and feature-gated preparation" +)] + +use std::fmt; +use std::sync::Arc; +use std::time::Duration; + +#[cfg(feature = "config")] +use edgezero_core::config_store::{ConfigStoreHandle, ConfigValue}; +#[cfg(feature = "kv")] +use edgezero_core::key_value_store::KvHandle; +use edgezero_core::time::{Deadline, MonotonicClock}; +#[cfg(feature = "config")] +use reqwest::Method; +#[cfg(feature = "config")] +use std::collections::{BTreeMap, HashMap}; + +use crate::PreparationError; +#[cfg(feature = "config")] +use crate::client::ResponseKind; +use crate::client::ScopedClient; +#[cfg(feature = "config")] +use crate::config::ConfigSnapshot; +#[cfg(feature = "kv")] +use crate::kv::CelldKvStore; +#[cfg(feature = "config")] +use crate::settings::CelldConfigSettings; +#[cfg(feature = "kv")] +use crate::settings::CelldKvSettings; +use crate::settings::{CelldAccess, CelldAccessToken, CelldPreparationLimits, endpoint}; + +/// Aggregate counters, not process memory measurements. Metadata counts bounded +/// identifier storage plus conservative per-record overhead. Staging includes at +/// least two document-sized buffers, but excludes allocator and HTTP chunk overhead. +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub struct PreparationStats { + pub mappings: u64, + pub unique_retained_payload_bytes: u64, + pub metadata_bytes: u64, + pub peak_transient_staging_bytes: u64, +} + +/// A session is permanently poisoned after any failed or cancelled preparation. +/// Handles already returned are immutable/config or independently live/KV. +pub struct CelldPreparation { + clients: Vec<(ClientIdentity, Arc)>, + clock: MonotonicClock, + deadline: Deadline, + limits: CelldPreparationLimits, + poisoned: bool, + stats: PreparationStats, + #[cfg(feature = "config")] + values: HashMap, +} + +impl fmt::Debug for CelldPreparation { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("CelldPreparation") + .field("stats", &self.stats) + .finish_non_exhaustive() + } +} + +impl CelldPreparation { + /// Starts one shared startup deadline, before any client construction. + /// + /// # Errors + /// Rejects invalid limits and unrepresentable deadlines. + #[inline] + pub fn new( + clock: MonotonicClock, + limits: CelldPreparationLimits, + ) -> Result { + limits.validate()?; + let deadline = clock + .now() + .checked_add(Duration::from_millis(limits.preparation_timeout_ms)) + .map(Deadline::at_instant) + .ok_or(PreparationError::InvalidLimits)?; + Ok(Self { + clients: Vec::new(), + clock, + deadline, + limits, + poisoned: false, + stats: PreparationStats::default(), + #[cfg(feature = "config")] + values: HashMap::new(), + }) + } + + /// Fetches all mapped complete UTF-8 documents, with no stale/local fallback. + /// + /// # Errors + /// Returns a closed category on validation, quota, deadline or protocol failure. + /// A failure poisons the session, including failed validation. + #[cfg(feature = "config")] + #[inline] + pub async fn prepare_config( + &mut self, + settings: &CelldConfigSettings, + token: Option<&CelldAccessToken>, + ) -> Result { + self.begin()?; + let result = self.config_inner(settings, token).await; + if result.is_ok() { + self.poisoned = false; + } + result + } + + /// Builds and accounts a live handle. This intentionally performs no remote + /// probe and makes no claim about connectivity or any operation permission. + /// Live calls start fresh deadlines, independent of this session's deadline. + /// + /// # Errors + /// Returns a closed category on invalid input, quota or client construction. + #[cfg(feature = "kv")] + #[inline] + #[expect( + clippy::unused_async, + reason = "the agreed preparation API is uniformly async; live KV setup deliberately performs no remote probe" + )] + pub async fn prepare_kv( + &mut self, + settings: &CelldKvSettings, + token: Option<&CelldAccessToken>, + ) -> Result { + self.begin()?; + let result = self.kv_inner(settings, token); + if result.is_ok() { + self.poisoned = false; + } + result + } + + /// Safe aggregate counters; no resource or authorization identifiers. + #[inline] + #[must_use] + pub const fn stats(&self) -> PreparationStats { + self.stats + } + + fn begin(&mut self) -> Result<(), PreparationError> { + if self.poisoned { + return Err(PreparationError::InvalidBinding); + } + self.poisoned = true; + self.check() + } + + fn check(&self) -> Result<(), PreparationError> { + if self.deadline.is_expired_at(self.clock.now()) { + return Err(PreparationError::Deadline); + } + Ok(()) + } + + fn charge_metadata(&mut self, bytes: usize) -> Result<(), PreparationError> { + self.stats.metadata_bytes = u64::try_from(bytes) + .ok() + .and_then(|count| self.stats.metadata_bytes.checked_add(count)) + .filter(|total| *total <= self.limits.max_metadata_bytes) + .ok_or(PreparationError::SizeLimit)?; + Ok(()) + } + + fn charge_mapping(&mut self, sizes: &[usize]) -> Result<(), PreparationError> { + self.stats.mappings = self + .stats + .mappings + .checked_add(1) + .filter(|count| *count <= self.limits.max_remote_values) + .ok_or(PreparationError::SizeLimit)?; + // Covers map nodes, strings in snapshot/cache targets and temporary target + // lists conservatively. This is a logical metadata quota, not RSS. + let bytes = sizes.iter().try_fold(512_usize, |total, size| { + size.checked_mul(3) + .and_then(|scaled_size| total.checked_add(scaled_size)) + .ok_or(PreparationError::SizeLimit) + })?; + self.charge_metadata(bytes) + } + + fn client( + &mut self, + endpoint_text: &str, + access: &CelldAccess, + token: Option<&CelldAccessToken>, + ) -> Result<(usize, Arc), PreparationError> { + if access.token_env().is_some() != token.is_some() { + return Err(PreparationError::Credentials); + } + let endpoint = endpoint(endpoint_text, access)?; + let identity = ClientIdentity { + endpoint: endpoint.as_str().to_owned(), + access: access.clone(), + token: token.map(|credential| credential.expose().to_owned()), + }; + if let Some((index, (_, client))) = self + .clients + .iter() + .enumerate() + .find(|(_, (key, _))| *key == identity) + { + return Ok((index, Arc::clone(client))); + } + if self.clients.len() >= 128 { + return Err(PreparationError::SizeLimit); + } + let metadata = [ + identity.endpoint.len(), + access.token_env().map_or(0, str::len), + token.map_or(0, |credential| credential.expose().len()), + ] + .into_iter() + .try_fold(1_024_usize, |total, size| { + size.checked_mul(3) + .and_then(|scaled_size| total.checked_add(scaled_size)) + .ok_or(PreparationError::SizeLimit) + })?; + self.charge_metadata(metadata)?; + let client = Arc::new(ScopedClient::new( + endpoint, + token, + self.clock.clone(), + self.limits, + )?); + self.check()?; + let index = self.clients.len(); + self.clients.push((identity, Arc::clone(&client))); + Ok((index, client)) + } + + #[cfg(feature = "config")] + async fn config_inner( + &mut self, + settings: &CelldConfigSettings, + token: Option<&CelldAccessToken>, + ) -> Result { + settings.validate()?; + // Charge every logical mapping before deduplication and before the first + // client/fetch, including mappings aliasing an already retained document. + for (key, document) in &settings.documents { + self.charge_mapping(&[ + key.len(), + settings.endpoint.len(), + document.store.len(), + document.key.len(), + ])?; + } + let (client_id, client) = self.client(&settings.endpoint, &settings.access, token)?; + let mut entries = BTreeMap::new(); + for (key, document) in &settings.documents { + self.check()?; + let target = TargetIdentity { + client_id, + store: document.store.clone(), + key: document.key.clone(), + }; + let value = if let Some(value) = self.values.get(&target) { + value.clone() + } else { + let remaining = self + .limits + .max_snapshot_bytes + .checked_sub(self.stats.unique_retained_payload_bytes) + .ok_or(PreparationError::SizeLimit)?; + let cap = usize::try_from(remaining.min(self.limits.max_config_document_bytes)) + .map_err(|_error| PreparationError::InvalidLimits)?; + let mut url = client.url(&document.store, "value")?; + url.query_pairs_mut().append_pair("key", &document.key); + let deadline = + Deadline::at_instant(self.deadline.instant().min(client.deadline()?.instant())); + let body = client + .execute( + Method::GET, + url, + None, + cap, + deadline, + ResponseKind::Document, + ) + .await? + .ok_or(PreparationError::Missing)?; + self.check()?; + let bytes = + u64::try_from(body.len()).map_err(|_error| PreparationError::SizeLimit)?; + self.stats.peak_transient_staging_bytes = self + .stats + .peak_transient_staging_bytes + .max(bytes.checked_mul(2).ok_or(PreparationError::SizeLimit)?); + let text = String::from_utf8(body).map_err(|_error| PreparationError::Malformed); + self.check()?; + let value = ConfigValue::from(text?); + self.stats.unique_retained_payload_bytes = self + .stats + .unique_retained_payload_bytes + .checked_add(bytes) + .filter(|total| *total <= self.limits.max_snapshot_bytes) + .ok_or(PreparationError::SizeLimit)?; + self.values.insert(target, value.clone()); + value + }; + entries.insert(key.clone(), value); + } + self.check()?; + Ok(ConfigStoreHandle::new(Arc::new(ConfigSnapshot::new( + entries, + )))) + } + + #[cfg(feature = "kv")] + fn kv_inner( + &mut self, + settings: &CelldKvSettings, + token: Option<&CelldAccessToken>, + ) -> Result { + settings.validate()?; + self.charge_mapping(&[settings.endpoint.len(), settings.store.len()])?; + let (_, client) = self.client(&settings.endpoint, &settings.access, token)?; + let endpoint = endpoint(&settings.endpoint, &settings.access)?; + self.check()?; + Ok(KvHandle::new(Arc::new(CelldKvStore::new( + client, + endpoint, + settings.store.clone(), + self.limits, + )))) + } +} + +// Session policy is immutable, so it is implicitly part of every cache identity. +// Access includes the variable reference and remote-HTTP policy, not just bearer +// bytes. Equal tokens selected through different references remain separate. +#[derive(Eq, PartialEq)] +struct ClientIdentity { + endpoint: String, + access: CelldAccess, + token: Option, +} + +impl fmt::Debug for ClientIdentity { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ClientIdentity").finish_non_exhaustive() + } +} + +#[cfg(feature = "config")] +#[derive(Eq, Hash, PartialEq)] +struct TargetIdentity { + client_id: usize, + store: String, + key: String, +} + +#[cfg(feature = "config")] +impl fmt::Debug for TargetIdentity { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("TargetIdentity").finish_non_exhaustive() + } +} + +#[cfg(test)] +mod tests { + use edgezero_core::time::MonotonicClock; + + use super::CelldPreparation; + use crate::PreparationError; + use crate::settings::{CelldAccess, CelldAccessToken, CelldPreparationLimits}; + + #[test] + fn cache_identity_includes_credentials_reference_and_complete_access_policy() { + let mut preparation = + CelldPreparation::new(MonotonicClock::default(), CelldPreparationLimits::default()) + .unwrap(); + let access = CelldAccess::Authenticated { + token_env: "FIRST".to_owned(), + }; + let token = CelldAccessToken::new("a".repeat(32)).unwrap(); + let (first, _) = preparation + .client("https://resource-sentinel.invalid", &access, Some(&token)) + .unwrap(); + let (same, _) = preparation + .client("https://resource-sentinel.invalid/", &access, Some(&token)) + .unwrap(); + assert_eq!(first, same); + let other = CelldAccessToken::new("b".repeat(32)).unwrap(); + assert_ne!( + preparation + .client("https://resource-sentinel.invalid", &access, Some(&other)) + .unwrap() + .0, + first + ); + let other_reference = CelldAccess::Authenticated { + token_env: "SECOND".to_owned(), + }; + assert_ne!( + preparation + .client( + "https://resource-sentinel.invalid", + &other_reference, + Some(&token) + ) + .unwrap() + .0, + first + ); + let trusted = CelldAccess::TrustedNetwork { + allow_remote_http: false, + }; + let trusted_opt_in = CelldAccess::TrustedNetwork { + allow_remote_http: true, + }; + let (trusted_id, _) = preparation + .client("https://resource-sentinel.invalid", &trusted, None) + .unwrap(); + assert_ne!( + preparation + .client("https://resource-sentinel.invalid", &trusted_opt_in, None) + .unwrap() + .0, + trusted_id + ); + assert!(matches!( + preparation.client("https://resource-sentinel.invalid", &trusted, Some(&token)), + Err(PreparationError::Credentials) + )); + assert!(matches!( + preparation.client("https://resource-sentinel.invalid", &access, None), + Err(PreparationError::Credentials) + )); + assert!(!format!("{preparation:?} {:?}", preparation.clients[0].0).contains("sentinel")); + assert_eq!(preparation.stats().unique_retained_payload_bytes, 0); + } + + #[test] + fn client_cache_and_metadata_are_finite_for_direct_callers() { + let access = CelldAccess::TrustedNetwork { + allow_remote_http: false, + }; + let mut preparation = + CelldPreparation::new(MonotonicClock::default(), CelldPreparationLimits::default()) + .unwrap(); + for index in 0..128_u16 { + preparation + .client( + &format!("https://127.0.0.1:{}", 1_000 + index), + &access, + None, + ) + .unwrap(); + } + assert!(matches!( + preparation.client("https://127.0.0.1:2000", &access, None), + Err(PreparationError::SizeLimit) + )); + assert_eq!(preparation.clients.len(), 128); + let limits = CelldPreparationLimits { + max_metadata_bytes: 1, + ..CelldPreparationLimits::default() + }; + let mut metadata_limited = + CelldPreparation::new(MonotonicClock::default(), limits).unwrap(); + assert!(matches!( + metadata_limited.client("https://127.0.0.1", &access, None), + Err(PreparationError::SizeLimit) + )); + assert!(metadata_limited.clients.is_empty()); + } +} + +#[cfg(test)] +#[cfg(feature = "config")] +mod config_tests { + use std::collections::BTreeMap; + use std::sync::Arc; + use std::sync::atomic::{AtomicU64, Ordering}; + use std::time::Duration; + + use edgezero_core::time::{MonotonicClock, MonotonicInstant}; + use tokio::time::sleep; + + use super::CelldPreparation; + use crate::PreparationError; + use crate::settings::{ + CelldAccess, CelldConfigSettings, CelldDocument, CelldPreparationLimits, + }; + use crate::test_support::{Fixture, Reply}; + + fn settings(fixture: &Fixture, mappings: &[(&str, &str)]) -> CelldConfigSettings { + CelldConfigSettings { + endpoint: fixture.endpoint().to_owned(), + access: CelldAccess::TrustedNetwork { + allow_remote_http: false, + }, + default_key: mappings + .first() + .expect("config fixture requires a default mapping") + .0 + .to_owned(), + documents: mappings + .iter() + .map(|(local, remote)| { + ( + (*local).to_owned(), + CelldDocument { + store: "config".to_owned(), + key: (*remote).to_owned(), + }, + ) + }) + .collect::>(), + } + } + + #[tokio::test(flavor = "current_thread")] + async fn exact_bytes_dedup_and_restart_adoption_without_request_time_io() { + let fixture = Fixture::new(vec![]).await; + let text = " {\"schema\": 1, \"payload\": {\"a\": 1}}\n"; + fixture.seed("config", "remote", text.as_bytes()).await; + fixture.seed("config", "empty", b"").await; + let settings = settings( + &fixture, + &[("first", "remote"), ("alias", "remote"), ("empty", "empty")], + ); + let mut preparation = + CelldPreparation::new(MonotonicClock::default(), CelldPreparationLimits::default()) + .unwrap(); + let snapshot = preparation.prepare_config(&settings, None).await.unwrap(); + assert_eq!(fixture.count().await, 2); + assert_eq!(preparation.stats().mappings, 3); + assert_eq!( + preparation.stats().unique_retained_payload_bytes, + u64::try_from(text.len()).unwrap() + ); + assert!( + preparation.stats().peak_transient_staging_bytes + >= u64::try_from(text.len() * 2).unwrap() + ); + assert_eq!(snapshot.get("first").await.unwrap().unwrap().as_str(), text); + assert_eq!(snapshot.get("alias").await.unwrap().unwrap().as_str(), text); + assert_eq!(snapshot.get("empty").await.unwrap().unwrap().as_str(), ""); + assert!(snapshot.get("unmapped").await.unwrap().is_none()); + fixture.seed("config", "remote", b"new revision").await; + let same_session = preparation.prepare_config(&settings, None).await.unwrap(); + assert_eq!( + same_session.get("first").await.unwrap().unwrap().as_str(), + text + ); + assert_eq!(preparation.stats().mappings, 6); + assert_eq!(fixture.count().await, 2); + drop(preparation); + assert_eq!(snapshot.get("first").await.unwrap().unwrap().as_str(), text); + let mut fresh = + CelldPreparation::new(MonotonicClock::default(), CelldPreparationLimits::default()) + .unwrap(); + let adopted = fresh.prepare_config(&settings, None).await.unwrap(); + assert_eq!( + adopted.get("first").await.unwrap().unwrap().as_str(), + "new revision" + ); + assert_eq!(fixture.count().await, 4); + } + + #[tokio::test(flavor = "current_thread")] + async fn mapping_and_metadata_caps_are_precharged_before_dedup_or_first_fetch() { + for limits in [ + CelldPreparationLimits { + max_remote_values: 1, + ..CelldPreparationLimits::default() + }, + CelldPreparationLimits { + max_metadata_bytes: 1, + ..CelldPreparationLimits::default() + }, + ] { + let fixture = Fixture::new(vec![]).await; + fixture.seed("config", "shared", b"v").await; + let settings = settings(&fixture, &[("a", "shared"), ("b", "shared")]); + let mut preparation = CelldPreparation::new(MonotonicClock::default(), limits).unwrap(); + assert!(matches!( + preparation.prepare_config(&settings, None).await, + Err(PreparationError::SizeLimit) + )); + assert_eq!(fixture.count().await, 0); + assert!(matches!( + preparation.prepare_config(&settings, None).await, + Err(PreparationError::InvalidBinding) + )); + } + } + + #[tokio::test(flavor = "current_thread")] + async fn retained_quota_clips_transfer_and_failure_permanently_poisons_session() { + let fixture = Fixture::new(vec![]).await; + fixture.seed("config", "one", b"123").await; + fixture.seed("config", "two", b"456").await; + let settings = settings(&fixture, &[("a", "one"), ("b", "two")]); + let limits = CelldPreparationLimits { + max_config_document_bytes: 4, + max_value_bytes: 4, + max_snapshot_bytes: 5, + ..CelldPreparationLimits::default() + }; + let mut preparation = CelldPreparation::new(MonotonicClock::default(), limits).unwrap(); + assert!(matches!( + preparation.prepare_config(&settings, None).await, + Err(PreparationError::SizeLimit) + )); + assert_eq!(preparation.stats().unique_retained_payload_bytes, 3); + assert_eq!(fixture.count().await, 2); + fixture.seed("config", "two", b"4").await; + assert!(matches!( + preparation.prepare_config(&settings, None).await, + Err(PreparationError::InvalidBinding) + )); + assert_eq!(fixture.count().await, 2); + } + + #[tokio::test(flavor = "current_thread")] + async fn required_misses_invalid_utf8_and_late_invalid_mapping_publish_nothing() { + for (bytes, expected) in [ + (Some(vec![255]), PreparationError::Malformed), + (None, PreparationError::Missing), + ] { + let fixture = Fixture::new(vec![]).await; + if let Some(payload) = bytes { + fixture.seed("config", "remote", &payload).await; + } + let settings = settings(&fixture, &[("a", "remote")]); + let mut preparation = + CelldPreparation::new(MonotonicClock::default(), CelldPreparationLimits::default()) + .unwrap(); + assert!( + matches!(preparation.prepare_config(&settings, None).await, Err(error) if error == expected) + ); + assert!(matches!( + preparation.prepare_config(&settings, None).await, + Err(PreparationError::InvalidBinding) + )); + assert_eq!(fixture.count().await, 1); + } + let fixture = Fixture::new(vec![]).await; + let settings = settings(&fixture, &[("a", "valid"), ("b", "invalid\n")]); + let mut preparation = + CelldPreparation::new(MonotonicClock::default(), CelldPreparationLimits::default()) + .unwrap(); + assert!(matches!( + preparation.prepare_config(&settings, None).await, + Err(PreparationError::InvalidResource) + )); + assert_eq!(fixture.count().await, 0); + } + + #[tokio::test(flavor = "current_thread")] + async fn aggregate_deadline_never_restarts_between_bindings() { + let fixture = Fixture::new(vec![]).await; + fixture.seed("config", "remote", b"v").await; + let start = MonotonicInstant::now(); + let elapsed = Arc::new(AtomicU64::new(0)); + let ticks = Arc::clone(&elapsed); + let clock = MonotonicClock::new(move || { + start + .checked_add(Duration::from_millis(ticks.load(Ordering::SeqCst))) + .unwrap() + }); + let mut preparation = + CelldPreparation::new(clock, CelldPreparationLimits::default()).unwrap(); + let settings = settings(&fixture, &[("a", "remote")]); + preparation.prepare_config(&settings, None).await.unwrap(); + elapsed.store(30_000, Ordering::SeqCst); + assert!(matches!( + preparation.prepare_config(&settings, None).await, + Err(PreparationError::Deadline) + )); + assert_eq!(fixture.count().await, 1); + } + + #[tokio::test(flavor = "current_thread")] + #[expect( + clippy::integer_division_remainder_used, + reason = "Tokio select generates bounded branch-index arithmetic" + )] + async fn cancellation_also_poisons_unpublished_partial_preparation() { + let fixture = Fixture::new(vec![Reply::Stall]).await; + let settings = settings(&fixture, &[("a", "remote")]); + let mut preparation = + CelldPreparation::new(MonotonicClock::default(), CelldPreparationLimits::default()) + .unwrap(); + let mut pending = Box::pin(preparation.prepare_config(&settings, None)); + tokio::select! { + biased; + _result = pending.as_mut() => panic!("stalled preparation completed before cancellation"), + () = async { + while fixture.count().await == 0 { + sleep(Duration::from_millis(1)).await; + } + } => {} + () = sleep(Duration::from_secs(3)) => panic!("request admission deadline"), + } + drop(pending); + assert!(matches!( + preparation.prepare_config(&settings, None).await, + Err(PreparationError::InvalidBinding) + )); + assert_eq!(fixture.count().await, 1); + } +} diff --git a/crates/edgezero-store-celld/src/settings.rs b/crates/edgezero-store-celld/src/settings.rs new file mode 100644 index 000000000..c8efe2cb3 --- /dev/null +++ b/crates/edgezero-store-celld/src/settings.rs @@ -0,0 +1,461 @@ +//! Pure structural validation. This module never reads the environment. + +#![expect( + clippy::arbitrary_source_item_ordering, + reason = "keep each public settings shape beside its validator and redacted Debug implementation" +)] + +use serde::Deserialize; +use std::collections::BTreeMap; +use std::fmt; +use url::{Host, Url}; + +use crate::PreparationError; + +/// Explicit listener access policy, not a credential value. +#[derive(Clone, Deserialize, Eq, Hash, PartialEq)] +#[serde(tag = "mode", rename_all = "kebab-case", deny_unknown_fields)] +pub enum CelldAccess { + TrustedNetwork { + #[serde(default)] + allow_remote_http: bool, + }, + Authenticated { + token_env: String, + }, +} + +impl CelldAccess { + /// Selected bootstrap variable, without looking it up. + #[inline] + #[must_use] + pub fn token_env(&self) -> Option<&str> { + match self { + Self::Authenticated { token_env } => Some(token_env), + Self::TrustedNetwork { .. } => None, + } + } +} + +impl fmt::Debug for CelldAccess { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("CelldAccess").finish_non_exhaustive() + } +} + +/// One complete remote document selected by a service-side store alias. +#[derive(Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CelldDocument { + pub store: String, + pub key: String, +} + +impl fmt::Debug for CelldDocument { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("CelldDocument").finish_non_exhaustive() + } +} + +/// Explicit local lookup mappings to complete UTF-8 documents. +#[derive(Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CelldConfigSettings { + pub endpoint: String, + pub access: CelldAccess, + pub default_key: String, + pub documents: BTreeMap, +} + +impl CelldConfigSettings { + /// Validates every mapping without credentials, runtime or network access. + /// + /// # Errors + /// Rejects invalid endpoint, access policy, default, lookup key or target. + #[inline] + pub fn validate(&self) -> Result<(), PreparationError> { + endpoint(&self.endpoint, &self.access)?; + if self.documents.is_empty() || !self.documents.contains_key(&self.default_key) { + return Err(PreparationError::InvalidBinding); + } + for (key, document) in &self.documents { + if !valid_key(key, false) || key.trim().is_empty() { + return Err(PreparationError::InvalidBinding); + } + if !valid_alias(&document.store) || !valid_key(&document.key, false) { + return Err(PreparationError::InvalidResource); + } + } + Ok(()) + } +} + +impl fmt::Debug for CelldConfigSettings { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("CelldConfigSettings") + .finish_non_exhaustive() + } +} + +/// One live scoped store alias. No physical namespace is accepted. +#[derive(Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CelldKvSettings { + pub endpoint: String, + pub access: CelldAccess, + pub store: String, +} + +impl CelldKvSettings { + /// Validates the binding without probing remote permissions. + /// + /// # Errors + /// Rejects invalid endpoint, access policy or store alias. + #[inline] + pub fn validate(&self) -> Result<(), PreparationError> { + endpoint(&self.endpoint, &self.access)?; + if !valid_alias(&self.store) { + return Err(PreparationError::InvalidResource); + } + Ok(()) + } +} + +impl fmt::Debug for CelldKvSettings { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("CelldKvSettings").finish_non_exhaustive() + } +} + +/// Captured application bearer token. Debug reveals neither value nor length. +#[derive(Clone)] +pub struct CelldAccessToken(String); + +impl CelldAccessToken { + /// Validates explicit bytes against the embedded service's token contract. + /// + /// # Errors + /// Rejects tokens outside 32..=512 ASCII graphic bytes. + #[inline] + pub fn new(value: String) -> Result { + if !(32..=512).contains(&value.len()) || !value.bytes().all(|byte| byte.is_ascii_graphic()) + { + return Err(PreparationError::Credentials); + } + Ok(Self(value)) + } + + /// Exposes bytes only where explicitly needed for transport. + #[inline] + #[must_use] + pub fn expose(&self) -> &str { + &self.0 + } +} + +impl fmt::Debug for CelldAccessToken { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("CelldAccessToken([redacted])") + } +} + +/// Aggregate preparation and independent live-operation bounds. +#[derive(Clone, Copy, Debug, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct CelldPreparationLimits { + pub max_remote_values: u64, + pub max_config_document_bytes: u64, + pub max_snapshot_bytes: u64, + pub max_metadata_bytes: u64, + pub max_value_bytes: u64, + pub max_page_bytes: u64, + pub preparation_timeout_ms: u64, + pub operation_timeout_ms: u64, + pub connect_timeout_ms: u64, +} + +impl Default for CelldPreparationLimits { + #[inline] + fn default() -> Self { + Self { + max_remote_values: 1_024, + max_config_document_bytes: 0x0010_0000, + max_snapshot_bytes: 0x0080_0000, + max_metadata_bytes: 0x0010_0000, + max_value_bytes: 0x0010_0000, + max_page_bytes: 0x0020_0000, + preparation_timeout_ms: 30_000, + operation_timeout_ms: 5_000, + connect_timeout_ms: 1_000, + } + } +} + +impl CelldPreparationLimits { + /// Checks positive, representable quotas and ordered finite deadlines. + /// + /// # Errors + /// Returns a fixed category for invalid bounds or service-ceiling violations. + #[inline] + pub fn validate(&self) -> Result<(), PreparationError> { + let quotas = [ + self.max_remote_values, + self.max_config_document_bytes, + self.max_snapshot_bytes, + self.max_metadata_bytes, + self.max_value_bytes, + self.max_page_bytes, + ]; + if quotas.iter().any(|value| *value == 0 || usize::try_from(*value).is_err()) + || self.max_value_bytes > 0x0010_0000 + || self.max_config_document_bytes > 0x0010_0000 + || self.max_config_document_bytes > self.max_snapshot_bytes + || self.max_page_bytes > 0x0020_0000 + || self.connect_timeout_ms == 0 + || self.connect_timeout_ms > self.operation_timeout_ms + || self.operation_timeout_ms > self.preparation_timeout_ms + // Match the portable clock's finite seven-day horizon. + || self.preparation_timeout_ms > 604_800_000 + { + return Err(PreparationError::InvalidLimits); + } + Ok(()) + } +} + +pub(crate) fn endpoint(value: &str, access: &CelldAccess) -> Result { + if value.len() > 4_096 + || value + .bytes() + .any(|byte| byte.is_ascii_control() || byte.is_ascii_whitespace()) + || value.contains(['\\', '?', '#', '@']) + { + return Err(PreparationError::InvalidEndpoint); + } + let origin_suffix = value + .strip_prefix("http://") + .or_else(|| value.strip_prefix("https://")) + .ok_or(PreparationError::InvalidEndpoint)?; + let (authority, path) = origin_suffix.split_once('/').unwrap_or((origin_suffix, "")); + if !path.is_empty() || authority.contains('%') || authority.ends_with(':') { + return Err(PreparationError::InvalidEndpoint); + } + let parsed = Url::parse(value).map_err(|_error| PreparationError::InvalidEndpoint)?; + if parsed.query().is_some() + || parsed.fragment().is_some() + || !parsed.username().is_empty() + || parsed.password().is_some() + || parsed.host_str().is_none() + || parsed.port() == Some(0) + || parsed.path() != "/" + { + return Err(PreparationError::InvalidEndpoint); + } + let raw_host = if authority.starts_with('[') { + authority + .split_once(']') + .map(|(host, _port)| host) + .and_then(|host| host.strip_prefix('[')) + } else { + authority.split(':').next() + } + .ok_or(PreparationError::InvalidEndpoint)?; + let loopback = match parsed.host() { + Some(Host::Ipv4(ip)) => { + if raw_host != ip.to_string() { + return Err(PreparationError::InvalidEndpoint); + } + ip.is_loopback() + } + Some(Host::Ipv6(ip)) => { + if raw_host != ip.to_string() { + return Err(PreparationError::InvalidEndpoint); + } + ip.is_loopback() + } + Some(Host::Domain(host)) => { + if host.ends_with('.') || host.is_empty() { + return Err(PreparationError::InvalidEndpoint); + } + false + } + None => return Err(PreparationError::InvalidEndpoint), + }; + match access { + CelldAccess::Authenticated { token_env } => { + if !valid_variable(token_env) { + return Err(PreparationError::InvalidBinding); + } + if parsed.scheme() != "https" { + return Err(PreparationError::InvalidEndpoint); + } + } + CelldAccess::TrustedNetwork { allow_remote_http } => { + if parsed.scheme() == "http" && !loopback && !allow_remote_http { + return Err(PreparationError::InvalidEndpoint); + } + } + } + Ok(parsed) +} + +pub(crate) fn valid_alias(value: &str) -> bool { + !value.is_empty() + && value.len() <= 64 + && value + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'-')) +} + +pub(crate) fn valid_key(value: &str, empty: bool) -> bool { + (empty || !value.is_empty()) && value.len() <= 512 && !value.chars().any(char::is_control) +} + +fn valid_variable(value: &str) -> bool { + let mut bytes = value.bytes(); + value.len() <= 128 + && bytes + .next() + .is_some_and(|byte| byte.is_ascii_alphabetic() || byte == b'_') + && bytes.all(|byte| byte.is_ascii_alphanumeric() || byte == b'_') +} + +#[cfg(test)] +mod tests { + use super::{CelldAccess, CelldAccessToken, CelldKvSettings, CelldPreparationLimits, endpoint}; + use crate::PreparationError; + + #[test] + fn endpoints_are_explicit_origins_not_normalized_paths_or_obfuscated_ips() { + let trusted = CelldAccess::TrustedNetwork { + allow_remote_http: false, + }; + for good in [ + "http://127.0.0.1:7780", + "http://[::1]:7780/", + "https://stores.example", + ] { + endpoint(good, &trusted).expect("explicit loopback or TLS origin must be accepted"); + } + for bad in [ + "http://localhost", + "http://127.1", + "http://2130706433", + "http://127.0.0.1/a/..", + "http://127.0.0.1//", + "http://127.0.0.1?", + "http://127.0.0.1#", + "http://user@127.0.0.1", + "http://127.0.0.1\\x", + " http://127.0.0.1", + "http://private.example", + "http://127.0.0.1:0", + "http://127.0.0.1:", + "https://stores.example.", + ] { + endpoint(bad, &trusted).expect_err("invalid endpoint origin must be rejected"); + } + let remote = CelldAccess::TrustedNetwork { + allow_remote_http: true, + }; + endpoint("http://private.example", &remote) + .expect("remote HTTP origin with explicit opt-in must be accepted"); + let authenticated = CelldAccess::Authenticated { + token_env: "TOKEN".to_owned(), + }; + endpoint("http://127.0.0.1", &authenticated) + .expect_err("authenticated access must reject plaintext HTTP"); + endpoint("https://stores.example", &authenticated) + .expect("authenticated TLS origin must be accepted"); + } + + #[test] + fn config_and_kv_value_caps_are_independent() { + let limits = CelldPreparationLimits { + max_value_bytes: 1, + ..CelldPreparationLimits::default() + }; + limits + .validate() + .expect("lower live KV cap does not constrain config snapshots"); + assert_eq!( + CelldPreparationLimits { + max_config_document_bytes: 0x0010_0001, + ..limits + } + .validate() + .expect_err("config retains its independent service ceiling"), + PreparationError::InvalidLimits + ); + } + + #[test] + fn tokens_match_service_bounds_and_debug_is_fully_redacted() { + for size in [0, 1, 31, 513] { + CelldAccessToken::new("x".repeat(size)) + .expect_err("token length outside service bounds must be rejected"); + } + for value in [" ".repeat(32), "\n".repeat(32), "\u{e9}".repeat(32)] { + CelldAccessToken::new(value) + .expect_err("token with non-graphic ASCII bytes must be rejected"); + } + let token = CelldAccessToken::new("s".repeat(32)).unwrap(); + assert_eq!(format!("{token:?}"), "CelldAccessToken([redacted])"); + assert_eq!(token.clone().expose().len(), 32); + CelldAccessToken::new("x".repeat(512)) + .expect("maximum-length graphic ASCII token must be accepted"); + } + + #[test] + fn settings_and_limits_reject_unknown_or_mixed_fields() { + for access in [ + r#"{"mode":"trusted-network","token_env":"TOKEN"}"#, + r#"{"mode":"authenticated","token_env":"TOKEN","allow_remote_http":true}"#, + "{}", + ] { + serde_json::from_str::(access) + .expect_err("mixed or missing access mode fields must be rejected"); + } + let settings: CelldKvSettings = serde_json::from_str(r#"{"endpoint":"http://127.0.0.1","store":"Case_Sensitive-1","access":{"mode":"trusted-network"}}"#).unwrap(); + settings.validate().unwrap(); + assert_eq!(settings.access.token_env(), None); + serde_json::from_str::(r#"{"unknown":1}"#) + .expect_err("unknown preparation limit field must be rejected"); + CelldPreparationLimits::default().validate().unwrap(); + for limits in [ + CelldPreparationLimits { + max_remote_values: 0, + ..CelldPreparationLimits::default() + }, + CelldPreparationLimits { + max_value_bytes: 0x0010_0001, + ..CelldPreparationLimits::default() + }, + CelldPreparationLimits { + max_snapshot_bytes: 1, + ..CelldPreparationLimits::default() + }, + CelldPreparationLimits { + max_metadata_bytes: 0, + ..CelldPreparationLimits::default() + }, + CelldPreparationLimits { + connect_timeout_ms: 5_001, + ..CelldPreparationLimits::default() + }, + CelldPreparationLimits { + preparation_timeout_ms: u64::MAX, + ..CelldPreparationLimits::default() + }, + ] { + limits + .validate() + .expect_err("invalid preparation quotas or deadlines must be rejected"); + } + } +} diff --git a/crates/edgezero-store-celld/src/test_support.rs b/crates/edgezero-store-celld/src/test_support.rs new file mode 100644 index 000000000..14e38041a --- /dev/null +++ b/crates/edgezero-store-celld/src/test_support.rs @@ -0,0 +1,323 @@ +//! Private real TCP fixtures shared by config and KV tests. No transport mocking. + +#![expect( + clippy::arbitrary_source_item_ordering, + reason = "keep request/reply models before fixture ownership and TCP serving" +)] +#![expect( + clippy::field_scoped_visibility_modifiers, + reason = "fixture records are inspected by sibling colocated test modules" +)] + +use std::collections::{BTreeMap, VecDeque}; +use std::mem; +use std::sync::Arc; +use std::time::Duration; + +use tokio::io::{ + AsyncBufReadExt as _, AsyncRead, AsyncReadExt as _, AsyncWrite, AsyncWriteExt as _, BufReader, +}; +use tokio::net::TcpListener; +use tokio::sync::Mutex; +use tokio::task::{JoinHandle, JoinSet}; +use tokio::time::{sleep, timeout}; +use tokio_rustls::TlsAcceptor; +use url::Url; + +pub(crate) struct Request { + pub(crate) body: Vec, + pub(crate) connection: usize, + pub(crate) method: String, + pub(crate) target: String, + pub(crate) authorization: Option, + pub(crate) encoding: Option, +} + +pub(crate) enum Reply { + Bytes(Vec), + #[cfg(feature = "kv")] + Drop, + Stall, + SlowBody, +} + +impl Reply { + pub(crate) fn response(status: u16, content_type: &str, body: &[u8]) -> Self { + Self::custom(status, content_type, "x-celld-store-version: 1\r\n", body) + } + + pub(crate) fn custom(status: u16, content_type: &str, headers: &str, body: &[u8]) -> Self { + let mut reply = format!("HTTP/1.1 {status} Fixture\r\nContent-Type: {content_type}\r\nContent-Length: {}\r\n{headers}\r\n", body.len()).into_bytes(); + reply.extend_from_slice(body); + Self::Bytes(reply) + } +} + +#[derive(Default)] +struct State { + requests: Vec, + replies: VecDeque, + values: BTreeMap<(String, String), Vec>, +} + +pub(crate) struct Fixture { + endpoint: String, + state: Arc>, + task: JoinHandle<()>, +} + +impl Fixture { + pub(crate) async fn new(replies: Vec) -> Self { + Self::start(replies, None).await + } + + pub(crate) async fn tls(certificate: &'static [u8], replies: Vec) -> Self { + use tokio_rustls::rustls::ServerConfig; + use tokio_rustls::rustls::crypto::aws_lc_rs::default_provider; + use tokio_rustls::rustls::pki_types::{CertificateDer, PrivateKeyDer, PrivatePkcs8KeyDer}; + + let key = PrivateKeyDer::Pkcs8(PrivatePkcs8KeyDer::from( + include_bytes!("fixtures/tls-key.der").to_vec(), + )); + let config = ServerConfig::builder_with_provider(default_provider().into()) + .with_safe_default_protocol_versions() + .expect("fixture provider supports default TLS protocol versions") + .with_no_client_auth() + .with_single_cert(vec![CertificateDer::from(certificate.to_vec())], key) + .expect("fixture certificate matches the bundled private key"); + Self::start(replies, Some(TlsAcceptor::from(Arc::new(config)))).await + } + + #[expect( + clippy::integer_division_remainder_used, + reason = "Tokio select generates bounded branch-index arithmetic" + )] + async fn start(replies: Vec, acceptor: Option) -> Self { + let listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("bind fixture loopback listener"); + let scheme = if acceptor.is_some() { "https" } else { "http" }; + let endpoint = format!( + "{scheme}://{}", + listener.local_addr().expect("read bound fixture address") + ); + let state = Arc::new(Mutex::new(State { + replies: replies.into(), + ..State::default() + })); + let server_state = Arc::clone(&state); + let task = tokio::spawn(async move { + let mut connections = JoinSet::new(); + let mut index = 0_usize; + loop { + tokio::select! { + biased; + accepted = listener.accept() => { + let Ok((stream, _address)) = accepted else { break; }; + let connection_state = Arc::clone(&server_state); + let connection_acceptor = acceptor.clone(); + index = index.checked_add(1).expect("fixture connection count must fit usize"); + connections.spawn(async move { + if let Some(tls_acceptor) = connection_acceptor { + if let Ok(Ok(tls_stream)) = timeout( + Duration::from_secs(3), tls_acceptor.accept(stream), + ).await { + serve(tls_stream, connection_state, index).await; + } + } else { + serve(stream, connection_state, index).await; + } + }); + } + Some(_result) = connections.join_next(), if !connections.is_empty() => {} + } + } + }); + Self { + endpoint, + state, + task, + } + } + + pub(crate) fn endpoint(&self) -> &str { + &self.endpoint + } + + pub(crate) async fn seed(&self, store: &str, key: &str, bytes: &[u8]) { + self.state + .lock() + .await + .values + .insert((store.to_owned(), key.to_owned()), bytes.to_vec()); + } + + pub(crate) async fn value(&self, store: &str, key: &str) -> Option> { + self.state + .lock() + .await + .values + .get(&(store.to_owned(), key.to_owned())) + .cloned() + } + + pub(crate) async fn count(&self) -> usize { + self.state.lock().await.requests.len() + } + + pub(crate) async fn requests(&self) -> Vec { + mem::take(&mut self.state.lock().await.requests) + } +} + +impl Drop for Fixture { + fn drop(&mut self) { + self.task.abort(); + } +} + +async fn serve( + socket: S, + state: Arc>, + connection: usize, +) { + let mut stream = BufReader::new(socket); + loop { + let mut first = String::new(); + if stream.read_line(&mut first).await.unwrap_or(0) == 0 { + return; + } + let mut parts = first.split_whitespace(); + let method = parts + .next() + .expect("fixture request line includes a method") + .to_owned(); + let target = parts + .next() + .expect("fixture request line includes a target") + .to_owned(); + let mut length = 0; + let mut authorization = None; + let mut encoding = None; + loop { + let mut line = String::new(); + if stream.read_line(&mut line).await.unwrap_or(0) == 0 { + return; + } + if line == "\r\n" { + break; + } + if let Some((name, raw_value)) = line.split_once(':') { + let value = raw_value.trim(); + if name.eq_ignore_ascii_case("content-length") { + length = value + .parse() + .expect("fixture Content-Length is a valid length"); + } + if name.eq_ignore_ascii_case("authorization") { + authorization = Some(value.to_owned()); + } + if name.eq_ignore_ascii_case("accept-encoding") { + encoding = Some(value.to_owned()); + } + } + } + let mut body = vec![0; length]; + if stream.read_exact(&mut body).await.is_err() { + return; + } + let request = Request { + body, + connection, + method, + target, + authorization, + encoding, + }; + let reply = { + let mut locked = state.lock().await; + let dynamic = apply(&request, &mut locked.values); + locked.requests.push(request); + locked.replies.pop_front().unwrap_or(dynamic) + }; + match reply { + Reply::Bytes(bytes) => { + if stream.get_mut().write_all(&bytes).await.is_err() { + return; + } + } + #[cfg(feature = "kv")] + Reply::Drop => return, + Reply::Stall => { + sleep(Duration::from_secs(5)).await; + return; + } + Reply::SlowBody => { + if stream.get_mut().write_all(b"HTTP/1.1 200 Fixture\r\nx-celld-store-version: 1\r\nContent-Type: application/octet-stream\r\nContent-Length: 2\r\n\r\nx").await.is_err() { return; } + sleep(Duration::from_secs(5)).await; + return; + } + } + } +} + +fn apply(request: &Request, values: &mut BTreeMap<(String, String), Vec>) -> Reply { + let url = Url::parse(&format!("http://fixture{}", request.target)) + .expect("fixture request target forms a valid URL"); + let segments: Vec<_> = url + .path_segments() + .expect("fixture URL has path segments") + .collect(); + let store = (*segments + .get(2) + .expect("fixture store path includes an alias")) + .to_owned(); + let params: BTreeMap<_, _> = url.query_pairs().into_owned().collect(); + let key = params.get("key").cloned().unwrap_or_default(); + let target = (store.clone(), key); + match request.method.as_str() { + "PUT" => { + values.insert(target, request.body.clone()); + Reply::response(204, "application/octet-stream", b"") + } + "DELETE" => { + values.remove(&target); + Reply::response(204, "application/octet-stream", b"") + } + "HEAD" => Reply::response( + if values.contains_key(&target) { + 200 + } else { + 404 + }, + "application/octet-stream", + b"", + ), + _ if url.path().ends_with("/keys") => { + let prefix = params.get("prefix").map_or("", String::as_str); + let after = params.get("cursor").map_or("", String::as_str); + let limit: usize = params + .get("limit") + .expect("fixture page request has a limit") + .parse() + .expect("fixture page limit is numeric"); + let mut keys: Vec<_> = values + .keys() + .filter(|(alias, stored_key)| { + *alias == store && stored_key.starts_with(prefix) && stored_key.as_str() > after + }) + .map(|(_, stored_key)| stored_key.clone()) + .collect(); + let more = keys.len() > limit; + keys.truncate(limit); + let cursor = if more { keys.last().cloned() } else { None }; + let bytes = serde_json::to_vec(&serde_json::json!({"keys": keys, "cursor": cursor})) + .expect("serialize fixture page JSON"); + Reply::response(200, "application/json", &bytes) + } + _ => values.get(&target).map_or_else( + || Reply::response(404, "application/json", br#"{"error":"not_found"}"#), + |bytes| Reply::response(200, "application/octet-stream", bytes), + ), + } +} diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts index 46f21e821..5b1f28d64 100644 --- a/docs/.vitepress/config.mts +++ b/docs/.vitepress/config.mts @@ -50,6 +50,11 @@ export default defineConfig({ { text: 'Cloudflare Workers', link: '/guide/adapters/cloudflare' }, { text: 'Fermyon Spin', link: '/guide/adapters/spin' }, { text: 'Axum (Native)', link: '/guide/adapters/axum' }, + { text: 'Axum diagnostics', link: '/guide/axum-diagnostics' }, + { text: 'Native Geo providers', link: '/guide/native-geo' }, + { text: 'Native AWS stores', link: '/guide/native-aws-stores' }, + { text: 'Native celld stores', link: '/guide/native-celld-stores' }, + { text: 'DynamoDB KV', link: '/guide/dynamodb-kv' }, ], }, { diff --git a/docs/guide/adapters/axum.md b/docs/guide/adapters/axum.md index 45addea2e..12b455bd8 100644 --- a/docs/guide/adapters/axum.md +++ b/docs/guide/adapters/axum.md @@ -209,6 +209,29 @@ field-section accounting or raw framing-rejection policy. Admission still receiv `IngressHeadAccounting::HostManaged` and `IngressFraming::HostManaged`; both raw-ingress capabilities remain `Unsupported`. See [Capabilities](/guide/capabilities). +## Checked ingress and diagnostics + +Managed hosting validates the original normalized request head, removes raw forwarding fields, +and exposes checked metadata from the actual socket peer. Proxy trust is off by default. +Configure a bounded `TrustedProxyPolicy` only for deployment-owned peers. Native request records +omit raw headers, addresses, paths/queries, and source error strings; application-owned loggers +and observers remain responsible for their own output. See +[Axum diagnostics and trusted proxies](../axum-diagnostics.md) for configuration, metadata, +request identity, counters, and deployment examples. + +## Native Geo providers + +Geo is off by default and selected only through `AxumRunOptions::with_geo`. The opt-in +`geo-cloudfront` feature adds a bounded local CloudFront parser, not AWS SDKs or store setup. +Both built-in and supplied providers require a separate `ProtectedCloudFrontAlb` policy using +the actual socket peer. Ordinary trusted-proxy configuration does not admit geographic assertions. + +Read the immutable outcome through `RequestContext::geo()`. Missing metadata, untrusted ingress, +invalid assertions, and provider failures remain distinct; none grants permission or chooses an +HTTP status. The [native Geo guide](../native-geo.md) covers the Rust API, mandatory HTTPS origin +credential gate and network isolation, header generation, disabled shared caching, rotation, +and the separate cloud/downstream acceptance checks. + ## Outbound HTTP The Axum adapter injects `AxumOutboundClient`, backed by `reqwest`. Application handlers use the @@ -315,27 +338,39 @@ The database file grows over time and does not shrink after deletions. To reclai space, delete the file in `.edgezero/`; the data is lost. See [KV Storage](/guide/kv) for the portable API. -## Secret Store - -A declared `[stores.secrets]` id resolves to an `EnvSecretStore`, which looks up each -secret name verbatim in the process environment. Axum lists `secrets` in its -`single_store_kinds`, so only one secrets id may be declared: - -```bash -API_KEY=mysecret edgezero serve --adapter axum -``` - -## Config Store - -For local development, each declared `[stores.config]` id resolves to a -local-file config store backed by `.edgezero/local-config-.json`. -The standard runner loads immutable snapshots before binding its listener. -Missing files create empty declared stores. Malformed files retain typed failing -bindings, while allocation-limit violations fail startup. Changes made by `config -push` take effect after restart, not on the next request. -The portable manifest carries no inline defaults — the -pre-rewrite `[stores.config.defaults]` table is gone (see -[the migration guide](../manifest-store-migration.md)). +## Native store providers + +Axum keeps the existing local defaults, but native applications can select a provider +per logical store ID. The portable manifest still declares IDs and defaults only. A +separate, strict native bindings file selects supported providers; it does not change +store metadata or add AWS dependencies to Fastly, Cloudflare, or Spin builds. + +Without a bindings file, config uses local `.edgezero/local-config-.json` +snapshots, secrets use environment variables, and KV uses the existing local backend. +With explicit AWS bindings, the runner fetches mapped values before its initializer +and before it accepts requests. It does not fall back to a local provider when an AWS +lookup or preparation fails. See the [native AWS stores guide](../native-aws-stores.md) +for the schema, feature selection, permissions, and lifecycle limits. +The independent [native celld providers](../native-celld-stores.md) add startup +configuration snapshots and live KV through a scoped listener inside celld. + +The native binding file path must be absolute. `EDGEZERO__STORE_BINDINGS_FILE` is +read only by the environment-loading runner. Explicit `AxumRunOptions` selection +uses only values supplied in code. Programmatic applications may also inject existing +handles or deferred setup through `run_app_with_native_stores`; deferred setup is +polled after source validation, in the current native runtime. Its construction must +not perform provider work. + +A declared `[stores.secrets]` ID still defaults to an environment-backed store when +not explicitly bound. It is no longer limited to one ID by Axum. Existing +`EDGEZERO__STORES__SECRETS____NAME` and config/KV local overrides retain their +meaning for implicit local bindings; they cannot override an explicitly bound ID. + +`config push --adapter axum` remains a local-file writer. It does not publish to +AppConfig. If passed an explicit `--store-bindings` file that binds the selected ID +to AWS, the guard rejects the operation before writing anything. A push +without that explicit file still only updates the local snapshot file; it does not +update an AWS-bound deployment. ```toml [stores.config] @@ -424,6 +459,22 @@ not guaranteed. Local file reads are synchronous; no finite startup I/O interrup claimed. The broad `config-read-allocation-bounds` capability remains `Unsupported`; this bounded local config-snapshot path does not certify every secret/provider allocation. +## Optional AWS features + +Generated native targets opt into `aws-appconfig-agent` and +`aws-secrets-manager` independently, or use the `aws-stores` alias for both. These +features are separate from the CLI binary's own dependencies. A bindings file cannot +enable a provider that was not compiled into the application. The AWS SDK and service +clients stay out of default native and WASM target graphs. + +The native AWS SDK can emit diagnostic fields at dependency `DEBUG`/`TRACE` levels. +Those events may include secret ARNs, credential-source metadata, or response details +when sensitive-body logging is enabled. EdgeZero redacts its own errors and diagnostics, +but the first version does not suppress dependency events because scoped suppression +can break an application's tracing-to-log fallback. Do not enable dependency +`DEBUG`/`TRACE` in production unless the process logging policy has been reviewed. +This limitation must be disclosed in release and PR review notes. + ## Container Deployment Build and deploy as a standard container: diff --git a/docs/guide/axum-diagnostics.md b/docs/guide/axum-diagnostics.md new file mode 100644 index 000000000..375951544 --- /dev/null +++ b/docs/guide/axum-diagnostics.md @@ -0,0 +1,444 @@ +# Axum diagnostics and trusted proxies + +The managed native host exposes checked ingress metadata, safe request records, and work +snapshots. Incoming proxy trust is off by default. A forwarding header never establishes +trust by itself. + +These APIs belong to `edgezero_adapter_axum`, with the `axum` feature enabled. They apply to +`run_app`, `run_production_app`, the explicit-options runners, and `AxumDevServer`. Other +adapters retain their existing behavior. Geographic source admission is a separate policy; +ordinary proxy trust never admits CloudFront geography. See [Native Geo providers](./native-geo.md). + +## Startup settings + +| Setting | Default | Accepted values | +| --------------------------------------------- | ------------------- | ---------------------------------------------------------------------- | +| `EDGEZERO__ADAPTER__TRUSTED_PROXY_CIDRS` | No trusted networks | Comma-separated literal IPv4/IPv6 addresses or CIDRs | +| `EDGEZERO__ADAPTER__FORWARDING_HEADER_FAMILY` | No selected family | `forwarded` or `x-forwarded`, required when the trust list is nonempty | +| `EDGEZERO__LOGGING__REQUEST_RECORDS` | `true` | Exactly `true` or `false` | + +For example, trust one deployment-owned proxy address and use RFC Forwarded: + +```bash +EDGEZERO__ADAPTER__TRUSTED_PROXY_CIDRS=10.20.0.8/32 \ +EDGEZERO__ADAPTER__FORWARDING_HEADER_FAMILY=forwarded \ +EDGEZERO__LOGGING__REQUEST_RECORDS=true \ +./my-app-adapter-axum +``` + +An unset trust list, an empty string, or a string containing only spaces/tabs means no trust. +The network parser trims spaces/tabs around entries. It rejects empty entries inside a list, +other control characters, invalid literals, DNS names, and tokens such as `private`, `all`, or +provider names. Family names are lowercase, with surrounding spaces/tabs allowed. Unknown +supplied families fail even with an empty trust list. A valid family alone enables nothing. +Request-record booleans accept no case changes or whitespace. + +Invalid supplied values, including non-Unicode environment values for these settings, fail +startup before readiness. They do not silently revert to defaults. Both development and +production validate these three settings strictly. Production also validates its other hosting +settings; this does not change unrelated development-setting fallback behavior. + +Network entries are truncated to their prefix and deduplicated. IPv4-mapped IPv6 peers and +assertions match their IPv4 equivalents. Mapped networks within `::ffff:0:0/96` become equivalent +IPv4 networks. `/0`, unions covering an entire address family, and IPv6 prefixes that would +trust the entire mapped IPv4 range are rejected. Private, loopback, and container networks +receive no implicit trust. EdgeZero does not discover provider ranges or refresh them. + +The trust configuration has a 16,384-byte text budget, including separators, and at most 256 +supplied entries. Duplicates count before normalization. Explicit policy iterators have the +same bounds and never silently truncate. + +### Explicit hosting + +```rust +use edgezero_adapter_axum::dev_server::AxumRunOptions; +use edgezero_adapter_axum::diagnostics::NativeDiagnosticsHandle; +use edgezero_adapter_axum::proxy::{ForwardingHeaderFamily, TrustedProxyPolicy}; + +let diagnostics = NativeDiagnosticsHandle::new(); +let policy = TrustedProxyPolicy::new( + ForwardingHeaderFamily::Forwarded, + ["10.20.0.8/32"], +)?; +let options = AxumRunOptions::new("127.0.0.1:8787".parse()?)? + .with_trusted_proxy_policy(policy) + .with_diagnostics(diagnostics.clone()) + .with_request_records(false); +``` + +Pass these options and a caller-owned stop future to +`dev_server::run_app_with_options::`. Use +`run_app_with_options_and_initializer` when startup needs application checks. The explicit +runners install no process-wide signal handlers. The corresponding `AxumDevServer` builders +are `with_trusted_proxy_policy`, `with_diagnostics`, and `with_request_records`. + +Explicit options/builders do not merge ambient proxy or request-record settings. Use +`AxumRunOptions::from_env()` to capture environment-driven settings deliberately, or +`from_vars` for supplied string pairs. Each run captures its settings once. +`TrustedProxyPolicy::no_trust()` is the explicit no-trust policy. + +## What makes a forwarding assertion trustworthy + +Protect the proxy-to-application network first. Only deployment-owned proxies should be able +to connect from addresses in the allowlist. A shared private subnet or loopback address is not +proof that every process using it is trustworthy. Trust is neither user authentication nor +permission to use any asserted host as an outbound destination. + +The service matches the socket-owned peer, not `Forwarded by=`, a header, URI, or +caller-supplied `ConnectInfo`. Without a matching peer, it does not parse forwarding assertions. +It uses direct facts instead. + +```mermaid +flowchart TD + A[Owned socket peer and received head] --> B[Validate and account original normalized head] + B --> C[Match explicit proxy policy] + C --> D[Validate selected family within bounds] + D --> E[Compute checked facts and provenance] + E --> F[Remove both raw forwarding families] + F --> G[Admission policy] + G --> H[Middleware and handlers] +``` + +Select exactly one family. `forwarded` reads ordered RFC 7239 elements, including repeated +field lines and quoted IPv6 nodes. `x-forwarded` reads an ordered X-Forwarded-For chain and +singleton X-Forwarded-Host and X-Forwarded-Proto values. Repeated or comma-listed host/proto +values are ambiguous and invalidate that family. The unselected family never fills a gap. + +All recognized assertions are validated before choosing a client boundary, including entries +that the eventual trust walk will ignore. The service walks predecessors right to left while +the current hop is trusted. The first untrusted numeric predecessor is the effective client +boundary. It never selects the leftmost value unconditionally or skips an unknown hop. + +| `ForwardingResult` | Meaning | +| ------------------ | ----------------------------------------------------------------------------------------------------------------------------------- | +| `Absent` | No usable selected assertions | +| `UntrustedPeer` | Forwarding fields are present but the direct peer cannot supply trusted assertions | +| `Accepted` | Client, host, and scheme assertions were accepted | +| `Partial` | Some fields were accepted; missing fields use direct facts | +| `UnresolvedChain` | A valid chain has a missing/unknown/obfuscated predecessor or exhausts only trusted addresses; client falls back to the direct peer | +| `Malformed` | Invalid syntax, recognized values, ambiguity, or bounds invalidate the whole selected assertion | + +An unresolved client chain can still supply valid final-proxy host and scheme assertions. +Malformed client text cannot. With malformed optional forwarding metadata, an otherwise valid +request continues using direct facts for every selected field. Existing HTTP head/framing +violations still follow their normal rejection or abort path. + +The selected assertion has a 16,384-byte budget, counting conceptual separators between +repeated field values, and at most 64 chain/list slots. Forwarded allows at most 16 parameter +slots per element. Empty slots count against bounds. RFC Forwarded tolerates empty list +members and empty semicolon slots. X-Forwarded-For does not: a present-empty value or empty +member invalidates the complete selected family, including otherwise-valid host/proto. +Absent X-Forwarded-For still permits individual host/proto acceptance. + +These bounds cover additional forwarding parsing and retained metadata. They do not cap +Hyper's initial receive allocation, total process memory, or concurrent work. The original +normalized head is validated and counted before stripping, so removed headers cannot evade +head limits. That accounting is normalized head size, not raw wire bytes. + +### Host and scheme belong to the final proxy + +For Forwarded, only the final, rightmost element can supply host/proto. For X-Forwarded-*, only +the singleton host/proto fields can supply them. Earlier host/proto values and the client-address +chain cannot establish the public origin. + +The final trusted ingress must overwrite these fields with values it verified. If another proxy +terminated public TLS, verify that upstream assertion or enforce a known HTTPS-only public +route. The final hop's own connection scheme is not necessarily the visitor's scheme. + +Hosts must be valid nonempty authorities with an optional numeric port, without userinfo, +whitespace, path, query, or fragment. Schemes normalize only `http` and `https`. There is no DNS +or IDNA lookup. Missing fields fall back independently. Direct host comes from the +HTTP-authoritative request head; unavailable authority stays unavailable. The native direct +transport scheme is HTTP, even when an absolute request URI says HTTPS. + +## Nginx terminating public TLS + +This example replaces the chain for a single public ingress. Put the `map` in the existing +Nginx `http` block and add the server alongside your TLS certificate configuration. The map +quotes and brackets IPv6 addresses for Forwarded; IPv4 remains unquoted. + +```nginx +map $remote_addr $edgezero_forwarded_for { + default $remote_addr; + ~: '"[$remote_addr]"'; +} + +server { + listen 443 ssl; + listen [::]:443 ssl; + server_name app.example.com; + # Configure ssl_certificate and ssl_certificate_key here. + + location / { + proxy_pass http://127.0.0.1:8787; + proxy_set_header Host app.example.com; + proxy_set_header Forwarded "for=$edgezero_forwarded_for;host=app.example.com;proto=https"; + proxy_set_header X-Forwarded-For ""; + proxy_set_header X-Forwarded-Host ""; + proxy_set_header X-Forwarded-Proto ""; + } +} +``` + +Configure EdgeZero with `TRUSTED_PROXY_CIDRS=127.0.0.1/32` and +`FORWARDING_HEADER_FAMILY=forwarded`, using the full environment prefixes above. Bind its +listener to loopback and keep untrusted local processes away from this trusted connection. +If Nginx uses IPv6 for the backend hop, configure that actual peer instead. + +The snippet assumes `$remote_addr` is the actual incoming peer, with no header-driven real-IP +rewrite. The HTTPS-only server establishes `proto=https`. It supplies a fixed public host, +not `$http_host`, and replaces visitor-supplied Forwarded values rather than appending them. +Configure your default virtual host to reject unexpected public hosts and handle HTTP +redirects separately. Test both IPv4 and IPv6 visitors in the deployed configuration. +See Nginx's [map](https://nginx.org/en/docs/http/ngx_http_map_module.html) and +[proxy header](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_set_header) +documentation for directive scope and inheritance. + +## Cloudflare plus a load balancer + +For `visitor → Cloudflare → load balancer → EdgeZero`, configure the final load balancer to +append its actual incoming Cloudflare peer to the verified predecessor chain. It must replace +X-Forwarded-Host with the allowed public authority and X-Forwarded-Proto with the verified +visitor scheme. Do not merely pass through a visitor-controlled host/proto field. + +For illustration, suppose the final peer is `10.20.0.8` and the Cloudflare peer is +`198.51.100.10`. These are example addresses, not provider ranges: + +```http +X-Forwarded-For: 203.0.113.7, 198.51.100.10 +X-Forwarded-Host: app.example.com +X-Forwarded-Proto: https +``` + +Select `x-forwarded` and explicitly trust the actual load-balancer addresses and relevant +Cloudflare ranges for your deployment. The walk can then stop at `203.0.113.7`. If only the +load balancer is trusted, it stops at the Cloudflare address. If every represented address is +trusted, it falls back to the direct peer as unresolved. + +Restrict origin access to the intended Cloudflare producers, including verification appropriate +to your zone and route. Restrict EdgeZero access to the final load balancer. Provider defaults, +address membership alone, or an HTTPS load-balancer hop do not prove correct visitor metadata. +Maintain the allowlist and inspect your deployed header mode. EdgeZero has no built-in Cloudflare/AWS forwarding trust list and does not consume +`CF-Connecting-IP` automatically. The separate protected CloudFront/ALB Geo policy does not +select a forwarding family or establish checked reader-IP metadata. +Cloudflare's [header documentation](https://developers.cloudflare.com/fundamentals/reference/http-request-headers/) +explains how Workers, transforms, and Pseudo IPv4 can change assertions. + +### A gateway can replace the chain + +If the load balancer cannot produce verified singleton host/proto fields, place a sanitizing +gateway between it and EdgeZero. The gateway must validate its upstream producer and chain, +reject or discard unverified fields, and replace the selected family entirely. For a verified +visitor address, it can emit: + +```http +Forwarded: for=203.0.113.7;host=app.example.com;proto=https +``` + +Trust only that gateway's actual outgoing peer and select `forwarded`. Emit HTTPS only after +verifying the public TLS assertion or enforcing the HTTPS-only public route. If the gateway +cannot establish the client boundary, `for=unknown` is honest; EdgeZero then uses direct client +fallback while retaining valid host/proto. Renaming untouched client input to a new header +does not verify it. + +## Migrate applications from raw headers + +Before admission, the managed host removes Forwarded and every case-insensitive +X-Forwarded-* header, including accepted fields and extensions. It overwrites incoming checked +metadata extensions with its own results. Host, URI, and +`context::AxumRequestContext::remote_addr` remain unchanged; `remote_addr` is always the direct +peer, not the effective client. + +At admission, read `IngressHead::extension::()`. In middleware or a +handler, read the same facts from the request context: + +```rust +use edgezero_adapter_axum::diagnostics::NativeIngressMetadata; + +if let Some(metadata) = ctx.extensions().get::() { + let authority = metadata.effective_host().authority(); + let scheme = metadata.effective_scheme().as_str(); + let client_boundary = metadata.effective_client(); + let source = metadata.host_source(); + // Apply application host authorization before using the public origin. +} +``` + +`request_id`, `direct_peer`, `client_source`, `scheme_source`, `forwarding_result`, and +`received_normalized_head` are also read-only accessors. Per-field sources use +`edgezero_core::ingress::CheckedHostSource`: `Direct`, `TrustedForwarded`, +`TrustedXForwarded`, or `Unavailable`. Read each source independently from the aggregate +forwarding result. Metadata describes original ingress, not later application mutations. + +`edgezero_core::ForwardedHost` now reads `CheckedEffectiveHost` first on the managed native +path, including accepted RFC Forwarded host values. Checked-unavailable uses its existing +`localhost` fallback without consulting raw forwarding fields. Prefer the metadata accessor +when you must distinguish absence from that convenience fallback. Without checked metadata, +other adapters and legacy conversions retain the extractor's old behavior. + +`request::into_core_request` is a low-level bypass. It does not perform managed admission or +socket-owned proxy checking, and raw `ForwardedHost` behavior can remain there. Supplying +ConnectInfo or setting an environment variable does not turn that converter into a trusted +ingress. Use managed hosting for this policy. The converter does not acquire managed-native +JSON error detail policy or a Geo outcome. Do not recreate removed headers for automatic +outbound propagation; outbound assertions need their own explicit authorization decision. + +## Records and logger ownership + +Default facade records use target `edgezero::native`. `request_terminal` describes begun +egress attempts; `request_ingress` describes no-response abort, draining, or abandonment. +`lifecycle` describes runner transitions and fixed failure categories. + +Request records contain a generated ID, known method or `other`, registered route template or +`unknown`, optional selected status and total duration, typed outcome/failure/forwarding +categories, and optional egress bytes/body/fallback accounting. Default framework request +records do not contain addresses, hosts, raw paths/queries, headers, payloads, config/secret values, +origin URLs, or source error strings. Route templates are application-owned names; do not +put secrets in them. Default output escapes route text to one line. + +Severity follows typed outcomes, not status alone. Ordinary completion, framework route misses, +and expected draining use INFO. Refusal, propagated errors, malformed forwarding, and +failed/abandoned attempts use WARN. Startup/runtime faults and unusable clock/identity state +use ERROR. An application-selected 503 is not automatically a framework failure. + +`Hooks::owns_logging()` controls installation only. Set `owns_logging = true` in the `app!` +declaration and install your logger once before entering the runner. For example, in your +Axum entrypoint with your own logger dependency: + +```rust +use edgezero_adapter_axum::dev_server::run_production_app; +use my_app_core::App; + +fn main() -> anyhow::Result<()> { + // App declares owns_logging = true. + simple_logger::SimpleLogger::new() + .with_level(log::LevelFilter::Info) + .init()?; + run_production_app::() +} +``` + +Keep the `edgezero::native` target enabled in your logger's filters when you want these records. +No competing logger/subscriber is installed when the application owns logging. + +`with_request_records(false)`, or `EDGEZERO__LOGGING__REQUEST_RECORDS=false` on an +environment-driven runner, suppresses default request-terminal and request-ingress logs only. +Native and application observers, lifecycle events, counts, and snapshots remain active. + +### Request identity + +`RequestId` displays 64 hexadecimal characters derived from a process nonce, hosting sequence, +and request sequence. IDs do not repeat within the process; separation between processes is +entropy-based, not a mathematical global uniqueness guarantee. Entropy failure prevents +startup; sequence exhaustion never wraps or reuses an ID. + +Incoming IDs are not adopted. There is no automatic response ID header, tracing-header +propagation, or authentication meaning. Applications can expose an ID deliberately. +Changing the public metadata extension cannot change the framework's private correlation ID. +Never use request IDs as metric labels. + +## Snapshots and optional metrics + +Retain a clone of the handle passed to `with_diagnostics` and sample it from application-owned +management code: + +```rust +let snapshot = diagnostics.snapshot(); +let active_requests = snapshot.active_requests; +let active_connections = snapshot.active_connections; +let idle_connections = snapshot.idle_connections; +``` + +Active requests include managed admission, body read, handlers, and egress until completion or +abandonment releases the guard. Active connections count owned connection futures, including +idle keep-alive connections; idle connections are their idle subset. Detached application work, +kernel buffers, and external services are not counted. + +A snapshot coherently reads the three counters at one moment. It is not atomic with lifecycle +phase or subsequent work. A request used to query counts counts itself while active, and its +connection also contributes. Sample externally or account for the query before asserting zero. +Accounting releases before native logger/observer delivery. + +Each runner creates its own handle by default. Explicitly sharing one handle aggregates its +counters across runners; clones of an observer-equipped handle also receive their records. +Each runner still has its own identity namespace and lifecycle. Attach the observer before +cloning the handle for runners: `with_request_observer` configures the returned handle, not +previous clones. + +Implement `NativeRequestObserver` for application metrics. This small example keeps fixed +counters and cumulative duration without per-request labels: + +```rust +use std::sync::{Arc, Mutex}; +use std::time::Duration; +use edgezero_adapter_axum::diagnostics::{ + NativeDiagnosticsHandle, NativeRequestObserver, NativeRequestRecord, +}; + +#[derive(Default)] +struct Totals { + observed: u64, + timed: u64, + duration: Duration, +} + +struct Metrics(Arc>); + +impl NativeRequestObserver for Metrics { + fn observe(&self, record: &NativeRequestRecord) { + if let Ok(mut totals) = self.0.lock() { + totals.observed = totals.observed.saturating_add(1); + if let Some(duration) = record.duration { + totals.timed = totals.timed.saturating_add(1); + totals.duration = totals.duration.saturating_add(duration); + } + } + } +} + +let totals = Arc::new(Mutex::new(Totals::default())); +let diagnostics = NativeDiagnosticsHandle::new() + .with_request_observer(Metrics(Arc::clone(&totals))); +``` + +Use your metrics library for histograms and bounded method, registered-route, status-class, or +outcome labels. Handle missing status/duration and future enum variants. Never label metrics +with IDs, addresses, raw URLs, payloads, or secrets. The native observer adds no exporter, +queue, metrics server, or public diagnostics endpoint and does not replace application egress +observers/completions. Do not capture an observer-bearing handle in its own observer and +create an ownership cycle. + +## Timing, failures, and privacy limits + +Native total duration runs from the parsed service boundary to the existing authoritative +terminal observation on the application monotonic clock. It includes pre-egress work but +excludes subsequent completion-callback and logger delay. `ResponseEgressReport::elapsed` +remains egress-only. A backwards clock produces absent duration and a safe clock-fault +category, not a wall-clock substitute. Optional application timing collectors do not replace +this clock or completion owner. + +Selected status is not proof that headers reached the wire or that the client received them. +A selected 200 may end with `SourceError`, `DeadlineExceeded`, `ClientDisconnected`, or +`TransportError`. A precommit fallback can select 500/504; a post-commit failure does not send +a second error response. `bytes_written` counts egress-accounted bytes, not delivered TCP +bytes. No-response paths have no invented status or egress report. + +On the managed native path, default framework routing 404/405 errors use fixed public text. +Custom renderers retain the original routing errors. Native-managed JSON deserialization +errors return HTTP 400 with the fixed message `invalid JSON payload`, without serde's value, +line, or type detail. Custom renderers also receive this fixed native-generated JSON error; +the serde detail is already discarded. Legacy/unmanaged JSON errors retain their existing +detail. Application-authored `EdgeError` messages and responses are not globally sanitized, +and custom renderers remain responsible for their own output. This integration adds no JSON +size selector or managed body carrier; existing body-read limits remain unchanged. + +Loggers and observers run synchronously. They can block body progress or shutdown, so keep +them short. Unwind-capable targets isolate diagnostic sink panics independently and release +native accounting, but panic-abort, SIGKILL, and process failure have no completion or log-flush +promise. There is no retry, durable delivery guarantee, or added shutdown flush budget. + +Deployment acceptance remains downstream. Parser tests and local socket/TLS fixtures do not +prove final-image behavior, proxy configuration, client receipt, signal delivery, or bounded +sink latency. Recheck the accepted prerequisite stack and test the actual image and ingress +before relying on these deployment claims. Packaging and final-image acceptance are separate +from this guide's API contract. diff --git a/docs/guide/dynamodb-kv.md b/docs/guide/dynamodb-kv.md new file mode 100644 index 000000000..f4a1603d8 --- /dev/null +++ b/docs/guide/dynamodb-kv.md @@ -0,0 +1,141 @@ +# Native DynamoDB KV + +DynamoDB is an optional live KV provider for native Axum apps. It uses the existing +`Kv` extractor, `KvHandle` and per-ID registry. Config and Secrets Manager stores +remain independent startup snapshots. DynamoDB calls happen during KV operations, +not only at startup. + +## Select a store + +Enable `aws-dynamodb` on the generated Axum package, for example in +`edgezero.toml`: + +```toml +[adapters.axum.build] +features = ["aws-dynamodb"] +``` + +`aws-stores` retains its existing AppConfig Agent and Secrets Manager membership. +Use `aws-stores,aws-dynamodb` to enable all three. DynamoDB is off by default and +its SDK is absent from settings-only, ordinary CLI/Axum, Agent-only and WASM +normal dependency graphs. + +Declare a portable KV ID in the application as usual, then select its native +provider in an absolute bindings file: + +```toml +version = 1 + +[kv.state] +provider = "aws-dynamodb" +region = "us-east-1" +table = "edgezero-kv-prod" +namespace = "checkout-prod/state" +``` + +Set `EDGEZERO__STORE_BINDINGS_FILE` to that file. Selection is explicit. A missing +feature, invalid binding or unavailable table fails before the listener opens. +There is no fallback to redb or another provider. A deployment with no local KV +IDs does not need `DATA_DIR`. Existing supplied/ready/deferred KV handles still +work and must not conflict with a binding for the same ID. + +The SDK uses its supported operator-owned credential chain, including workload +roles and configured developer profiles. Do not put credentials in bindings, +images or application config. Region/table/namespace are explicit, and the +binding schema has no endpoint field. SDK profile/environment endpoint settings +remain operator-owned. Structural CLI validation makes no AWS request and reports +provider availability as unknown, not an offline Cargo proof. + +## Operator-owned table + +Create the table outside EdgeZero, in the selected region. The first version +requires an ACTIVE, single-region table with no secondary/vector indexes or +replicas, and exactly these String keys: + +| Attribute | Role | +| --------------- | ---------------------------------------------------------- | +| `pk` | Partition key, the configured physical namespace | +| `sk` | Sort key, the exact portable UTF-8 key | +| `v` | Non-key Binary value, including empty values | +| `expires_at_ms` | Optional Number, logical Unix epoch millisecond expiration | +| `ttl_epoch_s` | Optional Number, background cleanup timestamp | + +Enable DynamoDB TTL on `ttl_epoch_s` for garbage collection. The provider does not +create or mutate the table or its TTL setting. Disabled cleanup retains expired +records and storage costs, although reads/listings still treat them as missing. + +Grant the runtime role only `dynamodb:DescribeTable`, `dynamodb:GetItem`, +`dynamodb:PutItem`, `dynamodb:DeleteItem` and `dynamodb:Query` on the reviewed table +ARN. Restrict `dynamodb:LeadingKeys` to approved namespaces where appropriate. +Table creation, schema/TTL configuration and other publication permissions belong +to the operator role, not the application. The bounded startup DescribeTable +checks shape/status, not write permissions, workload capacity or deployed IAM +correctness. + +Physical namespaces must be nonempty, control-free and at most 512 UTF-8 bytes. +Different namespaces isolate data in a shared table. Two bindings to the same +region/table/namespace deliberately share data. Choose an app/environment-specific +namespace rather than a bare common logical ID. One namespace can become a hot +partition; this provider does not shard or certify throughput. + +## Semantics and limits + +- Keys retain their exact UTF-8 bytes and portable 512-byte limit. No normalization + or case folding occurs. Prefix listing uses Query, never a whole-table Scan. +- Payloads are limited to **384 KiB**, below DynamoDB's 400 KiB whole-item ceiling. + The portable 25 MiB validation ceiling does not override this backend limit. + Excess input returns a fixed validation error before I/O. There is no chunking, + S3 fallback or truncation. +- Put replaces the whole item. Put without TTL removes prior expiration metadata. + TTL uses millisecond logical expiry and rounded-up seconds for service cleanup. + Get/Exists/List check logical expiry after the response. Reads do not delete an + expired item, avoiding a race with renewal. Synchronize application host clocks. +- Get and Query request strong consistency in the configured region. Pages are + not snapshots across calls. Concurrent mutations can affect later pages, and + `KvHandle::read_modify_write` remains non-atomic. There is no CAS, lock, atomic + counter or distributed identity-graph guarantee. +- Each list call makes one Query with an evaluated-item limit. DynamoDB's 1 MiB + evaluated-data boundary and expired rows can produce short or empty pages with + a cursor. Continue until `cursor` is absent, not until `keys` is empty. Cursors + are bounded and scoped to region/table/namespace/prefix. They are not secret or + authenticated tokens and must not be logged. +- One SDK attempt is made per operation, with no application retry loop. A timeout + or error can follow a committed Put/Delete. Retrying from application code can + overwrite or delete a concurrent writer's newer state. No exactly-once claim is + made. Missing/expired items alone are successful misses; provider failures are + errors, never misses. +- Default budgets are 5 seconds per operation, 1 second connect and 2 seconds per + attempt. A binding's strict `[kv.state.limits]` table can lower the payload cap + and set ordered positive timeout values. Runtime calls receive fresh deadlines; + the startup deadline is not reused after readiness. Dropping a future does not + guarantee termination of credential subprocesses, DNS or socket work. + +```mermaid +flowchart TD + Binding[Validate every source and compiled capability] --> Setup[Bounded client and table setup] + Setup --> Registry[Complete registry and application initializer] + Registry --> Listener[Open listener] + Listener --> KV[Handler uses portable KvHandle] + KV --> SDK[One bounded live DynamoDB operation] +``` + +## Diagnostics and verification scope + +EdgeZero-owned errors/debug output do not retain SDK response details or raw +keys, values, credentials or cursors. **Dependency DEBUG/TRACE is not suppressed.** +DynamoDB SDK model Debug can expose item keys and values, including binary bytes, +without Secrets Manager's sensitive-output protections. Review application logging +filters before enabling SDK diagnostics. The provider does not install subscriber +suppression or change global logging. Do not assume `LOG_SENSITIVE_BODIES=false` +redacts DynamoDB model Debug. + +Default verification uses deterministic fixtures and the actual SDK wire path +with explicit dummy credentials and literal-loopback endpoints. CI also runs a +required [independent local-service suite](./native-aws-stores#account-free-integration-tests) +against DynamoDB Local, including base-table Query pagination across the 1 MiB +evaluation boundary. Neither suite proves AWS IAM, capacity, deployed service +behavior or global-table correctness. Real AWS acceptance requires separate +authorization and disposable-resource ownership. + +See [native AWS stores](./native-aws-stores) for captured bootstrap and startup +composition. diff --git a/docs/guide/native-aws-stores.md b/docs/guide/native-aws-stores.md new file mode 100644 index 000000000..115d2eccf --- /dev/null +++ b/docs/guide/native-aws-stores.md @@ -0,0 +1,286 @@ +# Native AWS stores + +This guide is a draft for the first native implementation. It covers two opt-in providers: +AppConfig Agent for complete configuration documents and Secrets Manager for complete secret +values. They can be selected independently per logical store ID. They do not apply to Fastly, +Cloudflare, or Spin builds, and they do not add write, publish, provision, or rotation commands. + +A separate [DynamoDB KV provider](./dynamodb-kv) adds live mutable KV operations through the +independent `aws-dynamodb` feature. It does not change either snapshot provider or the +existing `aws-stores` alias. + +These two providers read at process startup and keep immutable in-memory snapshots. Restart the +application to adopt a new config document, secret value, or moved version stage. A running +process keeps secrets already loaded even after remote deletion or access revocation. + +## Build the providers you use + +Generated native targets can opt into either service: + +```toml +[adapters.axum.build] +features = ["aws-appconfig-agent", "aws-secrets-manager"] +``` + +`aws-stores` is a convenience alias for both. Config-only apps need not include the Secrets +Manager SDK. Secret-only apps need not include the AppConfig Agent HTTP client. All features +are off by default. A bindings file cannot enable a service that the application binary did +not compile. + +CLI bindings validation is structural and offline: + +```sh +edgezero config validate \ + --adapter axum \ + --store-bindings /etc/my-app/native-stores.toml \ + --target-features aws-appconfig-agent,aws-secrets-manager +``` + +The bindings path must be absolute. Target controls include +`--target-all-features`, `--target-no-default-features`, and `--target `. The command +normalizes these controls without resolving Cargo's target feature graph. It always reports +structural validation only and says provider availability is unknown. This check does not +load credentials, contact AWS or the Agent, prove resource access, or validate remote +contents. Runtime startup validates the actual selected inputs again. Set +`EDGEZERO__STORE_BINDINGS_FILE=/etc/my-app/native-stores.toml` before starting the generated +native target. Its entrypoint selects strict startup when this locator is present. Without +it, the generated target keeps its existing local development behavior. An empty or invalid +selected locator fails startup rather than selecting local stores. + +## Declare portable stores + +Keep provider resources out of `edgezero.toml`. The application declares logical IDs and +defaults, not AWS resources: + +```toml +[stores.config] +ids = ["application"] +default = "application" + +[stores.secrets] +ids = ["runtime"] +default = "runtime" +``` + +A deployment selects providers in a separate native file. This example uses fictional +resource names and account `111122223333`: + +```toml +version = 1 + +[config.application] +provider = "aws-appconfig-agent" +default_key = "settings" +endpoint = "http://127.0.0.1:2772" +access_token_env = "APPCONFIG_AGENT_TOKEN" + +[config.application.documents.settings] +application = "checkout" +environment = "production" +profile = "checkout-envelope" + +[secrets.runtime] +provider = "aws-secrets-manager" +region = "us-east-1" + +[secrets.runtime.entries.signing_key] +secret_id = "arn:aws:secretsmanager:us-east-1:111122223333:secret:checkout-signing-AbCdEf" +version_stage = "AWSCURRENT" +``` + +The schema rejects unknown fields, unsupported provider tags, empty mappings, undeclared IDs, +and conflicting selectors. The names are case-sensitive. Every configured mapping is required +during preparation. A missing mapped document or secret fails startup. There is no fallback to +`.edgezero/` or an environment secret. An unmapped lookup returns a normal miss and never +triggers another AWS request. Runtime errors use closed, fixed categories; they do not retain +SDK/HTTP error chains or raw resource values in EdgeZero-owned diagnostics. + +Use exactly one selector for each secret: no selector means `AWSCURRENT`; set `version_stage` +or `version_id` to pin another version. Do not set both. Supply a complete secret ARN and an +explicit region. The file contains identifiers and environment variable names, never +credentials, tokens, or secret values. Protect it as deployment configuration nonetheless. + +## Start the AppConfig Agent + +EdgeZero talks to a local Agent HTTP endpoint. It does not start or supervise the Agent, poll +AWS AppConfig, or run AppConfig Data session-token logic. The Agent owns its AWS role, region, +AWS polling, and cache. The application makes one GET per distinct mapped profile at startup. + +Run the Agent in the same network namespace as the application and configure it to bind to +`127.0.0.1`. The provider accepts only literal loopback IP endpoints, so endpoint hostnames do +not trigger DNS. It rejects redirects, proxy use, and non-identity content encoding. It sends +`Accept-Encoding: identity`. When the binding names a token variable, startup reads it once and +sends `Authorization: Bearer `. A missing or empty configured token fails startup. +Configure the Agent's `ACCESS_TOKEN` to the same value. Never put the token in a URL or the +bindings file. + +Pin the operator-reviewed Agent image by digest in deployment configuration, for example +`public.ecr.aws/aws-appconfig/aws-appconfig-agent@sha256:`. The +placeholder is not a usable image reference. Before publishing this guide, the operator must +select the exact Agent release, verify its image digest through the approved artifact process, +and record both in deployment documentation. Do not replace the placeholder with an +unverified or invented digest. Disable Agent disk backup/preload for this setup; this does not +make its in-memory cache fresh or prove that AWS was reachable when the application starts. A +successful read proves only that the Agent supplied acceptable bytes. + +Do not expose the Agent endpoint to untrusted processes. If processes share its network +namespace, require token authentication. A token does not isolate the endpoint from a +compromised application process. + +## Preserve complete values + +An AppConfig mapping returns the profile's entire UTF-8 document. It does not select a JSON +field, parse or reserialize a document, unquote a JSON string, or resolve secret references. +A raw profile containing `hello` returns those exact characters. A profile containing a +serialized EdgeZero `BlobEnvelope` returns that serialization unchanged; core validates the +envelope when the app loads typed config. + +For example, a free-form profile can contain one serialized envelope: + +```json +{ + "version": 1, + "generated_at": "2026-10-07T12:00:00Z", + "sha256": "", + "data": { "message": "hello", "api_token": "signing_key" } +} +``` + +The `api_token` field in this example is a secret reference name, not a token value. The +application's typed config must declare the appropriate `#[secret]` field and must explicitly +check required config and secret references during its startup initializer before readiness. +The provider does not know which values the application requires. It does not claim that +startup checks or application-specific validation have run merely because snapshots loaded. + +Secrets Manager returns each configured value in full. `SecretString` is kept as its exact +UTF-8 bytes; the SDK's `SecretBinary` is already decoded bytes and is not base64-decoded a +second time. JSON secrets remain whole JSON byte values. Typed string references still need +UTF-8; byte-oriented consumers can retain binary values. + +## Startup flow + +```mermaid +flowchart TD + Bindings[Capture explicit bindings and named token values] --> Validate[Validate every source and actual compiled providers] + Validate --> Prepare[Prepare config and secret snapshots sequentially in one session] + Prepare --> Registries[Publish the complete registries] + Registries --> Initialize[Run the application-owned startup initializer] + Initialize --> Listen[Bind listener and report readiness] + Listen --> Requests[Serve from immutable snapshots without provider requests] +``` + +The generated entrypoint prepares provider snapshots but does not invent application-specific +startup checks. To require typed config and nested secrets before listening, replace its native +main with `run_production_app_with_initializer::(...)`. The callback receives +`&mut App` and `&PreparedStores`; use core `AppConfig::::from_binding` with the selected +config binding, secret registry, extraction limits and app clock. Insert validated state with +`app.insert_state(...)`. The callback may borrow both arguments and need not be `Send`. The +framework waits for it before binding. The generated-consumer integration test exercises this +public callback, a complete envelope and nested secret resolution against loopback fixtures. + +## Limits and lifecycle + +The shared AWS preparation session defaults to an 8 MiB cap over unique retained config and +secret payload bytes. The limit is per standard runner preparation session, not process-wide +and not a bound on RSS. Separate sessions, caller-supplied handles, local providers, SDK +response allocations, credential providers, TLS buffers, typed application state, and allocator +overhead are outside this cap. Config documents and secret entries also have per-value and +mapping-count limits. Treat the 8 MiB value as a configurable provisional default, not an AWS +service limit or a measured workload requirement. + +Preparation is sequential in the initial implementation. It has finite operation and aggregate +budgets, but dropping a future does not prove that credential discovery, a subprocess, or every +socket operation stopped. A startup failure drops the runner's staged snapshot and stops before +readiness. Use the process supervisor for restart policy and backoff. + +Config values are snapshots by mapped document. The snapshots do not promise a coordinated +revision across profiles or across config and secrets. If multiple documents must change +atomically, place them in one envelope or have application-owned release validation reject +mismatched revisions. Restart to adopt changes. + +A running process retains loaded secrets until it exits. Remote deletion, rotation, or revoked +access does not erase those bytes. Restart to adopt the selected version or remove a secret. +`Bytes`/`Arc` ownership does not promise memory zeroization. Do not log values, ARNs, tokens, +credential sources, or response bodies. + +**Dependency logging limitation.** EdgeZero-owned startup errors and diagnostics use redacted +categories, but the first version does not suppress DEBUG/TRACE events emitted by the AWS SDK +and its dependencies. Those events can expose secret ARNs, credential-source metadata, and, +when `LOG_SENSITIVE_BODIES=true`, response details. Scoped suppression was not adopted because +it can permanently disable an application's tracing-to-log fallback. Keep AWS dependency +`DEBUG`/`TRACE` off in production unless operators have reviewed their logger, sinks, and +sensitive-body settings. This limitation must be called out in reviews of the first version. + +## Account-free integration tests + +CI runs the existing SDK loopback fixtures and a required independent-service +suite. Run the latter locally with Docker, Node.js and the repository Rust +toolchain, after fetching the locked Cargo dependencies: + +```sh +node scripts/run_native_aws_integration.mjs +``` + +The Node runner starts digest-pinned DynamoDB Local, Moto and the actual AWS +AppConfig Agent in local development mode. It creates disposable resources, +builds one CLI-generated Axum application with all three providers, and removes +its containers, network and temporary files afterward. Missing Docker, service +startup failures and zero executed tests are errors, not skips. Fixture +administration uses a dependency-free Node.js helper against the published +loopback endpoints. No Python helper or npm installation is needed. + +The suite checks exact binary/config/secret bytes, per-ID namespace isolation, +base-table Query continuation across the 1 MiB evaluation boundary, logically +expired rows that remain stored, TTL renewal/reset, startup rejection before +listening, typed initialization, and persistence/snapshot behavior across +application restarts. It waits until the real Agent independently serves updated +files before checking that the running app retains its old snapshot. + +Image acquisition requires network access, but the suite does not require an AWS +account. Generated application children use cleared environments, empty AWS +profile files, dummy credentials, disabled metadata discovery and validated +literal-loopback endpoints. Published container ports bind only to `127.0.0.1`. +The Docker bridge is not a hard outbound-network sandbox. These tests do not +establish deployed IAM/KMS permissions, capacity, production consistency or AWS +TTL cleanup timing. Agent local development mode does not exercise AWS polling, +deployment validation or rollback. + +Moto 5.2.2 has a verified limitation: manually moving `AWSCURRENT` updates its +labels but not its no-selector `GetSecretValue` result. The suite therefore tests +default-version adoption through normal `PutSecretValue`, and independently tests +explicit named-stage movement and pinned version IDs. It does not change the +provider to accommodate Moto or claim manual `AWSCURRENT` move parity. Test image +pins are recorded in the runner; they are not production deployment approval. + +## Credentials and permissions + +For the application, use the AWS SDK's supported credential chain. In production, prefer the +workload role attached to the service. For local development, use a named authenticated AWS +profile or another supported developer credential source. The bindings file and container +image must not contain static access keys. Secret-only applications initialize the Secrets +Manager client; AppConfig-only applications leave AWS credentials to the Agent. + +Grant the Agent role only the AppConfig Data read actions it needs: + +- `appconfig:StartConfigurationSession` for the selected application, environment, and profile. +- `appconfig:GetLatestConfiguration` for that configuration session/resource scope. + +Grant the application role `secretsmanager:GetSecretValue` for the explicitly mapped secret +ARNs, and `kms:Decrypt` only when a selected customer-managed key requires it. Runtime roles +do not need resource creation, AppConfig deployment, secret mutation, staging-label changes, +or rotation-Lambda management. Review the exact IAM resource scoping for the deployed AWS +service and account before deployment. This guide is not an IAM acceptance result. + +AWS charges and service quotas depend on the selected AWS resources, Agent polling, region, and +traffic. Estimate them from the deployment configuration. Keep a named operator responsible +for stopping the Agent and removing disposable AppConfig/Secrets Manager resources after +experiments; the runtime and CLI do not clean them up. + +## Push remains local + +`edgezero config push --adapter axum` writes local config snapshot files. It does not publish +AppConfig data. The `--store-bindings /absolute/path` push guard rejects a selection +that binds the chosen logical ID to AWS, before diffing, prompting, or writing. Without that +explicit file, a local push is still only a local write; it makes no claim about a separate +AWS deployment. There are no AWS publish, provision, secret-mutation, or cleanup commands in +this feature. diff --git a/docs/guide/native-celld-stores.md b/docs/guide/native-celld-stores.md new file mode 100644 index 000000000..4f8705254 --- /dev/null +++ b/docs/guide/native-celld-stores.md @@ -0,0 +1,190 @@ +# Native celld stores + +The native provider reads complete configuration documents at startup and exposes live KV operations through a scoped listener **inside celld**. There is no gateway, sidecar, Worker bridge or celld HTTP adapter. + +This is an unreleased integration. It requires the modified celld revision `f72ae21a515859570622ac9d6ca9c19cf6870b81`. Upstream celld v0.6.2 does not have this API. Local tests do not establish fleet failover, deployment security or production readiness. + +## Build only the providers you use + +Generated Axum targets forward two independent, default-off features: + +```toml +[adapters.axum.build] +features = ["celld-config", "celld-kv"] +``` + +The provider crate uses `config` and `kv`. Its settings compile without live transport dependencies. Applications do not depend on the celld runtime or V8. Existing AWS features and the `aws-stores` alias keep their current membership. + +A deployment file cannot enable a provider omitted from the application binary. Startup checks the compiled capabilities before preparing any selected store. + +## Keep declarations portable + +The application declares logical IDs in `edgezero.toml`: + +```toml +[stores.config] +ids = ["application"] +default = "application" + +[stores.kv] +ids = ["state"] +default = "state" +``` + +A separate native deployment file selects providers: + +```toml +version = 1 + +[config.application] +provider = "celld-config" +endpoint = "https://stores.example.internal:7780" +default_key = "settings" + +[config.application.access] +mode = "authenticated" +token_env = "CHECKOUT_STORE_TOKEN" + +[config.application.documents.settings] +store = "checkout-config" +key = "application-envelope" + +[kv.state] +provider = "celld-kv" +endpoint = "https://stores.example.internal:7780" +store = "checkout-state" + +[kv.state.access] +mode = "authenticated" +token_env = "CHECKOUT_STORE_TOKEN" +``` + +These are fictional resource names. `store` is a service-owned alias, never a physical namespace or runtime scope. The schema rejects unknown fields, malformed endpoints, undeclared IDs and conflicting sources. Endpoints are HTTP or HTTPS origins without userinfo, query, fragment or a non-root path. Keys and aliases retain their exact case and UTF-8 bytes. + +Select the absolute file path with `EDGEZERO__STORE_BINDINGS_FILE`. The environment-loading runner captures the selected file and the union of selected config/KV token names once. Explicit `AxumRunOptions` uses supplied bootstrap values, not ambient variables. Tokens are 32 through 512 header-safe ASCII graphic bytes. Generate random application tokens and inject them through deployment secret management. Do not put token values in TOML or URLs. + +Different IDs can select celld, AWS, local providers or supplied handles. Celld-only bindings need no AWS credentials or local roots. An unbound ID retains its existing local default and may require a local root. There is no celld secret provider. A typed celld config document may reference a separately bound AWS secret store. + +## Configure the listener in celld + +Add `--store-api-config /absolute/scoped-stores.json` to the existing celld deployment command. Without this selection, celld opens no scoped socket. + +An authenticated policy has this form: + +```json +{ + "version": 1, + "listen": "0.0.0.0:7780", + "tls": { + "cert": "/etc/celld/server-chain.pem", + "key": "/etc/celld/server-key.pem" + }, + "stores": [ + { "alias": "checkout-config", "namespace": "production-checkout-config" }, + { "alias": "checkout-state", "namespace": "production-checkout-state" } + ], + "access": { + "mode": "authenticated", + "tokens": [ + { + "id": "checkout", + "sha256": "", + "grants": [ + { "store": "checkout-config", "operations": ["read"] }, + { + "store": "checkout-state", + "operations": ["read", "write", "delete", "list"] + } + ] + } + ] + } +} +``` + +Replace the verifier placeholder through operator-owned provisioning. It is not a usable policy value. The application receives only its own bearer token. Celld owns the TLS key, policy, physical namespaces, fleet discovery and fleet credentials. + +Celld captures policy and certificates at startup. Restart celld to adopt changed grants, replace certificates or revoke a token. Overlapping credential IDs permit staged token replacement. There is no issuance service or hot reload. Loaded application config snapshots remain available after remote revocation until the application restarts. Live KV calls use current server authorization. + +### Trusted-network access + +A binding may instead select: + +```toml +[config.application.access] +mode = "trusted-network" +``` + +Literal loopback HTTP needs no further client opt-in. Remote HTTP requires `allow_remote_http = true`. This flag permits unencrypted transport; it does not establish network isolation. HTTPS still verifies the server certificate. + +The corresponding celld policy uses `mode: "trusted-network"` with a `grants` list. Every reachable caller receives those grants. Non-loopback service binding also requires `allow_remote_trusted_network: true`. Trusted-network mode rejects supplied credentials, and authentication failure never falls back to anonymous access. + +Restrict the scoped listener to intended workloads. Block application access to celld's internal operator listener and fleet credentials, including filesystem and process views. A private IP or successful TLS request does not prove these restrictions. + +```mermaid +flowchart TD + App[Native application with its application token] --> Scoped[celld scoped TLS listener] + Scoped --> Policy[Exact alias and operation authorization] + Policy --> Owner[Existing in-process owner dispatch] + Owner --> KV[Reserved celld KV storage] + Operator[Operator-owned policy and fleet credentials] --> Scoped +``` + +The diagram has no application route to the internal operator listener. Enforce that boundary in the actual deployment, not just in application code. + +## Configuration lifecycle + +Startup validates every source selection, policy, setting and selected credential before provider I/O or deferred preparation. It prepares config snapshots before running the application initializer, publishing readiness or accepting requests. Missing documents, invalid UTF-8, quota failures and remote errors fail startup without local or stale fallback. + +The provider preserves whole UTF-8 documents, including whitespace and empty documents. It does not project JSON fields or resolve secrets. Core validates `BlobEnvelope`, typed configuration and secret references when the application initializer requests them. + +Ordinary and bounded config reads use immutable shared values and perform no network I/O. Publish new complete documents through operator-owned tooling, then restart the app to adopt them. Multiple documents and providers are not an atomic release. `edgezero config push` refuses selected celld config bindings; it does not publish to celld or write a substitute local file. + +## Live KV and limits + +KV preparation builds a handle without probing the service. Startup success alone does not prove connectivity or operation grants. Each live call has a fresh deadline. + +| Limit | Default or service ceiling | +| ------------------------------------------------ | -------------------------------------------------------- | +| Raw value and config document | 1 MiB each | +| Retained config snapshot per preparation session | 8 MiB | +| Logical mappings before deduplication | 1,024, counting each config document and KV binding | +| Preparation metadata | 1 MiB | +| Key page JSON | 2 MiB | +| Page entries | At most 1,000 | +| Service cursor / provider cursor | 2 KiB / 8 KiB | +| Remote key | Nonempty UTF-8, at most 512 bytes, no controls | +| Store alias | At most 64 ASCII letters, digits, underscores or hyphens | +| Preparation / operation / connect timeout | 30 / 5 / 1 seconds | + +`[limits.celld]` or `with_celld_preparation_limits` selects one celld policy. File and explicit policies do not merge. AWS has an independent policy and accounting. Client caches hold at most 128 authorization-scoped entries. Accounting describes logical payload and metadata quotas, not process RSS. + +Binary and empty values remain distinct from missing keys. Missing delete succeeds. TTL is between 60 and 31,536,000 seconds. Fractional valid durations round upward without shortening lifetime; unsupported durations fail rather than clamp. TTL overwrite renews expiry. Plain put removes expiry. + +Listing returns one bounded page, including a short or empty continuing page. It is not a snapshot. Cursors bind endpoint, alias and prefix, but never confer authorization. Preserve a returned continuation rather than scanning repeatedly to fill a page. + +The client disables redirects, ambient proxies, retries and decompression. It validates the version header and bounds streamed values, pages and error bodies. Celld scoped dispatch also disables ambiguous replay. A failed or timed-out mutation may have applied. The provider does not promise exactly-once operations, CAS or transactions. + +Provider-owned errors and Debug output redact credentials and omit remote payloads, URLs and source chains. Dependency DEBUG/TRACE logging is outside that promise. Application-owned initializer errors and logs remain the application's responsibility. Review deployment logging before enabling it. + +## Validation and local proofs + +Offline validation remains structural-only: + +```sh +edgezero config validate --adapter axum \ + --store-bindings /etc/my-app/native-stores.toml \ + --target-features celld-config,celld-kv +``` + +Provider availability is always unknown offline. Target controls normalize and forward Cargo arguments; validation does not resolve the application's feature graph, capture tokens, connect to endpoints or verify grants/content. + +Normal CI uses deterministic loopback client and public-runner fixtures. The generated-consumer proof builds default-off, config-only, KV-only and combined targets. The separate real-service command requires an explicit modified celld executable, Linux namespace tools, curl and OpenSSL: + +```sh +CELLD_BIN=/absolute/path/to/modified/celld \ +EDGEZERO_GENERATED_TEST_TARGET_DIR=/absolute/path/to/disposable-build-cache \ +node scripts/run_native_celld_integration.mjs +``` + +Missing prerequisites fail the command. It uses disposable data and dummy credentials, checks private namespace isolation, and runs a generated native app against the embedded API with certificate verification. It does not access a real fleet or AWS account. The plan records executed results and remaining gates. Multi-node forwarding, reload, production storage/replication, upgrades and stress acceptance remain separate work. diff --git a/docs/guide/native-geo.md b/docs/guide/native-geo.md new file mode 100644 index 000000000..26a7c31a1 --- /dev/null +++ b/docs/guide/native-geo.md @@ -0,0 +1,514 @@ +# Native Geo providers + +Native Axum hosting can resolve estimated country and subdivision from an operator-supplied +local GeoLite database, protected CloudFront assertions, or a supplied local provider, once per +admitted request. Geo is off by default. There is no remote lookup, fallback provider, downloader, +hot reload, CLI option, environment selector, or manifest selector. + +Geography is an estimate, not proof of residence, identity, consent, or permission. Applications +own jurisdiction recognition and policy. A missing subdivision stays unknown, and a +syntax-valid subdivision is not necessarily assigned or recognized. + +## Select a provider in Rust + +The public selection API lives in `edgezero_adapter_axum::geo` with the `axum` feature. +Built-in implementations use default-off features. Choose `geo-maxmind` for a local database, +`geo-cloudfront` for protected assertions, or both to make both implementations available. +Compiling both still selects only one provider: + +```toml +[dependencies] +edgezero-adapter-axum = { version = "", default-features = false, features = ["geo-maxmind"] } +``` + +Use the same pinned EdgeZero release as your application. This feature enables `axum`, not +AWS stores or an AWS SDK. A Geo-only application needs no AWS credentials, config/data roots, +or declared stores. Existing declared stores keep their own startup requirements. + +### Off + +Every options constructor defaults to `Off`, including `from_env` and `from_vars`. Selection +is always programmatic, even when other hosting settings come from the environment. + +```rust +use edgezero_adapter_axum::dev_server::AxumRunOptions; +use edgezero_adapter_axum::geo::{NativeGeoCapability, NativeGeoSelection}; +let options = AxumRunOptions::new("127.0.0.1:8787".parse()?)? + .with_geo(NativeGeoSelection::Off); +assert_eq!(options.geo_capability()?, NativeGeoCapability::Off); +``` + +### Local GeoLite database + +Enable `geo-maxmind` and supply a regular MMDB file. Initially only metadata types +`GeoLite2-City` and `GeoLite2-Country`, with IP version 4 or 6, are supported. City supplies +country and optional first-level subdivision; Country supplies country only. Other MMDB +publishers or types are not supported merely because their format decodes. + +For a server directly receiving reader connections: + +```rust +use edgezero_adapter_axum::dev_server::AxumRunOptions; +use edgezero_adapter_axum::geo::{MaxMindGeoConfig, NativeGeoSelection, ReaderIpExpectation}; + +let options = AxumRunOptions::new("0.0.0.0:8787".parse()?)? + .with_geo(NativeGeoSelection::MaxMind { + config: MaxMindGeoConfig::new( + "/var/lib/edgezero/geo/GeoLite2-City.mmdb", + ReaderIpExpectation::PeerAllowed, + ), + }); +let _configured = options.geo_capability()?; +``` + +Behind a reverse proxy, require an explicitly resolved forwarded reader instead of geolocating +the proxy. Protect the proxy-to-server network and configure the actual proxy peer ranges: + +```rust +use edgezero_adapter_axum::dev_server::AxumRunOptions; +use edgezero_adapter_axum::geo::{MaxMindGeoConfig, NativeGeoSelection, ReaderIpExpectation}; +use edgezero_adapter_axum::proxy::{ForwardingHeaderFamily, TrustedProxyPolicy}; + +let proxy = TrustedProxyPolicy::new( + ForwardingHeaderFamily::Forwarded, ["10.20.0.8/32"], +)?; +let options = AxumRunOptions::new("0.0.0.0:8787".parse()?)? + .with_trusted_proxy_policy(proxy) + .with_geo(NativeGeoSelection::MaxMind { + config: MaxMindGeoConfig::new( + "/var/lib/edgezero/geo/GeoLite2-City.mmdb", + ReaderIpExpectation::ForwardedRequired, + ), + }); +``` + +Use `ForwardingHeaderFamily::XForwarded` for an ingress that owns the X-Forwarded chain. +See [trusted proxy configuration](./axum-diagnostics.md#what-makes-a-forwarding-assertion-trustworthy) +for chain validation and network controls. This is not the protected CloudFront/ALB policy. +No origin credential or geographic header is needed for local lookup. + +| Checked ingress | `PeerAllowed` | `ForwardedRequired` | +| ------------------------------------------------ | ------------------- | ------------------- | +| Trusted Forwarded or X-Forwarded client resolved | Use checked client | Use checked client | +| Direct client with actual socket peer | Use normalized peer | `MissingReaderIp` | +| No usable checked reader or no actual peer | `MissingReaderIp` | `MissingReaderIp` | + +IPv4-mapped IPv6 addresses retain ingress normalization. A resolved forwarded client remains +usable when host or scheme is absent. Malformed or unresolved forwarding can leave direct facts; +`PeerAllowed` deliberately permits that peer, so it is the wrong expectation for a proxy-only +reader path. Raw headers, forged public extensions and caller-supplied connection metadata cannot +supply the private checked reader. Geographic headers never affect local lookup. + +#### Mount and prepare the snapshot + +Download and verify the dataset in your deployment process, outside the server. Check its +publisher-provided integrity information and protect the distribution channel. Do not put +MaxMind credentials in the runtime environment or image. For example, mount an already populated +host directory read-only into your existing application image: + +```sh +docker run --rm \ + --mount type=bind,src=/srv/edgezero/geo,dst=/var/lib/edgezero/geo,readonly \ + -p 8787:8787 your-app-image +``` + +The Rust selection above supplies the path; the mount alone enables nothing. Make the file +readable by the runtime user. EdgeZero creates no database, sidecar or lock file and does not +need AWS credentials, a writable data root or a database server. File acquisition is currently +supported on Unix; other platforms reject preparation before opening the source. + +Preparation reads the file once into an immutable in-memory reader before application +initialization, binding and readiness. It rejects missing/nonregular files, unsupported metadata, +corruption and incompatible consumed field types. Upstream verification is followed by a bounded +recursive structural pass over reachable data records and borrowed geographic projection checks. +The pinned reader still accepts some malformed unused metadata fields. This is an open +[#438 acceptance blocker](https://github.com/IABTechLab/edgezero/issues/438); the data-record pass +does not close it. Validation also does not prove publisher authenticity. Structurally valid but +unsupported code strings fail only when looked up. + +`MaxMindGeoConfig::limits` is public `MaxMindPreparationLimits`, defaulting to +`max_database_bytes = 128 * 1024 * 1024` and `max_validation_networks = 32_000_000`. +The actual read is capped, including an extra-byte overflow check. The visit budget bounds the +upstream tree allowance and network iteration, including aliases and no-data leaves. The pinned +reader also bounds per-decode values, payload and depth; verification memoizes shared values, +while the structural pass expands them under decoder limits. Allocation grows with file bytes, +referenced values and visited records. These are not fixed-RSS or wall-clock guarantees. +Exceeding a bound fails startup, never skips validation. Raise limits deliberately and measure +preparation time and peak memory with your permitted production snapshot before rollout. + +Preparation is synchronous and pre-runtime, before signal registration and polling an embedding +stop future. An already-requested stop still waits for preparation; an invalid selected database +still fails startup first. Early signals follow the caller/OS disposition. Filesystem I/O time is +not bounded by byte/work limits. Cooperative startup cancellation applies only after this phase. + +After logging initialization, startup diagnostics identify the loaded database type, build epoch +and SHA-256 of the loaded bytes. These identify the snapshot, not freshness, accuracy or +signature verification. Configuration Debug redacts paths; reader input Debug redacts addresses. +Outcomes retain no reader address or database path. Default framework diagnostics do not log +reader IPs, geographic input headers or raw decode errors. Application code must preserve that +redaction and avoid address/code/checksum metric labels. + +#### Updates, licensing and deletion + +GeoLite data is free of charge. A free MaxMind account and download license key are required +for downloads, not for running a local snapshot. The `maxminddb` reader's ISC license is +separate from the dataset's [GeoLite EULA](https://www.maxmind.com/en/geolite/eula), incorporated +Creative Commons terms, attribution and redistribution obligations. Recheck the current terms +before distributing an image, dataset or derived data. EdgeZero does not bundle production data. + +Provide the required attribution in the relevant product documentation and advertising, following +[MaxMind's guidance](https://support.maxmind.com/knowledge-base/articles/sell-or-display-data-from-geolite-databases-and-web-services): + +> This product includes GeoLite Data created by MaxMind, available from https://www.maxmind.com. + +Operators own [database updates](https://dev.maxmind.com/geoip/updating-databases/): + +1. Download and verify a complete replacement outside the runtime. Never rewrite the source in + place during preparation. Publish the finished file or update the mount/reference atomically. +2. Restart every serving process. Existing processes keep their old in-memory snapshot even if + the source is replaced or deleted. A new process validates the replacement before readiness; + invalid data fails startup without silently reusing an older file. +3. Track release dates and retire old data. The EULA requires promptly using updates and ceasing + use of and destroying old versions within thirty days after an updated database is released. + This includes running in-memory snapshots and retained rollback copies, images and backups. + Stop/restart old processes and remove expired artifacts. Roll back only to data still allowed + by the terms. Termination has its own immediate destruction requirement. + +No updater, watcher, live swap or freshness cutoff is built in. File modification time, a +successful load and a checksum do not prove freshness. DB-IP City Lite is a separately licensed +MMDB alternative, not currently supported by this integration. + +### CloudFront + +First deploy and verify the [protected origin](#protect-the-cloudfront-origin). Selecting the +policy below declares that those infrastructure controls are enforced. It does not configure +or inspect AWS. + +```rust +use edgezero_adapter_axum::dev_server::{AxumRunOptions, run_app_with_options_and_initializer}; +use edgezero_adapter_axum::geo::{NativeGeoSelection, ProtectedCloudFrontAlb}; +use edgezero_core::app::Hooks; + +fn run() -> anyhow::Result<()> { + let policy = ProtectedCloudFrontAlb::new( + ["10.42.1.0/24", "10.42.2.0/24"], "x-origin-verify-randomsuffix", + )?; + let options = AxumRunOptions::new("0.0.0.0:8787".parse()?)? + .with_geo(NativeGeoSelection::CloudFront { policy }); + let capability = options.geo_capability()?; + run_app_with_options_and_initializer::( + options, std::future::pending::<()>(), move |_app, _stores| { + Box::pin(async move { + let _configured_capability = capability; + // Validate application requirements here, without inferring + // that every request will have geographic metadata. + Ok(()) + }) + }, + ) +} +``` + +The constructor accepts literal IPv4/IPv6 addresses or CIDRs, not DNS names or provider aliases. +The list must be nonempty, with at most 256 supplied entries and 16,384 input bytes including +separators. Duplicates count before normalization. Whole-address-family coverage, including +unions and unrestricted mapped-IPv4 ranges, is rejected. IPv4-mapped IPv6 peers match their +IPv4 equivalents. Use the narrowest actual ALB peer ranges and enforce their ownership through +security groups. A matching private address alone does not authenticate a distribution. + +The dedicated credential-header name must be a valid ASCII HTTP token of at most 128 bytes. +After case normalization it must start with `x-`. These names are forbidden: + +- Every `x-forwarded-*`, `x-amz-*`, `x-amzn-*`, and `x-edge-*` name. +- `x-real-ip`, `x-request-id`, and `x-correlation-id`. + +The `x-` rule also excludes standard authorization, cookie, transport, and CloudFront geographic +headers. Do not include a credential value in this policy. EdgeZero never compares that value. +Policy debug output and configuration errors omit both the name and network literals. + +Selecting `CloudFront` without compiling `geo-cloudfront` is a startup error, not `Off`. +The selector remains available so a missing implementation cannot silently disable a required +configuration. Invalid selections fail before runner resources, deferred store setup, the +application initializer, and listener binding. + +### Application-supplied provider + +Supply a local implementation of `edgezero_core::geo::GeoProvider`. It works with only `axum` +enabled, without either built-in implementation. This composition-only example +always returns missing metadata; replace `ExampleProvider` with your own bounded implementation: + +```rust +use std::sync::Arc; +use edgezero_adapter_axum::dev_server::AxumRunOptions; +use edgezero_adapter_axum::geo::{NativeGeoSelection, ProtectedCloudFrontAlb}; +use edgezero_core::geo::{GeoError, GeoInput, GeoProvider, GeoProviderId, GeoResolution}; + +struct ExampleProvider; +impl GeoProvider for ExampleProvider { + fn resolve(&self, input: &GeoInput) -> GeoResolution { + match input { + GeoInput::Assertions { .. } => GeoResolution::MissingMetadata, + GeoInput::ReaderIp { .. } => GeoResolution::Error(GeoError::ProviderFailure), + } + } +} +let policy = ProtectedCloudFrontAlb::new( + ["10.42.1.0/24"], "x-origin-verify-randomsuffix", +)?; +let options = AxumRunOptions::new("127.0.0.1:8787".parse()?)? + .with_geo(NativeGeoSelection::Supplied { + policy, + id: GeoProviderId::new("example.local")?, + provider: Arc::new(ExampleProvider), + }); +let _capability = options.geo_capability()?; +``` + +Provider IDs are public diagnostic labels, never secrets. They match +`[a-z][a-z0-9._-]{0,63}`. The built-in `cloudfront` and `maxmind` identities are reserved and cannot label a +supplied provider, including IDs obtained through their built-in constructors. + +`GeoProvider::resolve(&self, input: &GeoInput) -> GeoResolution` is synchronous, bounded local +computation. `GeoInput::Assertions { country, subdivision }` carries country and subdivision slots +of at most two and three bytes. Each slot is absent, bounded, or marked invalid. Preserve invalid +markers rather than turning them into missing data. `GeoInput::ReaderIp { address, source }` +carries a typed address and `GeoIpSource`, not arbitrary headers or service handles. Providers +must return `Error(GeoError::ProviderFailure)` for unsupported input variants, without probing +another input. Results use validated `CountryCode`, `SubdivisionCode` and `GeoLocation` types, +`NoMatch`, `MissingMetadata` or bounded `GeoError` categories. + +Do not perform blocking I/O, remote requests, filesystem access, or background preparation. +The portable trait has no `Send`/`Sync` supertraits; native ownership adds these bounds to the +supplied object. Application Rust is trusted code, not a sandbox. `Supplied` uses the same +protected CloudFront/ALB assertion contract. For checked reader input, use +`NativeGeoSelection::SuppliedReaderIp { expectation, id, provider }`, where expectation is +`ReaderIpExpectation::PeerAllowed` or `ForwardedRequired`, ID is an application `GeoProviderId` +and provider is `Arc`. It works with `axum` alone and uses the same +private checked-reader admission as MaxMind, without claiming to be a local database. + +There is exactly one selection. A later `with_geo` call replaces the entire earlier selection; +it does not merge policies or create a fallback chain. Provider failures do not switch providers. + +### Inspect startup capability + +Capture capability from the final options, then pass those same options to the existing +initializer-based runner. No extra initializer argument or App registry is needed. + +Capability is `NativeGeoCapability::Off` or `NativeGeoCapability::Selected { provider, input }`. +Input is `NativeGeoInputContract::ProtectedCloudFrontAlb` or +`NativeGeoInputContract::ReaderIp { expectation }`. Inspection validates configuration and feature +availability without opening, reading or preparing a database, even for a nonexistent path. +It cannot report loaded metadata or eventual per-request provenance. The CloudFront example +above captures capability in the initializer closure. + +This embedding example waits indefinitely; supply your own stop future in a real host. +Explicit-options runners install no process-wide signal handlers. Capability inspection returns +`Result`. It confirms validated selection, not +per-request coverage, infrastructure protection, or readiness of a downstream policy. + +## Read the immutable request outcome + +Middleware and handlers read `RequestContext::geo()`. An outcome does not set an HTTP status +or grant access. Call `describe(ctx.geo())` with the following classifier. Its labels describe +data availability, not permission. Applications must separately recognize each subdivision, +preserving unknown values rather than treating them as known non-regulated jurisdictions: + +```rust +use edgezero_core::geo::{GeoError, GeoOutcome, GeoStatus, GeoUnavailableReason}; +fn describe(outcome: &GeoOutcome) -> &'static str { + match outcome { + GeoOutcome::NoSelection => "not configured", + GeoOutcome::Selected { status, .. } => match status { + GeoStatus::Unavailable(GeoUnavailableReason::UntrustedIngress) => "source not admitted", + GeoStatus::Unavailable(GeoUnavailableReason::MissingMetadata) => "metadata missing", + GeoStatus::Unavailable(GeoUnavailableReason::MissingReaderIp) => "reader unavailable", + GeoStatus::Unavailable(GeoUnavailableReason::NoMatch) => "no geographic match", + GeoStatus::Error(GeoError::InvalidAssertion) => "assertion rejected", + GeoStatus::Error(GeoError::ProviderFailure) => "provider failed", + GeoStatus::Located { location, .. } if location.subdivision().is_none() => "country only", + GeoStatus::Located { .. } => "country and unverified subdivision assertion", + }, + } +} +``` + +Every selected outcome retains the selected provider identity, even when source admission +fails before invocation. Located assertion results carry `GeoSource::CloudFront` and +`GeoTrustBasis::DeclaredProtectedAlb`. Local MaxMind results carry `GeoSource::LocalDatabase`; +supplied reader results carry `GeoSource::CheckedReaderIp`. Both reader compositions carry +`GeoTrustBasis::CheckedReaderIp(source)` with `GeoIpSource::DirectPeer`, `TrustedForwarded` or +`TrustedXForwarded`. These describe admitted estimates, not identity or residence attestations. + +For local databases, absent reader input means `MissingReaderIp` and zero provider calls. +A valid reader with no record, neither selected code, or IPv6 against an IPv4-only dataset means +`NoMatch` and one call. A valid country without subdivision is country-only. City reads only +`country.iso_code` and `subdivisions[0].iso_code`; Country reads only the country code. +Registered/represented country, names and later subdivisions never fill gaps. Leading zeroes +are preserved. Empty/malformed present codes, unsupported countries such as `XK`, or subdivision +without country produce `InvalidAssertion`, not a downgraded country-only result. Unexpected +reader failures produce `ProviderFailure`. Unknown but syntax-valid subdivisions stay assertions. + +```mermaid +flowchart TD + Head[Account original request head] --> Strip[Remove every configured credential field] + Strip --> Admission[Validate and admit ingress] + Admission -->|Refused or aborted| Stop[No provider invocation] + Admission -->|Admitted| Selection{Selected input?} + Selection -->|Off| Off[NoSelection] + Selection -->|Assertions| Peer{Peer matches protected policy?} + Peer -->|No or missing| Untrusted[UntrustedIngress, no invocation] + Peer -->|Yes| Resolve[Resolve selected provider exactly once] + Selection -->|Reader IP| Reader{Checked reader meets expectation?} + Reader -->|No| Missing[MissingReaderIp, no invocation] + Reader -->|Yes| Resolve + Resolve --> Context[Store private outcome before middleware and handlers] +``` + +Ordinary trusted-proxy configuration, effective reader IP, forwarding headers, public +extensions, and caller-supplied `ConnectInfo` cannot create protected assertion admission. +Reader selections use only runner-owned checked ingress facts, not mutable public metadata. Admission +callbacks run before Geo exists. Header changes or extension insertion/removal by application +code cannot replace the managed outcome. Response finalization never resolves it again. + +The unmanaged request converter, legacy dispatch, Fastly, Cloudflare, and Spin expose +`NoSelection`. Returning a context to an ordinary request and constructing a new unmanaged +context does not preserve its private Geo result. + +## CloudFront assertion rules + +CloudFront must generate both `CloudFront-Viewer-Country` and +`CloudFront-Viewer-Country-Region` through an explicit origin request policy. Generic viewer +header forwarding is not enough. Never install static custom geographic values or an edge +function that rewrites the generated geography. + +| Received assertions from an admitted source | Outcome | +| --------------------------------------------------------------------------- | ---------------------------------------------------------------------------- | +| Neither present | `MissingMetadata` | +| Country only | `Located`, subdivision unknown | +| `us` and `ca` | `Located` with `US` and `CA` | +| `US` and `01` | `Located` with syntax-checked numeric subdivision | +| `US` and `ZZ` | `Located` with a syntax-valid assertion, not verified subdivision membership | +| Subdivision without country | `InvalidAssertion` | +| Unsupported/reserved country, including `XX`, `ZZ`, `UK`, or `XK` | `InvalidAssertion`, no aliases | +| Duplicate field instances, even identical, or comma-joined values | `InvalidAssertion` | +| Empty, whitespace, non-ASCII, malformed, or oversized value in either field | `InvalidAssertion` for the whole result | + +Country is exactly two ASCII letters, normalized to uppercase and checked against the pinned, +checked-in list of 249 officially assigned ISO 3166-1 alpha-2 codes. The source revision and +checksum live beside `edgezero-core/src/geo/country_codes.rs`; updates are reviewed code changes, +not runtime downloads. Subdivision is one to three ASCII letters/digits, normalized to uppercase, +without a country prefix or separator. There is no subdivision catalogue. No trimming, decoding, +truncation into validity, or guessed geography occurs. These rules apply to values exposed by +the HTTP parser; upstream parser-stripped whitespace cannot be detected retroactively. + +## Protect the CloudFront origin + +Use AWS's [origin-secret ALB pattern](https://docs.aws.amazon.com/AmazonCloudFront/latest/DeveloperGuide/restrict-access-to-load-balancer.html). +All of these controls are required together: + +1. Give the intended CloudFront distribution a dedicated high-entropy custom origin credential. + Do not share it with unrelated distributions. Keep both header name and value out of request + logs, responses, diagnostic exports, and test artifacts. +2. Require viewer HTTPS. Set the CloudFront origin protocol to HTTPS-only. For AWS's HTTPS ALB + recipe, include `Host` in the origin request policy and configure the ALB certificate to match + the routed hostname. Restrict served public hostnames; a viewer's arbitrary host is not an + authorization decision. +3. Forward from the ALB listener to the application target group only when that credential + matches an explicit non-wildcard value. The default action denies access. No alternate + listener or rule may forward unchecked traffic to this target group. +4. Restrict the ALB security group to CloudFront origin-facing managed prefix lists for every + enabled IP family. These ranges admit CloudFront infrastructure, not one distribution; + they do not replace the credential gate. +5. Keep the Axum port private and reachable only from the intended ALB security group. Keep + the ALB-to-Axum HTTP hop inside the isolated VPC. No public target, alternate proxy, listener, + or network route may bypass the gate. This feature adds no native TLS server. + +AWS documents that [custom origin headers replace viewer-supplied values](https://docs.aws.amazon.com/AmazonCloudFront/latest/DeveloperGuide/add-origin-custom-headers.html). +The ALB validates the credential. EdgeZero stores only its removal name, strips every occurrence +before application callbacks, and never receives the expected value as runtime configuration. +A correctly matched socket peer with an incorrectly configured ALB is still unsafe. EdgeZero +cannot inspect listener rules or security groups. + +### Generated headers and missing subdivision + +AWS documents [generated-header replacement after the viewer-request event](https://docs.aws.amazon.com/AmazonCloudFront/latest/DeveloperGuide/lambda-at-edge-function-restrictions.html#lambda-at-edge-restrictions-cloudfront-headers). +No Lambda@Edge function is required. The [header reference](https://docs.aws.amazon.com/AmazonCloudFront/latest/DeveloperGuide/adding-cloudfront-headers.html) +says subdivision is omitted for requests originating from the AWS network. Test spoofed viewer +subdivision in that case. Do not assume that omission of generated data safely removes a +viewer-supplied value. + +### Disable shared caching + +Apply CloudFront's managed [CachingDisabled policy](https://docs.aws.amazon.com/AmazonCloudFront/latest/DeveloperGuide/using-managed-cache-policies.html) +to these application routes, with minimum/default/maximum TTL zero. Set +[Error Caching Minimum TTL](https://docs.aws.amazon.com/AmazonCloudFront/latest/DeveloperGuide/custom-error-pages-expiration.html) +to zero for cacheable application-error statuses too; the default is ten seconds. Prevent any +other shared cache from reusing these responses. Forwarding geographic headers does not partition +cached responses by geography or permission. A custom Geo-aware cache-key recipe and automatic +response/cache-header mutation are outside this feature. + +### Rotate the credential value + +Keep the header name fixed during live rotation: + +1. Generate a new high-entropy value through your secret-management process, without printing it. +2. Temporarily configure the ALB gate to accept old and new exact values. +3. Update the intended distribution's custom origin value and wait for deployment completion. +4. Verify intended traffic, denial of unrelated traffic, and application-side header removal. +5. Remove the old accepted value and verify it no longer passes the gate. + +Renaming the credential header requires coordinated stop/reconfigure/restart so no in-flight +configuration leaves an unremoved credential field. V1 has no dual-name removal state. + +## Acceptance boundaries + +Local EdgeZero tests prove bounded parsing, selection failures, credential removal, real-socket +peer matching, one invocation, immutable outcomes, lifecycle preservation, and portable dependency +isolation. They do not prove AWS configuration or application permission policy. + +Before production use, record separate evidence against the actual distribution, ALB, and target: + +- Intended distribution traffic reaches the app. Direct ALB traffic and a different distribution + without the credential cannot. A network denial is valid when prefix lists block connection. +- Direct target access, alternate listeners/proxies, and unintended security-group sources cannot + bypass the credential gate. +- Forged viewer country/subdivision does not become accepted geography. Include the AWS-network + missing-subdivision case. +- Credential replacement, validation, removal, and value rotation work without exposing name or + value to application code, responses, or logs. +- Missing subdivision stays country-only and absent metadata stays missing, without fallback. +- Successful and cacheable error responses are not reused across requests by any shared cache. + +A failed deployment check blocks production use of the declared policy. It does not justify +broader header trust or a fallback. Downstream policy-path tests remain another acceptance gate. +Trusted Server adoption, jurisdiction lists, consent interpretation, and Edge Cookie/identity +behavior are separate work; this feature does not relax their fail-closed requirements. + +[AWS VPC origins](https://docs.aws.amazon.com/AmazonCloudFront/latest/DeveloperGuide/private-content-vpc-origins.html), +other adapters/sources, remote providers, hot reload and subdivision catalogues are deferred. VPC origins are not a second supported trust mode here. +See [Axum diagnostics and trusted proxies](./axum-diagnostics.md) for forwarding policy, +which supplies checked readers but remains independent from protected assertion admission. + +## Pre-release API migration + +This is a coordinated breaking migration in the unpublished workspace, not a compatibility shim. +External consumers must update before using the next release: + +- Replace `GeoInput { country, subdivision }` literals with + `GeoInput::Assertions { country, subdivision }`. Replace direct field access with matching; + explicitly handle or reject `ReaderIp` in custom providers. +- Handle `GeoResolution::NoMatch`, `GeoUnavailableReason::MissingReaderIp` and `NoMatch` in + exhaustive matches. Preserve uncertainty rather than granting permission on unavailable data. +- Handle `GeoSource::LocalDatabase`, `GeoSource::CheckedReaderIp` and + `GeoTrustBasis::CheckedReaderIp(GeoIpSource)` in provenance consumers. +- Match capability's `{ provider, input }` instead of the removed `{ provider, source, +trust_basis }` fields. Startup input contract is not request provenance. +- Rename any custom provider previously called `maxmind`; it is now reserved like `cloudfront`. + +MaxMind warns that GeoLite City is considerably less accurate than paid GeoIP City and does not +recommend it for commercial use cases. That accuracy warning is not a blanket commercial-use +prohibition. Evaluate state-level suitability on representative traffic. VPNs, carrier networks, +proxies and stale data can mislocate readers. No local fixture proves jurisdiction recognition, +consent compliance or production accuracy. Keep unknown geography fail-closed, test downstream +policy separately and prevent shared caches from reusing permission-sensitive responses across +visitors. No automatic response or cache header is added. diff --git a/docs/superpowers/plans/2026-10-07-native-store-providers.md b/docs/superpowers/plans/2026-10-07-native-store-providers.md new file mode 100644 index 000000000..f323eb3de --- /dev/null +++ b/docs/superpowers/plans/2026-10-07-native-store-providers.md @@ -0,0 +1,618 @@ +# Native store providers implementation plan + +- Status: Implementation complete through offline Stages 1-8. Native provider combinations, generated-consumer startup, workspace/native/WASM gates and documentation checks passed. Draft [PR #414](https://github.com/stackpop/edgezero/pull/414) was published via `gh stack` after rebasing and rerunning gates. Provider availability validation deliberately remains structural-only/unknown. Real AWS acceptance is declined and excluded. +- Date: 2026-10-07. +- Owning issue: [EdgeZero #410](https://github.com/stackpop/edgezero/issues/410). +- Governing design: [Native store providers independent of HTTP adapters](../specs/2026-10-07-native-store-providers-design.md), including the R1 Cargo feature-resolution correction. +- Inspected main: `683202c66948146ec360f490126f1d60f920288b`. +- Prerequisite evidence: PR #275 at `d1a81579afb96f200e7fe8fd294600f0b446b080`; PR #406 at `3ab22096fd117f7506cf437b5ba128e0789e059c`. Both remain open at planning time. Their inspected APIs are not APIs available on main. +- Evidence level: Executed combined-prerequisite tests, native service feature combinations, real SDK literal-loopback fixtures and actual CLI-generated consumer startup. No real AWS, IAM, image or deployment acceptance. + +## 1. Goal and scope + +Allow native applications to bind KV, configuration, and secrets independently of their HTTP runtime and independently for each logical store ID. Ship AppConfig Agent whole-document configuration and Secrets Manager whole-value secrets as optional native providers. Keep ordinary local providers, portable declarations, typed envelopes, nested secret references, and per-ID registries. + +Success means a generated native application can start with mixed local, custom, and AWS bindings, validate required typed application inputs before readiness, serve requests from the same completed registries, and adopt remote changes only on restart. A failed mapped remote fetch must stop startup without falling back to files or environment values. + +Do not implement DynamoDB, live reload, secret refresh, publishing, provisioning, a provider plugin registry, a new HTTP runtime, or application-specific schemas. Do not extract existing local providers into new crates. Do not change portable trait future bounds, store declarations, macro metadata, or core errors to expose AWS details. + +The user's subsequent instruction to implement authorizes execution of this plan and rebasing onto the open prerequisites. Stage proof gates remain; do not skip verification between stages. Real AWS acceptance is excluded. Subsequent approval authorizes #410 commits and a draft via `gh stack`, but not merging, deployment, or changes to existing prerequisite PRs. Rewriting the prerequisite commit as part of the expressly authorized rebase is recorded below. + +## 2. Decisions and approval status + +The configurable 8 MiB default, shared-session quota scope, and caller-owned preparation boundary were approved in conversation on 2026-10-07. After the initial Stage 0 audit, the user explicitly instructed: "Those pull requests are not blockers. Go ahead and implement. If you need changes from those pull requests, rebase on top of them." This authorizes implementation of the proposed design and stages, including use of open prerequisite heads. Remaining policy proposals are the implementation contract unless the user changes them. This original instruction did not authorize AWS service access or publishing. Later approval authorizes draft publication; real AWS acceptance remains excluded. + +The implementation follows these choices: + +1. Use a move-only `NativeStoreOverrides` argument separate from cloneable `AxumRunOptions`. Each supplied ID has either an existing handle or a deferred, owned, non-`Send` future. Validate the complete source plan before polling futures. Require deferred future construction to be side-effect-free, with provider work inside its async body. The runner guarantees no polling before validation; it cannot undo caller-side work during future construction. +2. Keep a cloneable `NativeStoreBindingsInput` selection inside `AxumRunOptions`: local defaults, an absolute bindings-file path, or an inline typed `NativeStoreBindings` value. File and inline settings are mutually exclusive; selection methods reject a second explicit source rather than overwriting it. Both normalize into the same validator and preparation path. Noncolliding supplied handles/futures may coexist as the separate overrides argument; same-ID collisions are errors. +3. Use one explicit `AwsPreparation` session for all built-in AWS bindings prepared by a runner invocation. It owns budgets, successful target deduplication, and client reuse. Do not provide convenience constructors that silently create an independent budget per binding. +4. Apply declared native build features to both standard native build and serve commands. Normalize manifest and explicit feature inputs once; use the same selection when offline validation can resolve the actual target. +5. Add a bindings-only mode to `config validate` when `--adapter axum --store-bindings ` is supplied. It does not require a local app-config TOML file or claim to validate remote contents. Preserve the existing validation mode without these arguments. +6. Use a Cargo-resolution parity spike before shipping provider-availability verdicts. A failed or ambiguous offline resolution produces an explicit structural-only result, never an inferred enabled/disabled result. +7. Keep provider failures as closed, sanitized native categories. The coordinator attaches only the baked store kind and logical ID. Do not retain or format SDK, HTTP, TOML, credential, or arbitrary custom-provider error chains. + +Approved accounting and ordering policy: + +- The standard runner uses one shared session to enforce combined quotas across all its built-in AWS bindings. Prebuilt opaque handles, local providers, and separately created sessions remain caller-owned and are not included. Independent sessions may together exceed 8 MiB; this is not a process-wide limit. +- Validate sources before opening runner-created providers or polling deferred supplied futures. Require side-effect-free deferred construction. A caller can already have opened a `Ready` handle or done other work before supplying it; the runner cannot undo that work. + +Do not introduce process-global accounting or mandatory introspection of portable handles. The former accepted-only prerequisite gate is superseded by the user's authorization to use open PRs. Re-audit and test the exact combined stack; never claim those contracts are accepted on main. Runtime proof, feature isolation, and separate AWS acceptance gates remain. + +## 3. Delivery order + +```mermaid +flowchart TD + Audit[Stage 0: approve policies and re-audit prerequisites] --> Compose[Stage 1: generic native composition] + Compose --> Settings[Stage 2: strict bootstrap and AWS settings] + Settings --> Session[Stage 3: shared preparation and snapshot contracts] + Session --> Agent[Stage 4: AppConfig Agent] + Session --> Secrets[Stage 5: Secrets Manager] + Agent --> Tooling[Stage 6: generated features and offline CLI] + Secrets --> Tooling + Tooling --> Runtime[Stage 7: generated production startup proof] + Runtime --> Gates[Stage 8: docs and native/WASM CI] + Gates --> Acceptance[Stage 9: separately authorized AWS acceptance] +``` + +Stages 4 and 5 may be investigated in parallel. Keep edits to shared settings, session state, Cargo manifests, and lockfiles under one owner. Serialize Cargo commands sharing a target directory. Stage 6's Cargo-resolution spike can be investigated earlier, but do not publish a capability verdict until it passes. + +## 4. File map and ownership + +Existing files to change narrowly: + +| Area | Files | Intended change | +| --------------------------- | --------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------- | +| Workspace | `Cargo.toml`, `Cargo.lock` | Add optional native provider member/dependencies and coherent SDK pins | +| Native runtime | `crates/edgezero-adapter-axum/src/dev_server.rs` | Source planning, sequential setup, completed registry publication, additive runner API | +| Accepted production options | `crates/edgezero-adapter-axum/src/run_options.rs` | Bootstrap selection and local-root validation after source selection; exists at inspected #406, not main | +| Axum features | `crates/edgezero-adapter-axum/{Cargo.toml,src/lib.rs}` | Native-bindings/settings support and independent AWS feature forwarding | +| Axum CLI/scaffolding | `crates/edgezero-adapter-axum/src/cli.rs`, `src/templates/Cargo.toml.hbs` | Standard native Cargo invocation and generated service features | +| CLI inputs | `crates/edgezero-cli/src/args.rs` | Bindings path, selected adapter, target feature/default/all controls, push guard input | +| CLI validation/push | `crates/edgezero-cli/src/config.rs` | Bindings-only validation and explicit remote publication refusal | +| CLI build routing | `crates/edgezero-cli/src/{lib,adapter}.rs` | Normalize standard Axum feature selection before both generated-command and fallback dispatch | +| Generator | `crates/edgezero-cli/src/generator.rs` | Structural assertions and any needed dependency/feature template context | +| Generated integration | `crates/edgezero-cli/tests/generated_project_builds.rs` | Native provider combinations and preserved generated WASM builds | +| CI | `.github/workflows/{test,format}.yml` | Enabled-native tests, isolated feature lint, generated proof, dependency-graph assertions | +| Guides | `docs/guide/adapters/{overview,axum}.md`, `docs/guide/{configuration,cli-reference}.md` | Setup, support matrix, CLI modes, bounds, restart and stale-value policies | + +New files, names proposed: + +```text +crates/edgezero-adapter-axum/src/ + native_bindings.rs # SDK-free parsing, source validation, local-root needs + native_build.rs # SDK-free standard native Cargo selection, if extraction is needed + +crates/edgezero-store-aws/ + Cargo.toml + src/ + lib.rs + settings.rs # Strict types, selectors, endpoint/ARN validation + preparation.rs # Shared session, budgets, dedup, sanitized errors + appconfig_agent.rs # Restricted HTTP reader and immutable config provider + secrets_manager.rs # SDK boundary and immutable secret provider + +crates/edgezero-cli/tests/ + native_target_features.rs # Cargo-resolution versus compiled-fixture parity + +scripts/ + check_native_store_dependency_graph.sh + smoke_test_native_stores.sh + accept_native_aws_stores.sh # Only if an opt-in harness is useful; never default CI +``` + +Keep unit tests colocated. Add standalone fixture files only when the real generated/runtime path needs them. Prefer generating a temporary application through the existing scaffold over maintaining a second hand-written application. Keep fixture lockfiles current where fixtures are committed separately. + +Core and macro production changes are not expected after prerequisites are accepted. If new work appears necessary there, stop and explain the missing contract instead of expanding this plan silently. + +## 5. Stage 0: prerequisites, policy approval, and baseline + +### Work + +- [x] Read current `CLAUDE.md`, governing spec, #410/#324, and exact selected #275/#406 revisions. +- [x] Record selected stack commit IDs and integration differences. Open-PR use is explicitly authorized; do not label them accepted on main or implement an alternate bounded-read API. +- [x] Confirm `ConfigValue`, `BoundedStoreRead`, required bounded config/secret methods, application monotonic clock, fallible `App::build`, `PreparedStores`, initializer state insertion, and readiness/shutdown behavior on the combined stack. +- [x] Record the approved configurable 8 MiB session quota and caller-owned preparation boundary in both spec and plan. +- [x] Record the user's authorization to implement the plan and use open prerequisites, superseding the original accepted-only gate. Separate AWS acceptance remains unapproved. +- [x] Inventory and reuse the selected prerequisites' tests. Add only missing native composition/provider behavior. +- [x] Record a baseline for relevant native tests, existing workspace gates, and installed WASM targets. Classify pre-existing failures separately. All eight Stage 0 main checks passed; see the execution record below. +- [ ] Select exact compatible `aws-config` and `aws-sdk-secretsmanager` versions for the repository Rust toolchain. Add direct Smithy dependencies only when a shipped API or test connector needs them. Record the SDK behavior version, TLS/runtime feature choices, and Agent image/version to use later. + +### Exit gate + +The implementation plan is authorized. Use the exact user-approved prerequisite stack, source/test inventory, and passing combined-contract tests. Both PRs remain open; they are not a blocker under the user's instruction. Do not modify main or claim these contracts have landed. SDK graph/image selection still needs evidence before the corresponding provider stages. + +## 6. Stage 1: generic per-ID native composition + +### API shape + +Use the existing `KvHandle`, `ConfigStoreBinding`, and `BoundSecretStore` output types. A proposed native-only builder owns maps keyed by logical ID: + +```rust,ignore +// Exact exported names are settled after the accepted prerequisite re-audit. +// No Send bound on this future. Existing provider objects remain Send + Sync. +pub type NativeStoreFuture = Pin> + 'static +>>; + +enum NativeStoreSource { + Ready(H), + Prepare(NativeStoreFuture), +} + +pub struct NativeStoreOverrides { + // private keyed maps for KV, config, and secrets +} +``` + +Expose small typed insertion methods for each kind and for ready/deferred inputs. Insertions reject duplicates instead of overwriting a map entry. Keep map/source internals private. Require all deferred provider opening to happen inside the future body when polled, not during construction; this is a caller contract, not something the type system can prove. Tests establish zero polls and zero runner-owned opens on validation failure, not absence of earlier caller-side side effects. Do not require another public provider-factory trait. + +Add one runner entrypoint accepting options, overrides, stop, and the accepted app initializer. The options carry `NativeStoreBindingsInput`; Stage 1 starts with its local-default case, and Stage 2 adds file/inline normalization. The validator compares the selected settings IDs with the separate override maps before any preparation. Existing public entrypoints delegate with empty overrides. Keep `AxumRunOptions` cloneable; do not put futures into it. Owned deferred futures can capture `Rc` and bootstrap data. They cannot borrow options or the app initializer's higher-ranked inputs. + +The validated source plan has separate local, supplied-ready, supplied-deferred, and built-in-settings variants. `NativeStoreFuture` is only for opaque custom providers. File or inline settings become built-in variants, not stored futures: the runner matches each typed settings variant and directly awaits the matching method on its one mutable `AwsPreparation` session. This avoids trying to borrow that session into a stored `'static` future. For secrets, pass the baked logical namespace into snapshot construction and wrap the returned `SecretHandle` in `BoundSecretStore` with the same namespace. Config construction returns a `ConfigStoreHandle` paired with the validated settings default key. File and inline inputs cannot coexist; either selected input can coexist with noncolliding custom overrides. Custom-provider quota enforcement belongs to that provider, not to introspection of its eventual handle. + +### Work + +- [x] Compile-probe the proposed API with an `Rc`-capturing setup future that returns a valid handle on the existing current-thread `LocalSet`. +- [x] Preserve the initializer's accepted higher-ranked borrowing signature. Prove it still permits borrowing stores across await and capturing non-`Send` application state during setup. +- [x] Separate options/scalar validation from local-root requirements. Determine required roots from the resolved per-ID sources, not just declared store kinds. +- [x] Build one source plan for all kinds. Validate defaults, declared IDs, duplicate supplied inputs, explicit scalar-overlay conflicts, and local-root needs before preparing any runner-owned provider. Stage 2 adds file/inline settings and cross-source collision coverage. +- [x] Preserve config default keys, secret bound namespaces, and local KV NAME-derived paths. Explicit sources retain supplied semantics; implicit local sources keep existing env behavior. +- [x] Assemble the app in the selected prerequisite order, then sequentially prepare sources under the existing runtime. No additional runtime or initializer-side provider selection. +- [x] Stage every handle, finalize registries once after all sources succeed, preserve declared defaults, and share the completed `PreparedStores` between initializer and request service. +- [x] Drop staged handles and preparation futures on failure or startup stop. No initializer, readiness, or listener after incomplete preparation. + +### Proof + +Add colocated tests for: + +- ready and deferred custom providers, mixed same-kind bindings, and the default ID; +- config default-key preservation and wrong-secret-namespace isolation; +- undeclared IDs, duplicate supplied IDs, and explicit NAME/KEY conflicts; supplied/file and supplied/inline collision tests follow in Stage 2; +- zero deferred polls and zero runner-owned provider opens on every source-validation failure; +- remote/custom-only config without CONFIG_DIR, custom-only KV without DATA_DIR, and roots required only for remaining local IDs; +- sequential preparation, failure of a later binding, partial-state drop, and no initializer/readiness on failure; +- initializer and request service observing the same prepared handles; +- existing local development and strict production defaults with empty overrides; +- stop during a pending preparation without claiming all external work is forcibly canceled. + +Run scoped Axum tests after each code increment. Exit when fake providers prove generic composition without importing AWS. + +## 7. Stage 2: strict bootstrap parsing and settings-only AWS crate + +### Ownership and feature shape + +Native `native_bindings.rs` owns the top-level versioned schema, kinds/IDs, tagged dispatch, source collisions, overlays, local-root needs, and file limits. `edgezero-store-aws::settings` owns AWS settings and resource-shape validation. Never put provider fields into portable declarations or generated metadata. + +Add a settings-only dependency on the provider crate to the native bindings feature. This does not enable either service. The runtime and CLI can both use native-bindings without enabling each other's runtime/registry features. + +Provider features: + +```text +edgezero-store-aws + default = [] + appconfig-agent -> HTTP client, streaming support, native timers + secrets-manager -> AWS config/service SDK and native runtime support + +edgezero-adapter-axum + native-bindings -> native parser + SDK-free AWS settings + axum -> existing runtime + native-bindings + cli -> existing CLI + native-bindings + aws-appconfig-agent -> provider appconfig-agent + aws-secrets-manager -> provider secrets-manager + aws-stores -> both explicit service features +``` + +Use native target dependency declarations and module gates. Do not add another Tokio dependency to an adapter or any AWS/native dependency to portable core or WASM adapters. Service-disabled settings parsing must work without the service clients. All-feature builds must not conceal missing single-feature cfgs. + +### Work + +- [x] Add the provider workspace member, `lib.rs`, strict settings structs, version selector enum, limits, and closed redacted preparation error categories. +- [x] Add native-only version 1 tagged parsing with unknown/irrelevant fields, unsupported KV/provider tags, duplicate tables, empty mappings and undeclared IDs rejected. +- [x] Require mapped config defaults and case-sensitive IDs/lookup keys, without parsing/projecting remote values. +- [x] Validate complete Secrets Manager ARNs, supported partition/region-family relationships, explicit matching region, exclusive selectors and API length bounds. Full cross-account ARNs remain explicit. +- [x] Validate literal-loopback HTTP endpoints and original authority/path spelling; construct escaped segments and independently test decoded round-trip equality. +- [x] Add mutually exclusive absolute file and typed inline selection. Explicit APIs use only supplied bootstrap values; `from_env` alone resolves ambient locator/token variables. +- [x] Read selected regular files once with cap+1 probing, redacted fixed errors and Unix nonblocking FIFO rejection. +- [x] Capture locator/scalar inputs once, reject relevant normalized duplicates, capture named tokens once per name, and redact Debug/errors. Preserve unrelated EnvConfig duplicate semantics; ambiguous explicitly supplied token values fail closed. +- [x] Reject disabled providers before deferred polling/local provider construction. Services are not implemented/enabled at this settings-only checkpoint; actual compiled capabilities are added with Stages 3-5. + +### Limits schema + +Use a trusted bootstrap `NativeBindingsLimits` policy for the file byte cap and total explicit-ID cap. Defaults are 256 KiB and 128 IDs. A bindings file cannot raise its own pre-parse byte limit. Programmatic overrides of bootstrap limits are explicit; offline validation reports the policy it used. + +Propose an optional `[limits.aws]` table for remote preparation settings: + +| Field | Default | +| --------------------------- | ------------------------------------------------------------- | +| `max_remote_values` | 1,024 mapped config documents plus secret entries | +| `max_config_document_bytes` | 1 MiB | +| `max_secret_value_bytes` | 64 KiB | +| `max_snapshot_bytes` | 8 MiB combined unique retained AWS payload per shared session | +| `preparation_timeout_ms` | 30,000 | +| `operation_timeout_ms` | 5,000 | +| `connect_timeout_ms` | 1,000 | +| `attempt_timeout_ms` | 2,000 | + +The approved 8 MiB default is provisional and not derived from AWS limits or workload measurements. Keep an aggregate cap because many values can each satisfy per-value limits while collectively retaining excessive payload. Document its rationale and scope; revisit the default with representative application payload sizes plus headroom. Explicitly raising or lowering it must remain possible. + +These are configurable defaults, not immutable hard ceilings. Explicit file/typed policies may change service limits subject to finite/nonzero checks, checked conversions/arithmetic, and consistency of per-operation/aggregate and per-value/combined limits. The file-byte/explicit-ID limits change only through trusted native bootstrap options. Keep retry count and concurrency fixed initially. Count mappings before deduplication; deduplication must not let unlimited aliases bypass entry limits. Reject simultaneous explicit bootstrap-supplied and file-supplied AWS limit policies. Otherwise use the one supplied policy, filling omitted fields from documented defaults, or use all defaults when both policies are omitted. Do not fieldwise merge two supplied policies. The selected policy applies to built-in settings prepared through the shared session, not opaque custom handles/futures or separately prepared AWS handles. + +### Proof + +Settings-only tests cover every rejection rule, blank/non-Unicode bootstrap inputs, token absence/emptiness, env versus explicit options, file overflow, unsupported features, selectors, limits, and source conflicts. Verify parser errors do not contain sentinel ARNs, tokens, credentials, or raw TOML values. + +Run settings-only provider tests and Axum parser tests in runtime-only and CLI-only configurations. Inspect reachable normal dependencies to confirm the SDK and HTTP client are absent from settings-only builds. + +## 8. Stage 3: shared AWS preparation and bounded snapshot reads + +### Work + +- [ ] Implement an explicit, per-startup `AwsPreparation` session with sequential `&mut self` preparation methods. No process-global client/cache/session. +- [ ] Start one aggregate deadline before credential/client initialization. Derive operation, attempt, and connect budgets from remaining aggregate time and configured caps. Credential/setup elapsed time consumes this budget; check expiry before and after awaits and discard results completed after expiry. Classify credential/blocking-work cancellation as best-effort or unsupported under the accepted capability contract. Do not promise a 30-second wall-clock completion bound or that dropping a future stops a credential process or socket. Audit whether the pinned SDK operation timeout includes each supported credential provider; do not assume it does. +- [ ] Track mapping counts, unique retained payload bytes, metadata, and transient staging separately. Use checked accounting and reject overflow before retaining the next payload. +- [ ] Cache only successful targets. Reuse immutable payload allocations between identical targets while retaining separate logical-key and namespace mappings. +- [ ] Include normalized endpoint, application/environment/profile, and authorization policy identity in AppConfig dedup keys. Different token policies must not merge. Keep token identity internal and redacted. +- [ ] Include region/client-policy identity, full ARN, and exact normalized selector in secret dedup keys. Normalize omitted/current selection consistently; never collapse pinned versions or distinct stages. +- [ ] Reuse one Secrets Manager client per region/policy in the shared session. Config-only sessions must not initialize credentials or an SDK client. +- [ ] Implement immutable config and secret snapshot providers under accepted #275 methods. They do not keep cloud clients or issue request-time cloud calls. +- [ ] Ordinary config reads return shared `ConfigValue`; secret reads return exact `Bytes`. Unknown keys or wrong secret namespaces return successful misses without probing a service. +- [ ] Bounded reads reject an expired deadline before lookup, honor independent backend/value caps, report hit payload bytes and zero bytes on misses, and recheck deadline precedence before returning. Reuse accepted finishing helpers where suitable. +- [ ] Preserve accepted conservative deadline/allocation classification. Document that SDK parsing, credential work, allocator overhead, and the Agent are outside retained-payload accounting. + +### Proof + +Use a deterministic injected monotonic clock and production accounting paths. Test exact limits and limit+1, explicit higher/lower quota settings, aggregate bytes across both services and IDs, mapping-count limits before dedup, checked overflow, auth/selector-sensitive dedup, preparation deadline exhaustion, sanitized Debug/display, and independent sessions without global state. Prove one runner shares the same quota across all built-in AWS bindings and that a quota breach drops staged state before readiness without eviction or local fallback. Independently prepared opaque handles are not included in that accounting. Add a delayed credential-provider fixture that completes after the aggregate deadline and prove no value is retained or published. Separately test a pending credential future under supported timers. Neither fixture proves forcible cancellation of a real credential process or blocking provider; keep that fact unproven with best-effort wording. + +Run existing core config/secret contract macros against preseeded snapshot providers. Add bounded-read tests for both ordinary hits and misses, empty values, namespace isolation, backend/value allowances, and deadline precedence. These tests are separate from startup transfer tests. + +Atomic publication is the native coordinator's responsibility. A programmatic caller that retains a handle returned by one successful preparation cannot expect the session to erase that handle after a later failure. The standard runner drops all staged handles on failure; prove that path in Stages 1 and 7. + +## 9. Stage 4: AppConfig Agent whole-document preparation + +### Work + +- [ ] Build a dedicated native HTTP client. Disable ambient proxies, redirects, and transparent decompression. Do not reuse the generic outbound proxy client. +- [ ] Construct GET `/applications/{application}/environments/{environment}/configurations/{profile}` using validated, escaped segments. +- [ ] Send `Accept-Encoding: identity`. Send `Authorization: Bearer ` only when explicitly configured. Do not enable request/header/body diagnostics that reveal bootstrap data. +- [ ] Perform one bounded request per distinct complete target, including streamed body consumption. Reject non-success/protocol responses and non-identity content encoding. +- [ ] Reject oversize declared content lengths early when available; always enforce actual streamed bytes before accumulation exceeds the allowed limit. Detect truncated/erroring streams. +- [ ] Validate UTF-8 and retain the exact text, including whitespace, JSON quoting, empty documents, and serialized envelopes. Do not parse, unquote, reserialize, or resolve secret references. +- [ ] Make every mapped profile mandatory during preparation. A missing/unavailable mapped profile is a startup error, never a snapshot miss or local fallback. +- [ ] Drop response bodies and source errors before producing categorized diagnostics. Publish no partial binding. + +### Proof + +Use authorized loopback-only fixtures with bounded lifetimes and isolated ports. Cover IPv4/IPv6 validation, exact escaped paths, auth/no-auth, absent/empty tokens, proxy/redirect refusal, encoding rejection, status errors, malformed UTF-8, empty/raw/envelope fidelity, content-length and streamed cap edges, truncation, connect/body deadline behavior, dedup across IDs, auth-policy separation, and immutable reads after the fixture changes. + +Test generated envelope extraction and nested secret resolution later through the app initializer. A fake Agent proves reader behavior, not AWS reachability, deployment freshness, or warm-cache policy. + +Run AppConfig-only provider tests. Exit with no Secrets Manager SDK in the config-only normal dependency graph. + +## 10. Stage 5: Secrets Manager preparation and real SDK behavior + +### Work + +- [ ] Load the pinned SDK using explicit region, fixed behavior defaults, supported credential discovery, capped standard retries, and explicit operation/attempt/connect budgets. +- [ ] Count credential/client initialization against the shared aggregate budget. Use workload roles in documented production setup and a documented authenticated developer path. Do not require static keys in bindings or images. +- [ ] Make exact `GetSecretValue` requests for mapped ARNs/selectors only. Load every mapped entry before completing that binding. +- [ ] Accept exactly one value field. Return `SecretString` UTF-8 bytes or the Rust SDK's already-decoded `SecretBinary` bytes. Preserve empty/binary values where allowed. Do not extract JSON fields or base64-decode a Blob again. +- [ ] Check payload size after SDK materialization and before retaining it. Describe this as a retained-value cap, not a bound on every SDK allocation. +- [ ] Classify service/credential/transport failures before dropping source details. Keep missing resource/version, denied, decryption, throttled, deadline, malformed, and size categories where exposed. +- [ ] Let the SDK own transient retries, at most three total attempts and bounded by remaining budgets. Add no outer retry loop. Verify denied/decryption/validation errors are not repeatedly retried under the pinned policy. +- [ ] Spike the real generated-child test path against the pinned SDK now: child-only dummy credentials, metadata disabled, and supported service endpoint override to a loopback scripted HTTP fixture. Verify client configuration honors that override without a production bindings field or public mock API before Stage 7 depends on it. +- [ ] Drop clients/session staging after preparation; snapshot handles contain only successful mapped payloads. Never persist snapshots to disk or claim zeroization. + +### Test layers + +1. A narrow private service-call seam exercises production snapshot building, mappings, selectors, payload fidelity, errors, and cleanup with deterministic responses. Do not expose a public mock/factory hierarchy. +2. The pinned SDK with a scripted connector exercises real serialization, error classification, retry attempts, and timeout settings. Supply fixture credentials explicitly to avoid ambient discovery or IMDS. A fake service-call method cannot prove SDK retry behavior. +3. Separately authorized AWS acceptance proves deployed credentials, IAM, and service version behavior. + +Cover binary data that resembles base64 text, neither/both fields, empty values, default/current versus pinned versions, selectors, wrong namespaces, unknown keys without calls, ARN/policy-sensitive dedup, missing version, expired credentials, denied/decryption/throttle/transport errors, capped attempts, exhausted budgets, and partial-startup cleanup. + +Capture EdgeZero-owned logs and returned errors with sentinel values. Assert they contain no secrets, ARNs, token/credential sources, response bodies, or raw SDK messages. Dependency DEBUG/TRACE output is outside that guarantee; the executed diagnostic spike documents its disclosure risk. For the approved first version, retain redacted EdgeZero-owned diagnostics and explicitly disclose dependency DEBUG/TRACE output to operators and draft reviewers. Do not install global or scoped subscribers to silence dependency events. The user approved this limitation after the diagnostic spike; hardening dependency diagnostics remains a review item. The separately operated Agent's own logging policy remains operator-owned. + +Run Secrets Manager-only and combined provider tests. Exit with actual SDK retry/classification evidence, not just fake-seam assertions. + +## 11. Stage 6: generated features, Cargo selection, offline validation, and push guard + +### Generated/native feature enablement + +- [ ] Add generated native `[features]` entries for `aws-appconfig-agent`, `aws-secrets-manager`, and the proposed `aws-stores` alias. Use `default = []` for these target service choices and forward each to the matching adapter feature. +- [ ] Keep default native and generated Fastly/Cloudflare/Spin targets AWS-free. Keep CLI service capabilities independent from the binary being validated. +- [ ] Introduce one standard native Cargo selection helper shared by build/serve and offline resolution. Requested names combine declared build features and explicit additions; provider availability comes from resolved capabilities, not those names. +- [ ] Handle both real dispatch paths. Generated Axum manifests currently use shell `cargo build -p ` / `cargo run -p ` commands through `edgezero-cli::adapter::execute`; the registered Axum `run_cargo` fallback is not the sole build path. +- [ ] Route the exact standard generated Cargo command forms and registered fallback through the shared normalization. Do not parse or rewrite arbitrary operator shell programs as Cargo commands. Preserve custom-command ownership and mark capability selection unknown when its build behavior cannot be represented. +- [ ] Normalize repeated/comma/space `--features`, `--features=...`, `--all-features`, and `--no-default-features` before the Cargo/run `--` separator. Runtime args after that separator must not affect compile features. Preserve unrelated passthrough arguments. +- [ ] Identify the exact native package/manifest and target. Treat manifest target `native` as host selection, not literal `cargo --target native`. Do not silently override operator package/target selections. +- [ ] Recognize equivalent supported target flags in bindings validation, proposed `--target-features`, `--target-all-features`, `--target-no-default-features`, and `--target`. Document defaults and reject target-control flags outside bindings validation. + +### Cargo-resolution correctness spike + +Candidate resolution is `cargo tree --offline --locked` for the exact native package, normal/build edges, the same target and feature controls, and narrowly parsed package/feature records. Use metadata for package identity/topology only when it can be obtained offline without changing the lockfile. Do not assume metadata's workspace-wide graph or tree's "pretty close" build view is exact. + +- [ ] Build local-only resolver-2 fixtures and compare predicted provider availability with the same-selection compiled fixture's cfg result. +- [ ] Cover default/no-default/all/explicit/alias selection and dependency feature propagation. +- [ ] Cover sibling workspace members, own/sibling dev dependencies, build dependencies, target-specific edges, renamed/duplicate package identities, and host versus explicitly selected native targets. +- [ ] Prove an unrelated member or dev dependency cannot falsely enable a normal-runtime provider. Check the Axum adapter service feature and reachable provider feature, not just a same-named feature on another crate. +- [ ] Classify unsupported package overrides, custom shell commands, `--all-targets`, `--workspace`, `--config`, ambiguous contexts, unexpected Cargo output, missing lock/cache, or failed offline resolution as unknown unless a parity test proves support. +- [ ] Emit structural-only when resolution is unknown. Never edit the lockfile, fetch dependencies, build application code, execute build scripts, or discover credentials during the user-facing offline check. Fixture compilation belongs to tests, not validation. + +Exit this spike with a documented supported invocation subset and tests. If the candidate cannot prove a case, keep that case structural-only rather than building a second approximate feature resolver. + +### Bindings-only validation + +- [ ] Add `--adapter` and `--store-bindings` inputs plus target controls to `ConfigValidateArgs` and its defaults/parser tests. +- [ ] Branch before `load_validation_context` for the new bindings-only mode. Load portable manifest declarations and explicit native bindings; do not require `.toml` or run typed remote-content checks. +- [ ] Require Axum selection and an absolute explicit bindings path. Reject irrelevant adapter/target combinations. Preserve existing raw/typed validation paths without the new mode. +- [ ] Use the SDK-free native source validator with only supplied deployment inputs. Never read ambient NAME/KEY overrides, the ambient bindings locator, or Agent token/credential values here. Startup validates actual deployment inputs again. +- [ ] Report either structural validation plus known target-provider compatibility, or structural validation only with provider availability unknown. Neither result proves resource existence, token availability, local-root availability, IAM, typed config validity, or build success. +- [ ] Permit programmatic validation to supply explicit overlay/override information through the same pure validator; CLI file validation does not invent future supplied handles. + +### Local publication guard + +- [ ] Add explicit `--store-bindings` to downstream typed `ConfigPushArgs` and its parser/default construction sites. +- [ ] Resolve the selected logical config ID, then parse and check the explicit bindings file before remote read-back/diff, prompts, adapter publication, or local writes. +- [ ] Reject an AWS-bound selected ID with a fixed unsupported-publication message, including dry-run and `--local`. Unrelated AWS IDs do not block a local selected ID. +- [ ] Do not consult the ambient deployment locator or claim a bindings-free local push updates an AWS-bound deployment. Keep the adapter trait unchanged and the local writer intact. +- [ ] Preserve ordinary local push, sibling-key retention, dry-run no-write, and all other adapters' behavior. Do not add AWS diff/gc/publication/mutation workflows. + +### Proof + +Extend generator structural tests and the real generated-project integration test. Test CLI/target feature mismatches in both directions, structural-only degradation without local app TOML, flags/aliases/default suppression, generated-command versus fallback feature forwarding, custom-command preservation, no lockfile mutation/network, and explicit push refusal with no file writes or provider calls. + +## 12. Stage 7: production startup and generated consumer proof + +### Work + +- [ ] Generate a native fixture through the actual CLI. Configure portable IDs, native binding settings, provider features, and an app-owned initializer under accepted lifecycle contracts. +- [ ] Use `#[action]` handlers and `edgezero_core` HTTP types. Load a real serialized envelope through accepted `AppConfig::from_binding` or its accepted equivalent, resolve nested secret references, validate typed configuration, and insert typed handler state before readiness. +- [ ] Exercise these combinations: AWS config plus environment secrets; local config plus AWS secrets; both AWS providers plus redb; same-kind local/custom/AWS bindings; remote-only config and injected KV without local roots. +- [ ] Use loopback Agent and scripted Secrets Manager HTTP fixtures through the real pinned SDK in the generated child process. Stage 5 must prove a supported SDK endpoint override plus explicit fixture credentials can select that loopback fixture in child-only bootstrap without ambient discovery or external calls. Clear inherited child bootstrap, supply only required fixture settings and dummy credentials/region, disable metadata credential lookup, and select only the loopback Secrets Manager endpoint. Keep this endpoint out of the production bindings schema. Assert request targets, bodies, selectors, and attempts without logging Authorization/signatures or payload values; unexpected destinations fail the test. A downstream binary cannot use a dependency's private `cfg(test)` service/connector seam; reserve that seam for provider unit tests. If the pinned SDK cannot support the generated fixture this way, stop and approve another test strategy rather than adding a public mock registry. Run real AWS through the same generated runtime only in Stage 9. +- [ ] Make a downstream fake KV use only generic handle injection. It must not import AWS or depend on a built-in tag. +- [ ] Change fixture versions while the process runs. Prove snapshot reads and retained typed state do not change; restart and prove new values are adopted. Test pinned/current secret selection separately. +- [ ] Fail a mapped profile, secret fetch, envelope validation, nested required secret lookup, and app initializer. Prove no readiness/listener acceptance and no fallback in each case. +- [ ] Stop while AWS preparation is pending. Reuse accepted shutdown semantics and observe staged handle drop without overstating credential/socket cancellation. +- [ ] Capture child process output and returned startup diagnostics. Assert sentinel secrets, tokens, ARNs, raw bindings, config documents, response bodies, and SDK chains never appear. + +### Exit gate + +A generated native application proves the preparation -> shared registries -> app initializer -> typed handler path. Record commit, feature set, fixture policy, commands, and results. Mock evidence is not IAM/Agent freshness evidence. + +## 13. Stage 8: docs, feature isolation, and CI + +### Documentation + +- [ ] Update existing local-only claims in the Axum guide. Document native binding selection, local defaults, strict production roots, explicit options versus env loading, generic supplied handles/deferred setup, and SDK-free CLI validation. +- [ ] Document Agent process/network ownership, literal loopback setup, explicit loopback bind, bearer token policy, pinned Agent version, backup/preload disabled, and warm-cache staleness. +- [ ] Document exact raw/envelope and string/binary semantics, required mappings, startup-only adoption, non-atomic cross-document snapshots, secret retention after remote deletion/revocation, and no zeroization guarantee. +- [ ] Document bounds and their scope, sequential startup costs, best-effort cancellation, service/credential failure categories, and supervisor-owned restart backoff. +- [ ] Document generated opt-in features, supported Cargo-resolution subset, structural-only results, and local-only `config push`. No publication/provisioning claims. +- [ ] Include fictional bindings, role permissions, developer authentication, native-only support, costs, and named cleanup ownership. Keep application schemas/required secret names out of framework examples. + +### Required repository checks + +Run these against a stable combined state after scoped tests have passed: + +```sh +cargo test --workspace --all-targets +cargo fmt --all -- --check +cargo clippy --workspace --all-targets --all-features -- -D warnings +cargo check --workspace --all-targets --features "fastly cloudflare spin" +cargo check -p edgezero-adapter-fastly --target wasm32-wasip1 --features fastly +cargo check -p edgezero-adapter-cloudflare --target wasm32-unknown-unknown --features cloudflare +cargo check -p edgezero-adapter-spin --target wasm32-wasip2 --features spin +``` + +Add isolated native-provider gates, proposed commands: + +```sh +cargo test -p edgezero-store-aws --no-default-features +cargo test -p edgezero-store-aws --no-default-features --features appconfig-agent +cargo test -p edgezero-store-aws --no-default-features --features secrets-manager +cargo test -p edgezero-store-aws --no-default-features --features "appconfig-agent secrets-manager" +cargo test -p edgezero-adapter-axum --no-default-features --features cli +cargo test -p edgezero-adapter-axum --no-default-features --features axum +cargo test -p edgezero-adapter-axum --no-default-features --features "axum aws-stores" +cargo test -p edgezero-cli --test native_target_features +cargo test -p edgezero-cli --test generated_project_builds -- --ignored +``` + +Also lint settings-only, AppConfig-only, Secrets-only, Axum runtime-only, and CLI-only modes so all-feature unification cannot hide cfg mistakes. Tests needing the native runtime stay in native crates. Keep portable contract suites on `futures::executor::block_on`. + +- [ ] Wire enabled native provider tests and resolver/generated fixture checks into CI. Default provider tests require no credentials and no external network. +- [ ] Add target-filtered reachable normal/build graph assertions for all three generated WASM adapters and the existing Fastly `fastly cli` combination. Assert no `edgezero-store-aws`, AWS service SDK/config/credential crates, or AWS Smithy runtime dependencies are reachable there. Match actual SDK package identities, not a blanket `aws-*` prefix that would also ban unrelated TLS cryptography such as `aws-lc-rs`. +- [ ] Assert default native has no service clients; config-only has no Secrets Manager SDK; secret-only has no Agent HTTP-client dependency introduced by the provider feature. Compare dependency causes rather than banning unrelated existing HTTP dependencies. +- [ ] Inspect the selected graph, not every package listed in Cargo metadata, the lockfile, or host workspace all-feature builds. Optional AWS packages in Cargo.lock are not evidence of WASM reachability. +- [ ] Preserve existing adapter WASM contract/runtime tests, generated target checks, and docs gates. Run excluded app-demo/fixture checks when their manifests or lockfiles change. +- [ ] Run `npm run format`, `npm run lint`, and `npm run build` in `docs` after approved guide changes. Internal `docs/superpowers` documents are excluded from Prettier/site inputs; do not reformat them globally. + +### Exit gate + +All applicable host, generated-native, isolated-feature, docs, and WASM gates pass, or each unavailable/pre-existing failure has explicit evidence and a named blocker. Do not call unrun targets passing. + +## 14. Stage 9: disposable AWS acceptance, separately authorized + +Before any AWS action, obtain approval naming account/region, temporary workload/developer identity, maximum cost/request count, finite runtime, resources, and cleanup owner. Do not put this into ordinary CI or launch it merely because local tests passed. + +### Work + +- [ ] Pin and record commit, Rust/SDK behavior version, Agent version/image, native feature set, deployment environment, and bindings policy. +- [ ] Have the authorized operator create/review disposable AppConfig application/environment/free-form profiles and Secrets Manager string/binary versions. Runtime code creates nothing. +- [ ] Use an Agent role limited to AppConfig Data `StartConfigurationSession` and `GetLatestConfiguration`, and an application role limited to exact secret `GetSecretValue` plus required customer-managed-key `kms:Decrypt`. +- [ ] Start a cold Agent without disk backups. Retrieve exact raw/envelope documents through the generated application and complete required typed/secret checks before readiness. +- [ ] Verify denied cold-Agent access fails startup; separately observe and record documented warm-cache behavior without presenting it as fail-on-revocation or latest-deployment proof. +- [ ] Verify current/pinned secret versions, binary fidelity, restart adoption, missing resources/versions, denied secret access, and denial/no-fallback startup diagnostics. +- [ ] Change config/secret versions and stage labels. Prove the running process retains its snapshots and a new process adopts the selected new values. +- [ ] Stop the application and Agent, remove disposable resources through operator tooling, and verify cleanup. Record incomplete cleanup and owner explicitly. + +Record resource identifiers only in an authorized private acceptance record. Public/runtime logs and checked-in examples remain redacted. Acceptance failure blocks claims about the deployed behavior it exercised; local fixtures cannot substitute for it. + +## 15. Evidence, review, rollout, and rollback + +For each stage record `check | exact revision or relevant file hashes | environment/features | result | evidence location`. Run scoped tests after every code change as required by the repository. Run broad checks at stable integration checkpoints. Reuse only evidence whose relevant inputs still match. + +Review two high-risk facts before release: + +1. Provider/bootstrap/service errors cannot expose values through EdgeZero-controlled output. Push from source/path inspection to captured generated-process output in Stage 7, including failure and diagnostic levels. +2. Required AWS values and app-owned typed checks finish before listener acceptance/readiness, with no local fallback. Push from source order to failing and successful generated startup tests, then authorized AWS acceptance. + +Source-only claims remain unproven. Timeout observation does not prove credential-process/socket cancellation; payload accounting does not certify RSS; warmed Agent reads do not certify AWS freshness. + +Roll out additively. Existing applications without bindings inputs keep current defaults and AWS-free builds. Compile provider features before selecting a remote binding. Migrate one ID at a time with operator-reviewed resource mappings and app-owned startup validation. No framework data migration or remote write is included. + +Rollback means select a reviewed local/custom binding and restart with the needed local inputs, or deploy the previous reviewed binary/bindings pair. Never automate fallback after a selected AWS binding fails. Restart-based rollback may still require app-owned config/secret version coordination. + +## 16. Planning investigation and chosen approach + +Three read-only workers investigated native composition, AWS providers, and CLI/build/CI. Each then proposed a minimal approach; no worker edited code or ran Cargo/AWS checks. + +- Native composition confirmed existing handles and registries suffice. Choose keyed ready/deferred inputs instead of a general factory registry or an async initializer that selects stores too late. +- AWS investigation confirmed an explicit shared preparation session can own quotas, dedup, and clients without globals. Choose it over independent hidden per-binding sessions; retain caller responsibility for opaque prebuilt handles. +- CLI investigation found generated manifest commands bypass the Axum fallback and that metadata alone cannot be assumed equivalent to selected-package compilation. Cover both standard dispatch paths and gate offline verdicts on Cargo parity fixtures, with explicit structural-only degradation. + +The chosen plan limits production changes to native composition, one optional provider crate, and necessary generated/CLI integration. Existing local implementations remain in place; core/macros and unrelated adapter behavior stay unchanged. + +## 17. Stage 0 execution record + +### Gate result + +Historical initial audit on 2026-10-07, before the user's subsequent implementation/rebase authorization. At that point the original accepted-only exit gate was not satisfied and no runtime implementation had started. The later authorization and execution record below supersede that gate result. + +Read `CLAUDE.md`, the handoff completely, this worktree's complete spec and plan, the referenced reusable lifecycle design, and current #410/#324 issue bodies. Both issues remain open. Read-only GitHub queries confirmed: + +| Prerequisite | Exact current head | Status | Accepted revision | +| ------------ | ------------------------------------------ | ------------------------------------ | --------------------- | +| #275 | `d1a81579afb96f200e7fe8fd294600f0b446b080` | OPEN, draft; unchanged from planning | None; no merge commit | +| #406 | `3ab22096fd117f7506cf437b5ba128e0789e059c` | OPEN, draft; unchanged from planning | None; no merge commit | + +#406's current PR body also records outstanding integration with newer #275 contracts. The #406 head is not a descendant of this worktree's main revision. Therefore there are no accepted revisions to re-audit or accepted-head API differences to record. Do not transplant either draft or implement against main's old contracts. Repeat the audit against the exact accepted commits after prerequisite acceptance and an approved worktree integration step. + +The remaining approvals are still pending: Agent staleness, startup-only secret retention after deletion/revocation, whole-document mappings, initial service set, external bindings and generic injection choices, remaining limits, generated feature enablement, and the Stage 1 implementation plan. The approved configurable 8 MiB shared-session quota and caller-owned boundary remain settled. AWS acceptance still requires its separate authorization. + +### Contract and test inventory + +These are source observations, not executed draft-contract proof: + +- Main `crates/edgezero-core/src/config_store.rs:214-220` still returns `Option`. Main has ordinary secret reads, no core `time.rs`, and infallible `Hooks::build_app`. Main Axum `dev_server.rs:333-383` has development startup only, binds before building registries, and has no production `PreparedStores` initializer. +- Draft #275 adds shared `ConfigValue`, `BoundedStoreRead`, mandatory `get_bounded` and `get_bytes_bounded` methods with injected `MonotonicClock`, absolute `Deadline`, and independent backend/value caps. See draft core `config_store.rs:295-412`, `secret_store.rs:385-402`, `time.rs:27-130`, and fallible `App::build` in `app.rs:143-168`. +- Draft #406 exposes borrowed registries through `PreparedStores`, with the initializer `for<'startup> FnOnce(&'startup mut App, &'startup PreparedStores) -> LocalStartupFuture<'startup>`. The boxed future has no `Send` bound. See draft Axum `dev_server.rs:63-94, 782-834`; `initializer_borrows_inputs_across_await_and_captures_rc` at `1111-1145`. +- Draft #406 `App::insert_state` retains `Clone + Send + Sync + 'static` handler state and uses router copy-on-write. See draft core `app.rs:285-297`, `router.rs:532-538`, and `startup_state_forks_router_and_invalidates_old_admission` at `app.rs:2459-2505`. Non-`Send` setup does not imply non-`Send` retained handler state. +- Draft #406 prepares stores and runs the initializer before binding/readiness. Its stop/drain path gates dispatch and releases service/registry ownership. Existing `production_hosting.rs` tests cover signals, finite/forced drain, startup stop without readiness, redacted failures, typed startup validation, embedding, persistence, aliases and restart snapshots. Reuse them after acceptance; they were not run in this worktree. +- Draft root validation remains kind-wide at `run_options.rs:144-189`. Stage 1 must derive required roots from resolved per-ID local sources, without replacing accepted lifecycle behavior. +- Exported `config_store_contract_tests!` and `secret_store_contract_tests!` exercise ordinary reads and are reusable downstream. The separate `ready_*_bounded_read!` helpers are private `cfg(test)` fixtures, not exported bounded contracts. Snapshot providers need their own bounded-read tests in addition to the exported ordinary suites. + +Full exact-head source/test inventories and conditional proposals are local artifacts at `/tmp/native-store-stage0-core/{report,proposal}.md` and `/tmp/native-store-stage0-startup/{report,proposal}.md`. Draft #275 text was retrieved through the read-only contents API because its Git object is absent locally; draft #406 text came from the existing Git object. All extracts remained under `/tmp`; no branches were copied. + +### Executed main baseline + +Tested code revision: `683202c66948146ec360f490126f1d60f920288b`, with no tracked code changes. Environment: Rust/Cargo 1.95.0, native `x86_64-unknown-linux-gnu`, `CARGO_NET_OFFLINE=true`, `CARGO_BUILD_JOBS=2`, worktree-local target directory. All three WASM targets were already installed. No dependencies or toolchains were installed. Cargo manifest and lockfile SHA-256 values were identical before and after: + +```text +Cargo.toml acbdefe5e403b9faa1c09b095811927cb4c16faa549ec287a43f2e307d9df844 +Cargo.lock ccc5cbeda40fab3f2334b9846c565ea6699753162c508998a891f6037a4cee6d +``` + +All Cargo commands sharing the target directory ran sequentially. Evidence directory: `/tmp/native-store-stage0-baseline/`, with exact `.command` files, full `.log` files, `results.txt`, environment, revision, and before/after hashes. + +| Check | Result | Seconds | Log | +| --------------------------------------------------------------------------------------------------------------------- | ---------------------------- | ------- | ------------------------ | +| `cargo test --offline --locked -p edgezero-adapter-axum --all-targets --all-features` | PASS, 149 tests | 64 | `axum-tests.log` | +| `cargo test --offline --locked --workspace --all-targets` | PASS, 1,440 tests; 1 ignored | 118 | `workspace-tests.log` | +| `cargo fmt --all -- --check` | PASS | 1 | `format.log` | +| `cargo clippy --offline --locked --workspace --all-targets --all-features -- -D warnings` | PASS | 90 | `clippy.log` | +| `cargo check --offline --locked --workspace --all-targets --features "fastly cloudflare spin"` | PASS | 8 | `workspace-features.log` | +| `cargo check --offline --locked -p edgezero-adapter-fastly --target wasm32-wasip1 --features fastly` | PASS | 28 | `fastly-wasm.log` | +| `cargo check --offline --locked -p edgezero-adapter-cloudflare --target wasm32-unknown-unknown --features cloudflare` | PASS | 19 | `cloudflare-wasm.log` | +| `cargo check --offline --locked -p edgezero-adapter-spin --target wasm32-wasip2 --features spin` | PASS | 19 | `spin-wasm.log` | + +No baseline failures were observed. The existing ignored `generated_workspace_compiles` test was not forced because it may fetch dependencies. WASM results are compilation evidence, not hosted execution. These baseline checks prove no new AWS provider behavior and do not validate the draft PR combination. + +### Dependency investigation and pending work + +Read-only registry evidence identifies candidate direct pins `aws-config = "=1.12.0"` and `aws-sdk-secretsmanager = "=1.120.0"`; both declare Rust 1.94.1 and are non-yanked. Their declared direct MSRVs fit installed Rust 1.95.0, but no complete transitive graph was resolved or compiled. Candidate fixed behavior is `BehaviorVersion::v2026_01_12()`, with explicit native runtime/current HTTPS-client features and reviewed developer credential features rather than `behavior-version-latest`. The documented default connect timeout is 3.1 seconds, so the provider must configure its selected budget explicitly. + +Final coherent SDK pins/features, transitive MSRV proof, diagnostic/timeout/retry audits, and the exact Agent image/version pin remain unfinished Stage 0 work. No container was pulled. Do not treat a floating Agent `2.x` tag or a Lambda-extension release as an exact acceptance pin. Research sources and limitations are recorded in `/tmp/native-store-stage0-baseline/sdk-research.md`. + +### Worker decisions and next step + +Two read-only repository explorers audited core contracts and production startup separately. Two read-only architecture workers then compared minimal conditional Stage 1 approaches. A read-only build validator checked the eight command/log/results artifacts, revision, environment, and unchanged hashes; its report is `/tmp/native-store-stage0-baseline/verification.md`. The coordinator inspected the source excerpts and actual logs directly. The chosen approach remains the existing plan: private per-ID ready/deferred overrides, full source validation, sequential temporary staging, and one complete registry publication. Complete-prebuilt-registry injection would shift mixed-source preparation onto callers; direct mutation of published registries would complicate failure cleanup. No production edits were needed or authorized for this audit. + +The initial audit recommended waiting for acceptance/approval. The user's later instruction superseded that recommendation; the current execution record follows. + +## 18. Authorized prerequisite integration and Stage 1 + +The user explicitly authorized implementation and rebasing onto the open PRs. No GitHub state changed. A local backup branch `backup/issue-410-main-stage0` retains the original main base. + +Selected #275 head: `d1a81579afb96f200e7fe8fd294600f0b446b080`. Selected #406 head: `3ab22096fd117f7506cf437b5ba128e0789e059c`, one unique commit after shared ancestor `5527b58e2a7cb6307d5c7a8926cd5dfc6f4f8b4a`. The #406 commit was rebased onto #275, producing combined base `1499062`. Both source PRs remain open; this is a local implementation stack, not an accepted main revision. + +Two read-only workers investigated and proposed integration choices, recorded at `/tmp/native-store-integration/{native,contracts}-proposal.md`. Central resolution preserved #275's required serde dependency, bounded local config snapshot parser/residency, HTTP head/deadline limits, connection capacity and panic isolation, logging/preflight ordering, and asynchronous egress. It added #406's HRTB initializer, state insertion, readiness/stop/signals and bounded production drain. The production fixture was migrated from String to shared ConfigValue. A bounded required-file loader replaces #406's older unbounded loader. + +Scoped integration tests passed: core all-targets, Axum all-targets/all-features, including 13 production-hosting subprocess tests. Initial integration runs exposed fixture value-type and category mismatches, plus regressions in connection panic/cancellation tests; these were fixed before the rebase completed. The combined owner keeps #275's bounded FuturesUnordered connection ownership and adds phase-aware graceful drain, rather than using #406's older parser or spawning unbounded tasks. + +Stage 1 adds `native_stores.rs`, private per-kind source maps, closed preparation categories, move-only ready/deferred overrides, and `run_app_with_native_stores`. Hosting options remain cloneable with local-default bootstrap selection. Complete source validation precedes local provider construction and deferred polling. Roots are required only for IDs still resolved locally. Sources are prepared sequentially, then all registries are finalized once before the initializer and request service. + +New colocated tests exercise non-Send Rc setup on the real runner, default/key preservation, mixed local/custom roots, ready/deferred KV and secrets, namespace misses, duplicates and redacted Debug, undeclared sources/overlay/root/metadata errors with zero deferred polls and no database files, later failure dropping staged handles and unpolled futures, startup stop without binding, and the initializer/request path using the same provider and inserted state. Existing dev and strict-production regression suites remain passing. + +Stage 1 exit gate passed. Evidence is `/tmp/native-store-integration/stage1-tests.log` and `stage1-clippy.log`, plus all twelve serialized workspace/native/WASM/isolation checks passing in `check-results.txt`. Runtime-only non-test `cargo check --offline --locked -p edgezero-adapter-axum --no-default-features --features axum --lib` also passed, closing the build review's dev-dependency-unification proof gap. Exact commands/logs and before/after hashes are retained there; current relevant source hashes still match `stage1-inputs.sha256`. + +Read-only runtime and build reviewers reported no confirmed findings in `stage1-{runtime,build}-review.md`. Their source conclusions were checked against executed logs and unchanged input fingerprints. The reviewer noted that cross-kind behavior was tested in separate fixtures rather than one combined invocation; later production/provider tests will exercise the combined path. Review workers and the broader check runner have finished and were released. Baseline main evidence is kept separate from this combined-stack proof. + +Stage 2 read-only investigations/proposals are `stage2-{bootstrap,aws-settings}-proposal.md`. Chosen approach is the shared native parser feature plus SDK-free AWS settings crate, with typed source decisions rather than unvalidated option bags. AWS resource/URL validation belongs in the provider crate; schema/collisions/bootstrap ownership stay native-side. Public API docs confirm Secrets Manager selector lengths and complete-ARN/name constraints. No AWS resource or credential access is needed for this unit. + +No #410 commit was created. No deployment, publishing, image pull, credential discovery, or AWS service call was performed. Further implementation is authorized. The user has since explicitly declined real AWS acceptance, so Stage 9 is excluded from the currently authorized task and will not be run or requested. + +## 19. Stage 2 execution and proof + +Added `edgezero-store-aws` as a default-off, settings-only member. Its normal graph has only serde, URL/percent encoding, and closed error formatting. The shared Axum `native-bindings` feature supports parser use without either runtime or CLI registration. AWS service feature bodies are intentionally empty until their provider stages; the native runner currently rejects all built-in AWS selections before setup. Do not describe this checkpoint as operational AWS support. + +The pure source validator now owns declarations/defaults, supplied/file/inline collisions, raw NAME/KEY conflicts and target capability checks. The runtime translates those decisions to its existing sources, checks local-root needs and retains Stage 1 failure/drop ordering. File contents are captured during selection into cloneable options, not reopened during preparation. Explicit constructors retain only supplied bootstrap values. `from_env` captures the locator and scalar inputs and resolves each selected token name once. Relevant normalized duplicate settings fail; unrelated core environment semantics remain unchanged. Ambiguous explicitly supplied token values cannot be used by a provider. + +ARN validation supports `aws`, `aws-cn`, and `aws-us-gov` region families, matching explicit region, full account/resource shape, names of 1-512 documented CreateSecret characters and the six-character ARN suffix. This is shape validation, not a served-region/resource existence claim. Selectors retain non-control Unicode and embedded spaces under the API's character-count lengths, with all-whitespace/control values rejected. Agent identifiers support escaped spaces/Unicode and other safe segment characters up to 128 bytes, while slash/backslash/percent/query/fragment/control and standalone dot segments fail. Endpoint validation rejects normalized integer/octal IPv4 and other path/authority ambiguities. + +Both Stage 2 reviewers found no confirmed defect. Their concrete test gaps were addressed: malformed UTF-8 and duplicate tables, independent decoded path assertions, selector/ARN boundaries, exact file/explicit policy selection, plus a real isolated subprocess through `AxumRunOptions::from_env` for named token capture and missing/empty/non-Unicode token/locator errors. File and inline failures run through the real runner and assert zero future polls, no initializer/database files, dropped supplied state and redacted output. + +Evidence is `/tmp/native-store-integration/stage2-check-results.txt`, exact command files and logs, and before/after hashes. Provider settings: 9 passed. Axum all features: 260 unit, 13 outbound, 13 production-hosting passed. CLI-only: 55 passed; runtime-only: 219 passed. Workspace tests, fmt, native feature check, three WASM builds, normal non-test CLI/runtime builds and SDK/HTTP-free settings/parser dependency assertions passed. One initial workspace Clippy run found two new test-only style errors; these were fixed and scoped settings tests plus full workspace strict Clippy and formatting passed. Final hashes differ from the broad-check hashes only in those test-only changes to `settings.rs`; `stage2-final-inputs.sha256` records the final state. Runtime/CLI sources and production settings validation remained unchanged. + +The shared-session and SDK diagnostic audits are the next read-only proposals. No AWS resource access, credential discovery, image pull, deployment, mutation or publishing occurred. Real AWS acceptance remains excluded by the user's instruction. + +## 20. Follow-up validation and diagnostic approval gate + +Stage 2 follow-up made source decisions borrow declaration IDs. The same validator now consumes macro-baked static metadata or strings owned by a CLI manifest, without leaking strings or changing core. Manifest adapters reject legacy fields and an omitted multi-ID default. Secret lookup keys and selected non-opaque namespaces are checked against core's 512-byte bound before preparation; supplied opaque namespace semantics remain caller-owned. Omitted/current and explicit `AWSCURRENT` selectors normalize to the same target identity. + +Executed evidence is `/tmp/native-store-integration/stage2-boundaries-*.log`: settings 10 passed, Axum all-features library 262 passed, workspace tests, strict workspace Clippy, formatting and combined native feature checks passed. Initial Clippy issues in the follow-up were corrected before the final successful run. The only subsequent source delta is comments, followed by settings tests and formatting again. No #410 commit was created. + +The isolated SDK spike is outside the repository. It resolves/builds Secrets Manager 1.120.0 with runtime 1.16.0 / HTTP client 1.5.0 on Rust 1.95.0; it does **not** establish the complete aws-config developer-chain graph. Explicit fictional credentials and literal-loopback fixtures were used with cleared process environments. Scoped full client construction/send suppresses sentinel ARN/value/provider metadata under a process-global TRACE collector, even with `LOG_SENSITIVE_BODIES=true`. Two non-sensitive throughput events escape on HTTP success. Source-only pooled-driver findings must be qualified by the selected path/features; that driver path was not active in the tested configuration. + +A separate executed tracing/log fixture establishes a policy blocker. With no global tracing subscriber, `WithSubscriber(NoSubscriber)` suppresses its inner event but permanently disables unrelated application tracing-to-log fallback after the scope (`before=true hidden=false after=false facade_after=true`). With additive `tracing/log-always`, it does not suppress the mirrored inner event (`hidden=true`). A scoped mute is therefore not a logging-preserving, feature-independent SDK diagnostic strategy. + +**Historical pause, superseded by explicit first-version approval.** The user authorized continuing with redacted/dropped EdgeZero diagnostics and a disclosed dependency DEBUG/TRACE limitation before reviews. Do not use scoped NoSubscriber or change global application logging. Keep EdgeZero-owned errors closed and payload-free; dependency diagnostic output remains caller/operator-owned and must be called out in docs and the draft PR. The user also authorized commits/publication to open a draft with `gh stack` after implementation and checks. Stages 3-8 are now in progress; Stage 9 remains declined/excluded and is not requested. Detailed sources, fixture boundaries and logs: `/tmp/native-store-integration/sdk-diagnostics-{proposal,execution}.md`. + +## 21. Offline implementation and verification checkpoint + +The coordinator integrated the common session, private snapshots, literal-loopback Agent transport and pinned Secrets Manager SDK. Config remains arbitrary whole UTF-8, including empty and quoted values. Secret payloads preserve SDK-decoded bytes, require exactly one payload field and deduplicate successful targets across namespaces. The native runner uses one shared session and completes all registries before the application initializer/listener. + +CLI build/serve dispatch shares SDK-free Cargo argument normalization. Only exact generated Axum commands are intercepted as direct argv; custom shell commands remain unchanged. Pinned dispatch reads selected manifest features, not a rediscovered manifest. Bindings-only validation normalizes manifest/explicit target controls and reports structural-only validity with provider availability unknown. The unproven Cargo feature resolver is deliberately not shipped. The public typed push API rejects selected AWS config bindings before app TOML, prompts or writes, including local/dry-run flags. + +Generated Axum entrypoints now use `run_generated_app`: no locator preserves existing local development behavior; any selected locator enters strict native startup and compiled-out providers fail closed. The ignored generated-consumer test invokes real `edgezero new` and `edgezero build`, compiles default-off/Agent-only/Secrets-only/both, rejects a compiled-out selection, then exercises a generated app's public HRTB initializer with gated provider responses, a serialized envelope, nested secret resolution and repeated snapshot-backed requests. This is separate from the five static production-consumer tests. + +Executed coordinator gates: `/tmp/native-store-integration/final-checks.sh`, exact commands/results/logs and stable before/after source fingerprint under `/tmp/native-store-integration/final-checks/`. All 24 gates passed. Workspace tests: 1,988 passed, two ignored generated tests; both ignored tests were separately executed successfully. Agent-only: 25; Secrets-only: 25; both providers: 40. Axum all-feature tests including subprocesses/contracts: 299. Demo tests: 54; lifecycle fixture tests: 31. Strict workspace and isolated Clippy, combined native features, all three WASM adapters, SDK-free normal graphs and strict rustdoc passed. The existing generated workspace test also passed its host/strict-Clippy/core-tests/typed-CLI/three-WASM checks (`generated-full-final.log`). Docs lint, full formatting and site build passed. + +A machine-local `/tmp` quota caused one interrupted test run. Checks use a disk-backed `TMPDIR=/home/pav/.cache/native-store-checks/tmp`; no unrelated temporary files were removed. CLI unit tests with every adapter disabled assume registered adapters and are not a supported test scope; CLI-only compilation/strict Clippy passes, and the default registered CLI suite passes 227 tests. + +Read-only final provider and CLI reviews found no confirmed boundary defects (`/tmp/native-store-final-review-{provider,cli}.md`). These reviews did not execute Cargo and do not replace coordinator gates. Generated `serve` CLI end-to-end behavior is not separately demonstrated; shared argument selection/action gating is source/unit-tested, and generated build plus production startup is executed. Remaining review items are dependency DEBUG/TRACE disclosure, operator-reviewed Agent release/digest and explicitly excluded real AWS acceptance. + +Publication completed with one draft stack member, [PR #414](https://github.com/stackpop/edgezero/pull/414), assigned to ChristianPavilonis. Trunk is `docs/outbound-http-spec` / #275 at `ea66eff045512cccc60646d6ec339e83ff44fed7`. The compatible lifecycle bridge is `a08f54c`; tested implementation checkpoint is `ab402f4fbdaa5c7c244ec16c11cfacec65123064`, tree `d6097c9c8947e8828338b3340287d6df0650380e`. Existing #275/#406 were not modified. The clean rebase required one lockfile dependency disambiguation (`wasm-streams 0.6.0`), followed by all 24 gates passing again and a successful existing generated workspace/WASM test. Exact commands, fingerprints and logs remain under `final-checks/`; `generated-full-rebased.log` records the additional consumer gate. The subsequent publication-record commit changes only documentation. No merge or deployment was performed. + +## Links + +- [Governing design](../specs/2026-10-07-native-store-providers-design.md). +- [Owning issue #410](https://github.com/stackpop/edgezero/issues/410). +- [Bounded config and lifecycle prerequisite PR #275](https://github.com/stackpop/edgezero/pull/275). +- [Production startup prerequisite PR #406](https://github.com/stackpop/edgezero/pull/406). +- [Raw configuration semantics #324](https://github.com/stackpop/edgezero/issues/324). +- [AWS Agent configuration](https://docs.aws.amazon.com/appconfig/latest/userguide/appconfig-integration-containers-agent-configuring.html). +- [Secrets Manager GetSecretValue](https://docs.aws.amazon.com/secretsmanager/latest/apireference/API_GetSecretValue.html). +- [Rust SDK timeouts](https://docs.aws.amazon.com/sdk-for-rust/latest/dg/timeouts.html) and [retries](https://docs.aws.amazon.com/sdk-for-rust/latest/dg/retries.html). +- [Cargo features](https://doc.rust-lang.org/cargo/reference/features.html), [metadata](https://doc.rust-lang.org/cargo/commands/cargo-metadata.html), and [tree](https://doc.rust-lang.org/cargo/commands/cargo-tree.html). diff --git a/docs/superpowers/plans/2026-10-14-dynamodb-kv.md b/docs/superpowers/plans/2026-10-14-dynamodb-kv.md new file mode 100644 index 000000000..b22929a71 --- /dev/null +++ b/docs/superpowers/plans/2026-10-14-dynamodb-kv.md @@ -0,0 +1,148 @@ +# Native DynamoDB KV implementation plan + +- Status: Implemented, verified and pushed to existing draft [PR #414](https://github.com/stackpop/edgezero/pull/414) on `issue-410-native-store-providers`. The eight pre-existing files' unrelated hunks remain preserved and uncommitted. +- Requested outcome: Native apps can select DynamoDB independently per declared KV store ID, through the existing portable KV interface. +- Base: Draft [PR #414](https://github.com/stackpop/edgezero/pull/414), published head `b764a883ce57a323250d9d7a20787dc5937ef90e` at research time. Re-check the published head before implementation. +- Safety: No AWS calls, ambient credential discovery, infrastructure changes, deployments or image pulls during research or default checks. Literal-loopback SDK fixtures use dummy credentials and cleared environments. Real AWS acceptance requires separate authorization. + +## Research findings + +Core `KvStore` supports raw-byte get, overwrite put, delete, exists, TTL and lexicographic prefix pages. `KvHandle` validates 512-byte UTF-8 keys, up to 1,000 keys per page and a 25 MiB value ceiling. That ceiling does not guarantee every provider can accept that size. `read_modify_write` is explicitly non-atomic. There is no portable CAS, conditional-create, transaction, consistency selector or snapshot-list interface. + +DynamoDB can implement the ordinary interface with a namespace partition key and the unchanged user key as sort key. Its item ceiling is 400 KiB including attribute names and values. Base-table Get/Query support strong reads, and Query sorts strings by UTF-8 bytes. Query evaluates its limit before filtering, stops at its 1 MiB data boundary, and can return an empty page with `LastEvaluatedKey`. Core already allows short/empty pages and terminates on cursor absence. + +DynamoDB TTL is asynchronous garbage collection, not an expiry guarantee. The provider must check logical expiration on reads and listing. The shared raw-trait contract includes a 1 ms TTL test, despite the public handle's 60-second minimum. + +A lost response or HTTP 500 can follow a committed write. Replaying an unconditional Put/Delete can overwrite or delete a concurrent writer's newer state. The first version will not automatically retry these calls or claim exactly-once effects. + +SDK generated `AttributeValue`, `PutItemInput` and `GetItemOutput` derive Debug without redacting item contents. The locked Smithy Blob implementation formats bytes as hex. Do not assume DynamoDB values have Secrets Manager's sensitive-model protection. EdgeZero-owned diagnostics will be closed/redacted. Dependency DEBUG/TRACE exposure must be disclosed, without installing subscriber suppression or changing application logging. + +Two read-only workers independently mapped the contracts/integration and researched service/SDK semantics, then proposed implementation alternatives. Reports and proposals are `/tmp/dynamodb-kv-{contract,service}-{research,proposal}.md`. No code, Cargo or AWS checks were run during research. The current checkout has unrelated ongoing modifications. Do not overwrite, stash, commit or migrate them as part of this task. + +## Approved design decisions + +### Scope and compatibility + +- Implement in `edgezero-store-aws`, not a new HTTP adapter and not portable core. +- Add default-off provider feature `dynamodb-kv` and Axum/generated alias `aws-dynamodb`. Preserve the existing `aws-stores` alias membership so existing config/secrets users do not acquire DynamoDB unexpectedly. All three can be selected explicitly with `aws-stores,aws-dynamodb`. +- Add strict v1 `[kv.]` binding dispatch with provider `aws-dynamodb`. No provider-specific fields in portable `edgezero.toml`. +- Preserve existing `NativeStoreBindings::new(config, secrets)` callers through an additive KV builder/accessor. Preserve ready/deferred KV overrides and local redb defaults. +- Leave CLI validation structural-only with provider availability unknown. Actual native runtime capabilities reject a selected but uncompiled provider before setup, local opens or deferred polling. +- DynamoDB is live mutable storage. Preparing a client is not a startup snapshot. Runtime KV calls perform remote I/O and are not covered by the config/secret retained-payload quota. + +```toml +version = 1 + +[kv.sessions] +provider = "aws-dynamodb" +region = "us-east-1" +table = "edgezero-kv-prod" +namespace = "checkout-prod/sessions" +``` + +### Table and key schema + +Use an operator-owned, single-region, unindexed table with String partition key `pk` and String sort key `sk`. This first version does not provision or migrate tables and does not support global tables or secondary indexes. A bounded startup `DescribeTable` verifies the schema/status before the handle is published. This does not certify write permissions or capacity. + +- `pk` is the explicitly configured physical namespace, nonempty, at most 512 UTF-8 bytes and without control characters. It is not derived from a bare logical ID. Two bindings with the same region/table/namespace intentionally share data. +- `sk` is the exact portable user key, without normalization, hashing, case folding or delimiter concatenation. Prefix matching remains literal. +- `v` is a Binary attribute containing the complete original bytes, including empty values. +- Optional `expires_at_ms` stores logical expiry. Optional `ttl_epoch_s` is the garbage-collection hint. +- Fixed attribute names avoid a configurable mapping/index framework. All namespaces in the table follow this schema. A high-traffic namespace can become a hot partition; no sharding or throughput certification is claimed. + +### Values, expiration and listing + +- Cap payloads at **384 KiB, 393,216 bytes**. Bound namespace/key/metadata and check item accounting before I/O. Return a fixed `KvError::Validation` for excessive inputs. Keep the portable 25 MiB ceiling unchanged. No chunking, S3 fallback or silent truncation. +- A normal Put replaces the whole item and clears both expiry fields. TTL Put computes a checked millisecond deadline without rounding expiration earlier than requested, plus `ttl_epoch_s = ceil(expires_at_ms / 1000)`. +- Get and the default Exists path enforce expiry at completion. Expired reads do not delete records, avoiding a race with concurrent renewal. DynamoDB TTL handles eventual cleanup if operators enable the documented attribute. Disabled cleanup can retain expired records and cost storage. +- List uses one ascending, strongly consistent base-table Query, constrained to configured `pk`, and `begins_with(sk, prefix)` for nonempty prefixes. Omit the sort condition for an empty prefix. Project keys/expiry, not values, and check expiration after receipt. This keeps malformed expiry handling explicit rather than trusting a filter to hide malformed records. +- Query's evaluated-item limit is the requested page size. A portable page may be short or empty with a continuation cursor. Do not loop to fill pages. Strong per-request reads do not make the page sequence a snapshot. +- Encode a bounded, versioned cursor containing region/table/namespace/prefix and the exact continuation key. Reject malformed/cross-scope cursors before provider I/O. Validate returned keys/order/continuation progress. Cursors are opaque, not secret or tamper-proof; every request remains constrained to the configured namespace and prefix. Never log them. + +### SDK, deadlines and errors + +- Exact pin `aws-sdk-dynamodb = "=1.120.0"`, defaults off with native HTTPS/Tokio support. Its manifest MSRV is 1.94.1. The selected graph compiled and passed isolated and combined tests without upgrading the existing AWS pins. The lockfile adds the SDK and its `http` 0.2 compatibility dependency; existing `http` 1.4 references are disambiguated. +- Reuse the existing fixed AWS behavior version and supported operator-owned credential chain. Settings remain SDK-free and contain no credentials or endpoint overrides. Tests inject only literal-loopback endpoints and dummy credentials. +- One retained client per compatible region/policy, with `max_attempts = 1` and no outer retry loop. This intentionally disables read retries too, avoiding dual-client policy complexity in the first slice. +- Start with a 5-second operation budget, 1-second connect timeout and 2-second attempt timeout. Validate finite positive policy values and ordering. Runtime deadlines are fresh per call, never the expired startup preparation deadline. +- Bound client/DescribeTable setup by the native startup preparation deadline. Reuse the existing sequential staging and publish only complete registries. Extend preparation accounting without pretending to retain live database contents. +- Missing and expired items alone return None/false. Access, credential, table/schema, throttling, transport and timeout failures remain closed errors. Discard SDK Display/Debug/source chains. Document that write failures may leave the effect unknown. +- No global/subscriber logging changes and no claim that dropping a future forcibly terminates credential subprocesses, DNS or socket work. + +```mermaid +flowchart TD + Capture[Capture selected KV bindings] --> Validate[Validate IDs conflicts settings and compiled capability] + Validate --> Prepare[Create bounded DynamoDB client and verify table] + Prepare --> Registry[Publish complete local and remote store registries] + Registry --> Initialize[Run application-owned initializer] + Initialize --> Listen[Bind listener] + Listen --> Handler[Handler obtains existing KvHandle] + Handler --> DynamoDB[Live bounded DynamoDB Get Put Delete or Query] +``` + +## Implementation sequence + +- [x] **0. Isolate the work and establish the focused baseline.** Use the existing branch and PR #414 as explicitly requested. Record and preserve the eight pre-existing modified files; stage only DynamoDB-owned changes. Keep current unrelated changes untouched. Reuse existing evidence only when exact relevant inputs still match. Read-only worker reports are maps, not executed proof. +- [x] **1. Validate settings and dependency boundaries.** Add strict DynamoDB settings/policy and exact optional native dependency. Prove settings-only/default/Agent-only/Secrets-only graphs exclude DynamoDB, and Dynamo-only excludes the Secrets SDK. Demonstrate generated SDK model Debug disclosure with dummy data, and closed EdgeZero errors through loopback failures. Disclose dependency leakage rather than muting application logging; do not claim an ambient TRACE capture or universal redaction. +- [x] **2. Implement the real KV provider.** Add checked row codec/size/expiry/cursor logic and SDK Get/Put/Delete/Query. Use core `KvStore`, `KvHandle` and `KvError`; no new portable methods or Tokio/core dependencies. Expose preparation returning a portable handle and keep SDK models private. +- [x] **3. Prove provider behavior.** Run the shared KV contract against the actual provider using deterministic fixtures, not a replacement mock implementation. Drive the real SDK request/response path against a loopback fixture with cleared environments. Cover Unicode NFC/NFD distinction, 512-byte keys, arbitrary/empty binary, overwrite/delete/missing, millisecond expiry and renewal, no-TTL reset, size bounds, malformed records, short/empty pages, 1 MiB continuation shape, cursor scope/progress, denial/missing table, stalls and one-attempt ambiguous writes. Assert no owned diagnostic contains fixture keys/values/credentials. +- [x] **4. Wire per-ID native selection.** Extend KV binding schema, declared-ID/limit/conflict checks, capability/source resolution, startup session selection and existing KV staging loop. Test local+Dynamo+supplied mixtures, intentional shared namespaces, zero opens/polls on invalid sources, no fallback and no partial registry/initializer/listener on setup failure. All-Dynamo KV must not require or create a local DATA_DIR. +- [x] **5. Integrate CLI and actual generated consumers.** Add the generated opt-in alias and structural parser coverage. Reuse the generic Cargo normalizer rather than another resolver or shell interceptor. Prove generated default-off rejection and Dynamo-enabled CRUD/list/TTL through the public `Kv` extractor, with real SDK loopback traffic. Restart the generated consumer against the same fixture to prove data is not local process state. Preserve existing config push rules and config/secrets feature combinations. +- [x] **6. Document and run affected final gates.** Document schema creation/TTL cleanup/IAM as operator instructions, credentials, cost/hot partitions, smaller values, non-atomic read-modify-write, non-snapshot pages, write ambiguity and dependency logging caveat. Run focused tests after each coherent code unit, then affected workspace/native/feature/WASM, fmt, strict Clippy, SDK graph isolation and generated-consumer gates at a stable checkpoint. Do not repeat unaffected successful checks without input changes. Record commands, checked state and limitations. Commit and push the completed DynamoDB work to existing PR #414 after checks. Merges and infrastructure acceptance remain unauthorized. + +## Likely affected files + +| Area | Files | +| ------------------- | --------------------------------------------------------------------------------------------------------------------------------------- | +| Provider | Root `Cargo.toml`/lock; AWS crate manifest, `src/settings.rs`, `src/lib.rs`, `src/preparation.rs`; new focused DynamoDB module/tests | +| Native integration | Axum manifest, `src/native_bindings.rs`, `src/native_stores.rs`, existing KV loop in `src/dev_server.rs`, relevant source/startup tests | +| Generated app / CLI | Axum `src/templates/Cargo.toml.hbs`; CLI generator/structural validation tests; a dedicated generated DynamoDB consumer proof | +| Evidence / docs | SDK isolation script and applicable CI feature jobs; focused DynamoDB guide linked from the existing native AWS guide | + +Avoid unrelated HTTP/runtime refactors, portable manifest changes, generic provider registration, hot reload, cache layers or provisioning code. + +## Approval and unverified work + +Approval should settle the ordinary-CRUD scope, 384 KiB payload limit, operator-owned table schema, single-attempt policy and dependency DEBUG/TRACE limitation. CAS or 25 MiB transparent values would require a different design, not an incidental addition to this plan. + +Implementation and offline verification are complete. No real DynamoDB/IAM/capacity/global-table acceptance is claimed or scheduled. The fixture verifies continuation wire shape, not the deployed service's 1 MiB evaluation boundary. SDK model Debug disclosure is executed evidence; no new ambient dependency TRACE capture or universal logging-safety claim is made. Generic native staging tests prove drop/atomicity behavior; mixed DynamoDB/local/ready/non-Send-deferred selection is directly tested, not presented as a deployed mixed-backend acceptance test. + +## Executed verification + +The exact staged source tree `2fa76fb26377cf41203c255f924d988d6b657002` was archived and checked independently of the eight dirty baseline files. Both CLI capability literals in the published source receive the new flag; the unrelated working-copy CLI refactor is excluded. Before/after SHA-256 manifests match. Subsequent publication-record edits are Markdown-only. + +- Workspace default tests: **1,999 passed, 0 failed, 16 ignored**. Required SDK/native feature scopes and the two native generated consumers were executed separately; this is not a claim that every unrelated ignored test ran. +- Provider library scopes: DynamoDB **44**, Agent **26**, Secrets **26**, all providers **74**. The all-provider suite used one test thread after an inherited Secrets timeout test transiently returned `Unavailable` instead of `Deadline` under parallel load. That test passed alone, then the complete serial suite passed. The initial failure log is retained; no production or unrelated test was changed to hide it. +- Native DynamoDB-only library: **229 passed**; native all-feature test/doc suites: **302 passed**. The publishable tree excludes six native tests added by the pre-existing unrelated changes. +- Actual CLI-generated DynamoDB consumer: **1 passed**, including default-off rejection before SDK I/O, enabled put/get/list/TTL-write/read, restart persistence and delete. Public-handle TTL is 60 seconds; elapsed millisecond expiry is exercised by the provider contract, not this subprocess proof. +- Existing CLI-generated AppConfig/Secrets consumer: **1 passed** again. +- Strict workspace all-target/all-feature Clippy, independent provider Clippy scopes, native default Clippy, strict provider/native rustdoc, formatting, dependency isolation and Cloudflare/Fastly/Spin WASM checks passed. +- Docs lint, formatting and production build passed after removing a link to process-only documents excluded from the documentation site. +- Loopback fixtures explicitly configure dummy credentials and endpoints. Put and Delete tests apply the mutation before a scripted 500, then assert `Unavailable`, one attempt and the retained/removed row. SDK GetItemOutput Debug contains a sentinel key and binary hex; owned error assertions remain source-free. + +Commands, logs and fingerprints: `/tmp/dynamodb-baseline/final-checks.sh`, `/tmp/dynamodb-baseline/final/`, `snapshot-{before,after}.sha256`. Read-only final provider review: `/tmp/dynamodb-provider-final-review.md`; no confirmed findings, and no execution claims from that review. + +## Publication + +Implementation commit `883354f98dfd4437eb60faaac12d4469f65aa158` was pushed to the existing branch and verified as PR #414's remote head. Only owned DynamoDB changes were staged, including narrowly projected edits in the two overlapping baseline files. The eight unrelated working-copy modifications remain unstaged; no stash, prerequisite rewrite, merge, deployment or AWS acceptance occurred. This final publication record changes only this Markdown plan, so the passing code gates remain applicable. + +## Sources + +- Portable implementation and shared tests: `crates/edgezero-core/src/key_value_store.rs`, especially `KvError`, `KvHandle`, `KvPage`, `KvStore` and `key_value_store_contract_tests!`. +- Native selection and staging: `crates/edgezero-adapter-axum/src/{native_bindings,native_stores,dev_server}.rs`. +- [DynamoDB constraints and item limits](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Constraints.html). +- [Query request/response and pagination](https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_Query.html). +- [Read consistency](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/HowItWorks.ReadConsistency.html). +- [Expired TTL records](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/ttl-expired-items.html). +- [Error handling and ambiguous writes](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Programming.Errors.html). +- [SDK 1.120.0 manifest](https://docs.rs/crate/aws-sdk-dynamodb/1.120.0/source/Cargo.toml). +- [AttributeValue source](https://docs.rs/aws-sdk-dynamodb/1.120.0/src/aws_sdk_dynamodb/types/_attribute_value.rs.html), [PutItemInput source](https://docs.rs/aws-sdk-dynamodb/1.120.0/src/aws_sdk_dynamodb/operation/put_item/_put_item_input.rs.html), [GetItemOutput source](https://docs.rs/aws-sdk-dynamodb/1.120.0/src/aws_sdk_dynamodb/operation/get_item/_get_item_output.rs.html), [Smithy Blob source](https://docs.rs/aws-smithy-types/1.8.1/src/aws_smithy_types/blob.rs.html). +- [SDK logging controls](https://docs.aws.amazon.com/sdk-for-rust/latest/dg/logging.html). +- [Current SDK registry metadata](https://crates.io/api/v1/crates/aws-sdk-dynamodb). + +## Related + +- [Native store composition design](../specs/2026-10-07-native-store-providers-design.md). +- [Native provider implementation and publication record](2026-10-07-native-store-providers.md). +- [Native AWS guide](../../guide/native-aws-stores.md). +- [Owning self-hosted support epic #391](https://github.com/stackpop/edgezero/issues/391). diff --git a/docs/superpowers/plans/2026-10-15-native-celld-store-providers.md b/docs/superpowers/plans/2026-10-15-native-celld-store-providers.md new file mode 100644 index 000000000..c6a608ac6 --- /dev/null +++ b/docs/superpowers/plans/2026-10-15-native-celld-store-providers.md @@ -0,0 +1,500 @@ +# Native celld store providers implementation plan + +- Status: embedded celld and EdgeZero provider integration are implemented. Provider, independent-feature, startup and generated-consumer checks have executed, including verified native TLS and disposable namespace isolation. The final integration checkpoint below records current results. Fleet, deployment and stress acceptance remain open. Work checkboxes describe implementation, not production acceptance. +- Governing spec: [Native celld store providers](../specs/2026-10-15-native-celld-store-providers-design.md). +- EdgeZero baseline: `350527e4b7387a86da3f4cdaea0e8459665aa0f6`, with the spec introduced at `32737dd29b8f1fefd6e035be8036780e58b2412c` and the approved structural-only CLI clarification applied afterward. +- celld research baseline: `90b43017241f81189453d326d05948f388b34652`, version 0.6.2. Recheck the selected source and executable before running proofs. +- Approved scope clarification: offline CLI validation stays structural-only and reports target provider availability as unknown. Do not add a Cargo feature-graph resolver. +- Authority: the user requested committing the spec/plan and starting implementation, then explicitly selected implementation inside celld with no separate service. This authorizes scoped celld source changes and EdgeZero integration under this plan. No push, production deployment or real fleet/AWS access is authorized. + +## 1. Outcome and invariants + +A native application can select celld independently for each declared configuration or KV store ID. Configuration is an immutable startup snapshot. KV operations remain live. AWS, celld, local stores and supplied handles can coexist in the same native runner without changing portable application declarations or HTTP handlers. + +Required invariants: + +- Keep core `ConfigStore`, `KvStore`, handles, registries, bounded-read contracts and `#[async_trait(?Send)]` future bounds unchanged. +- Applications call a scoped service API using store aliases. They never receive fleet HMAC secrets, fleet-bucket discovery credentials, internal node addresses or arbitrary runtime scopes. +- Support explicit trusted-network and authenticated listeners. Both modes enforce store and operation grants before backend dispatch. Authentication failure never selects anonymous access. +- Config preparation finishes before the app initializer, readiness and listener acceptance. A missing mapped document or failed remote call never falls back to local data. +- Validate the complete source plan, selected credentials, settings and compiled capabilities before opening runner-owned providers or polling supplied deferred futures. +- A celld-only deployment needs no AWS credentials and no local roots for remotely bound IDs. Local defaults still require their existing roots. +- No automatic replay of mutations with an ambiguous outcome, at either client or service layer. A timeout does not mean that no write occurred. +- Errors owned by EdgeZero contain fixed categories and validated logical IDs, not credentials, physical namespaces, URLs, response bodies or arbitrary source chains. +- Default-generated applications, AWS-only builds and WASM adapters do not acquire celld runtime, V8 or Worker tooling dependencies. + +Out of scope: a separate gateway, sidecar or proxy service; a celld HTTP adapter, Worker bridge or application runtime; celld secrets; fleet provisioning; config publication; secret management; live config refresh; CAS, transactions or counters; dynamic provider registration; extraction of existing local stores; AWS redesign; process-wide memory guarantees; automatic fleet failover acceptance. + +## 2. Embedded v1 decisions and release gates + +The scoped API does not exist in the researched upstream celld version. The initial implementation now lives inside celld, not another service. Its listener schema, protocol, limits and restart-based policy adoption are recorded in the governing spec. The table below defines what must stay explicit and what needs release/acceptance evidence; provider settings must not bypass those remaining gates. + +| Decision | Required output | Constraint | +| --- | --- | --- | +| API placement | Resolved by the user: inside the existing celld process | No separately operated gateway, sidecar or proxy. Celld implementation worktree: `/home/pav/worktrees/feat-scoped-store-api`, branch `feat/scoped-store-api`, based on the researched pin. Initial implementation revision: `f72ae21a515859570622ac9d6ca9c19cf6870b81`. Release/upgrade acceptance remains open. | +| Listener policy | Exact configuration schema for mode, bind address, alias mapping and operation grants | Disabled by default; tokenless loopback default; explicit non-loopback opt-in and separately enforced network restrictions. No client-supplied caller identity. | +| Authenticated access | Token verifier format, secret injection, reload, overlap, revocation and restart policy | Random application bearer tokens over TLS. No issuance service or application fleet credentials. A documented bounded restart-based service policy is acceptable if hot reload is unnecessary. | +| HTTP protocol | Version, methods, status/error schema, TTL units/rounding, cursor shape, malformed-input and body rules | Missing differs from empty. No mutation replay. An incompatible version fails explicitly. The implemented v1 protocol is pinned to the celld revision above; release acceptance remains open. | +| Transport policy | Native TLS in the celld scoped listener, certificate validation, trusted-network HTTP opt-in and internal network protection | Bearer tokens require TLS. A private address is not proof of isolation or encryption. Reuse existing in-process owner dispatch rather than a local HTTP proxy. The application client and any retained celld peer transport must not redirect or discover ambient proxies. | +| Limits and compatibility | Concrete service/client limits, deadlines, preparation quotas and supported celld revision policy | Account separately for raw bytes, backend JSON byte-array/base64 encoding, decoded payloads, key pages, cursors, metadata and retained snapshots. No claim that the portable 25 MiB ceiling is always reachable. | +| Public settings | Provider tags, feature names, alias/endpoint/access settings and policy ownership | The implemented bindings and defaults are documented in the native celld guide; they remain unreleased. Settings contain references, not credentials. | + +Keep the remaining release and operational acceptance gates explicit. The v1 protocol and binding examples below are implemented preview contracts, not a released compatibility promise. API placement is settled. The scoped listener remains inside celld. + +## 3. Delivery order + +```mermaid +flowchart TD + Baseline[Stage 0: baseline and protocol decisions] --> Service[Stage 1: one scoped service implementation] + Service --> Proof[Stage 2: native transport and authorization proof] + Proof --> Settings[Stage 3: provider crate and typed settings] + Settings --> HTTP[Stage 4: bounded scoped HTTP client] + HTTP --> KV[Stage 5: live KV] + HTTP --> Config[Stage 6: config snapshots] + KV --> Compose[Stage 7: native source composition] + Config --> Compose + Compose --> CLI[Stage 8: generated features and structural CLI] + CLI --> Consumer[Stage 9: generated application acceptance] + Consumer --> Gates[Stage 10: docs and final checks] +``` + +The settings and client stages may use fake scoped-service fixtures after the contract is frozen, but only after Stage 2 proves the chosen real celld path is feasible. KV and config work can be researched independently. Keep shared manifests, lockfiles, settings, session state and native composition under one edit owner. Serialize Cargo commands that share a target directory. + +## 4. File map and proposed API shape + +Existing production files expected to change narrowly: + +| Area | Files | Change | +| --- | --- | --- | +| Workspace | `Cargo.toml`, `Cargo.lock` | Native provider member, workspace dependency and reviewed HTTP feature selection | +| Native features | `crates/edgezero-adapter-axum/Cargo.toml`, `src/lib.rs` | Settings-only dependency and independent `celld-config` / `celld-kv` forwarding | +| Bindings | `crates/edgezero-adapter-axum/src/native_bindings.rs` | Strict config/KV variants, capabilities and independent provider policy validation | +| Credential capture | `crates/edgezero-adapter-axum/src/run_options.rs` | Capture selected celld token names once without coupling them to `AgentAccessToken` | +| Preparation | `crates/edgezero-adapter-axum/src/native_stores.rs`, `src/dev_server.rs` | Typed provider dispatch, separate preparation sessions and existing staging/readiness path | +| Generated features | `crates/edgezero-adapter-axum/src/templates/Cargo.toml.hbs`, CLI generator tests | Default-off independent celld aliases, preserving AWS alias membership | +| CLI | `crates/edgezero-cli/src/config.rs` | Structural celld validation, availability unknown and remote config-push refusal | +| Dependency checks and CI | `scripts/check_native_store_dependency_contract.sh`, `.github/workflows/test.yml`, applicable format workflow | Isolated feature graphs, focused provider/startup/generated checks and explicit service-proof job | +| Documentation | New `docs/guide/native-celld-stores.md`; existing Axum, configuration, CLI and navigation pages where applicable | Deployment boundaries, bindings, limits, operator publication and proof limitations | + +New EdgeZero files: + +```text +crates/edgezero-store-celld/ + Cargo.toml + src/lib.rs + src/settings.rs + src/client.rs + src/preparation.rs + src/config.rs + src/kv.rs + +crates/edgezero-adapter-axum/tests/ + native_celld_startup.rs + fixtures/native_celld_host.rs + +crates/edgezero-cli/tests/ + celld_generated_proof.rs + +scripts/ + run_native_celld_integration.mjs + run_native_celld_integration.test.mjs +``` + +Keep ordinary unit tests colocated. Separate test modules only when fixture size warrants them. Add one shared test-support module if actual repeated fixture code needs it, not a public transport/provider framework. + +The scoped listener lives inside the celld repository, not in a new EdgeZero service package. Work in `/home/pav/worktrees/feat-scoped-store-api` on branch `feat/scoped-store-api`, based on celld `90b43017241f81189453d326d05948f388b34652`; keep the research checkout unchanged. Inspect celld's `main.rs`, `startup.rs`, `operator_cell.rs`, `kv_cli.rs`, `peer_auth.rs`, `crates/logic/kv.rs` and `js/services/kv.js`. Reuse existing owner dispatch inside the process after authorization. Keep pure policy decisions in `crates/logic` and effectful listener/credential/TLS handling in `crates/celld`. Do not depend on the full celld crate or copy its runtime into the EdgeZero application provider. + +Implemented provider interface: + +- `edgezero-store-celld` has no default features. Strict settings and redacted credential types remain available without the HTTP client or Tokio. Native-only `config` and `kv` features share one internal scoped HTTP implementation. +- Typed config settings map each local lookup key to a scoped store alias and remote key. The default key must name a mapping. Typed KV settings select one scoped store alias. Physical namespace mapping stays in service policy. +- A `CelldPreparation` session owns the aggregate config deadline, mapping/metadata and snapshot accounting, and compatible client reuse. Programmatic callers create and reuse an explicit session. Live KV contents are outside snapshot quotas. +- Factories return existing `ConfigStoreHandle` and `KvHandle` outputs. No new portable store trait, global provider registry or executor. +- Preserve `NativeStoreBindings::new(config, secrets)`, `with_kv`, `with_aws_limits`, existing AWS settings and ready/deferred override behavior. Add celld policy inputs without forcing existing callers through a new constructor. + +Implemented deployment binding schema: + +```toml +version = 1 + +[config.app] +provider = "celld-config" +endpoint = "http://127.0.0.1:7780" +default_key = "settings" + +[config.app.access] +mode = "trusted-network" + +[config.app.documents.settings] +store = "checkout-config" +key = "application-envelope" + +[kv.state] +provider = "celld-kv" +endpoint = "https://stores.example.internal" +store = "checkout-state" + +[kv.state.access] +mode = "authenticated" +token_env = "CHECKOUT_STORE_TOKEN" +``` + +The first alias has read-only grants; the second has the required KV operations. The example hostname is fictional. The service owns physical namespace selection, TLS and grants. An authenticated binding with a selected missing/empty token fails startup; it never becomes anonymous. Neither binding contains a fleet credential. Represent `access` as a validated tagged type, not an untyped field bag. + +## 5. Stage 0: record baseline and close the contract gate + +### Work + +- [x] Record HEAD, dirty files and governing spec hash. Preserve the current approved spec edits and unrelated work. Recheck local repository instructions. +- [x] Inventory the actual core store contracts, native source plan, startup staging, bootstrap token capture and generated feature tests. Confirm that offline validation normalizes controls but does not resolve target capabilities. +- [x] Verify the selected celld source, executable version, backend protocol and test tooling. Record whether the scoped API will need an external change or release. At the researched pin, operator values use byte arrays inline and `put-base64`/`valueEncoding` above 1 MiB; listing uses a raw last-key `after`, not a scoped opaque cursor; there is no native `exists` operation. Specify the service's bounded existence implementation and cursor wrapper rather than assuming those HTTP operations map one-to-one. +- [ ] Finalize the remaining listener/protocol decisions in section 2 in the governing spec. Placement is resolved: the scoped API runs in celld, with native TLS when authenticated. Include an operation/error table, limit table and deployment trust-boundary diagram. Record implementation files and resulting celld revision. +- [x] Define a fixture policy using isolated local storage, dummy application tokens, explicit ports and resource ownership. No ambient cloud credentials, production fleet discovery, internet-exposed listeners or real AWS acceptance. +- [x] Record a focused baseline for existing source validation, startup, CLI structure and dependency graphs before production edits. Classify pre-existing failures once; do not silently exclude them. + +### Exit gate + +An embedded celld implementation, a frozen first-version contract and a safe runnable local setup. The user has selected placement and authorized implementation. Contract research alone is not runtime proof. + +## 6. Stage 1: implement the scoped service boundary + +### Work + +- [x] Implement a disabled-by-default scoped listener inside the existing celld process. Reuse existing owner-routing effects, not a loopback HTTP bridge or separate binary. Keep alias/grant policy separate from dispatch, with no administrative proxy or arbitrary scope/address input. +- [x] Validate all listener policy at startup. Disabled means no listener. Enabled tokenless access defaults to loopback. Missing/invalid mode cannot select trusted-network access. +- [x] Resolve aliases and authorize the exact operation before constructing a backend request. Config aliases are read-only. Whole-namespace grants and dedicated config namespaces are sufficient for v1. +- [x] Implement authenticated credential verification, native TLS on the scoped listener and the defined replacement/revocation policy. Never log supplied tokens or verifier material. Trusted-network listeners grant the same configured permissions to every reachable caller. +- [x] Implement the approved versioned raw-byte value operations, existence check and bounded JSON key pages. Reject duplicate/malformed parameters and unsupported operations. Return a distinct missing response for reads and body-free existence responses. +- [ ] Keep discovery, fleet authentication, namespace-to-scope derivation and wire compatibility inside this boundary. Bound privileged encoded requests/responses independently of the public raw-byte budget, including inline byte-array expansion and the pinned base64 form. Reject over-cap writes before backend dispatch and over-cap responses before unbounded decoding, including encoding paths excluded by approved value caps. +- [x] Dispatch authorized operations through celld's existing in-process owner path. Do not add external operator discovery, a local HTTP client or a new privileged service where existing dispatch suffices. Audit any reused peer transport for redirect/proxy behavior and bounded parsing. The pinned external `operator_cell::Reachable` helper is research material, not the implementation to copy. +- [x] Define one retry owner. Permit only explicitly proven pre-dispatch refusals if the approved contract needs them; never replay an ambiguous mutation or add nested client/service retry loops. +- [ ] Apply finite request deadlines and body, page, cursor and concurrency limits. Bound work and memory under slow callers and disconnects. Do not claim dropping a request undoes an applied backend write. + +### Tests and exit gate + +Drive public handlers with an instrumented backend. Prove zero backend calls for out-of-grant aliases, config writes, malformed input and denied operations in both modes. Prove credential isolation, missing/revoked-token rejection, no auth fallback, disabled/loopback defaults, remote-tokenless opt-in rejection, body/page limits, deadline termination and redacted error responses. Test both backend encodings and their encoded/decoded budgets, or explicit over-cap rejection where a path is unsupported. Verify authorized calls reach only the existing owner-dispatch path, and audit/test redirect/proxy restrictions on any network hop that path uses. Test the actual chosen service implementation, not a second policy implementation in EdgeZero fixtures. + +Before moving on, document how application workloads are prevented from reaching celld's internal listener or reading fleet credentials. An HTTP authorization test does not establish deployment network isolation. + +## 7. Stage 2: prove scoped native access against real celld + +### Work + +- [ ] Start one disposable local fleet and the chosen service using the pinned executable/revision. Record commands, environment, acquisition and storage of fixture-only fleet credentials, listener addresses and cleanup. +- [x] Use a small native client without an application Worker. Through the service, exercise binary and empty put/get, missing reads, exists, overwrite, idempotent delete, prefix pages and two isolated stores. +- [x] Exercise trusted-network loopback and authenticated TLS with valid certificate verification. Use a test CA through test setup, never a production `accept_invalid_certs` switch. +- [ ] Drive incompatible protocol versions, invalid tokens, out-of-grant stores and config write denial. Observe backend dispatch counts for authorization denials. +- [x] Verify real operator-wire TTL set, renewal and plain-put reset, and document how fractional `Duration` values will map to the approved wire units without shortening lifetime. Record the value encodings and size range exercised; a small-value result does not verify the bucket-backed/base64 path. +- [x] Exercise a listing larger than one page and record ordering and continuation behavior. Preserve empty/partial pages where the backend can return them; deterministic malformed-page cases belong in later fixtures. +- [ ] Demonstrate network/credential isolation using the actual deployment arrangement selected for the proof. Host loopback success alone does not establish container or multi-host isolation. + +### Exit gate + +Retain a rerunnable command or script and explicit observed results for transport and authorization. The client has no fleet credentials and sends no internal runtime requests. A missing executable, failed fleet startup, absent scoped API or skipped authenticated case leaves this gate unverified. Do not proceed on the strength of source inspection alone. + +## 8. Stage 3: add the native provider crate and strict settings + +### Work + +- [x] Add the workspace member/dependency with independent native-only `config` and `kv` features and no defaults. Keep settings usable in CLI-only builds without service clients, Tokio or celld runtime code. +- [x] Use existing workspace dependency versions where compatible. Review `reqwest` feature unification, since workspace defaults already include blocking/JSON support. Enable only necessary provider features and record the resolved graph rather than assuming manifest defaults prove isolation. +- [x] Implement strict endpoint, access, alias, document and limit settings. Reject unknown fields, invalid combinations, empty mappings, a missing default mapping, malformed URLs, userinfo, queries/fragments and unsupported endpoint paths. +- [x] Normalize caller input once into typed internal settings. Preserve exact remote keys and alias case; do not Unicode-normalize or reinterpret them as runtime scopes. +- [x] Validate finite positive limits, checked arithmetic and relationships among encoded/decoded/body/page and session budgets. Keep service limits and application/provider limits distinct. +- [x] Add a redacted celld token wrapper separate from AWS `AgentAccessToken`. A selected token source must resolve to a valid nonempty header-safe value. No ambient lookup inside the provider. +- [x] Define closed preparation/client error categories and fixed mappings into current portable errors. Do not add provider codes or source chains to core. + +### Tests and exit gate + +Settings-only tests cover every rejected shape, URL encoding boundary, token Debug redaction, case sensitivity, limit overflow and access-mode combination without network calls or ambient variables. Independently check settings-only, config-only and KV-only dependency graphs. No Axum, CLI, AWS, full celld, V8, `workers-rs` or `worker-build` dependency enters the provider graph. + +## 9. Stage 4: implement the bounded scoped HTTP client + +### Work + +- [x] Construct requests using an encoding library and the frozen contract. Test empty prefixes, Unicode, reserved delimiters and percent encoding without allowing path/query injection or double decoding. +- [x] Disable redirects, ambient proxies and implicit mutation replay. Set finite connect and whole-operation deadlines. Preserve normal application logging; do not install a logger or subscriber. +- [x] Require TLS for bearer credentials and validate certificates. Enforce the explicit trusted-network HTTP policy selected at Stage 0. Do not infer permission from a private hostname/IP. +- [x] Stream and cap response bodies before accumulation. Request identity encoding and either reject non-identity encoding initially or implement separately budgeted wire/decoded limits under an approved contract. Bound error bodies too. +- [x] Apply endpoint, header, request-value, decoded-value, page-entry, key-byte and cursor bounds. A bounded value count alone does not bound page JSON bytes or nesting. +- [x] Distinguish valid empty values from misses and protocol failures. Validate success/error status combinations and response shapes. Unsupported versions, malformed pages and auth failures must not become misses. +- [x] Reuse compatible clients by normalized endpoint and complete transport/authorization policy. Never reuse one credential's authorized state for another. Keep secrets out of cache-key Debug and diagnostics. +- [x] Close errors at the transport boundary. Document dependency DEBUG/TRACE limitations separately if testing finds them; do not promise universal redaction from sanitizing owned errors alone. + +### Tests and exit gate + +Exercise the actual HTTP client against loopback HTTP/TLS fixtures. Cover exact key bytes, binary/empty bodies, all operation/error categories, bad versions, slow headers/body, chunked bodies, oversized declared/streamed bodies, compression refusal, malformed JSON, page/cursor limits, disconnects and certificate errors. Have a fixture apply a PUT/DELETE and then lose the reply; assert one request, a closed error and the retained effect. Add redirect and proxy sentinels proving no alternate destination receives credentials or mutations. + +## 10. Stage 5: implement live KV + +### Work + +- [x] Implement existing `KvStore` get, put, TTL put, delete, exists and paginated prefix listing over the scoped client. Return existing `Bytes`/`KvPage` types; do not expose transport models. +- [x] Enforce actual provider/service limits in addition to `KvHandle` validation. Smaller configured/provider limits fail explicitly before requests, without truncation or chunking. +- [x] Preserve binary bytes, misses and empty values. Missing delete succeeds. Plain overwrite clears prior expiry; TTL overwrite renews it under verified celld semantics. +- [x] Validate and encode TTL with checked arithmetic and the approved rounding rule. Accept the public handle's 60-second minimum and reject unsupported direct-trait durations rather than silently clamping them. +- [x] Preserve bytewise lexicographic prefix order. Bind client continuation state to the configured endpoint/store/prefix and wire version as required; reauthorize every service page. A cursor never authorizes a different store or prefix. +- [x] Return one bounded page per operation, including short or empty continuation pages. Validate malformed, oversized, out-of-prefix, out-of-order or non-progressing continuation responses. Do not scan indefinitely to fill a page. +- [x] Create fresh runtime operation deadlines per call. Do not reuse an expired startup-preparation deadline for a live handle. + +### Tests and exit gate + +Run non-TTL portable contracts against the real provider using its scoped HTTP fixture and a native Tokio runtime. A current-thread runtime/`LocalSet` can drive the existing non-`Send` futures; `futures::executor::block_on` alone cannot drive Tokio HTTP/timers. The shared full contract macro contains direct sub-second TTL cases that celld rejects, so do not invoke it unchanged or change core semantics to fit celld. Add provider-specific TTL tests for valid durations, minimum rejection, overflow, rounding, renewal, expiry and reset. Cover namespace isolation, explicit shared aliases, NFC/NFD distinction, 512-byte keys, binary/empty values, page scope/progress and one-attempt ambiguous writes. Distinguish fixture expiry proof from the real 60-second expiry check in Stage 9. + +## 11. Stage 6: implement immutable config preparation + +### Work + +- [x] Fetch every explicitly mapped document during one bounded `CelldPreparation` session, sequentially initially. Require all mapped documents; an unmapped local lookup key remains a successful miss. +- [x] Validate UTF-8 without parsing, reserializing, projecting JSON fields or resolving secrets. Preserve whitespace, empty documents and exact complete `BlobEnvelope` text. +- [x] Account for each mapping and its metadata before deduplication. Deduplicate successful fetches only for identical endpoint, alias, remote key and full access/transport policy. Count shared immutable payload allocations once. +- [x] Enforce per-document transfer/value limits, total retained snapshot bytes and an aggregate preparation deadline across all runner-owned celld config IDs. Bound transient buffers too, without describing these counters as an RSS guarantee. +- [x] On any failure drop staged state and publish no partial config registry. Never reuse a prior process snapshot or add a stale/local fallback. +- [x] Implement both ordinary and bounded core reads over immutable `ConfigValue`. Use the existing bounded-read helper where appropriate: deadline precedence, independent backend/value allowances, correct payload-byte accounting and zero bytes for a miss. +- [x] Keep typed envelope validation and nested secret resolution in core/app startup. A celld document may reference an independently bound AWS secret. Multiple documents/providers are not an atomic release. + +### Tests and exit gate + +Cover complete document preservation, empty/raw/typed documents, invalid UTF-8, missing mapped documents, unmapped keys, duplicate-target sharing versus credential isolation, mapping/metadata/per-value/session cap boundaries, zero/expired deadlines, failure cleanup and bounded snapshot reads. Prove preparation reads once and later `get` calls perform no network I/O. A remote edit remains invisible to a running snapshot and is adopted by a fresh preparation. + +## 12. Stage 7: integrate existing native composition + +### Work + +- [x] Add typed celld config/KV variants and independent `NativeProviderCapabilities` fields. Replace current kind-map nonemptiness checks with actual-variant capability checks, so celld config/KV do not require AppConfig/DynamoDB features. Reject celld under secrets, unknown fields/tags and undeclared or conflicting sources. +- [x] Separate AWS and celld policy tables and mapping counters. Existing `[limits.aws]` documents and explicit `with_aws_limits` behavior remain compatible; celld-only policy must not require an AWS table, inherit AWS quotas or create an AWS session. A file celld policy must not conflict with an explicit AWS policy, or vice versa. Preserve existing public parse/read/validate signatures where possible through additive methods. Add only the celld policy inputs required by the approved budgets. +- [x] Replace the AWS-only config-source tuple with provider-specific typed source payloads carrying captured credentials. KV sources also need captured celld credentials. Do not reuse AWS token types or put opaque futures around session-borrowing built-in preparation. +- [x] Extend the existing bootstrap capture boundary to selected token names in celld config and KV bindings. `from_env` captures once; explicit options use only `with_bootstrap_values` or other supplied values. Missing, empty, duplicate and non-Unicode selected values fail before preparation. Unrelated ambient variables stay irrelevant. +- [x] Select preparation sessions by actual provider variants, not `BuiltIn(_)`. Create one celld session for selected celld bindings and the existing AWS session only for selected AWS bindings. Keep their state/budgets separate. +- [x] Extend the existing per-kind staging loops in `build_production_stores`. Preserve original defaults, shared handles, app initializer timing, readiness, failure cleanup and local root selection. +- [x] Register `fixtures/native_celld_host.rs` as a feature-gated `[[bin]]` and `native_celld_startup.rs` as the corresponding `[[test]]` in the Axum manifest, following the existing AWS fixture pattern. Verify all-target builds still work with celld features off. +- [x] Keep duplicate-source rejection and all-source validation before local opens, remote setup or supplied future polling. Existing ready handles remain caller-owned; the runner cannot undo their prior side effects. + +### Tests and exit gate + +Drive the public native runner in subprocess fixtures where environment capture matters. Cover local-only, AWS-only, celld-only and mixed providers across multiple IDs, plus ready and non-`Send` deferred inputs. Table-test actual-variant capabilities with celld-only/AWS-disabled, AWS-only/celld-disabled and each missing provider in mixed bindings. Separately prove mapping counters and policy-source conflicts are scoped to their provider. Count local opens, remote calls and deferred polls on invalid settings/capabilities/sources. Prove zero preparation on validation failure, no local fallback, no initializer/listener on failed setup, unchanged defaults and local roots only where needed. Include different config/KV tokens, one token used by both, explicit missing tokens and celld-only startup with ambient AWS discovery disabled. Reuse existing AWS regressions unchanged unless the changed dispatch requires an additive case. + +## 13. Stage 8: generated features and structural-only CLI + +### Work + +- [x] Forward independent default-off `celld-config` and `celld-kv` features from Axum and generated native target manifests. Do not expand `aws-stores` or add an all-provider alias without a concrete need. +- [x] Add only settings support to CLI/native-bindings dependencies. The CLI need not compile live celld clients merely to parse deployment files. +- [x] Extend existing structural bindings validation to celld syntax, policy, declared IDs and conflicts. Continue reporting target provider availability as unknown regardless of requested feature names. Do not resolve Cargo metadata, probe endpoints, fetch tokens or validate remote authorization/content. +- [x] Reuse current Cargo argument normalization and forwarding unchanged where possible. Do not add a parallel resolver or rewrite arbitrary custom shell commands. +- [x] Generalize the config-push guard by selected remote config variant so both AWS-bound and celld-bound config IDs reject unsupported publication before writing local files. Keep local-bound IDs' current behavior. +- [x] Update generator/parser assertions and CLI messages, preserving existing command arguments and AWS behavior. Check `crates/edgezero-cli/src/args.rs` help text for inherited claims of target-feature resolution and describe normalization/forwarding without an availability verdict. + +### Tests and exit gate + +Test valid/invalid celld deployment documents through public `config validate`, including absent tokens, an unreachable endpoint and disabled target provider features. A structurally valid document still reports availability unknown and performs zero remote/token lookup. Verify feature controls are normalized/forwarded, not interpreted as a capability verdict. Prove selected remote config push fails without local side effects and local config push still works. Compile generated targets independently with neither feature, each feature alone, both and combined AWS features; startup, not offline CLI validation, rejects an uncompiled selected provider. + +## 14. Stage 9: generated consumer and real-service acceptance + +### Work + +- [x] Generate an actual application using the existing CLI scaffold. Use portable logical declarations and handlers/extractors, the production native runner and a normal typed app initializer. Do not substitute a hand-written adapter host for this proof. +- [ ] First run against the deterministic scoped fixture to prove default-off rejection before I/O, feature forwarding, source conflicts, root handling, closed failures and CLI behavior. +- [ ] Then run the same generated native target against the chosen scoped service and real disposable celld. Test trusted-network loopback and authenticated TLS. Keep all fleet credentials in service/fleet setup, not the app's environment or filesystem view. +- [ ] Through normal application endpoints, observe binary/empty KV, put/get/delete/exists/list, multi-page order, two-store isolation and intentional sharing. Restart the app against the same fleet to prove persistence is not process-local state. +- [ ] Exercise a valid real TTL for at least the service's 60-second minimum, renewal and no-TTL reset. Use a bounded opt-in slow test and finite polling, not sub-second durations or a fixture clock presented as celld evidence. +- [ ] Publish a new complete config document using operator-owned test setup. The running app keeps its initialized snapshot; a fresh process adopts the new version. Missing, oversized, invalid UTF-8 or invalid typed config fails before readiness as applicable. +- [ ] Where installed fixtures permit it, compose celld config/KV with AWS secret fixtures and local bindings through the same generated app. Mark missing fixture coverage as unverified; do not obtain real AWS credentials to complete it. +- [ ] Verify network isolation and credential-file denial in the actual tested deployment arrangement. Retain authorization dispatch observations and fixture resource identities. + +### Harness and exit gate + +Add one explicit local-service harness using the repository's existing Node test-harness style if it reduces repeated setup. It must check tool/version availability, use isolated state and dummy/test credentials, clear ambient deployment credentials, set startup/operation/shutdown deadlines, reap owned process trees and clean up in a `finally` path. It must never kill an arbitrary listener or remove non-fixture storage. Unit-test orchestration and failure cleanup without launching real services. + +The real-service command must fail or report an explicit unverified gate when prerequisites are missing, not exit successfully after skipping all assertions. Default unit tests stay deterministic and require no external services or credentials. Run real-service acceptance explicitly in a prepared job or with operator permission. Record exact command, app/service/celld revisions, tested features, outcome, elapsed time and redacted log path. Single-node proof does not establish fleet failover, crash recovery, encrypted overlays, large-value bucket operation, upgrades or performance. + +## 15. Stage 10: documentation and stable integration checks + +### Work + +- [ ] Document the required celld build/version, embedded listener configuration, application bindings, operator-owned aliases/grants and both access modes. Include TLS and network restrictions, token injection, replacement/revocation, fleet credential separation and safe examples. +- [ ] Explain startup-only config adoption, non-atomic multi-document releases, optional independent secret providers, service/provider limits, TTL constraints, empty continuation pages, non-snapshot listing and ambiguous mutation outcomes. +- [ ] Explain that offline CLI validation is structural-only, startup checks compiled capabilities, and config push does not publish remote config. Keep resource provisioning/publication instructions operator-owned. +- [ ] Extend isolated dependency checks and focused CI commands. Use the nonzero-test wrapper for new ignored/generated suites so a mistaken test filter cannot produce a passing zero-test job. +- [ ] Add the real-service job only after the chosen service artifact and safe local setup are pinned and available. Keep it separate from default no-service unit suites. +- [ ] Inspect the combined diff, rerun affected gates at a stable checkpoint and record remaining blockers without changing unrelated work. Do not commit, push or deploy without a separate request. + +### Planned commands + +These are future checks, not results. New feature/test names below are proposed by this plan. Finalize them with the files they exercise. Use `--offline --locked` once the reviewed dependency graph is fetched and locked; dependency acquisition itself needs an explicit prepared environment. + +```sh +# Focused provider units after their corresponding stages. +cargo test --offline --locked -p edgezero-store-celld --no-default-features --lib +cargo test --offline --locked -p edgezero-store-celld --no-default-features --features config --lib +cargo test --offline --locked -p edgezero-store-celld --no-default-features --features kv --lib +cargo test --offline --locked -p edgezero-store-celld --all-features --lib + +# Native and CLI integration, serialized when using the same Cargo target directory. +cargo test --offline --locked -p edgezero-adapter-axum --no-default-features --features axum,celld-config,celld-kv --lib +cargo test --offline --locked -p edgezero-adapter-axum --features celld-config,celld-kv,test-utils --test native_celld_startup +cargo test --offline --locked -p edgezero-adapter-axum --features aws-stores,aws-dynamodb,celld-config,celld-kv --lib +cargo test --offline --locked -p edgezero-cli --lib +scripts/run_test_nonzero.sh --ignored generated_celld_default_off_kv_and_config_restart cargo test --offline --locked -p edgezero-cli --no-default-features --features cli,edgezero-adapter-axum --test celld_generated_proof generated_celld_default_off_kv_and_config_restart + +# Existing AWS regressions and isolated dependency contracts. +cargo test --offline --locked -p edgezero-store-aws --all-features --lib +cargo test --offline --locked -p edgezero-adapter-axum --features aws-stores,test-utils --test native_aws_startup +bash scripts/check_native_store_dependency_contract.sh + +# Stable combined checkpoint and repository-required gates. +cargo fmt --all -- --check +cargo clippy --workspace --all-targets --all-features -- -D warnings +cargo test --workspace --all-targets +cargo check --workspace --all-targets --features "fastly cloudflare spin" +cargo check -p edgezero-adapter-spin --target wasm32-wasip2 --features spin + +# Explicit local-service harness once implemented and its prerequisites are prepared. +node --test scripts/run_native_celld_integration.test.mjs +CELLD_BIN=/absolute/path/to/modified/celld node scripts/run_native_celld_integration.mjs +``` + +Also run independent no-default/config-only/KV-only Axum/provider lint and compile scopes, existing generated AWS/DynamoDB proofs affected by shared dispatch/template changes, and selected Fastly/Cloudflare/Spin WASM graph/build checks. Extend `scripts/check_adapter_feature_matrix.sh` or add explicit provider combinations; its existing adapter/CLI/test-utils matrix does not cover celld feature isolation. For published docs changes run `npm run lint`, `npm run format` and `npm run build` from `docs/` with dependencies already prepared. Do not infer formatting coverage for process documents excluded by `docs/.prettierignore`. + +Developer/CI dependency-graph audits do not add a capability resolver to the CLI. The dependency script must assert separate normal dependency graphs for settings-only CLI, new provider settings/config/KV modes, default native targets, each WASM adapter and AWS-only builds. Also inspect selected provider-root feature edges: default Axum already uses HTTP/Tokio, and CLI/default native builds may include celld settings types, so package presence alone cannot prove live-provider activation. Check both directions: celld must not add AWS service SDKs, and default/AWS-only graphs must not activate celld live transport features. Assert config-only versus KV-only feature separation and no full celld runtime, V8 or Worker tooling. Workspace `--all-features` alone cannot prove isolation. + +## 16. Evidence, stopping rules and handoff + +For each behavioral unit record `check | tested revision or input fingerprint | result | elapsed time | log`. Reuse passing evidence while its relevant source, fixture, dependency and environment inputs remain unchanged. Run narrow checks during editing, neighboring checks at each completed unit, and broader suites at the stable combined checkpoint. + +The two highest-risk facts need direct executed evidence: + +1. Out-of-grant access and config writes never reach the privileged backend in either mode. Handler source and policy design alone are insufficient. Instrument service dispatch, then exercise the real scoped service under the tested network boundary. +2. The native transport preserves values/expiry and never replays an ambiguous mutation. Inspect pinned wire handling, execute client fault fixtures and drive a generated app through the actual service/celld path. State exactly which claims each level proves. + +Stop and revise the plan if safe scoped access cannot be delivered without app fleet authority, a full celld runtime dependency, a Worker bridge, a portable core contract change or an additional unapproved service. Do not conceal that change behind custom overrides or a test-only transport. + +Handoff must include changed files, concrete protocol decisions, service/provider versions, completed stages, exact checks and results, known baseline failures, intentionally unexecuted acceptance cases and remaining operational risks. No stage is complete merely because a package compiles or a service starts. + +## Stage 0 execution record + +The user requested committing the spec/plan and starting implementation. Commit `a12265b52770c50ca7c5384bd1037037b892b0ea` contains both documentation changes. The checkout was clean immediately afterward; no production source changed. Governing spec blob: `6c84a41957fcb56047b2259eb6dd79fc3fd6beb6`. + +Executed against the unchanged production source at that revision: + +| Command | Outcome | +| --- | --- | +| `cargo test --offline --locked -p edgezero-adapter-axum --no-default-features --features axum --lib native_bindings::tests` | 11 passed; 35.9 seconds including initial compilation | +| `cargo test --offline --locked -p edgezero-adapter-axum --no-default-features --features axum --lib` | 238 passed; native startup, source validation, local stores and bootstrap included | +| `cargo test --offline --locked -p edgezero-cli --no-default-features --features cli,edgezero-adapter-axum --lib bindings_only` | 2 passed; structural validation without app TOML and shared validation/push errors | +| `bash scripts/check_native_store_dependency_contract.sh` | Passed existing isolated AWS/native/WASM dependency assertions; no celld implementation exists yet | + +The last three commands ran sequentially in 20.3 seconds total. Full command output is retained in the implementation conversation. No baseline failures occurred in these scopes. Broader all-provider/generated/runtime suites have not run for this task. + +The local celld research checkout is clean at the required pin. `celld` is not on PATH and no executable was found in that checkout's usual debug/release locations or the user Cargo/local bin locations. Docker is available, but no daemon/image/service check, image pull, fleet setup or credential access was performed. Source availability does not satisfy the real-service gate. + +The user subsequently resolved placement: implement inside celld with no separate gateway/service. An isolated celld worktree now exists at `/home/pav/worktrees/feat-scoped-store-api`, branch `feat/scoped-store-api`, based on the researched pin. The first embedded implementation and local proof are recorded below. No production deployment, real fleet/AWS access or additional service was used. + +## Embedded celld implementation checkpoint + +The initial celld implementation is committed as `f72ae21a515859570622ac9d6ca9c19cf6870b81` on `feat/scoped-store-api`; the celld worktree was clean. At this checkpoint, EdgeZero changes after `a12265b5` were documentation-only; the later provider checkpoint follows below. Do not label the initial listener as a released celld capability. The clean celld repository was moved to `/home/pav/Projects/celld` and its worktree metadata repaired so the implementation branch does not depend on temporary Git storage; `/tmp/edgezero-celld-research` remains a compatibility symlink with unchanged source contents. + +Implemented files: + +- `crates/logic/scoped_store.rs`, its explicit integration test and module/manifest registration: std-only exact alias/principal/operation policy, validated once, actually called by the HTTP handler. +- `crates/celld/scoped_store.rs` and library export: strict capped JSON policy capture, token verifier checks, native TLS configuration, closed errors, HTTP operations, binary values and scope-bound key-page cursors. +- `crates/celld/main/store_api.rs`, `main.rs` and `main/cli.rs`: disabled-by-default third listener, startup file selection, admission accounting, existing owner dispatch/output gates, body/operation/connection budgets and shutdown socket closure. +- `crates/celld/generation.rs`: required reserved KV registration during enabled generation construction/reload, plus local built-in KV-only bootstrap without application source. The existing generation host requires a default fetch; the built-in 404 is not a KV bridge and scoped traffic never passes through it. +- `crates/celld/main/peer_tunnel.rs`: explicit no-replay policy, preserving legacy behavior for existing callers. The regression applies a pooled request then loses its reply and proves no fresh attempt in the scoped mode. Outer dispatch also stops after a failed scoped attempt. +- `crates/celld/tests/scoped_store_startup.rs`: real local binary, cleared environment, isolated storage and owned child processes. Explicit TLS proof uses disposable certificate material; slow expiry proof uses actual elapsed time. +- `docs/services/scoped-stores.md` and `docs/security.md`: embedded deployment/configuration and protocol, with remaining acceptance limits disclosed. + +Two read-only investigations identified the peer-pool replay hazard and missing reserved-runtime registration before edits. The coordinator inspected their evidence. Zellij follow-up delivery failed, so worker assignments were not treated as acknowledged transfers; their processes were released and the coordinator completed the coupled implementation locally, preserving and inspecting the partial HTTP/config scaffold. + +Build preparation fetched the exact locked dependencies without changing the lockfile. The initial offline check had failed because the checkout lacked cached packages. All celld checks use installed Rust 1.98.1, not the older global default. + +Executed checks: + +```sh +cargo test --offline --locked -p celld-logic --test scoped_store +cargo test --offline --locked -p celld --all-targets +cargo test --offline --locked -p celld --test scoped_store_startup -- --ignored --nocapture +cargo clippy --offline --locked -p celld -p celld-logic --all-targets -- -D warnings +cargo fmt --all -- --check +``` + +The proof inventory is six pure-policy tests, seven actual HTTP-handler/config tests, one pooled replay regression, one ordinary real-binary test, one explicit native TLS test and one explicit real TTL test. All 17 passed. The two ignored acceptance tests were explicitly selected and both passed; they were not counted as passing skips. Strict all-target Clippy and workspace formatting checks passed. The final native/test/lint/format batch took 84.7 seconds, including the 61.4-second real expiry proof. Tested code fingerprint: `e909661efdf160a14ec8f62ffc224ff5fb590d921fc1dd635140dd08585ed55b`, covering 182 tracked/untracked workspace source/manifests. It is SHA-256 of compact JSON containing sorted pairs of repository-relative path and file SHA-256 from `git ls-files --cached --others --exclude-standard crates Cargo.toml Cargo.lock`; documentation-only changes do not alter it. The TLS certificate fixture initially failed certificate validation because it was marked as a CA; the corrected fixture is an end-entity certificate and normal certificate checking remains enabled. An initial KV-only generation without a default fetch also failed; the engine's default-fetch requirement is now explicit in the built-in bootstrap rather than hidden in an application script. The 61-second expiry test observes expiry, renewal beyond the original expiry and plain overwrite removing expiry. + +Remaining gates: deployment-level network/credential isolation, multi-node owner forwarding and reload acceptance, production persistence/replication and fault/shutdown/body-budget stress. The local standalone path emits `local reload requires replication`; it has not been claimed as a fleet recovery proof or silently suppressed. Celld's private conformance/simulation harness is not available in this checkout and has not run. No larger-value bucket, failover, upgrade or performance acceptance is claimed. Complete remaining scoped-access gates before treating Stage 2 as fully passed or adding stable EdgeZero provider tags. + +## EdgeZero provider integration checkpoint + +Implemented, still unreleased: + +- `edgezero-store-celld` with settings-only availability, independent `config`/`kv` features, captured redacted tokens, bounded reqwest transport, immutable config snapshots and live KV. Public core traits and non-`Send` futures are unchanged. +- Per-variant native capabilities, typed config/KV credentials, separate AWS/celld preparation sessions and policies, provider-specific accounting and complete pre-I/O validation. Existing constructors, AWS policy methods, local roots, supplied handles and startup staging remain. +- Default-off Axum/generated feature forwarding, structural-only CLI validation and selected remote config-push refusal. Provider availability remains unknown offline. +- Public-runner and generated-consumer fixtures, TLS certificate and proxy tests, no-replay fault cases, dependency isolation checks and CI wiring. The new [native celld guide](../../guide/native-celld-stores.md) documents deployment constraints. +- The mixed public-runner regression loads celld config, resolves an AWS Secrets Manager reference through the real pinned SDK and serves a separate local environment secret. Its celld document exceeds the independent 128-byte AWS snapshot quota. An unrelated Agent token must not enter the trusted celld request. + +The coordinator reviewed the workers' source-only handoffs, registered the test binaries and owned all Cargo commands. Worker formatting was never counted as compilation or behavior proof. Initial type/lint failures were fixed before the results below. One cancellation test initially cancelled before HTTP admission; it now observes an admitted stalled request before dropping the future. Config-document and live-KV value caps are independent and each retains the service ceiling. + +### Executed checks + +EdgeZero uses Rust/Cargo 1.95.0. All Cargo checks below used `--offline --locked` after the coordinator updated the lockfile offline. The lockfile adds only the new workspace package and dependency edges; existing registry versions did not move. Fixture TLS server dependencies are native dev dependencies. + +| Check | Result | Runtime / log | +| --- | --- | --- | +| Provider settings-only / config-only / KV-only / combined units | 4 / 27 / 30 / 38 passed | Runtime suites about 2.9 seconds each; `/tmp/edgezero-celld-provider*.log` | +| Independent provider feature Clippy, all targets | Passed in each mode | `/tmp/edgezero-celld-clippy-{settings,config,kv,provider}.log` | +| Axum native-only / config-only / KV-only units and lib Clippy | 250 passed in each mode; Clippy passed | `/tmp/edgezero-celld-native-{axum,celld-config,celld-kv}.log` | +| Combined AWS/celld native units | 253 passed | `/tmp/edgezero-celld-combined.log` | +| Public celld startup subprocesses | 4 passed | `/tmp/edgezero-celld-native.log`, repeated in combined run | +| AWS startup and mixed celld/AWS/local subprocesses | 5 AWS-only and 6 combined passed | 0.30 seconds runtime each; `/tmp/edgezero-celld-{aws-only,mixed-startup}-final.log` | +| Default CLI library | 233 passed | 0.12 seconds runtime; `/tmp/edgezero-celld-cli-default.log` | +| Native dependency and provider-root feature isolation | Passed | `/tmp/edgezero-celld-dependencies.log` | +| Generated celld consumer, five feature combinations | Passed; one explicitly selected ignored test | 16.40 seconds; `/tmp/edgezero-celld-generated-combined-final.log` | +| Real embedded celld, generated native TLS consumer and namespace isolation | Passed; one explicitly selected ignored test, including 61-second expiry | 64.48 seconds consumer runtime; `/tmp/edgezero-celld-real-final.log` | +| Existing generated AWS / DynamoDB fixtures | Both passed, one selected test each | 36.83 / 14.63 seconds runtime; `/tmp/edgezero-celld-{aws,dynamodb}-generated-regression.log` | +| Provider integration runner Node tests | 24 passed, including 9 celld tests | 1.19 seconds; `/tmp/edgezero-celld-node-all.log` | +| Workspace formatting and all-target/all-feature Clippy | Passed | Clippy 47.41 seconds; `/tmp/edgezero-celld-{fmt,clippy-workspace}.log` | +| Workspace all-target tests | 2,028 passed, 6 ignored across 34 targets | `/tmp/edgezero-celld-workspace-tests-final.log` | +| Workspace all-target check with Fastly/Cloudflare/Spin, then Spin WASI P2 check | Both passed | 16.55 / 19.53 seconds; `/tmp/edgezero-celld-{workspace-check,wasm-check}.log` | +| Docs offline dependency preparation, lint, format and site build | Passed | `/tmp/edgezero-celld-docs-*.log`; process specs/plans remain outside the site formatter's match set | +| Shell syntax and `git diff --check` | Passed | Both new/extended shell scripts checked | + +The first whole-workspace test attempt ran in a Zellij TTY and blocked in the existing non-TTY consent test. Its stdin was confirmed as `/dev/pts/2`, and the test source explicitly assumes noninteractive CI. Only that owned test process and its waiting scheduler were stopped. The rerun used `exec `. The portable trait itself has no `Send`/`Sync` supertraits and no future. | +| Compatibility | Preserve `AxumRunOptions: Clone + Send + Sync`, `App: Send + Sync`, existing initializer signatures and local/non-`Send` startup futures. Do not add a provider factory to support non-`Send` native provider objects. | +| Selection | One `NativeGeoSelection` value: `Off`, `CloudFront { policy }`, or `Supplied { policy, id, provider }`. Repeated consuming `with_geo` calls explicitly replace the whole selection before startup; there is no merge or precedence between independent inputs. | +| Capability inspection | `AxumRunOptions::geo_capability()` returns a validated, owned description. The caller captures it from finalized options in the existing initializer closure. No new `App` capability field, state registry or `PreparedStores` field. | +| Trust | One opaque `ProtectedCloudFrontAlb` policy, distinct from ordinary proxy trust. Source admission uses the actual connection peer and an adapter-private request-bound witness. | +| CIDRs | Extract and reuse #409's internal bounded network set and matcher. No second parser and no dummy forwarding-family policy. | +| Request storage | Private outcome field in `AdmittedIngress`, transferred to a private `RequestContext` field. No import from public extensions and no live setter. | +| Cross-crate handoff | One explicit `App::dispatch_admitted_with_geo` operation consumes an owned outcome alongside the existing admission token. Existing dispatch APIs supply `NoSelection`. | +| Geographic validation | Checked-in ISO country-code table with source attribution and pinned snapshot; syntax-only subdivision validation. No new runtime dependency or subdivision catalogue. | +| Request preparers | Neither the inspected native baseline nor pinned #409 has a request-preparer callback. Do not add one. Preserve existing Fastly preparer APIs and test that their scratch extensions cannot supply Geo. | + +The native-only provider bounds preserve existing options auto-traits. Portable provider implementations may still use `Rc` or `Cell`; they simply cannot be supplied to the native options API without satisfying its bounds. No application handler, startup future or portable interface gains a new future `Send` requirement. + +The core dispatch operation is a trusted Rust integration API, not proof of origin authentication. Rust code can deliberately start another core dispatch with an explicit outcome. Managed Axum never obtains its outcome from such caller-controlled metadata. The adapter-private witness establishes source admission for that managed path. + +## 3. Existing integration points and file map + +All unqualified anchors below refer to the planning baseline. Anchors marked #409 refer to the pinned commit and must be reconciled before implementation. + +| Existing location | Relevant behavior | +| --- | --- | +| `crates/edgezero-adapter-axum/src/run_options.rs`, `AxumRunOptions` | Cloneable explicit native settings. Constructors currently default Geo implicitly to absent because no Geo field exists. | +| `crates/edgezero-adapter-axum/src/dev_server.rs`, `run_production_selected` | Validates native store sources, builds the app/stores/transport and invokes the existing initializer. Geo validation must precede its call to `run_local`. | +| `dev_server.rs`, `PreparedApp`, `run_owned` | Private prepared resources flow into service construction before listener adoption/readiness. Carry Geo separately from stores. | +| #409 `crates/edgezero-adapter-axum/src/service.rs:329-418` | Actual-peer cleanup, original normalized-head summary, forwarding normalization/removal and admission. Insert protected-source resolution after successful admission. | +| #409 `service.rs:265-327`, `dispatch_prepared` | Converts the admitted body, installs stores and dispatches with `FrameworkErrorDetail::CategoryOnly`. Preserve this behavior. | +| #409 `crates/edgezero-adapter-axum/src/proxy.rs:115-174,293-359` | Bounded CIDR validation, IPv4-mapped normalization, whole-family union rejection and containment matching. | +| `crates/edgezero-core/src/ingress.rs`, `AdmittedIngress`, `PreparedIngress` | Private request-lifetime data already passes from admission to routed context. Do not expose a public Geo setter on either token. | +| #409 `crates/edgezero-core/src/app.rs:247-299` | Admitted dispatch and fixed framework error detail. Add an owned-outcome overload without losing failure classification. | +| `crates/edgezero-core/src/router.rs`, `dispatch_found` | App state overwrites public extension types before constructing the context. Geo must travel outside that extension map. | +| `crates/edgezero-core/src/context.rs`, `new`, `new_routed`, `parts_mut`, `extensions_mut` | Add private Geo storage; unmanaged construction ignores any forged Geo extension. | +| `crates/edgezero-adapter-axum/tests/production_hosting.rs` and `tests/fixtures/production_host.rs` | Existing child-process, readiness, raw-socket, initializer and shutdown fixtures. Extend these rather than building another server harness. | + +New files: + +```text +crates/edgezero-core/src/geo.rs +crates/edgezero-core/src/geo/country_codes.rs +crates/edgezero-adapter-axum/src/geo.rs +crates/edgezero-adapter-axum/src/geo/cloudfront.rs +crates/edgezero-adapter-axum/src/peer_networks.rs +scripts/check_geo_dependency_contract.sh +docs/guide/native-geo.md +``` + +Existing files expected to change narrowly: + +- Core `src/lib.rs`, `src/ingress.rs`, `src/context.rs` and `src/app.rs`. Keep the existing router production interface; add router tests only if needed for state-extension overwrite coverage. +- Axum `Cargo.toml`, `src/lib.rs`, `src/run_options.rs`, `src/dev_server.rs`, `src/service.rs` and reconciled `src/proxy.rs`. +- Axum `tests/fixtures/production_host.rs` and `tests/production_hosting.rs`. +- Existing adapter contract tests for portable no-selection behavior, including the Fastly preparer fixture where available. +- `.github/workflows/test.yml`, `docs/.vitepress/config.mts` and `docs/guide/adapters/axum.md`. + +No workspace member, production binary, CLI template or new third-party dependency is planned for Geo itself. #409 reconciliation may have its own manifest/lockfile changes; record those separately. Do not churn lockfiles or fold unrelated provider refactoring into this work. + +## 4. Execution order + +```mermaid +flowchart TD + Base[Stage 0: reconcile checked ingress and establish baseline] --> Types[Stage 1: portable values and provider contract] + Base --> Policy[Stage 2: shared peer validation and protected policy] + Types --> Context[Stage 3: immutable core dispatch handoff] + Types --> Selection[Stage 4: native selection and early validation] + Policy --> Selection + Types --> Parser[Stage 5: CloudFront resolution] + Context --> Dispatch[Stage 6: managed request integration] + Selection --> Dispatch + Parser --> Dispatch + Dispatch --> Proof[Stage 7: real hosting and compatibility proof] + Proof --> Docs[Stage 8: deployment guide and CI coverage] + Docs --> Final[Stage 9: final integration checks] + Final --> Deploy[Separate authorized deployment acceptance] +``` + +Work in coherent, testable units. Add a failing regression first, implement the behavior, run its focused checks, then continue. Record test names and commands before relying on filters. Use `scripts/run_test_nonzero.sh` for named sentinels so a renamed or feature-gated test cannot produce a false pass. + +One implementation owner should handle core/native dispatch integration. Independent policy/type research may overlap, but serialize shared Cargo targets, manifest edits and global formatting. Reuse passing checks while their relevant inputs remain unchanged. Do not run every workspace gate after every small edit. + +## 5. Stage 0: reconcile prerequisites and record the baseline + +### Work + +- [x] Read repository instructions, record HEAD/status and recompute the governing spec hash. Preserve uncommitted spec/plan and unrelated work. HEAD is still `5426ea65f642beafc59e6b4a4d198220d3c20e57`; the spec hash matches the reviewed hash above. Initially only the untracked spec and plan were present. +- [x] Split the local GeoIP/database/input-contract work into #438 and record the narrowed initial scope on #432. CloudFront does not satisfy that follow-up. +- [x] Reconcile the pinned #409 checked-ingress delta with explicit preservation of the newer native-store, async EOF and Connections owners. The narrower reconciliation is reviewed below; only the separate #407 buffered JSON-cap/body-carrier feature is excluded. No old whole-file replacement or second forwarding parser was used. +- [x] Preserve current `NativeSourcePlan`, native store overrides, asynchronous store preparation, current logging behavior and the reconciled body/lifecycle/egress ownership. The pinned branch and HEAD diverge beyond Geo; whole-file replacement would discard newer work. +- [x] Confirm original head accounting, missing-peer cleanup, checked proxy normalization, forwarding-header removal, native diagnostics and category-only framework error behavior survive the reconciliation. +- [x] Record the resulting integration revision and the exact baseline checks below. If integration uses a later #409 revision, inspect its changed contract rather than relying on this plan's old line numbers. + +Useful inspection commands, not merge instructions: + +```sh +git status --short +git rev-parse HEAD +sha256sum docs/superpowers/specs/2026-10-22-pluggable-geo-providers-design.md +git merge-base HEAD 66a05e25bee674ced69387467acec3c87b06c8b6 +git diff --stat HEAD 66a05e25bee674ced69387467acec3c87b06c8b6 -- crates/edgezero-core crates/edgezero-adapter-axum +git show 66a05e25bee674ced69387467acec3c87b06c8b6:crates/edgezero-adapter-axum/src/service.rs +``` + +### Baseline checks after reconciliation + +```sh +cargo test --locked -p edgezero-core --lib ingress:: +cargo test --locked -p edgezero-core --lib router:: +cargo test --locked -p edgezero-adapter-axum --lib proxy::tests +cargo test --locked -p edgezero-adapter-axum --lib run_options::tests +cargo test --locked -p edgezero-adapter-axum --lib dev_server::native_store_tests +cargo test --locked -p edgezero-adapter-axum --features test-utils --test production_hosting +``` + +Use the pinned sentinels `owned_peer_checked_metadata_precedes_all_managed_consumers` and `original_forwarding_head_overages_reject_before_stripping_or_body_poll` to confirm the relevant #409 tests survived. Record any renamed equivalents. These tests do not exist at the current planning baseline. + +### Exit gate + +A reviewed integrated ingress baseline with passing neighboring checks, or documented pre-existing failures isolated once. No second forwarding parser, no loss of current store startup behavior, and no unresolved ownership conflict with #409. Native Geo work does not bypass this gate. + +## 6. Stage 1: portable values and synchronous provider contract + +Files: new core `geo.rs` and `geo/country_codes.rs`; core `lib.rs` exports. + +### Work + +- [x] Add bounded `GeoProviderId`, `CountryCode`, `SubdivisionCode` and `GeoLocation` values with private fields and validated constructors. `GeoLocation` requires a country and accepts an optional subdivision. Avoid unchecked conversions or deserializers that bypass validation. +- [x] Give provider IDs the ASCII format `[a-z][a-z0-9._-]{0,63}`. Reserve `cloudfront` for the built-in provider. They are public diagnostic labels, never credentials. Native supplied selection receives the validated ID explicitly, so provider identity cannot vary per request. +- [x] Add bounded assertion slots, such as `GeoAssertion` with absent/value/invalid states. A private bytes/length representation prevents out-of-bounds access. `GeoInput` contains exactly a country slot of capacity two and a subdivision slot of capacity three, with no request/service/IP fields. +- [x] Keep provider results separate from framework outcomes. `GeoResolution` allows missing metadata, a validated location or a bounded provider error. Providers do not choose `NoSelection`, `UntrustedIngress`, selected identity or source provenance. +- [x] Add `GeoOutcome::NoSelection` and a selected outcome carrying a validated provider ID plus unavailable/located/error status. Located data includes `CloudFront` source and the declared protected-ALB trust basis. Errors contain only `InvalidAssertion` or `ProviderFailure`, not an arbitrary error chain. +- [x] Add the synchronous `GeoProvider::resolve(&self, input: &GeoInput) -> GeoResolution` trait. No async method, I/O context, mutable request, future or portable `Send`/`Sync` bound. Document bounded local computation and the trusted-Rust limitation. +- [x] Check in the officially assigned ISO 3166-1 alpha-2 codes. Record the authoritative ISO source URL, captured revision/date, imported-data checksum and applicable attribution in the table file. Review the exact imported list and any licence obligations before committing it. No runtime downloads, country-name database or dependency added just for two-letter membership. +- [x] Normalize ASCII case once in the value constructors. Country is two letters and must be assigned in the pinned list. Subdivision is one to three ASCII alphanumerics and preserves leading zeroes. It is not checked against a subdivision-membership database. +- [x] Make raw assertion debug output redacted. Outcome debug output may contain normalized codes and bounded public categories, never raw source bytes or credential data. + +### Tests + +Add sentinel `geo_country_codes_normalize_and_reject_unsupported`. Cover lowercase normalization, empty/non-ASCII/whitespace/oversize values, reserved and user-assigned country exclusions including `XX`, `ZZ`, `UK`, `XK`, and valid numeric subdivisions. Prove `US/ZZ` is a syntax-valid assertion rather than validated US membership. Validate the country table's ordering, uniqueness and recorded snapshot size. + +Add constructor, equality and debug tests. Compile a portable `GeoProvider` implementation containing `Rc>` to prove the trait itself has no native thread bounds. Verify no public constructor can create a subdivision-only location. + +```sh +scripts/run_test_nonzero.sh geo_country_codes_normalize_and_reject_unsupported cargo test --locked -p edgezero-core --lib geo:: +``` + +### Exit gate + +Portable types and provider behavior compile and pass their focused tests without a native dependency or country/subdivision inference. + +## 7. Stage 2: shared peer matching and protected-ALB policy + +Files: new Axum `peer_networks.rs` and `geo.rs`; reconciled `proxy.rs`; Axum module declarations. + +### Work + +- [x] Extract #409's existing literal-network validation and matching into a crate-private `PeerNetworks` type. Preserve its 256-entry and 16-KiB input budgets, accounting before deduplication, network truncation, mapped-address normalization, and rejection of whole-family unions as well as `/0`. +- [x] Delegate ordinary `TrustedProxyPolicy` to that helper without changing its public API, forwarding-family semantics or empty-policy behavior. Do not move forwarding interpretation into the helper. +- [x] Add opaque `ProtectedCloudFrontAlb` with a nonempty `PeerNetworks` and one credential-header name. It contains no credential value, distribution-authentication claim or AWS handle. +- [x] Use one bounded header-name rule: a valid literal HTTP token, at most 128 ASCII bytes, beginning with `x-` after case normalization. Reject `x-forwarded-*`, `x-real-ip`, `x-amz-*`, `x-amzn-*`, `x-edge-*`, and native request/correlation-ID header names. Standard transport, authorization, cookie and CloudFront Geo names are excluded by the custom-name rule. Do not trim malformed names into validity. +- [x] Keep the configured header name, network contents and parser input out of errors and `Debug`; emit fixed configuration categories and bounded counts only. Guide examples use a dedicated name such as `x-origin-verify-`. +- [x] Add a private request-bound admission helper that compares the actual socket peer with the protected policy. It must not accept `ConnectInfo`, `AxumRequestContext`, public checked metadata, effective reader IP or forwarding values as evidence. + +### Tests + +Preserve all existing proxy-policy assertions and coverage. Mechanically adapt tests that inspect private network fields, or move their helper-level assertions beside `PeerNetworks`; do not weaken them to make the extraction pass. Add sentinel `geo_protected_policy_reuses_network_bounds` covering empty policies, exact/over entry and byte limits, duplicates, IPv4/IPv6, mapped peers, multiple ranges covering an entire family, and missing/nonmatching peers. Add case-insensitive credential-name collision and redaction tests. + +```sh +cargo test --locked -p edgezero-adapter-axum --lib proxy::tests +scripts/run_test_nonzero.sh geo_protected_policy_reuses_network_bounds cargo test --locked -p edgezero-adapter-axum --no-default-features --features axum --lib geo:: +``` + +### Exit gate + +One reused network validator, distinct proxy and protected-source policies, and no credential comparison or forwarding-family dependency in Geo admission. + +## 8. Stage 3: immutable core handoff + +Files: core `app.rs`, `ingress.rs`, `context.rs`; tests in those modules and, if useful, `router.rs`. + +### Work + +- [x] Add private Geo outcome storage to `AdmittedIngress`, defaulting to `NoSelection` in every ordinary/fallback admission constructor. Add only a crate-private consuming installation method for core dispatch code. +- [x] Add `App::dispatch_admitted_with_geo(prepared, request, detail, outcome)`, preserving the pinned `FrameworkErrorDetail` parameter and classified failure behavior. This is the explicit public cross-crate handoff, not a public native trust witness. +- [x] Factor existing admitted dispatch methods through the same implementation with `NoSelection`. Preserve their signatures and existing rendering, response metadata, completion, grant and fallback behavior. +- [x] Install the owned outcome into private admitted state before passing it through the existing router path. The router's public signature need not change. +- [x] Transfer the result into a private `RequestContext` field in `new_routed`; add only `geo(&self) -> &GeoOutcome`. `RequestContext::new` explicitly defaults to `NoSelection` regardless of public extensions. +- [x] Do not preserve Geo by placing it in extensions when converting a context back into an ordinary request. Constructing a later unmanaged context creates a new no-selection dispatch. Keep existing body transfer semantics intact. + +### Tests + +Add sentinel `geo_owned_outcome_survives_public_request_mutation`. Read the same outcome in multiple middleware layers and a handler after changing geographic headers, inserting a contradictory `GeoOutcome` extension, removing it, clearing extensions and injecting same-type app state. Test legacy dispatch and unmanaged construction with forged selected-outcome extensions; both remain no-selection. + +Retain `App: Send + Sync`, token ownership, stale/foreign token rejection, method/path mutation rejection, grant-drop and category-only 404/405 tests. A Geo error must not change HTTP status automatically; the handler must be able to return an unrelated success response. + +```sh +scripts/run_test_nonzero.sh geo_owned_outcome_survives_public_request_mutation cargo test --locked -p edgezero-core --lib geo_ +cargo test --locked -p edgezero-core +``` + +### Exit gate + +There is no extension-based or public context setter path to replace a live result. Existing adapters still compile against unchanged dispatch APIs and obtain `NoSelection`. + +## 9. Stage 4: native selection and early startup validation + +Files: Axum `geo.rs`, `run_options.rs`, `dev_server.rs`, `service.rs`, `lib.rs` and `Cargo.toml`. + +### Work + +- [x] Add `geo-cloudfront = ["axum"]` without adding it to default features, AWS aggregates, CLI features or generated templates. Keep the selection enum and supplied-provider support under native Axum, not the implementation feature. +- [x] Add one private Geo selection field to `AxumRunOptions`, initialized to `Off` by every constructor. Add consuming `with_geo(selection)` and read-only `geo_capability()`; do not read any new environment variables. +- [x] Validate typed policy/identity and implementation availability through one pure validation function shared by capability inspection and runner setup. `CloudFront` without its feature returns a fixed unavailable-implementation category, never `Off`. +- [x] Represent a supplied provider with `Arc`, without requiring provider `Clone`. Keep the ID and policy on the same enum variant. The capability description contains only off/selected identity/source facts, not the provider or credential name. +- [x] Run Geo validation synchronously at the start of `run_production_selected`, before `run_local` creates runtime/signal resources. Keep existing store validation/preparation sequencing and initializer cancellation behavior. +- [x] Construct the stateless built-in provider locally and carry validated composition alongside the existing private prepared resources into `AxumServiceState`. Do not put it in `App`, request extensions or `PreparedStores`. Do not add an async preparation step. +- [x] Preserve every development/unconfigured entrypoint with off composition. The explicit options path is the only new selection path. +- [x] Demonstrate capability inspection by capturing the owned result of `finalized_options.geo_capability()?` in the existing initializer closure, then moving those same finalized options into the runner. The runner validates again. No initializer signature or global capability registry changes. + +### Tests + +Add compile-time assertions for `AxumRunOptions: Clone + Send + Sync` and retain the existing local-`Rc` initializer test. Test off defaults, explicit whole-selection replacement and supplied-provider composition. + +Add sentinel `geo_unavailable_selection_fails_before_runner_resources` under `cfg(not(feature = "geo-cloudfront"))`. Do not gate its module or the public selector on the implementation feature. Prove feature/policy failure precedes the application initializer and deferred store preparation. The test should also work when called from an existing runtime, demonstrating rejection before a nested runner runtime is created. Check no listener is bound and no local store root is created. + +```sh +scripts/run_test_nonzero.sh geo_unavailable_selection_fails_before_runner_resources cargo test --locked -p edgezero-adapter-axum --no-default-features --features axum --lib geo_ +cargo test --locked -p edgezero-adapter-axum --lib run_options::tests +cargo test --locked -p edgezero-adapter-axum --lib dev_server::native_store_tests +``` + +### Exit gate + +Default-off and unavailable-feature behavior are executed, native options retain their compatibility bounds, and Geo adds no dependency on store roots, AWS credentials or provider preparation. + +## 10. Stage 5: CloudFront resolution + +Files: Axum `geo/cloudfront.rs`, with unit tests; shared values remain in core. + +### Work + +- [x] Implement the stateless built-in `GeoProvider` behind `geo-cloudfront`. It accepts only the two bounded assertion slots and returns a `GeoResolution`. +- [x] Map absent country plus absent subdivision to missing metadata. A subdivision without country is invalid. Any invalid slot makes the whole result invalid. +- [x] Delegate country/subdivision normalization to the shared validated constructors. A valid country with absent subdivision is a located country-only result. Never repair an unsupported country or malformed subdivision. +- [x] Keep provider selection, source admission, provider ID and provenance outside the provider implementation. It performs no header lookup, HTTP status choice, logging policy or fallback. + +### Tests + +Add sentinel `geo_cloudfront_parsing_matches_spec`. Cover every trusted parsing row in spec section 6, including identical duplicates represented as invalid slots, joined values, invalid UTF-8/non-ASCII, zero/one/two/three/four-byte boundaries, lowercase, numeric subdivision components and leading zeroes. Cover `US` without a subdivision, `US/ZZ`, unsupported countries and region-only input. + +Use a table-driven test with expected typed outcomes rather than assertions against display strings. Keep provider failure testing in a supplied test implementation; header parsing does not invent internal failure cases. + +```sh +scripts/run_test_nonzero.sh geo_cloudfront_parsing_matches_spec cargo test --locked -p edgezero-adapter-axum --no-default-features --features geo-cloudfront --lib geo:: +``` + +### Exit gate + +The built-in returns the spec's outcomes with no external operations or silent loss of malformed assertions. + +## 11. Stage 6: managed request integration + +Files: reconciled Axum `service.rs`, `geo.rs` and service tests. Keep the unmanaged request converter unchanged. + +### Work and exact ordering + +- [x] Preserve #409's clock, native request guard, drain gate and actual-peer replacement/removal behavior. These early hooks receive no raw headers; do not change lifecycle timing merely to add Geo. +- [x] Split the unpolled request. Compute the original normalized-head validation/summary result before removing headers, so credential bytes and duplicate fields still count against limits. +- [x] Remove every instance of the configured credential header from live parts. Then handle the saved validation result. No callback that can observe the request or its headers may run before removal, including future callbacks added to this path. Do not retain values or reflect the name in validation diagnostics. +- [x] Run the existing checked forwarding normalization/removal and admission on sanitized parts. Keep #409's original count/byte summary. Refused, aborted, invalid-ingress and draining paths perform no Geo resolution. +- [x] After admission, use only the runner-owned `remote_addr` and selected protected policy to create the private request-bound witness. Off returns `NoSelection`; missing/nonmatching peer returns selected `UntrustedIngress`. Neither path inspects Geo values or calls a provider. +- [x] With a witness, inspect both original Geo header names in the still-owned parts. Admission received a cloned immutable head, so it cannot have changed these values. Detect duplicates and oversize input without collecting values or copying more than two/three bytes. Do not retain another raw HeaderMap. +- [x] Invoke the selected provider exactly once, including both-absent and invalid-slot inputs. Wrap its result with the selection-owned ID and, for located results, source/trust basis. Perform this before `dispatch_prepared` conversion/dispatch; later conversion failures do not retry resolution. +- [x] Pass the owned outcome separately through `dispatch_prepared` to `App::dispatch_admitted_with_geo`, retaining `FrameworkErrorDetail::CategoryOnly` and the existing native egress owner. +- [x] Do not expose the witness, policy, provider handle or a mutable outcome to request extensions, middleware, finalizers or the unmanaged converter. + +### Tests + +Add sentinel `geo_managed_resolution_runs_once_from_owned_ingress`. Use an atomic counting supplied provider to prove one invocation on trusted admitted requests, including absent and invalid input, and zero on off/untrusted/rejected/aborted/draining paths. Finishing, dropping or failing egress must not invoke it again. Test a reported provider failure without fallback or automatic HTTP rejection. + +Forge public `ConnectInfo`, `AxumRequestContext`, checked metadata and Geo outcome extensions while passing a missing/nonmatching actual peer. None may establish trust. An ordinary trusted-proxy configuration with Geo off must return `NoSelection`, not a located result. Active selection cannot omit its protected policy. Separately prove that a matching protected ALB peer admits geography without forwarding assertions or ordinary proxy trust. Do not require `effective_client()` to be absent: #409 initializes that field from the direct peer even when no forwarded viewer identity is resolved. + +Test credential removal in admission and handler observations, repeated credential fields, and validation-error paths. A request whose original head exceeds a limit must still fail even if stripping the credential would make it fit. Summary counters must include removed bytes/fields while logs and responses contain neither the name nor sentinel value. + +No managed-Axum preparer hook exists. Cover mutation at the actual middleware/context boundary and leave that hook absent. + +```sh +scripts/run_test_nonzero.sh geo_managed_resolution_runs_once_from_owned_ingress cargo test --locked -p edgezero-adapter-axum --no-default-features --features geo-cloudfront --lib geo_ +cargo test --locked -p edgezero-adapter-axum --features geo-cloudfront --lib service::tests +``` + +### Exit gate + +Actual managed dispatch uses owned peer/input facts and a private immutable result, with the specified call counts and unchanged admission/egress ownership. Tower-only calls with no socket peer must remain untrusted; they are not trusted-path proof. + +## 12. Stage 7: hosting and compatibility proof + +Files: existing native production fixture/tests; existing portable adapter contract tests where needed. + +### Work + +- [x] Add programmatic fixture modes for off, compiled-out CloudFront, enabled CloudFront, supplied provider, invalid policy and provider failure. Fixture-only CLI/environment knobs must not become production Geo selectors. +- [x] Use a no-store fixture profile so a CloudFront-only application proves it needs no config/data roots or AWS settings. Reuse existing `env_clear`, child output capture, readiness polling and shutdown helpers. +- [x] Add raw-header request support through the existing socket helper. Use a real loopback listener and a matching `/32` policy for successful source admission. Use a different policy range for real-peer rejection. Missing-peer and forged-extension cases remain private service tests. +- [x] Observe normalized outcomes and credential absence in a test handler. Do not turn test observation into built-in Geo response headers or a public diagnostics endpoint. +- [x] Test the same public `CloudFront` selector in feature-disabled and enabled fixture builds. Disabled selection must exit nonzero before readiness/initializer and never bind the selected port. +- [x] Prove a supplied provider works with the CloudFront implementation feature disabled. This distinguishes pluggable composition from availability of the built-in parser. +- [x] Retain local initializer, cancellation, shutdown, refused/aborted ingress and store-preparation regression checks. Match the existing terminal call counts rather than introducing another completion owner. +- [x] Add a no-selection assertion to existing Fastly, Cloudflare and Spin dispatch contracts. For Fastly's existing preparer, insert a forged Geo outcome into scratch extensions and verify no-selection at the handler. Preserve callback ordering and signatures; do not implement platform Geo. + +### Checks + +Add process sentinels `geo_hosting_proves_real_peer_and_shared_outcome` for the enabled path, `geo_hosting_rejects_unavailable_cloudfront` for the disabled path, and `geo_hosting_supplied_provider_without_builtin` for supplied composition. + +```sh +scripts/run_test_nonzero.sh geo_hosting_rejects_unavailable_cloudfront cargo test --locked -p edgezero-adapter-axum --no-default-features --features axum,test-utils --test production_hosting +scripts/run_test_nonzero.sh geo_hosting_supplied_provider_without_builtin cargo test --locked -p edgezero-adapter-axum --no-default-features --features axum,test-utils --test production_hosting +scripts/run_test_nonzero.sh geo_hosting_proves_real_peer_and_shared_outcome cargo test --locked -p edgezero-adapter-axum --no-default-features --features geo-cloudfront,test-utils --test production_hosting +``` + +The second disabled-build command may reuse the first suite's execution when its log lists both sentinels and the tested inputs are unchanged. Do not rebuild merely to repeat identical passing evidence. + +### Exit gate + +A real native listener proves the local path, supplied-provider composition and no-store startup. Other adapters retain no-selection behavior. All production-cloud assertions remain unproven until separate deployment checks. + +## 13. Stage 8: deployment guide and CI coverage + +### Documentation + +- [x] Add `docs/guide/native-geo.md`, link it from the Axum guide and navigation. Include programmatic examples for off, CloudFront, supplied provider, startup capability capture and application handling of every outcome. Keep the same snippets in native module rustdoc and execute its doc tests; use `no_run`, not `ignore`, for examples that would start a server. Compare the guide snippets with those compiled examples before acceptance. +- [x] Describe the selected HTTPS CloudFront/ALB path, dedicated origin credential, default-deny listener, CloudFront prefix-list restriction and target security-group isolation. Distinguish declared infrastructure protection from runtime source attestation. +- [x] Show the exact supported credential-header naming rule and a value-only rotation sequence. Renaming requires coordinated stop/reconfigure/restart. Redact both name and value in operational output. +- [x] Require explicit configuration of the two CloudFront-generated headers, not generic viewer-header forwarding or static geographic values. Explain the AWS-network missing-subdivision case and the required spoofing test. +- [x] Require `CachingDisabled`, zero error-caching TTLs for cacheable application-error statuses and no other shared reuse of these responses. No Geo-aware cache-key recipe or automatic cache mutation is added. +- [x] Link the AWS sources already recorded in the spec. If their behavior has changed by implementation time, review the difference before modifying the chosen contract. +- [x] Keep Trusted Server adoption and VPC origins in deferred-work notes, not executable steps in this feature. + +### Feature and dependency checks + +- [x] Add `scripts/check_geo_dependency_contract.sh` using the repository's shell/Cargo-tree conventions. It must fail on missing expected data, not merely print graphs. +- [x] Compare isolated native normal dependency sets with `axum` versus `geo-cloudfront`: the new feature adds implementation code but should add no third-party runtime dependencies. Assert it is absent from defaults and present when explicitly selected. +- [x] Inspect core and all three WASM adapter normal graphs for new Axum/native-runtime/AWS SDK leakage. Preserve the existing native-store graph checks rather than adding Geo to a store registry or changing store feature semantics. +- [x] Add CI steps for the enabled Geo unit suite, disabled-feature sentinel, supplied-provider sentinel, real hosting sentinel and dependency script. Existing default workspace tests do not exercise the opt-in implementation, and all-features tests do not prove disabled behavior. +- [x] Run the new no-selection assertions through the existing WASM contract-test jobs and their runtime runners. Keep them attached to established dispatch sentinels rather than adding a separate deployment harness. + +### Exit gate + +Published examples compile against the implemented API. CI selects nonzero tests in both feature states. No new CLI/environment interface or cloud acceptance claim appears in the guide. + +## 14. Stage 9: final integration checkpoint + +Run these after the coherent implementation is stable. Reuse recorded passing focused results for unchanged inputs. Prepare locked dependencies and installed targets first; an unavailable toolchain or dependency is a coverage gap, not a passing check. + +```sh +cargo fmt --all -- --check +cargo test --locked --workspace --all-targets +cargo test --locked -p edgezero-adapter-axum --all-targets --all-features +cargo test --locked -p edgezero-adapter-axum --features geo-cloudfront --doc +cargo clippy --locked --workspace --all-targets --all-features -- -D warnings +cargo check --locked --workspace --all-targets --features "fastly cloudflare spin" +cargo check --locked -p edgezero-adapter-axum --no-default-features +cargo check --locked -p edgezero-adapter-axum --no-default-features --features native-bindings +cargo check --locked -p edgezero-adapter-fastly --no-default-features --features fastly --target wasm32-wasip1 +cargo check --locked -p edgezero-adapter-cloudflare --no-default-features --features cloudflare --target wasm32-unknown-unknown +cargo check --locked -p edgezero-adapter-spin --no-default-features --features spin --target wasm32-wasip2 +bash scripts/check_geo_dependency_contract.sh +bash scripts/check_native_store_dependency_contract.sh +RUSTDOCFLAGS='-D warnings' cargo doc --locked --workspace --all-features --no-deps +cargo test --locked --manifest-path examples/app-demo/Cargo.toml --workspace --all-targets +cargo check --locked --manifest-path tests/fixtures/reusable-app/Cargo.toml -p fixture-axum +``` + +Also retain the explicit disabled/enabled hosting and unit commands from earlier stages. Run the existing WASM dispatch test jobs with their configured Viceroy, wasm-bindgen and Wasmtime runners. Compile-only WASM results do not satisfy those runtime tests. + +For published documentation changes, install the locked docs dependencies if needed, then run: + +```sh +cd docs +npm run lint +npm run format +npm run build +``` + +The internal `docs/superpowers` documents are excluded from published-site builds and Prettier. Check their relative links, section references, fenced blocks and whitespace separately. A documentation-only plan update does not require executing the implementation commands above. + +### Exit gate + +All applicable local gates pass or have explicit, evidenced baseline exclusions. Report changes, exact tested revisions/features, runtime versus compile-only evidence, and remaining cloud/downstream gaps. Do not call the feature production-accepted solely because local CI is green. + +## 15. Separate deployment acceptance + +This stage requires separate authorization and access to an isolated CloudFront/ALB deployment. It is not part of local CI and does not authorize provisioning from the provider implementation. + +- [ ] Record the application build, enabled features and sanitized CloudFront/ALB/network configuration identifiers. Never store origin credentials in the plan, shell history, screenshots or test logs. +- [ ] Prove an intended request reaches the application, while direct ALB requests and a different distribution without the credential cannot reach it. With prefix-list restrictions, a network denial is an acceptable direct-access result; do not require an HTTP 403 where no connection is admitted. +- [ ] Prove the application target cannot be reached through another listener, proxy, public address or unintended security-group source. +- [ ] Send forged country/subdivision values through the intended distribution. Compare with generated metadata and prove viewer assertions are not accepted. Include an AWS-network request with a forged subdivision while CloudFront omits its own subdivision. +- [ ] Verify country-only and missing-metadata outcomes without guesses or local/remote fallback. +- [ ] Exercise value rotation under the fixed credential-header name, including temporary old/new acceptance at the ALB, CloudFront deployment and old-value removal. Prove the application never sees the credential header and operational outputs do not reveal it. +- [ ] Exercise success and cacheable error responses across requests with different geographic/policy inputs. Verify CloudFront and any other shared cache do not reuse them. Do not infer cache safety from origin-request policy configuration alone. + +Any failure blocks production use of the declared protected policy. Do not work around it by trusting additional headers or adding fallback providers. Trusted Server policy-path tests remain a separate downstream adoption task even after these checks pass. + +## 16. Final evidence and completion + +### Tested source and reconciliation + +The implementation is committed as `d5ec14ed` and `5da2000f` on top of native celld `5426ea65`, with the final test-only split at `56eab85fee2499d46984b077fc9e167693bd4e67`. Remaining publication edits affect only this evidence record and design status. No production source changed after the native all-feature check. The final workspace suite, real-hosting suites and WASM runtime contracts cover the final test state. + +The pinned #409 branch shares merge-base `5527b58e` with the native celld stack but its checked-ingress implementation was absent here. Two read-only investigations compared direct-delta adaptation and selective reimplementation. The selected approach adapted only #409's own commit delta, reused its parser/counters/tests and extracted its bounded peer-network matcher. Newer native stores, local non-Send initialization, `Connections`/`FuturesUnordered`, ingress/config bounds and asynchronous EOF/zero-length validation remain authoritative. No old tree was copied over newer files. + +The separate #407 managed Body carrier, `JsonBodyLimit` setting and 413-cap behavior are intentionally not imported. Native JSON error privacy instead travels through private admitted/context state. Only #407-dependent cap/carrier tests and claims are excluded; checked ingress, original-head limits, routing/JSON privacy, terminal ownership and TLS proxy fixture tests are retained. This is checked-ingress reconciliation, not full #409-plus-#407 parity. + +The required #409 sentinels `owned_peer_checked_metadata_precedes_all_managed_consumers` and `original_forwarding_head_overages_reject_before_stripping_or_body_poll` pass in the native unit suites. A read-only correctness review found no remaining implementation findings. Its credential-error ordering concern was fixed before acceptance: save original validation Result, remove all credential occurrences, then handle the saved Result. Direct rejected-head and managed overage regressions execute that path. + +### Local checks + +All Cargo commands below used Rust/Cargo 1.95.0 and locked dependencies. The workspace test process used stdin from `/dev/null`, matching noninteractive CI consent tests. Logs are local temporary artifacts, not durable CI records. Rows run before `56eab85f` retain their evidence because that commit only split an already-passing core test into two tests; the production implementation is unchanged. The final workspace suite retests both. + +| Check | Tested features/state | Result | Elapsed | Local evidence | +| --- | --- | --- | --- | --- | +| `cargo test --locked --workspace --all-targets` | `56eab85f`, workspace defaults | 2,115 passed, 6 existing ignored, none failed | 55.754 s | `/tmp/geo-workspace-tests-final.log` | +| Native `--all-targets --all-features` tests | `5da2000f`, all native providers + Geo | 422 passed across 8 suites, including 367 units, 6 AWS/mixed startup, 4 celld startup and 32 hosting | 182.120 s | `/tmp/geo-native-all-features-tests.log` | +| Enabled native library | `--no-default-features --features geo-cloudfront` | 322 passed; parser and managed-resolution sentinels included | 13.192 s | `/tmp/geo-final-enabled-units.log` | +| Disabled native library | `--no-default-features --features axum` | 323 passed; resource-precedence, supplied and policy paths included | 19.779 s | `/tmp/geo-final-disabled-units.log` | +| Final core Geo regressions | Final test split, `--lib geo_` | 3 passed, including immutable roundtrip and legacy-forgery cases | 17.64 s compile | `/tmp/geo-core-final-regressions.log` | +| Final enabled real hosting | `56eab85f`, `--no-default-features --features geo-cloudfront,test-utils` | Nonzero sentinel and full suite, 32 passed | 50.890 s | `/tmp/geo-final-enabled-hosting.log` | +| Final disabled/supplied real hosting | `56eab85f`, `--no-default-features --features axum,test-utils` | Full suite, 33 passed; both unavailable and supplied-without-built-in sentinels logged | 16.919 s | `/tmp/geo-final-disabled-hosting.log` | +| Workspace Clippy | `--workspace --all-targets --all-features -- -D warnings` | Pass after splitting the long core regression | 27.72 s | `/tmp/geo-workspace-clippy.log` | +| Native disabled Clippy | `--no-default-features --features axum --lib -- -D warnings` | Pass, including absent-implementation cfg | 2.14 s | `/tmp/geo-final-disabled-clippy.log` | +| Workspace feature compilation | `--all-targets --features "fastly cloudflare spin"` | Pass | 78 s | `/tmp/geo-feature-compilation.log` | +| Minimal native / bindings-only | No defaults; then `native-bindings` only | Both pass | 3 / 4 s | `/tmp/geo-native-{minimal,bindings-only}.log` | +| Three WASM target checks | Fastly wasip1, Cloudflare unknown-unknown, Spin wasip2; adapter feature only | All pass, compile-only | 11 / 32 / 39 s | `/tmp/geo-*-wasm-check.log` | +| Fastly WASM contracts | Viceroy 0.21.0, `fastly,test-utils` on wasip1 | 24 runtime tests passed; preparer/closed-lifecycle sentinel separately passed | 35 / 6 s | `/tmp/geo-fastly-{wasm,preparer}-runtime.log` | +| Cloudflare WASM contracts | wasm-bindgen-test-runner 0.2.128, Node 24.12.0, `cloudflare,test-utils` | 29 runtime tests passed | 59 s | `/tmp/geo-cloudflare-wasm-runtime.log` | +| Spin WASM contracts | Wasmtime 48.0.2, async component/P3/HTTP enabled, `spin,test-utils` on wasip2 | 35 runtime tests passed, including managed Geo-forgery case | 32 s | `/tmp/geo-spin-wasm-runtime.log` | +| Geo / native-store dependency scripts | Isolated normal graphs from final manifests/locks | Both pass; Geo adds no third-party runtime package relative to Axum | 2 / 6 s | `/tmp/geo-{dependency-final,native-store-dependency}.log` | +| Native Geo public doc tests | `--features geo-cloudfront --doc` | 5 passed, including all four guide examples | 4.16 s compile + 2.16 s doc harness | `/tmp/geo-native-doc-tests-final.log` | +| Workspace rustdoc | `RUSTDOCFLAGS='-D warnings'`, all features, no deps | Pass | 64 s | `/tmp/geo-workspace-rustdoc.log` | +| App-demo workspace | Final consumer lock, all targets | 54 passed, none failed | 170.338 s | `/tmp/geo-demo-tests.log` | +| Reusable native consumer | Final consumer lock, `-p fixture-axum` | Pass | 109.243 s | `/tmp/geo-reusable-check.log` | +| Published docs | Locked npm dependencies; lint, format, site build | All pass, exact guide/rustdoc snippets matched | 3.272 / 2.912 / 8.401 s | `/tmp/geo-docs-{lint,format,build}.log` | +| Dependency-script failure cases | 16 mocked success/failure graph scenarios | All pass; real graphs also checked above | 3.206 s | `/tmp/geo-docs-dependency-mocks.log` | +| Formatting, whitespace and internal links | `cargo fmt --all -- --check`, `git diff --check`, local Markdown checks | Pass | Not separately timed | Final coordinator checks | + +The runtime contract commands use the existing `scripts/run_test_nonzero.sh` harness with their established dispatch sentinels. Fastly used the official Viceroy 0.21.0 binary, archive SHA-256 `52ed3c3ed59d926c81015d637509942c3195443c02c2eebbded9125484336a9e`; the installed 0.17.0 binary was not used. Cloudflare used the absolute installed 0.2.128 runner rather than an unconfigured mise shim. Spin used `wasmtime run -W component-model-async=y -S p3=y -S http=y`. + +### Dependencies and proof limits + +The root lockfile adds only four dependency edges for reconciled #409 runtime/test support. Each excluded consumer lockfile adds only two native dependency edges; no package versions were changed. Minimum compatible `getrandom = "0.4.2"` and `ipnet = "2.12.0"` constraints preserve independently locked consumer patch versions instead of forcing unrelated downgrades and WASI dependency churn. Root tests use the pinned lock's 0.4.2/2.12.0; app-demo tests also prove its existing 0.4.3/2.12.2 lock. These are ingress prerequisites, not new Geo feature dependencies. The portable country allowlist is checked in with source revision, attribution and checksums, with no runtime dataset download. + +The Geo graph script explicitly permits the existing Cloudflare `worker 0.8.5 -> tokio 1.52.3` normal edge only with empty Tokio features. Native runtime/network companions or changed versions/features still fail. This baseline is not newly introduced native-runtime leakage. + +The six normally ignored workspace tests are existing generated-consumer or real-service acceptance harnesses. They were not silently selected or treated as passes: generated celld/default-off/restart, real celld TLS with explicit modified binary, generated DynamoDB lifecycle, generated-workspace compilation, native AWS emulator services, and generated native AWS build/startup qualification. The final demo/reusable builds and native mixed-provider startup tests pass, but do not replace those external harnesses. No manual `edgezero serve` run or actual cloud deployment was performed. + +Published docs dependencies report seven existing audit vulnerabilities, two moderate and five high, in their unchanged lockfile. No unrelated dependency upgrades were attempted. ShellCheck was unavailable; Bash syntax, 16 failure-case regressions and actual graph execution passed. Existing ignored core rustdoc examples remain ignored; the new Geo examples execute. + +### Safety evidence and separate acceptance + +1. **Protected source ownership.** Runtime evidence now proves actual-peer policy matching, missing/nonmatching peer rejection despite forged public metadata, distinct ordinary proxy trust, stripping before callbacks/error handling, original-head accounting, and exactly-once resolution only for admitted trusted requests. Evidence is the managed-resolution sentinel and both final real-hosting suites. This proves the local declared-policy boundary, not an ALB listener rule. +2. **Immutable and partial outcomes.** Executed core and real-hosting tests prove consumers share one outcome across header/extension mutation, same-type app-state injection and cleared extensions. Managed context roundtrip and legacy dispatch reset to NoSelection. Country-only, syntax-valid unknown subdivisions, missing/error distinctions, source failures, cancellation/drop/failing egress and no fallback retain their defined behavior. Portable runtime tests prove raw or forged Geo metadata cannot select a source. + +Actual ALB credential enforcement, network isolation, alternate-path denial, CloudFront header replacement including missing subdivision, value rotation and shared-cache isolation remain unproven until separately authorized deployment checks in section 15. Trusted Server policy translation and interpretation are outside this change. Deferred local databases and reader-IP input extension are tracked in [#438](https://github.com/IABTechLab/edgezero/issues/438), with the initial split recorded on [#432](https://github.com/IABTechLab/edgezero/issues/432#issuecomment-6349594498). + +Completion: + +- [x] Full local implementation has its focused and integration proof; the #409/#407 prerequisite distinction is explicit. +- [x] Default-off, no-fallback and unknown/error distinctions survive real native dispatch. +- [x] Provider objects, source admission and outcomes each have one owner; public request mutation cannot rewrite a live managed outcome. +- [x] Existing store preparation, native lifecycle, framework error privacy and portable adapter behavior are preserved. +- [x] Deployment, database and downstream acceptance remain separate from local implementation. +- [x] User-authorized commits and draft publication target `feat/native-celld-stores`, on top of #422; no unrelated edits or automatic deployment are included. diff --git a/docs/superpowers/plans/2026-11-04-axum-local-geoip.md b/docs/superpowers/plans/2026-11-04-axum-local-geoip.md new file mode 100644 index 000000000..b29c0be43 --- /dev/null +++ b/docs/superpowers/plans/2026-11-04-axum-local-geoip.md @@ -0,0 +1,188 @@ +# Axum local GeoIP implementation plan + +- Status: draft implementation locally verified. Full #438 acceptance is blocked by unused-metadata validation and the remaining proof/scope gaps below. +- Date: 2026-11-04 +- Issue: [#438](https://github.com/IABTechLab/edgezero/issues/438). +- Contract: [approved local GeoIP spec](../specs/2026-11-04-axum-local-geoip-design.md). User-approved requirements remain authoritative; unresolved gaps below are not approved scope changes. +- Base: `72ceed7e`, `feature/pluggable-geo-providers`, already published as draft [PR #439](https://github.com/IABTechLab/edgezero/pull/439). +- Delivery branch: `feature/axum-local-geoip`, published as draft [PR #453](https://github.com/IABTechLab/edgezero/pull/453) against `feature/pluggable-geo-providers`. Local GeoIP remains separate from #439. + +## 1. Decisions and implementation gates + +### Reader, limits and stop ownership + +Pin `maxminddb = "=0.32.0"` with default features disabled. Enable neither mmap nor unsafe string decoding. Add it as a workspace dependency and optional native-only Axum dependency, selected by default-off `geo-maxmind`. SHA-256 uses the existing workspace `sha2` dependency behind that feature. + +Use upstream `Reader::verify()` for metadata, search-tree and reachable value-graph verification, followed by network iteration, a non-materializing structural Serde pass and typed geographic projection decoding. Research executed fixtures proving upstream verification accepts non-string map keys and pointer-to-pointer values. The extra pass uses the same bounded decoder with strict string keys and identifier-based recursive decoding to reject these in reachable data records. The library exposes no raw metadata decoder; corresponding unused metadata-field cases remain an explicit validation gap, not a completed all-corruption guarantee. This is not a second MMDB parser. Verify consumed field shapes at startup, while code membership/syntax validation remains request-time behavior. A valid string `XK` loads successfully and fails only when resolved. + +Initial configurable defaults are 128 MiB of actual input bytes and 32 million validation network visits. MaxMind's published May-August 2026 ranges show City MMDB files at 43.9-66.9 MB and about 4.8-5.9 million combined IPv4/IPv6 networks, with Country smaller. These are observed sizes, not vendor guarantees. The byte default gives roughly twice the published upper City size; the visit budget allows tree-verification overhead and growth. An operator can deliberately raise finite limits for a newer release. Source: [database sizes](https://dev.maxmind.com/geoip/docs/databases/city-and-country/). + +Before verifier entry, check that metadata node count times its upstream three-visits allowance fits the selected visit budget. Enforce the same budget on projection iteration. The verifier bounds value/string work to eight times section bytes and memoizes shared targets. Its allocation is proportional to distinct referenced values. The decoder has finite container/payload/depth bounds; projection borrowing avoids copying large strings or constructing full city records. Document actual guarantees and test exhaustion rather than claiming a wall-clock or fixed-RSS limit. + +Preparation stays synchronous, before native runtime/signals/stop polling, as explicitly accepted in the revised spec and the user's approval. A file-byte or work limit cannot interrupt blocked filesystem I/O. Reject nonregular sources without waiting for their contents. Preserve the existing cooperative stop contract after preparation. No detached worker or cosmetic timeout is introduced. + +A representative production GeoLite snapshot requires an operator's free account and licensed download. Do not obtain credentials or distribute production data for tests. If no permitted representative artifact is available, measure synthetic fixtures and explicitly report production-size time/memory measurement as unverified, not passed. + +### Public data shapes and compatibility + +Use a portable `GeoInput` enum: + +- `Assertions { country, subdivision }`, preserving bounded slot semantics. +- `ReaderIp { address: IpAddr, source: GeoIpSource }`. + +`GeoIpSource` identifies direct peer, trusted Forwarded and trusted X-Forwarded input. Custom Debug redacts reader addresses. Add `NoMatch` resolution and `MissingReaderIp`/`NoMatch` unavailable reasons. Add local-database and supplied-reader-IP source kinds and a checked-reader trust basis carrying the source. Reserve `maxmind` with `cloudfront`. + +Native capability becomes a configuration description with provider identity and typed input contract, including `PeerAllowed` or `ForwardedRequired` for reader input. It does not pretend to know eventual request provenance. Separate pure validation/capability from preparation. Keep selector/configuration types available in feature-disabled builds. + +The workspace is unpublished (`publish = false`, version 0.1.0). This is a coordinated pre-release public API migration, not a compatibility shim. Migrate repository consumers in the same change and document breaking changes in the changelog/guide and PR. Before any release, external consumers must update input construction and exhaustive enum matches. Do not bump unrelated crate versions in this work. + +## 2. Implementation checklist + +### Stage A: portable inputs and existing provider migration + +- [x] Add typed IP input/provenance, safe Debug, no-match/missing-input outcomes and reserved MaxMind identity in `crates/edgezero-core/src/geo.rs`. +- [x] Migrate core tests and built-in CloudFront parser/tests to `Assertions`. +- [x] Reject an unsupported provider input case as `ProviderFailure`, with no input probing/fallback. +- [x] Update native counted-provider assertions, test fixture outcome classifiers and exhaustive guide/rustdoc examples. + +Proof: `cargo test --locked -p edgezero-core --lib geo` and focused CloudFront parser tests. Existing country, immutable outcome and legacy-forgery regressions remain passing. + +### Stage B: bounded local database and fixtures + +- [x] Add `geo-maxmind` feature, pinned native dependencies and workspace/consumer lock updates. +- [x] Add feature-independent settings/limits and source-free error categories, with path-redacted Debug. +- [x] Implement read-only nonblocking-open handling for nonregular Unix sources, bounded regular-file read, immutable reader, metadata allowlist, verifier and projection validation. +- [x] Record loaded type/build timestamp/SHA-256, and map City/Country records using only `country.iso_code` and first subdivision code. +- [x] Add small reproducible synthetic MMDB fixtures and provenance, with no production download dependency. +- [x] Test IPv4/IPv6 and mapped input, IPv6 against IPv4-only data, wrong input, City/Country, code errors, no-match, first-entry rules and unknown subdivisions. +- [ ] Test metadata/field-type/corruption failures, off-path corruption, byte/visit limits, missing/unreadable/nonregular files and safe errors. + +Proof: focused MaxMind library suite with `--no-default-features --features geo-maxmind`. Source removal/replacement and snapshot identity pass. The unchecked corruption/permission item includes a confirmed malformed-unused-metadata failure, retained as an ignored acceptance regression, and untested unreadable-file denial. Production-size time/RSS is unverified. Consumer lockfiles required no changes and pass locked checks. + +### Stage C: native source composition and pure capability + +- [x] Add `MaxMind` and supplied-reader-IP selection, reader expectation and typed native capability. +- [x] Preserve existing supplied-assertion/CloudFront protected-source admission and credential stripping. +- [x] Split pure capability validation from runner-owned preparation in `geo.rs` and `run_options.rs`. +- [x] Capture reader input from private checked facts in `service.rs` before moving metadata into public extensions; invoke only after admission. +- [x] Distinguish actual direct peer from explicitly resolved forwarded reader using `client_source()`. Honor both modes, including partial forwarding success and unresolved chains. +- [x] Emit safe loaded-snapshot diagnostics after existing logging initialization in `dev_server.rs`. +- [x] Test repeat capability calls on missing paths without I/O, feature-disabled selection and supplied reader input without either built-in. + +Proof: source-admission unit matrix for both forwarding families, both reader expectations, missing/untrusted/forged peers, and counted managed dispatch. Run disabled, CloudFront-only, MaxMind-only and combined feature suites. + +### Stage D: real startup and replacement contracts + +- [x] Extend the existing no-store production fixture with local/supplied-IP modes, explicit proxy trust and database settings. Fixture environment inputs remain test-only. +- [x] Add real-listener tests for country/state, shared immutable outcome, private checked-input ownership, no-match and country-only behavior. +- [x] Add startup failures proving no initializer, bind or ready transition for unavailable implementation, invalid database and exhausted budgets. +- [x] Prove startup/read access on a read-only file without sidecar writes, stable active results after replacement/removal, restart loading the new snapshot, and invalid replacement rejection. +- [x] Retain counted-provider zero-call paths on refused/aborted/draining/off/inadmissible requests and no finalization lookup. +- [x] Prove an already-requested embedding stop is honored after preparation without initializer/bind/readiness. + +Proof: nonzero sentinels through `scripts/run_test_nonzero.sh`, `production_hosting` integration suite, affected native lifecycle tests. These are native local runtime proofs, not downstream consent-policy acceptance. + +### Stage E: documentation and dependency/CI coverage + +- [x] Extend `docs/guide/native-geo.md` with direct/proxied free GeoLite examples, supported schemas, licensing/attribution/update duties, mount/replacement instructions, limits, pre-runtime stop exception and migration guidance. +- [x] Keep guide examples in compiled module rustdoc. Handle all unavailable/error cases without granting permission. +- [x] Add changelog entry for breaking public API additions and migration. +- [x] Extend `scripts/check_geo_dependency_contract.sh` to prove MaxMind/sha2 native opt-in isolation and absence of mmap/unsafe decoder features, preserving CloudFront dependency equality. +- [x] Extend `.github/workflows/test.yml` with disabled, each-built-in and combined-feature MaxMind/supplied-IP/runtime sentinels. + +Proof: public doc tests, dependency script, shell checks and docs lint/format/build. Public schema examples and internal relative links must match the implemented API. + +## 3. Stable integration checkpoint + +Serialize Cargo commands using this checkout's target directory. Reuse focused results for unchanged inputs. Record exact feature/revision or relevant fingerprints, outcome, elapsed time and evidence path. Run: + +```sh +cargo fmt --all -- --check +cargo test --locked --workspace --all-targets < /dev/null +cargo test --locked -p edgezero-adapter-axum --all-targets --all-features +cargo test --locked -p edgezero-adapter-axum --no-default-features --features axum --lib +cargo test --locked -p edgezero-adapter-axum --no-default-features --features geo-cloudfront --lib +cargo test --locked -p edgezero-adapter-axum --no-default-features --features geo-maxmind --lib +cargo test --locked -p edgezero-adapter-axum --features geo-cloudfront,geo-maxmind --doc +cargo clippy --locked --workspace --all-targets --all-features -- -D warnings +cargo check --locked --workspace --all-targets --features "fastly cloudflare spin" +cargo check --locked -p edgezero-adapter-axum --no-default-features +cargo check --locked -p edgezero-adapter-axum --no-default-features --features native-bindings +cargo check --locked -p edgezero-adapter-fastly --no-default-features --features fastly --target wasm32-wasip1 +cargo check --locked -p edgezero-adapter-cloudflare --no-default-features --features cloudflare --target wasm32-unknown-unknown +cargo check --locked -p edgezero-adapter-spin --no-default-features --features spin --target wasm32-wasip2 +bash scripts/check_geo_dependency_contract.sh +bash scripts/check_native_store_dependency_contract.sh +RUSTDOCFLAGS='-D warnings' cargo doc --locked --workspace --all-features --no-deps +cargo test --locked --manifest-path examples/app-demo/Cargo.toml --workspace --all-targets +cargo check --locked --manifest-path tests/fixtures/reusable-app/Cargo.toml -p fixture-axum +``` + +Run explicit feature-disabled and feature-enabled hosting sentinels as well. Where installed, run existing WASM adapter dispatch contract runners, not just cross-target compilation. Missing runtimes or representative data remain named coverage gaps. + +## 4. Review and draft PR + +- [x] Inspect the combined diff against the approved spec and obtain independent correctness/architecture review. The capability issue is fixed; metadata rejection remains an explicit blocker in section 5. +- [x] Validate no unrelated work or credentials enter commits. Include the reviewed spec and this implementation/evidence record. +- [x] Commit after relevant tests, push `feature/axum-local-geoip`, and create draft #453 against `feature/pluggable-geo-providers` using the repository PR template. +- [x] Describe #439 stacking, #438 scope, public migration, exact check outcomes, pre-runtime preparation exception and unverified production-size/downstream behavior. The PR explicitly retains draft status and makes no consent or production-acceptance claim. + +## 5. Evidence and decisions during implementation + +Initial source inspection confirms `maxminddb` 0.32.0 is ISC-licensed and default-feature-free. Independent proposals favored either replacing provenance wholesale or extending the existing enums. The chosen path extends them to minimize the migration beyond the required typed input change. Source inspection also found stacked-base workflow filters were missing; both workflow filters now include the parent Geo branch. Source research and independent contract investigation use configured `openai-codex/gpt-6.1-sol` agents. Zellij pane creation failed without creating panes; research continued through background noninteractive agents. The coordinator owns implementation and shared Cargo commands. + +### Independent review and blockers + +The independent correctness/architecture review confirmed private checked-IP ownership, no fallback, read-only snapshots and side-effect-free capability. It found two concrete issues: + +1. **Open acceptance blocker:** upstream 0.32.0 accepts non-string keys and pointer chains in unused metadata fields. Our structural pass closes the data-record cases, but the library exposes no raw metadata decoder. Full malformed-metadata startup rejection therefore remains unmet. A strict metadata verifier/raw-decode dependency fix or an explicitly approved requirement change is needed. Do not vendor a fork, add a second MMDB parser or weaken the approved spec silently. The PR must remain draft and must not claim #438 acceptance until resolved. + Reproduced by executing `geo_maxmind_rejects_unused_metadata_nonstring_keys -- --ignored`: one expected failing test, `/tmp/geoip-known-metadata-blocker.log`. A real fixture process with `bad-metadata-key.mmdb` also reached readiness, `/tmp/geoip-known-metadata-hosting.log`. The default suite explicitly ignores this one regression; it is not a passing corruption check. +2. **Fixed:** non-Unix builds advertised MaxMind capability although acquisition is unavailable. Pure validation now includes target support and returns `UnavailableMaxMind` before preparation. Non-Unix support itself remains unimplemented and requires scope acceptance or an acquisition implementation. + +Other unverified acceptance work includes representative production GeoLite preparation time/RSS, real read-only mount execution, unreadable-file permission denial, and preparer-specific mutation coverage. No managed Axum preparer hook is added; the existing portable immutable-outcome regressions and new native middleware/header/extension regressions pass. Existing permission-mode, FIFO/device, middleware/header/extension, startup-stop and real listener fixtures prove narrower claims only. + +One MaxMind-only hosting run failed when a child could not bind its ephemeral reserved address. The unchanged existing harness releases its reservation before child bind. The identical full suite passed on rerun; no production retry or arbitrary port policy was added. Retain this transient test-harness limitation in the PR evidence. + +Check | Tested state | Result | Elapsed | Evidence +--- | --- | --- | --- | --- +Baseline core | `72ceed7e`, before source edits | 8 passed | 17.64 s compile | `/tmp/geoip-baseline-core.log` +Portable core | Typed input migration | 9 passed | 6.54 s compile | `/tmp/geoip-core-contract.log` +Reader admission/managed dispatch | Axum only, no built-ins | 5 passed | 6.10 s compile | `/tmp/geoip-reader-managed.log` +Combined native Geo | CloudFront + MaxMind, including ready-stop and corruption | 26 passed | 17.37 s compile, 0.09 s tests | `/tmp/geoip-combined-geo.log` +Native all features | Before test-only lint cleanup | Passed, including 40 real-hosting tests | 54.204 s | `/tmp/geoip-native-all-features.log` +MaxMind-only real hosting | After test-only lint cleanup | 12 passed on rerun after one bind race | 0.35 s tests | `/tmp/geoip-maxmind-hosting-rerun.log` +Workspace Clippy | All targets/all features, after test-only lint cleanup | Passed | 1.242 s final run | `/tmp/geoip-workspace-clippy.log` +Dependency contracts | Isolated normal graphs | Geo and native-store scripts passed | 1.532 / 3.112 s | `/tmp/geoip-dependency-contracts.log` +Final workspace tests | Final Rust/test/build inputs | 2,121 passed, 6 existing ignored | 9.409 s | `/tmp/geoip-workspace-tests-final.log` +Final native all features | All targets, final inputs | 447 passed, 1 known metadata regression ignored | 13.980 s | `/tmp/geoip-native-all-features-final.log` +Final workspace Clippy | All targets/all features | Passed | 3.993 s | `/tmp/geoip-workspace-clippy-final.log` +Isolated native matrix | `axum`, CloudFront, MaxMind, both | Units: 328 / 327 / 340 / 339 passed; reader lanes each ignore the known regression. Hosting: 9 / 8 / 12 / 11 passed. Seven doc tests and all-target Clippy pass in every lane | Per-command timings in logs | `/tmp/geoip-matrix-*-{units,hosting,docs,clippy}.log` +Workspace platform features | `fastly cloudflare spin`, all targets | Passed | 16.667 s | `/tmp/geoip-workspace-feature-check.log` +Minimal native / bindings | No defaults, then bindings-only | Both passed | 5.628 / 2.646 s | `/tmp/geoip-minimal-checks.log` +WASM target compilation | Fastly wasip1 / Cloudflare unknown-unknown / Spin wasip2 | All passed, compile-only | 18.211 / 24.332 / 12.851 s | `/tmp/geoip-*-wasm-check.log` +WASM runtime dispatch contracts | Viceroy 0.21.0 / wasm-bindgen runner 0.2.128 / Wasmtime 48.0.2 | Fastly 24 / Cloudflare 29 / Spin 35 passed | 14.531 / 24.740 / 10.418 s | `/tmp/geoip-{fastly,cloudflare,spin}-runtime.log` +Workspace rustdoc | All features, no deps, warnings denied | Passed | 24.362 s | `/tmp/geoip-rustdoc.log` +App-demo tests | Unchanged excluded-workspace lock, shared target directory | 54 passed | 50.013 s | `/tmp/geoip-demo-tests.log` +Reusable native consumer | Unchanged excluded-workspace lock | Passed | 6.062 s | `/tmp/geoip-reusable-check.log` +Consumer locked checks | Both excluded workspaces, offline/locked, shared target | Both passed; no lockfile updates needed | 26.357 / 0.163 s | `/tmp/geoip-consumer-lock-checks.log` +Published docs | Final guide, lint / format / build | All passed | 1.950 / 1.577 / 3.730 s | `/tmp/geoip-docs-final.log` +Synthetic fixtures | Original Apache-2.0 generator and 26 fixtures | Syntax/reproduction/checksums passed | Not timed | `node scripts/generate_geoip_fixtures.mjs --check` +Formatting/static checks | Final source, workflows and internal documents | Rustfmt, whitespace, internal links/fences and actionlint passed; actionlint ShellCheck disabled | Not timed | Coordinator checks +Known metadata defect | Explicit ignored regression and real process | Reproduced as expected failure; malformed metadata reaches readiness | Not timed | `/tmp/geoip-known-metadata-{blocker,hosting}.log` + +The Rust/runtime implementation is committed as `9dd7b83b`. Publication follow-ups update this plan and fix CI cache IDs only; Rust/runtime inputs are unchanged. Final Rust, crate-test, script, workflow and guide input aggregate SHA-256 is `c586481c33c350e05476a670a126d97d0e1dbe6e416b7d4783e4b3f2df2e9c75`, computed with `git ls-files -cm --others --exclude-standard -z -- crates Cargo.toml Cargo.lock scripts .github docs/guide | sort -zu | xargs -0 sha256sum | sha256sum`. Plan/spec evidence edits and the changelog are outside this aggregate. Toolchain: Rust/Cargo 1.95.0, Linux native; Node 24.12.0. Logs are local temporary evidence, not durable CI artifacts. The official Viceroy 0.21.0 archive SHA-256 is `52ed3c3ed59d926c81015d637509942c3195443c02c2eebbded9125484336a9e`; the installed 0.17.0 binary was not used for runtime proof. + +The coordinator inspected returned changes and ran repository-native integration. Bounded implementation workers owned the reader/fixtures, hosting fixtures/tests, and guide/CI respectively; no files overlapped. Their scratch/static results were not substituted for native tests. The independent review's known metadata blocker was reproduced both as an intentionally failing ignored regression and through a real runner. Its non-Unix capability finding was fixed; non-Unix runtime support remains unavailable. + +### First GitHub CI feedback + +The combined-feature job in run `41914870792` failed before compilation because Actions cache keys cannot contain commas. The matrix previously interpolated `geo-cloudfront,geo-maxmind` directly. Fixed the source by giving every lane a comma-free `cache_key` while keeping the real feature list unchanged. A focused old-versus-new matrix regression check, actionlint and scoped Prettier pass; evidence is `/tmp/geoip-ci-cache-{failure,regression}.log` and `/tmp/geoip-ci-static-final.log`. No Rust/runtime test inputs changed, so local suite results above remain applicable. GitHub checks for the follow-up commit remain pending, not claimed passing. + +### Work needed before acceptance + +- [ ] Obtain a strict metadata-verification/raw-decode fix in the reader, pin it, and prove the saved metadata fixture fails before initializer/bind/readiness. Remove the ignored regression only after it passes. +- [ ] Resolve non-Unix support versus an explicitly accepted Unix-only delivery scope. +- [ ] Measure a permitted representative City/Country artifact for compatibility, preparation time and peak memory under the documented limits. +- [ ] Complete mount-level and permission-denied proof where required, retaining the distinction from the passing file-mode and source-independence tests. + +The draft is an implementation checkpoint, not completed #438 acceptance, consent compliance or production qualification. diff --git a/docs/superpowers/specs/2026-10-05-axum-production-startup-lifecycle-design.md b/docs/superpowers/specs/2026-10-05-axum-production-startup-lifecycle-design.md new file mode 100644 index 000000000..77f43ee48 --- /dev/null +++ b/docs/superpowers/specs/2026-10-05-axum-production-startup-lifecycle-design.md @@ -0,0 +1,427 @@ +# Axum startup, storage, and bounded shutdown + +Date: 2026-10-05 +Status: initial implementation for review. The worktree implements the combined lifecycle and production-startup APIs below. Deployment guidance and the remaining acceptance checks are still open. + +Issues: [#392 lifecycle](https://github.com/stackpop/edgezero/issues/392) and [#396 configuration/storage](https://github.com/stackpop/edgezero/issues/396), under [Epic #391](https://github.com/stackpop/edgezero/issues/391). + +Base: [PR #275](https://github.com/stackpop/edgezero/pull/275), inspected at `5527b58e2a7cb6307d5c7a8926cd5dfc6f4f8b4a`. + +Worktree: `/home/pav/worktrees/issue-392-396-production-hosting-spec`. + +## Goal + +A production Axum application validates its runtime settings and initializes required stores before accepting traffic. Applications with additional configuration/bootstrap requirements validate them through the initializer variant before readiness. Its handlers can reuse startup-loaded state. On shutdown, readiness is revoked, new work stops, and dispatched work gets one finite drain budget. + +Preserve the existing permissive development entrypoint and local defaults. Strict production startup is an explicit API choice over the same runtime owner, service, and connection implementation. Do not infer production policy from release builds, containers, or the presence of path settings. + +Keep #275's fallible app assembly, owned HTTP/1 connections, local non-Send execution, and response-egress semantics. This work extends that runner rather than introducing another HTTP server or portable lifecycle framework. + +## Scope and ownership + +| Owner | Responsibility | +| ----------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| #392 | Native runtime ownership, signals/caller shutdown, lifecycle observation, dispatch gating, tracked draining, exit outcomes, and portable probe helpers | +| #396 | Explicit strict production settings/store preparation, development compatibility, app-owned initializer, shared typed-config loading, startup-only typed state, persistence/permissions/backup evidence, and production adoption | +| Application | Required schemas, config keys, extra secrets, signing/bootstrap policy, registered handler state, probe paths, and optional ongoing dependency checks | +| #395 | Residual outbound transport/wire acceptance; coordinate fallible transport startup here without building another client | +| #398/#399 | Container packaging and native architecture validation; consume this runtime contract for later integration checks | +| #397/#400 | Wider diagnostics/trusted-proxy work and deployment recipes, respectively | + +No live reload, filesystem watcher, online backup service, new backup CLI, distributed storage, automatic migrations, retry framework, tracing vendor, health configuration section, management listener, or public EdgeZero application image is included. No blanket Send bounds are added to portable futures or bodies. + +Trusted Server supplies concrete consumer requirements, but this spec does not implement its migration or certify its application image. + +## Current implementation and evidence + +All findings below are source inspection at the base above, not executed proof. Paths are repository-relative unless noted. + +| Area | Current behavior and source | +| ------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| App assembly | `crates/edgezero-core/src/app.rs`, `App::build` and `Hooks::configure`: install platform metadata and configure once. Configure receives no prepared native stores. | +| Standard entrypoint | `crates/edgezero-adapter-axum/src/dev_server.rs`, `run_app`: build app, resolve settings, bind, initialize stores, then serve. Generated main calls its no-argument form. | +| Local serving | The same file, `serve_local`: current-thread runtime/LocalSet, detached local connection tasks, Ctrl-C or internal oneshot exit. No SIGTERM or tracked graceful-drain phase. | +| Connections | `crates/edgezero-adapter-axum/src/connection.rs`, `serve_http1`: owned Hyper HTTP/1 keep-alive and response deadline enforcement. Extend this owner for draining. | +| Config files | `crates/edgezero-adapter-axum/src/config_store.rs`, `from_path` and `local_path`: flat string-map snapshots, missing-file empty stores, ancestor/cwd discovery. The runner drops malformed bindings with warnings. | +| KV identity | `dev_server.rs`, `kv_store_path` and `build_kv_registry`: name-derived slug/hash basenames under `.edgezero`. Required declared KV failures are already fatal. Each id currently attempts its own open. | +| Documented aliases | `docs/guide/kv.md:122`: ids with the same resolved platform name share a file. Per-id opens must be reconciled with this documented behavior. | +| Typed config | `crates/edgezero-core/src/extractor.rs`, `AppConfig` and `extract_from_handle`: bounded reads, envelope/integrity checks, secret resolution, schema validation, and redaction. This algorithm currently takes RequestContext. | +| Handler state | `crates/edgezero-core/src/router.rs`, `RouterBuilder::with_state`: typed state copied into requests, same-type last-write-wins. `App` currently has no post-build state setter. | +| Outbound startup | `crates/edgezero-adapter-axum/src/service.rs`, `AxumServiceState::from_app`: `try_transport().ok()` hides initialization failure; admitted requests subsequently fail. | +| Completion | `response.rs` and `crates/edgezero-core/src/response_egress.rs`: HostHandoff is not client receipt. Pre-egress cancellation can abandon callbacks while releasing captured resources. | +| Existing proof gap | `key_value_store.rs`, `data_persists_across_reopens`, tests reopen in one process, not subprocess restart, competing-writer rejection, or backup/restore. Existing server tests do not prove signal-driven draining. | + +Trusted Server at `666953a0de0dbec514b0f0bd5a8b03074f13e62b` was also inspected. Its Fastly adapter loads settings into Arc and captures that state in handlers, but its current entrypoint builds the app for each request. Native startup should retain loaded state for the native run, not assume Fastly cross-request retention. + +## Agreed runtime settings + +Use the existing `EDGEZERO__ADAPTER__*` convention. The table defines strict production options, not replacement defaults for the development runner. + +| Setting | Contract | +| ------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `HOST` | Omitted defaults to `127.0.0.1`. Supplied invalid values fail, rather than warn/fallback. Preserve current IP-literal support. | +| `PORT` | Omitted defaults to `8787`. Supplied malformed/out-of-range/zero values fail. Containers supply explicit binding values. | +| `CONFIG_DIR` | Explicit absolute, existing directory when local config stores are declared in production. | +| `DATA_DIR` | Explicit absolute, existing writable directory when embedded KV stores are declared in production. | +| `SHUTDOWN_GRACE_SECONDS` | Omitted defaults to 10. Supplied values are positive integer seconds; invalid, zero, overflow, or unusable timer values fail startup. No silent clamp/fallback. | + +Directory values are not required for undeclared store kinds. Validate supplied recognized settings instead of using invalid input as omission. Preserve native defaults for genuinely omitted optional settings. Do not add a global scanner for unrelated provider variables. + +The production standalone entrypoint reads environment settings once into native options. Embedding accepts explicit options and does not silently merge ambient hosting settings. An embedding caller can deliberately use the environment-loading constructor. Store-name/key overrides belong to that selected configuration snapshot too; this does not remove the environment-backed secret backend. + +Strictness includes recognized supplied native settings that previously fell back, such as logging level or declared-store name/key overrides. Keep shared EnvConfig fallback semantics unchanged for other adapters; perform strict native validation before using normalized values. Preserve `Hooks::owns_logging` and avoid installing a competing logger. + +There is no new CLI flag or runtime configuration file for these settings. Preserve development bind warnings/fallbacks, local config discovery, missing-file empty stores, malformed-binding warnings/omission, and cwd-relative KV defaults. Declared KV initialization failure remains fatal. Explicit production options never use those fallbacks. The new shutdown duration has no legacy permissive behavior to preserve; a supplied invalid duration fails in either entrypoint. + +The Axum CLI's existing development serve path keeps its permissive behavior. A wrapper launching a production entrypoint must pass supplied hosting settings through without replacing invalid input with defaults. Keep existing development wrapper normalization out of that production path; do not add a second CLI configuration policy. + +## Startup and initialization + +Proposed production ordering for the agreed invariant. Development uses the same lifecycle owner with its existing store/default policy and no additional initializer requirement: + +```mermaid +flowchart TD + A[Parse and validate native settings] --> B[Create owned local runtime and stop observation] + B --> C[App::build installs platform and configures once] + C --> D[Prepare required registries and shared transport] + D --> E[Await app-owned initializer] + E --> F[Register validated typed handler state] + F --> G[Bind and adopt listener] + G --> H[Ready and accepting] + C -->|Failure| X[Release resources and return non-success] + D -->|Failure| X + E -->|Failure| X + G -->|Failure| X +``` + +No acceptance or Ready transition can precede successful initialization. Binding after validation avoids a misleading open port during production standalone startup. An existing embedding path with a prebound listener can preserve that listener, but still cannot accept or become Ready early. + +Register standalone signals early enough to observe stop during initialization. Embedding supplies its stop source without installing process-wide handlers. A stop before Ready cancels cooperative initialization, releases prepared resources, and never enters Ready. Completed intentional stop exits zero. Initialization failure remains non-success. Cleanup uses the same shutdown budget; expiry or a second signal is forced stop. + +Run startup and connection serving on the same owned current-thread runtime/LocalSet. The initializer and its future need not be Send. Do not capture them in the existing spawn_blocking bridge. This design is a blocking native runner; it does not add another public async hosting framework. + +A supplied production initializer runs exactly once, after `App::build`/`Hooks::configure` and strict store preparation, before App is shared with connections. It receives mutable App and borrowed access to the same registry handles/bindings that requests will receive. It cannot turn failed declarations into absent bindings or replace prepared registries. Borrowed registry access does not make the underlying KV backend read-only; any app-owned bootstrap writes remain application policy. + +The application explicitly selects every required typed blob/schema and additional required secret or startup invariant. Store metadata contains ids/defaults, not schemas or required-key lists. A successful callback is evidence that the app's selected checks passed, not proof that the framework discovered every application requirement. + +Apps without application-specific startup requirements can use a production convenience entrypoint without an initializer argument. It still validates framework settings, declared stores, transport, and fallible app assembly before Ready. An explicit initializer variant adds app-selected blob/schema, secret, and bootstrap checks. A convenience entrypoint must not claim those application-specific checks occurred. + +Preserve the existing no-argument development entrypoint. Newly generated no-store apps remain runnable without a source migration or explicit no-op callback; a generated config Rust type alone does not declare a mandatory stored blob. Ready means the selected entrypoint's startup contract completed, not that the framework discovered every application requirement. Production applications with additional requirements must select the initializer variant. + +### Proposed native API shape + +These sketches describe the boundary, not compiled APIs or final Rust lifetime ergonomics. Public names are proposed. Declarations are API outline notation, not implementation code. + +```text +pub struct AxumRunOptions { + // Validated bind, roots, shutdown duration and selected store overrides. +} + +impl AxumRunOptions { + pub fn from_env() -> anyhow::Result; + // Explicit construction validates supplied native settings as well. +} + +pub struct PreparedStores { + // Private registry fields, None only for an undeclared/unattached kind. +} + +impl PreparedStores { + pub fn config(&self) -> Option<&ConfigRegistry>; + pub fn kv(&self) -> Option<&KvRegistry>; + pub fn secrets(&self) -> Option<&SecretRegistry>; +} + +pub type LocalStartupFuture<'a> = + Pin> + 'a>>; + +// Existing development convenience API and permissive local defaults. +pub fn run_app() -> anyhow::Result<()>; + +// Production convenience: strict environment options and SIGINT/Unix SIGTERM. +// Completes framework startup checks without claiming app-specific validation. +pub fn run_production_app() -> anyhow::Result<()>; + +// Production standalone with additional app-owned initialization. +pub fn run_production_app_with_initializer( + initialize: Initialize, +) -> anyhow::Result<()> +where + A: Hooks, + Initialize: for<'a> FnOnce(&'a mut App, &'a PreparedStores) + -> LocalStartupFuture<'a>; + +// Production embedding convenience: explicit options and caller-owned stop. +// Neither embedding variant installs global signal handlers. +pub fn run_app_with_options( + options: AxumRunOptions, + stop: Stop, +) -> anyhow::Result<()> +where + A: Hooks, + Stop: Future; + +// Production embedding with additional app-owned initialization. +pub fn run_app_with_options_and_initializer( + options: AxumRunOptions, + stop: Stop, + initialize: Initialize, +) -> anyhow::Result<()> +where + A: Hooks, + Stop: Future, + Initialize: for<'a> FnOnce(&'a mut App, &'a PreparedStores) + -> LocalStartupFuture<'a>; +``` + +Caller-owned stop can use an existing oneshot/watch/notification future. No new portable shutdown-token framework is needed. The native owner handles stop idempotently; repeat caller requests do not shorten the budget. The standalone signal source separately observes the second signal for forced teardown. + +These entrypoints are thin policy/initializer selections over one native runtime owner, not separate servers. Convenience variants supply an internal successful initializer; callers need not write a no-op. Retain the existing module location and existing development signatures unless an accepted prerequisite changes them. + +Router-only AxumDevServer callers retain their configuration and handle-attachment conveniences over that same owner; do not require an unrelated initializer migration. Normalize bare handles into the existing registry representation before any selected initialization. Tests may keep a private prebound-listener entrypoint through that same owner. + +## Shared typed-config loading and state retention + +Expose a context-free entrypoint to the existing typed loader, proposed as: + +```text +AppConfig::from_binding( + binding: &ConfigStoreBinding, + key: Option<&str>, + secrets: Option<&SecretRegistry>, + limits: ConfigExtractionLimits, + clock: MonotonicClock, +) -> Result; +``` + +This is signature notation, not literal Rust syntax. Implement it as an async inherent method with the existing `C: DeserializeOwned + AppConfigMeta + Validate + Send + 'static` bounds. A missing explicit key uses the binding's resolved default key. The app chooses default or named bindings; do not apply one guessed type to all declared stores. + +Factor only the RequestContext dependencies out of `extract_from_handle` and secret traversal. Request extraction and startup then share one parser, budget, envelope/version/hash checks, secret walk, optional-secret behavior, validation, and redacted error mapping. No fake request, route dispatch, second parser, operator TOML loader, or implicit cache is involved. + +Existing extraction limits apply per selected blob and its referenced secrets. They are not a deadline for the entire initializer or arbitrary signing/bootstrap work. They also do not bound the initial allocation of the outer local-file map. Do not add a separate startup-timeout setting or new numerical file-size policy in this slice. Operator-supplied files are trusted deployment inputs, and the supervisor owns the ultimate startup/stop bound. + +Add an explicit post-build method, proposed as: + +```text +App::insert_state(&mut self, value: T) +where + T: Clone + Send + Sync + 'static; +``` + +It uses the existing router state-extension mechanism. Same-type insertion replaces earlier state, including builder-registered state, for this App. Typical usage registers Arc and handlers extract State>. + +A router shared with another App must not be mutated globally. Use copy-on-write/forked router internals for state insertion; clone existing handler/middleware references and retain route metadata and matching behavior. Do not rebuild the app/routes, require Arc uniqueness, introduce a second state overlay with competing precedence, or use a global/deferred cell. Prove shared-router isolation in a test. + +State insertion is a startup operation and must finish before ingress admission or serving begins. Existing resolved-dispatch/admission tokens retain the prior router owner; copy-on-write insertion invalidates those tokens for the modified App. Do not promise that outstanding tokens survive insertion or introduce a new route-owner identity model. Document this boundary and test that a pre-insertion token is rejected while requests admitted after insertion dispatch normally. This API is not live state replacement. + +A non-Send initializer does not relax the existing Clone/Send/Sync bounds on registered handler state. Applications own state contents and any intentional interior mutability. Retention is explicit; ordinary AppConfig request extraction does not become a cache automatically. + +## Store and filesystem contract + +The requirements below define strict production preparation. Development retains its current local config discovery, empty/malformed binding policy, and cwd-relative data paths. File identities, alias handling, ownership limitations, snapshots, and safe recovery guidance apply in both paths. + +### Config + +- Read `/local-config-.json`, retaining the existing flat string-to-string format and logical-id identity. +- Every declared file must exist, be readable, and parse successfully. No empty fallback or dropped binding. +- Permit symlinks used by projected ConfigMap/Secret mounts. The opened target must be a readable regular file. Operator-supplied roots are trusted; no generic filesystem sandbox or source-tree containment restriction is added. +- A valid outer map is not proof of a valid typed blob or required secrets. Those checks belong to the initializer. +- Config stores remain startup snapshots. Updates require process restart; no watcher/reload path. + +### KV + +- Read/create `/kv--.redb`, preserving the existing basename derived from the resolved platform NAME. Config filenames still use logical ids. +- The root must already exist and be writable. The runner may create a new database file inside it, but must not fabricate a missing mount/root directory. +- Resolve aliases before opening: two logical ids with the same resolved NAME share one database handle and keyspace in this process. Different names select different files. Do not change file identity or silently move existing data. +- The same key through either alias refers to the same value. Alias ids are not isolated namespaces. Shared handles do not change operation guarantees or make the documented read-modify-write helper atomic. +- redb remains one owning process per database. A second process fails rather than silently sharing. Shared volumes do not establish multi-writer or multi-replica safety. +- Keep runner-owned handles alive until connection tasks settle, then relinquish them on completed startup failure, orderly stop, or cooperative forced cleanup. A database lock is released only when the last handle owner drops it. Embedding callers may retain an attached handle, and initializers may clone one outside the App; those owners must release their handles before reopening or backing up the database. The runner does not revoke caller-owned handles. + +### Permissions, updates, and recovery + +Operators provision roots, ownership, and permissions for the runtime UID, including the Docker recipe's numeric UID. Config may be read-only; data must support real database writes. No startup chown, chmod 777, automatic lock repair, database deletion, or corruption recovery is added. + +Externally supplied secret updates use process restart. The existing environment-backed secret backend remains unchanged; this is not a new secret reload/rotation system. + +Initial backup support is a documented stopped-writer procedure: stop the writer, release every database handle including caller-owned clones, copy the closed database, restore into a prepared directory with correct permissions, reopen, and read stored values back. Returning from an embedded runner alone does not prove the database is closed. Online copying of an active database is not a supported backup claim. Remove ordinary recovery advice that recommends deleting production databases for lock errors or capacity maintenance; do not invent a maintenance subsystem. + +The storage guide must include these operational limits and procedures: + +- TTL expiry is lazy. Reads and paginated listing clean up encountered expired entries; unvisited expired entries can remain. An application-owned maintenance pass can traverse the existing paginated listing API through the owning process, following cursors to exhaustion even when a page is empty. There is no automatic sweeper or new maintenance CLI. +- Deletes and expiry cleanup do not automatically shrink the database file. Monitor database size and available mount capacity, provision headroom for writes and backups, and do not treat TTL as a disk quota. This slice does not expose or certify compaction; unsupported file shrinking must be stated explicitly. +- For lock conflicts, identify and stop the other owning process or release retained embedding handles before retrying. Do not remove the database or bypass its lock. Prevent overlapping writers during restart/rollout. +- For capacity exhaustion, stop the writer if needed, preserve the database, and restore usable capacity by enlarging the mount or removing unrelated disposable files. Reopen and verify stored values once capacity is available; backup/restore copying alone is not a compaction procedure. +- For suspected corruption, stop all owners, preserve the damaged file for investigation, and restore a known-good stopped-writer backup into a prepared directory. Verify stored values before resuming traffic. Automated repair and untested salvage procedures remain unsupported. + +## Lifecycle, shutdown, and results + +One native owner controls Starting, Ready, Draining, and Stopped. Acceptance and phase observation derive from that owner, not separately maintained booleans or response reports. A run never returns to Ready after draining begins. + +```mermaid +flowchart TD + A[Starting] -->|Validated service and listener| B[Ready] + A -->|Stop before ready| C[Cleanup without readiness] + A -->|Initialization failure| G[Stopped with non-success] + B -->|First stop request| D[Draining: revoke ready and block new work] + D --> E[Graceful connection completion] + E -->|All tasks settled| F[Stopped with zero exit] + C -->|Cleanup completes| F + D -->|Budget expires or second signal| H[Force cancellation and settle tasks] + C -->|Budget expires or second signal| H + H --> G +``` + +Standalone SIGINT and Unix SIGTERM feed the same first-stop transition. Registration failure is a startup error. A second termination signal of either kind requests immediate forced teardown. Embedding has no implicit global handlers. + +On first stop: + +1. Revoke readiness and stop accepting new connections. +2. Gate new request dispatch on existing keep-alive connections too. Define existing work at the service-dispatch boundary, not merely an accepted socket or queued pipelined bytes. +3. Ask owned Hyper connections to stop keep-alive/new work and finish dispatched requests. Close idle connections promptly. +4. Keep polling tracked local tasks and request/response work until completion or the single process drain deadline. +5. On expiry or second signal, cancel remaining work and settle its task ownership before returning. + +Use native monotonic runtime time for the process budget. Do not depend on an application's injected request clock for supervisor termination. Earlier accepted request/response deadlines still win. Long-lived/SSE responses share the shutdown budget; this does not impose a new ordinary stream-lifetime or handler deadline. + +Collect connection task results instead of detaching them. Preserve ordinary connection-scoped failure/disconnect behavior. Terminal listener/runtime/runner failures revoke readiness and return non-success; do not add a general restart/retry policy here. + +Completed graceful drain or completed intentional stop during startup exits zero. Startup failure, runner failure, grace exhaustion, and signal-requested forced teardown report non-success. Retain enough internal outcome distinction for focused tests and safe diagnostics, without introducing a detailed numeric exit-code taxonomy. + +The budget is cooperative. Synchronous startup code, blocking handlers, callbacks, destructors, or a stalled runtime can prevent timer/cancellation progress. The process supervisor supplies the final hard-kill boundary. Deployment examples must set a longer supervisor interval, such as 30 seconds for a 20-second drain, rather than rely on Docker's default timeout. + +Preserve #275 completion semantics. HostHandoff does not prove client receipt. Dropping unfinished egress uses its existing terminal outcome. Pre-egress cancellation may release captured resources without a completion callback/report. Do not promise a terminal report for every accepted request, SIGKILL, or process abort; connection ownership determines drain completion. + +## App-owned probe routes + +Add a small portable `edgezero_core::probe` module with: + +- A Starting/Ready/Draining/Stopped observation enum. +- A cloneable read-only live reader, following the existing MonotonicClock Arc/callback pattern. +- Ordinary `#[action]` handlers for readiness and liveness. + +The native runner publishes its reader through existing request extensions. Core contains no lifecycle writer, shutdown token, listener, task, timer, Axum/Tokio dependency, new action capability, or portable async Hooks method. Other adapters initially publish no native lifecycle observation. + +Handlers read the current phase when executed, not a copied admission-time snapshot. + +| Observed phase | Readiness | Liveness | +| --------------------- | ---------------------------- | ---------------------------- | +| Starting | 503 | 200 | +| Ready | 200 | 200 | +| Draining | 503 | 200 | +| Stopped | 503 | 503 | +| Missing/unknown state | 503, probe state unavailable | 503, probe state unavailable | + +Use short non-sensitive text, plain-text content type, and Cache-Control: no-store. Missing state must not invent healthy behavior or lifecycle support on a provider. Apps can retain an independent portable health handler when that is their desired policy. Ready reflects the selected runner's contract: development readiness is not strict production validation, and production convenience readiness does not imply application-specific initialization. Native readiness does not periodically revalidate dependencies. Applications can add ongoing checks; the framework does not poll upstreams or trigger automatic outage restarts. + +Applications bind helpers through ordinary Rust or manifest routes. No automatic paths, reserved namespace, health flag, health config section, path environment variable, management listener, or admission exemption. Existing middleware/auth remains application-owned. Do not replace framework-owned probe/store metadata through app state or middleware; the native dispatch gate must use its private owner rather than an overrideable request extension. + +There is no reachable-probe guarantee before startup acceptance or after listener withdrawal. The response table describes a helper that executes. During drain, a previously dispatched probe can observe Draining; a newly attempted request may receive refusal/reset/EOF instead of 503. No successful readiness response means not ready. Supervisor liveness policy must account for intentional listener shutdown. + +Trusted Server already owns `/health`. Its Axum handler returns 200 on a successfully built router, while startup fallback returns errors even for `/health`. Fastly answers `/health` before app construction, so that route is liveness rather than readiness. Keep those downstream routes application-owned. Generated EdgeZero root/introspection paths are editable trigger rows, not reserved paths; the old automatic `/__edgezero/routes` mechanism was removed. + +## Failure diagnostics + +Startup failures identify the stage, recognized setting/logical store id, and safe failure category. Never print config content, resolved secrets, raw invalid settings, parser input fragments, or callback/provider source strings. A malformed config must not leak through a serde error wrapped in an ordinary log or anyhow termination chain. + +Return safe classified errors at the standard runner boundary. Do not retain an unredacted source chain where ordinary main termination/debug formatting will print it. Broader permitted internal diagnostics belong to #397; this spec does not add a logging framework or debug disclosure switch. + +Use the existing public HTTP error contract for request failures. Probe bodies contain no app configuration or infrastructure details. Preserve custom application logger ownership. These guarantees apply to framework-owned error conversion/logging; arbitrary app logging and panic payloads remain application responsibilities. + +## Compatibility and entrypoint migration + +Preserve #396's permissive development requirement and the existing no-argument `run_app` signature. This review decision supersedes the earlier interview's strict-everywhere and mandatory-initializer policies. No GitHub issue amendment is needed. + +Inventory Axum main templates, app-demo's native entrypoint, CLI demo runner, AxumDevServer/test embedding callers, and downstream consumers. Do not migrate development callers merely to make them pass a no-op initializer. Add production adoption examples that select the strict convenience entrypoint or, for application-required checks, its initializer variant. Both use the existing server implementation. Document their different readiness guarantees and any prerequisite-driven source migration. + +Generated no-store apps retain their runnable development entrypoint. Show the explicit production entrypoint choice and how required stored config adds typed loading and state registration. Container recipes under #398 must select that production contract rather than certify a permissive development binary from a passing HTTP probe. Do not add schema inference, a new app! keyword, optional-store policy machinery, or a native dependency in the shared app crate. + +The native CLI's serve/push/provision recipes retain local preparation and path conventions. Production adoption documentation supplies explicit absolute roots; development does not acquire a mandatory-root requirement. `config push` retains its current project-local output convention. No automatic production push/deployment mechanism is added. + +Post-build state registration remains portable and preserves router state semantics. The context-free config helper preserves all existing request-extraction outcomes on native/WASM targets. Coordinate #379's retained-provider-app work where APIs overlap, without making it a required native hosting dependency or assuming retained provider apps imply process readiness. + +## Originally proposed PR split + +The original plan below separated the issue boundaries. This worktree delivers the shared runtime and startup/storage changes together in one PR stacked on #275, not old main. The issue ownership and acceptance boundaries still apply. + +1. **#392 lifecycle PR, first on #275.** Establish the single local runtime owner, explicit initialization barrier, caller/signal stop handling, tracked connections, dispatch gating, drain/result behavior, and core probe observation/helpers. Its initialization barrier can accept a local future without knowing application schemas. Do not claim strict config/value certification before #396 integrates. +2. **#396 startup/storage PR, stacked on #392.** Supply explicit strict production preparation and optional app initialization to that barrier; implement root/settings validation, alias handle reuse, shared typed loader, startup-only post-build state, production adoption examples, development compatibility, and persistence/permissions/backup checks. +3. **Integrated acceptance on the second PR.** Exercise failed initialization, phase transitions, signals, pending handlers/streams, durable restart, mounted permissions, and probe wiring against the combined series. Close overlapping issue acceptance only with that evidence. + +These are behavioral review boundaries, not permission for two independent agents to rewrite the same runner concurrently. Preserve existing development signatures and keep any prerequisite-required caller changes at a coherent boundary. Add production entrypoints and real required-config initialization under #396 without forcing a development migration. Do not leave a broken intermediate commit or invent a compatibility server. + +#393/#394/#395/#397 then add only residual work/evidence against the accepted foundation. #398/#399 can prepare packaging independently; final container production checks consume the integrated result. #400 documents proven behavior. + +## Likely affected files + +- `crates/edgezero-adapter-axum/src/dev_server.rs`, `connection.rs`, `service.rs`, `config_store.rs`, and `key_value_store.rs`, with colocated tests. +- `crates/edgezero-adapter-axum/src/cli.rs` and its native main template where needed to preserve development behavior and document explicit production adoption. Strict launch wrappers must not normalize invalid production inputs. +- `crates/edgezero-core/src/app.rs`, `router.rs`, and `extractor.rs` for state registration and shared typed loading; new `probe.rs` plus exports in `lib.rs`. +- CLI demo runner, root README/core-config templates where preparation guidance is stale, generated-project tests, and app-demo's Axum main. +- Axum/config/KV guides and small native/portable fixtures needed for the acceptance checks. Do not pull Trusted Server sources or operator configs into framework fixtures. + +Keep this list bounded. No general generator/dependency resolver, provider lifecycle, portable body, or distributed store rewrite is implied. + +## Acceptance and planned validation + +The table below is the full acceptance plan, not a claim that every item is complete. The initial implementation has passed workspace tests, strict workspace Clippy, formatting, the Fastly/Cloudflare/Spin feature check, and the Spin WASM compilation check. The Axum all-features suite also passed all 13 subprocess hosting tests, including signals, finite drain, forced cancellation, startup failures, embedding, persistence, aliasing, stopped-writer backup/restore, and read-only config permissions. + +Before choosing implementation changes, re-audit the accepted #275 integration revision and record its exact commit, existing contracts/tests reused, checks already covered, and residual implementation/evidence needed for each issue. Repeat the inventory if prerequisite contracts change. The draft-head source inspection above is not this acceptance gate or executed production proof. + +| Owner | Focused evidence required | +| ----- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| Both | Accepted-#275 revision and covered/residual inventory are recorded; add only residual changes and missing acceptance evidence. | +| #392 | Subprocess SIGINT and Unix SIGTERM each cause graceful stop; second signal forces non-success. Embedding stop installs no global handlers. | +| #392 | In a running subprocess, dispatch a delayed finite response before sending process-wide SIGTERM; observe readiness withdrawal, full response completion, and zero exit. Repeat SIGTERM with a never-ending handler and open-ended stream to prove forced non-zero exit within the configured cooperative budget. Reuse these scenarios for final-image integration; separate signal and unit drain tests do not replace them. | +| Both | Production subprocess failures cover an occupied bind port, explicit initializer failure, invalid recognized settings, missing roots/declared config files, unreadable/non-regular/malformed config, and unusable/locked KV data. Each exits non-zero without Ready or request acceptance and identifies a safe stage/setting or logical id/category; sentinel values and unredacted source chains never appear in ordinary stdout/stderr diagnostics. | +| #392 | Never Ready during startup/failure; startup stop never accepts; Ready only after successful barrier and listener setup; drain revokes it immediately. | +| #392 | Delayed dispatched work finishes; idle keep-alive closes; new keep-alive/pipelined work does not dispatch after drain. | +| #392 | Never-ending handler/stream and zero-read peer reach controlled forced teardown within the configured cooperative budget. No detached work remains when the runner returns. | +| #392 | One budget is not restarted by repeat programmatic stop. Existing earlier response deadlines/outcomes remain intact. | +| #392 | Probe helpers cover every phase and missing state through ordinary router dispatch, preserve app route ownership, and observe a live transition rather than a stale snapshot. New drain probes need not reach HTTP. | +| #396 | Production entrypoints reject invalid supplied host/port/logging/known overrides/duration without normalization into defaults. Omitted bind/duration values use documented defaults. Production launch wrappers preserve this behavior. | +| #396 | Existing development entrypoints remain runnable without new root settings or initializer arguments. Preserve missing-config empty stores, malformed-binding warning/omission, local path discovery/cwd defaults, existing bind fallbacks, and fatal declared KV failures. Diagnostics remain redacted. Selecting production explicitly enforces strict checks regardless of cwd or build profile. | +| #396 | A standalone release subprocess using the production entrypoint starts from unrelated cwd with only explicit roots; missing roots, unreadable/non-regular/malformed config, and unusable/locked data fail before acceptance. | +| #396 | Valid flat map containing malformed/missing required typed blob, integrity failure, invalid required fields, or missing referenced secret fails the initializer before Ready. Sentinel values never appear in ordinary logs/returned diagnostics. | +| #396 | A supplied initializer runs once with exactly the bindings attached to requests. Production convenience entrypoints require no callback but still enforce framework checks and never claim app-specific validation. Native production option construction does not silently merge environment settings. | +| #396 | Shared typed loader and request extraction agree on default KEY overrides, named stores/secrets, optional secrets, redaction, byte/structure/deadline limits, and existing error categories. | +| #396 | Startup-registered Arc state reaches multiple requests; last same-type insertion wins; another App sharing the original router retains its own state. Pre-insertion resolved/admission tokens are rejected for the changed App, while tokens created after insertion dispatch normally. | +| #396 | Distinct names stay distinct; same-name aliases share one handle/keyspace; fresh database in prepared root works; second process cannot own the same file. | +| #396 | Value survives actual process stop/restart; stopped-writer backup restores readable values; runtime UID works with read-only config and writable data; startup never changes root ownership or deletes locked files. An embedding owner retaining a handle keeps the file locked after runner return; dropping the last owner permits reopening. | +| #396 | Storage guidance covers lazy expiry, paginated cleanup, non-shrinking files, capacity monitoring/headroom, safe lock/capacity/corruption recovery, and unsupported compaction/repair. Backup instructions require all handle owners to release the database. | +| #396 | Config snapshot stays unchanged after file update until restart. Document restart-based secret deployment updates without unsafe process-environment mutation tests. | +| Both | Non-Send local initializer fixture compiles/runs; same shared probe/typed-state handlers compile for supported WASM targets without native dependencies. | + +Tests use disposable data, bounded waits, explicit cleanup, and no platform credentials. Ordinary unit/compile tests do not require Docker or external services. Signal/filesystem subprocess tests use local fixtures. UID/container evidence can reuse #398's opt-in runner after integration with its template PR; unavailable Docker/architecture evidence remains a reported gap. + +At implementation checkpoints, use the repository's required checks: + +```sh +cargo test -p edgezero-core --all-targets +cargo test -p edgezero-adapter-axum --all-targets --all-features +cargo test -p edgezero-cli --all-targets +cargo test --workspace --all-targets +cargo fmt --all -- --check +cargo clippy --workspace --all-targets --all-features -- -D warnings +cargo check --workspace --all-targets --features "fastly cloudflare spin" +cargo check -p edgezero-adapter-spin --target wasm32-wasip2 --features spin +cargo test -p edgezero-cli --test generated_project_builds -- --ignored +``` + +Also run docs lint/format/build and the existing example/provider gates relevant to changed callers. Installed-target or container gaps must be disclosed, not presented as passes. Existing Fastly/Cloudflare/Spin gates must stay green. Container UID, standalone release, unreadable/unusable mount, retained-handle lock, and production-guide acceptance remain open. The accepted-prerequisite re-audit must also be repeated after #275 merges. + +## Risks and review checks + +- Compile the proposed higher-ranked initializer signature with borrowed inputs and Rc-capturing non-Send futures before committing to its ergonomics. No runtime ownership workaround may add blanket Send bounds. +- Verify copy-on-write router state insertion against the locked matchit/http types and existing Clone/Send/Sync state bounds. No uniqueness assumption or cross-App mutation is acceptable. +- Production strictness is opt-in. Preserve existing development signatures, fallbacks, and data filenames. Make production adoption and readiness guarantees explicit; do not silently certify a development binary or force unrelated callers to migrate. +- App-owned initialization/state can execute arbitrary code. Cooperative cancellation does not bound blocking code/destructors or prevent unsafe app overrides of request metadata. The supervisor and trusted app-composition contract remain explicit. +- Existing typed limits do not prove startup-file or whole-process memory bounds. This spec adds no concurrency/handler-deadline policy and does not relabel BestEffort/Unsupported guarantees from #275. +- A successfully sampled readiness response does not promise the server remains ready through transmission or prove client receipt. + +Read-only audits and alternative proposals were used for lifecycle, configuration, native startup API, and probe portability. Their useful findings are incorporated above. Rejected alternatives were a second server, inferred environment modes, native dependencies in shared probe handlers, a portable async startup hook, synthetic request validation, implicit settings caches, per-id path machinery, and immediate-stop-only hosting. This review preserves permissive development and makes strict production an explicit entrypoint choice, superseding the earlier strict-everywhere and mandatory-no-op proposals. + +## Review boundary + +The interview's policy decisions and subsequent review revisions are recorded here. The initial source and test implementation covers development compatibility, optional application initialization, startup-only state insertion, explicit embedding options, restart-based config updates, and the native lifecycle owner. API and behavioral review remain necessary before merge. + +The implementation is based on #275 at `5527b58e2a7cb6307d5c7a8926cd5dfc6f4f8b4a`. Its branch has since advanced to `dfd439c613af4f6ed295e71c032881b15bf7247c`, including overlapping Axum preflight and logging changes. Integration with that newer prerequisite and its accepted-revision re-audit remain open. This work does not certify a deployment or published container image, and it does not complete all acceptance criteria for #392 or #396. diff --git a/docs/superpowers/specs/2026-10-07-native-store-providers-design.md b/docs/superpowers/specs/2026-10-07-native-store-providers-design.md new file mode 100644 index 000000000..f08dd1a93 --- /dev/null +++ b/docs/superpowers/specs/2026-10-07-native-store-providers-design.md @@ -0,0 +1,425 @@ +# Native store providers independent of HTTP adapters + +- Status: Offline implementation verified and published as draft [PR #414](https://github.com/stackpop/edgezero/pull/414) via `gh stack`, based on open #275 with the compatible #406 lifecycle bridge. Real AWS service acceptance is declined and excluded. +- Date: 2026-10-07 +- Owning issue: [EdgeZero #410](https://github.com/stackpop/edgezero/issues/410). +- Audited main: `683202c66948146ec360f490126f1d60f920288b`. +- Initial host: Axum. The provider implementations must not depend on Axum. + +**2026-10-14 scope extension:** the separately approved [DynamoDB KV plan](../plans/2026-10-14-dynamodb-kv.md) adds live ordinary KV through the default-off `aws-dynamodb` feature and binding tag on the same branch/PR. DynamoDB deferral statements below describe the original AppConfig/Secrets snapshot phase, not the extended implementation. DynamoDB configuration, CAS/transactions and deployed AWS acceptance remain excluded. + +## 1. Outcome and invariants + +A native application chooses its HTTP runtime and its KV, configuration, and secret providers independently. It can also choose different providers for different logical IDs of the same store kind. + +For example, Axum can serve an application with redb application state, AppConfig configuration, and environment secrets. Another deployment of the same application can use local configuration and Secrets Manager. A future DynamoDB KV provider replaces a KV binding, not the HTTP runtime or configuration provider. + +The application continues to use `KvStore`, `ConfigStore`, `SecretStore`, their handles, and the existing per-ID registries. AWS resource names, credential discovery, protocols, and service errors belong behind that boundary. + +Required invariants: + +- Provider selection never changes the meaning of a logical store ID or its default. +- Selecting an AWS binding never silently falls back to local files or environment secrets. +- Raw configuration, serialized `BlobEnvelope` values, and nested secret references retain their meanings. +- AWS and native runtime dependencies stay outside portable core and provider-WASM dependency graphs. +- Preparing a provider does not claim application hot reload, safe signing-key rotation, or distributed KV guarantees. +- Provider failure and an absent lookup key remain distinguishable. + +## 2. Audited baseline and integration prerequisites + +These are current main behaviors, not proposed contracts. + +| Area | Current behavior | Source | +| ------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------- | +| Store declarations | Only logical IDs and defaults are portable. Provider fields in `[stores]` and per-adapter store tuning are rejected. | `crates/edgezero-core/src/manifest.rs:350-372`, `:469-505` | +| Macro metadata | `StoreMetadata` carries IDs/default only. No provider initialization is generated. | `crates/edgezero-core/src/app.rs:77-102`; `crates/edgezero-macros/src/app.rs:91-124` | +| Config | Read-only async key lookup returning `Option`. | `crates/edgezero-core/src/config_store.rs:206-214` | +| Secrets | Read-only named-store/key lookup returning bytes. | `crates/edgezero-core/src/secret_store.rs`, `SecretStore` | +| KV | Mutable bytes, TTL, existence, deletion, and prefix pagination. | `crates/edgezero-core/src/key_value_store.rs:880` | +| Composition | Registries already bind a provider handle per ID. Config adds a default key; secrets add a bound namespace. | `crates/edgezero-core/src/store_registry.rs:35-84`, `:115-230` | +| Native injection | Service and dev-server builders accept existing registries. Generated `run_app` hard-wires redb, local JSON, and environment secrets. | `crates/edgezero-adapter-axum/src/service.rs:49-123`; `src/dev_server.rs:142-192`, `:333-509` | +| Typed configuration | Request extraction reads and verifies an envelope, resolves nested secret references, then deserializes and validates. It is not automatically a startup check. | `crates/edgezero-core/src/extractor.rs:748-813`, `:970-1195` | + +Two open PRs supply the prerequisite contracts. After the initial audit, the user explicitly authorized implementation and rebasing onto them. The worktree now uses #275 at `d1a81579afb96f200e7fe8fd294600f0b446b080` with #406 rebased onto it as combined base `1499062`. Neither source PR is accepted on main. References below to accepted prerequisite contracts mean the exact selected and tested stack for this implementation, not a claim about main or GitHub approval. + +Initially inspected heads: + +- [PR #275](https://github.com/stackpop/edgezero/pull/275), inspected at `d1a81579afb96f200e7fe8fd294600f0b446b080`, introduces fallible `App::build::(platform)`, shared `ConfigValue`, and required `ConfigStore::get_bounded` and `SecretStore::get_bytes_bounded` methods with a clock, absolute deadline, and separate backend/value allowances. Do not implement against main's old `String`-only contract or invent a parallel bounded-read API. +- [PR #406](https://github.com/stackpop/edgezero/pull/406), inspected at `3ab22096fd117f7506cf437b5ba128e0789e059c`, implements #392/#396 production startup and an app initializer over `PreparedStores`. At that head, `PreparedStores` exposes read access, not provider override construction. `AxumRunOptions::validate_stores` requires config/data roots for every declared config/KV kind. `build_production_stores` constructs local providers directly. + +Re-audit and test the exact selected stack before runtime integration. Open status is not a blocker under the user's subsequent instruction. In particular, remote configuration must not require `CONFIG_DIR`, and an injected remote KV must not require `DATA_DIR`. Mixed deployments require roots only for bindings still using local defaults. Reuse the accepted production initializer, readiness, shutdown, and configuration extraction contracts. + +Coordinate raw-value semantics with [#324](https://github.com/stackpop/edgezero/issues/324). Do not copy Fastly's current envelope-only raw lookup behavior into a new provider. + +## 3. Proposed decisions + +| Decision | Proposal | +| -------------------------- | --------------------------------------------------------------------------------------------------------------------------------------- | +| Application API | Keep the existing store traits, handles, and registries. No unified `Store` trait. | +| Provider unit | A binding for one store kind and logical ID. Same-kind local and AWS bindings can coexist. | +| AWS packaging | One optional native `edgezero-store-aws` crate, independent of any HTTP adapter. | +| General extension | Application-supplied per-ID handles at native startup. No dynamic plugin registry or provider discovery. | +| Bootstrap | An explicitly located native bindings file, or programmatic bindings. Do not bake AWS references into portable declarations. | +| Initial config provider | AppConfig Agent, free-form whole-document reads. No direct AppConfig Data API implementation initially. | +| Initial secret provider | Secrets Manager through the AWS SDK for Rust. Whole-value reads with current or pinned version selection. | +| Config and secret adoption | Load all explicitly mapped AWS values at startup and publish immutable process snapshots. Restart to adopt updates. | +| DynamoDB config | Evaluated and deferred. AppConfig plus Secrets Manager is the initial #410 service set. | +| DynamoDB KV | A separately approved provider implementing the ordinary KV contract. The composition boundary supports it; #410 does not implement it. | +| Publication/provisioning | Runtime readers only. Existing operator tooling owns AWS resource creation and publication. | + +The user's implementation instruction authorizes these choices. Direct polling, live secrets, or DynamoDB configuration would change the approved protocol and scope. + +## 4. Architecture and ownership + +```mermaid +flowchart TD + App[Portable application and logical store IDs] --> Traits[Core store traits and handles] + Traits --> Registries[Per-ID KV, config, and secret registries] + Bootstrap[Native bootstrap settings and supplied handles] --> Prepare[Native startup composition] + Prepare --> Local[Existing redb, JSON, and environment providers] + Prepare --> AWS[Optional edgezero-store-aws factories] + Prepare --> Custom[Other providers implementing core traits] + AWS --> Agent[AppConfig Agent HTTP endpoint] + AWS --> SM[AWS Secrets Manager SDK] + Local --> Registries + AWS --> Registries + Custom --> Registries + Registries --> Startup[App-owned typed configuration and secret checks] + Startup --> Runtime[Axum initially, other native HTTP runtimes later] +``` + +### Core + +Own store contracts, opaque handles, logical metadata, registries, envelope validation, and typed extraction. Do not add AWS enums, credentials, resource references, remote scheduling, or native executor requirements. + +Traits retain `#[async_trait(?Send)]`. Store implementations remain `Send + Sync`; changing the future bound to `Send` would break the portable contract. + +AWS snapshot implementations must implement both ordinary and bounded reads under the accepted #275 contracts. Bounded reads reject an already-expired deadline, apply independent backend/value allowances, report the snapshot payload bytes exposed to the caller, and recheck deadline precedence before returning. A successful missing lookup reports zero payload bytes. No bounded method may delegate to an unbounded cloud operation; cloud access happens only during preparation. + +### AWS provider crate + +Expose typed settings and async preparation methods producing config/secret handles through an explicit shared `AwsPreparation` session. The standard runner owns one session for all its built-in AWS bindings; programmatic callers explicitly create and reuse a session. It owns aggregate budgets, successful-target deduplication, and client reuse. Do not hide independent per-binding sessions behind convenience constructors. Preparation methods own AWS access, strict resource mapping, sanitized error classification, and immutable snapshot construction. They do not serve HTTP requests, discover application secret names, create another runtime, or initialize global logging. + +Suggested independent features are `appconfig-agent` and `secrets-manager`, with no default features. Use target-specific native dependencies. AWS SDK and any Tokio timer dependency belong in this native provider crate, never core or the WASM adapters. Reuse the existing native runtime; do not add another Tokio dependency to an adapter. + +The AWS crate must not depend on Axum, the CLI adapter registry, or downstream application crates. Future provider crates can implement the same contracts without depending on it. + +### Native composition + +Extend the accepted production store-preparation step with per-kind, per-ID override inputs using existing `KvHandle`, `ConfigStoreBinding`, and `BoundSecretStore` types. Permit async setup inside the native runtime before registries are finalized. The application initializer consumes the completed registries; it is too late to select providers there. + +For each declared ID, choose exactly one source: + +1. An application-supplied binding. +2. An explicit bindings-file entry. +3. The existing local default for that kind. + +Reject duplicate supplied/file bindings for the same ID rather than silently choosing precedence. Reject undeclared IDs. Build each registry once and retain its declared default. Validate local root requirements after determining which bindings need those roots. Complete source validation before opening runner-owned providers or polling supplied deferred futures. Constructing a deferred future must be side-effect-free, with provider work inside its async body. This ordering guarantee cannot undo caller-side work or prevent a supplied ready handle from already being open. + +The standard Axum runner can dispatch the small set of built-in binding tags to constructors. A custom provider needs only a supplied handle, not a new tag in a global registry. Share these handles between startup checks and requests. + +Keep existing redb/file/env implementations in their current crate initially. Extracting all local providers into new crates would add unrelated migration work. A later native runtime can adopt or move them separately. + +## 5. Bootstrap and binding shape + +Keep `edgezero.toml` portable: + +```toml +[stores.kv] +ids = ["state"] + +[stores.config] +ids = ["app", "routing"] +default = "app" + +[stores.secrets] +ids = ["signing", "notifications"] +default = "signing" +``` + +The proposed native input is an absolute `EDGEZERO__STORE_BINDINGS_FILE` path, or an explicit options method. Read it once during startup. With no input, preserve existing development defaults and the accepted production defaults. A missing explicitly selected file is an error, not an empty binding set. + +Proposed file, with fictional resources: + +```toml +version = 1 + +[config.app] +provider = "aws-appconfig-agent" +default_key = "settings" +endpoint = "http://127.0.0.1:2772" +# Optional local agent authorization, read from the bootstrap environment. +access_token_env = "APPCONFIG_AGENT_TOKEN" + +[config.app.documents.settings] +application = "example-app" +environment = "production" +profile = "application-envelope" + +[config.routing] +provider = "aws-appconfig-agent" +default_key = "allowlist" +endpoint = "http://127.0.0.1:2772" + +[config.routing.documents.allowlist] +application = "example-app" +environment = "production" +profile = "routing-allowlist" + +[secrets.signing] +provider = "aws-secrets-manager" +region = "us-east-1" + +[secrets.signing.entries.active_key] +secret_id = "arn:aws:secretsmanager:us-east-1:111122223333:secret:example-signing-AbCdEf" +version_stage = "AWSCURRENT" +``` + +`state` remains redb and `notifications` remains environment-backed. Replacing either is an independent operation. The schema may later add a `dynamodb` KV tag, but v1 must reject it as unsupported rather than accepting a nonfunctional placeholder. + +Validation rules: + +- Require schema version 1. Reject unknown kinds, fields, provider tags, duplicate tables, empty mappings, and undeclared IDs. IDs and lookup keys are case-sensitive. +- Parse provider settings into tagged types. Reject irrelevant provider fields instead of retaining an unvalidated option bag. +- Require the config `default_key` to name a document mapping. Multiple lookup keys require explicit entries. +- Require an explicit Secrets Manager region and complete ARN for each secret. Validate ARN service, region, and resource shape. Cross-account reads are allowed only with explicit full ARNs and operator-managed permissions. +- Allow one version selector per secret: omitted/current stage, an explicit stage, or a pinned version ID. Reject simultaneous `version_id` and `version_stage`. Although AWS can accept a matching pair, v1 does not need that extra state. +- Existing `EDGEZERO__STORES______NAME` and `__KEY` behavior remains unchanged for implicit local bindings. Reject those supplied overrides for an explicitly bound ID. Do not repurpose a platform name as an AWS ARN. +- Explicit options and supplied handles do not silently merge ambient environment settings. The env-loading entry point is the only path that reads the file locator from the environment. +- The file contains resource references and environment variable names, never credential or secret values. No remote configuration can supply its own bootstrap credentials, endpoint, region, or binding file. + +Keep parsing native-side. The native bindings module owns the versioned top-level file, store kinds/IDs, source collisions, provider-tag dispatch, local defaults, scalar-overlay conflicts, and root requirements. The AWS crate owns strict AWS settings types and resource-shape validation. It must not require changes to `StoreDeclaration`, `StoreMetadata`, or macro-generated resource metadata. Programmatic callers use typed constructors and existing handles without a bindings file. + +The env-loading entry point captures the file locator and scalar overlays once. Explicit options use only supplied bootstrap settings. Both use the same source validator. Offline CLI validation does not treat its own ambient environment as the future process's deployment configuration; conflicts with actual deployment overlays are always checked again at startup. + +## 6. AppConfig configuration + +### Access path + +Use AWS AppConfig Agent initially. AWS recommends it for local caching, polling interval handling, and configuration token management. EdgeZero uses its local HTTP GET endpoint; it does not implement session tokens or a polling loop. + +The operator owns the agent process/container, its AWS role and region, its pinned version, and its network policy. EdgeZero neither launches nor supervises it. The binding identifies application, environment, and profile; the agent's AWS bootstrap is separate from the application binding. + +V1 permits only an HTTP endpoint with a literal loopback IP address. Reject URL userinfo, query strings, fragments, unexpected base paths, redirects, and non-loopback destinations. Construct path segments with proper escaping. Disable ambient HTTP proxies for this client. A token, when configured, comes from the named bootstrap environment variable and never enters diagnostics. Separate-pod or remote-agent access requires a later explicit transport/security design. + +When token authentication is configured, the operator sets Agent `ACCESS_TOKEN` to the same value as the provider's named bootstrap environment variable. The provider sends `Authorization: Bearer ` on its GET request. When unset, it sends no authorization header. A configured but absent/empty token variable fails startup. Never put the token in a URL or emit request-header diagnostics. + +Container deployments must share an appropriate network namespace. Configure the agent to bind loopback explicitly; AWS documents different binding defaults for ECS/EKS and EC2/on-premises. Require token authentication when untrusted processes share that namespace. This does not isolate the endpoint from a compromised application process. + +### Key mapping + +Each configured EdgeZero key maps to the entire deployed free-form document in one profile. The key selects a document, never a field inside it. Multiple independent values require explicit profile mappings: + +- `get("settings")` returns the profile's exact UTF-8 document bytes as `ConfigValue`. +- A profile containing a serialized `BlobEnvelope` returns that serialization unchanged. Core extraction validates it. +- A profile containing a raw string returns the raw string, including whitespace or an empty payload. Do not parse or flatten arbitrary JSON, unquote it, reserialize it, or resolve secret references inside the provider. +- Unknown lookup keys return `Ok(None)`. An unavailable explicitly mapped profile fails startup. +- Invalid UTF-8, response/protocol errors, and oversized documents are errors. + +Request identity content encoding and disable transparent HTTP decompression. V1 rejects responses with a non-identity `Content-Encoding`; compressed responses require a later explicit wire/decoded-limit contract. Bound streamed body bytes before retaining them. The retained UTF-8 body is the original profile text, not compressed transport bytes. + +V1 implements neither key/value document projection nor feature-flag evaluation. They need explicit contracts if later required. + +### Snapshot and freshness + +Fetch each distinct mapped profile once during startup, validate its bytes/limits, and retain an immutable snapshot. The agent can continue polling, but EdgeZero does not reread it after startup. There is no hot reload or app-state mutation. + +The snapshot is coherent per document, not an atomic revision across profiles, secret resources, or providers. Applications needing a coordinated release must use a single configuration envelope with explicit pinned secret versions or their own release validation. + +An agent can successfully return cached data while AWS is unavailable or access has changed. Therefore startup proves that the configured agent supplied an acceptable document, not that AWS was reachable at that instant or that the document is the latest deployment. Do not call this a freshness guarantee. + +Disable agent disk backup/preload for the initial documented setup. Even without disk backup, its in-memory cache can be stale. This proposal explicitly accepts that upstream agent policy; EdgeZero never adds another stale fallback after a failed fetch. If #410 consumers require a maximum config age or fail-on-revocation startup, this access-path choice needs revision before approval. + +Direct Data API polling is deferred. Choosing it later requires single-owner session tokens, one-use tokens, server polling intervals, unchanged empty-response handling, session recovery, ambiguous failure handling, and separate tests. Those requirements are not implemented by this spec's agent reader. + +## 7. Secrets Manager + +Bind the exact logical namespace and key to an explicit secret ARN/version selector. The resulting `BoundSecretStore` must preserve namespace isolation; a wrong store or unmapped key returns `Ok(None)` and never probes arbitrary AWS names. + +At startup, retrieve every explicitly mapped entry. All such entries are required to exist. This keeps a finite, reviewable access set and avoids per-request cloud calls. If an application does not need a secret in this deployment, omit its entry. Required app references to an omitted entry fail in the app-owned startup check. + +Return complete values: + +- `SecretString` becomes its exact UTF-8 bytes. +- The Rust SDK's `SecretBinary` blob is already decoded bytes. Do not base64-decode it again. +- Reject malformed responses with neither or both value fields. Preserve valid empty values where the service/client permits them; app validation decides whether empty is acceptable. +- Do not extract JSON fields. A JSON secret is one complete byte value. +- Typed string secret references still require UTF-8. Direct byte consumers retain binary fidelity. + +Omitted version selection means `AWSCURRENT` at startup. A pinned version ID remains pinned. Restart reads the selector again and constructs a new process snapshot; moving a staging label does not alter the running snapshot. + +There is no TTL cache, negative cache, or serve-stale-on-refresh path. Snapshot memory contains only configured successful values and remains bounded until shutdown. Deleting a secret or revoking AWS access does not erase an already loaded secret from a running process. Operators must stop/restart that process when immediate removal is required. This policy needs explicit approval for rotation-sensitive consumers. + +A failed subsequent restart does not reuse a prior process snapshot. Do not persist secret snapshots to disk. Avoid unnecessary byte/string copies and exclude values from `Debug`, logs, traces, and errors. Standard `Bytes`/`Arc` ownership does not promise memory zeroization; any future such guarantee requires a separate API review. + +## 8. Startup, budgets, and failures + +```mermaid +flowchart TD + Start[Start native process] --> Resolve[Validate bootstrap settings and per-ID sources] + Resolve --> Assemble[Accepted fallible framework assembly] + Assemble --> Load[Prepare local bindings and bounded AWS snapshots] + Load --> Registries[Finalize shared registries] + Registries --> Checks[App initializer loads typed config and required secrets] + Checks --> Ready[Start accepting traffic and expose readiness] + Resolve -->|Failure| Stop[Redacted startup failure, non-zero exit] + Assemble -->|Failure| Stop + Load -->|Failure| Stop + Checks -->|Failure| Stop +``` + +Reuse #392/#396 rather than introducing another readiness or mandatory-secret API. The application owns schema validation, required secret references, signing initialization, and publication of typed handler state. Framework assembly keeps its accepted order; provider preparation precedes the app initializer and listener acceptance. + +At the inspected #275 head, `AppConfig::from_binding` provides context-free typed loading with the shared envelope/secret algorithm. Use its accepted form in the production initializer, then retain validated typed state using #406's startup state insertion. Request extraction remains available and can repeat parsing against immutable snapshots; snapshot providers alone do not cache typed application objects. + +### Proposed v1 defaults + +All limits are validated finite, nonzero settings. Reuse accepted core snapshot/extraction limits where they apply; provider transfer and bootstrap limits are additional checks, not replacements. + +| Limit | Default | +| -------------------------------------------- | --------------------------------------------------------------------- | +| Bindings file bytes | 256 KiB | +| Explicit bindings | 128 total IDs | +| Remote mapped values | 1,024 config document and secret entry mappings per shared session | +| AppConfig document bytes | 1 MiB | +| Secret payload bytes | 64 KiB | +| Combined retained AWS snapshot payload bytes | 8 MiB unique retained payload per shared preparation session | +| Aggregate AWS preparation budget | 30 seconds per session, including credential/client setup and fetches | +| Individual service operation budget | At most 5 seconds, also bounded by remaining aggregate budget | +| Connect / attempt budgets | At most 1 second / 2 seconds, clipped to remaining operation budget | +| Concurrent remote preparation | One operation at a time initially | + +The 8 MiB limit is an approved provisional, configurable default, not an AWS service limit or a number justified by workload measurements. An aggregate cap prevents many individually valid documents and secrets from accumulating an unexpectedly large snapshot. Applications with reviewed larger inputs may explicitly raise it. Revisit the default using representative deployment payload sizes and headroom, rather than presenting it as a validated requirement. + +The standard runner's one shared session accounts for all built-in AWS bindings it prepares, across config/secrets and logical IDs. Programmatic callers use the same accounting by reusing an explicit session. Opaque supplied handles, local providers, and separately created sessions are outside that session's quota. Independent sessions can together exceed 8 MiB; no process-wide cap or global accounting is promised. Callers own the budget for those additional resources. + +Count metadata, mapping entries, retained payload, and transient staging separately. Count every mapping before deduplication, but count a shared immutable payload allocation only once against retained bytes. Bound agent body reads before accumulating the whole document. Reject duplicates before constructing snapshots. Deduplicate identical profile or ARN/version requests only when their complete normalized target and authorization/client policy match; do not accidentally merge namespaces. Check aggregate residency before retaining each payload. A quota breach fails preparation and the runner drops all staged state before readiness; it does not evict values or fall back to local sources. + +The SDK may materialize secret responses and allocate before EdgeZero checks payload size. Metadata, transient buffers, credential providers, TLS, SDK parsing, allocator overhead, local-provider payloads, typed application state, and the external agent are not covered by the retained-payload cap. Do not describe these limits as a certified RSS bound. Preserve #275's conservative allocation/deadline capability classification. + +### Retry ownership + +Reuse one session-owned Secrets Manager client per region/client policy during preparation, not one client per key or request. Configure supported temporary-credential discovery and pin SDK versions and behavior defaults. Production uses workload roles; local developers use a named authenticated profile or supported developer chain. Never require static keys in the bindings file or image. + +The SDK owns Secrets Manager retries, capped at three total attempts within the operation and aggregate budgets. EdgeZero adds no outer retry loop. Retry transient transport/service/throttling failures only; do not repeatedly retry validation, access denial, or decryption failures. Test the pinned SDK's actual classifications. + +The agent HTTP reader makes one bounded request per distinct profile. Agent AWS polling/retry policy is upstream. A failed preparation exits; an operator/supervisor owns any restart backoff. No retry across partially published snapshots is possible because publication occurs only after success. + +Use the accepted application monotonic clock for caller deadlines. Configure provider timers and SDK operation budgets, but do not claim forcible cancellation of credential-process discovery, blocking provider work, or all socket activity. Timeout observation is not proof that every underlying operation stopped. Retain best-effort wording until runtime evidence proves a stronger guarantee. + +### Error boundary + +Classify failures before dropping provider response/source details. Use a native sanitized preparation error with store kind, baked logical ID, and a fixed category. Suggested categories include invalid binding, missing resource/version, credential failure, access denied, decryption failure, throttled, deadline, transport unavailable, malformed value, and size limit. Keep these distinctions where the access path exposes them; the Agent may expose only an upstream-unavailable response, not AWS's original cause. + +Do not extend core errors merely to publish AWS service codes. Snapshot reads use the accepted portable errors for invalid keys, deadlines, and bounds. A missing local lookup is `Ok(None)`; a failed mapped AWS fetch is a startup error, never a successful miss. Agent HTTP authorization denial and missing resource responses remain failures even if detailed cloud causes are unavailable. + +Log only fixed categories and validated application logical IDs. Drop SDK/HTTP error chains before startup formatting. Never log credential sources, ARNs, endpoint tokens, response bodies, raw bootstrap values, config contents, or secrets. Keep service request IDs only if a separately reviewed safe diagnostic policy allows them. + +## 9. Why DynamoDB is separate + +DynamoDB is a good candidate for both config and ordinary KV, but those are separate implementations and permissions. + +For configuration it fits `get(key)` directly. A later config provider must specify table/key/value attributes, namespace encoding, string versus binary handling, version selection, consistency, and missing-item behavior. A DynamoDB item is limited to 400 KiB including attribute names and values, which can exclude complete application envelopes. AppConfig better matches the initial managed configuration deployment requirement; supporting both now would multiply acceptance work without a confirmed second requirement. + +A later DynamoDB KV adapter can implement `KvStore` and return a `KvHandle` to the same native override path. Its owning spec must settle: + +- Table and namespace/key schema, collision-free encoding, and per-binding isolation. +- Byte fidelity, item overhead, and backend value limits. Core's current maximum is not a promise that every provider accepts that size. +- TTL visibility and expiration during reads and listing, rather than assuming background deletion is immediate. +- Prefix pagination, ordering, cursors, consistency, and behavior under concurrent updates. +- Operation budgets, retry policy, permissions, and disposable AWS acceptance. + +Passing ordinary KV contracts would not certify CAS, atomic multi-key operations, strongly consistent prefix snapshots, EC identity-graph behavior, or safe distributed rate limiting. Do not advertise those capabilities without their owning contract and tests. + +Parameter Store remains a documented alternative, not an initial implementation. Valkey, PostgreSQL, S3, and non-AWS provider implementations are also separate tasks. Secrets stored as ordinary KV bytes do not acquire Secrets Manager's lifecycle. + +## 10. Generated applications, CLI, and permissions + +Offer the standard native runner independent opt-in `aws-appconfig-agent` and `aws-secrets-manager` features forwarding to the corresponding `edgezero-store-aws` service features. An optional `aws-stores` alias may enable both, but must not be the only path. Config-only builds need no Secrets Manager SDK; secret-only builds need no Agent HTTP client. The CLI and generated Axum Cargo target must make those dependency choices explicit. A bindings file does not add dependencies to an existing binary. A selected provider that was not compiled in fails safely at startup. + +Make forwarding `[adapters.axum.build].features` an explicit build deliverable. Main's Axum `run_cargo` forwards explicit extra args, while its blueprint has no provider features. Declared native build features and explicit Cargo `--features` additions form the requested feature names, not the resolved provider capabilities. Determine provider availability from Cargo's resolved feature graph for the selected native package and target, including dependency declarations, defaults, `--no-default-features`, `--all-features`, and alias/dependency feature expansion. Build and offline validation must use the same target-selection and feature-resolution inputs. Map generated target features to adapter service features. Add generated-native compile tests for config-only, secret-only, both, and disabled providers. Keep default-generated native and WASM targets AWS-free. + +Put the bindings-file parser and per-ID source validation in a native bindings module in `edgezero-adapter-axum`, available to its `axum` and `cli` features. Share TOML/settings dependencies through a native-bindings feature, rather than enabling `cli` from the runtime or `axum` from CLI-only validation. Main currently enables the TOML dependency only under `cli`. + +The AWS crate owns strict deserialization/validation of its settings types; those types must be available without linking the service SDK or HTTP client. This allows the existing CLI dependency on the Axum adapter to reuse the parser without another crate or portable-core changes. + +Propose `config validate --adapter axum --store-bindings /absolute/native-stores.toml`, with optional `--target-features` additions and target controls equivalent to Cargo's `--all-features` and `--no-default-features`, for offline structural validation against declared IDs, explicitly supplied source conflicts, and limit settings. Target-provider compatibility uses Cargo-resolved features under the same inputs as native build, not a union of feature names or the running CLI binary's compiled services. If target build information is incomplete or its feature graph cannot be resolved offline, explicitly report structural validation only, without a provider-availability verdict. Runtime validation remains authoritative for a prebuilt binary. Test both directions of a CLI/target feature mismatch, default-enabled providers, suppression through `--no-default-features`, `--all-features`, the `aws-stores` alias when offered, and dependency feature propagation. Compare each verdict with the corresponding compiled native target. Never resolve AWS credentials, probe resources, or fetch configuration/secret values during this check. + +No new config publishing, secret mutation, or AWS provisioning behavior is included. Existing `config push --adapter axum` remains a local-file writer and does not update AppConfig. Document that it does not affect an explicitly AWS-bound ID. If the command is given an explicit native bindings file selecting that ID, reject unsupported remote publication rather than report a successful AWS update. Secret-management commands remain under #284. + +Read permissions are independent of operator writes: + +- Agent role needs AppConfig Data `StartConfigurationSession` and `GetLatestConfiguration` for reviewed configuration resources. +- Application role needs Secrets Manager `GetSecretValue` for explicit ARNs and `kms:Decrypt` when the selected customer-managed key requires it. +- Runtime roles do not need resource creation, config deployment, secret mutation, staging-label mutation, or rotation-Lambda management. +- Config-only applications need no Secrets Manager client or application AWS credentials. The agent owns its role. Secret-only applications need no agent. + +Document native-only support, exact setup, restart adoption, upstream agent staleness, secret snapshot retention, least privilege, costs, and cleanup. Downstream apps own binding adoption and app-specific requirements; no Trusted Server or Axon schema belongs in these crates. + +## 11. Delivery sequence and proof + +The user's subsequent instruction to go ahead authorizes execution of the staged implementation plan. Stage proof gates remain; AWS acceptance still requires separate explicit authorization. + +1. Re-audit and integrate accepted #275/#406 contracts. Record existing coverage and add only missing native composition behavior. Test injected fake providers, mixed bindings, defaults, duplicate rejection, and remote-only startup without local roots. +2. Add typed bootstrap parsing and the optional native AWS crate skeleton. Test strict parsing, source conflicts, feature-disabled selection, namespace mappings, and budget validation without network access. +3. Add AppConfig Agent whole-document snapshot preparation. Use a deterministic local HTTP fixture for exact paths, bearer authorization and token redaction, redirects/proxy rejection, content-encoding rejection, raw UTF-8, envelope preservation, empty values, invalid UTF-8, response errors, truncation, deadlines, size limits, and snapshot immutability. Test bounded snapshot reads independently of startup transfer limits. +4. Add Secrets Manager snapshot preparation. Put a narrow test seam at the service-call boundary, not a public provider factory hierarchy. Test string/binary fidelity, no double base64 decoding, namespace isolation, selectors, deduplication, missing resource/version, expired credentials, denied/decryption/throttling/transport errors, and partial-startup cleanup. Test bounded snapshot reads for expired deadlines, separate backend/value allowances, hit byte accounting, zero-byte missing results, and deadline precedence before return. Test startup fetch limits separately; bounded lookup does not bound SDK response allocation during construction. +5. Wire generated native feature enablement and offline validation. Test AWS config plus env secrets, local config plus AWS secrets, both AWS providers plus redb, and same-kind local/custom/AWS bindings. Prove a downstream fake KV can use the generic injection path without importing AWS. +6. Exercise production startup through a generated native application. Required envelope validation and nested secret resolution must finish before readiness. Capture logs/public errors and assert fixture values, tokens, ARNs, and provider bodies never appear. Version changes affect a new process, not the running one. +7. Run an explicitly authorized disposable AWS acceptance using pinned Agent/SDK versions, least-privilege roles, finite cost/request limits, and named cleanup ownership. Record commit, environment, resources, result, and cleanup. Verify cold-agent retrieval, denied cold-agent access, documented warm-cache behavior, current/pinned secret versions, restart adoption, and denied secret access. + +Default tests require neither AWS credentials nor network access. Provider fixtures prove EdgeZero behavior, not deployed IAM, credential discovery, or Agent behavior. Local HTTP fixtures are authorized loopback tests, not external-network tests. + +Implementation gates include: + +```sh +cargo test --workspace --all-targets +cargo fmt --all -- --check +cargo clippy --workspace --all-targets --all-features -- -D warnings +cargo check --workspace --all-targets --features "fastly cloudflare spin" +cargo check -p edgezero-adapter-spin --target wasm32-wasip2 --features spin +``` + +Also add enabled native-provider tests/builds, generated application compilation, and existing Fastly/Cloudflare/Spin WASM target checks with AWS absent from their reachable dependency graph. Deterministic provider tests use `futures::executor::block_on` where possible. Native SDK/HTTP fixture tests may use the provider crate's native runtime; they must not move that requirement into portable contract suites. + +Stages 1 and 2 now have executed generic composition, strict bootstrap/settings, isolated-feature and native/WASM build proof in the maintained plan's execution records. Cloud providers and shared-session accounting remain unimplemented at this checkpoint. No deployed AWS behavior or forcible cancellation is proven. The user has explicitly declined real AWS acceptance; it is excluded from the currently authorized task. + +## 12. Alternatives and review questions + +Rejected for the initial design: + +- AWS services inside Axum implementation modules. Smaller at first, but prevents independent provider use by another native runtime. +- Provider settings embedded in portable `[stores]` metadata. Mixes application declarations with deployment references and changes macros unnecessarily. +- A general plugin registry. Existing traits and handle injection already solve extension; no dynamic discovery is required. +- Environment-only nested resource mappings. Scalar NAME/KEY overrides are useful, but finite key-to-profile/ARN maps are easier to validate in a typed file. +- A JSON string-map inside each AppConfig profile. It could pack several lookup keys into one document, but adds projection and envelope-string wrapping. Whole-document bindings preserve raw bytes with fewer decoding rules, at the cost of more profiles for independently deployed keys. +- Direct AppConfig Data API polling. Removes a sidecar dependency but makes EdgeZero own the token protocol, polling, cache, and recovery. Agent plus startup snapshots is the smaller initial implementation. +- Live Secrets Manager reads with a TTL cache. Requires refresh ownership, stale-on-error policy, and rotation adoption semantics. Immutable startup snapshots are simpler and avoid per-request SDK work. + +The approved implementation follows these policies: + +- AppConfig Agent deployment and its lack of a provider-visible maximum-staleness guarantee. +- Startup-only secrets, including retention after remote deletion/revocation until process shutdown. +- Whole-document key mappings rather than JSON projection. +- AppConfig and Secrets Manager as the initial service set, with DynamoDB config/KV deferred to separately scoped work. +- The external bindings file, generic per-ID injection, remaining proposed bounds, and generated feature-enable path. The configurable 8 MiB shared-session quota and caller-owned preparation boundary were approved on 2026-10-07. + +If the application requires live rotation or enforced config freshness, obtain approval to change these contracts. Do not add them implicitly during implementation. The user's implementation/rebase instruction supersedes the original remaining-design and accepted-only prerequisite gates; later approval authorizes draft publication via `gh stack`. Real AWS acceptance remains excluded. + +### SDK diagnostic policy: approved first-version limitation + +An isolated native spike resolves/builds the Secrets Manager-only candidate graph and verifies direct scoped suppression against fictional loopback data; it does not prove the complete credential-chain graph. A separate executed tracing/log probe finds that scoped `NoSubscriber` permanently disables unrelated application tracing-to-log fallback when no global tracing subscriber is installed. Additive `tracing/log-always` instead bypasses that suppression. This violates the logging-preservation/diagnostic contracts; do not silently install a global logger/subscriber or impose new caller requirements. + +The user has explicitly approved continuing the first version with redacted/dropped EdgeZero-owned diagnostics and a dependency DEBUG/TRACE caveat disclosed before reviews. Do not scope NoSubscriber or alter application logging. Dependency diagnostic hardening is a review item, not an implementation blocker. Offline Stages 1-8 are implemented and verified; provider availability validation remains explicitly structural-only/unknown. Real AWS acceptance remains declined and is not requested. Evidence and exact boundaries are recorded in the maintained plan's section 20 and `/tmp/native-store-integration/sdk-diagnostics-execution.md`. + +## References + +- [Issue #410 and ownership boundaries](https://github.com/stackpop/edgezero/issues/410). +- [Production local stores #396](https://github.com/stackpop/edgezero/issues/396) and [startup/readiness #392](https://github.com/stackpop/edgezero/issues/392). +- [Reusable application lifecycle](2026-09-15-reusable-app-lifecycle-design.md). Retaining an app does not refresh its typed state. +- [AWS AppConfig Agent](https://docs.aws.amazon.com/appconfig/latest/userguide/appconfig-agent.html). +- [Agent container settings, network binding, and backups](https://docs.aws.amazon.com/appconfig/latest/userguide/appconfig-integration-containers-agent-configuring.html). +- [AppConfig Data polling protocol](https://docs.aws.amazon.com/appconfig/2019-10-09/APIReference/API_appconfigdata_GetLatestConfiguration.html). +- [Secrets Manager GetSecretValue](https://docs.aws.amazon.com/secretsmanager/latest/apireference/API_GetSecretValue.html). +- [AWS SDK credential providers](https://docs.aws.amazon.com/sdk-for-rust/latest/dg/credproviders.html), [timeouts](https://docs.aws.amazon.com/sdk-for-rust/latest/dg/timeouts.html), and [retry configuration](https://docs.aws.amazon.com/sdk-for-rust/latest/dg/retries.html). +- [DynamoDB constraints](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Constraints.html). diff --git a/docs/superpowers/specs/2026-10-15-native-celld-store-providers-design.md b/docs/superpowers/specs/2026-10-15-native-celld-store-providers-design.md new file mode 100644 index 000000000..d00ae0cf3 --- /dev/null +++ b/docs/superpowers/specs/2026-10-15-native-celld-store-providers-design.md @@ -0,0 +1,227 @@ +# Native celld store providers + +Status: embedded celld and the EdgeZero provider are implemented, including independent config/KV features, native composition and structural-only CLI support. Local provider, startup, verified TLS, generated-consumer and disposable isolation checks have executed. Release, deployment, stress and fleet acceptance remain unfinished. See the [implementation plan](../plans/2026-10-15-native-celld-store-providers.md) for current evidence and remaining gates. + +Initial celld implementation checkpoint: `f72ae21a515859570622ac9d6ca9c19cf6870b81` on `feat/scoped-store-api`. This is not a released capability or fleet acceptance claim. + +## Goal and scope + +Extend the native provider composition on `issue-410-native-store-providers` so a native application can select celld for configuration and ordinary KV, independently of its HTTP adapter. Axum is the first consumer. AWS, celld and local providers may coexist across logical store IDs. + +Do not add a celld application runtime, Rust WASM entrypoint, Worker build tooling, or celld secret provider. Do not merge PR #420 as a prerequisite. Secrets may continue to use the existing AWS provider or another explicitly selected existing source. + +Planning baseline: EdgeZero commit `350527e4`, PR #414. PR #420 is reference material for celld semantics and prior local experiments, not implementation to import. The working tree was clean before adding this document. + +## What exists already + +The current branch provides the needed application boundary: + +- Core `ConfigStore`, `KvStore`, handles and per-ID registries remain provider-neutral. +- `NativeStoreOverrides` accepts ready handles or deferred preparation per logical ID. +- `NativeStoreBindings` selects built-in providers in a deployment file, outside portable `edgezero.toml` declarations. +- Native startup validates all source selections before polling deferred setup, prepares stores before readiness, and runs an application-owned initializer. +- AWS config and secrets are startup snapshots. DynamoDB KV is live. +- Native startup checks built-in provider availability against the running binary's compiled features. Offline CLI validation is structural-only and reports target provider availability as unknown. + +Keep these contracts. Do not introduce a universal store trait or a dynamic provider registry. + +## celld findings + +Documentation and source were inspected at celld `90b43017241f81189453d326d05948f388b34652`, version 0.6.2. + +1. The documented application KV API is a Worker binding. There is no documented standalone config service. Config can be stored as complete UTF-8 documents in KV. +2. Non-Worker KV access exists in `celld kv`. Its internal `operator_cell::Reachable` discovers live nodes through the fleet bucket, loads the shared fleet secret and signs requests to `/runtime/`. A receiving node forwards to the namespace owner. +3. That client is crate-private, not a standalone supported Rust client. Depending on the full `celld` crate would also bring its runtime dependencies, including V8. +4. The internal listener requires a trusted private network or encrypted overlay. celld terminates no TLS there. The operator API is alpha and may change between releases. +5. The fleet HMAC secret is fleet-wide authority, not a namespace-scoped application credential. Normal CLI discovery also requires access to fleet bucket metadata. Do not silently give an HTTP application these privileges. +6. A namespace has one owner/writer. Reads reach that owner and are documented not to return stale values. These facts do not provide application-key CAS, conditional create or an atomic multi-call read-modify-write. +7. Documented limits include 512-byte keys, 25 MiB values, pages of at most 1000 keys and a 60-second minimum expiry. Values above 1 MiB require a fleet bucket. Expired keys become invisible at expiry. + +These findings describe the researched upstream revision. Executed proof for the new embedded listener is recorded in the implementation plan; it does not certify deployment isolation or fleet behavior. + +## Agreed decisions + +- Applications use a scoped API implemented inside the existing celld process. Do not add a gateway, sidecar, Worker bridge or other separately operated service. Do not put the fleet HMAC secret or celld fleet-bucket discovery credentials in the EdgeZero process. +- Support explicit trusted-network and authenticated access modes. Application bearer tokens are optional, not a prerequisite for private deployments. Both modes restrict exposed stores and operations. +- Config uses startup snapshots, matching the native AWS config provider. Restart to adopt published changes. +- No celld Worker runtime or celld secret provider is added. + +A Worker HTTP bridge is excluded from this proposal. Spawning `celld kv` per application operation and reading or modifying celld's backing SQLite/bucket files are also excluded. + +## Scoped API inside celld + +Implement the scoped API inside celld itself. It is not an existing feature of the researched release, so this work includes celld source changes and a build containing them. The user explicitly rejected a separate gateway or service. Use a disabled-by-default, separately permissioned listener in the existing celld process; do not expose administrative routes by reusing the internal listener. Reuse celld's existing namespace-owner routing after grant checks, rather than making the application discover nodes or sign internal requests. Authenticated TLS terminates in celld so this feature does not require another proxy process. + +```mermaid +flowchart TD + App[Native application] --> API + subgraph Celld[Existing celld process] + API[Scoped store listener] --> Mode{Configured access mode} + Mode --> Network[Trusted network: listener grants] + Mode --> Auth[Authenticated TLS: verify credential and select grants] + Network --> Policy[Check store alias and allowed operation] + Auth --> Policy + Policy --> Dispatch[Existing authorized KV and owner routing] + end + Dispatch --> Fleet[celld namespace owner over protected internal network] + Operator[Operator-owned listener policy and optional credentials] --> API +``` + +### Access modes and grants + +Configure one explicit access mode per listener. Missing or invalid configuration must not silently select tokenless access. + +| Mode | Caller access control | Store permissions | +| --- | --- | --- | +| Trusted network | No application credentials. Firewall rules, security groups or network policies control who can connect. | Every caller that can reach the listener receives its configured store and operation grants. | +| Authenticated | Verify a caller credential. Operator-provisioned random bearer tokens over TLS are the proposed first method. | The verified credential selects explicit store and operation grants. | + +In trusted-network mode, reachability grants access. A private IP alone does not identify an application or isolate it from other workloads. Applications needing different permissions require separate listeners with enforced network isolation, or authenticated access. An application-supplied name or header is not caller identity. + +For authenticated mode, celld keeps SHA-256 token verifiers and grants, compares candidate digests in constant time and terminates TLS on the scoped listener. The application receives only its own random token through deployment secret injection, not through remote config or a celld secret store. Policy is captured once at celld startup. Replacement/revocation requires updating that policy and restarting the node; old grants remain active until the restart. Distinct credential IDs with overlapping grants support a staged token replacement without changing the application API. No credential-issuance service or policy hot reload is included. + +For example, a trusted-network listener or an authenticated credential may grant: + +| Store alias | Physical namespace, service-side only | Allowed operations | +| --- | --- | --- | +| `checkout-config` | `production-checkout-config` | Read | +| `checkout-state` | `production-checkout-state` | Read, write, delete, list | + +The client requests a store alias, never an arbitrary celld runtime scope, node address or bucket path. Resolve the alias and check its grant before any backend call in both modes. Start with whole-namespace grants and dedicated config namespaces. Prefix-based authorization is unnecessary for the initial requirement and would add key-normalization and listing risks. + +Do not expose fleet discovery, deployments, eviction, shutdown, arbitrary cell calls or an administrative proxy endpoint. Neither trusted-network reachability nor a compromised application token authorizes these actions. A compromised celld node already has fleet authority, so its host and internal network remain security boundaries. The new listener does not grant its callers that authority. + +### Defaults and transport protection + +- The API is disabled unless configured. When enabled, tokenless access binds to loopback by default. +- Non-loopback tokenless access requires explicit trusted-network configuration and deployment-level network restrictions. Binding to a private address does not prove those restrictions exist. +- Authentication failures never fall back to trusted-network access. Do not mix authenticated and anonymous grants on one listener in the initial design. +- The provider supports tokenless connections without application credential provisioning. If a binding explicitly selects a token source, a missing or empty token is an error, not a request to connect anonymously. +- Authentication and encryption are separate decisions. Bearer credentials require TLS. Trusted-network mode may use HTTP on loopback or an explicitly accepted protected network. A private network does not inherently provide confidentiality; use TLS or an encrypted overlay where needed. +- Fleet credentials remain inside celld in both modes. Optional application authentication does not disable authentication on celld's internal reserved-cell protocol. + +### Operations + +Use a small versioned HTTP contract with raw bytes for values and bounded JSON for key pages. Exact route spelling is a proposal, not a shipped API: + +- `GET /v1/stores/{alias}/value?key=...` reads bytes. Missing is distinct from an empty successful body. +- `HEAD` on the same route checks existence without returning the value body. +- `PUT` on the same route stores raw bytes. Optional `ttl_seconds` is an integer from 60 through 31,536,000. The provider must round supported fractional `Duration` inputs upward to seconds, never earlier than requested. Omitting TTL clears a previous expiry. +- `DELETE` on the same route deletes a key, including a missing key. +- `GET /v1/stores/{alias}/keys?prefix=...&cursor=...&limit=...` returns a bounded page and opaque continuation. + +Construct URLs using an encoding library and define one decoding rule. Reject malformed or duplicate parameters rather than accepting ambiguous input. Bind continuation to the authorized store and prefix, and reauthorize every page. Never treat a cursor as authority. + +The service and client both enforce finite deadlines and body/page limits. The service owns celld wire-version compatibility, namespace scope derivation, fleet authentication and node discovery/refresh. Raw-byte HTTP avoids exposing celld's JSON byte-array expansion to application clients, but the privileged backend transport still needs its own encoded-body budget. + +No automatic replay of ambiguous mutations in the first version. A timeout does not prove that a write failed. Error categories must distinguish authentication, authorization, missing values, invalid input, limits and backend unavailability without returning internal error chains. + +Authorization tests must prove out-of-grant store denial and config write denial before backend dispatch in both access modes. Authenticated mode must also prove isolation between credentials. Trusted-network tests must make shared listener permissions explicit. Test disabled and loopback defaults, rejection of implicit remote tokenless exposure, missing selected tokens, and no fallback after authentication failure. Deployment tests must separately prove network restrictions on the scoped listener and that application workloads cannot reach the internal listener or read fleet credentials. + +### Embedded listener v1 contract + +Enable the listener only with an absolute `--store-api-config` JSON policy path or `CELLD_STORE_API_CONFIG`. Without a selection there is no scoped socket. The policy requires `version = 1`, explicit access mode, store aliases and operation grants. Its default address is `127.0.0.1:7780`; explicit port zero is permitted for disposable tests. Non-loopback tokenless binding requires `allow_remote_trusted_network = true` as well as operator-enforced network restrictions. Authenticated mode requires native TLS certificate/key files and cannot enable the trusted-network opt-in. + +Policy accepts at most 128 store aliases, 128 credential IDs and 1,024 grant entries in a file capped at 256 KiB. Aliases/credential IDs are case-sensitive ASCII alphanumerics, underscore or hyphen, at most 64 bytes. Service-side namespace IDs are nonempty, at most 512 UTF-8 bytes and contain no control characters. Application callers never select those namespace IDs directly. + +Initial wire limits are 1 MiB raw values, 4 MiB plus 16 KiB encoded backend value responses, 2 MiB encoded key pages, 2 KiB cursors and at most 1,000 keys per page, default 100. TLS/header completion has a five-second budget, each operation a ten-second budget and the scoped listener admits at most 32 simultaneous connections. These are finite feature limits, not an RSS certification. Larger bucket-backed values remain outside the tested scope. + +Every scoped response includes `x-celld-store-version: 1`. Value GET returns 200 with raw bytes or 404 for a miss. HEAD returns 200/404 without a body. PUT and DELETE return 204 on success. Key pages return `{ "keys": ["name"], "cursor": null }` or a bounded opaque cursor. The service checks its version, alias/physical-namespace fingerprint, prefix and continuation progress on every page, and reauthorizes the operation. Cursors are not secret, signatures or authority. Invalid/non-progressing backend pages fail rather than triggering an unbounded scan. + +Errors use fixed categories `invalid_input`, `unauthenticated`, `forbidden`, `not_found`, `limit_exceeded`, `unsupported_version`, `unavailable` and `deadline`, with statuses 400, 401, 403, 404, 413, 426, 503 and 504 respectively. HEAD errors have no body. Requests reject malformed/duplicate/unknown query parameters, malformed percent escapes, invalid UTF-8 and unsupported content encoding. Invalid authentication never falls back to anonymous access, including a supplied bearer header on a trusted-network listener. + +The celld implementation reuses existing reserved KV classes, routing and output gates. A local store-only bootstrap uses celld's built-in KV runtime and a built-in 404 default required by its generation host, not application Worker source or a KV-over-Worker bridge. Enabled KV registration survives deployment generation construction/reload. Calls use an explicit no-replay path through both owner dispatch and the pooled peer tunnel; buffering request bytes alone is not proof that retrying a mutation is safe. + +## Proposed provider design + +Add a native-only `edgezero-store-celld` crate, independent of Axum, the CLI and `edgezero-store-aws`. Use the existing runtime rather than creating another executor. Do not pull the full celld runtime, V8, workers-rs or worker-build into native application dependencies. + +Suggested opt-in Axum features are `celld-config` and `celld-kv`. Keep them independent, disabled by default and compatible with AWS features. Final provider tags and settings depend on the transport decision. + +```mermaid +flowchart TD + App[Portable application and logical store IDs] --> Registries[Core config, KV and secret registries] + Bindings[Explicit native deployment bindings] --> Startup[Validate sources and prepare before readiness] + Startup --> Celld[edgezero-store-celld] + Startup --> AWS[edgezero-store-aws] + Startup --> Local[Existing local providers] + Celld --> Config[Immutable config snapshots] + Celld --> KV[Live KV handles] + Config --> Registries + KV --> Registries + AWS --> Registries + Local --> Registries + Celld --> Transport[Agreed native transport] + Transport --> Owner[celld namespace owner] +``` + +### Config + +Use startup snapshots to match native AWS config adoption: + +- Map each logical config lookup key explicitly to a remote namespace/key. Require the default key to name a mapping. +- Fetch each mapped complete document during bounded preparation. Missing required mappings, invalid UTF-8, limits and transport errors fail before readiness. +- Preserve exact document bytes as UTF-8. Do not parse JSON fields, reserialize envelopes or resolve secrets in the provider. +- Publish immutable config bindings implementing ordinary and bounded core reads. Unmapped keys are misses, not remote discovery. +- Restart to adopt remote changes. Multiple documents do not imply one atomic release. Applications needing coordinated versions validate them in the existing initializer. +- Typed `BlobEnvelope` validation and nested secret resolution remain in core and application startup. celld config may reference a separately bound AWS secret store. + +Live config refresh is a separate decision, not implied by using KV as storage. + +### KV + +Implement the existing byte-oriented operations: get, put, TTL put, delete, exists and paginated prefix listing. + +- Preserve binary bytes and distinguish absent values from empty values. +- Keep namespace identity explicit and test isolation between logical IDs. Reusing one physical namespace is explicit sharing, not automatic isolation. +- Enforce transport, encoded-body, decoded-value and page limits before unbounded allocation. JSON byte-array expansion on the internal protocol needs its own budget. +- Preserve native expiry semantics. A plain overwrite removes an old TTL; TTL overwrite renews it. Verify the actual wire behavior rather than assuming it matches the Worker API. +- Bind opaque continuation state to the relevant namespace and prefix. Handle partial and empty continuation pages without unbounded scans. +- Apply finite connection and whole-operation deadlines. Do not replay ambiguous writes after a timeout or disconnect. A relative-TTL retry can change expiry even when overwriting identical bytes. +- Return closed, redacted errors. Do not retain URLs, remote bodies, namespace names, credentials or raw error chains in EdgeZero-owned diagnostics. + +Do not claim conditional writes, atomic counters, snapshot listing or a stronger identity backend. + +### Native composition and CLI + +Extend existing built-in dispatch rather than creating parallel startup: + +- Add typed config/KV binding variants and compiled-provider capabilities. Reject celld under the secret kind. +- Preserve duplicate-source rejection, undeclared-ID checks and validation-before-I/O. +- Select provider-specific preparation sessions by their actual variants. Current AWS session selection treats any built-in binding as AWS, which must not remain true once celld exists. +- Separate celld connection credentials from the AWS-specific `AgentAccessToken` config-source tuple. Keep AWS preparation limits AWS-owned. Add only the celld budget mechanism needed by the selected transport. +- Preserve AWS-only and local-only startup behavior. A celld-only deployment must not require AWS credentials or local roots for remote-bound IDs. +- Extend generated target feature forwarding and structural binding validation for celld. Keep offline validation structural-only and report target provider availability as unknown. Do not add Cargo feature-graph resolution, remote connectivity or authorization checks. Startup rejects providers not compiled into the running binary. +- Generalize the AWS-specific config-push guard so selected celld config cannot accidentally report a local file write as remote publication. + +Initial publication remains operator-owned. Do not add fleet provisioning, deployment, config publication or secret management to this provider task. + +## Delivery sequence and proof + +1. **Specify and implement the scoped API in celld.** Placement is settled: the listener and grant enforcement run inside celld, with no extra service. Finalize listener grants, optional credential provisioning/revocation, versioning, network boundaries and operation/error contracts. Scoped access, the two access modes and startup config snapshots are already agreed. +2. **Prove scoped native access.** Use a disposable local celld setup and a small native client with no application Worker. Exercise tokenless loopback and authenticated access. Observe binary put/get/delete, missing versus empty values, namespace isolation, pagination and explicit version/authentication failures. Prove config-write and out-of-grant store denial before dispatch, plus no fallback between access modes. Keep all fleet credentials inside the service boundary and record their acquisition. This is a transport and authorization gate, not production acceptance. +3. **Implement live KV.** Add typed settings, strict offline validation, bounded transport and operation tests. Prove TTL renewal/reset, malformed responses, oversized encoded/decoded payloads, continuation and ambiguous-write behavior. Use provider-specific TTL tests because the shared native contract macro currently expects direct sub-second TTLs that celld rejects. +4. **Implement config snapshots.** Prove whole-document preservation, missing and invalid UTF-8 failures, per-value/session budgets, bounded snapshot reads and restart-based adoption. +5. **Integrate Axum and CLI.** Prove mixed AWS/celld/local source selection, structural-only offline validation with target provider availability reported as unknown, startup missing-feature rejection, source conflicts, root requirements, readiness ordering and config-push refusal through public entrypoints. +6. **Exercise a generated consumer.** Run a generated native Axum app against real celld. Check mutable KV, typed config initialization and restart adoption. Where available, compose celld config/KV with existing AWS fixtures for secrets. Run feature/dependency checks proving celld does not add Worker/V8 dependencies or change AWS-only builds. + +Each behavioral unit ends with its focused regression and neighboring tests. At the combined checkpoint run affected formatting, lint, builds and integration checks. Missing external tooling or a failed service startup is an unverified gate, not a passing skip. + +Fleet failover, encrypted-overlay operation, crash recovery, large-value bucket behavior, upgrades and performance need separate runtime acceptance. A single-node local result does not establish them. + +## Open decisions + +- Freeze the embedded v1 contract above against remaining fault, shutdown and deployment-isolation tests before exposing stable EdgeZero provider settings. +- Record the supported celld implementation revision and upgrade compatibility policy before release. Fleet operation, deployment reload and production persistence need acceptance beyond the executed local proofs. + +## References + +- [Native AWS provider PR #414](https://github.com/stackpop/edgezero/pull/414) +- [Experimental celld runtime PR #420](https://github.com/stackpop/edgezero/pull/420), reference only +- [celld KV](https://celld.dev/docs/services/kv/) +- [celld security](https://celld.dev/docs/security/) +- [celld guarantees](https://celld.dev/docs/guarantees/) +- [Pinned operator client](https://github.com/denoland/celld/blob/90b43017241f81189453d326d05948f388b34652/crates/celld/operator_cell.rs) +- [Pinned KV CLI](https://github.com/denoland/celld/blob/90b43017241f81189453d326d05948f388b34652/crates/celld/kv_cli.rs) +- [Pinned peer authentication](https://github.com/denoland/celld/blob/90b43017241f81189453d326d05948f388b34652/crates/celld/peer_auth.rs) +- Existing branch design: [native store providers](2026-10-07-native-store-providers-design.md) diff --git a/docs/superpowers/specs/2026-10-22-pluggable-geo-providers-design.md b/docs/superpowers/specs/2026-10-22-pluggable-geo-providers-design.md new file mode 100644 index 000000000..e373f0600 --- /dev/null +++ b/docs/superpowers/specs/2026-10-22-pluggable-geo-providers-design.md @@ -0,0 +1,217 @@ +# Pluggable Geo providers + +- Status: implemented and locally verified. The implementation plan records test evidence. Cloud deployment and downstream acceptance remain separate and unproven. +- Date: 2026-10-22 +- Owning issue: [EdgeZero #432](https://github.com/IABTechLab/edgezero/issues/432), under [#391](https://github.com/IABTechLab/edgezero/issues/391). +- Initial deployment: CloudFront through an HTTPS, origin-secret-gated ALB to network-isolated native Axum. +- Planning baseline: EdgeZero `5426ea65`. Checked native ingress from [PR #409 at `66a05e25bee674ced69387467acec3c87b06c8b6`](https://github.com/IABTechLab/edgezero/tree/66a05e25bee674ced69387467acec3c87b06c8b6) was reconciled by adapting its own commit delta onto the newer native-store stack. Current asynchronous egress and connection ownership remain intact. The separate #407 buffered JSON-cap/body-carrier feature was not imported. No second forwarding parser was added. +- Composition reference: [native startup lifecycle](2026-10-05-axum-production-startup-lifecycle-design.md). Geo remains separate from [native store providers](2026-10-07-native-store-providers-design.md) and [PR #414](https://github.com/IABTechLab/edgezero/pull/414). + +## 1. Outcome and scope + +Applications obtain an estimated country and optional first-level subdivision through one portable Geo contract. EdgeZero supplies geographic assertions and their provenance. Applications decide jurisdiction, consent, identity behavior and disclosure. An estimate is not proof of residence, identity or consent. + +The first implementation includes portable outcomes, a synchronous provider contract, explicit native selection, the CloudFront provider, a request accessor, focused tests and deployment guidance. Application-supplied providers use the same narrow, local-only contract. No local GeoIP database or external lookup service is included. + +This deliberately narrows #432. Its local-database, reader-IP input and database-lifecycle requirements are tracked separately in [#438](https://github.com/IABTechLab/edgezero/issues/438), not satisfied by CloudFront support. The split is recorded on #432. Trusted Server adoption is a separate follow-up, not an EdgeZero delivery step. + +Required invariants: + +- Select zero or one provider per deployment. No selection means unavailable geography. A selected provider never falls back to another source. +- Country-only geography remains partial. A US country without a state does not identify a privacy regime. +- Invalid configuration or an unavailable selected implementation fails native startup before listen/readiness. +- Missing or rejected request metadata produces an outcome, not a provider-selection change or an automatic HTTP error. +- One immutable outcome belongs to the request. Consumers do not resolve geography again or reinterpret forwarding headers. +- Provider calls use bounded, source-admitted assertions, not the request, reader IP or unrestricted services. +- Core acquires no AWS SDK, Axum, Tokio or filesystem dependency from this feature. + +## 2. Ownership and data flow + +| Concern | Owner | +| --- | --- | +| Socket peer, protected-path policy, admission and source provenance | Native ingress and adapter | +| Selection, assertion normalization and bounded failure categories | Geo composition and providers | +| Country/subdivision outcome and read-only accessor | Portable core | +| CloudFront policies, ALB rules, credentials and network isolation | Deployment operator | +| Jurisdiction, consent, identity and data disclosure | Application | + +```mermaid +flowchart TD + Options[Explicit native selection and protected-path policy] --> Startup[Validate before application initialization] + Request[Incoming request] --> Ingress[Checked ingress and admission] + Startup --> Provider[Selected synchronous provider] + Ingress --> Source{Protected source admitted?} + Source -->|Yes: bounded assertions| Provider + Source -->|No| Unavailable[Unavailable: untrusted ingress] + Provider --> Outcome[Immutable request outcome] + Unavailable --> Outcome + Outcome --> Consumers[Application middleware and handlers] + Consumers --> Policy[Application jurisdiction and permission policy] +``` + +An AWS deployment region, browser language, timezone, arbitrary country header or absent consent signal is not visitor geography. This feature introduces no identity lifecycle, regulatory lists, permission language, device assessments or response-header middleware. + +## 3. Provider and request contract + +Core owns the outcome types and a small synchronous provider interface. Resolution is bounded local computation. It performs no network, filesystem or other blocking I/O, launches no background work and returns an owned outcome without a future. These restrictions apply to application-supplied implementations as well as the built-in provider. + +Applications may supply an already-constructed provider through typed native composition. There is no plugin discovery, registry, per-ID Geo store, asynchronous preparation interface or provider-specific shutdown hook. Any application-owned construction precedes handing the provider to the runner; this feature adds no resource-loading lifecycle. The CloudFront implementation needs no AWS client, credentials or local directories. + +The initial input contains only the two geographic assertion slots described in section 6. Each slot is absent, a bounded value or an invalid-assertion marker. Ingress owns source admission, duplicate detection and byte bounds. The provider owns code validation and normalization. Source-admitted does not mean the geographic value is already valid. Preserve invalid markers rather than treating rejected input as absent. + +Do not include reader IP, cookies, authorization, query/path, arbitrary headers, application settings or service handles. Future GeoIP or remote providers require an explicit input-contract extension. Reuse #409 if reader-IP support is added later; this implementation neither requires a resolved reader IP nor parses forwarding headers for Geo. + +For managed Axum dispatch: + +1. Remove the configured origin-credential header before application callbacks can observe the request. Its contents do not establish trust inside EdgeZero. +2. Complete framework ingress validation and admission. Refused or aborted ingress performs no provider resolution. +3. Use the runner-owned socket peer and original ingress snapshot to establish source admission. No selection yields `Unavailable(NoSelection)`; a selected provider without source admission yields `Unavailable(UntrustedIngress)`. Neither invokes the provider. +4. For an admitted request from the configured source, invoke the selected provider exactly once, including when both assertion slots are absent. Store the outcome before request preparers, application middleware and handlers run. + +`RequestContext` exposes a read-only accessor. Header mutation or insertion/removal of public extensions cannot replace the runner-owned outcome. All consumers see the same result. Admission callbacks do not have Geo; no new Geo-dependent admission hook is introduced. + +No lookup occurs during response finalization. Other adapters and the unmanaged request converter expose `Unavailable(NoSelection)` in v1 and never infer trust from raw headers or caller-inserted extensions. Optional application response middleware can read the outcome, but implementing that middleware is outside this change. + +Application-supplied Rust remains trusted code. The input contract prevents accidental coupling; it is not a sandbox or enforcement against malicious provider code. + +## 4. Outcomes and provenance + +Names below describe behavior, not final Rust spelling. + +| Outcome | Meaning | +| --- | --- | +| `Unavailable(NoSelection)` | No provider is attached to this request path | +| `Unavailable(UntrustedIngress)` | A provider is selected, but the request lacks the required protected-source admission | +| `Unavailable(MissingMetadata)` | The source is admitted, but neither geographic assertion is present | +| `Located` | A supported country is present, with an optional syntax-checked subdivision assertion | +| `Error(InvalidAssertion)` | A selected assertion is malformed, duplicated, unsupported or inconsistent | +| `Error(ProviderFailure)` | A supplied provider reports an execution failure, without exposing a raw source error | + +Attach the selected provider's identity to every outcome except `NoSelection`, including source rejection before invocation. A located result also records the source kind and trust basis. For CloudFront, distinguish `CloudFront` as source from the declared protected-ALB path as trust basis. Neither is a cryptographic attestation of distribution identity. Do not retain raw headers, credentials or reader IP in the outcome. + +A located result requires a country. A subdivision cannot exist without its country. Shared validated types enforce the normalization rules for built-in and supplied-provider outputs alike. The subdivision is a provider assertion with ISO 3166-2 component syntax, not independently verified membership in a worldwide subdivision catalogue. + +These outcomes do not set an HTTP status or grant permissions. Applications distinguish missing geography from an error and decide whether unrelated functionality may proceed. Unknown or unrecognized subdivisions must not silently become known non-regulated jurisdictions. + +## 5. Selected CloudFront trust model + +### One supported deployment recipe + +Use AWS's documented [origin-secret ALB pattern](https://docs.aws.amazon.com/AmazonCloudFront/latest/DeveloperGuide/restrict-access-to-load-balancer.html), with these controls required together: + +1. CloudFront adds a dedicated, high-entropy origin credential using a custom header. Only the intended distribution and authorized operators possess it. Do not reuse it across unrelated distributions. Keep its name and value out of logs and responses; document value rotation. +2. Viewer access uses HTTPS. CloudFront connects to the ALB using HTTPS-only with a valid origin certificate. +3. The ALB listener forwards to the application target group only when the credential matches a configured non-wildcard value. Its default action denies access. No other listener or rule may forward unverified viewer traffic to that target group. +4. The ALB security group admits origin traffic only from the CloudFront origin-facing managed prefix lists for the enabled IP families. These lists admit CloudFront infrastructure, not a particular distribution, so they do not replace the credential check. +5. The Axum application port is private and accepts traffic only from the intended ALB security group. The ALB-to-Axum hop stays inside the isolated VPC. No public target port, alternate proxy or other route may bypass the ALB gate. Adding a native TLS server is outside this feature. + +The ALB validates the credential. EdgeZero does not receive its configured value, fetch AWS credentials or implement another authentication layer. AWS documents that [custom origin headers replace viewer-supplied values](https://docs.aws.amazon.com/AmazonCloudFront/latest/DeveloperGuide/add-origin-custom-headers.html). + +### Private ingress witness + +Add one explicit protected-ALB policy to native Geo composition, separate from ordinary proxy trust. It contains a nonempty, bounded literal CIDR allowlist for the actual ALB socket peers and the dedicated credential-header name to remove. Reuse #409's CIDR validation and address normalization, including rejecting unrestricted ranges. Reject a credential-header name that conflicts with geographic, forwarding or standard transport/authentication headers. + +Selecting this policy declares that every deployment control above is enforced. It cannot be inferred from generic trusted-proxy CIDRs, a forwarding header, a claimed distribution ID or an application-provided extension. + +After admission, the managed runner compares the actual connection peer with this policy. A match creates an adapter-private, request-bound witness; a missing or nonmatching peer does not. Only this witness admits the two Geo assertion slots. The low-level converter and request preparers cannot create or replace it. Ordinary proxy trust without the separate protected-ALB policy never admits geographic assertions. + +Remove every occurrence of the configured credential header before application admission callbacks, request preparers or middleware run. Neither its name nor value belongs in Geo diagnostics. Live rotation changes the value under a fixed header name: the ALB temporarily accepts old and new values while CloudFront deploys the new value, then removes the old one. Renaming the header requires a coordinated stop/reconfigure/restart; v1 does not add dual-name removal state. + +This witness means the request arrived on the operator-declared protected path. EdgeZero cannot inspect security groups or distinguish a correctly gated ALB from the same ALB with a broken listener rule. Local tests prove peer/policy enforcement, not the external credential check. Deployment tests must prove the latter before production use. + +### Why this option + +This recipe adds no AWS SDK integration, application-side credential comparison or new VPC-origin resources. It is the smallest documented option for this native-Axum feature, at the cost of operator-owned secret rotation and an internet-facing ALB protected by network and listener rules. + +[AWS VPC origins](https://docs.aws.amazon.com/AmazonCloudFront/latest/DeveloperGuide/private-content-vpc-origins.html) remove public origin access and are a valid future deployment option. They require separate VPC-origin setup and currently exclude gRPC and Lambda@Edge origin triggers. They are not a second supported trust mode in v1. A private subnet or a CloudFront network range alone still must not be described as authentication of one distribution. + +## 6. CloudFront assertions and normalization + +Explicitly configure CloudFront to add `CloudFront-Viewer-Country` and `CloudFront-Viewer-Country-Region` through an origin request policy. Merely forwarding arbitrary viewer headers is not sufficient. Do not install static custom values for these names or edge functions that rewrite the generated geography. + +[AWS documents header replacement](https://docs.aws.amazon.com/AmazonCloudFront/latest/DeveloperGuide/lambda-at-edge-function-restrictions.html#lambda-at-edge-restrictions-cloudfront-headers) when these CloudFront headers are configured: viewer-supplied values are overwritten after the viewer-request event. No Lambda@Edge function is required by this recipe. The [header reference](https://docs.aws.amazon.com/AmazonCloudFront/latest/DeveloperGuide/adding-cloudfront-headers.html) also says subdivision is omitted for requests originating from the AWS network. Deployment proof must cover spoofed values in that missing-subdivision case; do not assume that absence of generated data permits forwarding a viewer's value. + +Apply the following rules to values exposed by the HTTP parser. Do not promise detection of whitespace already removed by an upstream HTTP parser. + +| Input | Required behavior | +| --- | --- | +| No selection | `NoSelection`, without inspecting geographic values | +| Selected provider, source not admitted | `UntrustedIngress`, regardless of absent, valid or malformed values | +| Admitted source, both headers absent | `MissingMetadata` | +| Duplicate instances, including identical duplicates, or comma-joined values | `InvalidAssertion`; do not concatenate or choose one | +| Country value | Exactly two ASCII letters; normalize to uppercase, then require membership in the compiled-in ISO 3166-1 alpha-2 allowlist | +| Country sentinel, reserved/user-assigned or unsupported code, such as `XX`, `ZZ`, `UK` or `XK` | `InvalidAssertion`; no alias substitution or guessed country | +| Subdivision value | One to three ASCII letters or digits; normalize letters to uppercase; no country prefix or separator | +| Empty value, whitespace, non-ASCII data, invalid encoding or excess length in either slot | `InvalidAssertion`; do not trim, decode or truncate into a valid value | +| Subdivision present without country | `InvalidAssertion` | +| Valid country, subdivision absent | `Located` with unknown subdivision | +| Valid country, invalid subdivision assertion | `InvalidAssertion` for the whole result, not country-only success | +| `us` with `ca` | `Located` with `US` and `CA` | +| `US` with `ZZ` | `Located` with the syntax-checked `ZZ` assertion; no claim that it is an assigned US subdivision or a known jurisdiction | + +Use the officially assigned ISO country codes from a pinned, checked-in dataset or dependency. Record the source revision beside the allowlist and test reserved/user-assigned exclusions. Updates are explicit code/dependency changes, not runtime downloads. V1 has no subdivision-membership database. Consumers own recognition of subdivision assertions and must preserve unknowns. + +Inspect all occurrences of the two names, but copy at most two country bytes and three subdivision bytes. Mark oversize or duplicate input invalid before copying it. Existing ingress header limits remain in force. All malformed trusted assertions yield the same bounded `InvalidAssertion` category; raw values never enter errors or diagnostics. + +The initial deployment recipe disables shared caching for these application routes using CloudFront's [managed `CachingDisabled` policy](https://docs.aws.amazon.com/AmazonCloudFront/latest/DeveloperGuide/using-managed-cache-policies.html), with minimum/default/maximum TTL zero. Also set [Error Caching Minimum TTL](https://docs.aws.amazon.com/AmazonCloudFront/latest/DeveloperGuide/custom-error-pages-expiration.html) to zero for cacheable application-error statuses; CloudFront otherwise defaults to caching errors for ten seconds. Other shared caches must not reuse application responses. A custom Geo-aware cache key is outside this recipe. Forwarding Geo headers alone does not separate cached responses by geography or permissions. + +## 7. Selection and startup + +Use one typed selection value: off, built-in CloudFront, or supplied provider. Do not model selection as independent flags/provider slots with precedence. Both active variants require the protected-ALB input policy in v1. Other sources require a later contract extension. + +Selection is programmatic through native run options. New environment/manifest selectors and CLI configuration are outside v1. Keep the built-in implementation default-off and native-scoped. Its selector must remain recognizable when the implementation feature is absent, so selection returns a bounded startup error rather than silently behaving as off. + +Validate selection, implementation availability and policy inputs before opening runner-owned resources or calling the application initializer. Valid CloudFront selection constructs a local provider; no deferred Geo preparation is polled. Existing initialization, readiness, shutdown and response-terminal ownership remain unchanged. Geo is not part of `PreparedStores`. + +The existing application initializer may inspect the selected Geo capability. This confirms configuration, not per-request coverage or infrastructure protection. Off is valid general hosting. Missing, untrusted or malformed request metadata does not change startup readiness or select another provider. + +## 8. Downstream compatibility, separate follow-up + +A later Trusted Server task supplies the translation and tests through its real request/policy path. It must preserve unavailable/error distinctions, country-only and unrecognized-subdivision handling, and independence from Edge Cookies. This spec does not relax the fail-closed behavior in [Trusted Server #1134](https://github.com/IABTechLab/trusted-server/issues/1134). + +Jurisdiction lists, consent interpretation, assumed-jurisdiction settings, identity lifecycle, auction mapping and response disclosure remain application work. No Trusted Server code or policy change is part of EdgeZero acceptance. + +## 9. Acceptance and proof boundaries + +### EdgeZero implementation acceptance + +1. Selection tests cover off, unavailable implementation, invalid policy and typed supplied-provider composition. Validation precedes runner resource opening and application initialization. No Geo dependency on AWS store features, credentials or local roots appears. +2. Ingress tests cover the selected policy plus a matching real socket peer, missing/nonmatching peers, ordinary proxy trust without the protected policy, forged public extensions and credential-header removal before application callbacks. They do not claim to validate an ALB rule. +3. Parsing fixtures cover every row in section 6, both value-length boundaries, identical duplicates and numeric subdivision components. Supplied providers cannot construct an invalid located value through the shared public types. +4. Real managed dispatch proves one resolution for a trusted admitted request, zero for rejected/untrusted/off paths, immutable outcomes across consumers and preparers, and no finalization lookup. Other adapters and the unmanaged converter retain no-selection behavior. +5. Privacy tests prove bounded inputs and diagnostics with no credential, reader IP, arbitrary headers, path/query or raw error chain. A supplied local test provider proves composition without adding remote execution or general preparation machinery. +6. Default builds, existing adapters and native lifecycle tests remain intact. Fastly, Cloudflare and Spin WASM dependency checks prove no native-runtime/SDK leakage. No new portable `Send` future requirement is introduced. + +### Separate deployment acceptance + +Before claiming the CloudFront deployment is safe, exercise the actual distribution, ALB and application network: + +- Intended requests reach the application; direct ALB access and an unrelated CloudFront distribution without the credential cannot reach it. +- Direct target access and alternate listeners/proxies cannot bypass the ALB gate. +- Viewer-supplied country/subdivision values do not survive as accepted geography. Include AWS-network requests with a forged subdivision where CloudFront omits its own value. +- Origin credential replacement, validation, removal and rotation work without exposing the credential in application-visible headers, responses or logs. +- Missing subdivision remains country-only; missing metadata never becomes guessed geography. +- Responses and application errors are not shared across requests through CloudFront or another shared cache. + +An incorrectly configured ALB behind an otherwise matching socket peer is an external trust failure, not something local parser tests can detect. A failed deployment check blocks production use of the declared policy; it does not authorize fallback or broader header trust. + +Downstream policy tests remain a separate adoption gate. Neither local provider tests nor deployed origin protection proves that an application handles unknown geography safely. The implementation plan records local runtime evidence; this design claims no cloud deployment or downstream acceptance. + +## 10. Delivery order and deferred work + +1. Reconcile the pinned #409 ingress revision and implement the private protected-ALB policy/witness without duplicating forwarding interpretation. +2. Add the portable synchronous contract, validated outcomes, typed selection and startup validation. +3. Add CloudFront parsing, immutable request integration and focused local tests. +4. Publish the selected deployment recipe and acceptance checklist. Record local results separately from any deployed evidence. + +Deferred work: + +- Local GeoIP databases, reader-IP inputs and remote/asynchronous providers. +- VPC-origin and other trust modes, plus Fastly/Cloudflare providers. +- Geo preparation, persistence, hot reload and fallback chains. +- Environment/manifest configuration and CLI support. +- Richer geography and subdivision catalogues. +- Custom geographic shared caching and automatic response headers. +- Trusted Server adoption and its policy tests. + +The earlier Trusted Server [pluggable-module](https://github.com/IABTechLab/trusted-server/blob/a854526e8186c20c4f0320761db6c14cd2cb267f/docs/superpowers/specs/2026-07-30-pluggable-providers-design.md), [device-signal](https://github.com/IABTechLab/trusted-server/blob/a854526e8186c20c4f0320761db6c14cd2cb267f/docs/superpowers/specs/2026-10-12-device-signals-design.md) and [request-based Geo](https://github.com/IABTechLab/trusted-server/blob/79a0421fda634ce75ec06bc336af368daaa0a22f/docs/superpowers/specs/2026-10-15-geo-module-request-design.md) proposals are background references only. They do not expand this delivery or supply its API contract. diff --git a/docs/superpowers/specs/2026-11-04-axum-local-geoip-design.md b/docs/superpowers/specs/2026-11-04-axum-local-geoip-design.md new file mode 100644 index 000000000..d1b5265a7 --- /dev/null +++ b/docs/superpowers/specs/2026-11-04-axum-local-geoip-design.md @@ -0,0 +1,226 @@ +# Local GeoIP for Axum + +- Status: approved for implementation by the user. Implementation and outstanding acceptance gaps are tracked in the [implementation plan](../plans/2026-11-04-axum-local-geoip.md). No production acceptance is claimed. +- Date: 2026-11-04 +- Owning issue: [EdgeZero #438](https://github.com/IABTechLab/edgezero/issues/438), under [#391](https://github.com/IABTechLab/edgezero/issues/391). +- Baseline: `72ceed7e` on `feature/pluggable-geo-providers`. +- Extends: [pluggable Geo providers](2026-10-22-pluggable-geo-providers-design.md), specifically its deferred reader-IP input and database lifecycle. +- Uses: [native lifecycle ownership](2026-10-05-axum-production-startup-lifecycle-design.md), with the pre-runtime preparation limitation in section 6. + +## 1. Outcome and scope + +An ordinary native Axum deployment can resolve an estimated country and optional first-level subdivision from a checked reader IP, using an operator-supplied local database. CloudFront, AWS infrastructure, AWS credentials and paid data are not required. + +Website consent management motivates this work. EdgeZero supplies estimates, availability and provenance. Applications own jurisdiction recognition and permission decisions. Unknown geography must not mean "no consent requirements." + +Required invariants: + +- Geo stays off by default. A deployment explicitly selects at most one provider. +- Local GeoIP, CloudFront and supplied providers share the existing request-outcome architecture, not each other's source-admission rules. +- A selected provider never falls back to another source. +- Database preparation succeeds before the application initializer, listener binding and readiness. Invalid selected configuration is a startup error, not Geo off. +- Request lookup uses an immutable in-memory database and checked ingress facts. No request-time filesystem access, remote lookup or background work occurs. +- The managed request owns one immutable Geo outcome, established after admission and before middleware and handlers. +- Country-only, no-match, missing-input and error outcomes remain distinguishable. + +Scope includes the input contract, native provider, startup/replacement behavior, documentation and regression tests. No plugin framework, store binding, downloader, watcher, hot reload, remote lookup or consent engine is added. Trusted Server adoption remains downstream work. + +## 2. Database choice and cost + +### Initial supported dataset + +Use MMDB binary files with the Rust `maxminddb` reader. MMDB is a file format, GeoLite is a dataset, and `maxminddb` is a separately licensed library. No database server is needed. + +Support GeoLite2 City for country and first-level subdivision estimates, and GeoLite2 Country for country-only estimates. Both are free of charge. Consume only geographic codes, not city names, postal codes or coordinates. + +No paid license, subscription or paid upgrade is required. Operators downloading GeoLite data need a free MaxMind account and download license key. Download credentials stay in the operator's distribution process; the running application needs neither credentials nor an external lookup account. + +Document GeoLite's EULA, attribution, redistribution and update/deletion obligations separately from the reader's ISC license. The EULA reviewed on 2026-11-04 requires ceasing use of and destroying old data within 30 days of a new release, including running snapshots and rollback copies. Recheck terms before publishing instructions. + +MaxMind does not recommend GeoLite City for commercial use cases because of accuracy limitations, not a blanket commercial-use prohibition. Applications must evaluate state-level suitability against representative traffic. + +### Alternatives and schema boundary + +DB-IP City Lite is a free, monthly MMDB alternative under CC BY 4.0. It needs separate code-mapping/schema evaluation before support; MMDB format compatibility is insufficient. Other formats, including IP2Location BIN, also need separate integrations. Neither is included here. + +Accept metadata `database_type` values `GeoLite2-City` and `GeoLite2-Country`, with `ip_version` 4 or 6. Reject other types regardless of filename. Metadata identifies the expected schema, not publisher authenticity. Use synthetic or upstream-derived fixtures matching these types, with provenance and redistribution terms; never bundle production GeoLite data. + +## 3. Ownership and flow + +| Concern | Owner | +| ------------------------------------------------------------------------------ | ------------------------------- | +| Socket peer, trusted proxy configuration, forwarding parsing and normalization | Existing native checked ingress | +| Source-specific Geo inputs, selection and outcome provenance | Native Geo composition | +| Read-only snapshot loading, schema validation and bounded lookup | Local GeoIP implementation | +| Portable values, outcome types and `RequestContext::geo()` | Core | +| Downloading, licensing, attribution, distribution, freshness and replacement | Deployment operator | +| Jurisdiction recognition, consent, permission and response disclosure | Application | + +```mermaid +flowchart TD + Options[Explicit native Geo selection] --> Validate[Validate selection and feature availability] + Validate -->|MaxMind only| Load[Read and validate database before startup completes] + Load --> Snapshot[Shared immutable in-memory reader] + Request[Incoming request] --> Checked[Existing checked ingress normalization] + Checked --> Admission{Request admitted?} + Admission -->|No| Stop[No Geo lookup] + Admission -->|Yes| Input{Selected source} + Input -->|MaxMind or supplied reader-IP| Reader{Checked reader available?} + Input -->|CloudFront or supplied assertions| Assertions[Protected assertion admission and resolution] + Input -->|Off| Off[NoSelection] + Reader -->|Yes| Lookup[One selected provider call] + Reader -->|No| Unavailable[Unavailable reader IP] + Snapshot --> Lookup + Lookup --> Outcome[Immutable Geo outcome] + Assertions --> Outcome + Unavailable --> Outcome + Off --> Outcome + Outcome --> App[Application middleware and handlers] +``` + +## 4. Checked reader-IP contract + +Reuse `CheckedProxyMetadata` from `proxy.rs`. Do not add a forwarding parser, interpret raw forwarding headers in a provider, or obtain trusted input by reading caller-mutable request extensions. + +Capture the needed Geo input from the runner-owned checked result before moving that result into public ingress metadata. Resolve only after application ingress admission succeeds. Refused, aborted or draining requests invoke no provider. + +### Direct and proxied deployments + +Selection explicitly chooses `PeerAllowed` or `ForwardedRequired`, an input expectation rather than a forwarding policy. `PeerAllowed` declares that the socket peer is a usable reader if forwarding does not resolve one. Reverse proxies use `ForwardedRequired` to avoid geolocating the proxy. + +| Checked ingress facts | `PeerAllowed` | `ForwardedRequired` | +| ------------------------------------------------------------------------------- | -------------------------------------- | -------------------------------------- | +| Client source is trusted `Forwarded` or `X-Forwarded`, with an effective client | Use that checked client and its source | Use that checked client and its source | +| Client source is direct, with an actual peer | Use the normalized peer | Unavailable reader IP | +| No actual peer or no usable checked address | Unavailable reader IP | Unavailable reader IP | + +Unavailable input invokes no provider. Checked ingress initializes `effective_client()` from the peer, so address presence alone cannot prove reader resolution. Use `client_source()`, not an aggregate forwarding-success flag. A partial result that resolved the client remains usable even if host or scheme was absent. + +Local GeoIP never uses geographic headers. Existing ingress rules alone decide which forwarding assertions are trusted. It does not require `ProtectedCloudFrontAlb`, origin credentials or CloudFront headers. + +Retain existing normalization, including IPv4-mapped IPv6. Do not substitute addresses or invent private-address jurisdictions. Missing records remain no match. + +### Portable provider input + +Extend `GeoInput` to a typed input enum with two cases, using final names selected during implementation: + +- Geographic assertions: the existing bounded country and subdivision slots, with their absent/value/invalid states unchanged. +- Reader IP: a typed `IpAddr` plus portable source provenance identifying direct peer, trusted `Forwarded`, or trusted `X-Forwarded` resolution. + +Missing/inadmissible readers produce a framework-owned unavailable outcome, never a fabricated address. Input carries no request, arbitrary headers, cookies, consent signals, service handles or filesystem access. The adapter owns admission and output provenance. + +Keep `GeoProvider::resolve` synchronous, bounded and local-only, returning an owned `GeoResolution`. Native ownership adds `Send + Sync`; the portable trait retains its existing bounds. An input case unsupported by a provider returns `Error(ProviderFailure)` without retrying another case. Supplied Rust remains trusted application code, not sandboxed code. + +`GeoInput` changes, public enum additions and a reserved provider ID break external consumers, including exhaustive matches. Inventory consumers, migrate repository code/examples together, and document release/migration handling. + +## 5. Selection, capability and extensibility + +Extend `NativeGeoSelection` with a local MaxMind selection containing the database path, preparation limits and reader expectation. Use the default-off native feature `geo-maxmind`. The selector and its configuration types remain available without the feature; selecting an unavailable implementation returns a bounded startup error. + +Keep programmatic selection through `AxumRunOptions`. There are no new CLI, environment or manifest selectors, automatic path discovery, or ambient dataset choices. Repeated `with_geo` calls continue to replace the entire selection. + +Preserve CloudFront and supplied-assertion composition with its protected-source policy. Add application-constructed supplied reader-IP composition using the same checked-IP admission contract as MaxMind. It must work with only `axum`, without built-in Geo features. Selection declares the input contract; do not infer it from provider IDs or probe different inputs. + +Reserve provider ID `maxmind` alongside `cloudfront` in public ID construction and native supplied-provider validation. Located MaxMind results identify a local database; supplied reader-IP results identify checked-IP input without claiming a database implementation. Both retain direct, trusted `Forwarded` or trusted `X-Forwarded` provenance, never the address. + +`geo_capability()` reports provider identity, input contract and configured reader expectation. It validates configuration without file I/O or preparation, and cannot report eventual request provenance. Split the current `native_geo()` validation/construction path accordingly. Loaded metadata belongs to the prepared provider, outside `PreparedStores` and portable registries. + +## 6. Preparation, resource ownership and replacement + +### Preparation boundary + +Prepare the database once per runner invocation at the existing native Geo construction boundary, before serving resources and application initialization. Keep pure selection validation separate. CloudFront requires no preparation. + +1. Validate selection, feature availability and finite preparation limits before reading the file. +2. Open the operator-supplied local regular file read-only. Bound the actual read, not merely a prior metadata size check; reject pipes and devices without waiting for their contents. Do not create files or discover a different path. +3. Build an owned in-memory reader from those bytes, without mmap or `unsafe-str-decode`. +4. Validate metadata, structure and all reachable records, including off-path corruption. Check consumed country/subdivision field shapes; allow optional and unrelated fields. Reader construction alone is insufficient. Reuse its verifier and iteration APIs, not another MMDB parser. +5. Record dataset type, build timestamp and a SHA-256 checksum of the loaded bytes. These identify the snapshot, not its accuracy or authenticity. +6. Share the prepared immutable reader across requests through native ownership. Return a bounded error on failure before the initializer, bind or readiness. + +Structural corruption and incompatible field types fail startup. Geographic code validation remains request-time behavior from section 7. For example, a structurally valid record containing `XK` must not prevent the entire database from loading; a lookup of that record returns `InvalidAssertion`. + +Document the file-byte bound and traversal/allocation bounds for validation and lookup. Select concrete limits from the pinned reader's guarantees and representative supported files, and record their rationale in the implementation plan. Exceeding a limit fails preparation, never truncates the file or skips validation. + +Preparation is synchronous and pre-runtime, before signal registration and embedding-stop polling. The runner's cooperative startup-cancellation contract does not cover this phase; an early process signal follows the caller/OS signal disposition. Byte/work limits cannot bound filesystem I/O time. Add no detached task or cosmetic async timeout. Measure preparation time and peak memory on representative data and document this exception. + +### Lifetime and replacement + +The runner owns the reader; request/connection clones share it until teardown. Preparation failures drop partial state without creating files or requiring recovery. Startup retries repeat a read-only operation. + +Operators publish a complete replacement, update the file reference/mount and restart. Never rewrite a file in place during preparation. Active processes retain their snapshot after source changes/deletion. New processes validate the replacement before readiness; invalid replacements fail startup without substituting an older file. + +No update loop, live swap, update lock, persistence or database-specific shutdown hook is added. Operators may roll back only to data still permitted by its terms. + +## 7. Lookup and outcome semantics + +Extend resolution with an explicit no-match result and framework status with missing-reader-IP and no-match reasons. These must not collapse into CloudFront's missing-assertion outcome. Preserve selected provider identity in every selected outcome. + +| Situation | Outcome | +| ------------------------------------------------------------------------------------ | ------------------------------------------------ | +| Geo off | Existing `NoSelection` | +| Selected input expectation cannot admit a reader IP | Unavailable reader IP, no provider call | +| No record, no geographic codes, or IPv6 lookup against an IPv4-only database | Unavailable no match | +| Valid country with no subdivision | Located, country-only | +| Valid country and first-level subdivision | Located, country and subdivision assertion | +| Syntax-valid but application-unrecognized subdivision | Located; application recognition remains unknown | +| Malformed/unsupported country, malformed subdivision, or subdivision without country | Error, invalid geographic assertion | +| Unexpected decode/reader failure after successful preparation | Error, provider failure | +| CloudFront source not admitted or assertions absent/invalid | Existing CloudFront outcomes unchanged | + +Read `country.iso_code`, never registered/represented country as substitutes. For City records, use only `subdivisions[0].iso_code`; the schema orders entries largest to smallest. A missing/empty array or missing first-entry code means unknown subdivision, even if later entries have codes. Never promote a later district or infer codes from names. Country records are country-only. Decode only the geographic projection; structural verification may inspect unrelated fields. + +Use existing `CountryCode`, `SubdivisionCode` and `GeoLocation` validation. Preserve leading zeroes and normalize code case. Absent code fields are unknown; present empty or malformed strings are invalid. Unsupported country codes, including `XK`, remain errors without alias substitution. Never discard an invalid subdivision into country-only success. A record with neither selected code is no match; subdivision without country is an error. + +Provenance describes an estimate, not authenticated residence. Outcomes retain no IP, raw record or vendor error chain. Never resolve again during finalization. Unmanaged conversion and non-Axum adapters retain `NoSelection`. + +## 8. Deployment and privacy guidance + +Provide generic Axum examples with a read-only GeoLite City mount, explicit Rust selection and no stores/AWS/CloudFront dependencies. Cover direct `PeerAllowed` and reverse-proxy `ForwardedRequired` deployments using existing proxy trust. Explain that VPNs and intermediaries can make estimates unsuitable for jurisdiction decisions. + +Document external download, artifact verification, complete publication and restart. Record dataset version/build timestamp, checksum, source and terms. Keep download keys out of options, logs and images. Startup requires no writable config/data root or sidecar files. + +After logging initialization, emit a bounded native startup record with the loaded type, build timestamp and checksum. Earlier preparation failures return source-free error categories. New errors and input/selection `Debug` output redact IPs and paths. Expose no raw records, download keys, vendor error chains, diagnostic endpoint or automatic response header. + +Freshness remains the operator's obligation. Neither parsing nor file modification time proves it. No age threshold changes permission behavior or selects another source. Include attribution and update/deletion duties in the deployment checklist. + +Applications must preserve uncertainty, prevent cross-visitor shared caching of permission-sensitive responses and retain existing fail-closed behavior. Cache configuration, consent-cookie interpretation and real consent-policy tests remain downstream work. + +## 9. Acceptance and delivery checkpoints + +The input/lifecycle contract is reviewed before implementation. Deliver the following independently checkable units, reusing passing results while their inputs remain unchanged: + +| Unit | Required proof | +| ---------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Portable contract | Typed inputs, no-match/missing-reader outcomes, assertion validation, wrong-input behavior, reserved IDs and consumer migration compile and test | +| Checked-IP admission | Both reader modes with IPv4/IPv6, mapped addresses, both trusted forwarding families, partial client resolution, absent/malformed/unresolved chains, untrusted/missing peer and forged headers/extensions | +| Local reader | City/Country fixtures cover numeric and unknown subdivisions, missing first-entry code with a later code, absent/empty/invalid codes, unsupported country codes, no match and IPv6 against IPv4-only data | +| Startup preparation | Missing/unreadable/empty/truncated/oversized files, nonregular sources, unsupported metadata/schema, validation budget exhaustion and off-path corruption fail before initializer, bind and readiness; unsupported code strings alone do not fail preparation | +| Feature isolation | Test `axum` alone, each built-in separately and both together; unavailable selection fails startup, supplied reader-IP works without built-ins, off and CloudFront behavior stay unchanged | +| Real Axum hosting | One provider call per admitted request with admissible selected input, including no-match results; zero for off/inadmissible-input/refused/aborted/draining paths; immutable outcomes across preparers/middleware/handlers; no finalization lookup | +| Read-only lifetime | Read-only file/mount starts without sidecar writes; changing/removing the source does not change active results; restart loads a valid new snapshot and rejects an invalid replacement | +| Boundaries and lifecycle | Native startup/shutdown and WASM dependency regressions pass; after preparation, an already-requested embedding stop exits without initializer/bind/readiness; reader/filesystem dependencies stay native and default-off | +| Documentation and operations | Public Rust examples compile; mount/update/attribution instructions name supported schemas and terms; representative preparation time/memory and remaining accuracy gaps are recorded | + +Exercise startup failures through the real runner. Remove the loaded source and issue more requests to prove file independence. Assert loaded diagnostic values and redaction. Repeated capability inspection must perform no file I/O, even with a missing path. + +Extend `scripts/check_geo_dependency_contract.sh` and the existing CI feature matrix. Use focused tests, then native integration, formatting, lint, docs and affected adapter checks at the stable checkpoint. Report runtime evidence separately from compilation. + +Before implementation, the implementation plan must settle the remaining decisions: + +- Pin the reader version and confirm its verifier/decoder work bounds. Version 0.32.0 was inspected for this review; dependency compilation and representative-dataset measurements are not yet done. +- Set preparation limits and confirm the pre-runtime stop-observation exception is acceptable. +- Finalize Rust input/provenance/capability shapes and record the public API migration/release handling. + +These are explicit implementation gates, not claims of completed validation. DB-IP support and downstream consent-policy tests remain separate. + +## 10. Sources + +Database format and reader behavior are distinct from dataset licensing and accuracy. Recheck these sources when pinning implementation dependencies and publishing operator guidance: + +- [Rust `maxminddb` 0.32.0 reader](https://docs.rs/maxminddb/0.32.0/maxminddb/struct.Reader.html): owned in-memory construction, lookup, iteration and validation APIs; reader library uses the ISC license. +- [MaxMind City/Country database documentation](https://dev.maxmind.com/geoip/docs/databases/city-and-country/) and [City binary fields](https://dev.maxmind.com/geoip/docs/databases/city-and-country/city-binary/): code semantics, optional fields and subdivision ordering. +- [GeoLite integration and obligations](https://dev.maxmind.com/geoip/geolite2-free-geolocation-data/): free datasets, account/download keys, accuracy guidance and update/deletion obligations. +- [GeoLite EULA](https://www.maxmind.com/en/geolite/eula) and [external use/attribution guidance](https://support.maxmind.com/knowledge-base/articles/sell-or-display-data-from-geolite-databases-and-web-services): dataset terms, reviewed 2026-11-04 and to be rechecked before distribution. +- [MaxMind database updates](https://dev.maxmind.com/geoip/updating-databases/): operator-owned update process. +- [DB-IP Lite](https://db-ip.com/db/lite.php): alternative MMDB datasets, CC BY 4.0 attribution and monthly releases. diff --git a/examples/app-demo/Cargo.lock b/examples/app-demo/Cargo.lock index 738880a5d..d25d3cdfa 100644 --- a/examples/app-demo/Cargo.lock +++ b/examples/app-demo/Cargo.lock @@ -780,12 +780,16 @@ dependencies = [ "ctor", "edgezero-adapter", "edgezero-core", + "edgezero-store-aws", + "edgezero-store-celld", "futures", "futures-util", + "getrandom 0.4.3", "http", "http-body", "hyper", "hyper-util", + "ipnet", "libc", "log", "redb", @@ -961,6 +965,25 @@ dependencies = [ "validator", ] +[[package]] +name = "edgezero-store-aws" +version = "0.1.0" +dependencies = [ + "percent-encoding", + "serde", + "thiserror 2.0.21", + "url", +] + +[[package]] +name = "edgezero-store-celld" +version = "0.1.0" +dependencies = [ + "serde", + "thiserror 2.0.21", + "url", +] + [[package]] name = "either" version = "1.18.0" diff --git a/scripts/check_celld_scoped_isolation.sh b/scripts/check_celld_scoped_isolation.sh new file mode 100755 index 000000000..cd98a87fc --- /dev/null +++ b/scripts/check_celld_scoped_isolation.sh @@ -0,0 +1,111 @@ +#!/usr/bin/env bash +# Explicit disposable namespace proof. No host network or production credentials. +set -euo pipefail +if [[ $# != 1 || ! -x $1 || $1 != /* ]]; then + printf 'usage: %s /absolute/path/to/modified/celld\n' "$0" >&2 + exit 2 +fi +for tool in unshare ip nsenter setpriv mount curl python3; do + command -v "$tool" >/dev/null || { printf 'missing isolation prerequisite: %s\n' "$tool" >&2; exit 1; } +done +# Always create a private user/network/mount/PID world before any ip/mount work. +exec unshare --user --map-root-user --net --mount --pid --fork --mount-proc \ + env -i PATH="$PATH" bash -s -- "$1" <<'ISOLATED' +set -euo pipefail +binary=$1 +root=$(mktemp -d /tmp/edgezero-celld-isolation.XXXXXX) +server_pid= +app_wrapper= +app_pid= +cleanup() { + for pid in "$server_pid" "$app_wrapper" "$app_pid"; do + if [[ -n $pid ]]; then kill -KILL "$pid" 2>/dev/null || true; fi + done + wait 2>/dev/null || true + rm -rf -- "$root" +} +trap cleanup EXIT +mkdir -p "$root/service" "$root/control" +printf 'fixture-fleet-credential-not-a-real-secret\n' > "$root/service/fleet-credential" +ip link set lo up +ip link add scoped-server type veth peer name scoped-app +ip addr add 10.203.0.1/24 dev scoped-server +ip link set scoped-server up +cat > "$root/service/policy.json" <<'POLICY' +{"version":1,"listen":"10.203.0.1:7780","allow_remote_trusted_network":true,"stores":[{"alias":"state","namespace":"isolated-state"},{"alias":"config","namespace":"isolated-config"}],"access":{"mode":"trusted-network","grants":[{"store":"state","operations":["read","write","delete","list"]},{"store":"config","operations":["read"]}]}} +POLICY +env -i CELLD_TEST_DATA_DIR="$root/service/data" "$binary" \ + --listen 127.0.0.1:0 --internal-listen 127.0.0.1:0 \ + --store-api-config "$root/service/policy.json" > "$root/service/server.log" 2>&1 & +server_pid=$! +ready=false +for attempt in $(seq 1 100); do + status=$(curl --noproxy '*' --max-time 1 -s -o /dev/null -w '%{http_code}' \ + 'http://10.203.0.1:7780/v1/stores/state/value?key=probe' || true) + if [[ $status == 404 ]]; then ready=true; break; fi + kill -0 "$server_pid" 2>/dev/null || { printf 'celld startup failed\n' >&2; tail -40 "$root/service/server.log" >&2; exit 1; } + sleep 0.1 +done +[[ $ready == true ]] || { printf 'scoped listener startup deadline\n' >&2; exit 1; } +internal_port=$(python3 - "$root/service/server.log" <<'PY' +import re,sys +text=open(sys.argv[1]).read() +m=re.search(r'celld internal listening on 127\.0\.0\.1:(\d+)',text) +assert m, 'internal listener address absent' +print(m.group(1)) +PY +) +# Separate app network, mounts and process visibility. Hide the entire service +# tree before dropping capabilities in the actual app probe, not just chmod it. +unshare --net --mount --pid --fork --mount-proc bash -c ' + set -e + mount --make-rprivate / + mount -t tmpfs -o size=1m,mode=000 none "$1/service" + printf ready > "$1/control/app.ready" + exec sleep 120 +' bash "$root" & +app_wrapper=$! +for attempt in $(seq 1 100); do + if [[ -f $root/control/app.ready ]]; then + read -r app_pid < "/proc/$app_wrapper/task/$app_wrapper/children" || true + if [[ -n $app_pid ]]; then break; fi + fi + sleep 0.05 +done +[[ -n $app_pid ]] || { printf 'application namespace startup failed\n' >&2; exit 1; } +ip link set scoped-app netns "$app_pid" +nsenter --target "$app_pid" --net ip addr add 10.203.0.2/24 dev scoped-app +nsenter --target "$app_pid" --net ip link set scoped-app up +nsenter --target "$app_pid" --net ip link set lo up +nsenter --target "$app_pid" --net --mount --pid \ + setpriv --bounding-set=-all --inh-caps=-all --ambient-caps=-all \ + env -i PATH="$PATH" bash -s -- "$root" "$internal_port" <<'APP' +set -euo pipefail +root=$1 +internal=$2 +base=http://10.203.0.1:7780 +response="$root/control/app-response" +request() { curl --disable --noproxy '*' --max-time 3 -s -o "$response" -w '%{http_code}' "$@"; } +[[ $(request -X PUT --data-binary 'isolated-app-value' "$base/v1/stores/state/value?key=probe") == 204 ]] +[[ $(request "$base/v1/stores/state/value?key=probe") == 200 ]] +[[ $(< "$response") == isolated-app-value ]] +[[ $(request -X PUT --data-binary forbidden "$base/v1/stores/config/value?key=probe") == 403 ]] +[[ $(request "$base/v1/stores/undeclared/value?key=probe") == 403 ]] +[[ $(request "$base/shutdown") == 404 ]] +if curl --noproxy '*' --max-time 1 -sf "http://10.203.0.1:$internal/state" >/dev/null; then + printf 'application reached the internal listener by server address\n' >&2; exit 1 +fi +if curl --noproxy '*' --max-time 1 -sf "http://127.0.0.1:$internal/state" >/dev/null; then + printf 'application reached the server loopback listener\n' >&2; exit 1 +fi +[[ ! -r $root/service/fleet-credential ]] +[[ ! -r $root/service/policy.json ]] +for comm in /proc/[0-9]*/comm; do + if [[ $(< "$comm") == celld ]]; then + printf 'application can inspect the server process through procfs\n' >&2; exit 1 + fi +done +rm -f -- "$response" +printf 'Scoped API reachable; admin listener, service files and server procfs isolated.\n' +APP +ISOLATED diff --git a/scripts/check_geo_dependency_contract.sh b/scripts/check_geo_dependency_contract.sh new file mode 100755 index 000000000..de0a2a145 --- /dev/null +++ b/scripts/check_geo_dependency_contract.sh @@ -0,0 +1,132 @@ +#!/usr/bin/env bash +# Check isolated normal graphs, not workspace-unified or test dependencies. +set -euo pipefail +export LC_ALL=C + +scratch="$(mktemp -d)" +trap 'rm -rf "$scratch"' EXIT + +fail() { + printf 'Geo dependency contract: %s\n' "$*" >&2 + exit 1 +} + +graph() { + local name="$1" root="$2" + shift 2 + cargo tree --offline --locked --edges normal --prefix none --format '{p}|{f}' \ + -p "$root" "$@" > "$scratch/$name" + # Empty output, a wrong package, or a changed output format is not proof. + grep -Eq "^${root} v[^|]+[|]" "$scratch/$name" \ + || fail "missing expected root package in $name" + grep -Eq '^edgezero-core v[^|]+[|]' "$scratch/$name" \ + || fail "missing portable core in $name" +} + +features() { + local name="$1" root="$2" + awk -F '|' -v root="$root" '$1 ~ "^" root " v" { print $2; exit }' "$scratch/$name" +} + +assert_feature() { + local name="$1" feature="$2" expected="$3" selected + selected="$(features "$name" edgezero-adapter-axum)" + if grep -Eq "(^|,)${feature}(,|$)" <<< "$selected"; then + [[ "$expected" == on ]] || fail "$feature unexpectedly selected in $name" + else + [[ "$expected" == off ]] || fail "$feature missing in $name" + fi +} + +# CloudFront adds no dependencies; the local reader must remain native opt-in. +graph native-default edgezero-adapter-axum +graph native-off edgezero-adapter-axum --no-default-features --features axum +graph native-on edgezero-adapter-axum --no-default-features --features geo-cloudfront +graph native-maxmind edgezero-adapter-axum --no-default-features --features geo-maxmind +graph native-both edgezero-adapter-axum --no-default-features --features geo-cloudfront,geo-maxmind +for name in native-default native-off native-on native-maxmind native-both; do + assert_feature "$name" axum on + if grep -Eq '^aws-(config|sdk-[^ ]+) v' "$scratch/$name"; then + fail "AWS SDK leaked into $name" + fi +done +assert_feature native-default geo-cloudfront off +assert_feature native-off geo-cloudfront off +assert_feature native-on geo-cloudfront on +assert_feature native-maxmind geo-cloudfront off +assert_feature native-both geo-cloudfront on +for name in native-default native-off native-on; do + assert_feature "$name" geo-maxmind off + if grep -Eq '^maxminddb v' "$scratch/$name"; then + fail "MaxMind reader leaked into $name" + fi +done +for name in native-maxmind native-both; do + assert_feature "$name" geo-maxmind on + grep -Eq '^maxminddb v0[.]32[.]0[|]' "$scratch/$name" \ + || fail "missing pinned maxminddb 0.32.0 in $name" + grep -Eq '^sha2 v[^|]+[|]' "$scratch/$name" \ + || fail "missing snapshot SHA-256 dependency in $name" + if grep -Eq '^memmap2 v' "$scratch/$name"; then + fail "memory mapping dependency selected in $name" + fi + if ! awk -F '|' ' + /^maxminddb v/ { + sub(/ [(][*][)]$/, "", $2) + count = split($2, selected, ",") + for (i = 1; i <= count; i++) + if (selected[i] ~ /^(default|mmap|unsafe-str-decode)$/) exit 1 + } + ' "$scratch/$name"; then + fail "unsafe or default reader features selected in $name" + fi +done +# sha2 may already be present transitively in feature-off native graphs. +for pair in 'native-off native-on' 'native-maxmind native-both'; do + read -r baseline cloudfront <<< "$pair" + for name in "$baseline" "$cloudfront"; do + # Ignore feature annotations and duplicate-tree markers, retaining exact versions. + cut -d '|' -f 1 "$scratch/$name" | sort -u > "$scratch/$name.packages" + done + if ! cmp -s "$scratch/$baseline.packages" "$scratch/$cloudfront.packages"; then + diff -u "$scratch/$baseline.packages" "$scratch/$cloudfront.packages" >&2 || true + fail "geo-cloudfront changed the isolated normal dependency set in $cloudfront" + fi +done + +# Portable core and real platform graphs must not acquire native hosting/stores. +assert_portable() { + local name="$1" root="$2" + shift 2 + graph "$name" "$root" "$@" + if grep -Eq '^(edgezero-adapter-axum|edgezero-store-(aws|celld)|axum(-[^ ]+)?|tokio-[^ ]+|reqwest|hyper(-[^ ]+)?|mio|socket2|ipnet|maxminddb|memmap2|aws-config|aws-sdk-[^ ]+) v' "$scratch/$name"; then + fail "native runtime/store/SDK leaked into $name" + fi + # Pinned worker 0.8.5 already uses tokio 1.52.3 with default-features=false + # and no selected features on wasm32-unknown-unknown. That empty-feature + # dependency is not the native executor. Keep the exception exact so a worker + # upgrade or enabling runtime/net features requires a fresh isolation review. + if [[ "$name" == cloudflare ]]; then + grep -Eq '^worker v0[.]8[.]5[|]' "$scratch/$name" \ + || fail 'Cloudflare worker pin changed; review its empty-feature tokio exception' + grep -Eq '^tokio v1[.]52[.]3[|]$' "$scratch/$name" \ + || fail 'missing expected Cloudflare empty-feature tokio dependency' + fi + if ! awk -F '|' -v name="$name" ' + /^tokio v/ { + sub(/ [(][*][)]$/, "", $2) + if (name != "cloudflare" || $1 != "tokio v1.52.3" || $2 != "") exit 1 + } + ' "$scratch/$name"; then + fail "unexpected tokio version or native runtime features in $name" + fi +} +assert_portable core edgezero-core --no-default-features +for entry in 'cloudflare wasm32-unknown-unknown worker' 'fastly wasm32-wasip1 fastly' 'spin wasm32-wasip2 spin-sdk'; do + read -r adapter target sdk <<< "$entry" + assert_portable "$adapter" "edgezero-adapter-$adapter" \ + --no-default-features --features "$adapter" --target "$target" + grep -Eq "^${sdk} v[^|]+[|]" "$scratch/$adapter" \ + || fail "missing selected platform SDK in $adapter" +done +printf 'Geo default-off, CloudFront dependency equality, local reader opt-in, and portable isolation graphs passed.\n' diff --git a/scripts/check_native_store_dependency_contract.sh b/scripts/check_native_store_dependency_contract.sh new file mode 100755 index 000000000..7eb365d0a --- /dev/null +++ b/scripts/check_native_store_dependency_contract.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +# Independent normal dependency graphs; workspace --all-features is not isolation proof. +set -euo pipefail + +assert_no_sdk() { + local graph + graph="$(cargo tree --offline --locked --edges normal --prefix none "$@")" + if grep -Eq '^aws-(config|sdk-[^ ]+) v' <<< "$graph"; then + printf 'AWS SDK leaked into isolated graph: %s\n' "$*" >&2 + exit 1 + fi +} + +assert_no_sdk -p edgezero-cli --no-default-features --features cli,edgezero-adapter-axum +assert_no_sdk -p edgezero-store-aws --no-default-features +assert_no_sdk -p edgezero-store-aws --no-default-features --features appconfig-agent +assert_no_sdk -p edgezero-adapter-axum --no-default-features --features axum +assert_no_sdk -p edgezero-adapter-cloudflare --target wasm32-unknown-unknown --features cloudflare +assert_no_sdk -p edgezero-adapter-fastly --target wasm32-wasip1 --features fastly +assert_no_sdk -p edgezero-adapter-spin --target wasm32-wasip2 --features spin +dynamodb_graph="$(cargo tree --offline --locked --edges normal --prefix none -p edgezero-store-aws --no-default-features --features dynamodb-kv)" +secrets_graph="$(cargo tree --offline --locked --edges normal --prefix none -p edgezero-store-aws --no-default-features --features secrets-manager)" +stores_graph="$(cargo tree --offline --locked --edges normal --prefix none -p edgezero-adapter-axum --no-default-features --features aws-stores)" +if ! grep -Eq '^aws-sdk-dynamodb v' <<< "$dynamodb_graph" || ! grep -Eq '^aws-sdk-secretsmanager v' <<< "$secrets_graph"; then + printf 'Selected AWS service SDK is missing.\n' >&2 + exit 1 +fi +if grep -Eq '^aws-sdk-secretsmanager v' <<< "$dynamodb_graph" || grep -Eq '^aws-sdk-dynamodb v' <<< "$secrets_graph" || grep -Eq '^aws-sdk-dynamodb v' <<< "$stores_graph"; then + printf 'AWS service SDK leaked into an independently selected provider.\n' >&2 + exit 1 +fi +# Celld settings are present in bindings-only builds, but live transport is not. +celld_features() { + local graph + graph="$(cargo tree --offline --locked --edges normal --prefix none --format '{p}|{f}' "$@")" + awk -F'|' '/^edgezero-store-celld v/ { sub(/ [(][*][)]$/, "", $2); print $2; exit }' <<< "$graph" +} + +assert_celld_off() { + local selected + selected="$(celld_features "$@")" + if [[ -n "$selected" ]]; then + printf 'Live celld features leaked into isolated graph: %s: %s\n' "$*" "$selected" >&2 + exit 1 + fi +} +assert_celld_off -p edgezero-cli --no-default-features --features cli,edgezero-adapter-axum +assert_celld_off -p edgezero-adapter-axum --no-default-features --features axum +assert_celld_off -p edgezero-adapter-axum --no-default-features --features aws-stores,aws-dynamodb + +celld_settings="$(cargo tree --offline --locked --edges normal --prefix none -p edgezero-store-celld --no-default-features)" +if grep -Eq '^(reqwest|tokio|edgezero-core|aws-config|aws-sdk-[^ ]+) v' <<< "$celld_settings"; then + printf 'Runtime/client dependency leaked into settings-only celld.\n' >&2 + exit 1 +fi +for feature in config kv; do + assert_no_sdk -p edgezero-store-celld --no-default-features --features "$feature" + graph="$(cargo tree --offline --locked --edges normal --prefix none -p edgezero-store-celld --no-default-features --features "$feature")" + if grep -Eq '^(celld(-[^ ]+)?|v8|worker|worker-build|edgezero-adapter-[^ ]+) v' <<< "$graph"; then + printf 'Server/Worker/adapter dependency leaked into celld provider: %s\n' "$feature" >&2 + exit 1 + fi + selected="$(celld_features -p edgezero-store-celld --no-default-features --features "$feature")" + other=config + if [[ "$feature" == config ]]; then other=kv; fi + if ! grep -Eq "(^|,)$feature(,|$)" <<< "$selected" || grep -Eq "(^|,)$other(,|$)" <<< "$selected"; then + printf 'Celld service feature isolation failed: %s: %s\n' "$feature" "$selected" >&2 + exit 1 + fi +done +printf 'Native store SDK and celld feature isolation graphs passed.\n' diff --git a/scripts/check_outbound_legacy_api.sh b/scripts/check_outbound_legacy_api.sh index fdaf08c64..d4162f34d 100755 --- a/scripts/check_outbound_legacy_api.sh +++ b/scripts/check_outbound_legacy_api.sh @@ -968,10 +968,14 @@ assert_active_surface_fixture_rejected() { failed=0 scan \ - 'Proxy(Client|Handle|Request|Response|Service)|proxy_handle|edgezero_core::proxy|crate::proxy|pub mod proxy' \ + 'Proxy(Client|Handle|Request|Response|Service)|proxy_handle|edgezero_core::proxy' \ crates examples/app-demo tests/fixtures/reusable-app docs/guide README.md CLAUDE.md TODO.md Cargo.toml \ .claude/agents/code-architect.md || failed=1 +# Crate-local proxy paths identify the retired outbound module only in core. +# Axum's independent proxy module handles incoming proxy assertions. +scan 'crate::proxy|pub mod proxy' crates/edgezero-core || failed=1 + # Downstream response delivery is now adapter-owned. Keep the outbound-fetch # buffering helpers: they implement ResponseMode::Buffered and are unrelated to # response egress. diff --git a/scripts/generate_geoip_fixtures.mjs b/scripts/generate_geoip_fixtures.mjs new file mode 100644 index 000000000..fb3f756f2 --- /dev/null +++ b/scripts/generate_geoip_fixtures.mjs @@ -0,0 +1,218 @@ +#!/usr/bin/env node +// Original synthetic MMDB encoder. No vendor database or third-party data. +import { mkdir, readFile, readdir, writeFile } from 'node:fs/promises'; +import { fileURLToPath } from 'node:url'; +import { createHash } from 'node:crypto'; + +const directory = fileURLToPath(new URL('../crates/edgezero-adapter-axum/tests/fixtures/geoip/', import.meta.url)); +const check = process.argv.includes('--check'); +const fixtures = new Map(); +const join = (...parts) => Buffer.concat(parts.flat().map(part => Buffer.from(typeof part === 'number' ? [part] : part))); +function header(type, size) { + const extra = []; + let encoded = size; + if (size >= 65821) { encoded = 31; extra.push((size - 65821) >> 16, (size - 65821) >> 8, size - 65821); } + else if (size >= 285) { encoded = 30; extra.push((size - 285) >> 8, size - 285); } + else if (size >= 29) { encoded = 29; extra.push(size - 29); } + return Buffer.from(type < 8 ? [(type << 5) | encoded, ...extra] : [encoded, type - 7, ...extra]); +} +const string = value => join(header(2, Buffer.byteLength(value)), Buffer.from(value)); +function uint(type, value) { + let hex = BigInt(value).toString(16); + if (hex.length % 2) hex = `0${hex}`; + const bytes = value === 0 ? Buffer.alloc(0) : Buffer.from(hex, 'hex'); + return join(header(type, bytes.length), bytes); +} +const map = entries => join(header(7, entries.length), entries.map(([key, value]) => join(typeof key === 'string' ? string(key) : key, value))); +const array = entries => join(header(11, entries.length), entries); +const pointer = offset => join([0x38], uint32(offset)); +function uint32(value) { const bytes = Buffer.alloc(4); bytes.writeUInt32BE(value); return bytes; } +const code = value => map([['iso_code', string(value)]]); +const country = value => map([['country', code(value)]]); +const city = (c, s) => map([['country', code(c)], ['subdivisions', array([code(s)])]]); + +function addressBits(address, version) { + let value; + if (address.includes('.')) { + value = address.split('.').reduce((n, octet) => (n << 8n) | BigInt(octet), 0n); + } else { + const [left, right] = address.split('::'); + const a = left ? left.split(':') : []; + const b = right ? right.split(':') : []; + const words = right === undefined ? a : [...a, ...Array(8 - a.length - b.length).fill('0'), ...b]; + value = words.reduce((n, word) => (n << 16n) | BigInt(`0x${word}`), 0n); + } + return value.toString(2).padStart(version === 4 ? 32 : 128, '0'); +} + +function database(entries, { type = 'GeoLite2-City', version = 4, recordSize = 24, metadataExtra = [], metadataWorkLimit = false, epoch = 1793750400 } = {}) { + const root = {}; + for (const [cidr, record] of entries) { + const [address, prefix] = cidr.split('/'); + const bits = addressBits(address, version).slice(0, Number(prefix)); + let node = root; + for (const bit of bits) node = node[bit] ??= {}; + node.record = record; + } + const nodes = []; + function collect(node) { + if (!node || node.record !== undefined) return; + node.index = nodes.length; + nodes.push(node); + collect(node[0]); collect(node[1]); + } + collect(root); + const data = []; + const offsets = new Map(); + function child(node) { + if (!node) return nodes.length; + if (node.record === undefined) return node.index; + if (!offsets.has(node.record)) { + const offset = data.reduce((sum, bytes) => sum + bytes.length, 0); + offsets.set(node.record, offset); + data.push(typeof node.record === 'function' ? node.record(offset) : node.record); + } + return nodes.length + 16 + offsets.get(node.record); + } + const tree = nodes.map(node => { + const left = child(node[0]); const right = child(node[1]); + if (recordSize === 24) return join(uint32(left).subarray(1), uint32(right).subarray(1)); + if (recordSize === 28) return join(uint32(left & 0xffffff).subarray(1), [((left >>> 24) << 4) | (right >>> 24)], uint32(right & 0xffffff).subarray(1)); + return join(uint32(left), uint32(right)); + }); + const metadataEntries = [ + ['binary_format_major_version', uint(5, 2)], ['binary_format_minor_version', uint(5, 0)], + ['build_epoch', uint(9, epoch)], ['database_type', string(type)], + ['description', map([['en', string('Original synthetic EdgeZero test data')]])], + ['ip_version', uint(5, version)], ['languages', array([string('en')])], + ['node_count', uint(6, nodes.length)], ['record_size', uint(5, recordSize)], ...metadataExtra, + ]; + let metadata; + if (metadataWorkLimit) { + const count = 100; + const placeholder = array(Array.from({ length: count }, () => pointer(0))); + const start = map([...metadataEntries, ['unknown', placeholder]]).length; + metadata = join( + map([...metadataEntries, ['unknown', array(Array.from({ length: count }, (_, i) => pointer(start + i * 4)))]]), + Array.from({ length: count }, () => header(2, 65821)), Buffer.alloc(65821, 120), + ); + } else metadata = map(metadataEntries); + return join(tree, Buffer.alloc(16), data, Buffer.from('\xab\xcd\xefMaxMind.com', 'latin1'), metadata); +} +const add = (name, entries, options) => fixtures.set(`${name}.mmdb`, database(entries, options)); +const cityEntries = [ + ['127.0.0.0/8', city('US', 'CA')], + ['192.0.2.1/32', city('us', 'ca')], + ['192.0.2.2/32', country('GB')], + ['192.0.2.3/32', city('JP', '01')], + ['192.0.2.4/32', city('US', 'zz')], + ['192.0.2.5/32', map([['country', code('US')], ['subdivisions', array([map([]), code('NY')])]])], + ['192.0.2.6/32', map([['country', code('US')], ['subdivisions', array([])]])], + ['192.0.2.7/32', map([])], + ['192.0.2.8/32', country('XK')], + ['192.0.2.9/32', country('')], + ['192.0.2.10/32', city('US', '')], + ['192.0.2.11/32', city('US', 'US-CA')], + ['192.0.2.12/32', map([['subdivisions', array([code('CA')])]])], + ['192.0.2.13/32', map([['registered_country', code('US')], ['represented_country', code('US')]])], + ['192.0.2.14/32', map([['country', map([])]])], + ['192.0.2.15/32', map([['country', code('US')], ['country', code('GB')]])], + ['192.0.2.16/32', map([['country', code('US')], ['subdivisions', array([code('CA'), map([['iso_code', uint(5, 1)]])])]])], + ['192.0.2.17/32', city('US', 'a1B')], + ['192.0.2.18/32', country('UK')], + ['192.0.2.19/32', country(' U')], + ['192.0.2.20/32', city('US', 'é')], +]; +add('city', cityEntries); +add('city-replacement', [['127.0.0.0/8', city('US', 'NY')], ['192.0.2.1/32', city('US', 'NY')]], { epoch: 1793836800 }); +add('country', [['127.0.0.0/8', country('US')], ['192.0.2.1/32', map([['country', code('US')], ['subdivisions', uint(5, 1)]])]], { type: 'GeoLite2-Country' }); +// IPv4 embedding uses hexadecimal words in the IPv6 fixture. +add('ipv6', [['::c000:201/128', city('US', 'CA')], ['2001:db8::/32', city('GB', 'ENG')]], { version: 6 }); +for (const recordSize of [28, 32]) add(`record-${recordSize}`, [['192.0.2.0/24', city('US', 'CA')]], { recordSize }); +for (const [name, record] of [ + ['bad-country-parent', map([['country', string('US')]])], + ['bad-country-code', map([['country', map([['iso_code', uint(5, 1)]])]])], + ['bad-subdivisions-parent', map([['country', code('US')], ['subdivisions', string('CA')]])], + ['bad-first-subdivision', map([['country', code('US')], ['subdivisions', array([string('CA')])]])], + ['bad-subdivision-code', map([['country', code('US')], ['subdivisions', array([map([['iso_code', uint(5, 1)]])])]])], + ['bad-unrelated-key', map([['country', code('US')], ['unrelated', map([[uint(5, 0), string('x')]])]])], + ['bad-offpath-pointer', pointer(0xffffffff)], +]) add(name, [['192.0.2.1/32', city('US', 'CA')], ['203.0.113.1/32', record]]); +// The unknown field points to a pointer, which points to a valid string. +// Upstream verify accepts this; normal deserialize_any rejects the chain. +add('bad-unrelated-chain', [['192.0.2.0/24', offset => { + const base = map([['country', code('US')], ['unrelated', pointer(0)]]); + const target = offset + base.length; + return join(map([['country', code('US')], ['unrelated', pointer(target)]]), pointer(target + 5), string('x')); +}]]); +add('bad-root-chain', [['192.0.2.0/24', offset => join(pointer(offset + 5), pointer(offset + 10), country('US'))]]); +add('unsupported', [['192.0.2.0/24', country('US')]], { type: 'Other-City' }); +const cyclicTree = database([['192.0.2.0/24', country('US')]]); +cyclicTree.fill(0, 0, 6); +fixtures.set('bad-tree-cycle.mmdb', cyclicTree); +const badSeparator = database([['192.0.2.0/24', country('US')]]); +badSeparator[24 * 6] = 1; +fixtures.set('bad-separator.mmdb', badSeparator); +fixtures.set('truncated.mmdb', database([['192.0.2.0/24', country('US')]]).subarray(0, -10)); +add('bad-metadata', [['192.0.2.0/24', country('US')]], { metadataExtra: [['unknown', pointer(0xffffffff)]] }); +// Known upstream metadata-verifier gap. Retained for the ignored acceptance regression. +add('bad-metadata-key', [['192.0.2.0/24', country('US')]], { metadataExtra: [['unknown', map([[uint(5, 0), string('x')]])]] }); +// Structural expansion exhaustion. Tiny shared arrays recursively double, +// passing memoized verification but exceeding the normal decoder value budget. +add('expansion-limit', [['192.0.2.0/24', offset => { + const root = map([['country', code('US')], ['unrelated', pointer(0)]]); + const parts = [null]; + let current = offset + root.length; + for (let depth = 0; depth < 17; depth++) { + const next = current + header(11, 2).length + 10; + parts.push(join(header(11, 2), pointer(next), pointer(next))); + current = next; + } + parts[0] = map([['country', code('US')], ['unrelated', pointer(offset + root.length)]]); + parts.push(string('x')); + return join(parts); +}]]); +// Every normal MMDB scalar type plus ordinary single pointers remains valid. +add('all-values', [['192.0.2.0/24', offset => { + const fields = [ + ['country', code('US')], ['bytes', join(header(4, 2), [0xff, 0x00])], + ['u16', uint(5, 1)], ['u32', uint(6, 2)], ['u64', uint(9, 3)], ['u128', uint(10, 4)], + ['i32', join(header(8, 4), [255, 255, 255, 255])], + ['bool', header(14, 1)], ['double', join(header(3, 8), Buffer.alloc(8))], + ['float', join(header(15, 4), Buffer.alloc(4))], ['pointer', pointer(0)], + ]; + const root = map(fields); + fields[fields.length - 1][1] = pointer(offset + root.length); + return join(map(fields), map([['nested', array([string('hello')])]])); +}]]); +// Overlapping valid string targets exhaust verifier work rather than allocating +// their expanded contents. Each ASCII header is itself valid UTF-8 payload. +add('verification-limit', [['192.0.2.0/24', offset => { + const count = 100; + const initial = map([['country', code('US')], ['unrelated', array(Array.from({ length: count }, () => pointer(0)))]]); + const start = offset + initial.length; + const root = map([['country', code('US')], ['unrelated', array(Array.from({ length: count }, (_, i) => pointer(start + i * 4)))]]); + return join(root, Array.from({ length: count }, () => header(2, 65821)), Buffer.alloc(65821, 120)); +}]]); +add('metadata-verification-limit', [['192.0.2.0/24', country('US')]], { metadataWorkLimit: true }); +// An invalid UTF-8 scalar in an unconsumed record proves off-path verification. +add('bad-offpath-utf8', [['192.0.2.1/32', country('US')], ['203.0.113.1/32', map([['unrelated', join(header(2, 1), [255])]])]]); + +await mkdir(directory, { recursive: true }); +const hashes = []; +for (const [name, bytes] of fixtures) { + if (check) { + if (!(await readFile(`${directory}${name}`)).equals(bytes)) throw new Error(`Fixture differs: ${name}`); + } else await writeFile(`${directory}${name}`, bytes); + hashes.push(`${createHash('sha256').update(bytes).digest('hex')} ${name}`); +} +const sums = `${hashes.join('\n')}\n`; +if (check) { + if ((await readFile(`${directory}SHA256SUMS`, 'utf8')) !== sums) throw new Error('SHA256SUMS differs'); + const existing = (await readdir(directory)).filter(name => name.endsWith('.mmdb')); + if (existing.some(name => !fixtures.has(name))) throw new Error('Unexpected fixture file'); + console.log(`Verified ${fixtures.size} original fixtures and checksums`); +} else { + await writeFile(`${directory}SHA256SUMS`, sums); + console.log(`Wrote ${fixtures.size} original fixtures and checksums`); +} diff --git a/scripts/native_aws_test_admin.mjs b/scripts/native_aws_test_admin.mjs new file mode 100644 index 000000000..8cb66e2d8 --- /dev/null +++ b/scripts/native_aws_test_admin.mjs @@ -0,0 +1,167 @@ +#!/usr/bin/env node +// Local emulator fixture administration, never an application AWS client. +import assert from "node:assert/strict"; +import { randomUUID } from "node:crypto"; +import http from "node:http"; +import { pathToFileURL } from "node:url"; + +const TABLE = "edgezero-emulator-kv"; +const TEXT_SECRET = "edgezero-emulator-text"; +const VERSION_A = "a".repeat(32); +const VERSION_B = "b".repeat(32); + +function client(endpoint, service) { + const match = /^http:\/\/127\.0\.0\.1:([0-9]+)$/.exec(endpoint ?? ""); + if (!match || Number(match[1]) < 1 || Number(match[1]) > 65535) { + throw new Error(`Fixture endpoint must be literal http://127.0.0.1:, got: ${endpoint}`); + } + return (operation, payload = {}) => new Promise((resolve, reject) => { + const body = JSON.stringify(payload); + const date = new Date().toISOString().replace(/[:-]|\.\d{3}/g, ""); + // These emulators need a credential scope but do not validate signatures. + // Fixed dummy authorization is safe only because endpoints are strictly local. + const request = http.request(endpoint, { + method: "POST", + headers: { + "Content-Type": service === "dynamodb" ? "application/x-amz-json-1.0" : "application/x-amz-json-1.1", + "Content-Length": Buffer.byteLength(body), + "X-Amz-Target": `${service === "dynamodb" ? "DynamoDB_20120810" : "secretsmanager"}.${operation}`, + "X-Amz-Date": date, + Authorization: `AWS4-HMAC-SHA256 Credential=DUMMYLOCAL/${date.slice(0, 8)}/us-east-1/${service}/aws4_request, SignedHeaders=host;x-amz-date, Signature=${"0".repeat(64)}`, + }, + }, (response) => { + const chunks = []; + let size = 0; + response.on("data", (chunk) => { + size += chunk.length; + if (size > 1024 * 1024) request.destroy(new Error("Oversized fixture response")); + else chunks.push(chunk); + }); + response.on("error", reject); + response.on("end", () => { + const text = Buffer.concat(chunks).toString("utf8"); + if (response.statusCode !== 200) { + reject(new Error(`${service}.${operation} failed: HTTP ${response.statusCode} ${text}`)); + return; + } + try { resolve(JSON.parse(text)); } catch (error) { reject(error); } + }); + }); + // No environment proxy discovery, redirects, credential lookup or retries. + const timer = setTimeout(() => request.destroy(new Error("Fixture request timeout")), 3000); + request.on("error", reject); + request.on("close", () => clearTimeout(timer)); + request.end(body); + }); +} + +async function seed(dynamo, secrets) { + const keys = [ + { AttributeName: "pk", KeyType: "HASH" }, + { AttributeName: "sk", KeyType: "RANGE" }, + ]; + const attributes = [ + { AttributeName: "pk", AttributeType: "S" }, + { AttributeName: "sk", AttributeType: "S" }, + ]; + await dynamo("CreateTable", { + TableName: TABLE, KeySchema: keys, AttributeDefinitions: attributes, BillingMode: "PAY_PER_REQUEST", + }); + const wrong = "edgezero-emulator-wrong"; + await dynamo("CreateTable", { + TableName: wrong, KeySchema: keys.slice(0, 1), AttributeDefinitions: attributes.slice(0, 1), + BillingMode: "PAY_PER_REQUEST", + }); + const indexed = "edgezero-emulator-indexed"; + await dynamo("CreateTable", { + TableName: indexed, KeySchema: keys, AttributeDefinitions: attributes, BillingMode: "PAY_PER_REQUEST", + GlobalSecondaryIndexes: [{ + IndexName: "unsupported-index", KeySchema: [{ AttributeName: "sk", KeyType: "HASH" }], + Projection: { ProjectionType: "KEYS_ONLY" }, + }], + }); + for (const [key, value, expired] of [ + ["expired/0", "expired", true], ["expired/1", "live", false], ["renew/item", "old", true], + ]) { + const item = { pk: { S: "primary" }, sk: { S: key }, v: { B: Buffer.from(value).toString("base64") } }; + if (expired) Object.assign(item, { expires_at_ms: { N: "1" }, ttl_epoch_s: { N: "1" } }); + await dynamo("PutItem", { TableName: TABLE, Item: item }); + } + const text = await secrets("CreateSecret", { + Name: TEXT_SECRET, SecretString: "secret-A", ClientRequestToken: VERSION_A, + }); + const versionB = await secrets("PutSecretValue", { + SecretId: text.ARN, SecretString: "secret-B", ClientRequestToken: VERSION_B, VersionStages: ["CANDIDATE"], + }); + await secrets("UpdateSecretVersionStage", { + SecretId: text.ARN, VersionStage: "PINNED", MoveToVersionId: text.VersionId, + }); + const wholeJson = await secrets("CreateSecret", { + Name: "edgezero-emulator-json", SecretString: ' { "answer": 42 }\n', ClientRequestToken: randomUUID(), + }); + const binary = await secrets("CreateSecret", { + Name: "edgezero-emulator-binary", SecretBinary: Buffer.from([0, 255, 10, 34]).toString("base64"), + ClientRequestToken: randomUUID(), + }); + return { + table: TABLE, wrong_table: wrong, indexed_table: indexed, missing_table: "edgezero-emulator-missing", + secrets: { + text: { arn: text.ARN, version_a: text.VersionId, version_b: versionB.VersionId }, + json: { arn: wholeJson.ARN }, binary: { arn: binary.ARN }, + }, + }; +} + +async function rotate(secrets) { + // Moto 5.2.2 does not update no-selector reads after a manual AWSCURRENT move. + // Move PINNED first: PutSecretValue also resets the old current version's stages. + // The application snapshots stay A while independent reads confirm adoption of B. + await secrets("UpdateSecretVersionStage", { + SecretId: TEXT_SECRET, VersionStage: "PINNED", MoveToVersionId: VERSION_B, RemoveFromVersionId: VERSION_A, + }); + const current = await secrets("PutSecretValue", { + SecretId: TEXT_SECRET, SecretString: "secret-B", ClientRequestToken: "c".repeat(32), + }); + for (const selector of [{}, { VersionStage: "PINNED" }, { VersionId: VERSION_A }]) { + const actual = await secrets("GetSecretValue", { SecretId: TEXT_SECRET, ...selector }); + assert.equal(actual.SecretString, selector.VersionId ? "secret-A" : "secret-B"); + } + return { version_current: current.VersionId }; +} + +export async function fixtureAction(action, payload, endpoints = { + dynamo: process.env.EDGEZERO_AWS_TEST_DYNAMODB_ENDPOINT, + moto: process.env.EDGEZERO_AWS_TEST_SECRETS_ENDPOINT, +}) { + const dynamo = client(endpoints.dynamo, "dynamodb"); + const secrets = client(endpoints.moto, "secretsmanager"); + switch (action) { + case "ready": + await dynamo("ListTables"); + await secrets("ListSecrets"); + return {}; + case "seed": return seed(dynamo, secrets); + case "rotate": return rotate(secrets); + case "delete_secret": + await secrets("DeleteSecret", { SecretId: TEXT_SECRET, ForceDeleteWithoutRecovery: true }); + return {}; + case "item": { + const { Item: item } = await dynamo("GetItem", { + TableName: TABLE, Key: { pk: { S: payload.namespace }, sk: { S: payload.key } }, ConsistentRead: true, + }); + if (!item) return { present: false }; + return { + present: true, value: [...Buffer.from(item.v.B, "base64")], + expires_at_ms: item.expires_at_ms ? Number(item.expires_at_ms.N) : null, + ttl_epoch_s: item.ttl_epoch_s ? Number(item.ttl_epoch_s.N) : null, + }; + } + default: throw new Error(`unknown fixture action: ${action}`); + } +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + Promise.resolve().then(() => fixtureAction(process.argv[2], JSON.parse(process.argv[3]))) + .then((result) => console.log(JSON.stringify(result))) + .catch((error) => { console.error(error); process.exitCode = 1; }); +} diff --git a/scripts/native_aws_test_admin.test.mjs b/scripts/native_aws_test_admin.test.mjs new file mode 100644 index 000000000..922732368 --- /dev/null +++ b/scripts/native_aws_test_admin.test.mjs @@ -0,0 +1,134 @@ +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import http from "node:http"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; +import { fixtureAction } from "./native_aws_test_admin.mjs"; + +const exec = promisify(execFile); +const script = fileURLToPath(new URL("./native_aws_test_admin.mjs", import.meta.url)); + +async function emulator(t, reply = () => ({})) { + const calls = []; + const server = http.createServer(async (request, response) => { + const chunks = []; + for await (const chunk of request) chunks.push(chunk); + const call = { + target: request.headers["x-amz-target"], headers: request.headers, + payload: JSON.parse(Buffer.concat(chunks)), method: request.method, + }; + calls.push(call); + const result = reply(call); + response.writeHead(result.status ?? 200); + response.end(result.raw ?? JSON.stringify(result)); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + t.after(() => new Promise((resolve) => server.close(resolve))); + const endpoint = `http://127.0.0.1:${server.address().port}`; + return { calls, endpoints: { dynamo: endpoint, moto: endpoint } }; +} + +test("fixture administration rejects non-loopback endpoints before sending requests", async (t) => { + const setup = await emulator(t); + for (const endpoint of [ + undefined, "https://127.0.0.1:8000", "http://localhost:8000", "http://127.0.0.1:0", + "http://127.0.0.1:65536", "http://127.0.0.1:8000/", "http://127.0.0.1:8000@aws.example", + ]) { + for (const service of ["dynamo", "moto"]) { + await assert.rejects(fixtureAction("ready", {}, { ...setup.endpoints, [service]: endpoint }), /literal/); + } + } + assert.deepEqual(setup.calls, []); +}); + +test("readiness checks both emulator JSON APIs with dummy credentials", async (t) => { + const setup = await emulator(t); + assert.deepEqual(await fixtureAction("ready", {}, setup.endpoints), {}); + assert.deepEqual(setup.calls.map((call) => call.target), ["DynamoDB_20120810.ListTables", "secretsmanager.ListSecrets"]); + for (const call of setup.calls) { + assert.equal(call.method, "POST"); + assert.deepEqual(call.payload, {}); + assert.match(call.headers.authorization, /Credential=DUMMYLOCAL\/\d{8}\/us-east-1\//); + } +}); + +test("seed preserves table schemas, expiry rows, secret bytes and version IDs", async (t) => { + const setup = await emulator(t, ({ payload }) => ({ ARN: payload.Name ?? payload.SecretId, VersionId: payload.ClientRequestToken })); + const result = await fixtureAction("seed", {}, setup.endpoints); + const tables = setup.calls.filter((call) => call.target.endsWith(".CreateTable")).map((call) => call.payload); + assert.deepEqual(tables.map((table) => table.KeySchema.length), [2, 1, 2]); + assert.equal(tables[2].GlobalSecondaryIndexes[0].IndexName, "unsupported-index"); + assert.equal(result.table, tables[0].TableName); + assert.equal(result.wrong_table, tables[1].TableName); + assert.equal(result.indexed_table, tables[2].TableName); + const rows = setup.calls.filter((call) => call.target.endsWith(".PutItem")).map((call) => call.payload.Item); + assert.deepEqual(rows.map((row) => Buffer.from(row.v.B, "base64").toString()), ["expired", "live", "old"]); + assert.deepEqual(rows.map((row) => row.expires_at_ms?.N), ["1", undefined, "1"]); + assert.deepEqual(rows.map((row) => row.ttl_epoch_s?.N), ["1", undefined, "1"]); + const secrets = setup.calls.filter((call) => call.target.endsWith(".CreateSecret")).map((call) => call.payload); + assert.equal(secrets[1].SecretString, ' { "answer": 42 }\n'); + assert.deepEqual([...Buffer.from(secrets[2].SecretBinary, "base64")], [0, 255, 10, 34]); + for (const secret of secrets) assert.ok(secret.ClientRequestToken.length >= 32); + assert.equal(result.secrets.text.version_a, "a".repeat(32)); + assert.equal(result.secrets.text.version_b, "b".repeat(32)); +}); + +test("rotation moves PINNED before default adoption and independently checks selectors", async (t) => { + const setup = await emulator(t, ({ payload }) => ({ + VersionId: payload.ClientRequestToken, + SecretString: payload.VersionId ? "secret-A" : "secret-B", + })); + assert.deepEqual(await fixtureAction("rotate", {}, setup.endpoints), { version_current: "c".repeat(32) }); + assert.deepEqual(setup.calls.map((call) => call.target), [ + "secretsmanager.UpdateSecretVersionStage", "secretsmanager.PutSecretValue", + "secretsmanager.GetSecretValue", "secretsmanager.GetSecretValue", "secretsmanager.GetSecretValue", + ]); + assert.equal(setup.calls[0].payload.MoveToVersionId, "b".repeat(32)); + assert.equal(setup.calls[0].payload.RemoveFromVersionId, "a".repeat(32)); + assert.equal(setup.calls[3].payload.VersionStage, "PINNED"); + assert.equal(setup.calls[4].payload.VersionId, "a".repeat(32)); +}); + +test("rotation fails if independent reads do not confirm adoption", async (t) => { + const setup = await emulator(t, () => ({ SecretString: "secret-A" })); + await assert.rejects(fixtureAction("rotate", {}, setup.endpoints), /secret-A.*secret-B/s); +}); + +test("item readback decodes binary values and preserves TTL and absence output", async (t) => { + let item; + const setup = await emulator(t, () => ({ Item: item })); + const read = () => fixtureAction("item", { namespace: "primary", key: "key" }, setup.endpoints); + assert.deepEqual(await read(), { present: false }); + item = { v: { B: Buffer.from([0, 255, 10, 34]).toString("base64") }, expires_at_ms: { N: "1234" }, ttl_epoch_s: { N: "2" } }; + assert.deepEqual(await read(), { present: true, value: [0, 255, 10, 34], expires_at_ms: 1234, ttl_epoch_s: 2 }); + delete item.expires_at_ms; + delete item.ttl_epoch_s; + assert.deepEqual(await read(), { present: true, value: [0, 255, 10, 34], expires_at_ms: null, ttl_epoch_s: null }); + assert.deepEqual(setup.calls[0].payload.Key, { pk: { S: "primary" }, sk: { S: "key" } }); + assert.equal(setup.calls[0].payload.ConsistentRead, true); +}); + +test("CLI returns JSON only and rejects unknown actions and malformed input", async (t) => { + const setup = await emulator(t); + const options = { env: { + ...process.env, + EDGEZERO_AWS_TEST_DYNAMODB_ENDPOINT: setup.endpoints.dynamo, + EDGEZERO_AWS_TEST_SECRETS_ENDPOINT: setup.endpoints.moto, + } }; + const { stdout, stderr } = await exec(process.execPath, [script, "delete_secret", "{}"], options); + assert.equal(stdout, "{}\n"); + assert.equal(stderr, ""); + assert.deepEqual(setup.calls[0].payload, { SecretId: "edgezero-emulator-text", ForceDeleteWithoutRecovery: true }); + await assert.rejects(exec(process.execPath, [script, "unknown", "{}"], options), /unknown fixture action/); + await assert.rejects(exec(process.execPath, [script, "seed", "not JSON"], options)); + assert.equal(setup.calls.length, 1); +}); + +test("HTTP failures, redirects and malformed responses fail without retries", async (t) => { + for (const result of [{ status: 500 }, { status: 302 }, { raw: "not JSON" }]) { + const setup = await emulator(t, () => result); + await assert.rejects(fixtureAction("ready", {}, setup.endpoints)); + assert.equal(setup.calls.length, 1); + } +}); diff --git a/scripts/run_native_aws_integration.mjs b/scripts/run_native_aws_integration.mjs new file mode 100755 index 000000000..42c79702b --- /dev/null +++ b/scripts/run_native_aws_integration.mjs @@ -0,0 +1,196 @@ +#!/usr/bin/env node +// Required CI proof against independent local services, without an AWS account. +import { execFile, spawn } from "node:child_process"; +import { mkdtemp, mkdir, rm, writeFile } from "node:fs/promises"; +import http from "node:http"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { setTimeout as delay } from "node:timers/promises"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { promisify } from "node:util"; +import { fixtureAction } from "./native_aws_test_admin.mjs"; + +const exec = promisify(execFile); +const root = fileURLToPath(new URL("../", import.meta.url)); +// Multi-architecture index digests checked against the upstream registries. +// DynamoDB Local 3.3.1; Moto 5.2.2; AppConfig Agent 2.0.224095. +const images = { + dynamo: "amazon/dynamodb-local@sha256:ff89bd48ff32cd8d9be5fee8873b65b8854dc408f1afe881be6eb00247bc0dab", + moto: "ghcr.io/getmoto/motoserver@sha256:d8ae5edc2bf080e7e4c13f9bd4b29b53ac3b4427e92956318db3dbe23ec43eb7", + agent: "public.ecr.aws/aws-appconfig/aws-appconfig-agent@sha256:cd604064652c56d9d6dc8435aa0d4bf3e90aa348830fa3b8a7b8650e3500ec3e", +}; + +export function publishedEndpoint(address) { + const match = /^127\.0\.0\.1:([0-9]+)$/.exec(address.trim()); + if (!match || Number(match[1]) < 1 || Number(match[1]) > 65535) { + throw new Error(`Expected one exclusively loopback-published port, got: ${address}`); + } + return `http://${address.trim()}`; +} + +async function docker(args, timeout = 30_000) { + const { stdout, stderr } = await exec("docker", args, { + timeout, + killSignal: "SIGKILL", + maxBuffer: 10 * 1024 * 1024, + }); + return (args[0] === "logs" ? `${stdout}${stderr}` : stdout).trim(); +} + +function agentReady(endpoint) { + return new Promise((resolve, reject) => { + // http.request does not discover environment proxies or follow redirects. + const request = http.get( + `${endpoint}/applications/probe/environments/test/configurations/ready`, + { headers: { Authorization: "Bearer local-agent-token" } }, + (response) => { + let body = ""; + response.setEncoding("utf8"); + response.on("data", (chunk) => { + body += chunk; + if (body.length > 4096) request.destroy(new Error("Oversized readiness response")); + }); + response.on("error", reject); + response.on("end", () => resolve(response.statusCode === 200 && body === "agent-ready\n")); + }, + ); + const timer = setTimeout(() => request.destroy(new Error("Agent readiness timeout")), 3000); + request.on("error", reject); + request.on("close", () => clearTimeout(timer)); + }); +} + +export async function main() { + await docker(["info"], 20_000); + const work = await mkdtemp(join(tmpdir(), "edgezero-aws-test-")); + const prefix = `edgezero-aws-${work.split("/").at(-1)}`; + const network = `${prefix}-network`; + const names = { dynamo: `${prefix}-dynamo`, moto: `${prefix}-moto`, agent: `${prefix}-agent` }; + const configs = join(work, "configs"); + let testProcess; + let testGroup; + let interruptTimer; + let interrupted; + let failure; + const signalTest = (signal) => { + if (testGroup) { + try { process.kill(-testGroup, signal); } catch (error) { + if (error.code !== "ESRCH") throw error; + } + } + }; + const onInterrupt = (signal) => { + interrupted = signal; + signalTest("SIGTERM"); + if (!interruptTimer) interruptTimer = setTimeout(() => signalTest("SIGKILL"), 5000); + }; + const onSigint = () => onInterrupt("SIGINT"); + const onSigterm = () => onInterrupt("SIGTERM"); + process.on("SIGINT", onSigint); + process.on("SIGTERM", onSigterm); + const checkInterrupted = () => { + if (interrupted) throw new Error(`Interrupted by ${interrupted}`); + }; + + try { + for (const image of Object.values(images)) { + checkInterrupted(); + console.log(`Pulling pinned service ${image}`); + await docker(["pull", image], 180_000); + } + checkInterrupted(); + await mkdir(configs); + await writeFile(join(configs, "probe:test:ready"), "agent-ready\n"); + // A normal bridge supports published ports. It is not an egress-deny sandbox. + await docker(["network", "create", network]); + await docker([ + "run", "-d", "--name", names.dynamo, "--network", network, + "--network-alias", "dynamodb", "--publish", "127.0.0.1::8000", + images.dynamo, "-jar", "DynamoDBLocal.jar", "-inMemory", "-sharedDb", "-disableTelemetry", + ]); + checkInterrupted(); + await docker([ + "run", "-d", "--name", names.moto, "--network", network, + "--publish", "127.0.0.1::5000", images.moto, + ]); + await docker([ + "run", "-d", "--name", names.agent, "--network", network, + "--publish", "127.0.0.1::2772", "--mount", `type=bind,source=${configs},target=/configs,readonly`, + "--env", "LOCAL_DEVELOPMENT_DIRECTORY=/configs", "--env", "SERVICE_REGION=us-east-1", + "--env", "AWS_ACCESS_KEY_ID=DUMMYLOCAL", "--env", "AWS_SECRET_ACCESS_KEY=DUMMYLOCAL", + "--env", "AWS_EC2_METADATA_DISABLED=true", "--env", "ACCESS_TOKEN=local-agent-token", + "--env", "HTTP_HOST=0.0.0.0", "--env", "POLL_INTERVAL=1s", images.agent, + ]); + const endpoints = {}; + for (const [service, port] of [["dynamo", 8000], ["moto", 5000], ["agent", 2772]]) { + endpoints[service] = publishedEndpoint(await docker(["port", names[service], `${port}/tcp`])); + } + const deadline = Date.now() + 60_000; + let ready = false; + let readinessError; + while (Date.now() < deadline) { + checkInterrupted(); + try { + await fixtureAction("ready", {}, endpoints); + ready = await agentReady(endpoints.agent); + if (ready) break; + } catch (error) { readinessError = error; } + await delay(200); + } + if (!ready) throw new Error("Local AWS services did not become ready within 60 seconds", { cause: readinessError }); + checkInterrupted(); + console.log("Pinned local AWS services ready; running generated consumer contracts"); + const env = { + ...process.env, + TMPDIR: work, + EDGEZERO_AWS_TEST_DYNAMODB_ENDPOINT: endpoints.dynamo, + EDGEZERO_AWS_TEST_SECRETS_ENDPOINT: endpoints.moto, + EDGEZERO_AWS_TEST_AGENT_ENDPOINT: endpoints.agent, + EDGEZERO_AWS_TEST_AGENT_DIRECTORY: configs, + EDGEZERO_AWS_TEST_NODE: process.execPath, + EDGEZERO_AWS_TEST_AGENT_TOKEN: "local-agent-token", + }; + // The nonzero runner rejects misspelled filters and accidental cfg exclusion. + await new Promise((resolve, reject) => { + testProcess = spawn(join(root, "scripts/run_test_nonzero.sh"), [ + "--ignored", "generated_native_aws_emulator_contracts", "cargo", "test", + "--offline", "--locked", "-p", "edgezero-cli", "--test", "native_aws_emulator_proof", + ], { cwd: root, env, stdio: "inherit", detached: true }); + testGroup = testProcess.pid; + let timedOut = false; + const timer = setTimeout(() => { timedOut = true; signalTest("SIGKILL"); }, 20 * 60_000); + testProcess.once("error", (error) => { clearTimeout(timer); reject(error); }); + testProcess.once("close", (code, signal) => { + clearTimeout(timer); + testProcess = undefined; + if (code === 0 && !timedOut && !interrupted) resolve(); + else reject(new Error(`Native AWS integration failed: ${timedOut ? "20 minute timeout" : interrupted ?? signal ?? `exit ${code}`}`)); + }); + }); + } catch (error) { + failure = error; + for (const container of Object.values(names)) { + try { console.error(`--- ${container} logs ---\n${await docker(["logs", container], 10_000)}`); } + catch (logError) { console.error(logError.message); } + } + } finally { + clearTimeout(interruptTimer); + signalTest("SIGKILL"); + for (const container of Object.values(names)) { + try { await docker(["rm", "-f", container], 10_000); } + catch (error) { if (!failure) failure = error; } + } + try { await docker(["network", "rm", network], 20_000); } + catch (error) { if (!failure) failure = error; } + await rm(work, { recursive: true, force: true }); + clearTimeout(interruptTimer); + process.off("SIGINT", onSigint); + process.off("SIGTERM", onSigterm); + } + if (interrupted && !failure) failure = new Error(`Interrupted by ${interrupted} during cleanup`); + if (failure) throw failure; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch((error) => { console.error(error); process.exitCode = 1; }); +} diff --git a/scripts/run_native_aws_integration.test.mjs b/scripts/run_native_aws_integration.test.mjs new file mode 100644 index 000000000..75b0ace61 --- /dev/null +++ b/scripts/run_native_aws_integration.test.mjs @@ -0,0 +1,144 @@ +import assert from "node:assert/strict"; +import { execFile, spawn } from "node:child_process"; +import { mkdtemp, mkdir, readFile, readdir, rm, writeFile } from "node:fs/promises"; +import http from "node:http"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; +import { setTimeout as delay } from "node:timers/promises"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; +import { publishedEndpoint } from "./run_native_aws_integration.mjs"; + +const exec = promisify(execFile); +const runner = fileURLToPath(new URL("./run_native_aws_integration.mjs", import.meta.url)); + +test("published service ports must be one literal IPv4 loopback address", () => { + assert.equal(publishedEndpoint("127.0.0.1:8000\n"), "http://127.0.0.1:8000"); + for (const address of [ + "0.0.0.0:8000", "localhost:8000", "[::1]:8000", "127.0.0.1:0", + "127.0.0.1:65536", "127.0.0.1:8000\n0.0.0.0:8000", "127.0.0.1:8000/path", "", + ]) { + assert.throws(() => publishedEndpoint(address), /exclusively loopback/); + } +}); + +test("missing Docker fails instead of silently skipping the suite", async () => { + await assert.rejects( + exec(process.execPath, [runner], { env: { PATH: "/nonexistent" }, timeout: 10_000 }), + (error) => error.code !== 0 && error.stderr.includes("ENOENT"), + ); +}); + +async function fixture(t, { + failPull = false, port = 8000, emptySuite = false, hang = false, + suiteCode = 17, slowCleanup = false, +} = {}) { + const directory = await mkdtemp(join(tmpdir(), "edgezero-runner-unit-")); + t.after(() => rm(directory, { recursive: true, force: true })); + const temporary = join(directory, "temporary"); + await mkdir(temporary); + const log = join(directory, "docker.log"); + // These commands only record runner decisions. Real-service coverage lives in Rust. + await writeFile(join(directory, "docker"), `#!/bin/sh +printf '%s\\n' "$*" >> "$MOCK_DOCKER_LOG" +case "$1" in + pull) exit ${failPull ? 1 : 0} ;; + port) printf '127.0.0.1:${port}\\n' ;; + logs) printf 'synthetic-service-log\\n' >&2 ;; + rm) ${slowCleanup ? "printf 'cleanup-start\\n' >> \"$MOCK_DOCKER_LOG\"; sleep 0.2" : ":"} ;; + *) exit 0 ;; +esac +`, { mode: 0o755 }); + await writeFile(join(directory, "cargo"), `#!/bin/sh +case "$*" in + *--list*) ${emptySuite ? ":" : "printf 'generated_native_aws_emulator_contracts: test\\n'"} ;; + *) ${hang ? "mkdir \"$TMPDIR/generated-project\"; printf log > \"$TMPDIR/generated-log\"; printf 'runtime-start\\n' >> \"$MOCK_DOCKER_LOG\"; sleep 60" : `exit ${suiteCode}`} ;; +esac +`, { mode: 0o755 }); + return { + log, temporary, + options: { + env: { ...process.env, PATH: `${directory}:/usr/bin:/bin`, TMPDIR: temporary, MOCK_DOCKER_LOG: log }, + timeout: 15_000, + }, + }; +} + +test("failed pinned image acquisition still cleans containers, network and temp files", async (t) => { + const setup = await fixture(t, { failPull: true }); + await assert.rejects(exec(process.execPath, [runner], setup.options)); + const log = await readFile(setup.log, "utf8"); + assert.match(log, /pull amazon\/dynamodb-local@sha256:[a-f0-9]{64}/); + assert.match(log, /rm -f edgezero-aws-/); + assert.match(log, /network rm edgezero-aws-/); + assert.deepEqual(await readdir(setup.temporary), []); +}); + +async function readyServer(t) { + const server = http.createServer((request, response) => { + request.resume(); + response.end(request.method === "POST" ? "{}" : "agent-ready\n"); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + t.after(() => new Promise((resolve) => server.close(resolve))); + return server.address().port; +} + +test("suite failure is propagated and service stderr logs survive cleanup", async (t) => { + const setup = await fixture(t, { port: await readyServer(t) }); + await assert.rejects(exec(process.execPath, [runner], setup.options), (error) => { + assert.match(error.stderr, /Native AWS integration failed: exit 17/); + assert.match(error.stderr, /synthetic-service-log/); + return true; + }); + const log = await readFile(setup.log, "utf8"); + assert.match(log, /network rm edgezero-aws-/); + assert.doesNotMatch(log, /admin\.py|python|exec /); + assert.deepEqual(await readdir(setup.temporary), []); +}); + +test("a zero-test suite fails the nonzero guard", async (t) => { + const setup = await fixture(t, { port: await readyServer(t), emptySuite: true }); + await assert.rejects(exec(process.execPath, [runner], setup.options), (error) => { + assert.match(error.stderr, /selected test suite contains zero tests/); + return true; + }); + assert.deepEqual(await readdir(setup.temporary), []); +}); + +async function interruptAt(setup, marker) { + const child = spawn(process.execPath, [runner], { env: setup.options.env, stdio: "pipe" }); + child.stdout.resume(); + child.stderr.resume(); + const closed = new Promise((resolve, reject) => { + child.once("error", reject); + child.once("close", (code) => resolve(code)); + }); + const timer = setTimeout(() => child.kill("SIGKILL"), 12_000); + try { + const deadline = Date.now() + 5000; + while (!(await readFile(setup.log, "utf8").catch(() => "")).includes(marker)) { + assert.ok(Date.now() < deadline, `synthetic runner must reach ${marker}`); + await delay(20); + } + child.kill("SIGTERM"); + assert.notEqual(await closed, 0); + assert.match(await readFile(setup.log, "utf8"), /network rm edgezero-aws-/); + assert.deepEqual(await readdir(setup.temporary), []); + } finally { + clearTimeout(timer); + child.kill("SIGKILL"); + await closed; + } +} + +test("interrupting an active suite kills its process group and cleans generated files", async (t) => { + const setup = await fixture(t, { port: await readyServer(t), hang: true }); + await interruptAt(setup, "runtime-start"); +}); + +test("an interrupt during successful teardown cannot produce a green run", async (t) => { + const setup = await fixture(t, { port: await readyServer(t), suiteCode: 0, slowCleanup: true }); + await interruptAt(setup, "cleanup-start"); +}); diff --git a/scripts/run_native_celld_integration.mjs b/scripts/run_native_celld_integration.mjs new file mode 100644 index 000000000..a722633ce --- /dev/null +++ b/scripts/run_native_celld_integration.mjs @@ -0,0 +1,116 @@ +#!/usr/bin/env node +import { spawn } from "node:child_process"; +import { constants, createReadStream } from "node:fs"; +import { access } from "node:fs/promises"; +import { createHash } from "node:crypto"; +import { isAbsolute, resolve } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; + +const root = fileURLToPath(new URL("../", import.meta.url)); +const sentinel = "generated_native_consumer_uses_embedded_celld_over_tls"; +const inherited = [ + "PATH", "HOME", "CARGO_HOME", "RUSTUP_HOME", "RUSTUP_TOOLCHAIN", "RUSTC", "RUSTDOC", + "RUSTFLAGS", "CARGO_BUILD_JOBS", "CARGO_TARGET_DIR", "EDGEZERO_GENERATED_TEST_TARGET_DIR", +]; + +export function fixtureEnvironment(source, binary) { + const env = {}; + for (const name of inherited) { + if (source[name] !== undefined) env[name] = source[name]; + } + env.CELLD_BIN = binary; + return env; +} + +async function binaryDigest(path) { + const hash = createHash("sha256"); + for await (const chunk of createReadStream(path)) hash.update(chunk); + return hash.digest("hex"); +} + +// One process group per owned command. A failed or interrupted proof must not +// leave a generated app or celld descendant running. +export function runCommand(command, args, { cwd, env, timeoutMs = 600_000 }) { + return new Promise((resolveCommand, reject) => { + const child = spawn(command, args, { cwd, env, detached: process.platform !== "win32", stdio: ["ignore", "pipe", "pipe"] }); + let stdout = ""; + let stderr = ""; + let failure; + let force; + const kill = (signal) => { + try { + if (process.platform === "win32") child.kill(signal); + else process.kill(-child.pid, signal); + } catch (error) { + if (error.code !== "ESRCH") failure ??= error; + } + }; + const stop = (reason) => { + failure ??= new Error(reason); + kill("SIGTERM"); + force ??= setTimeout(() => kill("SIGKILL"), 2_000); + force.unref(); + }; + const onInterrupt = () => stop("native celld proof interrupted"); + process.once("SIGINT", onInterrupt); + process.once("SIGTERM", onInterrupt); + const timer = setTimeout(() => stop("native celld proof deadline exceeded"), timeoutMs); + child.stdout.on("data", (chunk) => { + process.stdout.write(chunk); + stdout = (stdout + chunk.toString()).slice(-65_536); + }); + child.stderr.on("data", (chunk) => { + process.stderr.write(chunk); + stderr = (stderr + chunk.toString()).slice(-65_536); + }); + const clean = () => { + clearTimeout(timer); + if (force) clearTimeout(force); + process.removeListener("SIGINT", onInterrupt); + process.removeListener("SIGTERM", onInterrupt); + }; + child.once("error", (error) => { clean(); reject(error); }); + child.once("close", (code, signal) => { + clean(); + // Reap any descendant which outlived a failed parent before returning. + if (failure || code !== 0) kill("SIGKILL"); + if (failure) reject(failure); + else if (code !== 0) reject(new Error(`${command} failed: code=${code} signal=${signal}\n${stderr}`)); + else resolveCommand({ stdout, stderr }); + }); + }); +} + +export async function runIntegration({ + env = process.env, + cwd = root, + command = runCommand, + executable = (path) => access(path, constants.X_OK), + digest = binaryDigest, + log = console.log, +} = {}) { + const selected = env.CELLD_BIN; + if (!selected || !isAbsolute(selected)) { + throw new Error("CELLD_BIN must explicitly select an absolute modified celld executable; no passing skip"); + } + const binary = resolve(selected); + await executable(binary); + const childEnv = fixtureEnvironment(env, binary); + const help = await command(binary, ["--help"], { cwd, env: childEnv, timeoutMs: 10_000 }); + if (!help.stdout.includes("--store-api-config")) throw new Error("selected celld has no embedded scoped API"); + const fingerprint = await digest(binary); + if (!/^[a-f0-9]{64}$/.test(fingerprint)) throw new Error("invalid celld executable fingerprint"); + log(`celld executable SHA-256 ${fingerprint}`); + await command("bash", ["scripts/check_celld_scoped_isolation.sh", binary], { cwd, env: childEnv, timeoutMs: 60_000 }); + await command("scripts/run_test_nonzero.sh", [ + "--ignored", sentinel, + "cargo", "test", "--offline", "--locked", "-p", "edgezero-cli", "--no-default-features", + "--features", "cli,edgezero-adapter-axum", "--test", "celld_generated_proof", sentinel, + ], { cwd, env: childEnv, timeoutMs: 600_000 }); + if (await digest(binary) !== fingerprint) throw new Error("celld executable changed during proof"); + log("Native generated celld TLS and disposable isolation proofs passed. Fleet acceptance remains separate."); +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + runIntegration().catch((error) => { console.error(error.message); process.exitCode = 1; }); +} diff --git a/scripts/run_native_celld_integration.test.mjs b/scripts/run_native_celld_integration.test.mjs new file mode 100644 index 000000000..81aa5d23f --- /dev/null +++ b/scripts/run_native_celld_integration.test.mjs @@ -0,0 +1,112 @@ +import assert from "node:assert/strict"; +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import test from "node:test"; +import { fixtureEnvironment, runCommand, runIntegration } from "./run_native_celld_integration.mjs"; + +const binary = resolve("fixture-celld"); + +function fixture(extra = {}) { + const calls = []; + const options = { + env: { CELLD_BIN: binary, PATH: "/fixture/path", HOME: "/fixture/home", AWS_SECRET_ACCESS_KEY: "must-not-propagate", HTTP_PROXY: "must-not-propagate", SSL_CERT_FILE: "must-not-propagate" }, + executable: async () => {}, + digest: async () => "f".repeat(64), + command: async (command, args, options) => { + calls.push({ command, args, options }); + return { stdout: "--store-api-config PATH\n", stderr: "" }; + }, + log: () => {}, + ...extra, + }; + return { calls, options }; +} + +test("fixture environment excludes deployment credentials, proxies and inherited trust inputs", () => { + assert.deepEqual(fixtureEnvironment({ + PATH: "path", HOME: "home", CARGO_HOME: "cargo", RUSTUP_TOOLCHAIN: "1.95.0", CELLD_BIN: "ignored", + AWS_ACCESS_KEY_ID: "secret", CELLD_APP_TOKEN: "secret", ALL_PROXY: "proxy", SSL_CERT_FILE: "ca", + }, binary), { PATH: "path", HOME: "home", CARGO_HOME: "cargo", RUSTUP_TOOLCHAIN: "1.95.0", CELLD_BIN: binary }); +}); + +test("missing binary is a failed prerequisite, never a passing skip", async () => { + const { calls, options } = fixture({ env: {} }); + await assert.rejects(runIntegration(options), /CELLD_BIN must explicitly/); + assert.equal(calls.length, 0); +}); + +test("unavailable executable stops before any command", async () => { + const { calls, options } = fixture({ executable: async () => { throw new Error("not executable"); } }); + await assert.rejects(runIntegration(options), /not executable/); + assert.equal(calls.length, 0); +}); + +test("vanilla celld cannot masquerade as embedded API acceptance", async () => { + const { options } = fixture({ command: async () => ({ stdout: "unmodified celld help", stderr: "" }) }); + await assert.rejects(runIntegration(options), /no embedded scoped API/); +}); + +test("isolation precedes the exact nonzero generated TLS proof", async () => { + const { calls, options } = fixture(); + await runIntegration(options); + assert.equal(calls.length, 3); + assert.equal(calls[0].command, binary); + assert.deepEqual(calls[1].args, ["scripts/check_celld_scoped_isolation.sh", binary]); + assert.equal(calls[2].command, "scripts/run_test_nonzero.sh"); + assert.equal(calls[2].args[0], "--ignored"); + const sentinel = "generated_native_consumer_uses_embedded_celld_over_tls"; + assert.equal(calls[2].args[1], sentinel); + assert.equal(calls[2].args.at(-1), sentinel, "test filter must exclude the independent deterministic suite"); + assert.ok(calls[2].args.includes("--offline")); + for (const call of calls) { + assert.equal(call.options.env.AWS_SECRET_ACCESS_KEY, undefined); + assert.equal(call.options.env.HTTP_PROXY, undefined); + assert.equal(call.options.env.SSL_CERT_FILE, undefined); + } +}); + +test("an isolation failure prevents generated acceptance", async () => { + let count = 0; + const { options } = fixture({ command: async () => { + count += 1; + if (count === 2) throw new Error("isolation unavailable"); + return { stdout: "--store-api-config", stderr: "" }; + } }); + await assert.rejects(runIntegration(options), /isolation unavailable/); + assert.equal(count, 2); +}); + +test("changing the tested executable invalidates acceptance", async () => { + let count = 0; + const { options } = fixture({ digest: async () => (++count === 1 ? "a" : "b").repeat(64) }); + await assert.rejects(runIntegration(options), /executable changed/); +}); + +test("command failure is observable", async () => { + await assert.rejects(runCommand(process.execPath, ["-e", "process.exit(7)"], { + cwd: process.cwd(), env: {}, timeoutMs: 5_000, + }), /code=7/); +}); + +test("command timeout terminates and reaps its owned descendant", { skip: process.platform === "win32" }, async () => { + const root = await mkdtemp(join(tmpdir(), "celld-runner-test-")); + const pidFile = join(root, "child.pid"); + try { + const source = [ + "const {spawn}=require('node:child_process');", + "const fs=require('node:fs');", + "const child=spawn(process.execPath,['-e','setInterval(()=>{},1000)'],{stdio:'ignore'});", + "fs.writeFileSync(process.argv[1],String(child.pid));", + "process.on('SIGTERM',()=>{child.once('exit',()=>process.exit(0));child.kill('SIGTERM');});", + "setInterval(()=>{},1000);", + ].join(""); + await assert.rejects(runCommand(process.execPath, ["-e", source, pidFile], { + cwd: root, env: {}, timeoutMs: 500, + }), /deadline exceeded/); + const pid = Number(await readFile(pidFile, "utf8")); + assert.throws(() => process.kill(pid, 0), { code: "ESRCH" }); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); diff --git a/tests/fixtures/reusable-app/Cargo.lock b/tests/fixtures/reusable-app/Cargo.lock index d395b42ea..aac13f144 100644 --- a/tests/fixtures/reusable-app/Cargo.lock +++ b/tests/fixtures/reusable-app/Cargo.lock @@ -523,12 +523,16 @@ dependencies = [ "axum", "bytes", "edgezero-core", + "edgezero-store-aws", + "edgezero-store-celld", "futures", "futures-util", + "getrandom 0.4.2", "http", "http-body", "hyper", "hyper-util", + "ipnet", "libc", "log", "redb", @@ -538,6 +542,7 @@ dependencies = [ "simple_logger", "thiserror 2.0.20", "tokio", + "toml", "tower", "tracing", ] @@ -663,6 +668,25 @@ dependencies = [ "validator", ] +[[package]] +name = "edgezero-store-aws" +version = "0.1.0" +dependencies = [ + "percent-encoding", + "serde", + "thiserror 2.0.20", + "url", +] + +[[package]] +name = "edgezero-store-celld" +version = "0.1.0" +dependencies = [ + "serde", + "thiserror 2.0.20", + "url", +] + [[package]] name = "either" version = "1.16.0"