From aeec8e5591e1dff65e5d7adc73b8d8ea37f50759 Mon Sep 17 00:00:00 2001 From: Ruslan Talpa Date: Tue, 20 Oct 2026 11:04:47 +0000 Subject: [PATCH 01/12] Add postgres_data_apis bundle resource Add the `postgres_data_apis` resource, which enables and configures the Data API (PostgREST) of a Lakebase Postgres database. The Data API is a per-database singleton with no ID of its own: its resource name is "{parent}/data-api", and `parent` is a provided id field, so changing it recreates. Create, update, and delete are long-running operations. Updates send the full spec with update_mask=spec, so a field removed from the config resets to its server default. spec is INPUT_ONLY in the API, so GET does not return it and remote spec changes are not detected (ignore_remote_changes spec:input_only, generated). Deleting the resource turns the Data API off; the database and its data are unaffected. The resource is labelled Private Preview, matching the API's launch stage, rather than the Beta its postgres_* siblings are pinned to. Co-authored-by: Isaac --- .nextchanges/bundles/postgres-data-apis.md | 1 + .../bind/postgres_data_api/databricks.yml | 10 + .../bind/postgres_data_api/out.test.toml | 2 + .../bind/postgres_data_api/output.txt | 91 +++++++++ .../deployment/bind/postgres_data_api/script | 31 +++ .../bind/postgres_data_api/test.toml | 5 + .../bundle/invariant/auto-migrate/test.toml | 2 + .../configs/postgres_data_api.yml.tmpl | 32 +++ .../bundle/invariant/continue_293/test.toml | 3 + .../invariant/delete_idempotent/out.test.toml | 1 + .../destroy_idempotent/out.test.toml | 1 + acceptance/bundle/invariant/migrate/test.toml | 2 + .../bundle/invariant/no_drift/out.test.toml | 1 + acceptance/bundle/invariant/test.toml | 2 + acceptance/bundle/refschema/out.fields.txt | 37 ++++ .../basic/databricks.yml.tmpl | 34 ++++ .../basic/out.requests.deploy.json | 50 +++++ .../basic/out.requests.remove.json | 4 + .../postgres_data_apis/basic/out.test.toml | 4 + .../postgres_data_apis/basic/output.txt | 121 +++++++++++ .../resources/postgres_data_apis/basic/script | 33 +++ .../out-of-band/databricks.yml.tmpl | 34 ++++ .../out-of-band/out.test.toml | 4 + .../postgres_data_apis/out-of-band/output.txt | 85 ++++++++ .../postgres_data_apis/out-of-band/script | 26 +++ .../postgres_data_apis/out-of-band/test.toml | 1 + .../recreate/databricks.yml.tmpl | 37 ++++ .../recreate/out.requests.recreate.json | 16 ++ .../postgres_data_apis/recreate/out.test.toml | 4 + .../postgres_data_apis/recreate/output.txt | 73 +++++++ .../postgres_data_apis/recreate/script | 21 ++ .../postgres_data_apis/script.prepare | 13 ++ .../resources/postgres_data_apis/test.toml | 16 ++ .../update/databricks.yml.tmpl | 34 ++++ .../update/out.plan.update.json | 30 +++ .../update/out.requests.remove_field.json | 15 ++ .../update/out.requests.update.json | 18 ++ .../postgres_data_apis/update/out.test.toml | 4 + .../postgres_data_apis/update/output.txt | 73 +++++++ .../postgres_data_apis/update/script | 31 +++ .../apply_bundle_permissions_test.go | 1 + .../resourcemutator/apply_target_mode_test.go | 7 + .../mutator/resourcemutator/run_as_test.go | 2 + bundle/config/resources.go | 3 + bundle/config/resources/postgres_data_api.go | 80 ++++++++ bundle/config/resources_test.go | 9 + bundle/direct/dresources/all.go | 1 + bundle/direct/dresources/all_test.go | 40 ++++ .../direct/dresources/apitypes.generated.yml | 2 + bundle/direct/dresources/apitypes.yml | 2 + .../configs/postgres_data_apis.generated.yml | 21 ++ .../dresources/configs/postgres_data_apis.yml | 10 + bundle/direct/dresources/postgres_data_api.go | 162 +++++++++++++++ bundle/internal/annotation/preview.go | 6 +- bundle/internal/schema/annotations.yml | 10 + bundle/internal/schema/parser_test.go | 30 +-- .../validation/generated/enum_fields.go | 2 + .../validation/generated/required_fields.go | 2 + bundle/schema/jsonschema.json | 120 +++++++++++ bundle/statemgmt/state_load_test.go | 21 ++ libs/testserver/bundledeployments.go | 1 + libs/testserver/fake_workspace.go | 2 + libs/testserver/handlers.go | 26 +++ libs/testserver/postgres.go | 192 ++++++++++++++++++ libs/testserver/postgres_data_api_test.go | 104 ++++++++++ 65 files changed, 1842 insertions(+), 16 deletions(-) create mode 100644 .nextchanges/bundles/postgres-data-apis.md create mode 100644 acceptance/bundle/deployment/bind/postgres_data_api/databricks.yml create mode 100644 acceptance/bundle/deployment/bind/postgres_data_api/out.test.toml create mode 100644 acceptance/bundle/deployment/bind/postgres_data_api/output.txt create mode 100644 acceptance/bundle/deployment/bind/postgres_data_api/script create mode 100644 acceptance/bundle/deployment/bind/postgres_data_api/test.toml create mode 100644 acceptance/bundle/invariant/configs/postgres_data_api.yml.tmpl create mode 100644 acceptance/bundle/resources/postgres_data_apis/basic/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/postgres_data_apis/basic/out.requests.deploy.json create mode 100644 acceptance/bundle/resources/postgres_data_apis/basic/out.requests.remove.json create mode 100644 acceptance/bundle/resources/postgres_data_apis/basic/out.test.toml create mode 100644 acceptance/bundle/resources/postgres_data_apis/basic/output.txt create mode 100644 acceptance/bundle/resources/postgres_data_apis/basic/script create mode 100644 acceptance/bundle/resources/postgres_data_apis/out-of-band/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/postgres_data_apis/out-of-band/out.test.toml create mode 100644 acceptance/bundle/resources/postgres_data_apis/out-of-band/output.txt create mode 100644 acceptance/bundle/resources/postgres_data_apis/out-of-band/script create mode 100644 acceptance/bundle/resources/postgres_data_apis/out-of-band/test.toml create mode 100644 acceptance/bundle/resources/postgres_data_apis/recreate/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/postgres_data_apis/recreate/out.requests.recreate.json create mode 100644 acceptance/bundle/resources/postgres_data_apis/recreate/out.test.toml create mode 100644 acceptance/bundle/resources/postgres_data_apis/recreate/output.txt create mode 100644 acceptance/bundle/resources/postgres_data_apis/recreate/script create mode 100644 acceptance/bundle/resources/postgres_data_apis/script.prepare create mode 100644 acceptance/bundle/resources/postgres_data_apis/test.toml create mode 100644 acceptance/bundle/resources/postgres_data_apis/update/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/postgres_data_apis/update/out.plan.update.json create mode 100644 acceptance/bundle/resources/postgres_data_apis/update/out.requests.remove_field.json create mode 100644 acceptance/bundle/resources/postgres_data_apis/update/out.requests.update.json create mode 100644 acceptance/bundle/resources/postgres_data_apis/update/out.test.toml create mode 100644 acceptance/bundle/resources/postgres_data_apis/update/output.txt create mode 100644 acceptance/bundle/resources/postgres_data_apis/update/script create mode 100644 bundle/config/resources/postgres_data_api.go create mode 100644 bundle/direct/dresources/configs/postgres_data_apis.generated.yml create mode 100644 bundle/direct/dresources/configs/postgres_data_apis.yml create mode 100644 bundle/direct/dresources/postgres_data_api.go create mode 100644 libs/testserver/postgres_data_api_test.go diff --git a/.nextchanges/bundles/postgres-data-apis.md b/.nextchanges/bundles/postgres-data-apis.md new file mode 100644 index 00000000000..b37e8e8dcda --- /dev/null +++ b/.nextchanges/bundles/postgres-data-apis.md @@ -0,0 +1 @@ +* Add the `postgres_data_apis` resource (Private Preview) for enabling and configuring the Data API of a Lakebase Postgres database. diff --git a/acceptance/bundle/deployment/bind/postgres_data_api/databricks.yml b/acceptance/bundle/deployment/bind/postgres_data_api/databricks.yml new file mode 100644 index 00000000000..978fa248926 --- /dev/null +++ b/acceptance/bundle/deployment/bind/postgres_data_api/databricks.yml @@ -0,0 +1,10 @@ +bundle: + name: test-bundle + +resources: + postgres_data_apis: + data_api1: + parent: projects/test-project/branches/production/databases/test-database + db_schemas: + - public + db_max_rows: 500 diff --git a/acceptance/bundle/deployment/bind/postgres_data_api/out.test.toml b/acceptance/bundle/deployment/bind/postgres_data_api/out.test.toml new file mode 100644 index 00000000000..19b5b15d11f --- /dev/null +++ b/acceptance/bundle/deployment/bind/postgres_data_api/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/deployment/bind/postgres_data_api/output.txt b/acceptance/bundle/deployment/bind/postgres_data_api/output.txt new file mode 100644 index 00000000000..7a8260aedf3 --- /dev/null +++ b/acceptance/bundle/deployment/bind/postgres_data_api/output.txt @@ -0,0 +1,91 @@ + +=== Enable the Data API on a database outside the bundle +=== Deploy fails because the Data API is already enabled +>>> musterr [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Error: cannot create resources.postgres_data_apis.data_api1: Data API is already enabled for database projects/test-project/branches/production/databases/test-database (409 ALREADY_EXISTS) + +Endpoint: POST [DATABRICKS_URL]/api/2.0/postgres/projects/test-project/branches/production/databases/test-database/data-api +HTTP Status: 409 Conflict +API error_code: ALREADY_EXISTS +API message: Data API is already enabled for database projects/test-project/branches/production/databases/test-database + +Files: 2 uploaded, 0 deleted + +=== Bind adopts the existing Data API +>>> [CLI] bundle deployment bind data_api1 projects/test-project/branches/production/databases/test-database/data-api --auto-approve +Successfully bound postgres_data_api with an id 'projects/test-project/branches/production/databases/test-database/data-api' +Run 'bundle deploy' to deploy changes to your workspace + +>>> [CLI] bundle summary +Name: test-bundle +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default +Resources: + Postgres Data APIs: + data_api1: + Name: projects/test-project/branches/production/databases/test-database/data-api + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Files: 0 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 1 unchanged + +>>> [CLI] postgres get-data-api projects/test-project/branches/production/databases/test-database/data-api +{ + "name": "projects/test-project/branches/production/databases/test-database/data-api", + "status": { + "db_schemas": [ + "public" + ], + "db_max_rows": null + } +} + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +=== Unbind leaves the Data API enabled +>>> [CLI] bundle deployment unbind data_api1 + +>>> [CLI] bundle summary +Name: test-bundle +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default +Resources: + Postgres Data APIs: + data_api1: + Name: + +>>> [CLI] postgres get-data-api projects/test-project/branches/production/databases/test-database/data-api +{ + "name": "projects/test-project/branches/production/databases/test-database/data-api" +} + +=== Bind again; destroy turns off the Data API +>>> [CLI] bundle deployment bind data_api1 projects/test-project/branches/production/databases/test-database/data-api --auto-approve +Successfully bound postgres_data_api with an id 'projects/test-project/branches/production/databases/test-database/data-api' +Run 'bundle deploy' to deploy changes to your workspace + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.postgres_data_apis.data_api1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Destroy: 1 deleted + +>>> musterr [CLI] postgres get-data-api projects/test-project/branches/production/databases/test-database/data-api +Error: data api id not found [TraceId: [UUID]] + +>>> [CLI] postgres get-database projects/test-project/branches/production/databases/test-database +{ + "name": "projects/test-project/branches/production/databases/test-database" +} diff --git a/acceptance/bundle/deployment/bind/postgres_data_api/script b/acceptance/bundle/deployment/bind/postgres_data_api/script new file mode 100644 index 00000000000..af860e3508d --- /dev/null +++ b/acceptance/bundle/deployment/bind/postgres_data_api/script @@ -0,0 +1,31 @@ +DATABASE_NAME="projects/test-project/branches/production/databases/test-database" +DATA_API_NAME="${DATABASE_NAME}/data-api" + +title "Enable the Data API on a database outside the bundle" +$CLI postgres create-project test-project --json '{"spec": {"display_name": "Test Project", "pg_version": 16}}' &> LOG.setup +$CLI postgres create-database projects/test-project/branches/production --database-id test-database --json '{"spec": {"postgres_database": "app_db", "role": "projects/test-project/branches/production/roles/owner"}}' &>> LOG.setup +$CLI postgres create-data-api "${DATABASE_NAME}" --json '{"spec": {"db_schemas": ["public"]}}' &>> LOG.setup + +title "Deploy fails because the Data API is already enabled" +trace musterr $CLI bundle deploy + +title "Bind adopts the existing Data API" +# Bind records the config as state, and spec is input-only (GET does not return it), so +# the configured spec is not applied until it changes in the config. +trace $CLI bundle deployment bind data_api1 "${DATA_API_NAME}" --auto-approve +trace $CLI bundle summary +trace $CLI bundle plan +trace $CLI bundle deploy +trace $CLI postgres get-data-api "${DATA_API_NAME}" | jq '{name, status: (.status | {db_schemas, db_max_rows})}' +trace $CLI bundle plan + +title "Unbind leaves the Data API enabled" +trace $CLI bundle deployment unbind data_api1 +trace $CLI bundle summary +trace $CLI postgres get-data-api "${DATA_API_NAME}" | jq '{name}' + +title "Bind again; destroy turns off the Data API" +trace $CLI bundle deployment bind data_api1 "${DATA_API_NAME}" --auto-approve +trace $CLI bundle destroy --auto-approve +trace musterr $CLI postgres get-data-api "${DATA_API_NAME}" +trace $CLI postgres get-database "${DATABASE_NAME}" | jq '{name}' diff --git a/acceptance/bundle/deployment/bind/postgres_data_api/test.toml b/acceptance/bundle/deployment/bind/postgres_data_api/test.toml new file mode 100644 index 00000000000..976a421dd9d --- /dev/null +++ b/acceptance/bundle/deployment/bind/postgres_data_api/test.toml @@ -0,0 +1,5 @@ +Cloud = false + +Ignore = [ + ".databricks" +] diff --git a/acceptance/bundle/invariant/auto-migrate/test.toml b/acceptance/bundle/invariant/auto-migrate/test.toml index e0be4eb605f..5e52234be9b 100644 --- a/acceptance/bundle/invariant/auto-migrate/test.toml +++ b/acceptance/bundle/invariant/auto-migrate/test.toml @@ -41,6 +41,8 @@ EnvMatrixExclude.no_cluster_libraries = ["INPUT_CONFIG=cluster_libraries.yml.tmp EnvMatrixExclude.no_model_service = ["INPUT_CONFIG=model_service.yml.tmpl"] # Features are direct-only; the terraform deploy that seeds the migration fails for them. EnvMatrixExclude.no_feature = ["INPUT_CONFIG=feature.yml.tmpl"] +# The Postgres Data API is direct-only; the terraform deploy that seeds the migration fails for it. +EnvMatrixExclude.no_postgres_data_api = ["INPUT_CONFIG=postgres_data_api.yml.tmpl"] # Cross-resource permission references (e.g. ${resources.jobs.job_b.permissions[0].level}) # don't work in terraform mode: the terraform interpolator converts the path to diff --git a/acceptance/bundle/invariant/configs/postgres_data_api.yml.tmpl b/acceptance/bundle/invariant/configs/postgres_data_api.yml.tmpl new file mode 100644 index 00000000000..5de5c7c9d57 --- /dev/null +++ b/acceptance/bundle/invariant/configs/postgres_data_api.yml.tmpl @@ -0,0 +1,32 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + postgres_projects: + project: + project_id: test-pg-project-$UNIQUE_NAME + display_name: Test Postgres Project + + # The Data API needs a read-write endpoint on the branch; the "production" + # branch created with the project comes with one. + postgres_roles: + owner: + parent: ${resources.postgres_projects.project.name}/branches/production + role_id: test-role-$UNIQUE_NAME + postgres_role: app_role + + postgres_databases: + database: + parent: ${resources.postgres_projects.project.name}/branches/production + database_id: test-database-$UNIQUE_NAME + postgres_database: app_db + role: ${resources.postgres_roles.owner.name} + + postgres_data_apis: + foo: + parent: ${resources.postgres_databases.database.name} + db_schemas: + - public + db_max_rows: 1000 # CAN_REMOVE + server_cors_allowed_origins: # CAN_REMOVE + - https://app.example.com # CAN_REMOVE diff --git a/acceptance/bundle/invariant/continue_293/test.toml b/acceptance/bundle/invariant/continue_293/test.toml index 15890459519..150d61a2c4b 100644 --- a/acceptance/bundle/invariant/continue_293/test.toml +++ b/acceptance/bundle/invariant/continue_293/test.toml @@ -43,6 +43,9 @@ EnvMatrixExclude.no_pydabs_1000_tasks = ["INPUT_CONFIG=job_pydabs_1000_tasks.yml # secret resource is not supported on v0.293.0 EnvMatrixExclude.no_secret = ["INPUT_CONFIG=secret.yml.tmpl"] +# postgres_data_apis resource is not supported on v0.293.0 +EnvMatrixExclude.no_postgres_data_api = ["INPUT_CONFIG=postgres_data_api.yml.tmpl"] + # $${...} escaping is not supported on v0.293.0: the seed deploy fails with # 'invalid dependency "${batch_id}", no such node ""' (issue #6480). EnvMatrixExclude.no_escaped_refs = ["INPUT_CONFIG=job_escaped_refs.yml.tmpl"] diff --git a/acceptance/bundle/invariant/delete_idempotent/out.test.toml b/acceptance/bundle/invariant/delete_idempotent/out.test.toml index d9bed152760..18735ec8919 100644 --- a/acceptance/bundle/invariant/delete_idempotent/out.test.toml +++ b/acceptance/bundle/invariant/delete_idempotent/out.test.toml @@ -41,6 +41,7 @@ EnvMatrix.INPUT_CONFIG = [ "pipeline_config_dots.yml.tmpl", "postgres_branch.yml.tmpl", "postgres_catalog.yml.tmpl", + "postgres_data_api.yml.tmpl", "postgres_database.yml.tmpl", "postgres_endpoint.yml.tmpl", "postgres_project.yml.tmpl", diff --git a/acceptance/bundle/invariant/destroy_idempotent/out.test.toml b/acceptance/bundle/invariant/destroy_idempotent/out.test.toml index d9bed152760..18735ec8919 100644 --- a/acceptance/bundle/invariant/destroy_idempotent/out.test.toml +++ b/acceptance/bundle/invariant/destroy_idempotent/out.test.toml @@ -41,6 +41,7 @@ EnvMatrix.INPUT_CONFIG = [ "pipeline_config_dots.yml.tmpl", "postgres_branch.yml.tmpl", "postgres_catalog.yml.tmpl", + "postgres_data_api.yml.tmpl", "postgres_database.yml.tmpl", "postgres_endpoint.yml.tmpl", "postgres_project.yml.tmpl", diff --git a/acceptance/bundle/invariant/migrate/test.toml b/acceptance/bundle/invariant/migrate/test.toml index a4c41dfba3d..0894d465c0b 100644 --- a/acceptance/bundle/invariant/migrate/test.toml +++ b/acceptance/bundle/invariant/migrate/test.toml @@ -41,6 +41,8 @@ EnvMatrixExclude.no_cluster_libraries = ["INPUT_CONFIG=cluster_libraries.yml.tmp EnvMatrixExclude.no_model_service = ["INPUT_CONFIG=model_service.yml.tmpl"] # Features are direct-only; the terraform deploy that seeds the migration fails for them. EnvMatrixExclude.no_feature = ["INPUT_CONFIG=feature.yml.tmpl"] +# The Postgres Data API is direct-only; the terraform deploy that seeds the migration fails for it. +EnvMatrixExclude.no_postgres_data_api = ["INPUT_CONFIG=postgres_data_api.yml.tmpl"] # Cross-resource permission references (e.g. ${resources.jobs.job_b.permissions[0].level}) # don't work in terraform mode: the terraform interpolator converts the path to diff --git a/acceptance/bundle/invariant/no_drift/out.test.toml b/acceptance/bundle/invariant/no_drift/out.test.toml index 39825248867..d44a9a95ab7 100644 --- a/acceptance/bundle/invariant/no_drift/out.test.toml +++ b/acceptance/bundle/invariant/no_drift/out.test.toml @@ -41,6 +41,7 @@ EnvMatrix.INPUT_CONFIG = [ "pipeline_config_dots.yml.tmpl", "postgres_branch.yml.tmpl", "postgres_catalog.yml.tmpl", + "postgres_data_api.yml.tmpl", "postgres_database.yml.tmpl", "postgres_endpoint.yml.tmpl", "postgres_project.yml.tmpl", diff --git a/acceptance/bundle/invariant/test.toml b/acceptance/bundle/invariant/test.toml index 2427b6a553f..9fe72ef15f5 100644 --- a/acceptance/bundle/invariant/test.toml +++ b/acceptance/bundle/invariant/test.toml @@ -55,6 +55,7 @@ EnvMatrix.INPUT_CONFIG = [ "pipeline_config_dots.yml.tmpl", "postgres_branch.yml.tmpl", "postgres_catalog.yml.tmpl", + "postgres_data_api.yml.tmpl", "postgres_database.yml.tmpl", "postgres_endpoint.yml.tmpl", "postgres_project.yml.tmpl", @@ -94,6 +95,7 @@ EnvMatrixExclude.no_postgres_catalog_on_cloud = ["CONFIG_Cloud=true", "INPUT_CON EnvMatrixExclude.no_postgres_synced_table_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=postgres_synced_table.yml.tmpl"] EnvMatrixExclude.no_postgres_role_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=postgres_role.yml.tmpl"] EnvMatrixExclude.no_postgres_database_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=postgres_database.yml.tmpl"] +EnvMatrixExclude.no_postgres_data_api_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=postgres_data_api.yml.tmpl"] # Lakebase v1 (database instances/catalogs/synced tables) is not available on GCP, # so exclude these from all cloud runs (same as the postgres resources above). EnvMatrixExclude.no_database_instance_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=database_instance.yml.tmpl"] diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index 978c468c0ac..2af472ca296 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -3995,6 +3995,43 @@ resources.postgres_catalogs.*.status.project string REMOTE resources.postgres_catalogs.*.uid string REMOTE resources.postgres_catalogs.*.update_time *time.Time REMOTE resources.postgres_catalogs.*.url string INPUT +resources.postgres_data_apis.*.create_time *time.Time REMOTE +resources.postgres_data_apis.*.db_aggregates_enabled bool ALL +resources.postgres_data_apis.*.db_extra_search_path []string ALL +resources.postgres_data_apis.*.db_extra_search_path[*] string ALL +resources.postgres_data_apis.*.db_max_rows int ALL +resources.postgres_data_apis.*.db_schemas []string ALL +resources.postgres_data_apis.*.db_schemas[*] string ALL +resources.postgres_data_apis.*.id string INPUT +resources.postgres_data_apis.*.jwt_cache_max_lifetime *duration.Duration ALL +resources.postgres_data_apis.*.jwt_role_claim_key string ALL +resources.postgres_data_apis.*.lifecycle resources.Lifecycle INPUT +resources.postgres_data_apis.*.lifecycle.prevent_destroy bool INPUT +resources.postgres_data_apis.*.modified_status string INPUT +resources.postgres_data_apis.*.name string REMOTE +resources.postgres_data_apis.*.openapi_mode postgres.OpenApiMode ALL +resources.postgres_data_apis.*.parent string ALL +resources.postgres_data_apis.*.server_cors_allowed_origins []string ALL +resources.postgres_data_apis.*.server_cors_allowed_origins[*] string ALL +resources.postgres_data_apis.*.server_timing_enabled bool ALL +resources.postgres_data_apis.*.status *postgres.DataApiDataApiStatus REMOTE +resources.postgres_data_apis.*.status.available_schemas []string REMOTE +resources.postgres_data_apis.*.status.available_schemas[*] string REMOTE +resources.postgres_data_apis.*.status.db_aggregates_enabled bool REMOTE +resources.postgres_data_apis.*.status.db_extra_search_path []string REMOTE +resources.postgres_data_apis.*.status.db_extra_search_path[*] string REMOTE +resources.postgres_data_apis.*.status.db_max_rows int REMOTE +resources.postgres_data_apis.*.status.db_schemas []string REMOTE +resources.postgres_data_apis.*.status.db_schemas[*] string REMOTE +resources.postgres_data_apis.*.status.jwt_cache_max_lifetime *duration.Duration REMOTE +resources.postgres_data_apis.*.status.jwt_role_claim_key string REMOTE +resources.postgres_data_apis.*.status.openapi_mode postgres.OpenApiMode REMOTE +resources.postgres_data_apis.*.status.server_cors_allowed_origins []string REMOTE +resources.postgres_data_apis.*.status.server_cors_allowed_origins[*] string REMOTE +resources.postgres_data_apis.*.status.server_timing_enabled bool REMOTE +resources.postgres_data_apis.*.status.url string REMOTE +resources.postgres_data_apis.*.update_time *time.Time REMOTE +resources.postgres_data_apis.*.url string INPUT resources.postgres_databases.*.create_time *time.Time REMOTE resources.postgres_databases.*.database_id string ALL resources.postgres_databases.*.id string INPUT diff --git a/acceptance/bundle/resources/postgres_data_apis/basic/databricks.yml.tmpl b/acceptance/bundle/resources/postgres_data_apis/basic/databricks.yml.tmpl new file mode 100644 index 00000000000..6a340faa555 --- /dev/null +++ b/acceptance/bundle/resources/postgres_data_apis/basic/databricks.yml.tmpl @@ -0,0 +1,34 @@ +bundle: + name: deploy-pg-data-api-$UNIQUE_NAME + +sync: + paths: [] + +resources: + postgres_projects: + my_project: + project_id: test-pg-proj-$UNIQUE_NAME + display_name: "Test Project for Data API" + pg_version: 16 + + # The Data API needs a read-write endpoint on the branch; the "production" + # branch created with the project comes with one. + postgres_roles: + owner: + parent: ${resources.postgres_projects.my_project.id}/branches/production + role_id: app-owner + postgres_role: app_owner + + postgres_databases: + my_database: + parent: ${resources.postgres_projects.my_project.id}/branches/production + database_id: my-database + postgres_database: app_db + role: ${resources.postgres_roles.owner.id} + + postgres_data_apis: + my_data_api: + parent: ${resources.postgres_databases.my_database.name} + db_schemas: + - public + db_max_rows: 100 diff --git a/acceptance/bundle/resources/postgres_data_apis/basic/out.requests.deploy.json b/acceptance/bundle/resources/postgres_data_apis/basic/out.requests.deploy.json new file mode 100644 index 00000000000..3de94b89854 --- /dev/null +++ b/acceptance/bundle/resources/postgres_data_apis/basic/out.requests.deploy.json @@ -0,0 +1,50 @@ +{ + "method": "POST", + "path": "/api/2.0/postgres/projects", + "q": { + "project_id": "test-pg-proj-[UNIQUE_NAME]" + }, + "body": { + "spec": { + "display_name": "Test Project for Data API", + "pg_version": 16 + } + } +} +{ + "method": "POST", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/production/roles", + "q": { + "role_id": "app-owner" + }, + "body": { + "spec": { + "postgres_role": "app_owner" + } + } +} +{ + "method": "POST", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases", + "q": { + "database_id": "my-database" + }, + "body": { + "spec": { + "postgres_database": "app_db", + "role": "projects/test-pg-proj-[UNIQUE_NAME]/branches/production/roles/app-owner" + } + } +} +{ + "method": "POST", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/my-database/data-api", + "body": { + "spec": { + "db_max_rows": 100, + "db_schemas": [ + "public" + ] + } + } +} diff --git a/acceptance/bundle/resources/postgres_data_apis/basic/out.requests.remove.json b/acceptance/bundle/resources/postgres_data_apis/basic/out.requests.remove.json new file mode 100644 index 00000000000..9d3a7aaf9da --- /dev/null +++ b/acceptance/bundle/resources/postgres_data_apis/basic/out.requests.remove.json @@ -0,0 +1,4 @@ +{ + "method": "DELETE", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/my-database/data-api" +} diff --git a/acceptance/bundle/resources/postgres_data_apis/basic/out.test.toml b/acceptance/bundle/resources/postgres_data_apis/basic/out.test.toml new file mode 100644 index 00000000000..ee153e82621 --- /dev/null +++ b/acceptance/bundle/resources/postgres_data_apis/basic/out.test.toml @@ -0,0 +1,4 @@ +Cloud = true +CloudEnvs.azure = false +CloudEnvs.gcp = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/postgres_data_apis/basic/output.txt b/acceptance/bundle/resources/postgres_data_apis/basic/output.txt new file mode 100644 index 00000000000..4cb0296ee70 --- /dev/null +++ b/acceptance/bundle/resources/postgres_data_apis/basic/output.txt @@ -0,0 +1,121 @@ + +>>> [CLI] bundle validate +Name: deploy-pg-data-api-[UNIQUE_NAME] +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/deploy-pg-data-api-[UNIQUE_NAME]/default + +Validation OK! + +>>> [CLI] bundle summary +Name: deploy-pg-data-api-[UNIQUE_NAME] +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/deploy-pg-data-api-[UNIQUE_NAME]/default +Resources: + Postgres Data APIs: + my_data_api: + Name: + Postgres databases: + my_database: + Name: + Postgres projects: + my_project: + Name: Test Project for Data API + Postgres roles: + owner: + Name: + +>>> [CLI] bundle plan +create postgres_data_apis.my_data_api +create postgres_databases.my_database +create postgres_projects.my_project +create postgres_roles.owner + +Plan: 4 to add, 0 to change, 0 to delete, 0 unchanged + +=== Deploy enables the Data API +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-pg-data-api-[UNIQUE_NAME]/default/files... +Created postgres_data_apis.my_data_api +Created postgres_databases.my_database +Created postgres_projects.my_project +Created postgres_roles.owner +Files: 0 uploaded, 0 deleted +Resources: 4 created, 0 changed, 0 deleted, 0 unchanged + +>>> [CLI] postgres get-data-api projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/my-database/data-api +{ + "name": "projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/my-database/data-api", + "parent": "projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/my-database", + "status": { + "db_schemas": [ + "public" + ], + "db_max_rows": 100 + } +} + +>>> [CLI] bundle summary +Name: deploy-pg-data-api-[UNIQUE_NAME] +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/deploy-pg-data-api-[UNIQUE_NAME]/default +Resources: + Postgres Data APIs: + my_data_api: + Name: projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/my-database/data-api + Postgres databases: + my_database: + Name: projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/my-database + Postgres projects: + my_project: + Name: Test Project for Data API + Postgres roles: + owner: + Name: projects/test-pg-proj-[UNIQUE_NAME]/branches/production/roles/app-owner + +=== Plan after deploy shows no changes +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 4 unchanged + +=== Removing the resource turns off the Data API but keeps the database +>>> [CLI] bundle plan +delete postgres_data_apis.my_data_api + +Plan: 0 to add, 0 to change, 1 to delete, 3 unchanged + +>>> [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-pg-data-api-[UNIQUE_NAME]/default/files... +Deleted postgres_data_apis.my_data_api +Files: 0 uploaded, 0 deleted +Resources: 0 created, 0 changed, 1 deleted, 3 unchanged + +>>> musterr [CLI] postgres get-data-api projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/my-database/data-api +Error: data api id not found [TraceId: [UUID]] + +>>> [CLI] postgres get-database projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/my-database +{ + "name": "projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/my-database" +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.postgres_databases.my_database + delete resources.postgres_projects.my_project + delete resources.postgres_roles.owner + +This action will result in the deletion of the following Lakebase projects along with +all their branches, databases, and endpoints. All data stored in them will be permanently lost: + delete resources.postgres_projects.my_project + +This action will result in the deletion of the following Lakebase databases. +All data stored in them will be permanently lost: + delete resources.postgres_databases.my_database + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/deploy-pg-data-api-[UNIQUE_NAME]/default + +Destroy: 3 deleted diff --git a/acceptance/bundle/resources/postgres_data_apis/basic/script b/acceptance/bundle/resources/postgres_data_apis/basic/script new file mode 100644 index 00000000000..39212785c46 --- /dev/null +++ b/acceptance/bundle/resources/postgres_data_apis/basic/script @@ -0,0 +1,33 @@ +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f "$OUT_REQUESTS" +} +trap cleanup EXIT + +database_name="projects/test-pg-proj-${UNIQUE_NAME}/branches/production/databases/my-database" +data_api_name="${database_name}/data-api" + +envsubst < databricks.yml.tmpl > databricks.yml + +trace $CLI bundle validate +trace $CLI bundle summary +trace $CLI bundle plan + +title "Deploy enables the Data API" +rm -f "$OUT_REQUESTS" +trace $CLI bundle deploy +print_postgres_requests deploy +trace $CLI postgres get-data-api "${data_api_name}" | data_api_fields +trace $CLI bundle summary + +title "Plan after deploy shows no changes" +trace $CLI bundle plan + +title "Removing the resource turns off the Data API but keeps the database" +sed '/postgres_data_apis:/,$d' databricks.yml > databricks.yml.new && mv databricks.yml.new databricks.yml +trace $CLI bundle plan +rm -f "$OUT_REQUESTS" +trace $CLI bundle deploy --auto-approve +print_postgres_requests remove +trace musterr $CLI postgres get-data-api "${data_api_name}" +trace $CLI postgres get-database "${database_name}" | jq '{name}' diff --git a/acceptance/bundle/resources/postgres_data_apis/out-of-band/databricks.yml.tmpl b/acceptance/bundle/resources/postgres_data_apis/out-of-band/databricks.yml.tmpl new file mode 100644 index 00000000000..bd1c6aa3878 --- /dev/null +++ b/acceptance/bundle/resources/postgres_data_apis/out-of-band/databricks.yml.tmpl @@ -0,0 +1,34 @@ +bundle: + name: oob-pg-data-api-$UNIQUE_NAME + +sync: + paths: [] + +resources: + postgres_projects: + my_project: + project_id: test-pg-proj-$UNIQUE_NAME + display_name: "Test Project for Data API Out of Band" + pg_version: 16 + + # The Data API needs a read-write endpoint on the branch; the "production" + # branch created with the project comes with one. + postgres_roles: + owner: + parent: ${resources.postgres_projects.my_project.id}/branches/production + role_id: app-owner + postgres_role: app_owner + + postgres_databases: + my_database: + parent: ${resources.postgres_projects.my_project.id}/branches/production + database_id: my-database + postgres_database: app_db + role: ${resources.postgres_roles.owner.id} + + postgres_data_apis: + my_data_api: + parent: ${resources.postgres_databases.my_database.name} + db_schemas: + - public + db_max_rows: 100 diff --git a/acceptance/bundle/resources/postgres_data_apis/out-of-band/out.test.toml b/acceptance/bundle/resources/postgres_data_apis/out-of-band/out.test.toml new file mode 100644 index 00000000000..ee153e82621 --- /dev/null +++ b/acceptance/bundle/resources/postgres_data_apis/out-of-band/out.test.toml @@ -0,0 +1,4 @@ +Cloud = true +CloudEnvs.azure = false +CloudEnvs.gcp = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/postgres_data_apis/out-of-band/output.txt b/acceptance/bundle/resources/postgres_data_apis/out-of-band/output.txt new file mode 100644 index 00000000000..7a6716be96f --- /dev/null +++ b/acceptance/bundle/resources/postgres_data_apis/out-of-band/output.txt @@ -0,0 +1,85 @@ + +=== Deploy enables the Data API +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/oob-pg-data-api-[UNIQUE_NAME]/default/files... +Created postgres_data_apis.my_data_api +Created postgres_databases.my_database +Created postgres_projects.my_project +Created postgres_roles.owner +Files: 0 uploaded, 0 deleted +Resources: 4 created, 0 changed, 0 deleted, 0 unchanged + +>>> [CLI] postgres get-data-api projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/my-database/data-api +{ + "name": "projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/my-database/data-api", + "parent": "projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/my-database", + "status": { + "db_schemas": [ + "public" + ], + "db_max_rows": 100 + } +} + +=== Change the spec out of band +>>> [CLI] postgres update-data-api projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/my-database/data-api spec --json {"spec": {"db_schemas": ["public"], "db_max_rows": 7}} +{ + "name": "projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/my-database/data-api", + "parent": "projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/my-database", + "status": { + "db_schemas": [ + "public" + ], + "db_max_rows": 7 + } +} + +=== Plan does not detect the change: spec is input-only and GET does not return it +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 4 unchanged + +=== Disable the Data API out of band +>>> [CLI] postgres delete-data-api projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/my-database/data-api + +=== Plan detects the Data API is gone and re-creates it +>>> [CLI] bundle plan +create postgres_data_apis.my_data_api + +Plan: 1 to add, 0 to change, 0 to delete, 3 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/oob-pg-data-api-[UNIQUE_NAME]/default/files... +Created postgres_data_apis.my_data_api +Files: 0 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 3 unchanged + +>>> [CLI] postgres get-data-api projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/my-database/data-api +{ + "name": "projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/my-database/data-api", + "parent": "projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/my-database", + "status": { + "db_schemas": [ + "public" + ], + "db_max_rows": 100 + } +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.postgres_data_apis.my_data_api + delete resources.postgres_databases.my_database + delete resources.postgres_projects.my_project + delete resources.postgres_roles.owner + +This action will result in the deletion of the following Lakebase projects along with +all their branches, databases, and endpoints. All data stored in them will be permanently lost: + delete resources.postgres_projects.my_project + +This action will result in the deletion of the following Lakebase databases. +All data stored in them will be permanently lost: + delete resources.postgres_databases.my_database + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/oob-pg-data-api-[UNIQUE_NAME]/default + +Destroy: 4 deleted diff --git a/acceptance/bundle/resources/postgres_data_apis/out-of-band/script b/acceptance/bundle/resources/postgres_data_apis/out-of-band/script new file mode 100644 index 00000000000..e665ad8a95e --- /dev/null +++ b/acceptance/bundle/resources/postgres_data_apis/out-of-band/script @@ -0,0 +1,26 @@ +cleanup() { + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +database_name="projects/test-pg-proj-${UNIQUE_NAME}/branches/production/databases/my-database" +data_api_name="${database_name}/data-api" + +title "Deploy enables the Data API" +envsubst < databricks.yml.tmpl > databricks.yml +trace $CLI bundle deploy +trace $CLI postgres get-data-api "${data_api_name}" | data_api_fields + +title "Change the spec out of band" +trace $CLI postgres update-data-api "${data_api_name}" spec --json '{"spec": {"db_schemas": ["public"], "db_max_rows": 7}}' | data_api_fields + +title "Plan does not detect the change: spec is input-only and GET does not return it" +trace $CLI bundle plan + +title "Disable the Data API out of band" +trace $CLI postgres delete-data-api "${data_api_name}" + +title "Plan detects the Data API is gone and re-creates it" +trace $CLI bundle plan +trace $CLI bundle deploy +trace $CLI postgres get-data-api "${data_api_name}" | data_api_fields diff --git a/acceptance/bundle/resources/postgres_data_apis/out-of-band/test.toml b/acceptance/bundle/resources/postgres_data_apis/out-of-band/test.toml new file mode 100644 index 00000000000..a030353d571 --- /dev/null +++ b/acceptance/bundle/resources/postgres_data_apis/out-of-band/test.toml @@ -0,0 +1 @@ +RecordRequests = false diff --git a/acceptance/bundle/resources/postgres_data_apis/recreate/databricks.yml.tmpl b/acceptance/bundle/resources/postgres_data_apis/recreate/databricks.yml.tmpl new file mode 100644 index 00000000000..5b535dc1ef2 --- /dev/null +++ b/acceptance/bundle/resources/postgres_data_apis/recreate/databricks.yml.tmpl @@ -0,0 +1,37 @@ +bundle: + name: recreate-pg-data-api-$UNIQUE_NAME + +sync: + paths: [] + +resources: + postgres_projects: + my_project: + project_id: test-pg-proj-$UNIQUE_NAME + display_name: "Test Project for Data API Recreate" + pg_version: 16 + + postgres_roles: + owner: + parent: ${resources.postgres_projects.my_project.id}/branches/production + role_id: app-owner + postgres_role: app_owner + + postgres_databases: + first: + parent: ${resources.postgres_projects.my_project.id}/branches/production + database_id: first-database + postgres_database: first_db + role: ${resources.postgres_roles.owner.id} + second: + parent: ${resources.postgres_projects.my_project.id}/branches/production + database_id: second-database + postgres_database: second_db + role: ${resources.postgres_roles.owner.id} + + postgres_data_apis: + my_data_api: + parent: ${resources.postgres_databases.first.name} + db_schemas: + - public + db_max_rows: 100 diff --git a/acceptance/bundle/resources/postgres_data_apis/recreate/out.requests.recreate.json b/acceptance/bundle/resources/postgres_data_apis/recreate/out.requests.recreate.json new file mode 100644 index 00000000000..33930f3cd6c --- /dev/null +++ b/acceptance/bundle/resources/postgres_data_apis/recreate/out.requests.recreate.json @@ -0,0 +1,16 @@ +{ + "method": "DELETE", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/first-database/data-api" +} +{ + "method": "POST", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/second-database/data-api", + "body": { + "spec": { + "db_max_rows": 100, + "db_schemas": [ + "public" + ] + } + } +} diff --git a/acceptance/bundle/resources/postgres_data_apis/recreate/out.test.toml b/acceptance/bundle/resources/postgres_data_apis/recreate/out.test.toml new file mode 100644 index 00000000000..ee153e82621 --- /dev/null +++ b/acceptance/bundle/resources/postgres_data_apis/recreate/out.test.toml @@ -0,0 +1,4 @@ +Cloud = true +CloudEnvs.azure = false +CloudEnvs.gcp = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/postgres_data_apis/recreate/output.txt b/acceptance/bundle/resources/postgres_data_apis/recreate/output.txt new file mode 100644 index 00000000000..ed761507f8a --- /dev/null +++ b/acceptance/bundle/resources/postgres_data_apis/recreate/output.txt @@ -0,0 +1,73 @@ + +=== Deploy the Data API on the first database +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/recreate-pg-data-api-[UNIQUE_NAME]/default/files... +Created postgres_data_apis.my_data_api +Created postgres_databases.first +Created postgres_databases.second +Created postgres_projects.my_project +Created postgres_roles.owner +Files: 0 uploaded, 0 deleted +Resources: 5 created, 0 changed, 0 deleted, 0 unchanged + +>>> [CLI] postgres get-data-api projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/first-database/data-api +{ + "name": "projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/first-database/data-api", + "parent": "projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/first-database", + "status": { + "db_schemas": [ + "public" + ], + "db_max_rows": 100 + } +} + +=== Moving the Data API to another database recreates it +>>> update_file.py databricks.yml postgres_databases.first.name postgres_databases.second.name + +>>> [CLI] bundle plan +recreate postgres_data_apis.my_data_api + +Plan: 1 to add, 0 to change, 1 to delete, 4 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/recreate-pg-data-api-[UNIQUE_NAME]/default/files... +Recreated postgres_data_apis.my_data_api +Files: 0 uploaded, 0 deleted +Resources: 1 created, 0 changed, 1 deleted, 4 unchanged + +>>> musterr [CLI] postgres get-data-api projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/first-database/data-api +Error: data api id not found [TraceId: [UUID]] + +>>> [CLI] postgres get-data-api projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/second-database/data-api +{ + "name": "projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/second-database/data-api", + "parent": "projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/second-database", + "status": { + "db_schemas": [ + "public" + ], + "db_max_rows": 100 + } +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.postgres_data_apis.my_data_api + delete resources.postgres_databases.first + delete resources.postgres_databases.second + delete resources.postgres_projects.my_project + delete resources.postgres_roles.owner + +This action will result in the deletion of the following Lakebase projects along with +all their branches, databases, and endpoints. All data stored in them will be permanently lost: + delete resources.postgres_projects.my_project + +This action will result in the deletion of the following Lakebase databases. +All data stored in them will be permanently lost: + delete resources.postgres_databases.first + delete resources.postgres_databases.second + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/recreate-pg-data-api-[UNIQUE_NAME]/default + +Destroy: 5 deleted diff --git a/acceptance/bundle/resources/postgres_data_apis/recreate/script b/acceptance/bundle/resources/postgres_data_apis/recreate/script new file mode 100644 index 00000000000..b28dd1a5c52 --- /dev/null +++ b/acceptance/bundle/resources/postgres_data_apis/recreate/script @@ -0,0 +1,21 @@ +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f "$OUT_REQUESTS" +} +trap cleanup EXIT + +branch_name="projects/test-pg-proj-${UNIQUE_NAME}/branches/production" + +title "Deploy the Data API on the first database" +envsubst < databricks.yml.tmpl > databricks.yml +trace $CLI bundle deploy +trace $CLI postgres get-data-api "${branch_name}/databases/first-database/data-api" | data_api_fields + +title "Moving the Data API to another database recreates it" +trace update_file.py databricks.yml 'postgres_databases.first.name' 'postgres_databases.second.name' +trace $CLI bundle plan +rm -f "$OUT_REQUESTS" +trace $CLI bundle deploy +print_postgres_requests recreate +trace musterr $CLI postgres get-data-api "${branch_name}/databases/first-database/data-api" +trace $CLI postgres get-data-api "${branch_name}/databases/second-database/data-api" | data_api_fields diff --git a/acceptance/bundle/resources/postgres_data_apis/script.prepare b/acceptance/bundle/resources/postgres_data_apis/script.prepare new file mode 100644 index 00000000000..3c715e24166 --- /dev/null +++ b/acceptance/bundle/resources/postgres_data_apis/script.prepare @@ -0,0 +1,13 @@ +# data_api_fields is the allow-list of Data API fields the acceptance tests +# assert on. Anything the backend returns outside this list is dropped, so a new +# field in the API response does not break the golden. The status echoes only +# the spec fields the tests set explicitly. +data_api_fields() { + jq '{name, parent, status: (.status | {db_schemas, db_max_rows})}' +} + +# print_postgres_requests writes the recorded postgres write requests to +# out.requests..json. +print_postgres_requests() { + print_requests.py '//postgres' > "out.requests.$1.json" +} diff --git a/acceptance/bundle/resources/postgres_data_apis/test.toml b/acceptance/bundle/resources/postgres_data_apis/test.toml new file mode 100644 index 00000000000..aa07fccc9c9 --- /dev/null +++ b/acceptance/bundle/resources/postgres_data_apis/test.toml @@ -0,0 +1,16 @@ +Cloud = true + +# Lakebase v2 (postgres) is only available in AWS as of January 2026 +CloudEnvs.gcp = false +CloudEnvs.azure = false + +Ignore = [ + "databricks.yml", + ".databricks", +] + +[[Repls]] +# Normalize endpoint UIDs (ep-xxx-yyy format, supports both word-based and hex-based UIDs) +Old = 'ep-[a-z0-9-]+' +New = '[ENDPOINT_UID]' +Order = 1 diff --git a/acceptance/bundle/resources/postgres_data_apis/update/databricks.yml.tmpl b/acceptance/bundle/resources/postgres_data_apis/update/databricks.yml.tmpl new file mode 100644 index 00000000000..5dd17ca68c9 --- /dev/null +++ b/acceptance/bundle/resources/postgres_data_apis/update/databricks.yml.tmpl @@ -0,0 +1,34 @@ +bundle: + name: update-pg-data-api-$UNIQUE_NAME + +sync: + paths: [] + +resources: + postgres_projects: + my_project: + project_id: test-pg-proj-$UNIQUE_NAME + display_name: "Test Project for Data API Update" + pg_version: 16 + + postgres_roles: + owner: + parent: ${resources.postgres_projects.my_project.id}/branches/production + role_id: app-owner + postgres_role: app_owner + + postgres_databases: + my_database: + parent: ${resources.postgres_projects.my_project.id}/branches/production + database_id: my-database + postgres_database: app_db + role: ${resources.postgres_roles.owner.id} + + postgres_data_apis: + my_data_api: + parent: ${resources.postgres_databases.my_database.name} + db_schemas: + - public + db_max_rows: 100 + server_cors_allowed_origins: + - https://app.example.com diff --git a/acceptance/bundle/resources/postgres_data_apis/update/out.plan.update.json b/acceptance/bundle/resources/postgres_data_apis/update/out.plan.update.json new file mode 100644 index 00000000000..0b009da0c1c --- /dev/null +++ b/acceptance/bundle/resources/postgres_data_apis/update/out.plan.update.json @@ -0,0 +1,30 @@ +{ + "action": "update", + "changes": { + "db_max_rows": { + "action": "update", + "old": 100, + "new": 500 + }, + "db_schemas": { + "action": "skip", + "reason": "spec:input_only", + "old": [ + "public" + ], + "new": [ + "public" + ] + }, + "server_cors_allowed_origins": { + "action": "skip", + "reason": "spec:input_only", + "old": [ + "https://app.example.com" + ], + "new": [ + "https://app.example.com" + ] + } + } +} diff --git a/acceptance/bundle/resources/postgres_data_apis/update/out.requests.remove_field.json b/acceptance/bundle/resources/postgres_data_apis/update/out.requests.remove_field.json new file mode 100644 index 00000000000..615bb91c67f --- /dev/null +++ b/acceptance/bundle/resources/postgres_data_apis/update/out.requests.remove_field.json @@ -0,0 +1,15 @@ +{ + "method": "PATCH", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/my-database/data-api", + "q": { + "update_mask": "spec" + }, + "body": { + "spec": { + "db_max_rows": 500, + "db_schemas": [ + "public" + ] + } + } +} diff --git a/acceptance/bundle/resources/postgres_data_apis/update/out.requests.update.json b/acceptance/bundle/resources/postgres_data_apis/update/out.requests.update.json new file mode 100644 index 00000000000..8affc021c9e --- /dev/null +++ b/acceptance/bundle/resources/postgres_data_apis/update/out.requests.update.json @@ -0,0 +1,18 @@ +{ + "method": "PATCH", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/my-database/data-api", + "q": { + "update_mask": "spec" + }, + "body": { + "spec": { + "db_max_rows": 500, + "db_schemas": [ + "public" + ], + "server_cors_allowed_origins": [ + "https://app.example.com" + ] + } + } +} diff --git a/acceptance/bundle/resources/postgres_data_apis/update/out.test.toml b/acceptance/bundle/resources/postgres_data_apis/update/out.test.toml new file mode 100644 index 00000000000..ee153e82621 --- /dev/null +++ b/acceptance/bundle/resources/postgres_data_apis/update/out.test.toml @@ -0,0 +1,4 @@ +Cloud = true +CloudEnvs.azure = false +CloudEnvs.gcp = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/postgres_data_apis/update/output.txt b/acceptance/bundle/resources/postgres_data_apis/update/output.txt new file mode 100644 index 00000000000..45a2173f99a --- /dev/null +++ b/acceptance/bundle/resources/postgres_data_apis/update/output.txt @@ -0,0 +1,73 @@ + +=== Initial deployment +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/update-pg-data-api-[UNIQUE_NAME]/default/files... +Created postgres_data_apis.my_data_api +Created postgres_databases.my_database +Created postgres_projects.my_project +Created postgres_roles.owner +Files: 0 uploaded, 0 deleted +Resources: 4 created, 0 changed, 0 deleted, 0 unchanged + +=== Change a spec field +>>> update_file.py databricks.yml db_max_rows: 100 db_max_rows: 500 + +>>> [CLI] bundle plan +update postgres_data_apis.my_data_api + +Plan: 0 to add, 1 to change, 0 to delete, 3 unchanged + +>>> [CLI] bundle plan -o json + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/update-pg-data-api-[UNIQUE_NAME]/default/files... +Updated postgres_data_apis.my_data_api +Files: 0 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 3 unchanged + +>>> [CLI] postgres get-data-api projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/my-database/data-api +{ + "name": "projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/my-database/data-api", + "parent": "projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/my-database", + "status": { + "db_schemas": [ + "public" + ], + "db_max_rows": 500 + } +} + +=== Remove a spec field: the whole spec is sent, so the field resets to its server default +>>> [CLI] bundle plan +update postgres_data_apis.my_data_api + +Plan: 0 to add, 1 to change, 0 to delete, 3 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/update-pg-data-api-[UNIQUE_NAME]/default/files... +Updated postgres_data_apis.my_data_api +Files: 0 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 3 unchanged + +=== Plan after update shows no changes +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 4 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.postgres_data_apis.my_data_api + delete resources.postgres_databases.my_database + delete resources.postgres_projects.my_project + delete resources.postgres_roles.owner + +This action will result in the deletion of the following Lakebase projects along with +all their branches, databases, and endpoints. All data stored in them will be permanently lost: + delete resources.postgres_projects.my_project + +This action will result in the deletion of the following Lakebase databases. +All data stored in them will be permanently lost: + delete resources.postgres_databases.my_database + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/update-pg-data-api-[UNIQUE_NAME]/default + +Destroy: 4 deleted diff --git a/acceptance/bundle/resources/postgres_data_apis/update/script b/acceptance/bundle/resources/postgres_data_apis/update/script new file mode 100644 index 00000000000..f369066fa79 --- /dev/null +++ b/acceptance/bundle/resources/postgres_data_apis/update/script @@ -0,0 +1,31 @@ +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f "$OUT_REQUESTS" +} +trap cleanup EXIT + +data_api_name="projects/test-pg-proj-${UNIQUE_NAME}/branches/production/databases/my-database/data-api" + +title "Initial deployment" +envsubst < databricks.yml.tmpl > databricks.yml +trace $CLI bundle deploy +rm -f "$OUT_REQUESTS" + +title "Change a spec field" +trace update_file.py databricks.yml "db_max_rows: 100" "db_max_rows: 500" +trace $CLI bundle plan +trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_data_apis.my_data_api" | {action, changes}' > out.plan.update.json +rm -f "$OUT_REQUESTS" +trace $CLI bundle deploy +print_postgres_requests update +trace $CLI postgres get-data-api "${data_api_name}" | data_api_fields + +title "Remove a spec field: the whole spec is sent, so the field resets to its server default" +sed '/server_cors_allowed_origins:/,$d' databricks.yml > databricks.yml.new && mv databricks.yml.new databricks.yml +trace $CLI bundle plan +rm -f "$OUT_REQUESTS" +trace $CLI bundle deploy +print_postgres_requests remove_field + +title "Plan after update shows no changes" +trace $CLI bundle plan diff --git a/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go b/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go index 4ead9785e06..bbc2f045be5 100644 --- a/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go +++ b/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go @@ -32,6 +32,7 @@ var unsupportedResources = []string{ "database_catalogs", "synced_database_tables", "postgres_branches", + "postgres_data_apis", "postgres_databases", "postgres_endpoints", "postgres_catalogs", diff --git a/bundle/config/mutator/resourcemutator/apply_target_mode_test.go b/bundle/config/mutator/resourcemutator/apply_target_mode_test.go index e9ea418d1d6..8ff6a717c4e 100644 --- a/bundle/config/mutator/resourcemutator/apply_target_mode_test.go +++ b/bundle/config/mutator/resourcemutator/apply_target_mode_test.go @@ -324,6 +324,13 @@ func mockBundle(mode config.Mode) *bundle.Bundle { }, }, }, + PostgresDataApis: map[string]*resources.PostgresDataApi{ + "postgres_data_api1": { + PostgresDataApiConfig: resources.PostgresDataApiConfig{ + Parent: "projects/project1/branches/branch1/databases/database1", + }, + }, + }, VectorSearchEndpoints: map[string]*resources.VectorSearchEndpoint{ "vs_endpoint1": { CreateEndpoint: vectorsearch.CreateEndpoint{ diff --git a/bundle/config/mutator/resourcemutator/run_as_test.go b/bundle/config/mutator/resourcemutator/run_as_test.go index 200939f2dd0..5662465c564 100644 --- a/bundle/config/mutator/resourcemutator/run_as_test.go +++ b/bundle/config/mutator/resourcemutator/run_as_test.go @@ -62,6 +62,7 @@ func allResourceTypes(t *testing.T) []string { "pipelines", "postgres_branches", "postgres_catalogs", + "postgres_data_apis", "postgres_databases", "postgres_endpoints", "postgres_projects", @@ -195,6 +196,7 @@ var allowList = []string{ "model_provider_services", "postgres_branches", "postgres_catalogs", + "postgres_data_apis", "postgres_databases", "postgres_endpoints", "postgres_projects", diff --git a/bundle/config/resources.go b/bundle/config/resources.go index 75fef099133..8077c76dfd1 100644 --- a/bundle/config/resources.go +++ b/bundle/config/resources.go @@ -47,6 +47,7 @@ type Resources struct { PostgresRoles map[string]*resources.PostgresRole `json:"postgres_roles,omitempty"` PostgresSyncedTables map[string]*resources.PostgresSyncedTable `json:"postgres_synced_tables,omitempty"` PostgresSnapshotSchedules map[string]*resources.PostgresSnapshotSchedule `json:"postgres_snapshot_schedules,omitempty"` + PostgresDataApis map[string]*resources.PostgresDataApi `json:"postgres_data_apis,omitempty"` VectorSearchEndpoints map[string]*resources.VectorSearchEndpoint `json:"vector_search_endpoints,omitempty"` VectorSearchIndexes map[string]*resources.VectorSearchIndex `json:"vector_search_indexes,omitempty"` InstancePools map[string]*resources.InstancePool `json:"instance_pools,omitempty"` @@ -144,6 +145,7 @@ func (r *Resources) AllResources() []ResourceGroup { collectResourceMap(descriptions["postgres_roles"], r.PostgresRoles), collectResourceMap(descriptions["postgres_synced_tables"], r.PostgresSyncedTables), collectResourceMap(descriptions["postgres_snapshot_schedules"], r.PostgresSnapshotSchedules), + collectResourceMap(descriptions["postgres_data_apis"], r.PostgresDataApis), collectResourceMap(descriptions["vector_search_endpoints"], r.VectorSearchEndpoints), collectResourceMap(descriptions["vector_search_indexes"], r.VectorSearchIndexes), collectResourceMap(descriptions["instance_pools"], r.InstancePools), @@ -232,6 +234,7 @@ func SupportedResources() map[string]resources.ResourceDescription { "postgres_roles": (&resources.PostgresRole{}).ResourceDescription(), "postgres_synced_tables": (&resources.PostgresSyncedTable{}).ResourceDescription(), "postgres_snapshot_schedules": (&resources.PostgresSnapshotSchedule{}).ResourceDescription(), + "postgres_data_apis": (&resources.PostgresDataApi{}).ResourceDescription(), "vector_search_endpoints": (&resources.VectorSearchEndpoint{}).ResourceDescription(), "vector_search_indexes": (&resources.VectorSearchIndex{}).ResourceDescription(), "internal_immutable_snapshots": (&resources.Snapshot{}).ResourceDescription(), diff --git a/bundle/config/resources/postgres_data_api.go b/bundle/config/resources/postgres_data_api.go new file mode 100644 index 00000000000..1d434eb03c7 --- /dev/null +++ b/bundle/config/resources/postgres_data_api.go @@ -0,0 +1,80 @@ +package resources + +import ( + "context" + "net/url" + + "github.com/databricks/cli/libs/log" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/marshal" + "github.com/databricks/databricks-sdk-go/service/postgres" +) + +type PostgresDataApiConfig struct { + postgres.DataApiDataApiSpec + + // Parent is the database whose Data API this resource manages. Format: + // "projects/{project_id}/branches/{branch_id}/databases/{database_id}". The + // Data API has no ID of its own: at most one exists per database, and its + // resource name (and this resource's ID) is "{parent}/data-api". + Parent string `json:"parent"` +} + +func (c *PostgresDataApiConfig) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, c) +} + +func (c PostgresDataApiConfig) MarshalJSON() ([]byte, error) { + return marshal.Marshal(c) +} + +type PostgresDataApi struct { + BaseResource + ID string `json:"id,omitempty" bundle:"readonly"` + PostgresDataApiConfig +} + +func (d *PostgresDataApi) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, d) +} + +func (d PostgresDataApi) MarshalJSON() ([]byte, error) { + return marshal.Marshal(d) +} + +func (d *PostgresDataApi) Exists(ctx context.Context, w *databricks.WorkspaceClient, name string) (bool, error) { + _, err := w.Postgres.GetDataApi(ctx, postgres.GetDataApiRequest{Name: name}) + if apierr.IsMissing(err) { + log.Debugf(ctx, "postgres data api %s does not exist", name) + return false, nil + } + if err != nil { + return false, err + } + return true, nil +} + +func (d *PostgresDataApi) ResourceDescription() ResourceDescription { + return ResourceDescription{ + SingularName: "postgres_data_api", + PluralName: "postgres_data_apis", + SingularTitle: "Postgres Data API", + PluralTitle: "Postgres Data APIs", + } +} + +func (d *PostgresDataApi) GetName() string { + // The Data API has no name field of its own; its resource name is its ID + // ("{parent}/data-api"). + return d.ID +} + +func (d *PostgresDataApi) GetURL() (string, bool) { + // The IDs in the API do not (yet) map to IDs in the web UI. + return "", false +} + +func (d *PostgresDataApi) InitializeURL(_ url.URL) { + // The IDs in the API do not (yet) map to IDs in the web UI. +} diff --git a/bundle/config/resources_test.go b/bundle/config/resources_test.go index 963e7efd4ff..d85f7e09cee 100644 --- a/bundle/config/resources_test.go +++ b/bundle/config/resources_test.go @@ -133,6 +133,7 @@ func TestBundleResourcePluralNamesResolveInWorkspaceURLs(t *testing.T) { // built in JobRun.InitializeURL via workspaceurls.JobRunURL instead. "job_runs": true, "postgres_branches": true, + "postgres_data_apis": true, "postgres_databases": true, "postgres_endpoints": true, "postgres_projects": true, @@ -368,6 +369,13 @@ func TestResourcesBindSupport(t *testing.T) { }, }, }, + PostgresDataApis: map[string]*resources.PostgresDataApi{ + "my_postgres_data_api": { + PostgresDataApiConfig: resources.PostgresDataApiConfig{ + Parent: "projects/my-postgres-project/branches/my-postgres-branch/databases/my-postgres-database", + }, + }, + }, VectorSearchEndpoints: map[string]*resources.VectorSearchEndpoint{ "my_vector_search_endpoint": { CreateEndpoint: vectorsearch.CreateEndpoint{ @@ -431,6 +439,7 @@ func TestResourcesBindSupport(t *testing.T) { m.GetMockPostgresAPI().EXPECT().GetRole(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockPostgresAPI().EXPECT().GetSyncedTable(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockPostgresAPI().EXPECT().GetSnapshotSchedule(mock.Anything, mock.Anything).Return(nil, nil) + m.GetMockPostgresAPI().EXPECT().GetDataApi(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockPostgresAPI().EXPECT().GetRole(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockVectorSearchEndpointsAPI().EXPECT().GetEndpoint(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockVectorSearchIndexesAPI().EXPECT().GetIndexByIndexName(mock.Anything, mock.Anything).Return(nil, nil) diff --git a/bundle/direct/dresources/all.go b/bundle/direct/dresources/all.go index 88f08048235..25039ca99fe 100644 --- a/bundle/direct/dresources/all.go +++ b/bundle/direct/dresources/all.go @@ -30,6 +30,7 @@ var SupportedResources = map[string]any{ "postgres_roles": (*ResourcePostgresRole)(nil), "postgres_synced_tables": (*ResourcePostgresSyncedTable)(nil), "postgres_snapshot_schedules": (*ResourcePostgresSnapshotSchedule)(nil), + "postgres_data_apis": (*ResourcePostgresDataApi)(nil), "alerts": (*ResourceAlert)(nil), "clusters": (*ResourceCluster)(nil), "registered_models": (*ResourceRegisteredModel)(nil), diff --git a/bundle/direct/dresources/all_test.go b/bundle/direct/dresources/all_test.go index 1b0da0602a2..58d664c3ac1 100644 --- a/bundle/direct/dresources/all_test.go +++ b/bundle/direct/dresources/all_test.go @@ -890,6 +890,46 @@ var testDeps = map[string]prepareWorkspace{ }, nil }, + "postgres_data_apis": func(ctx context.Context, client *databricks.WorkspaceClient) (any, error) { + // Creating the project implicitly provisions the "production" branch and + // its primary read-write endpoint, which the Data API requires. + _, err := client.Postgres.CreateProject(ctx, postgres.CreateProjectRequest{ + ProjectId: "test-project-for-data-api", + Project: postgres.Project{ + Spec: &postgres.ProjectSpec{ + DisplayName: "Test Project for Data API", + PgVersion: 16, + }, + }, + }) + if err != nil { + return nil, err + } + + waiter, err := client.Postgres.CreateDatabase(ctx, postgres.CreateDatabaseRequest{ + Parent: "projects/test-project-for-data-api/branches/production", + DatabaseId: "test-database", + Database: postgres.Database{ + Spec: &postgres.DatabaseDatabaseSpec{ + PostgresDatabase: "app_db", + Role: "projects/test-project-for-data-api/branches/production/roles/owner", + }, + }, + }) + if err != nil { + return nil, err + } + database, err := waiter.Wait(ctx) + if err != nil { + return nil, err + } + + return &resources.PostgresDataApi{ + Parent: database.Name, + DbSchemas: []string{"public"}, + }, nil + }, + "postgres_roles": func(ctx context.Context, client *databricks.WorkspaceClient) (any, error) { // Create parent project first _, err := client.Postgres.CreateProject(ctx, postgres.CreateProjectRequest{ diff --git a/bundle/direct/dresources/apitypes.generated.yml b/bundle/direct/dresources/apitypes.generated.yml index 317a04442c4..c60fba878b5 100644 --- a/bundle/direct/dresources/apitypes.generated.yml +++ b/bundle/direct/dresources/apitypes.generated.yml @@ -46,6 +46,8 @@ postgres_branches: postgres.BranchSpec postgres_catalogs: postgres.CatalogCatalogSpec +postgres_data_apis: postgres.DataApiDataApiSpec + postgres_databases: postgres.DatabaseDatabaseStatus postgres_endpoints: postgres.EndpointSpec diff --git a/bundle/direct/dresources/apitypes.yml b/bundle/direct/dresources/apitypes.yml index ba16fd72c6a..325754faca3 100644 --- a/bundle/direct/dresources/apitypes.yml +++ b/bundle/direct/dresources/apitypes.yml @@ -30,3 +30,5 @@ postgres_catalogs: postgres.CatalogCatalogSpec postgres_synced_tables: postgres.SyncedTableSyncedTableSpec postgres_roles: postgres.RoleRoleSpec + +postgres_data_apis: postgres.DataApiDataApiSpec diff --git a/bundle/direct/dresources/configs/postgres_data_apis.generated.yml b/bundle/direct/dresources/configs/postgres_data_apis.generated.yml new file mode 100644 index 00000000000..5a94ec39e45 --- /dev/null +++ b/bundle/direct/dresources/configs/postgres_data_apis.generated.yml @@ -0,0 +1,21 @@ +# Generated, do not edit. + +ignore_remote_changes: + - field: db_aggregates_enabled + reason: spec:input_only + - field: db_extra_search_path + reason: spec:input_only + - field: db_max_rows + reason: spec:input_only + - field: db_schemas + reason: spec:input_only + - field: jwt_cache_max_lifetime + reason: spec:input_only + - field: jwt_role_claim_key + reason: spec:input_only + - field: openapi_mode + reason: spec:input_only + - field: server_cors_allowed_origins + reason: spec:input_only + - field: server_timing_enabled + reason: spec:input_only diff --git a/bundle/direct/dresources/configs/postgres_data_apis.yml b/bundle/direct/dresources/configs/postgres_data_apis.yml new file mode 100644 index 00000000000..42718ab0d51 --- /dev/null +++ b/bundle/direct/dresources/configs/postgres_data_apis.yml @@ -0,0 +1,10 @@ +provided_id_fields: + # parent composes the Data API's hierarchical name ("{parent}/data-api"); + # changing it targets a different database's Data API, so it recreates + # (delete + create). + - field: parent + reason: id_field +stable_output_fields: + # name ("{parent}/data-api") is backend-assigned at creation and immutable; + # see the stable_output_fields doc in config.go and ES-2202624. + - field: name diff --git a/bundle/direct/dresources/postgres_data_api.go b/bundle/direct/dresources/postgres_data_api.go new file mode 100644 index 00000000000..8803eb5f41e --- /dev/null +++ b/bundle/direct/dresources/postgres_data_api.go @@ -0,0 +1,162 @@ +package dresources + +import ( + "context" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/common/types/fieldmask" + sdktime "github.com/databricks/databricks-sdk-go/common/types/time" + "github.com/databricks/databricks-sdk-go/marshal" + "github.com/databricks/databricks-sdk-go/service/postgres" +) + +// dataApiUpdateMask masks the whole spec. The direct engine sends the full +// desired spec on every update, so a field removed from the config resets to +// its server default. Masking only the changed leaves would instead be rejected +// for a removed field ("is in update_mask but not provided in request"). +var dataApiUpdateMask = []string{"spec"} + +// PostgresDataApiRemote is the return type for DoRead. It embeds +// DataApiDataApiSpec so that all paths in StateType are valid paths in +// RemoteType. GET does not return spec (it is INPUT_ONLY), so the embedded spec +// stays at its zero value and postgres_data_apis.generated.yml suppresses the +// resulting phantom drift via ignore_remote_changes with reason spec:input_only. +type PostgresDataApiRemote struct { + postgres.DataApiDataApiSpec + + Parent string `json:"parent,omitempty"` + + Name string `json:"name,omitempty"` + Status *postgres.DataApiDataApiStatus `json:"status,omitempty"` + CreateTime *sdktime.Time `json:"create_time,omitempty"` + UpdateTime *sdktime.Time `json:"update_time,omitempty"` +} + +// Custom marshaler needed because embedded DataApiDataApiSpec has its own +// MarshalJSON which would otherwise take over and ignore the additional fields. +func (s *PostgresDataApiRemote) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, s) +} + +func (s PostgresDataApiRemote) MarshalJSON() ([]byte, error) { + return marshal.Marshal(s) +} + +type ResourcePostgresDataApi struct { + client *databricks.WorkspaceClient +} + +type PostgresDataApiState = resources.PostgresDataApiConfig + +func (*ResourcePostgresDataApi) New(client *databricks.WorkspaceClient) *ResourcePostgresDataApi { + return &ResourcePostgresDataApi{client: client} +} + +func (*ResourcePostgresDataApi) PrepareState(input *resources.PostgresDataApi) *PostgresDataApiState { + return &PostgresDataApiState{ + DataApiDataApiSpec: input.DataApiDataApiSpec, + Parent: input.Parent, + } +} + +func (*ResourcePostgresDataApi) RemapState(remote *PostgresDataApiRemote) *PostgresDataApiState { + return &PostgresDataApiState{ + DataApiDataApiSpec: remote.DataApiDataApiSpec, + Parent: remote.Parent, + } +} + +func makePostgresDataApiRemote(dataApi *postgres.DataApi) *PostgresDataApiRemote { + var spec postgres.DataApiDataApiSpec + if dataApi.Spec != nil { + spec = *dataApi.Spec + } + return &PostgresDataApiRemote{ + DataApiDataApiSpec: spec, + Parent: dataApi.Parent, + Name: dataApi.Name, + Status: dataApi.Status, + CreateTime: dataApi.CreateTime, + UpdateTime: dataApi.UpdateTime, + } +} + +func (r *ResourcePostgresDataApi) DoRead(ctx context.Context, id string) (*PostgresDataApiRemote, error) { + dataApi, err := r.client.Postgres.GetDataApi(ctx, postgres.GetDataApiRequest{Name: id}) + if err != nil { + return nil, err + } + return makePostgresDataApiRemote(dataApi), nil +} + +// DoCreate enables the Data API on the parent database. If it is already +// enabled the API returns ALREADY_EXISTS; adopt it with `bundle deployment bind`. +func (r *ResourcePostgresDataApi) DoCreate(ctx context.Context, config *PostgresDataApiState) (string, *PostgresDataApiRemote, error) { + waiter, err := r.client.Postgres.CreateDataApi(ctx, postgres.CreateDataApiRequest{ + Parent: config.Parent, + DataApi: postgres.DataApi{ + Spec: &config.DataApiDataApiSpec, + + // Output-only fields; the parent is carried in the request path. + CreateTime: nil, + Name: "", + Parent: "", + Status: nil, + UpdateTime: nil, + ForceSendFields: nil, + }, + }) + if err != nil { + return "", nil, err + } + + result, err := waiter.Wait(ctx) + if err != nil { + return "", nil, err + } + + remote := makePostgresDataApiRemote(result) + return remote.Name, remote, nil +} + +func (r *ResourcePostgresDataApi) DoUpdate(ctx context.Context, id string, config *PostgresDataApiState, _ *PlanEntry) (*PostgresDataApiRemote, error) { + waiter, err := r.client.Postgres.UpdateDataApi(ctx, postgres.UpdateDataApiRequest{ + Name: id, + DataApi: postgres.DataApi{ + Spec: &config.DataApiDataApiSpec, + + // Output-only fields; the name is carried in the request path. + CreateTime: nil, + Name: "", + Parent: "", + Status: nil, + UpdateTime: nil, + ForceSendFields: nil, + }, + UpdateMask: fieldmask.FieldMask{ + Paths: dataApiUpdateMask, + }, + }) + if err != nil { + return nil, err + } + + result, err := waiter.Wait(ctx) + if err != nil { + return nil, err + } + return makePostgresDataApiRemote(result), nil +} + +// DoDelete disables the Data API. The parent database and its data are not +// affected. +func (r *ResourcePostgresDataApi) DoDelete(ctx context.Context, id string, _ *PostgresDataApiState) error { + waiter, err := r.client.Postgres.DeleteDataApi(ctx, postgres.DeleteDataApiRequest{ + Name: id, + }) + if err != nil { + return err + } + return waiter.Wait(ctx) +} diff --git a/bundle/internal/annotation/preview.go b/bundle/internal/annotation/preview.go index b4be7348526..0eb137449ec 100644 --- a/bundle/internal/annotation/preview.go +++ b/bundle/internal/annotation/preview.go @@ -27,8 +27,10 @@ func PreviewTag(stage clijson.LaunchStage) string { // fully-qualified Go type paths (PkgPath + "." + Name), matching getPath in // bundle/internal/schema. var launchStageOverrides = map[string]clijson.LaunchStage{ - "github.com/databricks/cli/bundle/config/resources.PostgresBranch": clijson.LaunchStagePublicBeta, - "github.com/databricks/cli/bundle/config/resources.PostgresCatalog": clijson.LaunchStagePublicBeta, + "github.com/databricks/cli/bundle/config/resources.PostgresBranch": clijson.LaunchStagePublicBeta, + "github.com/databricks/cli/bundle/config/resources.PostgresCatalog": clijson.LaunchStagePublicBeta, + // The Data API is at Private Preview, below the Beta its siblings are pinned to. + "github.com/databricks/cli/bundle/config/resources.PostgresDataApi": clijson.LaunchStagePrivatePreview, "github.com/databricks/cli/bundle/config/resources.PostgresDatabase": clijson.LaunchStagePublicBeta, "github.com/databricks/cli/bundle/config/resources.PostgresEndpoint": clijson.LaunchStagePublicBeta, "github.com/databricks/cli/bundle/config/resources.PostgresProject": clijson.LaunchStagePublicBeta, diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index 1a5c4ccbfdd..a74eced6bad 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -1622,6 +1622,16 @@ resources: "lifecycle": "description": |- Settings that control the deployment lifecycle of the resource, such as preventing it from being destroyed. + "postgres_data_apis": + "description": |- + The Postgres Data API definitions for the bundle, where each key is the name of the Data API. Each entry enables and configures the Data API (a REST interface) of a database on a Lakebase Autoscaling branch. Removing an entry turns off the Data API; the database and its data are not affected. + "$fields": + "lifecycle": + "description": |- + Settings that control the deployment lifecycle of the resource, such as preventing it from being destroyed. + "parent": + "description": |- + The database whose Data API this manages. Format: projects/{project_id}/branches/{branch_id}/databases/{database_id} "postgres_databases": "description": |- The Postgres database definitions for the bundle, where each key is the name of the database. Each entry defines a database on a Lakebase Autoscaling branch. diff --git a/bundle/internal/schema/parser_test.go b/bundle/internal/schema/parser_test.go index c89e7b8f832..9369e777177 100644 --- a/bundle/internal/schema/parser_test.go +++ b/bundle/internal/schema/parser_test.go @@ -72,8 +72,9 @@ func TestExtractAnnotationsNestedEmbeddedSDKType(t *testing.T) { } // TestPostgresResourcesHaveLaunchStageOverride asserts every Postgres resource -// type is pinned to Beta via a launch-stage override, so an upstream stage -// change cannot silently alter its label (see annotation.OverrideLaunchStage). +// type is pinned to its expected stage (Beta, or Private Preview for the Data +// API) via a launch-stage override, so an upstream stage change cannot silently +// alter its label (see annotation.OverrideLaunchStage). // // postgresResources is spelled out explicitly so the expected set is visible on // the page, but it is also cross-checked against the Postgres resources actually @@ -81,21 +82,22 @@ func TestExtractAnnotationsNestedEmbeddedSDKType(t *testing.T) { // completeness check, which is the reminder to add it here AND to // launchStageOverrides in bundle/internal/annotation/preview.go. func TestPostgresResourcesHaveLaunchStageOverride(t *testing.T) { - postgresResources := []reflect.Type{ - reflect.TypeFor[resources.PostgresBranch](), - reflect.TypeFor[resources.PostgresCatalog](), - reflect.TypeFor[resources.PostgresDatabase](), - reflect.TypeFor[resources.PostgresEndpoint](), - reflect.TypeFor[resources.PostgresProject](), - reflect.TypeFor[resources.PostgresRole](), - reflect.TypeFor[resources.PostgresSnapshotSchedule](), - reflect.TypeFor[resources.PostgresSyncedTable](), + postgresResources := map[reflect.Type]clijson.LaunchStage{ + reflect.TypeFor[resources.PostgresBranch](): clijson.LaunchStagePublicBeta, + reflect.TypeFor[resources.PostgresCatalog](): clijson.LaunchStagePublicBeta, + reflect.TypeFor[resources.PostgresDataApi](): clijson.LaunchStagePrivatePreview, + reflect.TypeFor[resources.PostgresDatabase](): clijson.LaunchStagePublicBeta, + reflect.TypeFor[resources.PostgresEndpoint](): clijson.LaunchStagePublicBeta, + reflect.TypeFor[resources.PostgresProject](): clijson.LaunchStagePublicBeta, + reflect.TypeFor[resources.PostgresRole](): clijson.LaunchStagePublicBeta, + reflect.TypeFor[resources.PostgresSnapshotSchedule](): clijson.LaunchStagePublicBeta, + reflect.TypeFor[resources.PostgresSyncedTable](): clijson.LaunchStagePublicBeta, } // The list must cover every Postgres resource on config.Resources, so a // newly added one can't slip through unpinned. listed := make(map[reflect.Type]bool, len(postgresResources)) - for _, typ := range postgresResources { + for typ := range postgresResources { listed[typ] = true } for _, f := range reflect.VisibleFields(reflect.TypeFor[config.Resources]()) { @@ -112,13 +114,13 @@ func TestPostgresResourcesHaveLaunchStageOverride(t *testing.T) { "resource %s (%s) is not covered by this test; add it to postgresResources", f.Name, getPath(elem)) } - for _, typ := range postgresResources { + for typ, want := range postgresResources { typePath := getPath(typ) // A configured type returns its override for any input; an unconfigured // one returns the input unchanged, so a GA input that survives as GA // (rather than becoming Beta) means the override is missing. stage := annotation.OverrideLaunchStage(typePath, clijson.LaunchStageGA) - assert.Equalf(t, clijson.LaunchStagePublicBeta, stage, + assert.Equalf(t, want, stage, "resource %s has no launch-stage override; add it to launchStageOverrides in bundle/internal/annotation/preview.go", typePath) } } diff --git a/bundle/internal/validation/generated/enum_fields.go b/bundle/internal/validation/generated/enum_fields.go index e67e605689d..65ef4a8af2b 100644 --- a/bundle/internal/validation/generated/enum_fields.go +++ b/bundle/internal/validation/generated/enum_fields.go @@ -266,6 +266,8 @@ var EnumFields = map[string][]string{ "resources.pipelines.*.permissions[*].level": {"CAN_MANAGE", "CAN_RUN", "CAN_VIEW", "IS_OWNER"}, "resources.pipelines.*.restart_window.days_of_week[*]": {"FRIDAY", "MONDAY", "SATURDAY", "SUNDAY", "THURSDAY", "TUESDAY", "WEDNESDAY"}, + "resources.postgres_data_apis.*.openapi_mode": {"OPEN_API_MODE_DISABLED", "OPEN_API_MODE_IGNORE_PRIVILEGES"}, + "resources.postgres_endpoints.*.endpoint_type": {"ENDPOINT_TYPE_READ_ONLY", "ENDPOINT_TYPE_READ_WRITE"}, "resources.postgres_projects.*.permissions[*].level": {"CAN_ATTACH_TO", "CAN_BIND", "CAN_CREATE", "CAN_CREATE_APP", "CAN_EDIT", "CAN_EDIT_METADATA", "CAN_MANAGE", "CAN_MANAGE_PRODUCTION_VERSIONS", "CAN_MANAGE_RUN", "CAN_MANAGE_STAGING_VERSIONS", "CAN_MONITOR", "CAN_MONITOR_ONLY", "CAN_QUERY", "CAN_READ", "CAN_RESTART", "CAN_RUN", "CAN_USE", "CAN_VIEW", "CAN_VIEW_METADATA", "IS_OWNER"}, diff --git a/bundle/internal/validation/generated/required_fields.go b/bundle/internal/validation/generated/required_fields.go index f43a07dddce..458659f8da4 100644 --- a/bundle/internal/validation/generated/required_fields.go +++ b/bundle/internal/validation/generated/required_fields.go @@ -351,6 +351,8 @@ var RequiredFields = map[string][]string{ "resources.postgres_catalogs.*": {"postgres_database", "catalog_id"}, + "resources.postgres_data_apis.*": {"parent"}, + "resources.postgres_databases.*": {"role", "database_id", "parent"}, "resources.postgres_endpoints.*": {"endpoint_type", "endpoint_id", "parent"}, diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 76a608b5275..f230f221060 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -2457,6 +2457,88 @@ } ] }, + "resources.PostgresDataApi": { + "oneOf": [ + { + "type": "object", + "description": "[Private Preview] Desired PostgREST configuration (input).", + "properties": { + "db_aggregates_enabled": { + "description": "[Private Preview] Enable aggregate functions (count, sum, avg, etc.) in Data API responses.\nDefault: true.", + "$ref": "#/$defs/bool", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + }, + "db_extra_search_path": { + "description": "[Private Preview] Additional schemas to include in the PostgreSQL search path.\nEach entry must be a valid PostgreSQL schema name.", + "$ref": "#/$defs/slice/string", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + }, + "db_max_rows": { + "description": "[Private Preview] Maximum number of rows returned in a single Data API response.\nMust be a positive integer.", + "$ref": "#/$defs/int", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + }, + "db_schemas": { + "description": "[Private Preview] Database schemas exposed through the Data API.\nEach entry must be a valid PostgreSQL schema name (1-63 chars, [a-zA-Z_][a-zA-Z0-9_$]*).\nMaximum 100 entries. Default: [\"public\"].", + "$ref": "#/$defs/slice/string", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + }, + "jwt_cache_max_lifetime": { + "description": "[Private Preview] Maximum lifetime for cached JWT tokens. Zero duration disables caching.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/common/types/duration.Duration", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + }, + "jwt_role_claim_key": { + "description": "[Private Preview] JSON path to the role claim in JWT tokens (e.g., \".sub\").\nDefault: \".sub\".", + "$ref": "#/$defs/string", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + }, + "lifecycle": { + "description": "Settings that control the deployment lifecycle of the resource, such as preventing it from being destroyed.", + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.Lifecycle" + }, + "openapi_mode": { + "description": "[Private Preview] OpenAPI documentation mode for the Data API endpoint.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/postgres.OpenApiMode", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + }, + "parent": { + "description": "The database whose Data API this manages. Format: projects/{project_id}/branches/{branch_id}/databases/{database_id}", + "$ref": "#/$defs/string" + }, + "server_cors_allowed_origins": { + "description": "[Private Preview] Allowed origins for CORS requests.\nEach entry should be a valid origin URL, or use \"*\" to allow all origins.", + "$ref": "#/$defs/slice/string", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + }, + "server_timing_enabled": { + "description": "[Private Preview] Enable the Server-Timing header in Data API responses.", + "$ref": "#/$defs/bool", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + } + }, + "additionalProperties": false, + "required": [ + "parent" + ], + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\._*\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "resources.PostgresDatabase": { "oneOf": [ { @@ -4200,6 +4282,10 @@ "description": "The Postgres catalog definitions for the bundle, where each key is the name of the catalog. Each entry binds a Unity Catalog catalog to a Postgres database on a Lakebase Autoscaling branch.", "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.PostgresCatalog" }, + "postgres_data_apis": { + "description": "The Postgres Data API definitions for the bundle, where each key is the name of the Data API. Each entry enables and configures the Data API (a REST interface) of a database on a Lakebase Autoscaling branch. Removing an entry turns off the Data API; the database and its data are not affected.", + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.PostgresDataApi" + }, "postgres_databases": { "description": "The Postgres database definitions for the bundle, where each key is the name of the database. Each entry defines a database on a Lakebase Autoscaling branch.", "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.PostgresDatabase" @@ -17293,6 +17379,26 @@ } ] }, + "postgres.OpenApiMode": { + "oneOf": [ + { + "type": "string", + "description": "Controls how the Data API exposes the OpenAPI documentation endpoint.\nOnly IGNORE_PRIVILEGES and DISABLED are supported today; \"follow-privileges\"\nis not implemented yet (it may be added later as value 3 — adding new enum\nvalues is backward-compatible).", + "enum": [ + "OPEN_API_MODE_IGNORE_PRIVILEGES", + "OPEN_API_MODE_DISABLED" + ], + "enumDescriptions": [ + "[Private Preview]", + "[Private Preview]" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\._*\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "postgres.ProjectCustomTag": { "oneOf": [ { @@ -20236,6 +20342,20 @@ } ] }, + "resources.PostgresDataApi": { + "oneOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.PostgresDataApi" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\._*\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "resources.PostgresDatabase": { "oneOf": [ { diff --git a/bundle/statemgmt/state_load_test.go b/bundle/statemgmt/state_load_test.go index 4ec77578d2b..d43f9cac7df 100644 --- a/bundle/statemgmt/state_load_test.go +++ b/bundle/statemgmt/state_load_test.go @@ -60,6 +60,7 @@ func TestStateToBundleEmptyLocalResources(t *testing.T) { "resources.postgres_roles.test_postgres_role": {ID: "projects/test-project/branches/main/roles/test-role"}, "resources.postgres_synced_tables.test_postgres_synced_table": {ID: "synced_tables/main.public.test_synced_table"}, "resources.postgres_snapshot_schedules.test_postgres_snapshot_schedule": {ID: "projects/test-project/branches/main/snapshot-schedule"}, + "resources.postgres_data_apis.test_postgres_data_api": {ID: "projects/test-project/branches/main/databases/test-db/data-api"}, "resources.vector_search_endpoints.test_vector_search_endpoint": {ID: "vs-endpoint-1"}, "resources.vector_search_indexes.test_vector_search_index": {ID: "vs-index-1"}, "resources.instance_pools.test_instance_pool": {ID: "1"}, @@ -154,6 +155,9 @@ func TestStateToBundleEmptyLocalResources(t *testing.T) { assert.Equal(t, "projects/test-project/branches/main/snapshot-schedule", config.Resources.PostgresSnapshotSchedules["test_postgres_snapshot_schedule"].ID) assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.PostgresSnapshotSchedules["test_postgres_snapshot_schedule"].ModifiedStatus) + assert.Equal(t, "projects/test-project/branches/main/databases/test-db/data-api", config.Resources.PostgresDataApis["test_postgres_data_api"].ID) + assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.PostgresDataApis["test_postgres_data_api"].ModifiedStatus) + assert.Equal(t, "vs-endpoint-1", config.Resources.VectorSearchEndpoints["test_vector_search_endpoint"].ID) assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.VectorSearchEndpoints["test_vector_search_endpoint"].ModifiedStatus) @@ -415,6 +419,13 @@ func TestStateToBundleEmptyRemoteResources(t *testing.T) { }, }, }, + PostgresDataApis: map[string]*resources.PostgresDataApi{ + "test_postgres_data_api": { + PostgresDataApiConfig: resources.PostgresDataApiConfig{ + Parent: "projects/test-project/branches/main/databases/test-db", + }, + }, + }, VectorSearchEndpoints: map[string]*resources.VectorSearchEndpoint{ "test_vector_search_endpoint": { CreateEndpoint: vectorsearch.CreateEndpoint{ @@ -566,6 +577,9 @@ func TestStateToBundleEmptyRemoteResources(t *testing.T) { assert.Empty(t, config.Resources.PostgresSnapshotSchedules["test_postgres_snapshot_schedule"].ID) assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.PostgresSnapshotSchedules["test_postgres_snapshot_schedule"].ModifiedStatus) + assert.Empty(t, config.Resources.PostgresDataApis["test_postgres_data_api"].ID) + assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.PostgresDataApis["test_postgres_data_api"].ModifiedStatus) + assert.Empty(t, config.Resources.VectorSearchEndpoints["test_vector_search_endpoint"].ID) assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.VectorSearchEndpoints["test_vector_search_endpoint"].ModifiedStatus) @@ -976,6 +990,13 @@ func TestStateToBundleModifiedResources(t *testing.T) { }, }, }, + PostgresDataApis: map[string]*resources.PostgresDataApi{ + "test_postgres_data_api": { + PostgresDataApiConfig: resources.PostgresDataApiConfig{ + Parent: "projects/test-project/branches/main/databases/test-db", + }, + }, + }, VectorSearchEndpoints: map[string]*resources.VectorSearchEndpoint{ "test_vector_search_endpoint": { CreateEndpoint: vectorsearch.CreateEndpoint{ diff --git a/libs/testserver/bundledeployments.go b/libs/testserver/bundledeployments.go index d5c190750c9..c8d158f0b1d 100644 --- a/libs/testserver/bundledeployments.go +++ b/libs/testserver/bundledeployments.go @@ -63,6 +63,7 @@ var dmsResourceTypes = []string{ "pipelines", "postgres_branches", "postgres_catalogs", + "postgres_data_apis", "postgres_databases", "postgres_endpoints", "postgres_projects", diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index 78234c5ebfa..16f4ee75260 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -265,6 +265,7 @@ type FakeWorkspace struct { PostgresRoles map[string]postgres.Role PostgresSyncedTables map[string]postgres.SyncedTable PostgresSnapshotSchedules map[string]postgres.SnapshotSchedule + PostgresDataApis map[string]postgres.DataApi PostgresOperations map[string]postgres.Operation // Branches and endpoints that the server provisioned implicitly together @@ -545,6 +546,7 @@ func NewFakeWorkspace(url, token string) *FakeWorkspace { PostgresRoles: map[string]postgres.Role{}, PostgresSyncedTables: map[string]postgres.SyncedTable{}, PostgresSnapshotSchedules: map[string]postgres.SnapshotSchedule{}, + PostgresDataApis: map[string]postgres.DataApi{}, PostgresOperations: map[string]postgres.Operation{}, postgresImplicitBranches: map[string]bool{}, postgresImplicitEndpoints: map[string]bool{}, diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index e1a14b9800b..e8ecdd45141 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -1396,6 +1396,11 @@ func AddDefaultHandlers(server *Server) { return req.Workspace.PostgresOperationGet(name) }) + server.Handle("GET", "/api/2.0/postgres/projects/{project_id}/branches/{branch_id}/databases/{database_id}/data-api/operations/{operation_id}", func(req Request) any { + name := "projects/" + req.Vars["project_id"] + "/branches/" + req.Vars["branch_id"] + "/databases/" + req.Vars["database_id"] + "/data-api/operations/" + req.Vars["operation_id"] + return req.Workspace.PostgresOperationGet(name) + }) + server.Handle("GET", "/api/2.0/postgres/projects/{project_id}/branches/{branch_id}/roles/{role_id}/operations/{operation_id}", func(req Request) any { name := "projects/" + req.Vars["project_id"] + "/branches/" + req.Vars["branch_id"] + "/roles/" + req.Vars["role_id"] + "/operations/" + req.Vars["operation_id"] return req.Workspace.PostgresOperationGet(name) @@ -1547,6 +1552,27 @@ func AddDefaultHandlers(server *Server) { return req.Workspace.PostgresDatabaseDelete(name) }) + // Postgres Data API (a per-database singleton): + server.Handle("POST", "/api/2.0/postgres/projects/{project_id}/branches/{branch_id}/databases/{database_id}/data-api", func(req Request) any { + parent := "projects/" + req.Vars["project_id"] + "/branches/" + req.Vars["branch_id"] + "/databases/" + req.Vars["database_id"] + return req.Workspace.PostgresDataApiCreate(req, parent) + }) + + server.Handle("GET", "/api/2.0/postgres/projects/{project_id}/branches/{branch_id}/databases/{database_id}/data-api", func(req Request) any { + name := "projects/" + req.Vars["project_id"] + "/branches/" + req.Vars["branch_id"] + "/databases/" + req.Vars["database_id"] + "/data-api" + return req.Workspace.PostgresDataApiGet(name) + }) + + server.Handle("PATCH", "/api/2.0/postgres/projects/{project_id}/branches/{branch_id}/databases/{database_id}/data-api", func(req Request) any { + name := "projects/" + req.Vars["project_id"] + "/branches/" + req.Vars["branch_id"] + "/databases/" + req.Vars["database_id"] + "/data-api" + return req.Workspace.PostgresDataApiUpdate(req, name) + }) + + server.Handle("DELETE", "/api/2.0/postgres/projects/{project_id}/branches/{branch_id}/databases/{database_id}/data-api", func(req Request) any { + name := "projects/" + req.Vars["project_id"] + "/branches/" + req.Vars["branch_id"] + "/databases/" + req.Vars["database_id"] + "/data-api" + return req.Workspace.PostgresDataApiDelete(name) + }) + // Postgres Roles: server.Handle("POST", "/api/2.0/postgres/projects/{project_id}/branches/{branch_id}/roles", func(req Request) any { parent := "projects/" + req.Vars["project_id"] + "/branches/" + req.Vars["branch_id"] diff --git a/libs/testserver/postgres.go b/libs/testserver/postgres.go index 2f59389aeb1..6801395296d 100644 --- a/libs/testserver/postgres.go +++ b/libs/testserver/postgres.go @@ -5,6 +5,7 @@ import ( "encoding/hex" "encoding/json" "fmt" + "reflect" "regexp" "slices" "strings" @@ -1360,6 +1361,195 @@ func (s *FakeWorkspace) PostgresDatabaseDelete(name string) Response { } delete(s.PostgresDatabases, name) + // The Data API is a sub-resource of the database and goes away with it. + delete(s.PostgresDataApis, name+dataApiSuffix) + + return Response{ + Body: s.createOperationLocked(name, nil), + } +} + +// dataApiSuffix is the final segment of a Data API's resource name: +// "projects/{project_id}/branches/{branch_id}/databases/{database_id}/data-api". +const dataApiSuffix = "/data-api" + +// dataApiUpdateMaskPaths is the update_mask the Data API is known to accept: the +// whole spec, which worked against a real workspace in July 2026. Leaf paths are +// unverified, so the fake rejects them. +var dataApiUpdateMaskPaths = []string{"spec"} + +// hasReadWriteEndpointLocked reports whether a branch has a read-write endpoint +// (caller must hold lock). +func (s *FakeWorkspace) hasReadWriteEndpointLocked(branchName string) bool { + prefix := branchName + "/endpoints/" + for name, endpoint := range s.PostgresEndpoints { + if strings.HasPrefix(name, prefix) && endpoint.Status != nil && endpoint.Status.EndpointType == postgres.EndpointTypeEndpointTypeReadWrite { + return true + } + } + return false +} + +// dataApiPreconditionLocked returns the error the API answers a Data API create, +// get, or update with when its database is missing (404) or the database's +// branch has no read-write endpoint (400), which the backend requires to serve +// the Data API. Returns nil when the request may proceed (caller must hold lock). +func (s *FakeWorkspace) dataApiPreconditionLocked(databaseName string) *Response { + // Project and branch deletion do not cascade to databases in this fake, so + // check the ancestors first, as PostgresDatabaseGet does. + branchName, _, _ := strings.Cut(databaseName, "/databases/") + projectName, _, _ := strings.Cut(branchName, "/branches/") + if _, exists := s.PostgresProjects[projectName]; !exists { + resp := postgresNotFoundResponse("project") + return &resp + } + if _, exists := s.PostgresBranches[branchName]; !exists { + resp := postgresNotFoundResponse("branch") + return &resp + } + if _, exists := s.PostgresDatabases[databaseName]; !exists { + resp := postgresNotFoundResponse("database") + return &resp + } + if !s.hasReadWriteEndpointLocked(branchName) { + resp := postgresErrorResponse(400, "BAD_REQUEST", "Data API requires a read-write endpoint on branch "+branchName) + return &resp + } + return nil +} + +// dataApiStatusFromSpec mirrors the spec onto Status, the way the real API +// reports the configuration it applied while leaving Spec=nil on GET. +func dataApiStatusFromSpec(spec *postgres.DataApiDataApiSpec) *postgres.DataApiDataApiStatus { + status := &postgres.DataApiDataApiStatus{ + AvailableSchemas: []string{"public"}, + } + if spec == nil { + return status + } + status.DbAggregatesEnabled = spec.DbAggregatesEnabled + status.DbExtraSearchPath = spec.DbExtraSearchPath + status.DbMaxRows = spec.DbMaxRows + status.DbSchemas = spec.DbSchemas + status.JwtCacheMaxLifetime = spec.JwtCacheMaxLifetime + status.JwtRoleClaimKey = spec.JwtRoleClaimKey + status.OpenapiMode = spec.OpenapiMode + status.ServerCorsAllowedOrigins = spec.ServerCorsAllowedOrigins + status.ServerTimingEnabled = spec.ServerTimingEnabled + return status +} + +// PostgresDataApiCreate enables the Data API on a database. At most one Data API +// exists per database, so a second create returns ALREADY_EXISTS. +func (s *FakeWorkspace) PostgresDataApiCreate(req Request, parent string) Response { + defer s.LockUnlock()() + + if errResp := s.dataApiPreconditionLocked(parent); errResp != nil { + return *errResp + } + + name := parent + dataApiSuffix + if _, exists := s.PostgresDataApis[name]; exists { + return postgresErrorResponse(409, "ALREADY_EXISTS", "Data API is already enabled for database "+parent) + } + + var dataApi postgres.DataApi + if len(req.Body) > 0 { + if err := json.Unmarshal(req.Body, &dataApi); err != nil { + return Response{ + StatusCode: 400, + Body: fmt.Sprintf("cannot unmarshal request body: %v", err), + } + } + } + + // The real API rejects enabling the Data API with an empty spec. A field sent + // explicitly with its zero value is recorded in ForceSendFields, so it counts. + if dataApi.Spec == nil || reflect.ValueOf(*dataApi.Spec).IsZero() { + return postgresErrorResponse(400, "INVALID_PARAMETER_VALUE", "Field 'data_api.spec' must have at least one subfield with a non-default value") + } + + now := nowTime() + dataApi.Name = name + dataApi.Parent = parent + dataApi.CreateTime = now + dataApi.UpdateTime = now + dataApi.Status = dataApiStatusFromSpec(dataApi.Spec) + // spec is INPUT_ONLY: the API only returns status. + dataApi.Spec = nil + + s.PostgresDataApis[name] = dataApi + + return Response{ + Body: s.createOperationLocked(name, dataApi), + } +} + +// PostgresDataApiGet retrieves the Data API of a database. +func (s *FakeWorkspace) PostgresDataApiGet(name string) Response { + defer s.LockUnlock()() + + if errResp := s.dataApiPreconditionLocked(strings.TrimSuffix(name, dataApiSuffix)); errResp != nil { + return *errResp + } + + dataApi, exists := s.PostgresDataApis[name] + if !exists { + return postgresNotFoundResponse("data api") + } + + return Response{ + Body: dataApi, + } +} + +// PostgresDataApiUpdate replaces the Data API configuration of a database. +func (s *FakeWorkspace) PostgresDataApiUpdate(req Request, name string) Response { + if resp := validateUpdateMask(req, dataApiUpdateMaskPaths, nil); resp != nil { + return *resp + } + + defer s.LockUnlock()() + + if errResp := s.dataApiPreconditionLocked(strings.TrimSuffix(name, dataApiSuffix)); errResp != nil { + return *errResp + } + + dataApi, exists := s.PostgresDataApis[name] + if !exists { + return postgresNotFoundResponse("data api") + } + + var updateDataApi postgres.DataApi + if len(req.Body) > 0 { + if err := json.Unmarshal(req.Body, &updateDataApi); err != nil { + return Response{ + StatusCode: 400, + Body: fmt.Sprintf("cannot unmarshal request body: %v", err), + } + } + } + + // "spec" is the only accepted mask path, so the whole configuration is replaced. + dataApi.Status = dataApiStatusFromSpec(updateDataApi.Spec) + dataApi.UpdateTime = nowTime() + s.PostgresDataApis[name] = dataApi + + return Response{ + Body: s.createOperationLocked(name, dataApi), + } +} + +// PostgresDataApiDelete disables the Data API of a database. The database and +// its data are not affected. +func (s *FakeWorkspace) PostgresDataApiDelete(name string) Response { + defer s.LockUnlock()() + + if _, exists := s.PostgresDataApis[name]; !exists { + return postgresNotFoundResponse("data api") + } + + delete(s.PostgresDataApis, name) return Response{ Body: s.createOperationLocked(name, nil), @@ -1740,6 +1930,8 @@ func (s *FakeWorkspace) createOperationLocked(resourceName string, response any) resourceType = "SyncedTable" case strings.HasSuffix(resourceName, "/snapshot-schedule"): resourceType = "SnapshotSchedule" + case strings.HasSuffix(resourceName, dataApiSuffix): + resourceType = "DataApi" case strings.Contains(resourceName, "/endpoints/"): resourceType = "Endpoint" case strings.Contains(resourceName, "/databases/"): diff --git a/libs/testserver/postgres_data_api_test.go b/libs/testserver/postgres_data_api_test.go new file mode 100644 index 00000000000..1c2cdde2878 --- /dev/null +++ b/libs/testserver/postgres_data_api_test.go @@ -0,0 +1,104 @@ +package testserver + +import ( + "net/url" + "testing" + + "github.com/databricks/databricks-sdk-go/service/postgres" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + testDataApiBranch = "projects/p/branches/production" + testDataApiDatabase = testDataApiBranch + "/databases/db" + testDataApiName = testDataApiDatabase + "/data-api" +) + +// dataApiWorkspace returns a workspace with a database on the project's root +// branch, which comes with a read-write endpoint. +func dataApiWorkspace(t *testing.T) *FakeWorkspace { + workspace := NewFakeWorkspace("http://test.test", "token") + resp := workspace.PostgresProjectCreate(Request{}, "p") + require.Equal(t, 0, resp.StatusCode) + resp = workspace.PostgresDatabaseCreate(Request{Body: []byte(`{"spec": {"postgres_database": "app_db", "role": "` + testDataApiBranch + `/roles/owner"}}`)}, testDataApiBranch, "db", false) + require.Equal(t, 0, resp.StatusCode) + return workspace +} + +func updateDataApi(workspace *FakeWorkspace, mask, body string) Response { + return workspace.PostgresDataApiUpdate(Request{ + URL: &url.URL{RawQuery: url.Values{"update_mask": {mask}}.Encode()}, + Body: []byte(body), + }, testDataApiName) +} + +func TestPostgresDataApiCreateGetUpdateDelete(t *testing.T) { + workspace := dataApiWorkspace(t) + + resp := workspace.PostgresDataApiCreate(Request{Body: []byte(`{"spec": {"db_schemas": ["public"], "db_max_rows": 100}}`)}, testDataApiDatabase) + require.Equal(t, 0, resp.StatusCode) + + resp = workspace.PostgresDataApiGet(testDataApiName) + require.Equal(t, 0, resp.StatusCode) + dataApi := resp.Body.(postgres.DataApi) + assert.Equal(t, testDataApiName, dataApi.Name) + assert.Equal(t, testDataApiDatabase, dataApi.Parent) + assert.Nil(t, dataApi.Spec) + require.NotNil(t, dataApi.Status) + assert.Equal(t, []string{"public"}, dataApi.Status.DbSchemas) + assert.Equal(t, 100, dataApi.Status.DbMaxRows) + + // The "spec" mask replaces the whole configuration, so db_max_rows is reset. + resp = updateDataApi(workspace, "spec", `{"spec": {"db_schemas": ["api"]}}`) + require.Equal(t, 0, resp.StatusCode) + dataApi = workspace.PostgresDataApis[testDataApiName] + assert.Equal(t, []string{"api"}, dataApi.Status.DbSchemas) + assert.Equal(t, 0, dataApi.Status.DbMaxRows) + + resp = workspace.PostgresDataApiDelete(testDataApiName) + require.Equal(t, 0, resp.StatusCode) + assert.Equal(t, 404, workspace.PostgresDataApiGet(testDataApiName).StatusCode) + assert.Equal(t, 404, workspace.PostgresDataApiDelete(testDataApiName).StatusCode) +} + +func TestPostgresDataApiCreateErrors(t *testing.T) { + workspace := dataApiWorkspace(t) + spec := Request{Body: []byte(`{"spec": {"db_schemas": ["public"]}}`)} + + resp := workspace.PostgresDataApiCreate(Request{Body: []byte(`{"spec": {}}`)}, testDataApiDatabase) + assert.Equal(t, 400, resp.StatusCode) + + resp = workspace.PostgresDataApiCreate(spec, testDataApiBranch+"/databases/missing") + assert.Equal(t, 404, resp.StatusCode) + + require.Equal(t, 0, workspace.PostgresDataApiCreate(spec, testDataApiDatabase).StatusCode) + resp = workspace.PostgresDataApiCreate(spec, testDataApiDatabase) + assert.Equal(t, 409, resp.StatusCode) +} + +func TestPostgresDataApiRequiresReadWriteEndpoint(t *testing.T) { + workspace := dataApiWorkspace(t) + delete(workspace.PostgresEndpoints, testDataApiBranch+"/endpoints/primary") + + resp := workspace.PostgresDataApiCreate(Request{Body: []byte(`{"spec": {"db_schemas": ["public"]}}`)}, testDataApiDatabase) + assert.Equal(t, 400, resp.StatusCode) + assert.Equal(t, 400, workspace.PostgresDataApiGet(testDataApiName).StatusCode) +} + +func TestPostgresDataApiUpdateRejectsLeafMask(t *testing.T) { + workspace := dataApiWorkspace(t) + require.Equal(t, 0, workspace.PostgresDataApiCreate(Request{Body: []byte(`{"spec": {"db_schemas": ["public"]}}`)}, testDataApiDatabase).StatusCode) + + resp := updateDataApi(workspace, "spec.db_max_rows", `{"spec": {"db_max_rows": 5}}`) + assert.Equal(t, 400, resp.StatusCode) +} + +func TestPostgresDataApiGoneWithDatabase(t *testing.T) { + workspace := dataApiWorkspace(t) + require.Equal(t, 0, workspace.PostgresDataApiCreate(Request{Body: []byte(`{"spec": {"db_schemas": ["public"]}}`)}, testDataApiDatabase).StatusCode) + + require.Equal(t, 0, workspace.PostgresDatabaseDelete(testDataApiDatabase).StatusCode) + assert.NotContains(t, workspace.PostgresDataApis, testDataApiName) + assert.Equal(t, 404, workspace.PostgresDataApiGet(testDataApiName).StatusCode) +} From a20f8653a4697d444ed6b464a8c9686494081e6b Mon Sep 17 00:00:00 2001 From: Ruslan Talpa Date: Tue, 20 Oct 2026 12:18:36 +0000 Subject: [PATCH 02/12] Match the testserver Data API fake to the real API Observed against a real workspace: - status reports the configuration in effect, server defaults included: db_aggregates_enabled=true, db_schemas=["public"], jwt_role_claim_key=".sub", openapi_mode=OPEN_API_MODE_DISABLED and server_cors_allowed_origins=["none"]. A bool that was never set and is false is omitted; once set it is reported, false included. - update_mask=spec keeps the current value of a field the request omits; it does not reset it to the default. - Errors: 400 INVALID_PARAMETER_VALUE "Field 'data_api' is required and must contain at least one subfield with a non-default value!" for an empty spec, 409 ALREADY_EXISTS "Data API is already enabled for this database", and 404 NOT_FOUND "Data API is not enabled for this database" for get, update and delete when it is not enabled. The acceptance tests now also print server_cors_allowed_origins, and the update test shows that a field removed from the config keeps its value. Co-authored-by: Isaac --- .../bind/postgres_data_api/output.txt | 6 +- .../postgres_data_apis/basic/output.txt | 7 +- .../postgres_data_apis/out-of-band/output.txt | 15 ++- .../postgres_data_apis/recreate/output.txt | 12 +- .../postgres_data_apis/script.prepare | 6 +- .../postgres_data_apis/update/output.txt | 22 +++- .../postgres_data_apis/update/script | 3 +- libs/testserver/postgres.go | 111 +++++++++++++----- libs/testserver/postgres_data_api_test.go | 46 +++++++- 9 files changed, 176 insertions(+), 52 deletions(-) diff --git a/acceptance/bundle/deployment/bind/postgres_data_api/output.txt b/acceptance/bundle/deployment/bind/postgres_data_api/output.txt index 7a8260aedf3..08a537a76d7 100644 --- a/acceptance/bundle/deployment/bind/postgres_data_api/output.txt +++ b/acceptance/bundle/deployment/bind/postgres_data_api/output.txt @@ -3,12 +3,12 @@ === Deploy fails because the Data API is already enabled >>> musterr [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... -Error: cannot create resources.postgres_data_apis.data_api1: Data API is already enabled for database projects/test-project/branches/production/databases/test-database (409 ALREADY_EXISTS) +Error: cannot create resources.postgres_data_apis.data_api1: Data API is already enabled for this database (409 ALREADY_EXISTS) Endpoint: POST [DATABRICKS_URL]/api/2.0/postgres/projects/test-project/branches/production/databases/test-database/data-api HTTP Status: 409 Conflict API error_code: ALREADY_EXISTS -API message: Data API is already enabled for database projects/test-project/branches/production/databases/test-database +API message: Data API is already enabled for this database Files: 2 uploaded, 0 deleted @@ -83,7 +83,7 @@ All files and directories at the following location will be deleted: /Workspace/ Destroy: 1 deleted >>> musterr [CLI] postgres get-data-api projects/test-project/branches/production/databases/test-database/data-api -Error: data api id not found [TraceId: [UUID]] +Error: Data API is not enabled for this database >>> [CLI] postgres get-database projects/test-project/branches/production/databases/test-database { diff --git a/acceptance/bundle/resources/postgres_data_apis/basic/output.txt b/acceptance/bundle/resources/postgres_data_apis/basic/output.txt index 4cb0296ee70..17e51e228f9 100644 --- a/acceptance/bundle/resources/postgres_data_apis/basic/output.txt +++ b/acceptance/bundle/resources/postgres_data_apis/basic/output.txt @@ -54,7 +54,10 @@ Resources: 4 created, 0 changed, 0 deleted, 0 unchanged "db_schemas": [ "public" ], - "db_max_rows": 100 + "db_max_rows": 100, + "server_cors_allowed_origins": [ + "none" + ] } } @@ -95,7 +98,7 @@ Files: 0 uploaded, 0 deleted Resources: 0 created, 0 changed, 1 deleted, 3 unchanged >>> musterr [CLI] postgres get-data-api projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/my-database/data-api -Error: data api id not found [TraceId: [UUID]] +Error: Data API is not enabled for this database >>> [CLI] postgres get-database projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/my-database { diff --git a/acceptance/bundle/resources/postgres_data_apis/out-of-band/output.txt b/acceptance/bundle/resources/postgres_data_apis/out-of-band/output.txt index 7a6716be96f..3543918abbb 100644 --- a/acceptance/bundle/resources/postgres_data_apis/out-of-band/output.txt +++ b/acceptance/bundle/resources/postgres_data_apis/out-of-band/output.txt @@ -17,7 +17,10 @@ Resources: 4 created, 0 changed, 0 deleted, 0 unchanged "db_schemas": [ "public" ], - "db_max_rows": 100 + "db_max_rows": 100, + "server_cors_allowed_origins": [ + "none" + ] } } @@ -30,7 +33,10 @@ Resources: 4 created, 0 changed, 0 deleted, 0 unchanged "db_schemas": [ "public" ], - "db_max_rows": 7 + "db_max_rows": 7, + "server_cors_allowed_origins": [ + "none" + ] } } @@ -61,7 +67,10 @@ Resources: 1 created, 0 changed, 0 deleted, 3 unchanged "db_schemas": [ "public" ], - "db_max_rows": 100 + "db_max_rows": 100, + "server_cors_allowed_origins": [ + "none" + ] } } diff --git a/acceptance/bundle/resources/postgres_data_apis/recreate/output.txt b/acceptance/bundle/resources/postgres_data_apis/recreate/output.txt index ed761507f8a..0e3258e4ae8 100644 --- a/acceptance/bundle/resources/postgres_data_apis/recreate/output.txt +++ b/acceptance/bundle/resources/postgres_data_apis/recreate/output.txt @@ -18,7 +18,10 @@ Resources: 5 created, 0 changed, 0 deleted, 0 unchanged "db_schemas": [ "public" ], - "db_max_rows": 100 + "db_max_rows": 100, + "server_cors_allowed_origins": [ + "none" + ] } } @@ -37,7 +40,7 @@ Files: 0 uploaded, 0 deleted Resources: 1 created, 0 changed, 1 deleted, 4 unchanged >>> musterr [CLI] postgres get-data-api projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/first-database/data-api -Error: data api id not found [TraceId: [UUID]] +Error: Data API is not enabled for this database >>> [CLI] postgres get-data-api projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/second-database/data-api { @@ -47,7 +50,10 @@ Error: data api id not found [TraceId: [UUID]] "db_schemas": [ "public" ], - "db_max_rows": 100 + "db_max_rows": 100, + "server_cors_allowed_origins": [ + "none" + ] } } diff --git a/acceptance/bundle/resources/postgres_data_apis/script.prepare b/acceptance/bundle/resources/postgres_data_apis/script.prepare index 3c715e24166..b965bc2df9d 100644 --- a/acceptance/bundle/resources/postgres_data_apis/script.prepare +++ b/acceptance/bundle/resources/postgres_data_apis/script.prepare @@ -1,9 +1,9 @@ # data_api_fields is the allow-list of Data API fields the acceptance tests # assert on. Anything the backend returns outside this list is dropped, so a new -# field in the API response does not break the golden. The status echoes only -# the spec fields the tests set explicitly. +# field in the API response does not break the golden. The status reports the +# configuration in effect, server defaults included. data_api_fields() { - jq '{name, parent, status: (.status | {db_schemas, db_max_rows})}' + jq '{name, parent, status: (.status | {db_schemas, db_max_rows, server_cors_allowed_origins})}' } # print_postgres_requests writes the recorded postgres write requests to diff --git a/acceptance/bundle/resources/postgres_data_apis/update/output.txt b/acceptance/bundle/resources/postgres_data_apis/update/output.txt index 45a2173f99a..c2aecd05327 100644 --- a/acceptance/bundle/resources/postgres_data_apis/update/output.txt +++ b/acceptance/bundle/resources/postgres_data_apis/update/output.txt @@ -33,11 +33,14 @@ Resources: 0 created, 1 changed, 0 deleted, 3 unchanged "db_schemas": [ "public" ], - "db_max_rows": 500 + "db_max_rows": 500, + "server_cors_allowed_origins": [ + "https://app.example.com" + ] } } -=== Remove a spec field: the whole spec is sent, so the field resets to its server default +=== Remove a spec field: update_mask=spec keeps a field the request omits, so it keeps its last value >>> [CLI] bundle plan update postgres_data_apis.my_data_api @@ -49,6 +52,21 @@ Updated postgres_data_apis.my_data_api Files: 0 uploaded, 0 deleted Resources: 0 created, 1 changed, 0 deleted, 3 unchanged +>>> [CLI] postgres get-data-api projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/my-database/data-api +{ + "name": "projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/my-database/data-api", + "parent": "projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/my-database", + "status": { + "db_schemas": [ + "public" + ], + "db_max_rows": 500, + "server_cors_allowed_origins": [ + "https://app.example.com" + ] + } +} + === Plan after update shows no changes >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 4 unchanged diff --git a/acceptance/bundle/resources/postgres_data_apis/update/script b/acceptance/bundle/resources/postgres_data_apis/update/script index f369066fa79..8403ad4a6a0 100644 --- a/acceptance/bundle/resources/postgres_data_apis/update/script +++ b/acceptance/bundle/resources/postgres_data_apis/update/script @@ -20,12 +20,13 @@ trace $CLI bundle deploy print_postgres_requests update trace $CLI postgres get-data-api "${data_api_name}" | data_api_fields -title "Remove a spec field: the whole spec is sent, so the field resets to its server default" +title "Remove a spec field: update_mask=spec keeps a field the request omits, so it keeps its last value" sed '/server_cors_allowed_origins:/,$d' databricks.yml > databricks.yml.new && mv databricks.yml.new databricks.yml trace $CLI bundle plan rm -f "$OUT_REQUESTS" trace $CLI bundle deploy print_postgres_requests remove_field +trace $CLI postgres get-data-api "${data_api_name}" | data_api_fields title "Plan after update shows no changes" trace $CLI bundle plan diff --git a/libs/testserver/postgres.go b/libs/testserver/postgres.go index 6801395296d..30e6dbfd50d 100644 --- a/libs/testserver/postgres.go +++ b/libs/testserver/postgres.go @@ -1373,11 +1373,22 @@ func (s *FakeWorkspace) PostgresDatabaseDelete(name string) Response { // "projects/{project_id}/branches/{branch_id}/databases/{database_id}/data-api". const dataApiSuffix = "/data-api" -// dataApiUpdateMaskPaths is the update_mask the Data API is known to accept: the -// whole spec, which worked against a real workspace in July 2026. Leaf paths are -// unverified, so the fake rejects them. +// dataApiUpdateMaskPaths is the update_mask the fake accepts: the whole spec, +// which is what the CLI sends. The real API also accepts leaf paths, which reset +// an omitted field instead of keeping it; the CLI never sends them, so the fake +// does not model them. var dataApiUpdateMaskPaths = []string{"spec"} +// dataApiNotEnabledMessage is the message the real API answers a get, update, or +// delete with when the database's Data API is not enabled. +const dataApiNotEnabledMessage = "Data API is not enabled for this database" + +// dataApiNotEnabledResponse is the 404 the real API returns for a Data API that +// is not enabled. +func dataApiNotEnabledResponse() Response { + return postgresErrorResponse(404, "NOT_FOUND", dataApiNotEnabledMessage) +} + // hasReadWriteEndpointLocked reports whether a branch has a read-write endpoint // (caller must hold lock). func (s *FakeWorkspace) hasReadWriteEndpointLocked(branchName string) bool { @@ -1418,25 +1429,67 @@ func (s *FakeWorkspace) dataApiPreconditionLocked(databaseName string) *Response return nil } -// dataApiStatusFromSpec mirrors the spec onto Status, the way the real API -// reports the configuration it applied while leaving Spec=nil on GET. -func dataApiStatusFromSpec(spec *postgres.DataApiDataApiSpec) *postgres.DataApiDataApiStatus { - status := &postgres.DataApiDataApiStatus{ - AvailableSchemas: []string{"public"}, +// dataApiDefaultStatus is the status the real API reports right after enabling +// the Data API with only db_schemas set: the server defaults, which the status +// echoes because it reports every setting in effect. +func dataApiDefaultStatus() *postgres.DataApiDataApiStatus { + return &postgres.DataApiDataApiStatus{ + AvailableSchemas: []string{"public"}, + DbAggregatesEnabled: true, + DbSchemas: []string{"public"}, + JwtRoleClaimKey: ".sub", + OpenapiMode: postgres.OpenApiModeOpenApiModeDisabled, + ServerCorsAllowedOrigins: []string{"none"}, } +} + +// applyDataApiSpec returns the status after applying the fields the spec sets, +// the way the real API applies a create or an update_mask=spec update: a field +// the request omits keeps its current value. +func applyDataApiSpec(current *postgres.DataApiDataApiStatus, spec *postgres.DataApiDataApiSpec) *postgres.DataApiDataApiStatus { + status := *current + status.ForceSendFields = slices.Clone(current.ForceSendFields) if spec == nil { - return status + return &status + } + applyDataApiBool(&status, spec, "DbAggregatesEnabled", &status.DbAggregatesEnabled, spec.DbAggregatesEnabled) + applyDataApiBool(&status, spec, "ServerTimingEnabled", &status.ServerTimingEnabled, spec.ServerTimingEnabled) + if spec.DbExtraSearchPath != nil { + status.DbExtraSearchPath = spec.DbExtraSearchPath + } + if spec.DbMaxRows != 0 { + status.DbMaxRows = spec.DbMaxRows + } + if spec.DbSchemas != nil { + status.DbSchemas = spec.DbSchemas + } + if spec.JwtCacheMaxLifetime != nil { + status.JwtCacheMaxLifetime = spec.JwtCacheMaxLifetime + } + if spec.JwtRoleClaimKey != "" { + status.JwtRoleClaimKey = spec.JwtRoleClaimKey + } + if spec.OpenapiMode != "" { + status.OpenapiMode = spec.OpenapiMode + } + if spec.ServerCorsAllowedOrigins != nil { + status.ServerCorsAllowedOrigins = spec.ServerCorsAllowedOrigins + } + return &status +} + +// applyDataApiBool sets a bool status field when the spec sets it. A false sent +// explicitly is recorded in the spec's ForceSendFields, so it counts as set. Once +// set, the real API reports the bool even when it is false, so the status forces +// it too; a bool that was never set and is false is omitted. +func applyDataApiBool(status *postgres.DataApiDataApiStatus, spec *postgres.DataApiDataApiSpec, field string, dst *bool, value bool) { + if !value && !slices.Contains(spec.ForceSendFields, field) { + return + } + *dst = value + if !slices.Contains(status.ForceSendFields, field) { + status.ForceSendFields = append(status.ForceSendFields, field) } - status.DbAggregatesEnabled = spec.DbAggregatesEnabled - status.DbExtraSearchPath = spec.DbExtraSearchPath - status.DbMaxRows = spec.DbMaxRows - status.DbSchemas = spec.DbSchemas - status.JwtCacheMaxLifetime = spec.JwtCacheMaxLifetime - status.JwtRoleClaimKey = spec.JwtRoleClaimKey - status.OpenapiMode = spec.OpenapiMode - status.ServerCorsAllowedOrigins = spec.ServerCorsAllowedOrigins - status.ServerTimingEnabled = spec.ServerTimingEnabled - return status } // PostgresDataApiCreate enables the Data API on a database. At most one Data API @@ -1450,7 +1503,7 @@ func (s *FakeWorkspace) PostgresDataApiCreate(req Request, parent string) Respon name := parent + dataApiSuffix if _, exists := s.PostgresDataApis[name]; exists { - return postgresErrorResponse(409, "ALREADY_EXISTS", "Data API is already enabled for database "+parent) + return postgresErrorResponse(409, "ALREADY_EXISTS", "Data API is already enabled for this database") } var dataApi postgres.DataApi @@ -1466,7 +1519,7 @@ func (s *FakeWorkspace) PostgresDataApiCreate(req Request, parent string) Respon // The real API rejects enabling the Data API with an empty spec. A field sent // explicitly with its zero value is recorded in ForceSendFields, so it counts. if dataApi.Spec == nil || reflect.ValueOf(*dataApi.Spec).IsZero() { - return postgresErrorResponse(400, "INVALID_PARAMETER_VALUE", "Field 'data_api.spec' must have at least one subfield with a non-default value") + return postgresErrorResponse(400, "INVALID_PARAMETER_VALUE", "Field 'data_api' is required and must contain at least one subfield with a non-default value!") } now := nowTime() @@ -1474,8 +1527,8 @@ func (s *FakeWorkspace) PostgresDataApiCreate(req Request, parent string) Respon dataApi.Parent = parent dataApi.CreateTime = now dataApi.UpdateTime = now - dataApi.Status = dataApiStatusFromSpec(dataApi.Spec) - // spec is INPUT_ONLY: the API only returns status. + dataApi.Status = applyDataApiSpec(dataApiDefaultStatus(), dataApi.Spec) + // spec is INPUT_ONLY: GET only returns status. dataApi.Spec = nil s.PostgresDataApis[name] = dataApi @@ -1495,7 +1548,7 @@ func (s *FakeWorkspace) PostgresDataApiGet(name string) Response { dataApi, exists := s.PostgresDataApis[name] if !exists { - return postgresNotFoundResponse("data api") + return dataApiNotEnabledResponse() } return Response{ @@ -1503,7 +1556,7 @@ func (s *FakeWorkspace) PostgresDataApiGet(name string) Response { } } -// PostgresDataApiUpdate replaces the Data API configuration of a database. +// PostgresDataApiUpdate updates the Data API configuration of a database. func (s *FakeWorkspace) PostgresDataApiUpdate(req Request, name string) Response { if resp := validateUpdateMask(req, dataApiUpdateMaskPaths, nil); resp != nil { return *resp @@ -1517,7 +1570,7 @@ func (s *FakeWorkspace) PostgresDataApiUpdate(req Request, name string) Response dataApi, exists := s.PostgresDataApis[name] if !exists { - return postgresNotFoundResponse("data api") + return dataApiNotEnabledResponse() } var updateDataApi postgres.DataApi @@ -1530,8 +1583,8 @@ func (s *FakeWorkspace) PostgresDataApiUpdate(req Request, name string) Response } } - // "spec" is the only accepted mask path, so the whole configuration is replaced. - dataApi.Status = dataApiStatusFromSpec(updateDataApi.Spec) + // The "spec" mask keeps the current value of a field the request omits. + dataApi.Status = applyDataApiSpec(dataApi.Status, updateDataApi.Spec) dataApi.UpdateTime = nowTime() s.PostgresDataApis[name] = dataApi @@ -1546,7 +1599,7 @@ func (s *FakeWorkspace) PostgresDataApiDelete(name string) Response { defer s.LockUnlock()() if _, exists := s.PostgresDataApis[name]; !exists { - return postgresNotFoundResponse("data api") + return dataApiNotEnabledResponse() } delete(s.PostgresDataApis, name) diff --git a/libs/testserver/postgres_data_api_test.go b/libs/testserver/postgres_data_api_test.go index 1c2cdde2878..3638cb3c3ff 100644 --- a/libs/testserver/postgres_data_api_test.go +++ b/libs/testserver/postgres_data_api_test.go @@ -49,17 +49,51 @@ func TestPostgresDataApiCreateGetUpdateDelete(t *testing.T) { assert.Equal(t, []string{"public"}, dataApi.Status.DbSchemas) assert.Equal(t, 100, dataApi.Status.DbMaxRows) - // The "spec" mask replaces the whole configuration, so db_max_rows is reset. + // The "spec" mask keeps the current value of a field the request omits, so + // db_max_rows stays. resp = updateDataApi(workspace, "spec", `{"spec": {"db_schemas": ["api"]}}`) require.Equal(t, 0, resp.StatusCode) dataApi = workspace.PostgresDataApis[testDataApiName] assert.Equal(t, []string{"api"}, dataApi.Status.DbSchemas) - assert.Equal(t, 0, dataApi.Status.DbMaxRows) + assert.Equal(t, 100, dataApi.Status.DbMaxRows) resp = workspace.PostgresDataApiDelete(testDataApiName) require.Equal(t, 0, resp.StatusCode) - assert.Equal(t, 404, workspace.PostgresDataApiGet(testDataApiName).StatusCode) - assert.Equal(t, 404, workspace.PostgresDataApiDelete(testDataApiName).StatusCode) + assert.Equal(t, dataApiNotEnabledResponse(), workspace.PostgresDataApiGet(testDataApiName)) + assert.Equal(t, dataApiNotEnabledResponse(), workspace.PostgresDataApiDelete(testDataApiName)) +} + +func TestPostgresDataApiStatusReportsDefaults(t *testing.T) { + workspace := dataApiWorkspace(t) + require.Equal(t, 0, workspace.PostgresDataApiCreate(Request{Body: []byte(`{"spec": {"db_max_rows": 10}}`)}, testDataApiDatabase).StatusCode) + + status := workspace.PostgresDataApis[testDataApiName].Status + assert.Equal(t, &postgres.DataApiDataApiStatus{ + AvailableSchemas: []string{"public"}, + DbAggregatesEnabled: true, + DbMaxRows: 10, + DbSchemas: []string{"public"}, + JwtRoleClaimKey: ".sub", + OpenapiMode: postgres.OpenApiModeOpenApiModeDisabled, + ServerCorsAllowedOrigins: []string{"none"}, + }, status) +} + +func TestPostgresDataApiUpdateBools(t *testing.T) { + workspace := dataApiWorkspace(t) + require.Equal(t, 0, workspace.PostgresDataApiCreate(Request{Body: []byte(`{"spec": {"db_schemas": ["public"]}}`)}, testDataApiDatabase).StatusCode) + + // An omitted bool keeps its value; an explicit false is applied and then + // reported, unlike a false that was never set. + require.Equal(t, 0, updateDataApi(workspace, "spec", `{"spec": {"db_schemas": ["public"]}}`).StatusCode) + status := workspace.PostgresDataApis[testDataApiName].Status + assert.True(t, status.DbAggregatesEnabled) + assert.Empty(t, status.ForceSendFields) + + require.Equal(t, 0, updateDataApi(workspace, "spec", `{"spec": {"db_aggregates_enabled": false, "server_timing_enabled": false}}`).StatusCode) + status = workspace.PostgresDataApis[testDataApiName].Status + assert.False(t, status.DbAggregatesEnabled) + assert.ElementsMatch(t, []string{"DbAggregatesEnabled", "ServerTimingEnabled"}, status.ForceSendFields) } func TestPostgresDataApiCreateErrors(t *testing.T) { @@ -67,14 +101,14 @@ func TestPostgresDataApiCreateErrors(t *testing.T) { spec := Request{Body: []byte(`{"spec": {"db_schemas": ["public"]}}`)} resp := workspace.PostgresDataApiCreate(Request{Body: []byte(`{"spec": {}}`)}, testDataApiDatabase) - assert.Equal(t, 400, resp.StatusCode) + assert.Equal(t, postgresErrorResponse(400, "INVALID_PARAMETER_VALUE", "Field 'data_api' is required and must contain at least one subfield with a non-default value!"), resp) resp = workspace.PostgresDataApiCreate(spec, testDataApiBranch+"/databases/missing") assert.Equal(t, 404, resp.StatusCode) require.Equal(t, 0, workspace.PostgresDataApiCreate(spec, testDataApiDatabase).StatusCode) resp = workspace.PostgresDataApiCreate(spec, testDataApiDatabase) - assert.Equal(t, 409, resp.StatusCode) + assert.Equal(t, postgresErrorResponse(409, "ALREADY_EXISTS", "Data API is already enabled for this database"), resp) } func TestPostgresDataApiRequiresReadWriteEndpoint(t *testing.T) { From ffb2f1b85781d148600b70bade05702ac1f81062 Mon Sep 17 00:00:00 2001 From: Ruslan Talpa Date: Tue, 20 Oct 2026 12:19:10 +0000 Subject: [PATCH 03/12] Detect postgres_data_apis spec drift by reading the spec from status GET does not return spec (it is INPUT_ONLY), so the plan could not see a remote spec change, and a bound Data API kept its old configuration until the bundle config changed. status reports every spec field in effect, so DoRead now fills the remote spec from it. The generated spec:input_only rules no longer apply, so apitypes.yml drops the type for postgres_data_apis and the field rules are hand-written: every spec field is a backend_default with no allowed values. A field the config omits is left to the server: create applies the server default, and update_mask=spec keeps the current value of an omitted field, so a field removed from the config keeps its last value. Leaf update masks would reset it, but resetting db_max_rows or jwt_role_claim_key that way leaves the Data API unable to serve requests, so the mask stays at spec. Status omits a false bool that was never set, so the remote spec forces db_aggregates_enabled and server_timing_enabled: an explicit false in the config compares against false, not against an unknown value. With this, binding an existing Data API plans an update that applies the configured spec, and an out-of-band spec change shows up in the plan. Co-authored-by: Isaac --- .../bind/postgres_data_api/output.txt | 9 ++- .../deployment/bind/postgres_data_api/script | 4 +- .../postgres_data_apis/out-of-band/output.txt | 28 +++++++- .../postgres_data_apis/out-of-band/script | 5 +- .../update/out.plan.update.json | 35 +++++----- bundle/direct/dresources/apitypes.yml | 6 +- .../configs/postgres_data_apis.generated.yml | 21 ------ .../dresources/configs/postgres_data_apis.yml | 17 +++++ bundle/direct/dresources/postgres_data_api.go | 46 ++++++++++--- .../dresources/postgres_data_api_test.go | 69 +++++++++++++++++++ 10 files changed, 183 insertions(+), 57 deletions(-) delete mode 100644 bundle/direct/dresources/configs/postgres_data_apis.generated.yml create mode 100644 bundle/direct/dresources/postgres_data_api_test.go diff --git a/acceptance/bundle/deployment/bind/postgres_data_api/output.txt b/acceptance/bundle/deployment/bind/postgres_data_api/output.txt index 08a537a76d7..6d40b177023 100644 --- a/acceptance/bundle/deployment/bind/postgres_data_api/output.txt +++ b/acceptance/bundle/deployment/bind/postgres_data_api/output.txt @@ -29,12 +29,15 @@ Resources: Name: projects/test-project/branches/production/databases/test-database/data-api >>> [CLI] bundle plan -Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged +update postgres_data_apis.data_api1 + +Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Updated postgres_data_apis.data_api1 Files: 0 uploaded, 0 deleted -Resources: 0 created, 0 changed, 0 deleted, 1 unchanged +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged >>> [CLI] postgres get-data-api projects/test-project/branches/production/databases/test-database/data-api { @@ -43,7 +46,7 @@ Resources: 0 created, 0 changed, 0 deleted, 1 unchanged "db_schemas": [ "public" ], - "db_max_rows": null + "db_max_rows": 500 } } diff --git a/acceptance/bundle/deployment/bind/postgres_data_api/script b/acceptance/bundle/deployment/bind/postgres_data_api/script index af860e3508d..bcfe75291a8 100644 --- a/acceptance/bundle/deployment/bind/postgres_data_api/script +++ b/acceptance/bundle/deployment/bind/postgres_data_api/script @@ -10,8 +10,8 @@ title "Deploy fails because the Data API is already enabled" trace musterr $CLI bundle deploy title "Bind adopts the existing Data API" -# Bind records the config as state, and spec is input-only (GET does not return it), so -# the configured spec is not applied until it changes in the config. +# Bind records the config as state. The plan compares it with the spec read back +# from status, so the configured db_max_rows is applied on the next deploy. trace $CLI bundle deployment bind data_api1 "${DATA_API_NAME}" --auto-approve trace $CLI bundle summary trace $CLI bundle plan diff --git a/acceptance/bundle/resources/postgres_data_apis/out-of-band/output.txt b/acceptance/bundle/resources/postgres_data_apis/out-of-band/output.txt index 3543918abbb..a5c18b4ffd6 100644 --- a/acceptance/bundle/resources/postgres_data_apis/out-of-band/output.txt +++ b/acceptance/bundle/resources/postgres_data_apis/out-of-band/output.txt @@ -40,7 +40,33 @@ Resources: 4 created, 0 changed, 0 deleted, 0 unchanged } } -=== Plan does not detect the change: spec is input-only and GET does not return it +=== Plan detects the change from status, and deploy restores the config +>>> [CLI] bundle plan +update postgres_data_apis.my_data_api + +Plan: 0 to add, 1 to change, 0 to delete, 3 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/oob-pg-data-api-[UNIQUE_NAME]/default/files... +Updated postgres_data_apis.my_data_api +Files: 0 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 3 unchanged + +>>> [CLI] postgres get-data-api projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/my-database/data-api +{ + "name": "projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/my-database/data-api", + "parent": "projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/my-database", + "status": { + "db_schemas": [ + "public" + ], + "db_max_rows": 100, + "server_cors_allowed_origins": [ + "none" + ] + } +} + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 4 unchanged diff --git a/acceptance/bundle/resources/postgres_data_apis/out-of-band/script b/acceptance/bundle/resources/postgres_data_apis/out-of-band/script index e665ad8a95e..ab1d205a1b7 100644 --- a/acceptance/bundle/resources/postgres_data_apis/out-of-band/script +++ b/acceptance/bundle/resources/postgres_data_apis/out-of-band/script @@ -14,7 +14,10 @@ trace $CLI postgres get-data-api "${data_api_name}" | data_api_fields title "Change the spec out of band" trace $CLI postgres update-data-api "${data_api_name}" spec --json '{"spec": {"db_schemas": ["public"], "db_max_rows": 7}}' | data_api_fields -title "Plan does not detect the change: spec is input-only and GET does not return it" +title "Plan detects the change from status, and deploy restores the config" +trace $CLI bundle plan +trace $CLI bundle deploy +trace $CLI postgres get-data-api "${data_api_name}" | data_api_fields trace $CLI bundle plan title "Disable the Data API out of band" diff --git a/acceptance/bundle/resources/postgres_data_apis/update/out.plan.update.json b/acceptance/bundle/resources/postgres_data_apis/update/out.plan.update.json index 0b009da0c1c..66214332aee 100644 --- a/acceptance/bundle/resources/postgres_data_apis/update/out.plan.update.json +++ b/acceptance/bundle/resources/postgres_data_apis/update/out.plan.update.json @@ -1,30 +1,31 @@ { "action": "update", "changes": { + "db_aggregates_enabled": { + "action": "skip", + "reason": "backend_default", + "remote": true + }, "db_max_rows": { "action": "update", "old": 100, - "new": 500 + "new": 500, + "remote": 100 + }, + "jwt_role_claim_key": { + "action": "skip", + "reason": "backend_default", + "remote": ".sub" }, - "db_schemas": { + "openapi_mode": { "action": "skip", - "reason": "spec:input_only", - "old": [ - "public" - ], - "new": [ - "public" - ] + "reason": "backend_default", + "remote": "OPEN_API_MODE_DISABLED" }, - "server_cors_allowed_origins": { + "server_timing_enabled": { "action": "skip", - "reason": "spec:input_only", - "old": [ - "https://app.example.com" - ], - "new": [ - "https://app.example.com" - ] + "reason": "empty", + "remote": false } } } diff --git a/bundle/direct/dresources/apitypes.yml b/bundle/direct/dresources/apitypes.yml index 325754faca3..7832c70e58f 100644 --- a/bundle/direct/dresources/apitypes.yml +++ b/bundle/direct/dresources/apitypes.yml @@ -31,4 +31,8 @@ postgres_synced_tables: postgres.SyncedTableSyncedTableSpec postgres_roles: postgres.RoleRoleSpec -postgres_data_apis: postgres.DataApiDataApiSpec +# No generated rules for postgres_data_apis. The API marks spec INPUT_ONLY, which +# would generate ignore_remote_changes for every spec field, but DoRead fills +# the spec from status, which echoes the configuration in effect. The field +# rules are hand-written in configs/postgres_data_apis.yml instead. +postgres_data_apis: null diff --git a/bundle/direct/dresources/configs/postgres_data_apis.generated.yml b/bundle/direct/dresources/configs/postgres_data_apis.generated.yml deleted file mode 100644 index 5a94ec39e45..00000000000 --- a/bundle/direct/dresources/configs/postgres_data_apis.generated.yml +++ /dev/null @@ -1,21 +0,0 @@ -# Generated, do not edit. - -ignore_remote_changes: - - field: db_aggregates_enabled - reason: spec:input_only - - field: db_extra_search_path - reason: spec:input_only - - field: db_max_rows - reason: spec:input_only - - field: db_schemas - reason: spec:input_only - - field: jwt_cache_max_lifetime - reason: spec:input_only - - field: jwt_role_claim_key - reason: spec:input_only - - field: openapi_mode - reason: spec:input_only - - field: server_cors_allowed_origins - reason: spec:input_only - - field: server_timing_enabled - reason: spec:input_only diff --git a/bundle/direct/dresources/configs/postgres_data_apis.yml b/bundle/direct/dresources/configs/postgres_data_apis.yml index 42718ab0d51..661fba45782 100644 --- a/bundle/direct/dresources/configs/postgres_data_apis.yml +++ b/bundle/direct/dresources/configs/postgres_data_apis.yml @@ -8,3 +8,20 @@ stable_output_fields: # name ("{parent}/data-api") is backend-assigned at creation and immutable; # see the stable_output_fields doc in config.go and ES-2202624. - field: name +backend_defaults: + # DoRead fills the spec from status, which reports every setting in effect. + # A field the config omits is left to the server: create applies the server + # default, and update_mask=spec keeps the current value of any field the + # request omits, so a field removed from the config keeps its last value. + # No allowed values are listed because that last value can be anything. + # Server defaults on create: + - field: db_aggregates_enabled # true + - field: db_schemas # ["public"] + - field: jwt_role_claim_key # ".sub" + - field: openapi_mode # OPEN_API_MODE_DISABLED + - field: server_cors_allowed_origins # ["none"] + # Unset by default: + - field: db_extra_search_path + - field: db_max_rows + - field: jwt_cache_max_lifetime + - field: server_timing_enabled diff --git a/bundle/direct/dresources/postgres_data_api.go b/bundle/direct/dresources/postgres_data_api.go index 8803eb5f41e..14f9f0e6066 100644 --- a/bundle/direct/dresources/postgres_data_api.go +++ b/bundle/direct/dresources/postgres_data_api.go @@ -2,6 +2,7 @@ package dresources import ( "context" + "slices" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/databricks-sdk-go" @@ -11,17 +12,22 @@ import ( "github.com/databricks/databricks-sdk-go/service/postgres" ) -// dataApiUpdateMask masks the whole spec. The direct engine sends the full -// desired spec on every update, so a field removed from the config resets to -// its server default. Masking only the changed leaves would instead be rejected -// for a removed field ("is in update_mask but not provided in request"). +// dataApiUpdateMask masks the whole spec. With this mask the API sets every +// field the request carries and keeps the current value of a field it omits, +// so a field removed from the config keeps its last value. Leaf paths would +// reset an omitted field instead, but resetting db_max_rows or +// jwt_role_claim_key that way leaves the Data API unable to serve requests. var dataApiUpdateMask = []string{"spec"} +// dataApiBoolFields are the spec's bool fields. Status omits a bool that is +// false and was never set, so the remote spec forces them: a missing bool in +// status means false, not unknown. +var dataApiBoolFields = []string{"DbAggregatesEnabled", "ServerTimingEnabled"} + // PostgresDataApiRemote is the return type for DoRead. It embeds // DataApiDataApiSpec so that all paths in StateType are valid paths in // RemoteType. GET does not return spec (it is INPUT_ONLY), so the embedded spec -// stays at its zero value and postgres_data_apis.generated.yml suppresses the -// resulting phantom drift via ignore_remote_changes with reason spec:input_only. +// is filled from status, which reports the configuration in effect. type PostgresDataApiRemote struct { postgres.DataApiDataApiSpec @@ -68,12 +74,8 @@ func (*ResourcePostgresDataApi) RemapState(remote *PostgresDataApiRemote) *Postg } func makePostgresDataApiRemote(dataApi *postgres.DataApi) *PostgresDataApiRemote { - var spec postgres.DataApiDataApiSpec - if dataApi.Spec != nil { - spec = *dataApi.Spec - } return &PostgresDataApiRemote{ - DataApiDataApiSpec: spec, + DataApiDataApiSpec: dataApiSpecFromStatus(dataApi.Status), Parent: dataApi.Parent, Name: dataApi.Name, Status: dataApi.Status, @@ -82,6 +84,28 @@ func makePostgresDataApiRemote(dataApi *postgres.DataApi) *PostgresDataApiRemote } } +// dataApiSpecFromStatus returns the spec in effect, as reported by status. The +// spec on the resource cannot be used: GET omits it, and the one in an operation +// response leaves out the server defaults. +func dataApiSpecFromStatus(status *postgres.DataApiDataApiStatus) postgres.DataApiDataApiSpec { + if status == nil { + var spec postgres.DataApiDataApiSpec + return spec + } + return postgres.DataApiDataApiSpec{ + DbAggregatesEnabled: status.DbAggregatesEnabled, + DbExtraSearchPath: status.DbExtraSearchPath, + DbMaxRows: status.DbMaxRows, + DbSchemas: status.DbSchemas, + JwtCacheMaxLifetime: status.JwtCacheMaxLifetime, + JwtRoleClaimKey: status.JwtRoleClaimKey, + OpenapiMode: status.OpenapiMode, + ServerCorsAllowedOrigins: status.ServerCorsAllowedOrigins, + ServerTimingEnabled: status.ServerTimingEnabled, + ForceSendFields: slices.Clone(dataApiBoolFields), + } +} + func (r *ResourcePostgresDataApi) DoRead(ctx context.Context, id string) (*PostgresDataApiRemote, error) { dataApi, err := r.client.Postgres.GetDataApi(ctx, postgres.GetDataApiRequest{Name: id}) if err != nil { diff --git a/bundle/direct/dresources/postgres_data_api_test.go b/bundle/direct/dresources/postgres_data_api_test.go new file mode 100644 index 00000000000..0fa5aa50ba2 --- /dev/null +++ b/bundle/direct/dresources/postgres_data_api_test.go @@ -0,0 +1,69 @@ +package dresources + +import ( + "testing" + "time" + + "github.com/databricks/databricks-sdk-go/common/types/duration" + "github.com/databricks/databricks-sdk-go/service/postgres" + "github.com/stretchr/testify/assert" +) + +func TestDataApiSpecFromStatus(t *testing.T) { + jwtCache := duration.New(5 * time.Minute) + + tests := []struct { + name string + status *postgres.DataApiDataApiStatus + want postgres.DataApiDataApiSpec + }{ + { + name: "no status", + status: nil, + want: postgres.DataApiDataApiSpec{}, + }, + { + name: "every spec field is read back; url and available_schemas are not", + status: &postgres.DataApiDataApiStatus{ + AvailableSchemas: []string{"public", "api"}, + DbAggregatesEnabled: true, + DbExtraSearchPath: []string{"extensions"}, + DbMaxRows: 100, + DbSchemas: []string{"api"}, + JwtCacheMaxLifetime: jwtCache, + JwtRoleClaimKey: ".sub", + OpenapiMode: postgres.OpenApiModeOpenApiModeIgnorePrivileges, + ServerCorsAllowedOrigins: []string{"https://app.example.com"}, + ServerTimingEnabled: true, + Url: "https://ep.example.com/rest/app_db", + }, + want: postgres.DataApiDataApiSpec{ + DbAggregatesEnabled: true, + DbExtraSearchPath: []string{"extensions"}, + DbMaxRows: 100, + DbSchemas: []string{"api"}, + JwtCacheMaxLifetime: jwtCache, + JwtRoleClaimKey: ".sub", + OpenapiMode: postgres.OpenApiModeOpenApiModeIgnorePrivileges, + ServerCorsAllowedOrigins: []string{"https://app.example.com"}, + ServerTimingEnabled: true, + ForceSendFields: []string{"DbAggregatesEnabled", "ServerTimingEnabled"}, + }, + }, + { + // Status omits a false bool that was never set; it still means false. + name: "omitted bools are false", + status: &postgres.DataApiDataApiStatus{DbSchemas: []string{"public"}}, + want: postgres.DataApiDataApiSpec{ + DbSchemas: []string{"public"}, + ForceSendFields: []string{"DbAggregatesEnabled", "ServerTimingEnabled"}, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, dataApiSpecFromStatus(tt.status)) + }) + } +} From 9bff5ee96ef59135e9e69f3ab91b926e3f856083 Mon Sep 17 00:00:00 2001 From: Ruslan Talpa Date: Wed, 21 Oct 2026 06:55:39 +0000 Subject: [PATCH 04/12] Add PR link to postgres_data_apis changelog entry Co-authored-by: Isaac --- .nextchanges/bundles/postgres-data-apis.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.nextchanges/bundles/postgres-data-apis.md b/.nextchanges/bundles/postgres-data-apis.md index b37e8e8dcda..ae114d67572 100644 --- a/.nextchanges/bundles/postgres-data-apis.md +++ b/.nextchanges/bundles/postgres-data-apis.md @@ -1 +1 @@ -* Add the `postgres_data_apis` resource (Private Preview) for enabling and configuring the Data API of a Lakebase Postgres database. +* Add the `postgres_data_apis` resource (Private Preview) for enabling and configuring the Data API of a Lakebase Postgres database. ([#7139](https://github.com/databricks/cli/pull/7139)) From 8870cf1651a7e796e5f3658a255c60bb199e8e90 Mon Sep 17 00:00:00 2001 From: Ruslan Talpa Date: Wed, 21 Oct 2026 08:12:11 +0000 Subject: [PATCH 05/12] Report status.url from the testserver Data API fake The real API reports the serving URL in status.url: https://{read-write endpoint host}/api/2.0/workspace/{workspace_id}/rest/{postgres_database}. The fake now fills it in GET and in the create and update operation responses, from the branch's read-write endpoint host, the fake workspace ID and the database's Postgres name. The new url_reference acceptance test has a job parameter that references ${resources.postgres_data_apis..status.url} and shows it resolves on the first deploy. It also records that an in-place update of the Data API currently plans an update of the job too. Co-authored-by: Isaac --- .../url_reference/databricks.yml.tmpl | 39 ++++++++++ .../url_reference/out.test.toml | 4 ++ .../url_reference/output.txt | 72 +++++++++++++++++++ .../postgres_data_apis/url_reference/script | 28 ++++++++ libs/testserver/postgres.go | 34 ++++++--- libs/testserver/postgres_data_api_test.go | 2 + 6 files changed, 170 insertions(+), 9 deletions(-) create mode 100644 acceptance/bundle/resources/postgres_data_apis/url_reference/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/postgres_data_apis/url_reference/out.test.toml create mode 100644 acceptance/bundle/resources/postgres_data_apis/url_reference/output.txt create mode 100644 acceptance/bundle/resources/postgres_data_apis/url_reference/script diff --git a/acceptance/bundle/resources/postgres_data_apis/url_reference/databricks.yml.tmpl b/acceptance/bundle/resources/postgres_data_apis/url_reference/databricks.yml.tmpl new file mode 100644 index 00000000000..33ea8827068 --- /dev/null +++ b/acceptance/bundle/resources/postgres_data_apis/url_reference/databricks.yml.tmpl @@ -0,0 +1,39 @@ +bundle: + name: url-ref-pg-data-api-$UNIQUE_NAME + +sync: + paths: [] + +resources: + postgres_projects: + my_project: + project_id: test-pg-proj-$UNIQUE_NAME + display_name: "Test Project for Data API URL" + pg_version: 16 + + postgres_roles: + owner: + parent: ${resources.postgres_projects.my_project.id}/branches/production + role_id: app-owner + postgres_role: app_owner + + postgres_databases: + my_database: + parent: ${resources.postgres_projects.my_project.id}/branches/production + database_id: my-database + postgres_database: app_db + role: ${resources.postgres_roles.owner.id} + + postgres_data_apis: + my_data_api: + parent: ${resources.postgres_databases.my_database.name} + db_schemas: + - public + db_max_rows: 100 + + jobs: + my_job: + name: data-api-url-$UNIQUE_NAME + parameters: + - name: data_api_url + default: ${resources.postgres_data_apis.my_data_api.status.url} diff --git a/acceptance/bundle/resources/postgres_data_apis/url_reference/out.test.toml b/acceptance/bundle/resources/postgres_data_apis/url_reference/out.test.toml new file mode 100644 index 00000000000..ee153e82621 --- /dev/null +++ b/acceptance/bundle/resources/postgres_data_apis/url_reference/out.test.toml @@ -0,0 +1,4 @@ +Cloud = true +CloudEnvs.azure = false +CloudEnvs.gcp = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/postgres_data_apis/url_reference/output.txt b/acceptance/bundle/resources/postgres_data_apis/url_reference/output.txt new file mode 100644 index 00000000000..8732cee3708 --- /dev/null +++ b/acceptance/bundle/resources/postgres_data_apis/url_reference/output.txt @@ -0,0 +1,72 @@ + +=== The first deploy resolves status.url into the job parameter +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/url-ref-pg-data-api-[UNIQUE_NAME]/default/files... +Created jobs.my_job +Created postgres_data_apis.my_data_api +Created postgres_databases.my_database +Created postgres_projects.my_project +Created postgres_roles.owner +Files: 0 uploaded, 0 deleted +Resources: 5 created, 0 changed, 0 deleted, 0 unchanged + +>>> [CLI] postgres get-data-api projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/my-database/data-api +{ + "url": "https://[RW_ENDPOINT_HOST]/api/2.0/workspace/[WORKSPACE_ID]/rest/app_db" +} + +>>> [CLI] jobs get [MY_JOB_ID] +[ + { + "default": "https://[RW_ENDPOINT_HOST]/api/2.0/workspace/[WORKSPACE_ID]/rest/app_db", + "name": "data_api_url" + } +] + +=== An in-place update of the Data API does not update the job that references status.url +>>> update_file.py databricks.yml db_max_rows: 100 db_max_rows: 500 + +>>> [CLI] bundle plan +update jobs.my_job +update postgres_data_apis.my_data_api + +Plan: 0 to add, 2 to change, 0 to delete, 3 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/url-ref-pg-data-api-[UNIQUE_NAME]/default/files... +Updated jobs.my_job +Updated postgres_data_apis.my_data_api +Files: 0 uploaded, 0 deleted +Resources: 0 created, 2 changed, 0 deleted, 3 unchanged + +>>> [CLI] jobs get [MY_JOB_ID] +[ + { + "default": "https://[RW_ENDPOINT_HOST]/api/2.0/workspace/[WORKSPACE_ID]/rest/app_db", + "name": "data_api_url" + } +] + +=== Plan after update shows no changes +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 5 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.my_job + delete resources.postgres_data_apis.my_data_api + delete resources.postgres_databases.my_database + delete resources.postgres_projects.my_project + delete resources.postgres_roles.owner + +This action will result in the deletion of the following Lakebase projects along with +all their branches, databases, and endpoints. All data stored in them will be permanently lost: + delete resources.postgres_projects.my_project + +This action will result in the deletion of the following Lakebase databases. +All data stored in them will be permanently lost: + delete resources.postgres_databases.my_database + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/url-ref-pg-data-api-[UNIQUE_NAME]/default + +Destroy: 5 deleted diff --git a/acceptance/bundle/resources/postgres_data_apis/url_reference/script b/acceptance/bundle/resources/postgres_data_apis/url_reference/script new file mode 100644 index 00000000000..e422c0626b7 --- /dev/null +++ b/acceptance/bundle/resources/postgres_data_apis/url_reference/script @@ -0,0 +1,28 @@ +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f "$OUT_REQUESTS" +} +trap cleanup EXIT + +branch_name="projects/test-pg-proj-${UNIQUE_NAME}/branches/production" +data_api_name="${branch_name}/databases/my-database/data-api" + +envsubst < databricks.yml.tmpl > databricks.yml + +title "The first deploy resolves status.url into the job parameter" +trace $CLI bundle deploy +rm -f "$OUT_REQUESTS" +# The URL is served from the branch's read-write endpoint host. +add_repl "$($CLI postgres get-endpoint "${branch_name}/endpoints/primary" | jq -r .status.hosts.host)" RW_ENDPOINT_HOST +trace $CLI postgres get-data-api "${data_api_name}" | jq '{url: .status.url}' +trace $CLI jobs get "$(read_id.py my_job)" | jq '.settings.parameters' + +title "An in-place update of the Data API does not update the job that references status.url" +trace update_file.py databricks.yml "db_max_rows: 100" "db_max_rows: 500" +trace $CLI bundle plan +trace $CLI bundle deploy +rm -f "$OUT_REQUESTS" +trace $CLI jobs get "$(read_id.py my_job)" | jq '.settings.parameters' + +title "Plan after update shows no changes" +trace $CLI bundle plan diff --git a/libs/testserver/postgres.go b/libs/testserver/postgres.go index 30e6dbfd50d..b8d73ec685f 100644 --- a/libs/testserver/postgres.go +++ b/libs/testserver/postgres.go @@ -1389,16 +1389,32 @@ func dataApiNotEnabledResponse() Response { return postgresErrorResponse(404, "NOT_FOUND", dataApiNotEnabledMessage) } -// hasReadWriteEndpointLocked reports whether a branch has a read-write endpoint -// (caller must hold lock). -func (s *FakeWorkspace) hasReadWriteEndpointLocked(branchName string) bool { +// readWriteEndpointLocked returns the read-write endpoint of a branch (caller +// must hold lock). +func (s *FakeWorkspace) readWriteEndpointLocked(branchName string) (postgres.Endpoint, bool) { prefix := branchName + "/endpoints/" for name, endpoint := range s.PostgresEndpoints { if strings.HasPrefix(name, prefix) && endpoint.Status != nil && endpoint.Status.EndpointType == postgres.EndpointTypeEndpointTypeReadWrite { - return true + return endpoint, true } } - return false + return postgres.Endpoint{}, false +} + +// withDataApiURLLocked returns the Data API with status.url set the way the real +// API reports it: https://{read-write endpoint host}/api/2.0/workspace/{workspace +// ID}/rest/{Postgres database name}. It is derived from the current endpoint and +// database on every response rather than stored. The caller must hold the lock +// and have passed dataApiPreconditionLocked. +func (s *FakeWorkspace) withDataApiURLLocked(dataApi postgres.DataApi) postgres.DataApi { + branchName, _, _ := strings.Cut(dataApi.Parent, "/databases/") + endpoint, _ := s.readWriteEndpointLocked(branchName) + database := s.PostgresDatabases[dataApi.Parent] + + status := *dataApi.Status + status.Url = fmt.Sprintf("https://%s/api/2.0/workspace/%d/rest/%s", endpoint.Status.Hosts.Host, TestWorkspaceID, database.Status.PostgresDatabase) + dataApi.Status = &status + return dataApi } // dataApiPreconditionLocked returns the error the API answers a Data API create, @@ -1422,7 +1438,7 @@ func (s *FakeWorkspace) dataApiPreconditionLocked(databaseName string) *Response resp := postgresNotFoundResponse("database") return &resp } - if !s.hasReadWriteEndpointLocked(branchName) { + if _, ok := s.readWriteEndpointLocked(branchName); !ok { resp := postgresErrorResponse(400, "BAD_REQUEST", "Data API requires a read-write endpoint on branch "+branchName) return &resp } @@ -1534,7 +1550,7 @@ func (s *FakeWorkspace) PostgresDataApiCreate(req Request, parent string) Respon s.PostgresDataApis[name] = dataApi return Response{ - Body: s.createOperationLocked(name, dataApi), + Body: s.createOperationLocked(name, s.withDataApiURLLocked(dataApi)), } } @@ -1552,7 +1568,7 @@ func (s *FakeWorkspace) PostgresDataApiGet(name string) Response { } return Response{ - Body: dataApi, + Body: s.withDataApiURLLocked(dataApi), } } @@ -1589,7 +1605,7 @@ func (s *FakeWorkspace) PostgresDataApiUpdate(req Request, name string) Response s.PostgresDataApis[name] = dataApi return Response{ - Body: s.createOperationLocked(name, dataApi), + Body: s.createOperationLocked(name, s.withDataApiURLLocked(dataApi)), } } diff --git a/libs/testserver/postgres_data_api_test.go b/libs/testserver/postgres_data_api_test.go index 3638cb3c3ff..d577b3967d1 100644 --- a/libs/testserver/postgres_data_api_test.go +++ b/libs/testserver/postgres_data_api_test.go @@ -48,6 +48,8 @@ func TestPostgresDataApiCreateGetUpdateDelete(t *testing.T) { require.NotNil(t, dataApi.Status) assert.Equal(t, []string{"public"}, dataApi.Status.DbSchemas) assert.Equal(t, 100, dataApi.Status.DbMaxRows) + host := workspace.PostgresEndpoints[testDataApiBranch+"/endpoints/primary"].Status.Hosts.Host + assert.Equal(t, "https://"+host+"/api/2.0/workspace/900800700600/rest/app_db", dataApi.Status.Url) // The "spec" mask keeps the current value of a field the request omits, so // db_max_rows stays. From 7338262c47ca0a46d36261bec1291a482d219481 Mon Sep 17 00:00:00 2001 From: Ruslan Talpa Date: Wed, 21 Oct 2026 08:13:14 +0000 Subject: [PATCH 06/12] Declare postgres_data_apis status.url a stable output field status.url is built from the read-write endpoint host, the workspace and the database's Postgres name. An update of the Data API changes none of them, but without this the reference resolver delays any ${resources.postgres_data_apis..status.url} reference during an in-place update, so every dependent got a no-op update planned too. The url_reference acceptance test now shows that changing db_max_rows plans an update of the Data API only, and the next plan is clean. Co-authored-by: Isaac --- .../resources/postgres_data_apis/url_reference/output.txt | 6 ++---- bundle/direct/dresources/configs/postgres_data_apis.yml | 5 +++++ 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/acceptance/bundle/resources/postgres_data_apis/url_reference/output.txt b/acceptance/bundle/resources/postgres_data_apis/url_reference/output.txt index 8732cee3708..5ff1e94437e 100644 --- a/acceptance/bundle/resources/postgres_data_apis/url_reference/output.txt +++ b/acceptance/bundle/resources/postgres_data_apis/url_reference/output.txt @@ -27,17 +27,15 @@ Resources: 5 created, 0 changed, 0 deleted, 0 unchanged >>> update_file.py databricks.yml db_max_rows: 100 db_max_rows: 500 >>> [CLI] bundle plan -update jobs.my_job update postgres_data_apis.my_data_api -Plan: 0 to add, 2 to change, 0 to delete, 3 unchanged +Plan: 0 to add, 1 to change, 0 to delete, 4 unchanged >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/url-ref-pg-data-api-[UNIQUE_NAME]/default/files... -Updated jobs.my_job Updated postgres_data_apis.my_data_api Files: 0 uploaded, 0 deleted -Resources: 0 created, 2 changed, 0 deleted, 3 unchanged +Resources: 0 created, 1 changed, 0 deleted, 4 unchanged >>> [CLI] jobs get [MY_JOB_ID] [ diff --git a/bundle/direct/dresources/configs/postgres_data_apis.yml b/bundle/direct/dresources/configs/postgres_data_apis.yml index 661fba45782..fdef02f93a0 100644 --- a/bundle/direct/dresources/configs/postgres_data_apis.yml +++ b/bundle/direct/dresources/configs/postgres_data_apis.yml @@ -8,6 +8,11 @@ stable_output_fields: # name ("{parent}/data-api") is backend-assigned at creation and immutable; # see the stable_output_fields doc in config.go and ES-2202624. - field: name + # url is built from the read-write endpoint host, the workspace and the + # database's Postgres name, none of which a Data API update can change, so a + # ${resources.postgres_data_apis..status.url} reference resolves during + # an in-place update instead of planning a no-op update of its dependents. + - field: status.url backend_defaults: # DoRead fills the spec from status, which reports every setting in effect. # A field the config omits is left to the server: create applies the server From b60bdeb41a6626b587dc47dbd4a38da90bfad84b Mon Sep 17 00:00:00 2001 From: Ruslan Talpa Date: Wed, 21 Oct 2026 08:16:19 +0000 Subject: [PATCH 07/12] Cover bools, durations and openapi_mode in the Data API invariant config The invariant config now also sets db_aggregates_enabled: false, server_timing_enabled: true, jwt_cache_max_lifetime: 300s and openapi_mode: OPEN_API_MODE_IGNORE_PRIVILEGES, so no_drift checks the status readback for a forced false bool, a true bool, a duration and an enum. Every optional field is tagged CAN_REMOVE, the bools included. Removing either bool on its own plans an update and converges: the server keeps the last value, status reports it, and the field is a backend default. Co-authored-by: Isaac --- .../bundle/invariant/configs/postgres_data_api.yml.tmpl | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/acceptance/bundle/invariant/configs/postgres_data_api.yml.tmpl b/acceptance/bundle/invariant/configs/postgres_data_api.yml.tmpl index 5de5c7c9d57..2474d6b5995 100644 --- a/acceptance/bundle/invariant/configs/postgres_data_api.yml.tmpl +++ b/acceptance/bundle/invariant/configs/postgres_data_api.yml.tmpl @@ -30,3 +30,9 @@ resources: db_max_rows: 1000 # CAN_REMOVE server_cors_allowed_origins: # CAN_REMOVE - https://app.example.com # CAN_REMOVE + jwt_cache_max_lifetime: 300s # CAN_REMOVE + openapi_mode: OPEN_API_MODE_IGNORE_PRIVILEGES # CAN_REMOVE + # Removing a bool converges like the other fields: the server keeps the + # last value and status reports it. + db_aggregates_enabled: false # CAN_REMOVE + server_timing_enabled: true # CAN_REMOVE From f4d32af8b103955758185bceea98657c2efc8ba5 Mon Sep 17 00:00:00 2001 From: Ruslan Talpa Date: Wed, 21 Oct 2026 08:17:21 +0000 Subject: [PATCH 08/12] Describe postgres_data_apis and OpenApiMode in the bundle schema The resource's schema description was inherited from the embedded SDK spec type ("Desired PostgREST configuration (input)."). Override it to say what the resource is, that a database has at most one Data API, where the serving URL is (status.url) and how durations are written (300s). Also override the postgres.OpenApiMode description, which mentioned a value that may be added later, with what the two supported values do. Co-authored-by: Isaac --- bundle/internal/schema/annotations.yml | 7 +++++++ bundle/schema/jsonschema.json | 4 ++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index a74eced6bad..c35d73319ff 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -1625,10 +1625,17 @@ resources: "postgres_data_apis": "description": |- The Postgres Data API definitions for the bundle, where each key is the name of the Data API. Each entry enables and configures the Data API (a REST interface) of a database on a Lakebase Autoscaling branch. Removing an entry turns off the Data API; the database and its data are not affected. + "$type": + "description": |- + The Data API (a PostgREST-compatible REST interface) of a Postgres database on a Lakebase Autoscaling branch. A database has at most one Data API. Its serving URL is available as `status.url`; reference it as `${resources.postgres_data_apis..status.url}`. Durations use the protobuf seconds format, such as `300s`. "$fields": "lifecycle": "description": |- Settings that control the deployment lifecycle of the resource, such as preventing it from being destroyed. + "openapi_mode": + "$type": + "description": |- + Controls how the Data API exposes the OpenAPI documentation endpoint. `OPEN_API_MODE_IGNORE_PRIVILEGES` serves the OpenAPI document without filtering it by the caller's privileges; `OPEN_API_MODE_DISABLED` turns the endpoint off. "parent": "description": |- The database whose Data API this manages. Format: projects/{project_id}/branches/{branch_id}/databases/{database_id} diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index f230f221060..3023edbaed1 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -2461,7 +2461,7 @@ "oneOf": [ { "type": "object", - "description": "[Private Preview] Desired PostgREST configuration (input).", + "description": "[Private Preview] The Data API (a PostgREST-compatible REST interface) of a Postgres database on a Lakebase Autoscaling branch. A database has at most one Data API. Its serving URL is available as `status.url`; reference it as `${resources.postgres_data_apis.\u003ckey\u003e.status.url}`. Durations use the protobuf seconds format, such as `300s`.", "properties": { "db_aggregates_enabled": { "description": "[Private Preview] Enable aggregate functions (count, sum, avg, etc.) in Data API responses.\nDefault: true.", @@ -17383,7 +17383,7 @@ "oneOf": [ { "type": "string", - "description": "Controls how the Data API exposes the OpenAPI documentation endpoint.\nOnly IGNORE_PRIVILEGES and DISABLED are supported today; \"follow-privileges\"\nis not implemented yet (it may be added later as value 3 — adding new enum\nvalues is backward-compatible).", + "description": "Controls how the Data API exposes the OpenAPI documentation endpoint. `OPEN_API_MODE_IGNORE_PRIVILEGES` serves the OpenAPI document without filtering it by the caller's privileges; `OPEN_API_MODE_DISABLED` turns the endpoint off.", "enum": [ "OPEN_API_MODE_IGNORE_PRIVILEGES", "OPEN_API_MODE_DISABLED" From d14b9dac330ddd45308410f45a3fcc8dc60ea55d Mon Sep 17 00:00:00 2001 From: Ruslan Talpa Date: Wed, 21 Oct 2026 08:17:51 +0000 Subject: [PATCH 09/12] Keep the launchStageOverrides alignment unchanged The comment above the PostgresDataApi entry split gofmt's alignment block, which re-indented the unrelated PostgresBranch and PostgresCatalog lines. Make it a trailing comment, so the diff against main is only the added entry. Co-authored-by: Isaac --- bundle/internal/annotation/preview.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/bundle/internal/annotation/preview.go b/bundle/internal/annotation/preview.go index 0eb137449ec..33392256e48 100644 --- a/bundle/internal/annotation/preview.go +++ b/bundle/internal/annotation/preview.go @@ -27,10 +27,9 @@ func PreviewTag(stage clijson.LaunchStage) string { // fully-qualified Go type paths (PkgPath + "." + Name), matching getPath in // bundle/internal/schema. var launchStageOverrides = map[string]clijson.LaunchStage{ - "github.com/databricks/cli/bundle/config/resources.PostgresBranch": clijson.LaunchStagePublicBeta, - "github.com/databricks/cli/bundle/config/resources.PostgresCatalog": clijson.LaunchStagePublicBeta, - // The Data API is at Private Preview, below the Beta its siblings are pinned to. - "github.com/databricks/cli/bundle/config/resources.PostgresDataApi": clijson.LaunchStagePrivatePreview, + "github.com/databricks/cli/bundle/config/resources.PostgresBranch": clijson.LaunchStagePublicBeta, + "github.com/databricks/cli/bundle/config/resources.PostgresCatalog": clijson.LaunchStagePublicBeta, + "github.com/databricks/cli/bundle/config/resources.PostgresDataApi": clijson.LaunchStagePrivatePreview, // The Data API is at Private Preview, below its siblings' Beta. "github.com/databricks/cli/bundle/config/resources.PostgresDatabase": clijson.LaunchStagePublicBeta, "github.com/databricks/cli/bundle/config/resources.PostgresEndpoint": clijson.LaunchStagePublicBeta, "github.com/databricks/cli/bundle/config/resources.PostgresProject": clijson.LaunchStagePublicBeta, From 1a763c3d0559bf1060a7c1ade5e082d16743c7e0 Mon Sep 17 00:00:00 2001 From: Ruslan Talpa Date: Thu, 22 Oct 2026 11:01:20 +0000 Subject: [PATCH 10/12] Update changelog entry to link #7160 Co-authored-by: Isaac --- .nextchanges/bundles/postgres-data-apis.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.nextchanges/bundles/postgres-data-apis.md b/.nextchanges/bundles/postgres-data-apis.md index ae114d67572..503541f6280 100644 --- a/.nextchanges/bundles/postgres-data-apis.md +++ b/.nextchanges/bundles/postgres-data-apis.md @@ -1 +1 @@ -* Add the `postgres_data_apis` resource (Private Preview) for enabling and configuring the Data API of a Lakebase Postgres database. ([#7139](https://github.com/databricks/cli/pull/7139)) +* Add the `postgres_data_apis` resource (Private Preview) for enabling and configuring the Data API of a Lakebase Postgres database. ([#7160](https://github.com/databricks/cli/pull/7160)) From 5485444387f1630351a42e613b9dfdb760222352 Mon Sep 17 00:00:00 2001 From: Ruslan Talpa Date: Thu, 22 Oct 2026 11:39:58 +0000 Subject: [PATCH 11/12] Use bash 3.2-compatible redirect in postgres_data_api bind test Co-authored-by: Isaac --- acceptance/bundle/deployment/bind/postgres_data_api/script | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/acceptance/bundle/deployment/bind/postgres_data_api/script b/acceptance/bundle/deployment/bind/postgres_data_api/script index bcfe75291a8..e1e57ea0f20 100644 --- a/acceptance/bundle/deployment/bind/postgres_data_api/script +++ b/acceptance/bundle/deployment/bind/postgres_data_api/script @@ -3,8 +3,8 @@ DATA_API_NAME="${DATABASE_NAME}/data-api" title "Enable the Data API on a database outside the bundle" $CLI postgres create-project test-project --json '{"spec": {"display_name": "Test Project", "pg_version": 16}}' &> LOG.setup -$CLI postgres create-database projects/test-project/branches/production --database-id test-database --json '{"spec": {"postgres_database": "app_db", "role": "projects/test-project/branches/production/roles/owner"}}' &>> LOG.setup -$CLI postgres create-data-api "${DATABASE_NAME}" --json '{"spec": {"db_schemas": ["public"]}}' &>> LOG.setup +$CLI postgres create-database projects/test-project/branches/production --database-id test-database --json '{"spec": {"postgres_database": "app_db", "role": "projects/test-project/branches/production/roles/owner"}}' >> LOG.setup 2>&1 +$CLI postgres create-data-api "${DATABASE_NAME}" --json '{"spec": {"db_schemas": ["public"]}}' >> LOG.setup 2>&1 title "Deploy fails because the Data API is already enabled" trace musterr $CLI bundle deploy From 300f4d841b1d3be7b8fd1f4b14f69c8c23f2d669 Mon Sep 17 00:00:00 2001 From: Ruslan Talpa Date: Fri, 23 Oct 2026 07:06:52 +0000 Subject: [PATCH 12/12] Skip DMS runs for postgres_data_apis until deployment history accepts it The Integration Tests failed on AWS for every postgres_data_apis test in the DMS=true variant. Deployment history rejects the new resource-key prefix: failed to create deployment version: operations.resource_key must have a known resource type prefix (e.g. 'jobs.', 'pipelines.'): postgres_data_apis.my_data_api (400 INVALID_PARAMETER_VALUE) The testserver's dmsResourceTypes is meant to mirror the real service, so listing postgres_data_apis there hid the failure locally. Remove it, so the fake rejects the prefix the same way the real service does, and exclude DMS=true for the postgres_data_apis resource tests and the invariant config. This matches how the features resource handles the same gap. Co-authored-by: Isaac --- acceptance/bundle/invariant/test.toml | 2 ++ .../bundle/resources/postgres_data_apis/basic/out.test.toml | 2 +- .../resources/postgres_data_apis/out-of-band/out.test.toml | 2 +- .../resources/postgres_data_apis/recreate/out.test.toml | 2 +- acceptance/bundle/resources/postgres_data_apis/test.toml | 4 ++++ .../bundle/resources/postgres_data_apis/update/out.test.toml | 2 +- .../resources/postgres_data_apis/url_reference/out.test.toml | 2 +- libs/testserver/bundledeployments.go | 1 - 8 files changed, 11 insertions(+), 6 deletions(-) diff --git a/acceptance/bundle/invariant/test.toml b/acceptance/bundle/invariant/test.toml index 9fe72ef15f5..33899e103f5 100644 --- a/acceptance/bundle/invariant/test.toml +++ b/acceptance/bundle/invariant/test.toml @@ -121,6 +121,8 @@ EnvMatrixExclude.no_model_service_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG # Deployment history has no "features." resource-key prefix, so the deploy fails before the feature # is created; see dmsResourceTypes in libs/testserver/bundledeployments.go. EnvMatrixExclude.no_feature_dms = ["DMS=true", "INPUT_CONFIG=feature.yml.tmpl"] +# Same for the "postgres_data_apis." prefix. +EnvMatrixExclude.no_postgres_data_api_dms = ["DMS=true", "INPUT_CONFIG=postgres_data_api.yml.tmpl"] # Fake SQL endpoint for local tests [[Server]] diff --git a/acceptance/bundle/resources/postgres_data_apis/basic/out.test.toml b/acceptance/bundle/resources/postgres_data_apis/basic/out.test.toml index ee153e82621..a026907d175 100644 --- a/acceptance/bundle/resources/postgres_data_apis/basic/out.test.toml +++ b/acceptance/bundle/resources/postgres_data_apis/basic/out.test.toml @@ -1,4 +1,4 @@ Cloud = true CloudEnvs.azure = false CloudEnvs.gcp = false -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/postgres_data_apis/out-of-band/out.test.toml b/acceptance/bundle/resources/postgres_data_apis/out-of-band/out.test.toml index ee153e82621..a026907d175 100644 --- a/acceptance/bundle/resources/postgres_data_apis/out-of-band/out.test.toml +++ b/acceptance/bundle/resources/postgres_data_apis/out-of-band/out.test.toml @@ -1,4 +1,4 @@ Cloud = true CloudEnvs.azure = false CloudEnvs.gcp = false -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/postgres_data_apis/recreate/out.test.toml b/acceptance/bundle/resources/postgres_data_apis/recreate/out.test.toml index ee153e82621..a026907d175 100644 --- a/acceptance/bundle/resources/postgres_data_apis/recreate/out.test.toml +++ b/acceptance/bundle/resources/postgres_data_apis/recreate/out.test.toml @@ -1,4 +1,4 @@ Cloud = true CloudEnvs.azure = false CloudEnvs.gcp = false -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/postgres_data_apis/test.toml b/acceptance/bundle/resources/postgres_data_apis/test.toml index aa07fccc9c9..1555d272216 100644 --- a/acceptance/bundle/resources/postgres_data_apis/test.toml +++ b/acceptance/bundle/resources/postgres_data_apis/test.toml @@ -4,6 +4,10 @@ Cloud = true CloudEnvs.gcp = false CloudEnvs.azure = false +# Deployment history has no "postgres_data_apis." resource-key prefix, so the deploy fails before the +# Data API is created. See dmsResourceTypes in libs/testserver/bundledeployments.go, which mirrors Universe. +EnvMatrixExclude.no_data_api_dms = ["DMS=true"] + Ignore = [ "databricks.yml", ".databricks", diff --git a/acceptance/bundle/resources/postgres_data_apis/update/out.test.toml b/acceptance/bundle/resources/postgres_data_apis/update/out.test.toml index ee153e82621..a026907d175 100644 --- a/acceptance/bundle/resources/postgres_data_apis/update/out.test.toml +++ b/acceptance/bundle/resources/postgres_data_apis/update/out.test.toml @@ -1,4 +1,4 @@ Cloud = true CloudEnvs.azure = false CloudEnvs.gcp = false -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/postgres_data_apis/url_reference/out.test.toml b/acceptance/bundle/resources/postgres_data_apis/url_reference/out.test.toml index ee153e82621..a026907d175 100644 --- a/acceptance/bundle/resources/postgres_data_apis/url_reference/out.test.toml +++ b/acceptance/bundle/resources/postgres_data_apis/url_reference/out.test.toml @@ -1,4 +1,4 @@ Cloud = true CloudEnvs.azure = false CloudEnvs.gcp = false -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [""] diff --git a/libs/testserver/bundledeployments.go b/libs/testserver/bundledeployments.go index c8d158f0b1d..d5c190750c9 100644 --- a/libs/testserver/bundledeployments.go +++ b/libs/testserver/bundledeployments.go @@ -63,7 +63,6 @@ var dmsResourceTypes = []string{ "pipelines", "postgres_branches", "postgres_catalogs", - "postgres_data_apis", "postgres_databases", "postgres_endpoints", "postgres_projects",