From 1b30844cb7dd4e2fde4269c7910483f12b3d7fe1 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 7 Oct 2026 12:46:51 +0200 Subject: [PATCH 1/6] bundle: keep pending secret scope rename when migrating Terraform state When migrating Terraform state to the direct engine, reconcileIDFields looks up ID field values from the Terraform state to detect user-initiated renames/recreates not yet deployed. For secret scopes, the direct state field is `scope` but Terraform stores it as `name`. The lookup was using the unmapped DABs field name and failing silently, so pending renames were treated as normalization instead of being flagged for recreation. Add mapping: secret_scopes.scope <-> Terraform secret_scopes.name Fix: add the field rename mapping in terraform_dabs_map so LookupTFField can find the correct Terraform attribute name. Also add acceptance test to verify the fix works. Co-authored-by: Isaac --- .../secret-scope-rename/databricks.yml.tmpl | 10 ++++++ .../auto/secret-scope-rename/out.test.toml | 2 ++ .../auto/secret-scope-rename/output.txt | 33 +++++++++++++++++++ .../migrate/auto/secret-scope-rename/script | 23 +++++++++++++ .../terraform-requests.json | 9 +++++ .../secret-scope-rename/terraform.tfstate | 29 ++++++++++++++++ .../auto/secret-scope-rename/test.toml | 10 ++++++ 7 files changed, 116 insertions(+) create mode 100644 acceptance/bundle/migrate/auto/secret-scope-rename/databricks.yml.tmpl create mode 100644 acceptance/bundle/migrate/auto/secret-scope-rename/out.test.toml create mode 100644 acceptance/bundle/migrate/auto/secret-scope-rename/output.txt create mode 100644 acceptance/bundle/migrate/auto/secret-scope-rename/script create mode 100644 acceptance/bundle/migrate/auto/secret-scope-rename/terraform-requests.json create mode 100644 acceptance/bundle/migrate/auto/secret-scope-rename/terraform.tfstate create mode 100644 acceptance/bundle/migrate/auto/secret-scope-rename/test.toml diff --git a/acceptance/bundle/migrate/auto/secret-scope-rename/databricks.yml.tmpl b/acceptance/bundle/migrate/auto/secret-scope-rename/databricks.yml.tmpl new file mode 100644 index 00000000000..d5878cba064 --- /dev/null +++ b/acceptance/bundle/migrate/auto/secret-scope-rename/databricks.yml.tmpl @@ -0,0 +1,10 @@ +bundle: + name: test-bundle + +workspace: + root_path: ~/.bundle/test-bundle + +resources: + secret_scopes: + my_scope: + name: my-scope-$UNIQUE_NAME diff --git a/acceptance/bundle/migrate/auto/secret-scope-rename/out.test.toml b/acceptance/bundle/migrate/auto/secret-scope-rename/out.test.toml new file mode 100644 index 00000000000..19b5b15d11f --- /dev/null +++ b/acceptance/bundle/migrate/auto/secret-scope-rename/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/migrate/auto/secret-scope-rename/output.txt b/acceptance/bundle/migrate/auto/secret-scope-rename/output.txt new file mode 100644 index 00000000000..ca6007ff990 --- /dev/null +++ b/acceptance/bundle/migrate/auto/secret-scope-rename/output.txt @@ -0,0 +1,33 @@ + +=== Replay the prior terraform deployment (captured from a real terraform run) +>>> replay_tfstate.py terraform-requests.json terraform.tfstate + +=== Rename the secret scope (an updatable id field). With the fix, the migration detects it. +>>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle deploy --auto-approve +Warn: migration to direct: secret_scopes.my_scope: scope differs between config ("my-scope-[UNIQUE_NAME]-renamed") and terraform state ("my-scope-[UNIQUE_NAME]"); this resource will be recreated. +Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/files... +Migrated 2 resources to direct deployment engine. +Recreated secret_scopes.my_scope +Updated secret_scopes.my_scope.permissions +Files: 2 uploaded, 0 deleted +Resources: 1 created, 1 changed, 1 deleted, 0 unchanged + +=== Redeploy on direct: the scope now matches config, so the plan should be a clean no-op +>>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/files... +Files: 0 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 2 unchanged + +=== Verify the scope exists with the new name +>>> [CLI] secrets list-scopes +my-scope-[UNIQUE_NAME]-renamed DATABRICKS + +=== Cleanup +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.secret_scopes.my_scope + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle + +Destroy: 1 deleted diff --git a/acceptance/bundle/migrate/auto/secret-scope-rename/script b/acceptance/bundle/migrate/auto/secret-scope-rename/script new file mode 100644 index 00000000000..727c09b6c1f --- /dev/null +++ b/acceptance/bundle/migrate/auto/secret-scope-rename/script @@ -0,0 +1,23 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + title "Cleanup" + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +title "Replay the prior terraform deployment (captured from a real terraform run)" +trace replay_tfstate.py terraform-requests.json terraform.tfstate +rm -f "$OUT_REQUESTS" + +title "Rename the secret scope (an updatable id field). With the fix, the migration detects it." +# name is an updatable id field, so a rename requires the resource to be recreated/renamed. +# reconcileIDFields should detect the mismatch between config and deployed value. +update_file.py databricks.yml "my-scope-$UNIQUE_NAME" "my-scope-$UNIQUE_NAME-renamed" +trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle deploy --auto-approve + +title "Redeploy on direct: the scope now matches config, so the plan should be a clean no-op" +trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle deploy + +title "Verify the scope exists with the new name" +trace $CLI secrets list-scopes | grep "my-scope-$UNIQUE_NAME-renamed" diff --git a/acceptance/bundle/migrate/auto/secret-scope-rename/terraform-requests.json b/acceptance/bundle/migrate/auto/secret-scope-rename/terraform-requests.json new file mode 100644 index 00000000000..37bca7e3d6f --- /dev/null +++ b/acceptance/bundle/migrate/auto/secret-scope-rename/terraform-requests.json @@ -0,0 +1,9 @@ +[ + { + "method": "POST", + "path": "/api/2.0/secrets/scopes/create", + "body": { + "scope": "my-scope-[UNIQUE_NAME]" + } + } +] diff --git a/acceptance/bundle/migrate/auto/secret-scope-rename/terraform.tfstate b/acceptance/bundle/migrate/auto/secret-scope-rename/terraform.tfstate new file mode 100644 index 00000000000..7735667e570 --- /dev/null +++ b/acceptance/bundle/migrate/auto/secret-scope-rename/terraform.tfstate @@ -0,0 +1,29 @@ +{ + "version": 4, + "terraform_version": "1.5.5", + "serial": 1, + "lineage": "abcd1234-5678-abcd-1234-567890abcdef", + "outputs": {}, + "resources": [ + { + "mode": "managed", + "type": "databricks_secret_scope", + "name": "my_scope", + "provider": "provider[\"registry.terraform.io/databricks/databricks\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "backend_type": "DATABRICKS", + "id": "my-scope-[UNIQUE_NAME]", + "name": "my-scope-[UNIQUE_NAME]", + "keyvault_metadata": [] + }, + "sensitive_attributes": [], + "private": "bnVsbA==" + } + ] + } + ], + "check_results": null +} diff --git a/acceptance/bundle/migrate/auto/secret-scope-rename/test.toml b/acceptance/bundle/migrate/auto/secret-scope-rename/test.toml new file mode 100644 index 00000000000..6880cf5c651 --- /dev/null +++ b/acceptance/bundle/migrate/auto/secret-scope-rename/test.toml @@ -0,0 +1,10 @@ +# Local-only: the replay fixtures are captured against the fake server, so they reproduce a +# terraform state only there, not on a real workspace (replaying them on cloud fails auth and +# hits backend validation the mock does not enforce). +Cloud = false +RecordRequests = true + +Ignore = [ + ".databricks", + "databricks.yml", +] From 72ed51fa77d6d4077a8dc3ac9db8583e41b02ca9 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 7 Oct 2026 12:47:15 +0200 Subject: [PATCH 2/6] Add terraform_dabs_map entry for secret_scopes field name translation Co-authored-by: Isaac --- bundle/terraform_dabs_map/generated.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/bundle/terraform_dabs_map/generated.go b/bundle/terraform_dabs_map/generated.go index 30d61163258..502b80a8b08 100644 --- a/bundle/terraform_dabs_map/generated.go +++ b/bundle/terraform_dabs_map/generated.go @@ -72,6 +72,9 @@ var TerraformToDABsFieldMap = map[string]RenameTree{ "postgres_branches": { "spec": {Unwrap: true}, }, + "secret_scopes": { + "name": {NewName: "scope"}, + }, "postgres_catalogs": { "spec": {Unwrap: true}, }, @@ -642,6 +645,9 @@ var DABsToTerraformRenameMap = map[string]RenameTree{ "libraries": {NewName: "library"}, "notifications": {NewName: "notification"}, }, + "secret_scopes": { + "scope": {NewName: "name"}, + }, } // DABsToTerraformWrappers maps DABs group name → the TF wrapper segment to prepend. From 2b76625eecebc16f264baa420cef6865376b1432 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 7 Oct 2026 13:11:23 +0200 Subject: [PATCH 3/6] Revert "Add terraform_dabs_map entry for secret_scopes field name translation" This reverts commit 78d4918b616c135d14b7210b775dc82518fa8599. --- bundle/terraform_dabs_map/generated.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/bundle/terraform_dabs_map/generated.go b/bundle/terraform_dabs_map/generated.go index 502b80a8b08..30d61163258 100644 --- a/bundle/terraform_dabs_map/generated.go +++ b/bundle/terraform_dabs_map/generated.go @@ -72,9 +72,6 @@ var TerraformToDABsFieldMap = map[string]RenameTree{ "postgres_branches": { "spec": {Unwrap: true}, }, - "secret_scopes": { - "name": {NewName: "scope"}, - }, "postgres_catalogs": { "spec": {Unwrap: true}, }, @@ -645,9 +642,6 @@ var DABsToTerraformRenameMap = map[string]RenameTree{ "libraries": {NewName: "library"}, "notifications": {NewName: "notification"}, }, - "secret_scopes": { - "scope": {NewName: "name"}, - }, } // DABsToTerraformWrappers maps DABs group name → the TF wrapper segment to prepend. From 9f0fc1b001cf8a58181e90747fa8431972e38f4b Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 7 Oct 2026 13:13:37 +0200 Subject: [PATCH 4/6] bundle: keep pending secret scope rename when migrating Terraform state When migrating Terraform state to the direct engine, reconcileIDFields looks up ID field values from the Terraform state to detect user-initiated renames/recreates not yet deployed. For secret scopes, the direct state field is `scope` but Terraform stores it as `name`. The lookup was using the DABs field name and failing silently, so pending renames were treated as normalization instead of being flagged for recreation. Fix: add a fallback lookup in reconcileIDFields specifically for secret_scopes that translates the direct state field name to the Terraform attribute name. This keeps the fix localized to the migration context and does not affect field name translation for references or other parts of the system. Co-authored-by: Isaac --- bundle/migrate/build_state.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/bundle/migrate/build_state.go b/bundle/migrate/build_state.go index 0f0776152eb..2aba8167f44 100644 --- a/bundle/migrate/build_state.go +++ b/bundle/migrate/build_state.go @@ -286,7 +286,14 @@ func reconcileIDFields(ctx context.Context, adapter *dresources.Adapter, group, } deployedVal, err := LookupTFField(tfAttrs, group, name, path) if err != nil { - continue + // For secret_scopes, the direct state field "scope" maps to Terraform attribute "name". + // Try a fallback lookup for this specific case. + if group == "secret_scopes" && rule.Field.String() == "scope" { + deployedVal, err = LookupTFField(tfAttrs, group, name, structpath.NewStringKey(nil, "name")) + } + if err != nil { + continue + } } // Every id-composing field is a string (a name, catalog_name, storage path, ...). From e58d3af3693826912f85eaf2df0dcaa320077b34 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 7 Oct 2026 13:29:53 +0200 Subject: [PATCH 5/6] Map the secret scope state field explicitly instead of a fallback lookup Look up the Terraform attribute through a small state-field table before the lookup rather than retrying with "name" after a failure, so there is one code path and the mapping is visible. Co-authored-by: Isaac --- bundle/migrate/build_state.go | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/bundle/migrate/build_state.go b/bundle/migrate/build_state.go index 2aba8167f44..ec6cf41369d 100644 --- a/bundle/migrate/build_state.go +++ b/bundle/migrate/build_state.go @@ -250,6 +250,13 @@ func BuildStateFromTF( return warningsSeen, nil } +// stateIDFieldToTF maps an id-composing direct state field to its Terraform attribute where the +// two names differ and terraform_dabs_map (which maps bundle config fields) does not cover it: +// the secret scope state stores the CreateScope request field "scope", Terraform calls it "name". +var stateIDFieldToTF = map[string]map[string]string{ + "secret_scopes": {"scope": "name"}, +} + // reconcileIDFields aligns each id-composing field (provided_id_fields, updatable_id_fields) // in the migrated state with the deployed terraform state. The state is otherwise seeded from // config; for id fields, recording the current config would snapshot a pending change as @@ -284,16 +291,13 @@ func reconcileIDFields(ctx context.Context, adapter *dresources.Adapter, group, if err != nil { continue } - deployedVal, err := LookupTFField(tfAttrs, group, name, path) + tfPath := path + if tfField, ok := stateIDFieldToTF[group][rule.Field.String()]; ok { + tfPath = structpath.NewStringKey(nil, tfField) + } + deployedVal, err := LookupTFField(tfAttrs, group, name, tfPath) if err != nil { - // For secret_scopes, the direct state field "scope" maps to Terraform attribute "name". - // Try a fallback lookup for this specific case. - if group == "secret_scopes" && rule.Field.String() == "scope" { - deployedVal, err = LookupTFField(tfAttrs, group, name, structpath.NewStringKey(nil, "name")) - } - if err != nil { - continue - } + continue } // Every id-composing field is a string (a name, catalog_name, storage path, ...). From 6445c48ace9f670d6a38f764853473baa0df5169 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 7 Oct 2026 13:30:40 +0200 Subject: [PATCH 6/6] Add changelog fragment Co-authored-by: Isaac --- .nextchanges/bundles/migrate-secret-scope-rename.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 .nextchanges/bundles/migrate-secret-scope-rename.md diff --git a/.nextchanges/bundles/migrate-secret-scope-rename.md b/.nextchanges/bundles/migrate-secret-scope-rename.md new file mode 100644 index 00000000000..21df0c4c0ec --- /dev/null +++ b/.nextchanges/bundles/migrate-secret-scope-rename.md @@ -0,0 +1 @@ +* Fix a pending secret scope rename being dropped when Terraform state is migrated to the direct engine. ([#6971](https://github.com/databricks/cli/pull/6971))