bundle: keep pending secret scope rename when migrating Terraform state

.nextchanges/bundles/migrate-secret-scope-rename.md

@@ -0,0 +1 @@
+* Fix a pending secret scope rename being dropped when Terraform state is migrated to the direct engine. ([#6971](https://github.com/databricks/cli/pull/6971))

acceptance/bundle/migrate/auto/secret-scope-rename/databricks.yml.tmpl

@@ -0,0 +1,10 @@
+bundle:
+  name: test-bundle
+
+workspace:
+  root_path: ~/.bundle/test-bundle
+
+resources:
+  secret_scopes:
+    my_scope:
+      name: my-scope-$UNIQUE_NAME

acceptance/bundle/migrate/auto/secret-scope-rename/out.test.toml

@@ -0,0 +1,2 @@
+Cloud = false
+EnvMatrix.DMS = [""]

acceptance/bundle/migrate/auto/secret-scope-rename/output.txt

@@ -0,0 +1,33 @@
+
+=== Replay the prior terraform deployment (captured from a real terraform run)
+>>> replay_tfstate.py terraform-requests.json terraform.tfstate
+
+=== Rename the secret scope (an updatable id field). With the fix, the migration detects it.
+>>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle deploy --auto-approve
+Warn: migration to direct: secret_scopes.my_scope: scope differs between config ("my-scope-[UNIQUE_NAME]-renamed") and terraform state ("my-scope-[UNIQUE_NAME]"); this resource will be recreated.
+Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765).
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/files...
+Migrated 2 resources to direct deployment engine.
+Recreated secret_scopes.my_scope
+Updated secret_scopes.my_scope.permissions
+Files: 2 uploaded, 0 deleted
+Resources: 1 created, 1 changed, 1 deleted, 0 unchanged
+
+=== Redeploy on direct: the scope now matches config, so the plan should be a clean no-op
+>>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/files...
+Files: 0 uploaded, 0 deleted
+Resources: 0 created, 0 changed, 0 deleted, 2 unchanged
+
+=== Verify the scope exists with the new name
+>>> [CLI] secrets list-scopes
+my-scope-[UNIQUE_NAME]-renamed  DATABRICKS
+
+=== Cleanup
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.secret_scopes.my_scope
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle
+
+Destroy: 1 deleted

acceptance/bundle/migrate/auto/secret-scope-rename/script

@@ -0,0 +1,23 @@
+envsubst < databricks.yml.tmpl > databricks.yml
+
+cleanup() {
+  title "Cleanup"
+  trace $CLI bundle destroy --auto-approve
+}
+trap cleanup EXIT
+
+title "Replay the prior terraform deployment (captured from a real terraform run)"
+trace replay_tfstate.py terraform-requests.json terraform.tfstate
+rm -f "$OUT_REQUESTS"
+
+title "Rename the secret scope (an updatable id field). With the fix, the migration detects it."
+# name is an updatable id field, so a rename requires the resource to be recreated/renamed.
+# reconcileIDFields should detect the mismatch between config and deployed value.
+update_file.py databricks.yml "my-scope-$UNIQUE_NAME" "my-scope-$UNIQUE_NAME-renamed"
+trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle deploy --auto-approve
+
+title "Redeploy on direct: the scope now matches config, so the plan should be a clean no-op"
+trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle deploy
+
+title "Verify the scope exists with the new name"
+trace $CLI secrets list-scopes | grep "my-scope-$UNIQUE_NAME-renamed"

acceptance/bundle/migrate/auto/secret-scope-rename/terraform-requests.json

@@ -0,0 +1,9 @@
+[
+  {
+    "method": "POST",
+    "path": "/api/2.0/secrets/scopes/create",
+    "body": {
+      "scope": "my-scope-[UNIQUE_NAME]"
+    }
+  }
+]

acceptance/bundle/migrate/auto/secret-scope-rename/terraform.tfstate

@@ -0,0 +1,29 @@
+{
+  "version": 4,
+  "terraform_version": "1.5.5",
+  "serial": 1,
+  "lineage": "abcd1234-5678-abcd-1234-567890abcdef",
+  "outputs": {},
+  "resources": [
+    {
+      "mode": "managed",
+      "type": "databricks_secret_scope",
+      "name": "my_scope",
+      "provider": "provider[\"registry.terraform.io/databricks/databricks\"]",
+      "instances": [
+        {
+          "schema_version": 0,
+          "attributes": {
+            "backend_type": "DATABRICKS",
+            "id": "my-scope-[UNIQUE_NAME]",
+            "name": "my-scope-[UNIQUE_NAME]",
+            "keyvault_metadata": []
+          },
+          "sensitive_attributes": [],
+          "private": "bnVsbA=="
+        }
+      ]
+    }
+  ],
+  "check_results": null
+}

acceptance/bundle/migrate/auto/secret-scope-rename/test.toml

@@ -0,0 +1,10 @@
+# Local-only: the replay fixtures are captured against the fake server, so they reproduce a
+# terraform state only there, not on a real workspace (replaying them on cloud fails auth and
+# hits backend validation the mock does not enforce).
+Cloud = false
+RecordRequests = true
+
+Ignore = [
+    ".databricks",
+    "databricks.yml",
+]

bundle/migrate/build_state.go

@@ -250,6 +250,13 @@ func BuildStateFromTF(
 	return warningsSeen, nil
 }
 
+// stateIDFieldToTF maps an id-composing direct state field to its Terraform attribute where the
+// two names differ and terraform_dabs_map (which maps bundle config fields) does not cover it:
+// the secret scope state stores the CreateScope request field "scope", Terraform calls it "name".
+var stateIDFieldToTF = map[string]map[string]string{
+	"secret_scopes": {"scope": "name"},
+}
+
 // reconcileIDFields aligns each id-composing field (provided_id_fields, updatable_id_fields)
 // in the migrated state with the deployed terraform state. The state is otherwise seeded from
 // config; for id fields, recording the current config would snapshot a pending change as
@@ -284,7 +291,11 @@ func reconcileIDFields(ctx context.Context, adapter *dresources.Adapter, group,
 			if err != nil {
 				continue
 			}
-			deployedVal, err := LookupTFField(tfAttrs, group, name, path)
+			tfPath := path
+			if tfField, ok := stateIDFieldToTF[group][rule.Field.String()]; ok {
+				tfPath = structpath.NewStringKey(nil, tfField)
+			}
+			deployedVal, err := LookupTFField(tfAttrs, group, name, tfPath)
 			if err != nil {
 				continue
 			}