acc: record config edge cases for variables, YAML merge keys and diagnostic locations

acceptance/bundle/run/job-ref-job-id/databricks.yml

@@ -0,0 +1,13 @@
+bundle:
+  name: test-bundle
+
+resources:
+  jobs:
+    bar:
+      name: job bar
+    foo:
+      name: job foo
+      tasks:
+        - task_key: job_task
+          run_job_task:
+            job_id: ${resources.jobs.bar.id}

acceptance/bundle/run/job-ref-job-id/out.test.toml

@@ -0,0 +1,2 @@
+Cloud = false
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/run/job-ref-job-id/output.txt

@@ -0,0 +1,20 @@
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
+Created jobs.bar
+Created jobs.foo
+Files: 1 uploaded, 0 deleted
+Resources: 2 created, 0 changed, 0 deleted, 0 unchanged
+
+=== foo's task points at bar's job id
+>>> [CLI] jobs get [FOO_ID]
+{
+  "job_id": [BAR_ID]
+}
+
+=== run resolves the referenced job id into the int job_id field
+>>> [CLI] bundle run foo
+Run URL: [DATABRICKS_URL]/jobs/[FOO_ID]/runs/[RUN_ID]?o=[WORKSPACE_ID]
+
+[TIMESTAMP] "job foo" RUNNING
+[TIMESTAMP] "job foo" TERMINATED SUCCESS

acceptance/bundle/run/job-ref-job-id/script

@@ -0,0 +1,8 @@
+trace $CLI bundle deploy
+replace_ids.py
+
+title "foo's task points at bar's job id"
+trace $CLI jobs get $(read_id.py foo) | jq '.settings.tasks[0].run_job_task'
+
+title "run resolves the referenced job id into the int job_id field"
+trace $CLI bundle run foo

acceptance/bundle/run/job-ref-job-id/test.toml

@@ -0,0 +1,3 @@
+[[Repls]]
+Old = '/runs/\d+'
+New = '/runs/[RUN_ID]'

acceptance/bundle/validate/complex_variable_default_merge/a.yml

@@ -0,0 +1,6 @@
+variables:
+  libs:
+    type: complex
+    default:
+      - one
+      - two

acceptance/bundle/validate/complex_variable_default_merge/b.yml

@@ -0,0 +1,5 @@
+variables:
+  libs:
+    type: complex
+    default:
+      - three

acceptance/bundle/validate/complex_variable_default_merge/databricks.yml

@@ -0,0 +1,6 @@
+bundle:
+  name: test-bundle
+
+include:
+  - a.yml
+  - b.yml

acceptance/bundle/validate/complex_variable_default_merge/out.test.toml

@@ -0,0 +1,2 @@
+Cloud = false
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/validate/complex_variable_default_merge/output.txt

@@ -0,0 +1,15 @@
+
+>>> [CLI] bundle validate -o json
+{
+  "default": [
+    "one",
+    "two",
+    "three"
+  ],
+  "type": "complex",
+  "value": [
+    "one",
+    "two",
+    "three"
+  ]
+}

acceptance/bundle/validate/complex_variable_default_merge/script

@@ -0,0 +1 @@
+trace $CLI bundle validate -o json | jq .variables.libs

acceptance/bundle/validate/compute_id_location/databricks.yml

@@ -0,0 +1,9 @@
+bundle:
+  name: test-bundle
+  compute_id: root-cluster
+
+targets:
+  dev:
+    compute_id: dev-cluster
+  other:
+    default: true

acceptance/bundle/validate/compute_id_location/out.test.toml

@@ -0,0 +1,2 @@
+Cloud = false
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/validate/compute_id_location/output.txt

@@ -0,0 +1,36 @@
+
+=== default target without compute_id
+>>> [CLI] bundle validate
+Warning: compute_id is deprecated, please use cluster_id instead
+  at bundle.compute_id
+  in databricks.yml:3:15
+
+Warning: compute_id is deprecated, please use cluster_id instead
+  at compute_id
+  in databricks.yml:7:17
+
+Name: test-bundle
+Target: other
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle/other
+
+Found 2 warnings
+
+=== target with compute_id
+>>> [CLI] bundle validate -t dev
+Warning: compute_id is deprecated, please use cluster_id instead
+  at bundle.compute_id
+  in databricks.yml:3:15
+
+Warning: compute_id is deprecated, please use cluster_id instead
+  at compute_id
+  in databricks.yml:7:17
+
+Name: test-bundle
+Target: dev
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle/dev
+
+Found 2 warnings

acceptance/bundle/validate/compute_id_location/script

@@ -0,0 +1,5 @@
+title "default target without compute_id"
+trace $CLI bundle validate
+
+title "target with compute_id"
+trace $CLI bundle validate -t dev

acceptance/bundle/validate/compute_id_location/test.toml

@@ -0,0 +1 @@
+Badness = "Target-level compute_id is reported without the target in the path (at compute_id instead of at targets.dev.compute_id), and is reported even when that target is not selected."

acceptance/bundle/validate/dashboard_key_special_chars/databricks.yml

@@ -0,0 +1,14 @@
+bundle:
+  name: test-bundle
+
+targets:
+  bracket:
+    resources:
+      dashboards:
+        "reports[":
+          serialized_dashboard: "{}"
+  comma:
+    resources:
+      dashboards:
+        "a,b":
+          serialized_dashboard: "{}"

acceptance/bundle/validate/dashboard_key_special_chars/out.test.toml

@@ -0,0 +1,2 @@
+Cloud = false
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/validate/dashboard_key_special_chars/output.txt

@@ -0,0 +1,38 @@
+
+>>> [CLI] bundle validate -t bracket
+Error: dashboard display_name is required
+  at resources.dashboards['reports[']
+  in databricks.yml:9:11
+
+Error: dashboard warehouse_id is required
+  at resources.dashboards['reports[']
+  in databricks.yml:9:11
+
+Name: test-bundle
+Target: bracket
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle/bracket
+
+Found 2 errors
+
+Exit code: 1
+
+>>> [CLI] bundle validate -t comma
+Error: dashboard display_name is required
+  at resources.dashboards['a,b']
+  in databricks.yml:14:11
+
+Error: dashboard warehouse_id is required
+  at resources.dashboards['a,b']
+  in databricks.yml:14:11
+
+Name: test-bundle
+Target: comma
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle/comma
+
+Found 2 errors
+
+Exit code: 1

acceptance/bundle/validate/dashboard_key_special_chars/script

@@ -0,0 +1,9 @@
+# Dashboards missing display_name / warehouse_id must be reported even when the
+# resource key contains characters that are path syntax ("[", ","). The required-field
+# check looks the dashboard up by path, so a key that does not survive being turned
+# into a path string would silently drop these errors.
+#
+# Each key is in its own target: with both in one config the errors are reported in
+# a nondeterministic order.
+errcode trace $CLI bundle validate -t bracket
+errcode trace $CLI bundle validate -t comma

acceptance/bundle/validate/override_effective_location/databricks.yml

@@ -0,0 +1,21 @@
+bundle:
+  name: test-bundle
+  engine: direct
+
+presets:
+  trigger_pause_status: PAUSED
+  name_prefix: "${workspace.current_user.short_name}_"
+
+targets:
+  bad_engine:
+    bundle:
+      engine: invalid_value
+  dev_trigger:
+    mode: development
+    default: true
+    presets:
+      trigger_pause_status: UNPAUSED
+  dev_prefix:
+    mode: development
+    presets:
+      name_prefix: nouser_

acceptance/bundle/validate/override_effective_location/out.test.toml

@@ -0,0 +1,2 @@
+Cloud = false
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/validate/override_effective_location/output.txt

@@ -0,0 +1,42 @@
+
+=== engine overridden in target
+>>> [CLI] bundle validate -t bad_engine
+Error: invalid value "invalid_value" for bundle.engine (expected "direct")
+  in databricks.yml:12:15
+
+Name: test-bundle
+Target: bad_engine
+
+Found 1 error
+
+Exit code: 1
+
+=== trigger_pause_status overridden in development target
+>>> [CLI] bundle validate -t dev_trigger
+Error: target with 'mode: development' cannot set trigger pause status to UNPAUSED by default
+  in databricks.yml:17:29
+
+Name: test-bundle
+Target: dev_trigger
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle/dev_trigger
+
+Found 1 error
+
+Exit code: 1
+
+=== name_prefix overridden in development target
+>>> [CLI] bundle validate -t dev_prefix
+Error: prefix should contain the current username or ${workspace.current_user.short_name} to ensure uniqueness when using 'mode: development'
+  in databricks.yml:21:20
+
+Name: test-bundle
+Target: dev_prefix
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle/dev_prefix
+
+Found 1 error
+
+Exit code: 1

acceptance/bundle/validate/override_effective_location/script

@@ -0,0 +1,8 @@
+title "engine overridden in target"
+errcode trace $CLI bundle validate -t bad_engine
+
+title "trigger_pause_status overridden in development target"
+errcode trace $CLI bundle validate -t dev_trigger
+
+title "name_prefix overridden in development target"
+errcode trace $CLI bundle validate -t dev_prefix

acceptance/bundle/validate/pipeline_development_reference/databricks.yml

@@ -0,0 +1,10 @@
+bundle:
+  name: test-bundle
+
+resources:
+  pipelines:
+    other:
+      name: other
+    main:
+      name: main
+      development: ${resources.pipelines.other.development}

acceptance/bundle/validate/pipeline_development_reference/out.test.toml

@@ -0,0 +1,2 @@
+Cloud = false
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/validate/pipeline_development_reference/output.txt

@@ -0,0 +1,14 @@
+
+>>> [CLI] bundle validate
+Warning: The "development" property of pipelines is deprecated. Use "mode: development" in your databricks.yml instead. See https://docs.databricks.com/dev-tools/bundles/deployment-modes.
+  at resources.pipelines.main.development
+  in databricks.yml:10:20
+
+Name: test-bundle
+Target: default
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default
+
+Found 1 warning
+"${resources.pipelines.other.development}"

acceptance/bundle/validate/pipeline_development_reference/script

@@ -0,0 +1,3 @@
+trace $CLI bundle validate
+
+trace $CLI bundle validate -o json 2>/dev/null | jq .resources.pipelines.main.development

acceptance/bundle/validate/target_variable_full_def_merge_key/databricks.yml

@@ -0,0 +1,13 @@
+bundle:
+  name: test-bundle
+
+variables:
+  x: &x
+    default: old
+
+targets:
+  dev:
+    variables:
+      x:
+        <<: *x
+        default: new

acceptance/bundle/validate/target_variable_full_def_merge_key/out.test.toml

@@ -0,0 +1,2 @@
+Cloud = false
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/validate/target_variable_full_def_merge_key/output.txt

@@ -0,0 +1,6 @@
+
+>>> [CLI] bundle validate -t dev -o json
+{
+  "default": "new",
+  "value": "new"
+}

acceptance/bundle/validate/target_variable_full_def_merge_key/script

@@ -0,0 +1 @@
+trace $CLI bundle validate -t dev -o json | jq .variables.x

acceptance/bundle/validate/target_variable_merge_key_precedence/databricks.yml

@@ -0,0 +1,16 @@
+bundle:
+  name: test-bundle
+
+variables:
+  x:
+
+definitions:
+  a: &a
+    x: from-a
+  b: &b
+    x: from-b
+
+targets:
+  dev:
+    variables:
+      <<: [*a, *b]

acceptance/bundle/validate/target_variable_merge_key_precedence/out.test.toml

@@ -0,0 +1,2 @@
+Cloud = false
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/validate/target_variable_merge_key_precedence/output.txt

@@ -0,0 +1,8 @@
+
+>>> [CLI] bundle validate -t dev -o json
+{
+  "x": {
+    "default": "from-b",
+    "value": "from-b"
+  }
+}

acceptance/bundle/validate/target_variable_merge_key_precedence/script

@@ -0,0 +1 @@
+trace $CLI bundle validate -t dev -o json | jq .variables

acceptance/bundle/validate/target_variable_merge_key_precedence/test.toml

@@ -0,0 +1 @@
+Badness = "With <<: [*a, *b] the later anchor wins; the YAML merge key spec gives precedence to the earlier one."