diff --git a/acceptance/bundle/override/cluster_id_clears_new_cluster_reference/databricks.yml b/acceptance/bundle/override/cluster_id_clears_new_cluster_reference/databricks.yml new file mode 100644 index 00000000000..d2e913f1abe --- /dev/null +++ b/acceptance/bundle/override/cluster_id_clears_new_cluster_reference/databricks.yml @@ -0,0 +1,49 @@ +bundle: + name: test-bundle + cluster_id: existing-cluster + +variables: + cluster: + type: complex + default: + spark_version: 15.4.x-scala2.12 + node_type_id: i3.xlarge + num_workers: 1 + +resources: + clusters: + source: + cluster_name: source + spark_version: 15.4.x-scala2.12 + node_type_id: i3.xlarge + num_workers: 2 + jobs: + job: + name: job + tasks: + - task_key: task + notebook_task: + notebook_path: /Workspace/notebook + # Replaced by bundle.cluster_id: the reference below new_cluster must go too. + new_cluster: + spark_version: 15.4.x-scala2.12 + node_type_id: i3.xlarge + num_workers: ${resources.clusters.source.num_workers} + - task_key: task_var + notebook_task: + notebook_path: /Workspace/notebook + # A reference at new_cluster must not bring it back either. + new_cluster: ${var.cluster} + - task_key: task_resource_ref + notebook_task: + notebook_path: /Workspace/notebook + # A reference to the new_cluster of another job is resolved after the override. + new_cluster: ${resources.jobs.other.job_clusters[0].new_cluster} + other: + name: other + job_clusters: + - job_cluster_key: shared + new_cluster: + spark_version: 15.4.x-scala2.12 + node_type_id: i3.xlarge + num_workers: 3 diff --git a/acceptance/bundle/override/cluster_id_clears_new_cluster_reference/out.test.toml b/acceptance/bundle/override/cluster_id_clears_new_cluster_reference/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/override/cluster_id_clears_new_cluster_reference/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/override/cluster_id_clears_new_cluster_reference/output.txt b/acceptance/bundle/override/cluster_id_clears_new_cluster_reference/output.txt new file mode 100644 index 00000000000..14e1e4bbefd --- /dev/null +++ b/acceptance/bundle/override/cluster_id_clears_new_cluster_reference/output.txt @@ -0,0 +1,27 @@ + +>>> [CLI] bundle validate -o json +[ + { + "task_key": "task", + "existing_cluster_id": "existing-cluster", + "new_cluster": null + }, + { + "task_key": "task_resource_ref", + "existing_cluster_id": "existing-cluster", + "new_cluster": null + }, + { + "task_key": "task_var", + "existing_cluster_id": "existing-cluster", + "new_cluster": null + } +] + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created clusters.source +Created jobs.job +Created jobs.other +Files: 1 uploaded, 0 deleted +Resources: 3 created, 0 changed, 0 deleted, 0 unchanged diff --git a/acceptance/bundle/override/cluster_id_clears_new_cluster_reference/script b/acceptance/bundle/override/cluster_id_clears_new_cluster_reference/script new file mode 100644 index 00000000000..2016268365d --- /dev/null +++ b/acceptance/bundle/override/cluster_id_clears_new_cluster_reference/script @@ -0,0 +1,3 @@ +# bundle.cluster_id replaces new_cluster on every task, also when new_cluster holds a reference. +trace $CLI bundle validate -o json | jq '[.resources.jobs.job.tasks[] | {task_key, existing_cluster_id, new_cluster}]' +trace $CLI bundle deploy diff --git a/acceptance/bundle/override/preset_clears_forced_schedule_reference/databricks.yml b/acceptance/bundle/override/preset_clears_forced_schedule_reference/databricks.yml new file mode 100644 index 00000000000..25315e8078e --- /dev/null +++ b/acceptance/bundle/override/preset_clears_forced_schedule_reference/databricks.yml @@ -0,0 +1,39 @@ +bundle: + name: test-bundle + +resources: + quality_monitors: + src: + table_name: main.default.src + assets_dir: /Workspace/monitors/src + output_schema_name: main.default + inference_log: + granularities: [1 day] + model_id_col: model_id + prediction_col: prediction + problem_type: PROBLEM_TYPE_CLASSIFICATION + timestamp_col: timestamp + schedule: + quartz_cron_expression: "0 0 12 * * ?" + timezone_id: UTC + dst: + table_name: main.default.dst + assets_dir: /Workspace/monitors/dst + output_schema_name: main.default + inference_log: + granularities: [1 day] + model_id_col: model_id + prediction_col: prediction + problem_type: PROBLEM_TYPE_CLASSIFICATION + timestamp_col: timestamp + # Forced null: the target override replaces it with a reference. + schedule: null + +targets: + dev: + presets: + trigger_pause_status: PAUSED + resources: + quality_monitors: + dst: + schedule: ${resources.quality_monitors.src.schedule} diff --git a/acceptance/bundle/override/preset_clears_forced_schedule_reference/out.test.toml b/acceptance/bundle/override/preset_clears_forced_schedule_reference/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/override/preset_clears_forced_schedule_reference/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/override/preset_clears_forced_schedule_reference/output.txt b/acceptance/bundle/override/preset_clears_forced_schedule_reference/output.txt new file mode 100644 index 00000000000..eb07e4ec538 --- /dev/null +++ b/acceptance/bundle/override/preset_clears_forced_schedule_reference/output.txt @@ -0,0 +1,13 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/dev/files... +Created quality_monitors.dst +Created quality_monitors.src +Files: 1 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //tables/main.default.dst/monitor +{ + "has_schedule": false, + "schedule": null +} diff --git a/acceptance/bundle/override/preset_clears_forced_schedule_reference/script b/acceptance/bundle/override/preset_clears_forced_schedule_reference/script new file mode 100644 index 00000000000..37d355db8c8 --- /dev/null +++ b/acceptance/bundle/override/preset_clears_forced_schedule_reference/script @@ -0,0 +1,4 @@ +# trigger_pause_status: PAUSED removes the schedules. The target override replaces the null schedule of dst +# with a reference to the schedule of src, which is removed too: the request carries no schedule. +trace $CLI bundle deploy +trace print_requests.py //tables/main.default.dst/monitor | jq '.body | {has_schedule: has("schedule"), schedule}' diff --git a/acceptance/bundle/override/preset_clears_forced_schedule_reference/test.toml b/acceptance/bundle/override/preset_clears_forced_schedule_reference/test.toml new file mode 100644 index 00000000000..159efe02696 --- /dev/null +++ b/acceptance/bundle/override/preset_clears_forced_schedule_reference/test.toml @@ -0,0 +1 @@ +RecordRequests = true diff --git a/acceptance/bundle/presets/name_prefix_reference_to_name/databricks.yml b/acceptance/bundle/presets/name_prefix_reference_to_name/databricks.yml new file mode 100644 index 00000000000..fa7ad2b0783 --- /dev/null +++ b/acceptance/bundle/presets/name_prefix_reference_to_name/databricks.yml @@ -0,0 +1,17 @@ +bundle: + name: test-bundle + +presets: + name_prefix: "pre-" + +resources: + jobs: + base: + name: base + schedule: {quartz_cron_expression: "0 0 * * * ?", timezone_id: UTC} + trigger: {file_arrival: {url: "s3://bucket/path"}} + tasks: [{task_key: t, notebook_task: {notebook_path: ./nb.py}}] + viaref: + name: viaref + name: ${resources.jobs.base.name} + tasks: [{task_key: t, notebook_task: {notebook_path: ./nb.py}}] diff --git a/acceptance/bundle/presets/name_prefix_reference_to_name/nb.py b/acceptance/bundle/presets/name_prefix_reference_to_name/nb.py new file mode 100644 index 00000000000..7c87acfa794 --- /dev/null +++ b/acceptance/bundle/presets/name_prefix_reference_to_name/nb.py @@ -0,0 +1,2 @@ +# Databricks notebook source +print(1) diff --git a/acceptance/bundle/presets/name_prefix_reference_to_name/out.test.toml b/acceptance/bundle/presets/name_prefix_reference_to_name/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/presets/name_prefix_reference_to_name/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/presets/name_prefix_reference_to_name/output.txt b/acceptance/bundle/presets/name_prefix_reference_to_name/output.txt new file mode 100644 index 00000000000..e07fab23d09 --- /dev/null +++ b/acceptance/bundle/presets/name_prefix_reference_to_name/output.txt @@ -0,0 +1,5 @@ + +>>> [CLI] bundle plan -o json +{ + "resources.jobs.viaref": "pre-pre-base" +} diff --git a/acceptance/bundle/presets/name_prefix_reference_to_name/script b/acceptance/bundle/presets/name_prefix_reference_to_name/script new file mode 100644 index 00000000000..f43b4a8b5aa --- /dev/null +++ b/acceptance/bundle/presets/name_prefix_reference_to_name/script @@ -0,0 +1,2 @@ +# name_prefix is added to a name that is a reference to the name of another job, which has the prefix too. +trace $CLI bundle plan -o json | jq '.plan | del(.["resources.jobs.base"]) | map_values(.new_state.value.name)' diff --git a/acceptance/bundle/presets/name_prefix_reference_to_name/test.toml b/acceptance/bundle/presets/name_prefix_reference_to_name/test.toml new file mode 100644 index 00000000000..0f1ffaaa6c1 --- /dev/null +++ b/acceptance/bundle/presets/name_prefix_reference_to_name/test.toml @@ -0,0 +1 @@ +Badness = "name_prefix is added a second time to a name that is a reference to the name of another job." diff --git a/acceptance/bundle/presets/trigger_pause_status_whole_schedule_reference/databricks.yml b/acceptance/bundle/presets/trigger_pause_status_whole_schedule_reference/databricks.yml new file mode 100644 index 00000000000..0ce3005fef5 --- /dev/null +++ b/acceptance/bundle/presets/trigger_pause_status_whole_schedule_reference/databricks.yml @@ -0,0 +1,17 @@ +bundle: + name: test-bundle + +presets: + trigger_pause_status: PAUSED + +resources: + jobs: + base: + name: base + schedule: {quartz_cron_expression: "0 0 * * * ?", timezone_id: UTC} + trigger: {file_arrival: {url: "s3://bucket/path"}} + tasks: [{task_key: t, notebook_task: {notebook_path: ./nb.py}}] + viaref: + name: viaref + schedule: ${resources.jobs.base.schedule} + tasks: [{task_key: t, notebook_task: {notebook_path: ./nb.py}}] diff --git a/acceptance/bundle/presets/trigger_pause_status_whole_schedule_reference/nb.py b/acceptance/bundle/presets/trigger_pause_status_whole_schedule_reference/nb.py new file mode 100644 index 00000000000..7c87acfa794 --- /dev/null +++ b/acceptance/bundle/presets/trigger_pause_status_whole_schedule_reference/nb.py @@ -0,0 +1,2 @@ +# Databricks notebook source +print(1) diff --git a/acceptance/bundle/presets/trigger_pause_status_whole_schedule_reference/out.test.toml b/acceptance/bundle/presets/trigger_pause_status_whole_schedule_reference/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/presets/trigger_pause_status_whole_schedule_reference/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/presets/trigger_pause_status_whole_schedule_reference/output.txt b/acceptance/bundle/presets/trigger_pause_status_whole_schedule_reference/output.txt new file mode 100644 index 00000000000..3709ef8711a --- /dev/null +++ b/acceptance/bundle/presets/trigger_pause_status_whole_schedule_reference/output.txt @@ -0,0 +1,11 @@ + +>>> musterr [CLI] bundle validate +Error: expected a map to index "pause_status", found string + +Name: test-bundle +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Found 1 error diff --git a/acceptance/bundle/presets/trigger_pause_status_whole_schedule_reference/script b/acceptance/bundle/presets/trigger_pause_status_whole_schedule_reference/script new file mode 100644 index 00000000000..a7d47981807 --- /dev/null +++ b/acceptance/bundle/presets/trigger_pause_status_whole_schedule_reference/script @@ -0,0 +1,3 @@ +# trigger_pause_status needs to set pause_status inside the schedule, but the schedule of viaref is a reference to the whole +# schedule of another job and is still unresolved at that point. +trace musterr $CLI bundle validate diff --git a/acceptance/bundle/presets/trigger_pause_status_whole_schedule_reference/test.toml b/acceptance/bundle/presets/trigger_pause_status_whole_schedule_reference/test.toml new file mode 100644 index 00000000000..04077d22022 --- /dev/null +++ b/acceptance/bundle/presets/trigger_pause_status_whole_schedule_reference/test.toml @@ -0,0 +1 @@ +Badness = "trigger_pause_status fails with a confusing error when the schedule is a reference to the whole schedule of another job." diff --git a/acceptance/bundle/presets/trigger_pause_status_whole_trigger_reference/databricks.yml b/acceptance/bundle/presets/trigger_pause_status_whole_trigger_reference/databricks.yml new file mode 100644 index 00000000000..5c536a73738 --- /dev/null +++ b/acceptance/bundle/presets/trigger_pause_status_whole_trigger_reference/databricks.yml @@ -0,0 +1,17 @@ +bundle: + name: test-bundle + +presets: + trigger_pause_status: PAUSED + +resources: + jobs: + base: + name: base + schedule: {quartz_cron_expression: "0 0 * * * ?", timezone_id: UTC} + trigger: {file_arrival: {url: "s3://bucket/path"}} + tasks: [{task_key: t, notebook_task: {notebook_path: ./nb.py}}] + viaref: + name: viaref + trigger: ${resources.jobs.base.trigger} + tasks: [{task_key: t, notebook_task: {notebook_path: ./nb.py}}] diff --git a/acceptance/bundle/presets/trigger_pause_status_whole_trigger_reference/nb.py b/acceptance/bundle/presets/trigger_pause_status_whole_trigger_reference/nb.py new file mode 100644 index 00000000000..7c87acfa794 --- /dev/null +++ b/acceptance/bundle/presets/trigger_pause_status_whole_trigger_reference/nb.py @@ -0,0 +1,2 @@ +# Databricks notebook source +print(1) diff --git a/acceptance/bundle/presets/trigger_pause_status_whole_trigger_reference/out.test.toml b/acceptance/bundle/presets/trigger_pause_status_whole_trigger_reference/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/presets/trigger_pause_status_whole_trigger_reference/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/presets/trigger_pause_status_whole_trigger_reference/output.txt b/acceptance/bundle/presets/trigger_pause_status_whole_trigger_reference/output.txt new file mode 100644 index 00000000000..3709ef8711a --- /dev/null +++ b/acceptance/bundle/presets/trigger_pause_status_whole_trigger_reference/output.txt @@ -0,0 +1,11 @@ + +>>> musterr [CLI] bundle validate +Error: expected a map to index "pause_status", found string + +Name: test-bundle +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Found 1 error diff --git a/acceptance/bundle/presets/trigger_pause_status_whole_trigger_reference/script b/acceptance/bundle/presets/trigger_pause_status_whole_trigger_reference/script new file mode 100644 index 00000000000..6646655bab8 --- /dev/null +++ b/acceptance/bundle/presets/trigger_pause_status_whole_trigger_reference/script @@ -0,0 +1,2 @@ +# Same as with a schedule: pause_status cannot be set inside a trigger that is a reference to the whole trigger of another job. +trace musterr $CLI bundle validate diff --git a/acceptance/bundle/presets/trigger_pause_status_whole_trigger_reference/test.toml b/acceptance/bundle/presets/trigger_pause_status_whole_trigger_reference/test.toml new file mode 100644 index 00000000000..b1915e567cd --- /dev/null +++ b/acceptance/bundle/presets/trigger_pause_status_whole_trigger_reference/test.toml @@ -0,0 +1 @@ +Badness = "trigger_pause_status fails with a confusing error when the trigger is a reference to the whole trigger of another job." diff --git a/acceptance/bundle/resource_deps/preset_tags_not_applied_to_whole_map_reference/databricks.yml b/acceptance/bundle/resource_deps/preset_tags_not_applied_to_whole_map_reference/databricks.yml new file mode 100644 index 00000000000..9b2e3df1ddf --- /dev/null +++ b/acceptance/bundle/resource_deps/preset_tags_not_applied_to_whole_map_reference/databricks.yml @@ -0,0 +1,18 @@ +bundle: + name: test-bundle + +presets: + tags: + preset_tag: ${resources.jobs.j.id} + +resources: + schemas: + s: + name: s + catalog_name: main + properties: {owner: fixed} + jobs: + j: + name: j + # tags is a reference to a whole map, and the preset tag above is not added to it. + tags: ${resources.schemas.s.properties} diff --git a/acceptance/bundle/resource_deps/preset_tags_not_applied_to_whole_map_reference/out.test.toml b/acceptance/bundle/resource_deps/preset_tags_not_applied_to_whole_map_reference/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/resource_deps/preset_tags_not_applied_to_whole_map_reference/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resource_deps/preset_tags_not_applied_to_whole_map_reference/output.txt b/acceptance/bundle/resource_deps/preset_tags_not_applied_to_whole_map_reference/output.txt new file mode 100644 index 00000000000..06316029ed1 --- /dev/null +++ b/acceptance/bundle/resource_deps/preset_tags_not_applied_to_whole_map_reference/output.txt @@ -0,0 +1,13 @@ + +>>> [CLI] bundle plan -o json +{ + "tags": { + "owner": "fixed" + }, + "depends_on": [ + { + "node": "resources.schemas.s", + "label": "${resources.schemas.s.properties}" + } + ] +} diff --git a/acceptance/bundle/resource_deps/preset_tags_not_applied_to_whole_map_reference/script b/acceptance/bundle/resource_deps/preset_tags_not_applied_to_whole_map_reference/script new file mode 100644 index 00000000000..a28b61a766a --- /dev/null +++ b/acceptance/bundle/resource_deps/preset_tags_not_applied_to_whole_map_reference/script @@ -0,0 +1,5 @@ +# The preset tag is not applied to a job whose tags is a reference to a whole map: the job gets the +# tags of the referenced schema only, and depends on the schema. +# The preset value references the job itself. If the preset tag were applied, this plan would fail +# with a cycle error. +trace $CLI bundle plan -o json | jq '.plan["resources.jobs.j"] | {tags: .new_state.value.tags, depends_on}' diff --git a/acceptance/bundle/resource_deps/preset_tags_not_applied_to_whole_map_reference/test.toml b/acceptance/bundle/resource_deps/preset_tags_not_applied_to_whole_map_reference/test.toml new file mode 100644 index 00000000000..e329f5af27f --- /dev/null +++ b/acceptance/bundle/resource_deps/preset_tags_not_applied_to_whole_map_reference/test.toml @@ -0,0 +1 @@ +Badness = "presets.tags are silently not applied to a job whose tags is a reference to a whole map." diff --git a/acceptance/bundle/validate/required_struct_reference/databricks.yml b/acceptance/bundle/validate/required_struct_reference/databricks.yml new file mode 100644 index 00000000000..60e02ad20cb --- /dev/null +++ b/acceptance/bundle/validate/required_struct_reference/databricks.yml @@ -0,0 +1,17 @@ +bundle: + name: test-bundle + +resources: + clusters: + source: + cluster_name: source + spark_version: 15.4.x-scala2.12 + is_single_node: true + target: + cluster_name: target + spark_version: 15.4.x-scala2.12 + # clients is a required field of workload_type. Both of its fields are references. + workload_type: + clients: + notebooks: ${resources.clusters.source.is_single_node} + jobs: ${resources.clusters.source.is_single_node} diff --git a/acceptance/bundle/validate/required_struct_reference/out.test.toml b/acceptance/bundle/validate/required_struct_reference/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/validate/required_struct_reference/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/validate/required_struct_reference/output.txt b/acceptance/bundle/validate/required_struct_reference/output.txt new file mode 100644 index 00000000000..2fc3e5bc32e --- /dev/null +++ b/acceptance/bundle/validate/required_struct_reference/output.txt @@ -0,0 +1,8 @@ + +>>> [CLI] bundle validate -o json +{ + "clients": { + "jobs": "${resources.clusters.source.is_single_node}", + "notebooks": "${resources.clusters.source.is_single_node}" + } +} diff --git a/acceptance/bundle/validate/required_struct_reference/script b/acceptance/bundle/validate/required_struct_reference/script new file mode 100644 index 00000000000..5d37f63d12b --- /dev/null +++ b/acceptance/bundle/validate/required_struct_reference/script @@ -0,0 +1,2 @@ +# workload_type.clients is required. A clients whose fields are all references is not reported as missing. +trace $CLI bundle validate -o json | jq .resources.clusters.target.workload_type