diff --git a/acceptance/bundle/validate/app_resource_duplicate_name_location/databricks.yml b/acceptance/bundle/validate/app_resource_duplicate_name_location/databricks.yml new file mode 100644 index 00000000000..7ae4a68fa02 --- /dev/null +++ b/acceptance/bundle/validate/app_resource_duplicate_name_location/databricks.yml @@ -0,0 +1,17 @@ +bundle: + name: test-bundle + +resources: + apps: + app: + name: test-app + source_code_path: . + resources: + - name: job + job: + id: "1" + permission: INVALID + - name: job + job: + id: "2" + permission: INVALID diff --git a/acceptance/bundle/validate/anchor_containers/out.test.toml b/acceptance/bundle/validate/app_resource_duplicate_name_location/out.test.toml similarity index 100% rename from acceptance/bundle/validate/anchor_containers/out.test.toml rename to acceptance/bundle/validate/app_resource_duplicate_name_location/out.test.toml diff --git a/acceptance/bundle/validate/app_resource_duplicate_name_location/output.txt b/acceptance/bundle/validate/app_resource_duplicate_name_location/output.txt new file mode 100644 index 00000000000..a8eac9a3ace --- /dev/null +++ b/acceptance/bundle/validate/app_resource_duplicate_name_location/output.txt @@ -0,0 +1,28 @@ + +>>> [CLI] bundle validate +Warning: invalid value "INVALID" for enum field. Valid values are [CAN_MANAGE CAN_MANAGE_RUN CAN_VIEW IS_OWNER] + at resources.apps.app.resources[0].job.permission + in databricks.yml:13:25 + +Name: test-bundle +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Found 1 warning + +>>> [CLI] bundle validate -o json +Warning: invalid value "INVALID" for enum field. Valid values are [CAN_MANAGE CAN_MANAGE_RUN CAN_VIEW IS_OWNER] + at resources.apps.app.resources[0].job.permission + in databricks.yml:13:25 + +[ + { + "job": { + "id": "2", + "permission": "INVALID" + }, + "name": "job" + } +] diff --git a/acceptance/bundle/validate/app_resource_duplicate_name_location/script b/acceptance/bundle/validate/app_resource_duplicate_name_location/script new file mode 100644 index 00000000000..3531c01dcef --- /dev/null +++ b/acceptance/bundle/validate/app_resource_duplicate_name_location/script @@ -0,0 +1,5 @@ +# A diagnostic on a field of a duplicated app resource entry points at the first entry when the duplicate's value is the same. +errcode trace $CLI bundle validate + +# The entries are merged by name, the later one wins. +trace $CLI bundle validate -o json | jq .resources.apps.app.resources diff --git a/acceptance/bundle/validate/includes/clears_unforced_preset_values/databricks.yml b/acceptance/bundle/validate/includes/clears_unforced_preset_values/databricks.yml new file mode 100644 index 00000000000..0a6c0479fee --- /dev/null +++ b/acceptance/bundle/validate/includes/clears_unforced_preset_values/databricks.yml @@ -0,0 +1,15 @@ +bundle: + name: test-bundle + +include: + - override.yml + +presets: + name_prefix: "base-" + jobs_max_concurrent_runs: 7 + artifacts_dynamic_version: true + +resources: + jobs: + j: + name: job diff --git a/acceptance/bundle/validate/complex_variable_default_merge/out.test.toml b/acceptance/bundle/validate/includes/clears_unforced_preset_values/out.test.toml similarity index 100% rename from acceptance/bundle/validate/complex_variable_default_merge/out.test.toml rename to acceptance/bundle/validate/includes/clears_unforced_preset_values/out.test.toml diff --git a/acceptance/bundle/validate/includes/clears_unforced_preset_values/output.txt b/acceptance/bundle/validate/includes/clears_unforced_preset_values/output.txt new file mode 100644 index 00000000000..82b92b1e6ad --- /dev/null +++ b/acceptance/bundle/validate/includes/clears_unforced_preset_values/output.txt @@ -0,0 +1,9 @@ + +>>> [CLI] bundle validate -o json +{ + "presets": {}, + "job": { + "name": "job", + "max_concurrent_runs": 1 + } +} diff --git a/acceptance/bundle/validate/includes/clears_unforced_preset_values/override.yml b/acceptance/bundle/validate/includes/clears_unforced_preset_values/override.yml new file mode 100644 index 00000000000..dc413ce0648 --- /dev/null +++ b/acceptance/bundle/validate/includes/clears_unforced_preset_values/override.yml @@ -0,0 +1,4 @@ +presets: + name_prefix: "" + jobs_max_concurrent_runs: 0 + artifacts_dynamic_version: false diff --git a/acceptance/bundle/validate/includes/clears_unforced_preset_values/script b/acceptance/bundle/validate/includes/clears_unforced_preset_values/script new file mode 100644 index 00000000000..c8d6c1aace9 --- /dev/null +++ b/acceptance/bundle/validate/includes/clears_unforced_preset_values/script @@ -0,0 +1,2 @@ +# An include can override fields with "", 0 and false. +trace $CLI bundle validate -o json | jq '{presets, job: (.resources.jobs.j | {name, max_concurrent_runs})}' diff --git a/acceptance/bundle/validate/include_locations/databricks.yml b/acceptance/bundle/validate/includes/locations/databricks.yml similarity index 100% rename from acceptance/bundle/validate/include_locations/databricks.yml rename to acceptance/bundle/validate/includes/locations/databricks.yml diff --git a/acceptance/bundle/validate/definitions_yaml_anchors/out.test.toml b/acceptance/bundle/validate/includes/locations/out.test.toml similarity index 100% rename from acceptance/bundle/validate/definitions_yaml_anchors/out.test.toml rename to acceptance/bundle/validate/includes/locations/out.test.toml diff --git a/acceptance/bundle/validate/include_locations/output.dev.json b/acceptance/bundle/validate/includes/locations/output.dev.json similarity index 100% rename from acceptance/bundle/validate/include_locations/output.dev.json rename to acceptance/bundle/validate/includes/locations/output.dev.json diff --git a/acceptance/bundle/validate/include_locations/output.prod.json b/acceptance/bundle/validate/includes/locations/output.prod.json similarity index 100% rename from acceptance/bundle/validate/include_locations/output.prod.json rename to acceptance/bundle/validate/includes/locations/output.prod.json diff --git a/acceptance/bundle/validate/include_locations/output.txt b/acceptance/bundle/validate/includes/locations/output.txt similarity index 100% rename from acceptance/bundle/validate/include_locations/output.txt rename to acceptance/bundle/validate/includes/locations/output.txt diff --git a/acceptance/bundle/validate/include_locations/override.yml b/acceptance/bundle/validate/includes/locations/override.yml similarity index 100% rename from acceptance/bundle/validate/include_locations/override.yml rename to acceptance/bundle/validate/includes/locations/override.yml diff --git a/acceptance/bundle/validate/include_locations/resources/job.yml b/acceptance/bundle/validate/includes/locations/resources/job.yml similarity index 100% rename from acceptance/bundle/validate/include_locations/resources/job.yml rename to acceptance/bundle/validate/includes/locations/resources/job.yml diff --git a/acceptance/bundle/validate/include_locations/script b/acceptance/bundle/validate/includes/locations/script similarity index 100% rename from acceptance/bundle/validate/include_locations/script rename to acceptance/bundle/validate/includes/locations/script diff --git a/acceptance/bundle/validate/include_locations/src/dev.py b/acceptance/bundle/validate/includes/locations/src/dev.py similarity index 100% rename from acceptance/bundle/validate/include_locations/src/dev.py rename to acceptance/bundle/validate/includes/locations/src/dev.py diff --git a/acceptance/bundle/validate/include_locations/src/prod.py b/acceptance/bundle/validate/includes/locations/src/prod.py similarity index 100% rename from acceptance/bundle/validate/include_locations/src/prod.py rename to acceptance/bundle/validate/includes/locations/src/prod.py diff --git a/acceptance/bundle/validate/includes/merge_failure_short_circuits_diagnostics/a.yml b/acceptance/bundle/validate/includes/merge_failure_short_circuits_diagnostics/a.yml new file mode 100644 index 00000000000..5a6fbc1e893 --- /dev/null +++ b/acceptance/bundle/validate/includes/merge_failure_short_circuits_diagnostics/a.yml @@ -0,0 +1,3 @@ +variables: + x: + default: incompatible-scalar diff --git a/acceptance/bundle/validate/includes/merge_failure_short_circuits_diagnostics/b.yml b/acceptance/bundle/validate/includes/merge_failure_short_circuits_diagnostics/b.yml new file mode 100644 index 00000000000..61f0de50e9e --- /dev/null +++ b/acceptance/bundle/validate/includes/merge_failure_short_circuits_diagnostics/b.yml @@ -0,0 +1,2 @@ +bundle: + unexpected_after_failure: true diff --git a/acceptance/bundle/validate/includes/merge_failure_short_circuits_diagnostics/databricks.yml b/acceptance/bundle/validate/includes/merge_failure_short_circuits_diagnostics/databricks.yml new file mode 100644 index 00000000000..fdd8bfa87b3 --- /dev/null +++ b/acceptance/bundle/validate/includes/merge_failure_short_circuits_diagnostics/databricks.yml @@ -0,0 +1,12 @@ +bundle: + name: test-bundle + +include: + - a.yml + - b.yml + +variables: + x: + type: complex + default: + key: value diff --git a/acceptance/bundle/validate/duplicate_yaml_merge_key/out.test.toml b/acceptance/bundle/validate/includes/merge_failure_short_circuits_diagnostics/out.test.toml similarity index 100% rename from acceptance/bundle/validate/duplicate_yaml_merge_key/out.test.toml rename to acceptance/bundle/validate/includes/merge_failure_short_circuits_diagnostics/out.test.toml diff --git a/acceptance/bundle/validate/includes/merge_failure_short_circuits_diagnostics/output.txt b/acceptance/bundle/validate/includes/merge_failure_short_circuits_diagnostics/output.txt new file mode 100644 index 00000000000..f5f6443e809 --- /dev/null +++ b/acceptance/bundle/validate/includes/merge_failure_short_circuits_diagnostics/output.txt @@ -0,0 +1,9 @@ + +>>> [CLI] bundle validate +Error: cannot merge map with string + +Name: test-bundle + +Found 1 error + +Exit code: 1 diff --git a/acceptance/bundle/validate/includes/merge_failure_short_circuits_diagnostics/script b/acceptance/bundle/validate/includes/merge_failure_short_circuits_diagnostics/script new file mode 100644 index 00000000000..6f34e8010d2 --- /dev/null +++ b/acceptance/bundle/validate/includes/merge_failure_short_circuits_diagnostics/script @@ -0,0 +1,2 @@ +# Includes after one that fails to merge do not report their diagnostics. +errcode trace $CLI bundle validate diff --git a/acceptance/bundle/validate/includes/merge_failure_short_circuits_diagnostics/test.toml b/acceptance/bundle/validate/includes/merge_failure_short_circuits_diagnostics/test.toml new file mode 100644 index 00000000000..b11d0b76cd4 --- /dev/null +++ b/acceptance/bundle/validate/includes/merge_failure_short_circuits_diagnostics/test.toml @@ -0,0 +1 @@ +Badness = "The merge error has no location or path, and diagnostics of includes loaded after the failed merge (the unknown field in b.yml) are not reported, although they do not depend on the merge." diff --git a/acceptance/bundle/validate/nulls/resources_container/databricks.yml b/acceptance/bundle/validate/nulls/resources_container/databricks.yml new file mode 100644 index 00000000000..339364daf47 --- /dev/null +++ b/acceptance/bundle/validate/nulls/resources_container/databricks.yml @@ -0,0 +1,4 @@ +bundle: + name: test-bundle + +resources: null diff --git a/acceptance/bundle/validate/include_locations/out.test.toml b/acceptance/bundle/validate/nulls/resources_container/out.test.toml similarity index 100% rename from acceptance/bundle/validate/include_locations/out.test.toml rename to acceptance/bundle/validate/nulls/resources_container/out.test.toml diff --git a/acceptance/bundle/validate/nulls/resources_container/output.txt b/acceptance/bundle/validate/nulls/resources_container/output.txt new file mode 100644 index 00000000000..1241151fda5 --- /dev/null +++ b/acceptance/bundle/validate/nulls/resources_container/output.txt @@ -0,0 +1,9 @@ + +>>> [CLI] bundle validate +Error: expected a map at "resources", found nil + +Name: test-bundle + +Found 1 error + +Exit code: 1 diff --git a/acceptance/bundle/validate/nulls/resources_container/script b/acceptance/bundle/validate/nulls/resources_container/script new file mode 100644 index 00000000000..253cc3d205d --- /dev/null +++ b/acceptance/bundle/validate/nulls/resources_container/script @@ -0,0 +1,2 @@ +# resources: null is rejected. +errcode trace $CLI bundle validate diff --git a/acceptance/bundle/validate/nulls/resources_container/test.toml b/acceptance/bundle/validate/nulls/resources_container/test.toml new file mode 100644 index 00000000000..022e0f2c7e9 --- /dev/null +++ b/acceptance/bundle/validate/nulls/resources_container/test.toml @@ -0,0 +1 @@ +Badness = 'An empty (null) resources section is an error without a location, while empty targets, variables or sync sections are accepted.' diff --git a/acceptance/bundle/validate/nulls/schedule_default_materialization/databricks.yml b/acceptance/bundle/validate/nulls/schedule_default_materialization/databricks.yml new file mode 100644 index 00000000000..d68996de3ce --- /dev/null +++ b/acceptance/bundle/validate/nulls/schedule_default_materialization/databricks.yml @@ -0,0 +1,8 @@ +bundle: + name: test-bundle + +resources: + jobs: + j: + name: test + schedule: null diff --git a/acceptance/bundle/validate/target_variable_full_def_merge_key/out.test.toml b/acceptance/bundle/validate/nulls/schedule_default_materialization/out.test.toml similarity index 100% rename from acceptance/bundle/validate/target_variable_full_def_merge_key/out.test.toml rename to acceptance/bundle/validate/nulls/schedule_default_materialization/out.test.toml diff --git a/acceptance/bundle/validate/nulls/schedule_default_materialization/output.txt b/acceptance/bundle/validate/nulls/schedule_default_materialization/output.txt new file mode 100644 index 00000000000..8cc844569f5 --- /dev/null +++ b/acceptance/bundle/validate/nulls/schedule_default_materialization/output.txt @@ -0,0 +1,13 @@ + +>>> [CLI] bundle validate -o json +Warning: required field "quartz_cron_expression" is not set + at resources.jobs.j.schedule + in databricks.yml:8:17 + +Warning: required field "timezone_id" is not set + at resources.jobs.j.schedule + in databricks.yml:8:17 + +{ + "schedule": null +} diff --git a/acceptance/bundle/validate/nulls/schedule_default_materialization/script b/acceptance/bundle/validate/nulls/schedule_default_materialization/script new file mode 100644 index 00000000000..0c5e31164ff --- /dev/null +++ b/acceptance/bundle/validate/nulls/schedule_default_materialization/script @@ -0,0 +1,2 @@ +# Defaults are not applied under an explicitly null block (schedule stays null). +trace $CLI bundle validate -o json | jq '.resources.jobs.j | {schedule}' diff --git a/acceptance/bundle/validate/nulls/schedule_default_materialization/test.toml b/acceptance/bundle/validate/nulls/schedule_default_materialization/test.toml new file mode 100644 index 00000000000..68e81befdb7 --- /dev/null +++ b/acceptance/bundle/validate/nulls/schedule_default_materialization/test.toml @@ -0,0 +1 @@ +Badness = 'An explicitly null schedule is reported as missing its required fields.' diff --git a/acceptance/bundle/validate/references/resolved_invalid_map_value/databricks.yml b/acceptance/bundle/validate/references/resolved_invalid_map_value/databricks.yml new file mode 100644 index 00000000000..3fffd4ad478 --- /dev/null +++ b/acceptance/bundle/validate/references/resolved_invalid_map_value/databricks.yml @@ -0,0 +1,16 @@ +bundle: + name: test-bundle + +variables: + bad: + type: complex + default: + x: 1 + +resources: + jobs: + j: + name: test + tags: + "bad.key[0]": ${var.bad} + good: ok diff --git a/acceptance/bundle/validate/target_variable_merge_key_precedence/out.test.toml b/acceptance/bundle/validate/references/resolved_invalid_map_value/out.test.toml similarity index 100% rename from acceptance/bundle/validate/target_variable_merge_key_precedence/out.test.toml rename to acceptance/bundle/validate/references/resolved_invalid_map_value/out.test.toml diff --git a/acceptance/bundle/validate/references/resolved_invalid_map_value/output.txt b/acceptance/bundle/validate/references/resolved_invalid_map_value/output.txt new file mode 100644 index 00000000000..07bfe6c7450 --- /dev/null +++ b/acceptance/bundle/validate/references/resolved_invalid_map_value/output.txt @@ -0,0 +1,9 @@ + +>>> [CLI] bundle validate -o json +Warning: expected string, found map + at resources.jobs.j.tags['bad.key[0]'] + in databricks.yml:15:23 + +{ + "good": "ok" +} diff --git a/acceptance/bundle/validate/references/resolved_invalid_map_value/script b/acceptance/bundle/validate/references/resolved_invalid_map_value/script new file mode 100644 index 00000000000..835edef52e9 --- /dev/null +++ b/acceptance/bundle/validate/references/resolved_invalid_map_value/script @@ -0,0 +1,2 @@ +# A map value whose reference resolves to the wrong type is dropped, with a warning at its path. +trace $CLI bundle validate -o json | jq .resources.jobs.j.tags diff --git a/acceptance/bundle/validate/references/resolved_invalid_sequence_item/databricks.yml b/acceptance/bundle/validate/references/resolved_invalid_sequence_item/databricks.yml new file mode 100644 index 00000000000..90dd5f6d362 --- /dev/null +++ b/acceptance/bundle/validate/references/resolved_invalid_sequence_item/databricks.yml @@ -0,0 +1,15 @@ +bundle: + name: test-bundle + +variables: + bad: + type: complex + default: + x: 1 + +resources: + jobs: + j: + name: test + email_notifications: + on_failure: ["${var.bad}", "ok@example.test"] diff --git a/acceptance/bundle/validate/target_variable_shorthand_alias/out.test.toml b/acceptance/bundle/validate/references/resolved_invalid_sequence_item/out.test.toml similarity index 100% rename from acceptance/bundle/validate/target_variable_shorthand_alias/out.test.toml rename to acceptance/bundle/validate/references/resolved_invalid_sequence_item/out.test.toml diff --git a/acceptance/bundle/validate/references/resolved_invalid_sequence_item/output.txt b/acceptance/bundle/validate/references/resolved_invalid_sequence_item/output.txt new file mode 100644 index 00000000000..03952e879af --- /dev/null +++ b/acceptance/bundle/validate/references/resolved_invalid_sequence_item/output.txt @@ -0,0 +1,11 @@ + +>>> [CLI] bundle validate -o json +Warning: expected string, found map + at resources.jobs.j.email_notifications.on_failure[0] + in databricks.yml:15:22 + +{ + "on_failure": [ + "ok@example.test" + ] +} diff --git a/acceptance/bundle/validate/references/resolved_invalid_sequence_item/script b/acceptance/bundle/validate/references/resolved_invalid_sequence_item/script new file mode 100644 index 00000000000..61ba88b4d75 --- /dev/null +++ b/acceptance/bundle/validate/references/resolved_invalid_sequence_item/script @@ -0,0 +1,2 @@ +# A list element whose reference resolves to the wrong type is dropped, with a warning at its path. +trace $CLI bundle validate -o json | jq .resources.jobs.j.email_notifications diff --git a/acceptance/bundle/validate/references/to_omitted_struct_defaults/databricks.yml b/acceptance/bundle/validate/references/to_omitted_struct_defaults/databricks.yml new file mode 100644 index 00000000000..b3d428b5e16 --- /dev/null +++ b/acceptance/bundle/validate/references/to_omitted_struct_defaults/databricks.yml @@ -0,0 +1,7 @@ +bundle: + name: test-bundle + +variables: + git_info: + type: complex + default: ${bundle.git} diff --git a/acceptance/bundle/validate/variable_overrides_file_complex_path/out.test.toml b/acceptance/bundle/validate/references/to_omitted_struct_defaults/out.test.toml similarity index 100% rename from acceptance/bundle/validate/variable_overrides_file_complex_path/out.test.toml rename to acceptance/bundle/validate/references/to_omitted_struct_defaults/out.test.toml diff --git a/acceptance/bundle/validate/references/to_omitted_struct_defaults/output.txt b/acceptance/bundle/validate/references/to_omitted_struct_defaults/output.txt new file mode 100644 index 00000000000..479ce92fe43 --- /dev/null +++ b/acceptance/bundle/validate/references/to_omitted_struct_defaults/output.txt @@ -0,0 +1,19 @@ + +>>> [CLI] bundle validate -o json +{ + "default": { + "actual_branch": "", + "branch": "", + "bundle_root_path": ".", + "commit": "", + "origin_url": "" + }, + "type": "complex", + "value": { + "actual_branch": "", + "branch": "", + "bundle_root_path": ".", + "commit": "", + "origin_url": "" + } +} diff --git a/acceptance/bundle/validate/references/to_omitted_struct_defaults/script b/acceptance/bundle/validate/references/to_omitted_struct_defaults/script new file mode 100644 index 00000000000..cb09a7b5bec --- /dev/null +++ b/acceptance/bundle/validate/references/to_omitted_struct_defaults/script @@ -0,0 +1,3 @@ +# A reference to a struct that is not set in the configuration (here outside a git repository) +# resolves to the struct with its fields filled in, as empty values. +trace $CLI bundle validate -o json | jq .variables.git_info diff --git a/acceptance/bundle/validate/targets/inherited_from_root_merge/databricks.yml b/acceptance/bundle/validate/targets/inherited_from_root_merge/databricks.yml new file mode 100644 index 00000000000..e5a45a569c9 --- /dev/null +++ b/acceptance/bundle/validate/targets/inherited_from_root_merge/databricks.yml @@ -0,0 +1,15 @@ +bundle: + name: test-bundle + +variables: + x: + default: base + +definitions: + root_defaults: &root_defaults + targets: + dev: + variables: + x: overridden + +<<: *root_defaults diff --git a/acceptance/bundle/validate/yaml_anchor_cycle_nested/out.test.toml b/acceptance/bundle/validate/targets/inherited_from_root_merge/out.test.toml similarity index 100% rename from acceptance/bundle/validate/yaml_anchor_cycle_nested/out.test.toml rename to acceptance/bundle/validate/targets/inherited_from_root_merge/out.test.toml diff --git a/acceptance/bundle/validate/targets/inherited_from_root_merge/output.txt b/acceptance/bundle/validate/targets/inherited_from_root_merge/output.txt new file mode 100644 index 00000000000..d3be1ebf0eb --- /dev/null +++ b/acceptance/bundle/validate/targets/inherited_from_root_merge/output.txt @@ -0,0 +1,6 @@ + +>>> [CLI] bundle validate -t dev -o json +{ + "default": "overridden", + "value": "overridden" +} diff --git a/acceptance/bundle/validate/targets/inherited_from_root_merge/script b/acceptance/bundle/validate/targets/inherited_from_root_merge/script new file mode 100644 index 00000000000..b51a0e30bfe --- /dev/null +++ b/acceptance/bundle/validate/targets/inherited_from_root_merge/script @@ -0,0 +1,2 @@ +# Target variable shorthands in a targets section inherited through a root merge key are applied. +trace $CLI bundle validate -t dev -o json | jq .variables.x diff --git a/acceptance/bundle/validate/targets/scalar_rewrite_diagnostic/databricks.yml b/acceptance/bundle/validate/targets/scalar_rewrite_diagnostic/databricks.yml new file mode 100644 index 00000000000..2583829e912 --- /dev/null +++ b/acceptance/bundle/validate/targets/scalar_rewrite_diagnostic/databricks.yml @@ -0,0 +1,4 @@ +bundle: + name: test-bundle + +targets: false diff --git a/acceptance/bundle/validate/targets/scalar_rewrite_diagnostic/out.test.toml b/acceptance/bundle/validate/targets/scalar_rewrite_diagnostic/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/validate/targets/scalar_rewrite_diagnostic/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/validate/targets/scalar_rewrite_diagnostic/output.txt b/acceptance/bundle/validate/targets/scalar_rewrite_diagnostic/output.txt new file mode 100644 index 00000000000..8020d180559 --- /dev/null +++ b/acceptance/bundle/validate/targets/scalar_rewrite_diagnostic/output.txt @@ -0,0 +1,8 @@ + +>>> [CLI] bundle validate +Error: failed to rewrite [TEST_TMP_DIR]/databricks.yml: expected a map or sequence, found bool + + +Found 1 error + +Exit code: 1 diff --git a/acceptance/bundle/validate/targets/scalar_rewrite_diagnostic/script b/acceptance/bundle/validate/targets/scalar_rewrite_diagnostic/script new file mode 100644 index 00000000000..4f218738be7 --- /dev/null +++ b/acceptance/bundle/validate/targets/scalar_rewrite_diagnostic/script @@ -0,0 +1,2 @@ +# A scalar targets section is an error. +errcode trace $CLI bundle validate diff --git a/acceptance/bundle/validate/targets/scalar_rewrite_diagnostic/test.toml b/acceptance/bundle/validate/targets/scalar_rewrite_diagnostic/test.toml new file mode 100644 index 00000000000..b1bb8151234 --- /dev/null +++ b/acceptance/bundle/validate/targets/scalar_rewrite_diagnostic/test.toml @@ -0,0 +1 @@ +Badness = 'The error names an internal step ("failed to rewrite") and has no location, instead of reporting a type error at targets.' diff --git a/acceptance/bundle/validate/targets/test.toml b/acceptance/bundle/validate/targets/test.toml new file mode 100644 index 00000000000..e9cede5e28a --- /dev/null +++ b/acceptance/bundle/validate/targets/test.toml @@ -0,0 +1,4 @@ +# Errors for invalid YAML print the full path of the file; normalize Windows separators. +[[Repls]] +Old = '\\' +New = '/' diff --git a/acceptance/bundle/validate/complex_variable_default_merge/a.yml b/acceptance/bundle/validate/variables/complex_default_merge/a.yml similarity index 100% rename from acceptance/bundle/validate/complex_variable_default_merge/a.yml rename to acceptance/bundle/validate/variables/complex_default_merge/a.yml diff --git a/acceptance/bundle/validate/complex_variable_default_merge/b.yml b/acceptance/bundle/validate/variables/complex_default_merge/b.yml similarity index 100% rename from acceptance/bundle/validate/complex_variable_default_merge/b.yml rename to acceptance/bundle/validate/variables/complex_default_merge/b.yml diff --git a/acceptance/bundle/validate/complex_variable_default_merge/databricks.yml b/acceptance/bundle/validate/variables/complex_default_merge/databricks.yml similarity index 100% rename from acceptance/bundle/validate/complex_variable_default_merge/databricks.yml rename to acceptance/bundle/validate/variables/complex_default_merge/databricks.yml diff --git a/acceptance/bundle/validate/variables/complex_default_merge/out.test.toml b/acceptance/bundle/validate/variables/complex_default_merge/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/validate/variables/complex_default_merge/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/validate/complex_variable_default_merge/output.txt b/acceptance/bundle/validate/variables/complex_default_merge/output.txt similarity index 100% rename from acceptance/bundle/validate/complex_variable_default_merge/output.txt rename to acceptance/bundle/validate/variables/complex_default_merge/output.txt diff --git a/acceptance/bundle/validate/complex_variable_default_merge/script b/acceptance/bundle/validate/variables/complex_default_merge/script similarity index 100% rename from acceptance/bundle/validate/complex_variable_default_merge/script rename to acceptance/bundle/validate/variables/complex_default_merge/script diff --git a/acceptance/bundle/validate/variables/complex_default_nested_timestamp/databricks.yml b/acceptance/bundle/validate/variables/complex_default_nested_timestamp/databricks.yml new file mode 100644 index 00000000000..43221f5f33a --- /dev/null +++ b/acceptance/bundle/validate/variables/complex_default_nested_timestamp/databricks.yml @@ -0,0 +1,8 @@ +bundle: + name: test-bundle + +variables: + payload: + type: complex + default: + created: 2020-01-02 diff --git a/acceptance/bundle/validate/variables/complex_default_nested_timestamp/out.test.toml b/acceptance/bundle/validate/variables/complex_default_nested_timestamp/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/validate/variables/complex_default_nested_timestamp/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/validate/variables/complex_default_nested_timestamp/output.txt b/acceptance/bundle/validate/variables/complex_default_nested_timestamp/output.txt new file mode 100644 index 00000000000..3c236fb1c2b --- /dev/null +++ b/acceptance/bundle/validate/variables/complex_default_nested_timestamp/output.txt @@ -0,0 +1,10 @@ + +>>> [CLI] bundle validate +Warn: unable to convert typed configuration to dynamic configuration: cannot convert struct field to dynamic type "time": src=time.Date(2020, time.January, 2, 0, 0, 0, 0, time.UTC) ref=time.Date(2020, time.January, 2, 0, 0, 0, 0, time.UTC) +Error: exit error: cannot convert struct field to dynamic type "time": src=time.Date(2020, time.January, 2, 0, 0, 0, 0, time.UTC) ref=time.Date(2020, time.January, 2, 0, 0, 0, 0, time.UTC) + +Name: test-bundle + +Found 1 error + +Exit code: 1 diff --git a/acceptance/bundle/validate/variables/complex_default_nested_timestamp/script b/acceptance/bundle/validate/variables/complex_default_nested_timestamp/script new file mode 100644 index 00000000000..e4d20dafe80 --- /dev/null +++ b/acceptance/bundle/validate/variables/complex_default_nested_timestamp/script @@ -0,0 +1,2 @@ +# A timestamp nested inside a complex variable default. +errcode trace $CLI bundle validate diff --git a/acceptance/bundle/validate/variables/complex_default_nested_timestamp/test.toml b/acceptance/bundle/validate/variables/complex_default_nested_timestamp/test.toml new file mode 100644 index 00000000000..b1b00a5e46a --- /dev/null +++ b/acceptance/bundle/validate/variables/complex_default_nested_timestamp/test.toml @@ -0,0 +1 @@ +Badness = "A valid config fails with an internal conversion error: the nested timestamp cannot be converted back to the dynamic configuration." diff --git a/acceptance/bundle/validate/variables/complex_unknown_field_location/databricks.yml b/acceptance/bundle/validate/variables/complex_unknown_field_location/databricks.yml new file mode 100644 index 00000000000..a337abdcff1 --- /dev/null +++ b/acceptance/bundle/validate/variables/complex_unknown_field_location/databricks.yml @@ -0,0 +1,13 @@ +bundle: + name: test-bundle + +variables: + job_config: + type: complex + default: + name: test + misspelled_field: value + +resources: + jobs: + j: ${var.job_config} diff --git a/acceptance/bundle/validate/variables/complex_unknown_field_location/out.test.toml b/acceptance/bundle/validate/variables/complex_unknown_field_location/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/validate/variables/complex_unknown_field_location/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/validate/variables/complex_unknown_field_location/output.txt b/acceptance/bundle/validate/variables/complex_unknown_field_location/output.txt new file mode 100644 index 00000000000..118399cba34 --- /dev/null +++ b/acceptance/bundle/validate/variables/complex_unknown_field_location/output.txt @@ -0,0 +1,13 @@ + +>>> [CLI] bundle validate +Warning: unknown field: misspelled_field + at resources.jobs.j + in databricks.yml:9:7 + +Name: test-bundle +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Found 1 warning diff --git a/acceptance/bundle/validate/variables/complex_unknown_field_location/script b/acceptance/bundle/validate/variables/complex_unknown_field_location/script new file mode 100644 index 00000000000..4b8218079e8 --- /dev/null +++ b/acceptance/bundle/validate/variables/complex_unknown_field_location/script @@ -0,0 +1,2 @@ +# An unknown field inside a complex variable used as a whole resource keeps the location of its key. +errcode trace $CLI bundle validate diff --git a/acceptance/bundle/validate/variable_overrides_file_complex_path/databricks.yml b/acceptance/bundle/validate/variables/overrides_file_complex_path/databricks.yml similarity index 100% rename from acceptance/bundle/validate/variable_overrides_file_complex_path/databricks.yml rename to acceptance/bundle/validate/variables/overrides_file_complex_path/databricks.yml diff --git a/acceptance/bundle/validate/variables/overrides_file_complex_path/out.test.toml b/acceptance/bundle/validate/variables/overrides_file_complex_path/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/validate/variables/overrides_file_complex_path/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/validate/variable_overrides_file_complex_path/output.txt b/acceptance/bundle/validate/variables/overrides_file_complex_path/output.txt similarity index 100% rename from acceptance/bundle/validate/variable_overrides_file_complex_path/output.txt rename to acceptance/bundle/validate/variables/overrides_file_complex_path/output.txt diff --git a/acceptance/bundle/validate/variable_overrides_file_complex_path/script b/acceptance/bundle/validate/variables/overrides_file_complex_path/script similarity index 100% rename from acceptance/bundle/validate/variable_overrides_file_complex_path/script rename to acceptance/bundle/validate/variables/overrides_file_complex_path/script diff --git a/acceptance/bundle/validate/variable_overrides_file_complex_path/src/lib.whl b/acceptance/bundle/validate/variables/overrides_file_complex_path/src/lib.whl similarity index 100% rename from acceptance/bundle/validate/variable_overrides_file_complex_path/src/lib.whl rename to acceptance/bundle/validate/variables/overrides_file_complex_path/src/lib.whl diff --git a/acceptance/bundle/validate/variable_overrides_file_complex_path/src/main.py b/acceptance/bundle/validate/variables/overrides_file_complex_path/src/main.py similarity index 100% rename from acceptance/bundle/validate/variable_overrides_file_complex_path/src/main.py rename to acceptance/bundle/validate/variables/overrides_file_complex_path/src/main.py diff --git a/acceptance/bundle/validate/variable_overrides_file_complex_path/test.toml b/acceptance/bundle/validate/variables/overrides_file_complex_path/test.toml similarity index 100% rename from acceptance/bundle/validate/variable_overrides_file_complex_path/test.toml rename to acceptance/bundle/validate/variables/overrides_file_complex_path/test.toml diff --git a/acceptance/bundle/validate/variable_overrides_file_complex_path/variable-overrides.json b/acceptance/bundle/validate/variables/overrides_file_complex_path/variable-overrides.json similarity index 100% rename from acceptance/bundle/validate/variable_overrides_file_complex_path/variable-overrides.json rename to acceptance/bundle/validate/variables/overrides_file_complex_path/variable-overrides.json diff --git a/acceptance/bundle/validate/variables/target_inherited_from_target_merge/databricks.yml b/acceptance/bundle/validate/variables/target_inherited_from_target_merge/databricks.yml new file mode 100644 index 00000000000..497c570326c --- /dev/null +++ b/acceptance/bundle/validate/variables/target_inherited_from_target_merge/databricks.yml @@ -0,0 +1,15 @@ +bundle: + name: test-bundle + +variables: + x: + default: base + +definitions: + target_defaults: &target_defaults + variables: + x: overridden + +targets: + dev: + <<: *target_defaults diff --git a/acceptance/bundle/validate/variables/target_inherited_from_target_merge/out.test.toml b/acceptance/bundle/validate/variables/target_inherited_from_target_merge/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/validate/variables/target_inherited_from_target_merge/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/validate/variables/target_inherited_from_target_merge/output.txt b/acceptance/bundle/validate/variables/target_inherited_from_target_merge/output.txt new file mode 100644 index 00000000000..d3be1ebf0eb --- /dev/null +++ b/acceptance/bundle/validate/variables/target_inherited_from_target_merge/output.txt @@ -0,0 +1,6 @@ + +>>> [CLI] bundle validate -t dev -o json +{ + "default": "overridden", + "value": "overridden" +} diff --git a/acceptance/bundle/validate/variables/target_inherited_from_target_merge/script b/acceptance/bundle/validate/variables/target_inherited_from_target_merge/script new file mode 100644 index 00000000000..4339582a088 --- /dev/null +++ b/acceptance/bundle/validate/variables/target_inherited_from_target_merge/script @@ -0,0 +1,2 @@ +# Target variable shorthands inherited through a merge key on the target are applied. +trace $CLI bundle validate -t dev -o json | jq .variables.x diff --git a/acceptance/bundle/validate/variables/target_timestamp_shorthand/databricks.yml b/acceptance/bundle/validate/variables/target_timestamp_shorthand/databricks.yml new file mode 100644 index 00000000000..84eeb5183bd --- /dev/null +++ b/acceptance/bundle/validate/variables/target_timestamp_shorthand/databricks.yml @@ -0,0 +1,11 @@ +bundle: + name: test-bundle + +variables: + x: + default: base + +targets: + dev: + variables: + x: 2020-01-02 diff --git a/acceptance/bundle/validate/variables/target_timestamp_shorthand/out.test.toml b/acceptance/bundle/validate/variables/target_timestamp_shorthand/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/validate/variables/target_timestamp_shorthand/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/validate/variables/target_timestamp_shorthand/output.txt b/acceptance/bundle/validate/variables/target_timestamp_shorthand/output.txt new file mode 100644 index 00000000000..07eae4cbd80 --- /dev/null +++ b/acceptance/bundle/validate/variables/target_timestamp_shorthand/output.txt @@ -0,0 +1,10 @@ + +>>> [CLI] bundle validate -t dev -o json +Warning: expected map, found time + at targets.dev.variables.x + in databricks.yml:11:10 + +{ + "default": "base", + "value": "base" +} diff --git a/acceptance/bundle/validate/variables/target_timestamp_shorthand/script b/acceptance/bundle/validate/variables/target_timestamp_shorthand/script new file mode 100644 index 00000000000..e5418dbd3df --- /dev/null +++ b/acceptance/bundle/validate/variables/target_timestamp_shorthand/script @@ -0,0 +1,2 @@ +# A timestamp as a target variable shorthand. +trace $CLI bundle validate -t dev -o json | jq .variables.x diff --git a/acceptance/bundle/validate/variables/target_timestamp_shorthand/test.toml b/acceptance/bundle/validate/variables/target_timestamp_shorthand/test.toml new file mode 100644 index 00000000000..83f0c9957ac --- /dev/null +++ b/acceptance/bundle/validate/variables/target_timestamp_shorthand/test.toml @@ -0,0 +1 @@ +Badness = 'The timestamp shorthand is dropped with "expected map, found time" instead of being used as the default.' diff --git a/acceptance/bundle/validate/yaml_anchors/alias_mutates_complex_default/databricks.yml b/acceptance/bundle/validate/yaml_anchors/alias_mutates_complex_default/databricks.yml new file mode 100644 index 00000000000..87f970afec3 --- /dev/null +++ b/acceptance/bundle/validate/yaml_anchors/alias_mutates_complex_default/databricks.yml @@ -0,0 +1,14 @@ +bundle: + name: test-bundle + +variables: + x: + default: base + payload: + type: complex + default: &overrides + x: overridden + +targets: + dev: + variables: *overrides diff --git a/acceptance/bundle/validate/yaml_anchors/alias_mutates_complex_default/out.test.toml b/acceptance/bundle/validate/yaml_anchors/alias_mutates_complex_default/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/validate/yaml_anchors/alias_mutates_complex_default/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/validate/yaml_anchors/alias_mutates_complex_default/output.txt b/acceptance/bundle/validate/yaml_anchors/alias_mutates_complex_default/output.txt new file mode 100644 index 00000000000..e324cc30cf1 --- /dev/null +++ b/acceptance/bundle/validate/yaml_anchors/alias_mutates_complex_default/output.txt @@ -0,0 +1,17 @@ + +>>> [CLI] bundle validate -t dev -o json +{ + "x": { + "default": "overridden", + "value": "overridden" + }, + "payload": { + "default": { + "x": "overridden" + }, + "type": "complex", + "value": { + "x": "overridden" + } + } +} diff --git a/acceptance/bundle/validate/yaml_anchors/alias_mutates_complex_default/script b/acceptance/bundle/validate/yaml_anchors/alias_mutates_complex_default/script new file mode 100644 index 00000000000..56f3e365324 --- /dev/null +++ b/acceptance/bundle/validate/yaml_anchors/alias_mutates_complex_default/script @@ -0,0 +1,2 @@ +# Target variables given as an alias of a complex default do not change that default. +trace $CLI bundle validate -t dev -o json | jq '.variables | {x, payload}' diff --git a/acceptance/bundle/validate/anchor_containers/databricks.yml b/acceptance/bundle/validate/yaml_anchors/containers/databricks.yml similarity index 100% rename from acceptance/bundle/validate/anchor_containers/databricks.yml rename to acceptance/bundle/validate/yaml_anchors/containers/databricks.yml diff --git a/acceptance/bundle/validate/yaml_anchors/containers/out.test.toml b/acceptance/bundle/validate/yaml_anchors/containers/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/validate/yaml_anchors/containers/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/validate/anchor_containers/output.txt b/acceptance/bundle/validate/yaml_anchors/containers/output.txt similarity index 100% rename from acceptance/bundle/validate/anchor_containers/output.txt rename to acceptance/bundle/validate/yaml_anchors/containers/output.txt diff --git a/acceptance/bundle/validate/anchor_containers/script b/acceptance/bundle/validate/yaml_anchors/containers/script similarity index 100% rename from acceptance/bundle/validate/anchor_containers/script rename to acceptance/bundle/validate/yaml_anchors/containers/script diff --git a/acceptance/bundle/validate/yaml_anchor_cycle_nested/databricks.yml b/acceptance/bundle/validate/yaml_anchors/cycle_nested/databricks.yml similarity index 100% rename from acceptance/bundle/validate/yaml_anchor_cycle_nested/databricks.yml rename to acceptance/bundle/validate/yaml_anchors/cycle_nested/databricks.yml diff --git a/acceptance/bundle/validate/yaml_anchors/cycle_nested/out.test.toml b/acceptance/bundle/validate/yaml_anchors/cycle_nested/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/validate/yaml_anchors/cycle_nested/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/validate/yaml_anchor_cycle_nested/output.txt b/acceptance/bundle/validate/yaml_anchors/cycle_nested/output.txt similarity index 100% rename from acceptance/bundle/validate/yaml_anchor_cycle_nested/output.txt rename to acceptance/bundle/validate/yaml_anchors/cycle_nested/output.txt diff --git a/acceptance/bundle/validate/yaml_anchor_cycle_nested/script b/acceptance/bundle/validate/yaml_anchors/cycle_nested/script similarity index 100% rename from acceptance/bundle/validate/yaml_anchor_cycle_nested/script rename to acceptance/bundle/validate/yaml_anchors/cycle_nested/script diff --git a/acceptance/bundle/validate/yaml_anchor_cycle_nested/test.toml b/acceptance/bundle/validate/yaml_anchors/cycle_nested/test.toml similarity index 100% rename from acceptance/bundle/validate/yaml_anchor_cycle_nested/test.toml rename to acceptance/bundle/validate/yaml_anchors/cycle_nested/test.toml diff --git a/acceptance/bundle/validate/definitions_yaml_anchors/databricks.yml b/acceptance/bundle/validate/yaml_anchors/definitions/databricks.yml similarity index 100% rename from acceptance/bundle/validate/definitions_yaml_anchors/databricks.yml rename to acceptance/bundle/validate/yaml_anchors/definitions/databricks.yml diff --git a/acceptance/bundle/validate/yaml_anchors/definitions/out.test.toml b/acceptance/bundle/validate/yaml_anchors/definitions/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/validate/yaml_anchors/definitions/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/validate/definitions_yaml_anchors/output.txt b/acceptance/bundle/validate/yaml_anchors/definitions/output.txt similarity index 100% rename from acceptance/bundle/validate/definitions_yaml_anchors/output.txt rename to acceptance/bundle/validate/yaml_anchors/definitions/output.txt diff --git a/acceptance/bundle/validate/definitions_yaml_anchors/script b/acceptance/bundle/validate/yaml_anchors/definitions/script similarity index 100% rename from acceptance/bundle/validate/definitions_yaml_anchors/script rename to acceptance/bundle/validate/yaml_anchors/definitions/script diff --git a/acceptance/bundle/validate/duplicate_yaml_merge_key/databricks.yml b/acceptance/bundle/validate/yaml_anchors/duplicate_merge_key_in_cluster/databricks.yml similarity index 100% rename from acceptance/bundle/validate/duplicate_yaml_merge_key/databricks.yml rename to acceptance/bundle/validate/yaml_anchors/duplicate_merge_key_in_cluster/databricks.yml diff --git a/acceptance/bundle/validate/yaml_anchors/duplicate_merge_key_in_cluster/out.test.toml b/acceptance/bundle/validate/yaml_anchors/duplicate_merge_key_in_cluster/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/validate/yaml_anchors/duplicate_merge_key_in_cluster/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/validate/duplicate_yaml_merge_key/output.txt b/acceptance/bundle/validate/yaml_anchors/duplicate_merge_key_in_cluster/output.txt similarity index 100% rename from acceptance/bundle/validate/duplicate_yaml_merge_key/output.txt rename to acceptance/bundle/validate/yaml_anchors/duplicate_merge_key_in_cluster/output.txt diff --git a/acceptance/bundle/validate/duplicate_yaml_merge_key/script b/acceptance/bundle/validate/yaml_anchors/duplicate_merge_key_in_cluster/script similarity index 100% rename from acceptance/bundle/validate/duplicate_yaml_merge_key/script rename to acceptance/bundle/validate/yaml_anchors/duplicate_merge_key_in_cluster/script diff --git a/acceptance/bundle/validate/yaml_anchors/duplicate_merge_keys/databricks.yml b/acceptance/bundle/validate/yaml_anchors/duplicate_merge_keys/databricks.yml new file mode 100644 index 00000000000..638c26ee0ad --- /dev/null +++ b/acceptance/bundle/validate/yaml_anchors/duplicate_merge_keys/databricks.yml @@ -0,0 +1,18 @@ +bundle: + name: test-bundle + +variables: + x: + default: base + +definitions: + a: &a + x: first + b: &b + x: second + +targets: + dev: + variables: + <<: *a + <<: *b diff --git a/acceptance/bundle/validate/yaml_anchors/duplicate_merge_keys/out.test.toml b/acceptance/bundle/validate/yaml_anchors/duplicate_merge_keys/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/validate/yaml_anchors/duplicate_merge_keys/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/validate/yaml_anchors/duplicate_merge_keys/output.txt b/acceptance/bundle/validate/yaml_anchors/duplicate_merge_keys/output.txt new file mode 100644 index 00000000000..2efbf8faf73 --- /dev/null +++ b/acceptance/bundle/validate/yaml_anchors/duplicate_merge_keys/output.txt @@ -0,0 +1,9 @@ + +>>> [CLI] bundle validate -t dev +Error: duplicate YAML merge key ('<<') is not allowed; to merge multiple maps, use a sequence: '<<: [*anchor1, *anchor2]' + in databricks.yml:18:7 + + +Found 1 error + +Exit code: 1 diff --git a/acceptance/bundle/validate/yaml_anchors/duplicate_merge_keys/script b/acceptance/bundle/validate/yaml_anchors/duplicate_merge_keys/script new file mode 100644 index 00000000000..e316f5883d8 --- /dev/null +++ b/acceptance/bundle/validate/yaml_anchors/duplicate_merge_keys/script @@ -0,0 +1,2 @@ +# Two '<<' keys in target variables are an error. +errcode trace $CLI bundle validate -t dev diff --git a/acceptance/bundle/validate/yaml_anchors/inline_merge_mapping/databricks.yml b/acceptance/bundle/validate/yaml_anchors/inline_merge_mapping/databricks.yml new file mode 100644 index 00000000000..c79cadfe0a8 --- /dev/null +++ b/acceptance/bundle/validate/yaml_anchors/inline_merge_mapping/databricks.yml @@ -0,0 +1,11 @@ +bundle: + name: test-bundle + +variables: + x: + default: base + +targets: + dev: + variables: + <<: {x: overridden} diff --git a/acceptance/bundle/validate/yaml_anchors/inline_merge_mapping/out.test.toml b/acceptance/bundle/validate/yaml_anchors/inline_merge_mapping/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/validate/yaml_anchors/inline_merge_mapping/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/validate/yaml_anchors/inline_merge_mapping/output.txt b/acceptance/bundle/validate/yaml_anchors/inline_merge_mapping/output.txt new file mode 100644 index 00000000000..85dc1ee379b --- /dev/null +++ b/acceptance/bundle/validate/yaml_anchors/inline_merge_mapping/output.txt @@ -0,0 +1,8 @@ + +>>> [CLI] bundle validate +Error: failed to load [TEST_TMP_DIR]/databricks.yml: yaml ([TEST_TMP_DIR]/databricks.yml:11:11): map merge requires map or sequence of maps as the value + + +Found 1 error + +Exit code: 1 diff --git a/acceptance/bundle/validate/yaml_anchors/inline_merge_mapping/script b/acceptance/bundle/validate/yaml_anchors/inline_merge_mapping/script new file mode 100644 index 00000000000..1724b135e5e --- /dev/null +++ b/acceptance/bundle/validate/yaml_anchors/inline_merge_mapping/script @@ -0,0 +1,2 @@ +# An inline mapping as the value of '<<' in target variables. +errcode trace $CLI bundle validate diff --git a/acceptance/bundle/validate/yaml_anchors/inline_merge_mapping/test.toml b/acceptance/bundle/validate/yaml_anchors/inline_merge_mapping/test.toml new file mode 100644 index 00000000000..84101454028 --- /dev/null +++ b/acceptance/bundle/validate/yaml_anchors/inline_merge_mapping/test.toml @@ -0,0 +1 @@ +Badness = "The YAML merge key spec allows a single mapping as the value of the merge key, but it is rejected." diff --git a/acceptance/bundle/validate/yaml_anchors/invalid_merge_sequence_member/databricks.yml b/acceptance/bundle/validate/yaml_anchors/invalid_merge_sequence_member/databricks.yml new file mode 100644 index 00000000000..2c0bf7fe5b8 --- /dev/null +++ b/acceptance/bundle/validate/yaml_anchors/invalid_merge_sequence_member/databricks.yml @@ -0,0 +1,15 @@ +bundle: + name: test-bundle + +variables: + x: + default: base + +definitions: + overrides: &overrides + x: overridden + +targets: + dev: + variables: + <<: [*overrides, 17] diff --git a/acceptance/bundle/validate/yaml_anchors/invalid_merge_sequence_member/out.test.toml b/acceptance/bundle/validate/yaml_anchors/invalid_merge_sequence_member/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/validate/yaml_anchors/invalid_merge_sequence_member/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/validate/yaml_anchors/invalid_merge_sequence_member/output.txt b/acceptance/bundle/validate/yaml_anchors/invalid_merge_sequence_member/output.txt new file mode 100644 index 00000000000..770f0193a58 --- /dev/null +++ b/acceptance/bundle/validate/yaml_anchors/invalid_merge_sequence_member/output.txt @@ -0,0 +1,8 @@ + +>>> [CLI] bundle validate -t dev +Error: failed to load [TEST_TMP_DIR]/databricks.yml: yaml ([TEST_TMP_DIR]/databricks.yml:15:11): map merge requires map or sequence of maps as the value + + +Found 1 error + +Exit code: 1 diff --git a/acceptance/bundle/validate/yaml_anchors/invalid_merge_sequence_member/script b/acceptance/bundle/validate/yaml_anchors/invalid_merge_sequence_member/script new file mode 100644 index 00000000000..497e2fefa39 --- /dev/null +++ b/acceptance/bundle/validate/yaml_anchors/invalid_merge_sequence_member/script @@ -0,0 +1,2 @@ +# A scalar among the maps merged with '<<: [...]' is an error. +errcode trace $CLI bundle validate -t dev diff --git a/acceptance/bundle/validate/target_variable_full_def_merge_key/databricks.yml b/acceptance/bundle/validate/yaml_anchors/target_variable_full_def/databricks.yml similarity index 100% rename from acceptance/bundle/validate/target_variable_full_def_merge_key/databricks.yml rename to acceptance/bundle/validate/yaml_anchors/target_variable_full_def/databricks.yml diff --git a/acceptance/bundle/validate/yaml_anchors/target_variable_full_def/out.test.toml b/acceptance/bundle/validate/yaml_anchors/target_variable_full_def/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/validate/yaml_anchors/target_variable_full_def/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/validate/target_variable_full_def_merge_key/output.txt b/acceptance/bundle/validate/yaml_anchors/target_variable_full_def/output.txt similarity index 100% rename from acceptance/bundle/validate/target_variable_full_def_merge_key/output.txt rename to acceptance/bundle/validate/yaml_anchors/target_variable_full_def/output.txt diff --git a/acceptance/bundle/validate/target_variable_full_def_merge_key/script b/acceptance/bundle/validate/yaml_anchors/target_variable_full_def/script similarity index 100% rename from acceptance/bundle/validate/target_variable_full_def_merge_key/script rename to acceptance/bundle/validate/yaml_anchors/target_variable_full_def/script diff --git a/acceptance/bundle/validate/target_variable_merge_key_precedence/databricks.yml b/acceptance/bundle/validate/yaml_anchors/target_variable_precedence/databricks.yml similarity index 100% rename from acceptance/bundle/validate/target_variable_merge_key_precedence/databricks.yml rename to acceptance/bundle/validate/yaml_anchors/target_variable_precedence/databricks.yml diff --git a/acceptance/bundle/validate/yaml_anchors/target_variable_precedence/out.test.toml b/acceptance/bundle/validate/yaml_anchors/target_variable_precedence/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/validate/yaml_anchors/target_variable_precedence/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/validate/target_variable_merge_key_precedence/output.txt b/acceptance/bundle/validate/yaml_anchors/target_variable_precedence/output.txt similarity index 100% rename from acceptance/bundle/validate/target_variable_merge_key_precedence/output.txt rename to acceptance/bundle/validate/yaml_anchors/target_variable_precedence/output.txt diff --git a/acceptance/bundle/validate/target_variable_merge_key_precedence/script b/acceptance/bundle/validate/yaml_anchors/target_variable_precedence/script similarity index 100% rename from acceptance/bundle/validate/target_variable_merge_key_precedence/script rename to acceptance/bundle/validate/yaml_anchors/target_variable_precedence/script diff --git a/acceptance/bundle/validate/target_variable_merge_key_precedence/test.toml b/acceptance/bundle/validate/yaml_anchors/target_variable_precedence/test.toml similarity index 100% rename from acceptance/bundle/validate/target_variable_merge_key_precedence/test.toml rename to acceptance/bundle/validate/yaml_anchors/target_variable_precedence/test.toml diff --git a/acceptance/bundle/validate/target_variable_shorthand_alias/databricks.yml b/acceptance/bundle/validate/yaml_anchors/target_variable_shorthand_alias/databricks.yml similarity index 100% rename from acceptance/bundle/validate/target_variable_shorthand_alias/databricks.yml rename to acceptance/bundle/validate/yaml_anchors/target_variable_shorthand_alias/databricks.yml diff --git a/acceptance/bundle/validate/yaml_anchors/target_variable_shorthand_alias/out.test.toml b/acceptance/bundle/validate/yaml_anchors/target_variable_shorthand_alias/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/validate/yaml_anchors/target_variable_shorthand_alias/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/validate/target_variable_shorthand_alias/output.txt b/acceptance/bundle/validate/yaml_anchors/target_variable_shorthand_alias/output.txt similarity index 100% rename from acceptance/bundle/validate/target_variable_shorthand_alias/output.txt rename to acceptance/bundle/validate/yaml_anchors/target_variable_shorthand_alias/output.txt diff --git a/acceptance/bundle/validate/target_variable_shorthand_alias/script b/acceptance/bundle/validate/yaml_anchors/target_variable_shorthand_alias/script similarity index 100% rename from acceptance/bundle/validate/target_variable_shorthand_alias/script rename to acceptance/bundle/validate/yaml_anchors/target_variable_shorthand_alias/script diff --git a/acceptance/bundle/validate/yaml_anchors/test.toml b/acceptance/bundle/validate/yaml_anchors/test.toml new file mode 100644 index 00000000000..e9cede5e28a --- /dev/null +++ b/acceptance/bundle/validate/yaml_anchors/test.toml @@ -0,0 +1,4 @@ +# Errors for invalid YAML print the full path of the file; normalize Windows separators. +[[Repls]] +Old = '\\' +New = '/' diff --git a/acceptance/bundle/validate/yaml_duplicate_keys/resource_fields/databricks.yml b/acceptance/bundle/validate/yaml_duplicate_keys/resource_fields/databricks.yml new file mode 100644 index 00000000000..d29746866db --- /dev/null +++ b/acceptance/bundle/validate/yaml_duplicate_keys/resource_fields/databricks.yml @@ -0,0 +1,16 @@ +bundle: + name: test-bundle + +resources: + jobs: + j: + name: first + max_concurrent_runs: 1 + tags: + a: one + shared: first + name: second + max_concurrent_runs: 5 + tags: + b: two + shared: second diff --git a/acceptance/bundle/validate/yaml_duplicate_keys/resource_fields/out.test.toml b/acceptance/bundle/validate/yaml_duplicate_keys/resource_fields/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/validate/yaml_duplicate_keys/resource_fields/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/validate/yaml_duplicate_keys/resource_fields/output.txt b/acceptance/bundle/validate/yaml_duplicate_keys/resource_fields/output.txt new file mode 100644 index 00000000000..c395fba7081 --- /dev/null +++ b/acceptance/bundle/validate/yaml_duplicate_keys/resource_fields/output.txt @@ -0,0 +1,19 @@ + +>>> [CLI] bundle validate +Name: test-bundle +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Validation OK! + +>>> [CLI] bundle validate -o json +{ + "name": "second", + "max_concurrent_runs": 5, + "tags": { + "b": "two", + "shared": "second" + } +} diff --git a/acceptance/bundle/validate/yaml_duplicate_keys/resource_fields/script b/acceptance/bundle/validate/yaml_duplicate_keys/resource_fields/script new file mode 100644 index 00000000000..7b6d7caa93b --- /dev/null +++ b/acceptance/bundle/validate/yaml_duplicate_keys/resource_fields/script @@ -0,0 +1,5 @@ +# Duplicate keys in a mapping are accepted without a warning, although YAML requires keys to be unique. +trace $CLI bundle validate + +# The later value replaces the earlier one as a whole: maps are not merged ("a" is gone). +trace $CLI bundle validate -o json | jq '.resources.jobs.j | {name, max_concurrent_runs, tags}' diff --git a/acceptance/bundle/validate/yaml_duplicate_keys/resource_fields/test.toml b/acceptance/bundle/validate/yaml_duplicate_keys/resource_fields/test.toml new file mode 100644 index 00000000000..af65b8438c4 --- /dev/null +++ b/acceptance/bundle/validate/yaml_duplicate_keys/resource_fields/test.toml @@ -0,0 +1 @@ +Badness = "Duplicate keys are accepted silently (the later one wins); YAML requires keys to be unique." diff --git a/acceptance/bundle/validate/yaml_duplicate_keys/shadowed_invalid_subtree/databricks.yml b/acceptance/bundle/validate/yaml_duplicate_keys/shadowed_invalid_subtree/databricks.yml new file mode 100644 index 00000000000..03d9373eed1 --- /dev/null +++ b/acceptance/bundle/validate/yaml_duplicate_keys/shadowed_invalid_subtree/databricks.yml @@ -0,0 +1,11 @@ +bundle: + name: test-bundle + +resources: + jobs: + j: + name: test + tags: + 7: invalid-key + tags: + good: ok diff --git a/acceptance/bundle/validate/yaml_duplicate_keys/shadowed_invalid_subtree/out.test.toml b/acceptance/bundle/validate/yaml_duplicate_keys/shadowed_invalid_subtree/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/validate/yaml_duplicate_keys/shadowed_invalid_subtree/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/validate/yaml_duplicate_keys/shadowed_invalid_subtree/output.txt b/acceptance/bundle/validate/yaml_duplicate_keys/shadowed_invalid_subtree/output.txt new file mode 100644 index 00000000000..8c0f7149fad --- /dev/null +++ b/acceptance/bundle/validate/yaml_duplicate_keys/shadowed_invalid_subtree/output.txt @@ -0,0 +1,8 @@ + +>>> [CLI] bundle validate +Error: failed to load [TEST_TMP_DIR]/databricks.yml: yaml ([TEST_TMP_DIR]/databricks.yml:9:9): invalid key tag: !!int + + +Found 1 error + +Exit code: 1 diff --git a/acceptance/bundle/validate/yaml_duplicate_keys/shadowed_invalid_subtree/script b/acceptance/bundle/validate/yaml_duplicate_keys/shadowed_invalid_subtree/script new file mode 100644 index 00000000000..58b74e2f574 --- /dev/null +++ b/acceptance/bundle/validate/yaml_duplicate_keys/shadowed_invalid_subtree/script @@ -0,0 +1,2 @@ +# A value replaced by a later duplicate key is still loaded, so its invalid (integer) key is rejected. +errcode trace $CLI bundle validate diff --git a/acceptance/bundle/validate/yaml_duplicate_keys/shadowed_invalid_subtree/test.toml b/acceptance/bundle/validate/yaml_duplicate_keys/shadowed_invalid_subtree/test.toml new file mode 100644 index 00000000000..bad67f27bd1 --- /dev/null +++ b/acceptance/bundle/validate/yaml_duplicate_keys/shadowed_invalid_subtree/test.toml @@ -0,0 +1 @@ +Badness = 'The duplicate tags key is accepted silently (the later one wins); YAML requires keys to be unique.' diff --git a/acceptance/bundle/validate/yaml_duplicate_keys/target_variable_default/databricks.yml b/acceptance/bundle/validate/yaml_duplicate_keys/target_variable_default/databricks.yml new file mode 100644 index 00000000000..38c97681a98 --- /dev/null +++ b/acceptance/bundle/validate/yaml_duplicate_keys/target_variable_default/databricks.yml @@ -0,0 +1,13 @@ +bundle: + name: test-bundle + +variables: + x: + default: base + +targets: + dev: + variables: + x: + default: first + default: second diff --git a/acceptance/bundle/validate/yaml_duplicate_keys/target_variable_default/out.test.toml b/acceptance/bundle/validate/yaml_duplicate_keys/target_variable_default/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/validate/yaml_duplicate_keys/target_variable_default/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/validate/yaml_duplicate_keys/target_variable_default/output.txt b/acceptance/bundle/validate/yaml_duplicate_keys/target_variable_default/output.txt new file mode 100644 index 00000000000..5c429a0449c --- /dev/null +++ b/acceptance/bundle/validate/yaml_duplicate_keys/target_variable_default/output.txt @@ -0,0 +1,6 @@ + +>>> [CLI] bundle validate -t dev -o json +{ + "default": "second", + "value": "second" +} diff --git a/acceptance/bundle/validate/yaml_duplicate_keys/target_variable_default/script b/acceptance/bundle/validate/yaml_duplicate_keys/target_variable_default/script new file mode 100644 index 00000000000..771eda950ef --- /dev/null +++ b/acceptance/bundle/validate/yaml_duplicate_keys/target_variable_default/script @@ -0,0 +1,2 @@ +# With a duplicate default key in a target variable override, the last one wins and it is a full definition. +trace $CLI bundle validate -t dev -o json | jq .variables.x diff --git a/acceptance/bundle/validate/yaml_duplicate_keys/target_variable_default/test.toml b/acceptance/bundle/validate/yaml_duplicate_keys/target_variable_default/test.toml new file mode 100644 index 00000000000..42d04e8a1ef --- /dev/null +++ b/acceptance/bundle/validate/yaml_duplicate_keys/target_variable_default/test.toml @@ -0,0 +1 @@ +Badness = 'The duplicate default key is accepted silently (the later one wins); YAML requires keys to be unique.' diff --git a/acceptance/bundle/validate/yaml_duplicate_keys/test.toml b/acceptance/bundle/validate/yaml_duplicate_keys/test.toml new file mode 100644 index 00000000000..e9cede5e28a --- /dev/null +++ b/acceptance/bundle/validate/yaml_duplicate_keys/test.toml @@ -0,0 +1,4 @@ +# Errors for invalid YAML print the full path of the file; normalize Windows separators. +[[Repls]] +Old = '\\' +New = '/'