From 6674d4b9bcddac2783ae5fe1e9f64ccd4d360cca Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 15 Oct 2026 17:18:07 +0200 Subject: [PATCH 01/22] structpath: add NewPattern with AnyKey/AnyIndex wildcards NewPattern builds a pattern from parts like NewPath does for paths, so callers don't nest the single-step constructors or parse pattern strings. Co-authored-by: Isaac --- libs/structs/structpath/path.go | 31 ++++++++++++++++++++++++++++ libs/structs/structpath/path_test.go | 10 +++++++++ 2 files changed, 41 insertions(+) diff --git a/libs/structs/structpath/path.go b/libs/structs/structpath/path.go index adf2be51331..8e608253e33 100644 --- a/libs/structs/structpath/path.go +++ b/libs/structs/structpath/path.go @@ -360,6 +360,37 @@ func NewPatternBracketStar(prev *PatternNode) *PatternNode { }) } +// wildcard is a NewPattern part that matches any key or element. +type wildcard int + +// Wildcards for [NewPattern]: AnyKey matches any map key or field (rendered ".*") and +// AnyIndex matches any sequence element (rendered "[*]"). +const ( + AnyKey wildcard = tagDotStar + AnyIndex wildcard = tagBracketStar +) + +// NewPattern appends parts to prev like [NewPath]; AnyKey and AnyIndex append wildcards. +func NewPattern(prev *PatternNode, parts ...any) *PatternNode { + for _, part := range parts { + switch v := part.(type) { + case string: + prev = NewPatternStringKey(prev, v) + case int: + prev = NewPatternIndex(prev, v) + case wildcard: + if v == AnyKey { + prev = NewPatternDotStar(prev) + } else { + prev = NewPatternBracketStar(prev) + } + default: + panic(fmt.Sprintf("structpath.NewPattern: unsupported part %#v", part)) + } + } + return prev +} + func NewPatternKeyValue(prev *PatternNode, key, value string) *PatternNode { return (*PatternNode)(NewKeyValue((*PathNode)(prev), key, value)) } diff --git a/libs/structs/structpath/path_test.go b/libs/structs/structpath/path_test.go index 40fc0282c2b..3b7b3d3c896 100644 --- a/libs/structs/structpath/path_test.go +++ b/libs/structs/structpath/path_test.go @@ -1297,3 +1297,13 @@ func TestNewPathSlice(t *testing.T) { assert.Equal(t, "resources.jobs['${var.env}_job']", paths[0].String()) assert.Equal(t, "sync.paths[3]", NewPathSlice("sync", "paths", 3)[0].String()) } + +func TestNewPattern(t *testing.T) { + p := NewPattern(nil, "resources", "jobs", AnyKey, "tasks", AnyIndex, "libraries") + assert.Equal(t, "resources.jobs.*.tasks[*].libraries", p.String()) + + ext := NewPattern(p, AnyIndex, "whl", 0) + assert.Equal(t, "resources.jobs.*.tasks[*].libraries[*].whl[0]", ext.String()) + + assert.Panics(t, func() { NewPattern(nil, 1.5) }) +} From 6079e945d1c0b7ba672ad6482c7d11a45bc9f599 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 7 Oct 2026 16:06:41 +0200 Subject: [PATCH 02/22] bundle: delete libs/dyn; configuration is typed structs with structvar refs and locations Bundle configuration is the typed struct plus structvar (pure ${...} references in non-string fields and a tree of source locations). YAML and JSON decode straight into the structs; mutators, validators, variable resolution, configsync and generate work on structvar views and structpath paths. libs/dyn is deleted. Co-authored-by: Isaac --- .agents/rules/style-guide-go.md | 4 +- .github/workflows/python_push.yml | 2 +- .nextchanges/bundles/6954.md | 1 + AGENTS.md | 4 +- acceptance/invariant_test.go | 32 +- bundle/apps/validate.go | 45 +- bundle/artifacts/expand_globs.go | 145 ++- bundle/artifacts/prepare.go | 2 +- bundle/bundle_test.go | 4 - bundle/config/loader/process_include.go | 66 +- bundle/config/loader/process_root_includes.go | 55 +- .../loader/process_root_includes_test.go | 6 +- .../config/loctable}/locations.go | 46 +- .../apply_source_linked_deployment_preset.go | 17 +- .../mutator/collect_escape_telemetry.go | 48 +- bundle/config/mutator/compute_id_compat.go | 71 +- bundle/config/mutator/drop_empty_strings.go | 96 +- bundle/config/mutator/environments_compat.go | 55 +- .../config/mutator/initialize_volume_paths.go | 52 +- bundle/config/mutator/load_dbalert_files.go | 28 +- bundle/config/mutator/normalize_paths.go | 39 +- bundle/config/mutator/normalize_paths_test.go | 30 +- .../mutator/paths/alert_paths_visitor.go | 14 +- .../config/mutator/paths/app_paths_visitor.go | 14 +- .../mutator/paths/app_paths_visitor_test.go | 6 +- .../mutator/paths/artifact_paths_visitor.go | 23 +- .../paths/artifact_paths_visitor_test.go | 6 +- .../mutator/paths/dashboard_paths_visitor.go | 14 +- .../paths/dashboard_paths_visitor_test.go | 6 +- .../paths/genie_space_paths_visitor.go | 14 +- .../paths/job_libraries_paths_visitor.go | 58 +- .../paths/job_libraries_paths_visitor_test.go | 18 +- .../config/mutator/paths/job_paths_visitor.go | 48 +- .../mutator/paths/job_paths_visitor_test.go | 30 +- .../mutator/paths/job_run_paths_visitor.go | 17 +- .../paths/job_run_paths_visitor_test.go | 6 +- .../mutator/paths/pipeline_paths_visitor.go | 81 +- .../paths/pipeline_paths_visitor_test.go | 16 +- bundle/config/mutator/paths/visitor.go | 32 +- bundle/config/mutator/paths/visitor_test.go | 15 +- bundle/config/mutator/populate_locations.go | 54 +- .../mutator/prepend_workspace_prefix.go | 60 +- .../mutator/python/apply_python_output.go | 127 +- .../python/apply_python_output_test.go | 124 +- .../mutator/python/python_diagnostics.go | 7 +- .../config/mutator/python/python_locations.go | 41 +- .../mutator/python/python_locations_test.go | 91 +- .../config/mutator/python/python_mutator.go | 234 ++-- .../mutator/python/python_mutator_test.go | 65 +- .../mutator/resolve_variable_references.go | 385 ++++--- .../resolve_variable_references_test.go | 8 +- .../apply_bundle_permissions.go | 114 +- .../apply_default_task_source.go | 85 +- .../capture_uc_dependencies.go | 8 +- .../configure_cluster_policy_definition.go | 70 +- ...nfigure_dashboards_serialized_dashboard.go | 99 +- .../configure_genie_space_serialized_space.go | 103 +- .../expand_pipeline_glob_paths.go | 144 +-- .../expand_pipeline_glob_paths_test.go | 1 - .../resourcemutator/fix_permissions.go | 212 ++-- .../mutator/resourcemutator/keyed_merge.go | 80 ++ .../mutator/resourcemutator/merge_apps.go | 66 +- .../mutator/resourcemutator/merge_grants.go | 131 ++- .../resourcemutator/merge_job_clusters.go | 29 +- .../resourcemutator/merge_job_parameters.go | 29 +- .../resourcemutator/merge_job_tasks.go | 37 +- .../merge_pipeline_clusters.go | 35 +- .../process_static_resources.go | 16 +- .../mutator/resourcemutator/resource_key.go | 27 +- .../resourcemutator/resource_key_test.go | 35 +- .../resourcemutator/resource_mutator.go | 283 ++--- .../resourcemutator/resource_mutator_test.go | 145 +-- .../config/mutator/resourcemutator/run_as.go | 7 +- .../mutator/resourcemutator/run_as_test.go | 70 +- .../resourcemutator/secret_scope_fixups.go | 5 +- .../resourcemutator/validate_target_mode.go | 3 +- bundle/config/mutator/rewrite_sync_paths.go | 87 +- .../mutator/rewrite_workspace_prefix.go | 57 +- bundle/config/mutator/set_variables.go | 98 +- bundle/config/mutator/set_variables_test.go | 50 +- bundle/config/mutator/sync_default_path.go | 20 +- .../config/mutator/sync_default_path_test.go | 22 +- bundle/config/mutator/sync_infer_root.go | 5 +- bundle/config/mutator/translate_paths.go | 61 +- bundle/config/mutator/translate_paths_apps.go | 9 +- .../mutator/translate_paths_artifacts.go | 9 +- .../mutator/translate_paths_dashboards.go | 9 +- .../mutator/translate_paths_genie_spaces.go | 9 +- bundle/config/mutator/translate_paths_jobs.go | 23 +- .../mutator/translate_paths_pipelines.go | 15 +- bundle/config/mutator/translate_paths_test.go | 8 +- .../validate_secret_value_is_variable.go | 39 +- bundle/config/resources_types_test.go | 44 +- bundle/config/root.go | 691 +++++------ bundle/config/root_test.go | 147 ++- .../validate/all_resources_have_values.go | 33 +- bundle/config/validate/enum.go | 34 +- bundle/config/validate/files_to_sync.go | 6 +- .../validate/interpolation_in_auth_config.go | 17 +- .../validate/job_cluster_key_defined.go | 7 +- .../config/validate/job_task_cluster_spec.go | 16 +- .../no_interpolation_in_bundle_name.go | 4 +- .../no_variable_reference_in_resource_key.go | 48 +- bundle/config/validate/pattern_set.go | 37 + .../pipeline_development_deprecated.go | 35 +- bundle/config/validate/required.go | 83 +- bundle/config/validate/scripts.go | 26 +- bundle/config/validate/single_node_cluster.go | 71 +- .../validate/single_node_cluster_test.go | 63 +- bundle/config/validate/tf_only_references.go | 24 +- .../validate/tf_only_references_test.go | 19 +- .../config/validate/unique_resource_keys.go | 41 +- .../config/validate/validate_artifact_path.go | 16 +- .../validate/validate_artifact_path_test.go | 5 +- .../validate/validate_dashboard_etags.go | 2 +- .../validate/validate_deployment_fields.go | 14 +- bundle/config/validate/validate_engine.go | 6 +- .../validate/validate_genie_space_etags.go | 2 +- .../validate_job_run_idempotency_token.go | 4 +- bundle/configsync/diff.go | 23 +- bundle/configsync/resolve.go | 53 +- bundle/configsync/sequences.go | 78 +- bundle/configsync/telemetry_test.go | 22 +- bundle/configsync/variables.go | 244 ++-- bundle/configsync/variables_test.go | 99 +- .../check_dashboards_modified_remotely.go | 4 +- bundle/deploy/metadata/compute.go | 6 +- bundle/direct/bundle_plan.go | 76 +- bundle/direct/bundle_plan_test.go | 39 +- bundle/direct/dresources/adapter.go | 2 +- bundle/direct/dresources/grants.go | 1 + bundle/direct/dresources/permissions.go | 1 + bundle/direct/dresources/secret_scope_acls.go | 1 + bundle/direct/dresources/serialize_test.go | 2 +- bundle/direct/resolve_config.go | 122 +- bundle/direct/unescape.go | 4 +- bundle/generate/alert.go | 17 +- bundle/generate/app.go | 66 +- bundle/generate/dashboard.go | 17 +- bundle/generate/genie_space.go | 21 +- bundle/generate/job.go | 47 +- bundle/generate/pipeline.go | 14 +- bundle/internal/bundletest/benchmark.go | 27 - bundle/internal/bundletest/benchmark_test.go | 51 +- bundle/internal/bundletest/location.go | 27 +- bundle/internal/bundletest/mutate.go | 17 - .../bundletest/mutator_benchmark_test.go | 7 +- bundle/internal/schema/annotations.go | 59 +- bundle/internal/schema/annotations_file.go | 277 +++-- bundle/internal/schema/main.go | 6 +- bundle/internal/schema/main_test.go | 58 +- bundle/libraries/expand_glob_references.go | 232 ++-- .../libraries/expand_glob_references_test.go | 22 +- bundle/libraries/remote_path.go | 142 +-- bundle/libraries/same_name_libraries.go | 87 +- bundle/libraries/upload.go | 5 +- bundle/migrate/build_state.go | 26 +- bundle/migrate/build_state_test.go | 15 +- bundle/migrate/resolve.go | 5 +- bundle/mutator.go | 41 - bundle/mutator_test.go | 82 +- bundle/phases/plan.go | 12 +- bundle/phases/plan_test.go | 70 +- bundle/phases/telemetry.go | 7 +- bundle/set_default.go | 76 +- bundle/statemgmt/direct_migration.go | 15 +- bundle/statemgmt/reverse_interpolate.go | 75 +- bundle/statemgmt/reverse_interpolate_test.go | 139 +-- bundle/statemgmt/state_load.go | 108 +- bundle/trampoline/python_dbr_warning.go | 10 +- cmd/air/convert_to_dabs.go | 216 ++-- cmd/air/convert_to_dabs_test.go | 75 +- cmd/apps/import.go | 78 +- cmd/apps/import_test.go | 73 +- cmd/bundle/generate/alert.go | 17 +- cmd/bundle/generate/app.go | 14 +- cmd/bundle/generate/dashboard.go | 17 +- cmd/bundle/generate/genie_space.go | 17 +- cmd/bundle/generate/job.go | 17 +- cmd/bundle/generate/pipeline.go | 27 +- cmd/bundle/run.go | 4 +- cmd/bundle/utils/process.go | 4 +- cmd/bundle/validate.go | 7 +- cmd/pipelines/generate.go | 101 +- cmd/pipelines/generate_test.go | 53 +- .../assumptions/dashboard_assumptions_test.go | 28 +- libs/dyn/convert/end_to_end_test.go | 171 --- libs/dyn/convert/error.go | 16 - libs/dyn/convert/from_typed.go | 362 ------ libs/dyn/convert/from_typed_test.go | 927 --------------- libs/dyn/convert/normalize.go | 507 -------- libs/dyn/convert/normalize_test.go | 1018 ----------------- libs/dyn/convert/sdk_native_types.go | 97 -- libs/dyn/convert/sdk_native_types_test.go | 417 ------- libs/dyn/convert/struct_info_test.go | 228 ---- libs/dyn/convert/to_typed.go | 350 ------ libs/dyn/convert/to_typed_test.go | 790 ------------- libs/dyn/drop_keys.go | 27 - libs/dyn/drop_keys_test.go | 24 - libs/dyn/dynassert/assert.go | 36 - libs/dyn/dynvar/lookup.go | 21 - libs/dyn/dynvar/lookup_test.go | 27 - libs/dyn/dynvar/resolve.go | 255 ----- libs/dyn/dynvar/resolve_test.go | 430 ------- libs/dyn/jsonloader/json.go | 134 --- libs/dyn/jsonloader/json_test.go | 161 --- libs/dyn/jsonloader/locations.go | 49 - libs/dyn/kind_test.go | 38 - libs/dyn/mapping.go | 158 --- libs/dyn/mapping_test.go | 182 --- libs/dyn/merge/elements_by_key.go | 107 -- libs/dyn/merge/elements_by_key_test.go | 109 -- libs/dyn/merge/merge.go | 119 -- libs/dyn/merge/merge_test.go | 301 ----- libs/dyn/merge/override.go | 200 ---- libs/dyn/merge/override_test.go | 520 --------- libs/dyn/merge/select.go | 53 - libs/dyn/merge/select_test.go | 55 - libs/dyn/path.go | 146 --- libs/dyn/path_string.go | 89 -- libs/dyn/path_string_test.go | 115 -- libs/dyn/path_test.go | 157 --- libs/dyn/pattern.go | 191 ---- libs/dyn/pattern_string.go | 102 -- libs/dyn/pattern_string_test.go | 147 --- libs/dyn/pattern_test.go | 110 -- libs/dyn/pattern_trie.go | 172 --- libs/dyn/pattern_trie_test.go | 195 ---- libs/dyn/structpath.go | 41 - libs/dyn/structpath_test.go | 39 - libs/dyn/suggest_test.go | 164 --- libs/dyn/time.go | 66 -- libs/dyn/time_test.go | 41 - libs/dyn/value.go | 245 ---- libs/dyn/value_eq_test.go | 77 -- libs/dyn/value_test.go | 92 -- libs/dyn/value_underlying.go | 138 --- libs/dyn/value_underlying_test.go | 162 --- libs/dyn/visit.go | 210 ---- libs/dyn/visit_get.go | 31 - libs/dyn/visit_get_test.go | 83 -- libs/dyn/visit_map.go | 89 -- libs/dyn/visit_map_test.go | 477 -------- libs/dyn/visit_set.go | 77 -- libs/dyn/visit_set_test.go | 90 -- libs/dyn/visit_test.go | 36 - libs/dyn/walk.go | 95 -- libs/dyn/walk_read_only.go | 44 - libs/dyn/walk_read_only_test.go | 245 ---- libs/dyn/walk_test.go | 269 ----- libs/dyn/yamlloader/loader.go | 291 ----- libs/dyn/yamlloader/testdata/anchor_01.yml | 12 - libs/dyn/yamlloader/testdata/anchor_02.yml | 13 - libs/dyn/yamlloader/testdata/anchor_03.yml | 10 - libs/dyn/yamlloader/testdata/anchor_04.yml | 16 - libs/dyn/yamlloader/testdata/anchor_05.yml | 15 - libs/dyn/yamlloader/testdata/anchor_06.yml | 5 - libs/dyn/yamlloader/testdata/anchor_07.yml | 10 - libs/dyn/yamlloader/testdata/anchor_08.yml | 10 - libs/dyn/yamlloader/testdata/anchor_09.yml | 2 - libs/dyn/yamlloader/testdata/anchor_10.yml | 5 - libs/dyn/yamlloader/testdata/empty.yml | 0 libs/dyn/yamlloader/testdata/error_01.yml | 6 - libs/dyn/yamlloader/testdata/error_02.yml | 6 - libs/dyn/yamlloader/testdata/error_03.yml | 4 - libs/dyn/yamlloader/testdata/mix_01.yml | 90 -- libs/dyn/yamlloader/testdata/mix_02.yml | 49 - .../yamlloader/testdata/spec_example_2.1.yml | 5 - .../yamlloader/testdata/spec_example_2.10.yml | 10 - .../yamlloader/testdata/spec_example_2.11.yml | 10 - .../yamlloader/testdata/spec_example_2.12.yml | 10 - .../yamlloader/testdata/spec_example_2.13.yml | 6 - .../yamlloader/testdata/spec_example_2.14.yml | 6 - .../yamlloader/testdata/spec_example_2.15.yml | 10 - .../yamlloader/testdata/spec_example_2.16.yml | 9 - .../yamlloader/testdata/spec_example_2.17.yml | 9 - .../yamlloader/testdata/spec_example_2.18.yml | 8 - .../yamlloader/testdata/spec_example_2.19.yml | 15 - .../yamlloader/testdata/spec_example_2.2.yml | 5 - .../yamlloader/testdata/spec_example_2.20.yml | 7 - .../yamlloader/testdata/spec_example_2.21.yml | 5 - .../yamlloader/testdata/spec_example_2.22.yml | 6 - .../yamlloader/testdata/spec_example_2.23.yml | 15 - .../yamlloader/testdata/spec_example_2.24.yml | 16 - .../yamlloader/testdata/spec_example_2.25.yml | 9 - .../yamlloader/testdata/spec_example_2.26.yml | 9 - .../yamlloader/testdata/spec_example_2.27.yml | 31 - .../yamlloader/testdata/spec_example_2.28.yml | 28 - .../yamlloader/testdata/spec_example_2.3.yml | 10 - .../yamlloader/testdata/spec_example_2.4.yml | 10 - .../yamlloader/testdata/spec_example_2.5.yml | 5 - .../yamlloader/testdata/spec_example_2.6.yml | 7 - .../yamlloader/testdata/spec_example_2.7.yml | 12 - .../yamlloader/testdata/spec_example_2.8.yml | 12 - .../yamlloader/testdata/spec_example_2.9.yml | 10 - libs/dyn/yamlloader/yaml.go | 22 - libs/dyn/yamlloader/yaml_anchor_test.go | 140 --- libs/dyn/yamlloader/yaml_error_test.go | 36 - libs/dyn/yamlloader/yaml_mix_test.go | 26 - libs/dyn/yamlloader/yaml_spec_test.go | 824 ------------- libs/dyn/yamlloader/yaml_test.go | 35 - libs/dyn/yamlsaver/order.go | 33 - libs/dyn/yamlsaver/order_test.go | 24 - libs/dyn/yamlsaver/saver.go | 172 --- libs/dyn/yamlsaver/saver_test.go | 275 ----- libs/dyn/yamlsaver/utils.go | 94 -- libs/dyn/yamlsaver/utils_test.go | 104 -- libs/flags/json_flag.go | 60 +- libs/flags/json_flag_test.go | 7 +- libs/structs/structaccess/bundle_test.go | 51 +- libs/structs/structpath/path.go | 28 + libs/structs/structpath/path_test.go | 26 +- libs/structs/structvar/decode.go | 776 +++++++++++++ libs/structs/structvar/decode_json.go | 126 ++ libs/structs/structvar/decode_yaml.go | 327 ++++++ libs/{dyn => structs/structvar}/kind.go | 32 +- libs/structs/structvar/locations.go | 262 +++++ libs/structs/structvar/lookup.go | 157 +++ libs/structs/structvar/merge.go | 213 ++++ libs/structs/structvar/override.go | 163 +++ libs/structs/structvar/parity_walk_test.go | 71 ++ libs/structs/structvar/pattern.go | 110 ++ libs/{dyn/dynvar => structs/structvar}/ref.go | 25 +- .../dynvar => structs/structvar}/ref_test.go | 59 +- .../structvar}/ref_vectors_test.go | 7 +- libs/structs/structvar/resolve.go | 200 ++++ .../structvar}/struct_info.go | 96 +- libs/structs/structvar/structvar.go | 10 +- libs/{dyn => structs/structvar}/suggest.go | 50 +- .../testdata/reference_vectors.json | 2 +- libs/structs/structvar/view.go | 504 ++++++++ libs/structs/structvar/walk.go | 47 + libs/structs/structvar/write.go | 352 ++++++ libs/structs/structyaml/structyaml.go | 187 +++ libs/structs/structyaml/structyaml_test.go | 163 +++ python/Taskfile.yml | 2 +- python/codegen/codegen/packages.py | 2 +- python/databricks/bundles/core/_transform.py | 6 +- .../core/test_variable_reference_vectors.py | 2 +- yamlfmt.yml | 1 - 340 files changed, 8467 insertions(+), 20122 deletions(-) create mode 100644 .nextchanges/bundles/6954.md rename {libs/dyn/dynloc => bundle/config/loctable}/locations.go (61%) create mode 100644 bundle/config/mutator/resourcemutator/keyed_merge.go create mode 100644 bundle/config/validate/pattern_set.go delete mode 100644 bundle/internal/bundletest/mutate.go delete mode 100644 libs/dyn/convert/end_to_end_test.go delete mode 100644 libs/dyn/convert/error.go delete mode 100644 libs/dyn/convert/from_typed.go delete mode 100644 libs/dyn/convert/from_typed_test.go delete mode 100644 libs/dyn/convert/normalize.go delete mode 100644 libs/dyn/convert/normalize_test.go delete mode 100644 libs/dyn/convert/sdk_native_types.go delete mode 100644 libs/dyn/convert/sdk_native_types_test.go delete mode 100644 libs/dyn/convert/struct_info_test.go delete mode 100644 libs/dyn/convert/to_typed.go delete mode 100644 libs/dyn/convert/to_typed_test.go delete mode 100644 libs/dyn/drop_keys.go delete mode 100644 libs/dyn/drop_keys_test.go delete mode 100644 libs/dyn/dynassert/assert.go delete mode 100644 libs/dyn/dynvar/lookup.go delete mode 100644 libs/dyn/dynvar/lookup_test.go delete mode 100644 libs/dyn/dynvar/resolve.go delete mode 100644 libs/dyn/dynvar/resolve_test.go delete mode 100644 libs/dyn/jsonloader/json.go delete mode 100644 libs/dyn/jsonloader/json_test.go delete mode 100644 libs/dyn/jsonloader/locations.go delete mode 100644 libs/dyn/kind_test.go delete mode 100644 libs/dyn/mapping.go delete mode 100644 libs/dyn/mapping_test.go delete mode 100644 libs/dyn/merge/elements_by_key.go delete mode 100644 libs/dyn/merge/elements_by_key_test.go delete mode 100644 libs/dyn/merge/merge.go delete mode 100644 libs/dyn/merge/merge_test.go delete mode 100644 libs/dyn/merge/override.go delete mode 100644 libs/dyn/merge/override_test.go delete mode 100644 libs/dyn/merge/select.go delete mode 100644 libs/dyn/merge/select_test.go delete mode 100644 libs/dyn/path.go delete mode 100644 libs/dyn/path_string.go delete mode 100644 libs/dyn/path_string_test.go delete mode 100644 libs/dyn/path_test.go delete mode 100644 libs/dyn/pattern.go delete mode 100644 libs/dyn/pattern_string.go delete mode 100644 libs/dyn/pattern_string_test.go delete mode 100644 libs/dyn/pattern_test.go delete mode 100644 libs/dyn/pattern_trie.go delete mode 100644 libs/dyn/pattern_trie_test.go delete mode 100644 libs/dyn/structpath.go delete mode 100644 libs/dyn/structpath_test.go delete mode 100644 libs/dyn/suggest_test.go delete mode 100644 libs/dyn/time.go delete mode 100644 libs/dyn/time_test.go delete mode 100644 libs/dyn/value.go delete mode 100644 libs/dyn/value_eq_test.go delete mode 100644 libs/dyn/value_test.go delete mode 100644 libs/dyn/value_underlying.go delete mode 100644 libs/dyn/value_underlying_test.go delete mode 100644 libs/dyn/visit.go delete mode 100644 libs/dyn/visit_get.go delete mode 100644 libs/dyn/visit_get_test.go delete mode 100644 libs/dyn/visit_map.go delete mode 100644 libs/dyn/visit_map_test.go delete mode 100644 libs/dyn/visit_set.go delete mode 100644 libs/dyn/visit_set_test.go delete mode 100644 libs/dyn/visit_test.go delete mode 100644 libs/dyn/walk.go delete mode 100644 libs/dyn/walk_read_only.go delete mode 100644 libs/dyn/walk_read_only_test.go delete mode 100644 libs/dyn/walk_test.go delete mode 100644 libs/dyn/yamlloader/loader.go delete mode 100644 libs/dyn/yamlloader/testdata/anchor_01.yml delete mode 100644 libs/dyn/yamlloader/testdata/anchor_02.yml delete mode 100644 libs/dyn/yamlloader/testdata/anchor_03.yml delete mode 100644 libs/dyn/yamlloader/testdata/anchor_04.yml delete mode 100644 libs/dyn/yamlloader/testdata/anchor_05.yml delete mode 100644 libs/dyn/yamlloader/testdata/anchor_06.yml delete mode 100644 libs/dyn/yamlloader/testdata/anchor_07.yml delete mode 100644 libs/dyn/yamlloader/testdata/anchor_08.yml delete mode 100644 libs/dyn/yamlloader/testdata/anchor_09.yml delete mode 100644 libs/dyn/yamlloader/testdata/anchor_10.yml delete mode 100644 libs/dyn/yamlloader/testdata/empty.yml delete mode 100644 libs/dyn/yamlloader/testdata/error_01.yml delete mode 100644 libs/dyn/yamlloader/testdata/error_02.yml delete mode 100644 libs/dyn/yamlloader/testdata/error_03.yml delete mode 100644 libs/dyn/yamlloader/testdata/mix_01.yml delete mode 100644 libs/dyn/yamlloader/testdata/mix_02.yml delete mode 100644 libs/dyn/yamlloader/testdata/spec_example_2.1.yml delete mode 100644 libs/dyn/yamlloader/testdata/spec_example_2.10.yml delete mode 100644 libs/dyn/yamlloader/testdata/spec_example_2.11.yml delete mode 100644 libs/dyn/yamlloader/testdata/spec_example_2.12.yml delete mode 100644 libs/dyn/yamlloader/testdata/spec_example_2.13.yml delete mode 100644 libs/dyn/yamlloader/testdata/spec_example_2.14.yml delete mode 100644 libs/dyn/yamlloader/testdata/spec_example_2.15.yml delete mode 100644 libs/dyn/yamlloader/testdata/spec_example_2.16.yml delete mode 100644 libs/dyn/yamlloader/testdata/spec_example_2.17.yml delete mode 100644 libs/dyn/yamlloader/testdata/spec_example_2.18.yml delete mode 100644 libs/dyn/yamlloader/testdata/spec_example_2.19.yml delete mode 100644 libs/dyn/yamlloader/testdata/spec_example_2.2.yml delete mode 100644 libs/dyn/yamlloader/testdata/spec_example_2.20.yml delete mode 100644 libs/dyn/yamlloader/testdata/spec_example_2.21.yml delete mode 100644 libs/dyn/yamlloader/testdata/spec_example_2.22.yml delete mode 100644 libs/dyn/yamlloader/testdata/spec_example_2.23.yml delete mode 100644 libs/dyn/yamlloader/testdata/spec_example_2.24.yml delete mode 100644 libs/dyn/yamlloader/testdata/spec_example_2.25.yml delete mode 100644 libs/dyn/yamlloader/testdata/spec_example_2.26.yml delete mode 100644 libs/dyn/yamlloader/testdata/spec_example_2.27.yml delete mode 100644 libs/dyn/yamlloader/testdata/spec_example_2.28.yml delete mode 100644 libs/dyn/yamlloader/testdata/spec_example_2.3.yml delete mode 100644 libs/dyn/yamlloader/testdata/spec_example_2.4.yml delete mode 100644 libs/dyn/yamlloader/testdata/spec_example_2.5.yml delete mode 100644 libs/dyn/yamlloader/testdata/spec_example_2.6.yml delete mode 100644 libs/dyn/yamlloader/testdata/spec_example_2.7.yml delete mode 100644 libs/dyn/yamlloader/testdata/spec_example_2.8.yml delete mode 100644 libs/dyn/yamlloader/testdata/spec_example_2.9.yml delete mode 100644 libs/dyn/yamlloader/yaml.go delete mode 100644 libs/dyn/yamlloader/yaml_anchor_test.go delete mode 100644 libs/dyn/yamlloader/yaml_error_test.go delete mode 100644 libs/dyn/yamlloader/yaml_mix_test.go delete mode 100644 libs/dyn/yamlloader/yaml_spec_test.go delete mode 100644 libs/dyn/yamlloader/yaml_test.go delete mode 100644 libs/dyn/yamlsaver/order.go delete mode 100644 libs/dyn/yamlsaver/order_test.go delete mode 100644 libs/dyn/yamlsaver/saver.go delete mode 100644 libs/dyn/yamlsaver/saver_test.go delete mode 100644 libs/dyn/yamlsaver/utils.go delete mode 100644 libs/dyn/yamlsaver/utils_test.go create mode 100644 libs/structs/structvar/decode.go create mode 100644 libs/structs/structvar/decode_json.go create mode 100644 libs/structs/structvar/decode_yaml.go rename libs/{dyn => structs/structvar}/kind.go (54%) create mode 100644 libs/structs/structvar/locations.go create mode 100644 libs/structs/structvar/lookup.go create mode 100644 libs/structs/structvar/merge.go create mode 100644 libs/structs/structvar/override.go create mode 100644 libs/structs/structvar/parity_walk_test.go create mode 100644 libs/structs/structvar/pattern.go rename libs/{dyn/dynvar => structs/structvar}/ref.go (88%) rename libs/{dyn/dynvar => structs/structvar}/ref_test.go (56%) rename libs/{dyn/dynvar => structs/structvar}/ref_vectors_test.go (91%) create mode 100644 libs/structs/structvar/resolve.go rename libs/{dyn/convert => structs/structvar}/struct_info.go (65%) rename libs/{dyn => structs/structvar}/suggest.go (60%) rename libs/{dyn/dynvar => structs/structvar}/testdata/reference_vectors.json (99%) create mode 100644 libs/structs/structvar/view.go create mode 100644 libs/structs/structvar/walk.go create mode 100644 libs/structs/structvar/write.go create mode 100644 libs/structs/structyaml/structyaml.go create mode 100644 libs/structs/structyaml/structyaml_test.go diff --git a/.agents/rules/style-guide-go.md b/.agents/rules/style-guide-go.md index ab2dad3b6e9..bedd86132ca 100644 --- a/.agents/rules/style-guide-go.md +++ b/.agents/rules/style-guide-go.md @@ -113,7 +113,7 @@ return fieldPaths ### Encoding -**RULE: When mutating an API response by round-tripping JSON, decode with `json.Decoder` + `UseNumber()`.** The naive `json.Marshal` → `map[string]any` → `Marshal` path corrupts any int64 larger than 2^53 (it degrades to a float64 mantissa — e.g. a real `spark_context_id`) and alphabetizes object keys. `libs/dyn/jsonloader` preserves key order but also lacks `UseNumber`, so it shares the int64 hazard. +**RULE: When mutating an API response by round-tripping JSON, decode with `json.Decoder` + `UseNumber()`.** The naive `json.Marshal` → `map[string]any` → `Marshal` path corrupts any int64 larger than 2^53 (it degrades to a float64 mantissa — e.g. a real `spark_context_id`) and alphabetizes object keys. **RULE: Be careful with `encoding/csv` `Writer.UseCRLF = true`.** It rewrites both record terminators AND embedded newlines inside quoted fields to `\r\n`, so tests for quoted multiline fields must expect `\r\n`, not just the line endings between rows. @@ -191,7 +191,7 @@ func NewService(deps ServiceDeps) *Service { ... } ### Configuration patterns -- Bundle config uses `dyn.Value` for dynamic typing +- Bundle config is typed structs; `libs/structs/structvar` carries `${...}` references and source locations - Config loading supports includes, variable interpolation, and target overrides - Schema generation is automated from Go struct tags diff --git a/.github/workflows/python_push.yml b/.github/workflows/python_push.yml index 1a391719665..a2db548ddac 100644 --- a/.github/workflows/python_push.yml +++ b/.github/workflows/python_push.yml @@ -39,7 +39,7 @@ jobs: acceptance/bundle/python/ \ go.mod \ go.sum \ - libs/dyn/dynvar/testdata/reference_vectors.json \ + libs/structs/structvar/testdata/reference_vectors.json \ python/ \ tools/task/ \ ) diff --git a/.nextchanges/bundles/6954.md b/.nextchanges/bundles/6954.md new file mode 100644 index 00000000000..73ef9f56fe8 --- /dev/null +++ b/.nextchanges/bundles/6954.md @@ -0,0 +1 @@ +* Speed up bundle commands on large bundles: configuration is kept as typed structs instead of being converted to and from a dynamic tree (`bundle validate` and `bundle plan` with 1000 jobs are about 10x faster and use about 3.5x less memory). ([#6954](https://github.com/databricks/cli/pull/6954)) diff --git a/AGENTS.md b/AGENTS.md index 02b63df8169..c0cf2704b63 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -14,7 +14,7 @@ This is the Databricks CLI, a command-line interface for interacting with Databr **RULE: Keep each PR focused on one change.** If you notice an unrelated cleanup, bug fix, or refactor while making your primary change, leave it alone or put it in a separate PR. Reviewers consistently ask to split mixed PRs, especially when a dependency bump or schema diff rides along with a feature change. -**RULE: Before adding a new helper, search the codebase for an existing one.** Common homes: `libs/` (shared utilities), `libs/databrickscfg/` (config), `libs/git/`, `libs/filer/`, `libs/cmdio/` (CLI I/O, spinners, prompts), `libs/env/` (env vars), `libs/testserver/`, `libs/structpath/` and `libs/dyn/` (path / dynamic values), `acceptance/bin/` (acceptance test helpers), `internal/mocks/` (generated mocks). A function that duplicates an existing name and signature in the same package is a compile error waiting to happen; grep before you name. +**RULE: Before adding a new helper, search the codebase for an existing one.** Common homes: `libs/` (shared utilities), `libs/databrickscfg/` (config), `libs/git/`, `libs/filer/`, `libs/cmdio/` (CLI I/O, spinners, prompts), `libs/env/` (env vars), `libs/testserver/`, `libs/structs/structpath/` and `libs/structs/structvar/` (paths / typed config values), `acceptance/bin/` (acceptance test helpers), `internal/mocks/` (generated mocks). A function that duplicates an existing name and signature in the same package is a compile error waiting to happen; grep before you name. # Development Commands @@ -73,7 +73,7 @@ GIT_EDITOR=true GIT_SEQUENCE_EDITOR=true VISUAL=true GIT_PAGER=cat git rebase or - `bundle/phases/` - High-level deployment phases **libs/** - Shared libraries and utilities -- `libs/dyn/` - Dynamic configuration value manipulation +- `libs/structs/structvar/` - Typed configuration values with references and locations - `libs/filer/` - File system abstraction (local, DBFS, workspace) - `libs/auth/` - Databricks authentication handling - `libs/sync/` - File synchronization between local and remote diff --git a/acceptance/invariant_test.go b/acceptance/invariant_test.go index ad8e1d14022..99ac4f4707c 100644 --- a/acceptance/invariant_test.go +++ b/acceptance/invariant_test.go @@ -8,10 +8,9 @@ import ( "testing" "github.com/databricks/cli/bundle/config" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/yamlloader" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.yaml.in/yaml/v3" ) const invariantConfigsDir = "bundle/invariant/configs" @@ -88,31 +87,22 @@ func scanInvariantConfigs(t *testing.T) (present, withPermissions, withGrants ma contents, err := os.ReadFile(path) require.NoError(t, err) - v, err := yamlloader.LoadYAML(path, strings.NewReader(string(contents))) - require.NoError(t, err, "failed to parse %s", path) - - resources := v.Get("resources") - if resources.Kind() != dyn.KindMap { - // Some configs (e.g. PyDABs) declare resources outside of YAML. - continue + var doc struct { + Resources map[string]any `yaml:"resources"` } + require.NoError(t, yaml.Unmarshal(contents, &doc), "failed to parse %s", path) - for _, group := range resources.MustMap().Pairs() { - groupName := group.Key.MustString() + // Some configs (e.g. PyDABs) declare resources outside of YAML. + for groupName, group := range doc.Resources { present[groupName] = true - if group.Value.Kind() != dyn.KindMap { - continue - } - for _, resource := range group.Value.MustMap().Pairs() { - cfg := resource.Value - if cfg.Kind() != dyn.KindMap { - continue - } - if cfg.Get("permissions").Kind() != dyn.KindInvalid { + resources, _ := group.(map[string]any) + for _, resource := range resources { + cfg, _ := resource.(map[string]any) + if _, ok := cfg["permissions"]; ok { withPermissions[groupName] = true } - if cfg.Get("grants").Kind() != dyn.KindInvalid { + if _, ok := cfg["grants"]; ok { withGrants[groupName] = true } } diff --git a/bundle/apps/validate.go b/bundle/apps/validate.go index 2ab3384f991..e00a134bcf5 100644 --- a/bundle/apps/validate.go +++ b/bundle/apps/validate.go @@ -8,8 +8,8 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" "github.com/databricks/databricks-sdk-go/service/apps" ) @@ -28,7 +28,7 @@ func (v *validate) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics Severity: diag.Error, Summary: "Missing app source code path or git source", Detail: fmt.Sprintf("app resource '%s' should have either source_code_path or git_source field", key), - Locations: b.Config.GetLocationsOf(structpath.NewPath(nil, "resources", "apps", key)), + Locations: b.Config.GetLocations("resources.apps." + key), }) continue } @@ -38,7 +38,7 @@ func (v *validate) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics Severity: diag.Error, Summary: "Both source_code_path and git_source fields are set", Detail: fmt.Sprintf("app resource '%s' should have either source_code_path or git_source field, not both", key), - Locations: b.Config.GetLocationsOf(structpath.NewPath(nil, "resources", "apps", key)), + Locations: b.Config.GetLocations("resources.apps." + key), }) continue } @@ -48,7 +48,7 @@ func (v *validate) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics Severity: diag.Error, Summary: "Duplicate app source code path", Detail: fmt.Sprintf("app resource '%s' has the same source code path as app resource '%s', this will lead to the app configuration being overridden by each other", key, usedSourceCodePaths[app.SourceCodePath]), - Locations: b.Config.GetLocationsOf(structpath.NewPath(nil, "resources", "apps", key, "source_code_path")), + Locations: b.Config.GetLocations(fmt.Sprintf("resources.apps.%s.source_code_path", key)), }) } usedSourceCodePaths[app.SourceCodePath] = key @@ -94,32 +94,26 @@ type appResourceReference struct { // hasPermissions checks if a bundle resource at the given dyn path has a non-empty permissions list. func hasPermissions(b *bundle.Bundle, resourcePath string) bool { - pv, err := dyn.Get(b.Config.Value(), resourcePath+".permissions") + pv, err := valueAt(b, resourcePath+".permissions") if err != nil { return false } - s, ok := pv.AsSequence() - return ok && len(s) > 0 + for range pv.Sequence() { + return true + } + return false } // hasAppSPInPermissions checks if any permission entry for the given resource // references the app's service principal via variable interpolation. func hasAppSPInPermissions(b *bundle.Bundle, resourcePath, appKey string) bool { appSPRef := fmt.Sprintf("${resources.apps.%s.service_principal_client_id}", appKey) - pv, err := dyn.Get(b.Config.Value(), resourcePath+".permissions") + pv, err := valueAt(b, resourcePath+".permissions") if err != nil { return false } - s, ok := pv.AsSequence() - if !ok { - return false - } - for _, entry := range s { - spn, err := dyn.Get(entry, "service_principal_name") - if err != nil { - continue - } - if str, ok := spn.AsString(); ok && str == appSPRef { + for _, entry := range pv.Sequence() { + if str, ok := entry.Get("service_principal_name").AsString(); ok && str == appSPRef { return true } } @@ -155,7 +149,7 @@ func warnForAppResourcePermissions(b *bundle.Bundle, appKey string, app *resourc continue } - appPath := structpath.NewPath(nil, "resources", "apps", appKey) + appPath := "resources.apps." + appKey diags = append(diags, diag.Diagnostic{ Severity: diag.Warning, Summary: fmt.Sprintf("app %q references %s %q which has permissions set. To prevent permission override after deploying the app, please add the app service principal to the %s permissions", appKey, refType, resourceKey, refType), @@ -173,8 +167,8 @@ func warnForAppResourcePermissions(b *bundle.Bundle, appKey string, app *resourc ref.permission, appKey, ), - Paths: []*structpath.PathNode{appPath}, - Locations: b.Config.GetLocationsOf(appPath), + Paths: []*structpath.PathNode{structpath.NewStringKeys(nil, "resources", "apps", appKey)}, + Locations: b.Config.GetLocations(appPath), }) } @@ -188,3 +182,12 @@ func (v *validate) Name() string { func Validate() bundle.Mutator { return &validate{} } + +// valueAt returns the view of the configuration value at path. +func valueAt(b *bundle.Bundle, path string) (structvar.View, error) { + p, err := structpath.ParsePath(path) + if err != nil { + return structvar.View{}, err + } + return b.Config.View().Lookup(p), nil +} diff --git a/bundle/artifacts/expand_globs.go b/bundle/artifacts/expand_globs.go index cc5d1f3fb60..0f5a7585aa2 100644 --- a/bundle/artifacts/expand_globs.go +++ b/bundle/artifacts/expand_globs.go @@ -6,16 +6,17 @@ import ( "path/filepath" "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/patchwheel" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) -func createGlobError(v dyn.Value, p dyn.Path, message string) diag.Diagnostic { +func createGlobError(v structvar.View, p *structpath.PathNode, message string) diag.Diagnostic { // The pattern contained in v is an absolute path. // Make it relative to the value's location to make it more readable. - source := v.MustString() + source, _ := v.AsString() if l := v.Location(); l.File != "" { rel, err := filepath.Rel(filepath.Dir(l.File), source) if err == nil { @@ -27,7 +28,7 @@ func createGlobError(v dyn.Value, p dyn.Path, message string) diag.Diagnostic { Severity: diag.Error, Summary: fmt.Sprintf("%s: %s", source, message), Locations: []diag.Location{v.Location()}, - Paths: dyn.ToStructPaths(p), + Paths: []*structpath.PathNode{p}, } } @@ -42,82 +43,78 @@ func (e expandGlobs) Name() string { func (e expandGlobs) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { // Base path for this mutator. // This path is set with the list of expanded globs when done. - base := dyn.NewPath( - dyn.Key("artifacts"), - dyn.Key(e.name), - dyn.Key("files"), - ) + base := structpath.NewStringKeys(nil, "artifacts", e.name, "files") // Pattern to match the source key in the files sequence. - pattern := dyn.NewPatternFromPath(base).Append( - dyn.AnyIndex(), - dyn.Key("source"), - ) + pattern := structpath.NewPatternStringKey(structpath.NewPatternBracketStar(structpath.NewPatternStringKey(structpath.NewPatternStringKey(structpath.NewPatternStringKey(nil, "artifacts"), e.name), "files")), "source") + + artifact := b.Config.Artifacts[e.name] + if artifact == nil { + return nil + } var diags diag.Diagnostics - err := b.Config.Mutate(func(rootv dyn.Value) (dyn.Value, error) { - var output []dyn.Value - _, err := dyn.MapByPattern(rootv, pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - if v.Kind() != dyn.KindString { - return v, nil - } - - source := v.MustString() - - // Expand any glob reference in files source path - matches, err := filepath.Glob(source) - if err != nil { - diags = diags.Append(createGlobError(v, p, err.Error())) - - // Drop this value from the list; this does not matter since we've raised an error anyway - return v, nil - } - - // Note, we're applying this for all artifact types, not just "whl". - // Rationale: - // 1. type is optional - // 2. if you have wheels in other artifact type, maybe you still want the filter logic? impossible to say. - matches = patchwheel.FilterLatestWheels(ctx, matches) - - if len(matches) == 1 && matches[0] == source { - // No glob expansion was performed. - // Keep node unchanged. We need to ensure that "patched" field remains and not wiped out by code below. - parent, err := dyn.GetByPath(rootv, p[0:len(p)-1]) - if err != nil { - log.Debugf(ctx, "Failed to get parent of %s", p.String()) - } else { - output = append(output, parent) - } - return v, nil - } - - if len(matches) == 0 { - diags = diags.Append(createGlobError(v, p, "no matching files")) - - // Drop this value from the list; this does not matter since we've raised an error anyway - return v, nil - } - - for _, match := range matches { - output = append(output, dyn.V( - map[string]dyn.Value{ - "source": dyn.NewValue(match, v.Locations()), - }, - )) - } - - return v, nil - }) - - if err != nil || diags.HasError() { - return rootv, err + var output []config.ArtifactFile + var sources [][]int + var locations [][]diag.Location + err := structvar.ForEach(b.Config.View(), pattern, func(np *structpath.PathNode, v structvar.View) error { + if v.Kind() != structvar.KindString { + return nil + } + + index, _ := np.Parent().Index() + source, _ := v.AsString() + + // Expand any glob reference in files source path + matches, err := filepath.Glob(source) + if err != nil { + diags = diags.Append(createGlobError(v, np, err.Error())) + + // Drop this value from the list; this does not matter since we've raised an error anyway + return nil + } + + // Note, we're applying this for all artifact types, not just "whl". + // Rationale: + // 1. type is optional + // 2. if you have wheels in other artifact type, maybe you still want the filter logic? impossible to say. + matches = patchwheel.FilterLatestWheels(ctx, matches) + + if len(matches) == 1 && matches[0] == source { + // No glob expansion was performed. + // Keep node unchanged. We need to ensure that "patched" field remains and not wiped out by code below. + output = append(output, artifact.Files[index]) + sources = append(sources, []int{index}) + locations = append(locations, nil) + return nil } - // Set the expanded globs back into the configuration. - return dyn.SetByPath(rootv, base, dyn.V(output)) + if len(matches) == 0 { + diags = diags.Append(createGlobError(v, np, "no matching files")) + + // Drop this value from the list; this does not matter since we've raised an error anyway + return nil + } + + for _, match := range matches { + output = append(output, config.ArtifactFile{Source: match}) + sources = append(sources, []int{index}) + locations = append(locations, v.Locations()) + } + + return nil }) - if err != nil { - diags = diags.Extend(diag.FromErr(err)) + if err != nil || diags.HasError() { + return diags.Extend(diag.FromErr(err)) + } + + // Set the expanded globs back into the configuration. + artifact.Files = output + b.Config.UpdateSequence(base, sources) + for i, locs := range locations { + if locs != nil { + b.Config.SetLocations(structpath.NewStringKey(structpath.NewIndex(base, i), "source"), locs) + } } return diags diff --git a/bundle/artifacts/prepare.go b/bundle/artifacts/prepare.go index e285eb1ee30..d5330208eeb 100644 --- a/bundle/artifacts/prepare.go +++ b/bundle/artifacts/prepare.go @@ -61,7 +61,7 @@ func (m *prepare) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics logdiag.LogError(ctx, fmt.Errorf("artifact %q: a tgz artifact needs a `files` entry naming the output path", artifactName)) } - l := b.Config.GetLocation("artifacts." + artifactName) + l := b.Config.DefinitionLocation("artifacts." + artifactName) dirPath := filepath.Dir(l.File) // Check if source paths are absolute, if not, make them absolute diff --git a/bundle/bundle_test.go b/bundle/bundle_test.go index 9bd667afe62..05df7f14228 100644 --- a/bundle/bundle_test.go +++ b/bundle/bundle_test.go @@ -9,7 +9,6 @@ import ( "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/env" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/logdiag" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -157,9 +156,6 @@ func TestBundleGetResourceConfigJobsPointer(t *testing.T) { }, } - // Initialize the dynamic representation so GetResourceConfig can query it. - require.NoError(t, rootCfg.Mutate(func(v dyn.Value) (dyn.Value, error) { return v, nil })) - b := &Bundle{Config: rootCfg} res, err := b.Config.GetResourceConfig("resources.jobs.my_job") diff --git a/bundle/config/loader/process_include.go b/bundle/config/loader/process_include.go index 064855caca2..9fc6e13dfc5 100644 --- a/bundle/config/loader/process_include.go +++ b/bundle/config/loader/process_include.go @@ -10,11 +10,11 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) -func validateFileFormat(configRoot dyn.Value, filePath string) diag.Diagnostics { +func validateFileFormat(configRoot structvar.View, filePath string) diag.Diagnostics { for _, resourceDescription := range config.SupportedResources() { singularName := resourceDescription.SingularName @@ -29,10 +29,10 @@ func validateFileFormat(configRoot dyn.Value, filePath string) diag.Diagnostics return nil } -func validateSingleResourceDefined(configRoot dyn.Value, ext, typ string) diag.Diagnostics { +func validateSingleResourceDefined(configRoot structvar.View, ext, typ string) diag.Diagnostics { type resource struct { - path dyn.Path - value dyn.Value + path *structpath.PathNode + value structvar.View typ string key string } @@ -41,34 +41,34 @@ func validateSingleResourceDefined(configRoot dyn.Value, ext, typ string) diag.D supportedResources := config.SupportedResources() // Gather all resources defined in the resources block. - _, err := dyn.MapByPattern( + err := structvar.ForEach( configRoot, - dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey()), - func(p dyn.Path, v dyn.Value) (dyn.Value, error) { + structpath.MustParsePattern("resources.*.*"), + func(np *structpath.PathNode, v structvar.View) error { // The key for the resource, e.g. "my_job" for jobs.my_job. - k := p[2].Key() + k := np.KeyAt(2) // The type of the resource, e.g. "job" for jobs.my_job. - typ := supportedResources[p[1].Key()].SingularName + typ := supportedResources[np.KeyAt(1)].SingularName - resources = append(resources, resource{path: p, value: v, typ: typ, key: k}) - return v, nil + resources = append(resources, resource{path: np, value: v, typ: typ, key: k}) + return nil }) if err != nil { return diag.FromErr(err) } // Gather all resources defined in a target block. - _, err = dyn.MapByPattern( + err = structvar.ForEach( configRoot, - dyn.NewPattern(dyn.Key("targets"), dyn.AnyKey(), dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey()), - func(p dyn.Path, v dyn.Value) (dyn.Value, error) { + structpath.MustParsePattern("targets.*.resources.*.*"), + func(np *structpath.PathNode, v structvar.View) error { // The key for the resource, e.g. "my_job" for jobs.my_job. - k := p[4].Key() + k := np.KeyAt(4) // The type of the resource, e.g. "job" for jobs.my_job. - typ := supportedResources[p[3].Key()].SingularName + typ := supportedResources[np.KeyAt(3)].SingularName - resources = append(resources, resource{path: p, value: v, typ: typ, key: k}) - return v, nil + resources = append(resources, resource{path: np, value: v, typ: typ, key: k}) + return nil }) if err != nil { return diag.FromErr(err) @@ -112,7 +112,7 @@ func validateSingleResourceDefined(configRoot dyn.Value, ext, typ string) diag.D var paths []*structpath.PathNode for _, rr := range resources { locations = append(locations, rr.value.Locations()...) - paths = append(paths, dyn.ToStructPath(rr.path)) + paths = append(paths, rr.path) } // Sort the locations and paths to make the output deterministic. slices.SortFunc(locations, func(a, b diag.Location) int { @@ -151,15 +151,29 @@ func (m *processInclude) Name() string { } func (m *processInclude) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { - this, diags := config.Load(m.fullPath) + this, diags := m.load() if diags.HasError() { return diags } + err := b.Config.Merge(this) + if err != nil { + diags = diags.Extend(diag.FromErr(err)) + } + return diags +} + +// load loads and validates the included file. +func (m *processInclude) load() (*config.Root, diag.Diagnostics) { + this, diags := config.Load(m.fullPath) + if diags.HasError() { + return nil, diags + } + // Add any diagnostics associated with the file format. - diags = append(diags, validateFileFormat(this.Value(), m.relPath)...) + diags = append(diags, validateFileFormat(this.View(), m.relPath)...) if diags.HasError() { - return diags + return nil, diags } if len(this.Include) > 0 { @@ -173,9 +187,5 @@ Only includes defined in databricks.yml are applied.`, }) } - err := b.Config.Merge(this) - if err != nil { - diags = diags.Extend(diag.FromErr(err)) - } - return diags + return this, diags } diff --git a/bundle/config/loader/process_root_includes.go b/bundle/config/loader/process_root_includes.go index 655ca27162b..36166fa8512 100644 --- a/bundle/config/loader/process_root_includes.go +++ b/bundle/config/loader/process_root_includes.go @@ -10,7 +10,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/logdiag" ) type processRootIncludes struct{} @@ -38,7 +38,7 @@ func hasGlobCharacters(path string) (string, bool) { } func (m *processRootIncludes) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - var out []bundle.Mutator + var out []*processInclude // Preserve the raw include patterns before they are replaced below with the // expanded list of loaded files, so IsFileIncluded can re-match against them. @@ -127,39 +127,40 @@ func (m *processRootIncludes) Apply(ctx context.Context, b *bundle.Bundle) diag. slices.Sort(includes) files = append(files, includes...) for _, include := range includes { - out = append(out, ProcessInclude(filepath.Join(b.BundleRootPath, include), include)) + out = append(out, &processInclude{fullPath: filepath.Join(b.BundleRootPath, include), relPath: include}) } } - // Swap out the original includes list with the expanded globs. This goes through - // Mutate so the dynamic tree is updated too: the includes below are applied without - // their own mutator scope, so nothing converts the typed field back into the dynamic - // tree afterwards, and the next ToTyped would otherwise restore the raw patterns. - err := b.Config.Mutate(func(root dyn.Value) (dyn.Value, error) { - // Include is omitempty in the typed configuration, so an empty list must stay - // absent from the dynamic tree rather than be written as []. - if len(files) == 0 { - return dyn.DropKeys(root, []string{"include"}) - } - - includeValues := make([]dyn.Value, 0, len(files)) - for _, file := range files { - includeValues = append(includeValues, dyn.V(file)) - } - return dyn.Set(root, "include", dyn.NewValue(includeValues, root.Get("include").Locations())) - }) - if err != nil { - return diag.FromErr(err) + // Swap out the original includes list with the expanded globs. + // Include is omitempty in the typed configuration, so an empty list must stay + // absent rather than be written as []. + b.Config.Include = nil + if len(files) > 0 { + b.Config.Include = files } // Track number of bundle YAML (or JSON) files in the configuration. The +1 is there // to account for the root databricks.yaml file. b.Metrics.ConfigurationFileCount = int64(len(files)) + 1 - // ProcessInclude merges into the configuration via [config.Root.Merge], so it does - // not need its own mutator scope. Giving each included file one would re-convert the - // whole accumulated configuration per file, making load quadratic in the number of - // included files (~20 minutes for 6000 files). - bundle.ApplySeqInScopeContext(ctx, b, out...) + // Load all included files first and merge them into the configuration at once. + // Merging them one by one would convert the whole accumulated configuration per + // file, making load quadratic in the number of included files. + var roots []*config.Root + for _, m := range out { + this, diags := m.load() + for _, d := range diags { + logdiag.LogDiag(ctx, d) + } + if diags.HasError() { + return nil + } + roots = append(roots, this) + } + + err := b.Config.Merge(roots...) + if err != nil { + return diag.FromErr(err) + } return nil } diff --git a/bundle/config/loader/process_root_includes_test.go b/bundle/config/loader/process_root_includes_test.go index e7a850a835b..7403c1e01c5 100644 --- a/bundle/config/loader/process_root_includes_test.go +++ b/bundle/config/loader/process_root_includes_test.go @@ -9,7 +9,7 @@ import ( "github.com/databricks/cli/bundle/config/loader" "github.com/databricks/cli/internal/testutil" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structvar" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -119,7 +119,7 @@ func TestProcessRootIncludesUpdatesDynamicValue(t *testing.T) { require.NoError(t, diags.Error()) assert.Equal(t, []string{"a.yml"}, b.Config.Include) - assert.Equal(t, []any{"a.yml"}, b.Config.Value().Get("include").AsAny()) + assert.Equal(t, []any{"a.yml"}, b.Config.View().Get("include").AsAny()) } // An empty include list must stay absent from the dynamic tree: the typed field is @@ -139,7 +139,7 @@ func TestProcessRootIncludesEmptyOmitsDynamicValue(t *testing.T) { diags := bundle.Apply(t.Context(), b, loader.ProcessRootIncludes()) require.NoError(t, diags.Error()) assert.Empty(t, b.Config.Include) - assert.Equal(t, dyn.KindInvalid, b.Config.Value().Get("include").Kind()) + assert.Equal(t, structvar.KindInvalid, b.Config.View().Get("include").Kind()) } func TestProcessRootIncludesNotExists(t *testing.T) { diff --git a/libs/dyn/dynloc/locations.go b/bundle/config/loctable/locations.go similarity index 61% rename from libs/dyn/dynloc/locations.go rename to bundle/config/loctable/locations.go index d343e24533c..e7307e04c1e 100644 --- a/libs/dyn/dynloc/locations.go +++ b/bundle/config/loctable/locations.go @@ -1,4 +1,4 @@ -package dynloc +package loctable import ( "fmt" @@ -7,7 +7,6 @@ import ( "slices" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" ) const ( @@ -16,7 +15,7 @@ const ( Version = 1 ) -// Locations is a structure that holds location information for (a subset of) a [dyn.Value] value. +// Locations is a structure that holds location information for (a subset of) a configuration. type Locations struct { // Version is the version of the location information. Version int `json:"version"` @@ -24,9 +23,9 @@ type Locations struct { // Files is a list of file paths. Files []string `json:"files"` - // Locations maps the string representation of a [dyn.Path] to a list of 3-tuples that represent the index + // Locations maps the string representation of a path to a list of 3-tuples that represent the index // of the file in the [Files] array, followed by the line and column number. - // A single [dyn.Path] can have multiple locations (e.g. the effective location and original definition). + // A single path can have multiple locations (e.g. the effective location and original definition). Locations map[string][][]int `json:"locations"` // fileToIndex maps file paths to their index in the [Files] array. @@ -38,29 +37,6 @@ type Locations struct { basePath string } -func (l *Locations) gatherLocations(v dyn.Value) (map[string][]diag.Location, error) { - locs := map[string][]diag.Location{} - patterns := []dyn.Pattern{ - dyn.NewPattern(dyn.AnyKey()), // Top level fields - dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey()), // Resource groups ("resources.jobs") - dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey()), // Resources for all types ("resources.jobs.my_job") - dyn.NewPattern(dyn.Key("resources"), dyn.Key("jobs"), dyn.AnyKey(), dyn.Key("tasks")), // Job tasks ("resources.jobs.my_job.tasks") - dyn.NewPattern(dyn.Key("resources"), dyn.Key("jobs"), dyn.AnyKey(), dyn.Key("tasks"), dyn.AnyIndex()), // Job task items ("resources.jobs.my_job.tasks[2]") - } - - for _, pattern := range patterns { - _, err := dyn.MapByPattern(v, pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - locs[p.String()] = v.Locations() - return v, nil - }) - if err != nil { - return nil, err - } - } - - return locs, nil -} - func (l *Locations) normalizeFilePath(file string) (string, error) { var err error @@ -113,7 +89,7 @@ func (l *Locations) addLocation(path, file string, line, col int) error { // Expect the file to be present in the lookup map. if _, ok := l.fileToIndex[file]; !ok { // This indicates a logic problem below, but we rather not panic. - return fmt.Errorf("dynloc: unknown file %q", file) + return fmt.Errorf("loctable: unknown file %q", file) } // Add the location to the map. @@ -125,8 +101,8 @@ func (l *Locations) addLocation(path, file string, line, col int) error { return nil } -// Build constructs a [Locations] object from a [dyn.Value]. -func Build(v dyn.Value, basePath string) (Locations, error) { +// Build constructs a [Locations] object from locations keyed by the string representation of a path. +func Build(pathToLocations map[string][]diag.Location, basePath string) (Locations, error) { l := Locations{ Version: Version, Files: make([]string, 0), @@ -137,15 +113,9 @@ func Build(v dyn.Value, basePath string) (Locations, error) { basePath: basePath, } - // Traverse the value and collect locations. - pathToLocations, err := l.gatherLocations(v) - if err != nil { - return l, err - } - // Normalize file paths and add locations. // This step adds files to the [Files] array in alphabetical order. - err = l.registerFileNames(slices.Concat(slices.Collect(maps.Values(pathToLocations))...)) + err := l.registerFileNames(slices.Concat(slices.Collect(maps.Values(pathToLocations))...)) if err != nil { return l, err } diff --git a/bundle/config/mutator/apply_source_linked_deployment_preset.go b/bundle/config/mutator/apply_source_linked_deployment_preset.go index 695edabc68a..99e06d56d24 100644 --- a/bundle/config/mutator/apply_source_linked_deployment_preset.go +++ b/bundle/config/mutator/apply_source_linked_deployment_preset.go @@ -13,6 +13,11 @@ import ( type applySourceLinkedDeploymentPreset struct{} +func targetPresetPath(target string) *structpath.PathNode { + p := structpath.NewStringKeys(nil, "targets", target) + return structpath.NewStringKeys(p, "presets", "source_linked_deployment") +} + // Apply source-linked deployment preset func ApplySourceLinkedDeploymentPreset() *applySourceLinkedDeploymentPreset { return &applySourceLinkedDeploymentPreset{} @@ -33,13 +38,13 @@ func (m *applySourceLinkedDeploymentPreset) Apply(ctx context.Context, b *bundle if config.IsExplicitlyEnabled(b.Config.Presets.SourceLinkedDeployment) { if !isDatabricksWorkspace { - path := structpath.NewPath(nil, "targets", target, "presets", "source_linked_deployment") + path := targetPresetPath(target) diags = diags.Append( diag.Diagnostic{ Severity: diag.Warning, Summary: "source-linked deployment is available only in the Databricks Workspace", Paths: []*structpath.PathNode{path}, - Locations: b.Config.GetLocationsOf(path.SkipPrefix(2)), + Locations: b.Config.GetLocations("presets.source_linked_deployment"), }, ) @@ -51,13 +56,13 @@ func (m *applySourceLinkedDeploymentPreset) Apply(ctx context.Context, b *bundle b.Metrics.AddBoolValue("source_linked_set_for_non_development", b.Config.Bundle.Mode != config.Development) if b.Config.Bundle.Mode != config.Development { - path := structpath.NewPath(nil, "targets", target, "presets", "source_linked_deployment") + path := targetPresetPath(target) diags = diags.Append( diag.Diagnostic{ Severity: diag.Warning, Summary: "source-linked deployment in non-development mode is deprecated and will not be supported in a future release", Paths: []*structpath.PathNode{path}, - Locations: b.Config.GetLocationsOf(path.SkipPrefix(2)), + Locations: b.Config.GetLocations("presets.source_linked_deployment"), }, ) } @@ -70,14 +75,14 @@ func (m *applySourceLinkedDeploymentPreset) Apply(ctx context.Context, b *bundle // This mutator runs before workspace paths are defaulted so it's safe to check for the user-defined value if b.Config.Workspace.FilePath != "" && config.IsExplicitlyEnabled(b.Config.Presets.SourceLinkedDeployment) { - path := structpath.NewPath(nil, "workspace", "file_path") + path := structpath.MustParsePath("workspace.file_path") diags = diags.Append( diag.Diagnostic{ Severity: diag.Warning, Summary: "workspace.file_path setting will be ignored in source-linked deployment mode", Detail: "In source-linked deployment files are not copied to the destination and resources use source files instead", Paths: []*structpath.PathNode{path}, - Locations: b.Config.GetLocationsOf(path), + Locations: b.Config.GetLocations(path.String()), }, ) } diff --git a/bundle/config/mutator/collect_escape_telemetry.go b/bundle/config/mutator/collect_escape_telemetry.go index b24922bb5b3..dad06bb2312 100644 --- a/bundle/config/mutator/collect_escape_telemetry.go +++ b/bundle/config/mutator/collect_escape_telemetry.go @@ -2,11 +2,11 @@ package mutator import ( "context" + "reflect" "strings" "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" ) // Temporary: this mutator collects telemetry on escape patterns ($${}, $$, \${}, \$) @@ -24,12 +24,7 @@ func (*collectEscapeTelemetry) Name() string { func (*collectEscapeTelemetry) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { var hasDoubleDollarBrace, hasDoubleDollar, hasBackslashDollarBrace, hasBackslashDollar bool - _, err := dyn.Walk(b.Config.Value(), func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - s, ok := v.AsString() - if !ok { - return v, nil - } - + walkStrings(reflect.ValueOf(&b.Config), func(s string) { if !hasDoubleDollarBrace && strings.Contains(s, "$${") { hasDoubleDollarBrace = true } @@ -42,12 +37,7 @@ func (*collectEscapeTelemetry) Apply(ctx context.Context, b *bundle.Bundle) diag if !hasBackslashDollar && containsBackslashDollarWithoutBrace(s) { hasBackslashDollar = true } - - return v, nil }) - if err != nil { - return diag.FromErr(err) - } if hasDoubleDollarBrace { b.Metrics.SetBoolValue("config_has_double_dollar_brace", true) @@ -65,6 +55,40 @@ func (*collectEscapeTelemetry) Apply(ctx context.Context, b *bundle.Bundle) diag return nil } +// walkStrings calls fn for every string value in the configuration value v, +// including strings stored in interface values (e.g. variable defaults). +func walkStrings(v reflect.Value, fn func(string)) { + switch v.Kind() { + case reflect.String: + fn(v.String()) + case reflect.Pointer, reflect.Interface: + if !v.IsNil() { + walkStrings(v.Elem(), fn) + } + case reflect.Struct: + t := v.Type() + for i := range t.NumField() { + f := t.Field(i) + // Like the configuration conversion, skip fields without a JSON name. + name, _, _ := strings.Cut(f.Tag.Get("json"), ",") + if !f.IsExported() || (!f.Anonymous && (name == "" || name == "-")) { + continue + } + walkStrings(v.Field(i), fn) + } + case reflect.Slice: + for i := range v.Len() { + walkStrings(v.Index(i), fn) + } + case reflect.Map: + iter := v.MapRange() + for iter.Next() { + walkStrings(iter.Value(), fn) + } + default: + } +} + // containsDoubleDollarWithoutBrace returns true if s contains "$$" not followed by "{". func containsDoubleDollarWithoutBrace(s string) bool { for i := range len(s) - 1 { diff --git a/bundle/config/mutator/compute_id_compat.go b/bundle/config/mutator/compute_id_compat.go index 35568441ce8..605d4db5a30 100644 --- a/bundle/config/mutator/compute_id_compat.go +++ b/bundle/config/mutator/compute_id_compat.go @@ -2,11 +2,14 @@ package mutator import ( "context" + "maps" "slices" "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) type computeIdToClusterId struct{} @@ -22,68 +25,48 @@ func (m *computeIdToClusterId) Name() string { func (m *computeIdToClusterId) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { var diags diag.Diagnostics + // Skip if "compute_id" is not set anywhere. + if b.Config.Bundle.ComputeId == "" && !slices.ContainsFunc(slices.Collect(maps.Values(b.Config.Targets)), func(t *config.Target) bool { + return t != nil && t.ComputeId != "" + }) { + return nil + } + // The "compute_id" key is set; rewrite it to "cluster_id". - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { - v, d := rewriteComputeIdToClusterId(v, dyn.NewPath(dyn.Key("bundle")), nil) - diags = diags.Extend(d) + view := b.Config.View() + bundlePath := structpath.NewStringKey(nil, "bundle") + diags = diags.Extend(rewriteComputeIdToClusterId(b, bundlePath, view.Get("bundle"))) - // Check if the "compute_id" key is set in any target overrides. - return dyn.MapByPattern(v, dyn.NewPattern(dyn.Key("targets"), dyn.AnyKey()), func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - v, d := rewriteComputeIdToClusterId(v, dyn.Path{}, p) - diags = diags.Extend(d) - return v, nil - }) + // Check if the "compute_id" key is set in any target overrides. + err := structvar.ForEach(view, structpath.MustParsePattern("targets.*"), func(p *structpath.PathNode, v structvar.View) error { + diags = diags.Extend(rewriteComputeIdToClusterId(b, p, v)) + return nil }) diags = diags.Extend(diag.FromErr(err)) return diags } -// rewriteComputeIdToClusterId rewrites p.compute_id to p.cluster_id in v; prefix is the -// path of v in the configuration, used in the diagnostic. -func rewriteComputeIdToClusterId(v dyn.Value, p, prefix dyn.Path) (dyn.Value, diag.Diagnostics) { +// rewriteComputeIdToClusterId rewrites the "compute_id" key of the map v at path p to "cluster_id". +func rewriteComputeIdToClusterId(b *bundle.Bundle, p *structpath.PathNode, v structvar.View) diag.Diagnostics { var diags diag.Diagnostics - computeIdPath := p.Append(dyn.Key("compute_id")) - computeId, err := dyn.GetByPath(v, computeIdPath) + computeId := v.Get("compute_id") // If the "compute_id" key is not set, we don't need to do anything. - if err != nil { - return v, nil - } - - if computeId.Kind() == dyn.KindInvalid { - return v, nil + if !computeId.IsValid() { + return nil } diags = diags.Append(diag.Diagnostic{ Severity: diag.Warning, Summary: "compute_id is deprecated, please use cluster_id instead", Locations: computeId.Locations(), - Paths: dyn.ToStructPaths(slices.Concat(prefix, computeIdPath)), + Paths: []*structpath.PathNode{structpath.NewStringKey(p, "compute_id")}, }) - clusterIdPath := p.Append(dyn.Key("cluster_id")) - nv, err := dyn.SetByPath(v, clusterIdPath, computeId) + err := b.Config.Assign(structpath.NewStringKey(p, "cluster_id"), computeId) if err != nil { - return dyn.InvalidValue, diag.FromErr(err) + return diags.Extend(diag.FromErr(err)) } // Drop the "compute_id" key. - vout, err := dyn.Walk(nv, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - switch len(p) { - case 0: - return v, nil - case 1: - if p[0] == dyn.Key("compute_id") { - return v, dyn.ErrDrop - } - return v, nil - case 2: - if p[1] == dyn.Key("compute_id") { - return v, dyn.ErrDrop - } - } - return v, dyn.ErrSkip - }) - - diags = diags.Extend(diag.FromErr(err)) - return vout, diags + return diags.Extend(diag.FromErr(b.Config.Delete(structpath.NewStringKey(p, "compute_id")))) } diff --git a/bundle/config/mutator/drop_empty_strings.go b/bundle/config/mutator/drop_empty_strings.go index 028351ed943..f80a7d1f583 100644 --- a/bundle/config/mutator/drop_empty_strings.go +++ b/bundle/config/mutator/drop_empty_strings.go @@ -2,12 +2,12 @@ package mutator import ( "context" + "reflect" "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/convert" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) type dropEmptyStrings struct{} @@ -17,10 +17,8 @@ type dropEmptyStrings struct{} // either literally (policy_id: "") or via a variable that resolved to "", so // this must run after variable resolution. // -// The direct engine converts the resolved config through convert.ToTyped, which -// force-sends explicitly-set zero values, defeating the omitempty tag. Dropping -// here fixes it and makes the result visible in `bundle validate -o json`, which -// serializes the dynamic value. +// An explicitly-set zero value is force-sent, defeating the omitempty tag. +// Dropping it here fixes it and makes the result visible in `bundle validate -o json`. func DropEmptyStrings() bundle.Mutator { return &dropEmptyStrings{} } @@ -30,31 +28,71 @@ func (m *dropEmptyStrings) Name() string { } func (m *dropEmptyStrings) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - var diags diag.Diagnostics - err := b.Config.Mutate(func(root dyn.Value) (dyn.Value, error) { - root, err := dyn.Map(root, "resources", func(_ dyn.Path, resources dyn.Value) (dyn.Value, error) { - // Normalize against the resources type so omitempty is known per field. - // Only DropEmptyStrings is set: existing values are kept as-is otherwise. - out, normDiags := convert.Normalize(config.Resources{}, resources, convert.DropEmptyStrings) - diags = diags.Extend(normDiags) - return out, nil - }) + resourcesPath := structpath.NewStringKey(nil, "resources") + + // Drop the empty strings on all struct fields except the ones that must send their zero value. + var empty []*structpath.PathNode + collectEmptyStrings(b.Config.View().Get("resources"), resourcesPath, &empty) + for _, p := range empty { + if err := b.Config.Delete(p); err != nil { + return diag.FromErr(err) + } + } + + // Keep an empty apps description instead of dropping it: an app update without + // the field leaves the remote description unchanged, so removing description + // from config would never converge. Force-sending "" clears it. + appsPath := structpath.NewStringKey(resourcesPath, "apps") + var apps []string + for name, app := range b.Config.View().Lookup(appsPath).MapItems() { + if !app.Get("description").IsValid() { + apps = append(apps, name) + } + } + for _, name := range apps { + err := b.Config.Set(structpath.NewStringKeys(appsPath, name, "description"), "") if err != nil { - return root, err + return diag.FromErr(err) + } + } + return nil +} + +// collectEmptyStrings appends to out the paths of the struct fields in v that are empty strings, +// unless the field is tagged without omitempty (i.e. its zero value must be sent). +func collectEmptyStrings(v structvar.View, p *structpath.PathNode, out *[]*structpath.PathNode) { + switch v.Kind() { + case structvar.KindSequence: + for i, e := range v.Sequence() { + collectEmptyStrings(e, structpath.NewIndex(p, i), out) + } + case structvar.KindMap: + r := v.Reflect() + for r.Kind() == reflect.Pointer { + r = r.Elem() } - // Keep an empty apps description instead of dropping it: an app update without - // the field leaves the remote description unchanged, so removing description - // from config would never converge. Force-sending "" clears it. - return dyn.MapByPattern(root, dyn.NewPattern(dyn.Key("resources"), dyn.Key("apps"), dyn.AnyKey()), func(_ dyn.Path, app dyn.Value) (dyn.Value, error) { - if _, err := dyn.Get(app, "description"); err != nil { - return dyn.Set(app, "description", dyn.V("")) + var info *structvar.StructInfo + switch { + case r.Kind() == reflect.Struct && !structvar.IsSDKNativeType(r.Type()): + i := structvar.GetStructInfo(r.Type()) + info = &i + case r.Kind() == reflect.Map: + default: + return + } + + for k, c := range v.MapItems() { + if info != nil && !info.ForceEmpty[k] { + if _, ok := info.Fields[k]; ok { + if s, ok := c.AsString(); ok && s == "" { + *out = append(*out, structpath.NewStringKey(p, k)) + continue + } + } } - return app, nil - }) - }) - if err != nil { - diags = diags.Extend(diag.FromErr(err)) + collectEmptyStrings(c, structpath.NewStringKey(p, k), out) + } + default: } - return diags } diff --git a/bundle/config/mutator/environments_compat.go b/bundle/config/mutator/environments_compat.go index fb898edea36..b00a7d824a6 100644 --- a/bundle/config/mutator/environments_compat.go +++ b/bundle/config/mutator/environments_compat.go @@ -2,11 +2,10 @@ package mutator import ( "context" - "fmt" "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" ) type environmentsToTargets struct{} @@ -27,40 +26,26 @@ func (m *environmentsToTargets) Apply(ctx context.Context, b *bundle.Bundle) dia } // The "environments" key is set; validate and rewrite it to "targets". - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { - environments := v.Get("environments") - targets := v.Get("targets") - - // Return an error if both "environments" and "targets" are set. - if environments.Kind() != dyn.KindInvalid && targets.Kind() != dyn.KindInvalid { - return dyn.InvalidValue, fmt.Errorf( - "both 'environments' and 'targets' are specified; only 'targets' should be used: %s", - environments.Location().String(), - ) - } + view := b.Config.View() + environments := view.Get("environments") + targets := view.Get("targets") + + // Return an error if both "environments" and "targets" are set. + if environments.IsValid() && targets.IsValid() { + return diag.Errorf( + "both 'environments' and 'targets' are specified; only 'targets' should be used: %s", + environments.Location().String(), + ) + } - // Rewrite "environments" to "targets". - if environments.Kind() != dyn.KindInvalid && targets.Kind() == dyn.KindInvalid { - nv, err := dyn.Set(v, "targets", environments) - if err != nil { - return dyn.InvalidValue, err - } - // Drop the "environments" key. - return dyn.Walk(nv, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - switch len(p) { - case 0: - return v, nil - case 1: - if p[0] == dyn.Key("environments") { - return v, dyn.ErrDrop - } - } - return v, dyn.ErrSkip - }) + // Rewrite "environments" to "targets" and drop the "environments" key. + if environments.IsValid() && !targets.IsValid() { + err := b.Config.Assign(structpath.NewStringKey(nil, "targets"), environments) + if err != nil { + return diag.FromErr(err) } + return diag.FromErr(b.Config.Delete(structpath.NewStringKey(nil, "environments"))) + } - return v, nil - }) - - return diag.FromErr(err) + return nil } diff --git a/bundle/config/mutator/initialize_volume_paths.go b/bundle/config/mutator/initialize_volume_paths.go index 20ea9ae6b38..fa971a92298 100644 --- a/bundle/config/mutator/initialize_volume_paths.go +++ b/bundle/config/mutator/initialize_volume_paths.go @@ -5,11 +5,9 @@ import ( "fmt" "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/convert" - "github.com/databricks/cli/libs/dyn/dynvar" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) type initializeVolumePaths struct{} @@ -32,26 +30,26 @@ func (m *initializeVolumePaths) Name() string { } func (m *initializeVolumePaths) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { - err := b.Config.Mutate(func(root dyn.Value) (dyn.Value, error) { - pattern := dyn.NewPattern(dyn.Key("resources"), dyn.Key("volumes"), dyn.AnyKey()) - return dyn.MapByPattern(root, pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - // volume_path is computed and read-only; reject a user-provided value instead of overwriting it. - if existing, ok := v.Get("volume_path").AsString(); ok && existing != "" { - return dyn.InvalidValue, fmt.Errorf("%s.volume_path is computed and read-only; remove it from the configuration", p.String()) - } + // Skip converting the configuration if there are no volumes. + if len(b.Config.Resources.Volumes) == 0 { + return nil + } - var vol resources.Volume - if err := convert.ToTyped(&vol, v); err != nil { - return dyn.InvalidValue, err - } + view := b.Config.View() + pattern := structpath.MustParsePattern("resources.volumes.*") + err := structvar.ForEach(view, pattern, func(p *structpath.PathNode, v structvar.View) error { + // volume_path is computed and read-only; reject a user-provided value instead of overwriting it. + if existing, ok := v.Get("volume_path").AsString(); ok && existing != "" { + return fmt.Errorf("%s.volume_path is computed and read-only; remove it from the configuration", p.String()) + } - // Resolve references to compute the path only; the field values in v are left untouched. - vol.CatalogName = resolveResourceReference(root, vol.CatalogName) - vol.SchemaName = resolveResourceReference(root, vol.SchemaName) - vol.Name = resolveResourceReference(root, vol.Name) + // Resolve references to compute the path only; the field values are left untouched. + vol := *b.Config.Resources.Volumes[p.KeyAt(2)] + vol.CatalogName = resolveResourceReference(view, vol.CatalogName) + vol.SchemaName = resolveResourceReference(view, vol.SchemaName) + vol.Name = resolveResourceReference(view, vol.Name) - return dyn.Set(v, "volume_path", dyn.V(vol.ComputeVolumePath())) - }) + return b.Config.Set(structpath.NewStringKey(p, "volume_path"), vol.ComputeVolumePath()) }) if err != nil { return diag.FromErr(err) @@ -62,16 +60,12 @@ func (m *initializeVolumePaths) Apply(_ context.Context, b *bundle.Bundle) diag. // resolveResourceReference resolves a pure ${resources....} reference by looking it up in root. // Values that are not such a reference, or cannot be resolved, are returned unchanged (still // containing "${"), so the caller embeds the reference verbatim to be resolved later. -func resolveResourceReference(root dyn.Value, s string) string { - p, ok := dynvar.PureReferenceToPath(s) - if !ok || p[0].Key() != "resources" { - return s - } - rv, err := dyn.GetByPath(root, p) - if err != nil { +func resolveResourceReference(root structvar.View, s string) string { + p, ok := structvar.PureReferenceToPath(s) + if !ok || p.KeyAt(0) != "resources" { return s } - rs, ok := rv.AsString() + rs, ok := root.Lookup(p).AsString() if !ok { return s } diff --git a/bundle/config/mutator/load_dbalert_files.go b/bundle/config/mutator/load_dbalert_files.go index 437095aa218..00f30b920b9 100644 --- a/bundle/config/mutator/load_dbalert_files.go +++ b/bundle/config/mutator/load_dbalert_files.go @@ -11,9 +11,8 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/dynvar" "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" "github.com/databricks/databricks-sdk-go/marshal" "github.com/databricks/databricks-sdk-go/service/sql" ) @@ -57,26 +56,21 @@ func (m *loadDBAlertFiles) Apply(ctx context.Context, b *bundle.Bundle) diag.Dia continue } - alertV, err := dyn.GetByPath(b.Config.Value(), dyn.NewPath(dyn.Key("resources"), dyn.Key("alerts"), dyn.Key(alertKey))) - if err != nil { - return diag.FromErr(err) - } + alertPath := structpath.NewStringKeys(nil, "resources", "alerts", alertKey) + alertV := b.Config.View().Lookup(alertPath) // No other fields other than allowedInYAML should be set in the bundle YAML. - m, ok := alertV.AsMap() - if !ok { + if alertV.Kind() != structvar.KindMap { return diag.FromErr(fmt.Errorf("internal error: alert value is not a map, got %s", alertV.Kind())) } - for _, p := range m.Pairs() { - k := p.Key.MustString() - v := p.Value + for k, v := range alertV.MapItems() { if slices.Contains(allowedInYAML, k) { continue } - if v.Kind() == dyn.KindNil || v.Kind() == dyn.KindInvalid { + if v.Kind() == structvar.KindNil || v.Kind() == structvar.KindInvalid { continue } @@ -86,7 +80,7 @@ func (m *loadDBAlertFiles) Apply(ctx context.Context, b *bundle.Bundle) diag.Dia Severity: diag.Error, Summary: fmt.Sprintf("field %s is not allowed in the bundle configuration.", k), Detail: "When a .dbalert.json is specified, only the following fields are allowed in the bundle configuration: " + strings.Join(allowedInYAML, ", "), - Paths: structpath.NewPathSlice("resources", "alerts", alertKey, k), + Paths: []*structpath.PathNode{structpath.NewStringKey(alertPath, k)}, Locations: v.Locations(), }, } @@ -108,7 +102,7 @@ func (m *loadDBAlertFiles) Apply(ctx context.Context, b *bundle.Bundle) diag.Dia Severity: diag.Error, Summary: fmt.Sprintf("failed to read .dbalert.json file %s: %s", alert.FilePath, err), Detail: "", - Paths: structpath.NewPathSlice("resources", "alerts", alertKey, "file_path"), + Paths: []*structpath.PathNode{structpath.NewStringKey(alertPath, "file_path")}, Locations: alertV.Get("file_path").Locations(), }, } @@ -123,21 +117,21 @@ func (m *loadDBAlertFiles) Apply(ctx context.Context, b *bundle.Bundle) diag.Dia Severity: diag.Error, Summary: fmt.Sprintf("failed to parse .dbalert.json file %s: %s", alert.FilePath, err), Detail: "", - Paths: structpath.NewPathSlice("resources", "alerts", alertKey, "file_path"), + Paths: []*structpath.PathNode{structpath.NewStringKey(alertPath, "file_path")}, Locations: alertV.Get("file_path").Locations(), }, } } // Check that the file does not have any variable interpolations. - if dynvar.ContainsVariableReference(string(content)) { + if structvar.ContainsVariableReference(string(content)) { return diag.Diagnostics{ { ID: diag.ID(""), Severity: diag.Error, Summary: fmt.Sprintf(".alert file %s must not contain variable interpolations.", alert.FilePath), Detail: "Please inline the alert configuration in the bundle configuration to use variables", - Paths: structpath.NewPathSlice("resources", "alerts", alertKey, "file_path"), + Paths: []*structpath.PathNode{structpath.NewStringKey(alertPath, "file_path")}, Locations: alertV.Get("file_path").Locations(), }, } diff --git a/bundle/config/mutator/normalize_paths.go b/bundle/config/mutator/normalize_paths.go index 7256ebad4b7..bbf322b3429 100644 --- a/bundle/config/mutator/normalize_paths.go +++ b/bundle/config/mutator/normalize_paths.go @@ -14,7 +14,8 @@ import ( "github.com/databricks/cli/bundle/config/mutator/paths" "github.com/databricks/cli/bundle/libraries" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) type normalizePaths struct{} @@ -43,41 +44,39 @@ func (a normalizePaths) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnost // Do not normalize job task paths if using git source gitSourcePaths := collectGitSourcePaths(b) - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { - return paths.VisitPaths(v, func(path dyn.Path, kind paths.TranslateMode, v dyn.Value) (dyn.Value, error) { - if slices.ContainsFunc(gitSourcePaths, path.HasPrefix) { - return v, nil - } + err := paths.VisitPaths(b.Config.View(), func(path *structpath.PathNode, kind paths.TranslateMode, v structvar.View) error { + if slices.ContainsFunc(gitSourcePaths, path.HasPrefix) { + return nil + } - value, ok := v.AsString() - if !ok { - return dyn.InvalidValue, fmt.Errorf("value at %s is not a string", path.String()) - } + value, ok := v.AsString() + if !ok { + return fmt.Errorf("value at %s is not a string", path.String()) + } - newValue, err := normalizePath(value, v.Location(), b.BundleRootPath) - if err != nil { - return dyn.InvalidValue, err - } + newValue, err := normalizePath(value, v.Location(), b.BundleRootPath) + if err != nil { + return err + } - return dyn.NewValue(newValue, v.Locations()), nil - }) + return b.Config.Set(path, newValue) }) if err != nil { return diag.FromErr(fmt.Errorf("failed to normalize paths: %w", err)) } - return diag.FromErr(err) + return nil } -func collectGitSourcePaths(b *bundle.Bundle) []dyn.Path { - var jobs []dyn.Path +func collectGitSourcePaths(b *bundle.Bundle) []*structpath.PathNode { + var jobs []*structpath.PathNode for name, job := range b.Config.Resources.Jobs { if job == nil { continue } if job.GitSource != nil { - jobs = append(jobs, dyn.NewPath(dyn.Key("resources"), dyn.Key("jobs"), dyn.Key(name))) + jobs = append(jobs, structpath.NewStringKeys(nil, "resources", "jobs", name)) } } diff --git a/bundle/config/mutator/normalize_paths_test.go b/bundle/config/mutator/normalize_paths_test.go index 9c9085a27c6..faefb4b2a63 100644 --- a/bundle/config/mutator/normalize_paths_test.go +++ b/bundle/config/mutator/normalize_paths_test.go @@ -8,7 +8,7 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -38,20 +38,14 @@ func TestNormalizePaths(t *testing.T) { // update config as if 'notebook_path' property is defined in resources/job_1.yml location := diag.Location{File: filepath.Join(tmpDir, "resources", "job_1.yml")} - path := dyn.MustPathFromString("resources.jobs.job1.tasks[0].notebook_task.notebook_path") - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { - return dyn.MapByPath(v, path, func(path dyn.Path, value dyn.Value) (dyn.Value, error) { - return dyn.NewValue(value.MustString(), []diag.Location{location}), nil - }) - }) - require.NoError(t, err) + path := structpath.MustParsePath("resources.jobs.job1.tasks[0].notebook_task.notebook_path") + b.Config.SetLocations(path, []diag.Location{location}) diags := bundle.Apply(t.Context(), b, m) require.NoError(t, diags.Error()) - newValue, err := dyn.GetByPath(b.Config.Value(), path) - require.NoError(t, err) - require.Equal(t, "src/notebook.py", newValue.MustString()) + require.Equal(t, "src/notebook.py", b.Config.Resources.Jobs["job1"].Tasks[0].NotebookTask.NotebookPath) + require.Equal(t, []diag.Location{location}, b.Config.LocationsAt(path)) } func TestNormalizePaths_jobRunOnFileChange(t *testing.T) { @@ -77,20 +71,14 @@ func TestNormalizePaths_jobRunOnFileChange(t *testing.T) { } location := diag.Location{File: filepath.Join(tmpDir, "resources", "run.yml")} - path := dyn.MustPathFromString("resources.job_runs.run1.lifecycle.triggers[0].on_file_change") - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { - return dyn.MapByPath(v, path, func(path dyn.Path, value dyn.Value) (dyn.Value, error) { - return dyn.NewValue(value.MustString(), []diag.Location{location}), nil - }) - }) - require.NoError(t, err) + path := structpath.MustParsePath("resources.job_runs.run1.lifecycle.triggers[0].on_file_change") + b.Config.SetLocations(path, []diag.Location{location}) diags := bundle.Apply(t.Context(), b, m) require.NoError(t, diags.Error()) - newValue, err := dyn.GetByPath(b.Config.Value(), path) - require.NoError(t, err) - require.Equal(t, "data/*.txt", newValue.MustString()) + require.Equal(t, "data/*.txt", *b.Config.Resources.JobRuns["run1"].Lifecycle.Triggers[0].OnFileChange) + require.Equal(t, []diag.Location{location}, b.Config.LocationsAt(path)) } func TestNormalizePath_absolutePath(t *testing.T) { diff --git a/bundle/config/mutator/paths/alert_paths_visitor.go b/bundle/config/mutator/paths/alert_paths_visitor.go index 1f1a010c798..91265736264 100644 --- a/bundle/config/mutator/paths/alert_paths_visitor.go +++ b/bundle/config/mutator/paths/alert_paths_visitor.go @@ -1,18 +1,14 @@ package paths import ( - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) -func VisitAlertPaths(value dyn.Value, fn VisitFunc) (dyn.Value, error) { - pattern := dyn.NewPattern( - dyn.Key("resources"), - dyn.Key("alerts"), - dyn.AnyKey(), - dyn.Key("file_path"), - ) +func VisitAlertPaths(root structvar.View, fn VisitFunc) error { + pattern := structpath.MustParsePattern("resources.alerts.*.file_path") - return dyn.MapByPattern(value, pattern, func(path dyn.Path, value dyn.Value) (dyn.Value, error) { + return structvar.ForEach(root, pattern, func(path *structpath.PathNode, value structvar.View) error { return fn(path, TranslateModeLocalRelative, value) }) } diff --git a/bundle/config/mutator/paths/app_paths_visitor.go b/bundle/config/mutator/paths/app_paths_visitor.go index 21a7e467ae2..25040263f67 100644 --- a/bundle/config/mutator/paths/app_paths_visitor.go +++ b/bundle/config/mutator/paths/app_paths_visitor.go @@ -1,18 +1,14 @@ package paths import ( - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) -func VisitAppPaths(value dyn.Value, fn VisitFunc) (dyn.Value, error) { - pattern := dyn.NewPattern( - dyn.Key("resources"), - dyn.Key("apps"), - dyn.AnyKey(), - dyn.Key("source_code_path"), - ) +func VisitAppPaths(root structvar.View, fn VisitFunc) error { + pattern := structpath.MustParsePattern("resources.apps.*.source_code_path") - return dyn.MapByPattern(value, pattern, func(path dyn.Path, value dyn.Value) (dyn.Value, error) { + return structvar.ForEach(root, pattern, func(path *structpath.PathNode, value structvar.View) error { return fn(path, TranslateModeDirectory, value) }) } diff --git a/bundle/config/mutator/paths/app_paths_visitor_test.go b/bundle/config/mutator/paths/app_paths_visitor_test.go index 9ec01aeaf83..79dc3df22ef 100644 --- a/bundle/config/mutator/paths/app_paths_visitor_test.go +++ b/bundle/config/mutator/paths/app_paths_visitor_test.go @@ -5,7 +5,7 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/resources" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/databricks-sdk-go/service/apps" "github.com/stretchr/testify/assert" ) @@ -24,9 +24,7 @@ func TestAppPathsVisitor(t *testing.T) { } actual := collectVisitedPaths(t, root, VisitAppPaths) - expected := []dyn.Path{ - dyn.MustPathFromString("resources.apps.app0.source_code_path"), - } + expected := structpath.MustParsePaths("resources.apps.app0.source_code_path") assert.ElementsMatch(t, expected, actual) } diff --git a/bundle/config/mutator/paths/artifact_paths_visitor.go b/bundle/config/mutator/paths/artifact_paths_visitor.go index c37ae19fee2..4f934e42ba8 100644 --- a/bundle/config/mutator/paths/artifact_paths_visitor.go +++ b/bundle/config/mutator/paths/artifact_paths_visitor.go @@ -1,42 +1,37 @@ package paths import ( - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) type artifactRewritePattern struct { - pattern dyn.Pattern + pattern *structpath.PatternNode mode TranslateMode } func artifactRewritePatterns() []artifactRewritePattern { // Base pattern to match all artifacts. - base := dyn.NewPattern( - dyn.Key("artifacts"), - dyn.AnyKey(), - ) + base := "artifacts.*" // Compile list of configuration paths to rewrite. return []artifactRewritePattern{ { - pattern: base.Append(dyn.Key("path")), + pattern: structpath.MustParsePattern(base + ".path"), mode: TranslateModeLocalAbsoluteDirectory, }, } } -func VisitArtifactPaths(value dyn.Value, fn VisitFunc) (dyn.Value, error) { - var err error - newValue := value - +func VisitArtifactPaths(root structvar.View, fn VisitFunc) error { for _, rewritePattern := range artifactRewritePatterns() { - newValue, err = dyn.MapByPattern(newValue, rewritePattern.pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { + err := structvar.ForEach(root, rewritePattern.pattern, func(p *structpath.PathNode, v structvar.View) error { return fn(p, rewritePattern.mode, v) }) if err != nil { - return dyn.InvalidValue, err + return err } } - return newValue, nil + return nil } diff --git a/bundle/config/mutator/paths/artifact_paths_visitor_test.go b/bundle/config/mutator/paths/artifact_paths_visitor_test.go index 5d43036b813..9e9cb2b9ce7 100644 --- a/bundle/config/mutator/paths/artifact_paths_visitor_test.go +++ b/bundle/config/mutator/paths/artifact_paths_visitor_test.go @@ -4,7 +4,7 @@ import ( "testing" "github.com/databricks/cli/bundle/config" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" "github.com/stretchr/testify/assert" ) @@ -18,9 +18,7 @@ func TestArtifactPathsVisitor(t *testing.T) { } actual := collectVisitedPaths(t, root, VisitArtifactPaths) - expected := []dyn.Path{ - dyn.MustPathFromString("artifacts.artifact0.path"), - } + expected := structpath.MustParsePaths("artifacts.artifact0.path") assert.ElementsMatch(t, expected, actual) } diff --git a/bundle/config/mutator/paths/dashboard_paths_visitor.go b/bundle/config/mutator/paths/dashboard_paths_visitor.go index b0a25dfe2ff..40c9de427bd 100644 --- a/bundle/config/mutator/paths/dashboard_paths_visitor.go +++ b/bundle/config/mutator/paths/dashboard_paths_visitor.go @@ -1,18 +1,14 @@ package paths import ( - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) -func VisitDashboardPaths(value dyn.Value, fn VisitFunc) (dyn.Value, error) { - pattern := dyn.NewPattern( - dyn.Key("resources"), - dyn.Key("dashboards"), - dyn.AnyKey(), - dyn.Key("file_path"), - ) +func VisitDashboardPaths(root structvar.View, fn VisitFunc) error { + pattern := structpath.MustParsePattern("resources.dashboards.*.file_path") - return dyn.MapByPattern(value, pattern, func(path dyn.Path, value dyn.Value) (dyn.Value, error) { + return structvar.ForEach(root, pattern, func(path *structpath.PathNode, value structvar.View) error { return fn(path, TranslateModeLocalRelative, value) }) } diff --git a/bundle/config/mutator/paths/dashboard_paths_visitor_test.go b/bundle/config/mutator/paths/dashboard_paths_visitor_test.go index 4827e49d3ff..88a0ef4679e 100644 --- a/bundle/config/mutator/paths/dashboard_paths_visitor_test.go +++ b/bundle/config/mutator/paths/dashboard_paths_visitor_test.go @@ -5,7 +5,7 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/resources" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" "github.com/stretchr/testify/assert" ) @@ -21,9 +21,7 @@ func TestVisitDashboardPaths(t *testing.T) { } actual := collectVisitedPaths(t, root, VisitDashboardPaths) - expected := []dyn.Path{ - dyn.MustPathFromString("resources.dashboards.dashboard0.file_path"), - } + expected := structpath.MustParsePaths("resources.dashboards.dashboard0.file_path") assert.ElementsMatch(t, expected, actual) } diff --git a/bundle/config/mutator/paths/genie_space_paths_visitor.go b/bundle/config/mutator/paths/genie_space_paths_visitor.go index edd6ff2d8df..8cc61bc09c4 100644 --- a/bundle/config/mutator/paths/genie_space_paths_visitor.go +++ b/bundle/config/mutator/paths/genie_space_paths_visitor.go @@ -1,18 +1,14 @@ package paths import ( - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) -func VisitGenieSpacePaths(value dyn.Value, fn VisitFunc) (dyn.Value, error) { - pattern := dyn.NewPattern( - dyn.Key("resources"), - dyn.Key("genie_spaces"), - dyn.AnyKey(), - dyn.Key("file_path"), - ) +func VisitGenieSpacePaths(root structvar.View, fn VisitFunc) error { + pattern := structpath.MustParsePattern("resources.genie_spaces.*.file_path") - return dyn.MapByPattern(value, pattern, func(path dyn.Path, value dyn.Value) (dyn.Value, error) { + return structvar.ForEach(root, pattern, func(path *structpath.PathNode, value structvar.View) error { return fn(path, TranslateModeLocalRelative, value) }) } diff --git a/bundle/config/mutator/paths/job_libraries_paths_visitor.go b/bundle/config/mutator/paths/job_libraries_paths_visitor.go index ad56b1933dc..e96e136f814 100644 --- a/bundle/config/mutator/paths/job_libraries_paths_visitor.go +++ b/bundle/config/mutator/paths/job_libraries_paths_visitor.go @@ -2,18 +2,19 @@ package paths import ( "github.com/databricks/cli/bundle/libraries" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) -func jobTaskLibrariesRewritePatterns(base dyn.Pattern) []jobRewritePattern { +func jobTaskLibrariesRewritePatterns(base string) []jobRewritePattern { return []jobRewritePattern{ { - base.Append(dyn.Key("libraries"), dyn.AnyIndex(), dyn.Key("whl")), + structpath.MustParsePattern(base + ".libraries[*].whl"), TranslateModeLocalRelative, noSkipRewrite, }, { - base.Append(dyn.Key("libraries"), dyn.AnyIndex(), dyn.Key("jar")), + structpath.MustParsePattern(base + ".libraries[*].jar"), TranslateModeLocalRelative, noSkipRewrite, }, @@ -22,27 +23,12 @@ func jobTaskLibrariesRewritePatterns(base dyn.Pattern) []jobRewritePattern { func jobLibrariesRewritePatterns() []jobRewritePattern { // Base pattern to match all tasks in all jobs. - base := dyn.NewPattern( - dyn.Key("resources"), - dyn.Key("jobs"), - dyn.AnyKey(), - dyn.Key("tasks"), - dyn.AnyIndex(), - ) + base := "resources.jobs.*.tasks[*]" // Compile list of patterns and their respective rewrite functions. jobEnvironmentsPatterns := []jobRewritePattern{ { - dyn.NewPattern( - dyn.Key("resources"), - dyn.Key("jobs"), - dyn.AnyKey(), - dyn.Key("environments"), - dyn.AnyIndex(), - dyn.Key("spec"), - dyn.Key("dependencies"), - dyn.AnyIndex(), - ), + structpath.MustParsePattern("resources.jobs.*.environments[*].spec.dependencies[*]"), TranslateModeLocalRelativeWithPrefix, func(s string) bool { return !libraries.IsLibraryLocal(s) @@ -52,16 +38,7 @@ func jobLibrariesRewritePatterns() []jobRewritePattern { jobEnvironmentsWithRequirementsPatterns := []jobRewritePattern{ { - dyn.NewPattern( - dyn.Key("resources"), - dyn.Key("jobs"), - dyn.AnyKey(), - dyn.Key("environments"), - dyn.AnyIndex(), - dyn.Key("spec"), - dyn.Key("dependencies"), - dyn.AnyIndex(), - ), + structpath.MustParsePattern("resources.jobs.*.environments[*].spec.dependencies[*]"), TranslateModeEnvironmentPipFlag, func(s string) bool { _, _, ok := libraries.IsLocalPathInPipFlag(s) @@ -71,7 +48,7 @@ func jobLibrariesRewritePatterns() []jobRewritePattern { } taskPatterns := jobTaskLibrariesRewritePatterns(base) - forEachPatterns := jobTaskLibrariesRewritePatterns(base.Append(dyn.Key("for_each_task"), dyn.Key("task"))) + forEachPatterns := jobTaskLibrariesRewritePatterns(base + ".for_each_task.task") allPatterns := append(taskPatterns, jobEnvironmentsPatterns...) allPatterns = append(allPatterns, jobEnvironmentsWithRequirementsPatterns...) allPatterns = append(allPatterns, forEachPatterns...) @@ -79,22 +56,13 @@ func jobLibrariesRewritePatterns() []jobRewritePattern { } // VisitJobLibrariesPaths visits all libraries related paths in job resources and applies a function to each path. -func VisitJobLibrariesPaths(value dyn.Value, fn VisitFunc) (dyn.Value, error) { - var err error - newValue := value - +func VisitJobLibrariesPaths(root structvar.View, fn VisitFunc) error { for _, rewritePattern := range jobLibrariesRewritePatterns() { - newValue, err = dyn.MapByPattern(newValue, rewritePattern.pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - if rewritePattern.skipRewrite(v.MustString()) { - return v, nil - } - - return fn(p, rewritePattern.mode, v) - }) + err := visitString(root, rewritePattern.pattern, rewritePattern.mode, rewritePattern.skipRewrite, fn) if err != nil { - return dyn.InvalidValue, err + return err } } - return newValue, nil + return nil } diff --git a/bundle/config/mutator/paths/job_libraries_paths_visitor_test.go b/bundle/config/mutator/paths/job_libraries_paths_visitor_test.go index 07b3c6d6641..789a2cc72b5 100644 --- a/bundle/config/mutator/paths/job_libraries_paths_visitor_test.go +++ b/bundle/config/mutator/paths/job_libraries_paths_visitor_test.go @@ -5,7 +5,7 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/resources" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/databricks-sdk-go/service/compute" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/stretchr/testify/assert" @@ -71,10 +71,10 @@ func TestVisitJobLibrariesPaths(t *testing.T) { } actual := collectVisitedPaths(t, root, VisitJobLibrariesPaths) - expected := []dyn.Path{ - dyn.MustPathFromString("resources.jobs.job0.tasks[4].libraries[0].whl"), - dyn.MustPathFromString("resources.jobs.job0.tasks[5].libraries[0].jar"), - } + expected := structpath.MustParsePaths( + "resources.jobs.job0.tasks[4].libraries[0].whl", + "resources.jobs.job0.tasks[5].libraries[0].jar", + ) assert.ElementsMatch(t, expected, actual) } @@ -103,10 +103,10 @@ func TestVisitJobLibrariesPaths_environments(t *testing.T) { } actual := collectVisitedPaths(t, root, VisitJobLibrariesPaths) - expected := []dyn.Path{ - dyn.MustPathFromString("resources.jobs.job0.environments[0].spec.dependencies[0]"), - dyn.MustPathFromString("resources.jobs.job0.environments[0].spec.dependencies[1]"), - } + expected := structpath.MustParsePaths( + "resources.jobs.job0.environments[0].spec.dependencies[0]", + "resources.jobs.job0.environments[0].spec.dependencies[1]", + ) assert.ElementsMatch(t, expected, actual) } diff --git a/bundle/config/mutator/paths/job_paths_visitor.go b/bundle/config/mutator/paths/job_paths_visitor.go index a3abd0090a4..1e6244b21ed 100644 --- a/bundle/config/mutator/paths/job_paths_visitor.go +++ b/bundle/config/mutator/paths/job_paths_visitor.go @@ -1,11 +1,12 @@ package paths import ( - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) type jobRewritePattern struct { - pattern dyn.Pattern + pattern *structpath.PatternNode mode TranslateMode skipRewrite func(string) bool } @@ -14,35 +15,35 @@ func noSkipRewrite(string) bool { return false } -func jobTaskRewritePatterns(base dyn.Pattern) []jobRewritePattern { +func jobTaskRewritePatterns(base string) []jobRewritePattern { return []jobRewritePattern{ { - base.Append(dyn.Key("notebook_task"), dyn.Key("notebook_path")), + structpath.MustParsePattern(base + ".notebook_task.notebook_path"), TranslateModeNotebook, noSkipRewrite, }, { - base.Append(dyn.Key("spark_python_task"), dyn.Key("python_file")), + structpath.MustParsePattern(base + ".spark_python_task.python_file"), TranslateModeFile, noSkipRewrite, }, { - base.Append(dyn.Key("dbt_task"), dyn.Key("project_directory")), + structpath.MustParsePattern(base + ".dbt_task.project_directory"), TranslateModeDirectory, noSkipRewrite, }, { - base.Append(dyn.Key("sql_task"), dyn.Key("file"), dyn.Key("path")), + structpath.MustParsePattern(base + ".sql_task.file.path"), TranslateModeFile, noSkipRewrite, }, { - base.Append(dyn.Key("alert_task"), dyn.Key("workspace_path")), + structpath.MustParsePattern(base + ".alert_task.workspace_path"), TranslateModeFile, noSkipRewrite, }, { - base.Append(dyn.Key("libraries"), dyn.AnyIndex(), dyn.Key("requirements")), + structpath.MustParsePattern(base + ".libraries[*].requirements"), TranslateModeFile, noSkipRewrite, }, @@ -50,7 +51,7 @@ func jobTaskRewritePatterns(base dyn.Pattern) []jobRewritePattern { // The AI Runtime task runs this bash script on each node; the backend // reads it as a workspace file, so translate the local path to its // remote (or immutable-snapshot) location like any other file. - base.Append(dyn.Key("ai_runtime_task"), dyn.Key("deployments"), dyn.AnyIndex(), dyn.Key("command_path")), + structpath.MustParsePattern(base + ".ai_runtime_task.deployments[*].command_path"), TranslateModeFile, noSkipRewrite, }, @@ -59,16 +60,10 @@ func jobTaskRewritePatterns(base dyn.Pattern) []jobRewritePattern { func jobRewritePatterns() []jobRewritePattern { // Task-level base pattern for all jobs. - base := dyn.NewPattern( - dyn.Key("resources"), - dyn.Key("jobs"), - dyn.AnyKey(), - dyn.Key("tasks"), - dyn.AnyIndex(), - ) + base := "resources.jobs.*.tasks[*]" taskPatterns := jobTaskRewritePatterns(base) - forEachPatterns := jobTaskRewritePatterns(base.Append(dyn.Key("for_each_task"), dyn.Key("task"))) + forEachPatterns := jobTaskRewritePatterns(base + ".for_each_task.task") patterns := append(taskPatterns, forEachPatterns...) return append(patterns, jobRewritePattern{ @@ -103,22 +98,13 @@ func jobRewritePatterns() []jobRewritePattern { } // VisitJobPaths visits all paths in job resources and applies a function to each path. -func VisitJobPaths(value dyn.Value, fn VisitFunc) (dyn.Value, error) { - var err error - newValue := value - +func VisitJobPaths(root structvar.View, fn VisitFunc) error { for _, rewritePattern := range jobRewritePatterns() { - newValue, err = dyn.MapByPattern(newValue, rewritePattern.pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - if rewritePattern.skipRewrite(v.MustString()) { - return v, nil - } - - return fn(p, rewritePattern.mode, v) - }) + err := visitString(root, rewritePattern.pattern, rewritePattern.mode, rewritePattern.skipRewrite, fn) if err != nil { - return dyn.InvalidValue, err + return err } } - return newValue, nil + return nil } diff --git a/bundle/config/mutator/paths/job_paths_visitor_test.go b/bundle/config/mutator/paths/job_paths_visitor_test.go index b9fe29d154e..23d8b3fa652 100644 --- a/bundle/config/mutator/paths/job_paths_visitor_test.go +++ b/bundle/config/mutator/paths/job_paths_visitor_test.go @@ -5,7 +5,7 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/resources" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/databricks-sdk-go/service/compute" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/stretchr/testify/assert" @@ -85,15 +85,15 @@ func TestVisitJobPaths(t *testing.T) { } actual := collectVisitedPaths(t, root, VisitJobPaths) - expected := []dyn.Path{ - dyn.MustPathFromString("resources.jobs.job0.tasks[0].notebook_task.notebook_path"), - dyn.MustPathFromString("resources.jobs.job0.tasks[1].spark_python_task.python_file"), - dyn.MustPathFromString("resources.jobs.job0.tasks[2].dbt_task.project_directory"), - dyn.MustPathFromString("resources.jobs.job0.tasks[3].sql_task.file.path"), - dyn.MustPathFromString("resources.jobs.job0.tasks[6].libraries[0].requirements"), - dyn.MustPathFromString("resources.jobs.job0.tasks[7].alert_task.workspace_path"), - dyn.MustPathFromString("resources.jobs.job0.tasks[8].ai_runtime_task.deployments[0].command_path"), - } + expected := structpath.MustParsePaths( + "resources.jobs.job0.tasks[0].notebook_task.notebook_path", + "resources.jobs.job0.tasks[1].spark_python_task.python_file", + "resources.jobs.job0.tasks[2].dbt_task.project_directory", + "resources.jobs.job0.tasks[3].sql_task.file.path", + "resources.jobs.job0.tasks[6].libraries[0].requirements", + "resources.jobs.job0.tasks[7].alert_task.workspace_path", + "resources.jobs.job0.tasks[8].ai_runtime_task.deployments[0].command_path", + ) assert.ElementsMatch(t, expected, actual) } @@ -124,7 +124,7 @@ func TestVisitJobPaths_environments(t *testing.T) { } actual := collectVisitedPaths(t, root, VisitJobPaths) - expected := []dyn.Path{ + expected := []*structpath.PathNode{ dyn.MustPathFromString("resources.jobs.job0.environments[1].spec.project_environment"), dyn.MustPathFromString("resources.jobs.job0.environments[2].spec.project_environment"), } @@ -199,10 +199,10 @@ func TestVisitJobPaths_foreach(t *testing.T) { } actual := collectVisitedPaths(t, root, VisitJobPaths) - expected := []dyn.Path{ - dyn.MustPathFromString("resources.jobs.job0.tasks[0].for_each_task.task.notebook_task.notebook_path"), - dyn.MustPathFromString("resources.jobs.job0.tasks[1].for_each_task.task.alert_task.workspace_path"), - } + expected := structpath.MustParsePaths( + "resources.jobs.job0.tasks[0].for_each_task.task.notebook_task.notebook_path", + "resources.jobs.job0.tasks[1].for_each_task.task.alert_task.workspace_path", + ) assert.ElementsMatch(t, expected, actual) } diff --git a/bundle/config/mutator/paths/job_run_paths_visitor.go b/bundle/config/mutator/paths/job_run_paths_visitor.go index 3bc56bacf72..507a5343b44 100644 --- a/bundle/config/mutator/paths/job_run_paths_visitor.go +++ b/bundle/config/mutator/paths/job_run_paths_visitor.go @@ -1,24 +1,17 @@ package paths import ( - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) // VisitJobRunPaths visits local paths on job_runs so NormalizePaths can rewrite // them relative to the bundle root. Not used by TranslatePaths: hashing still // needs a local glob, not a workspace path. -func VisitJobRunPaths(value dyn.Value, fn VisitFunc) (dyn.Value, error) { - pattern := dyn.NewPattern( - dyn.Key("resources"), - dyn.Key("job_runs"), - dyn.AnyKey(), - dyn.Key("lifecycle"), - dyn.Key("triggers"), - dyn.AnyIndex(), - dyn.Key("on_file_change"), - ) +func VisitJobRunPaths(root structvar.View, fn VisitFunc) error { + pattern := structpath.MustParsePattern("resources.job_runs.*.lifecycle.triggers[*].on_file_change") - return dyn.MapByPattern(value, pattern, func(path dyn.Path, value dyn.Value) (dyn.Value, error) { + return structvar.ForEach(root, pattern, func(path *structpath.PathNode, value structvar.View) error { return fn(path, TranslateModeLocalRelative, value) }) } diff --git a/bundle/config/mutator/paths/job_run_paths_visitor_test.go b/bundle/config/mutator/paths/job_run_paths_visitor_test.go index 93df73d3c9e..ac552583a68 100644 --- a/bundle/config/mutator/paths/job_run_paths_visitor_test.go +++ b/bundle/config/mutator/paths/job_run_paths_visitor_test.go @@ -5,7 +5,7 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/resources" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" "github.com/stretchr/testify/assert" ) @@ -27,9 +27,7 @@ func TestVisitJobRunPaths(t *testing.T) { } actual := collectVisitedPaths(t, root, VisitJobRunPaths) - expected := []dyn.Path{ - dyn.MustPathFromString("resources.job_runs.run0.lifecycle.triggers[0].on_file_change"), - } + expected := structpath.MustParsePaths("resources.job_runs.run0.lifecycle.triggers[0].on_file_change") assert.ElementsMatch(t, expected, actual) } diff --git a/bundle/config/mutator/paths/pipeline_paths_visitor.go b/bundle/config/mutator/paths/pipeline_paths_visitor.go index b5af086dd34..ff50c637db8 100644 --- a/bundle/config/mutator/paths/pipeline_paths_visitor.go +++ b/bundle/config/mutator/paths/pipeline_paths_visitor.go @@ -2,11 +2,12 @@ package paths import ( "github.com/databricks/cli/bundle/libraries" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) type pipelineRewritePattern struct { - pattern dyn.Pattern + pattern *structpath.PatternNode mode TranslateMode // If function defined in skipRewrite returns true, we skip rewriting the path. @@ -15,32 +16,28 @@ type pipelineRewritePattern struct { } // Base pattern to match all libraries in all pipelines. -var base = dyn.NewPattern( - dyn.Key("resources"), - dyn.Key("pipelines"), - dyn.AnyKey(), -) +var base = "resources.pipelines.*" func pipelineRewritePatterns() []pipelineRewritePattern { // Compile list of configuration paths to rewrite. allPatterns := []pipelineRewritePattern{ { - pattern: base.Append(dyn.Key("libraries"), dyn.AnyIndex(), dyn.Key("notebook"), dyn.Key("path")), + pattern: structpath.MustParsePattern(base + ".libraries[*].notebook.path"), mode: TranslateModeNotebook, skipRewrite: noSkipRewrite, }, { - pattern: base.Append(dyn.Key("libraries"), dyn.AnyIndex(), dyn.Key("file"), dyn.Key("path")), + pattern: structpath.MustParsePattern(base + ".libraries[*].file.path"), mode: TranslateModeFile, skipRewrite: noSkipRewrite, }, { - pattern: base.Append(dyn.Key("libraries"), dyn.AnyIndex(), dyn.Key("glob"), dyn.Key("include")), + pattern: structpath.MustParsePattern(base + ".libraries[*].glob.include"), mode: TranslateModeGlob, skipRewrite: noSkipRewrite, }, { - pattern: base.Append(dyn.Key("root_path")), + pattern: structpath.MustParsePattern(base + ".root_path"), mode: TranslateModeDirectory, skipRewrite: noSkipRewrite, }, @@ -52,15 +49,8 @@ func pipelineRewritePatterns() []pipelineRewritePattern { func pipelineLibrariesRewritePatterns() []pipelineRewritePattern { pipelineEnvironmentsPatterns := []pipelineRewritePattern{ { - pattern: dyn.NewPattern( - dyn.Key("resources"), - dyn.Key("pipelines"), - dyn.AnyKey(), - dyn.Key("environment"), - dyn.Key("dependencies"), - dyn.AnyIndex(), - ), - mode: TranslateModeLocalRelativeWithPrefix, + pattern: structpath.MustParsePattern("resources.pipelines.*.environment.dependencies[*]"), + mode: TranslateModeLocalRelativeWithPrefix, skipRewrite: func(s string) bool { return !libraries.IsLibraryLocal(s) }, @@ -69,14 +59,7 @@ func pipelineLibrariesRewritePatterns() []pipelineRewritePattern { pipelineEnvironmentsPatternsWithPipFlags := []pipelineRewritePattern{ { - dyn.NewPattern( - dyn.Key("resources"), - dyn.Key("pipelines"), - dyn.AnyKey(), - dyn.Key("environment"), - dyn.Key("dependencies"), - dyn.AnyIndex(), - ), + structpath.MustParsePattern("resources.pipelines.*.environment.dependencies[*]"), TranslateModeEnvironmentPipFlag, func(s string) bool { _, _, ok := libraries.IsLocalPathInPipFlag(s) @@ -88,50 +71,24 @@ func pipelineLibrariesRewritePatterns() []pipelineRewritePattern { return append(pipelineEnvironmentsPatterns, pipelineEnvironmentsPatternsWithPipFlags...) } -func VisitPipelinePaths(value dyn.Value, fn VisitFunc) (dyn.Value, error) { - var err error - newValue := value - +func VisitPipelinePaths(root structvar.View, fn VisitFunc) error { for _, rewritePattern := range pipelineRewritePatterns() { - newValue, err = dyn.MapByPattern(newValue, rewritePattern.pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - sv, ok := v.AsString() - if !ok { - return v, nil - } - if rewritePattern.skipRewrite(sv) { - return v, nil - } - - return fn(p, rewritePattern.mode, v) - }) + err := visitString(root, rewritePattern.pattern, rewritePattern.mode, rewritePattern.skipRewrite, fn) if err != nil { - return dyn.InvalidValue, err + return err } } - return newValue, nil + return nil } -func VisitPipelineLibrariesPaths(value dyn.Value, fn VisitFunc) (dyn.Value, error) { - var err error - newValue := value - +func VisitPipelineLibrariesPaths(root structvar.View, fn VisitFunc) error { for _, rewritePattern := range pipelineLibrariesRewritePatterns() { - newValue, err = dyn.MapByPattern(newValue, rewritePattern.pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - sv, ok := v.AsString() - if !ok { - return v, nil - } - if rewritePattern.skipRewrite(sv) { - return v, nil - } - - return fn(p, rewritePattern.mode, v) - }) + err := visitString(root, rewritePattern.pattern, rewritePattern.mode, rewritePattern.skipRewrite, fn) if err != nil { - return dyn.InvalidValue, err + return err } } - return newValue, nil + return nil } diff --git a/bundle/config/mutator/paths/pipeline_paths_visitor_test.go b/bundle/config/mutator/paths/pipeline_paths_visitor_test.go index eb9748bba37..6a7b2c6183d 100644 --- a/bundle/config/mutator/paths/pipeline_paths_visitor_test.go +++ b/bundle/config/mutator/paths/pipeline_paths_visitor_test.go @@ -5,7 +5,7 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/resources" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/databricks-sdk-go/service/pipelines" "github.com/stretchr/testify/assert" ) @@ -47,13 +47,13 @@ func TestVisitPipelinePaths(t *testing.T) { actual := collectVisitedPaths(t, root, VisitPipelinePaths) actual = append(actual, collectVisitedPaths(t, root, VisitPipelineLibrariesPaths)...) - expected := []dyn.Path{ - dyn.MustPathFromString("resources.pipelines.pipeline0.libraries[0].file.path"), - dyn.MustPathFromString("resources.pipelines.pipeline0.libraries[1].notebook.path"), - dyn.MustPathFromString("resources.pipelines.pipeline0.libraries[2].glob.include"), - dyn.MustPathFromString("resources.pipelines.pipeline0.root_path"), - dyn.MustPathFromString("resources.pipelines.pipeline0.environment.dependencies[0]"), - } + expected := structpath.MustParsePaths( + "resources.pipelines.pipeline0.libraries[0].file.path", + "resources.pipelines.pipeline0.libraries[1].notebook.path", + "resources.pipelines.pipeline0.libraries[2].glob.include", + "resources.pipelines.pipeline0.root_path", + "resources.pipelines.pipeline0.environment.dependencies[0]", + ) assert.ElementsMatch(t, expected, actual) } diff --git a/bundle/config/mutator/paths/visitor.go b/bundle/config/mutator/paths/visitor.go index cadc52b75fe..02662c8b43b 100644 --- a/bundle/config/mutator/paths/visitor.go +++ b/bundle/config/mutator/paths/visitor.go @@ -1,14 +1,17 @@ package paths import ( - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) -type VisitFunc func(path dyn.Path, mode TranslateMode, value dyn.Value) (dyn.Value, error) +// VisitFunc is called for every matching value. It may change the configuration +// at path (e.g. with [config.Root.Set]). +type VisitFunc func(path *structpath.PathNode, mode TranslateMode, value structvar.View) error // VisitPaths visits all paths in bundle configuration -func VisitPaths(root dyn.Value, fn VisitFunc) (dyn.Value, error) { - visitors := []func(dyn.Value, VisitFunc) (dyn.Value, error){ +func VisitPaths(root structvar.View, fn VisitFunc) error { + visitors := []func(structvar.View, VisitFunc) error{ VisitJobPaths, VisitJobRunPaths, VisitJobLibrariesPaths, @@ -21,14 +24,23 @@ func VisitPaths(root dyn.Value, fn VisitFunc) (dyn.Value, error) { VisitPipelineLibrariesPaths, } - newRoot := root for _, visitor := range visitors { - updatedRoot, err := visitor(newRoot, fn) - if err != nil { - return dyn.InvalidValue, err + if err := visitor(root, fn); err != nil { + return err } - newRoot = updatedRoot } - return newRoot, nil + return nil +} + +// visitString calls fn for every string value matching pattern unless skip reports true for it. +// Values that are not strings are not visited. +func visitString(root structvar.View, pattern *structpath.PatternNode, mode TranslateMode, skip func(string) bool, fn VisitFunc) error { + return structvar.ForEach(root, pattern, func(p *structpath.PathNode, v structvar.View) error { + s, ok := v.AsString() + if !ok || skip(s) { + return nil + } + return fn(p, mode, v) + }) } diff --git a/bundle/config/mutator/paths/visitor_test.go b/bundle/config/mutator/paths/visitor_test.go index 019342404c8..80dc112237d 100644 --- a/bundle/config/mutator/paths/visitor_test.go +++ b/bundle/config/mutator/paths/visitor_test.go @@ -4,18 +4,17 @@ import ( "testing" "github.com/databricks/cli/bundle/config" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" "github.com/stretchr/testify/require" ) // collectVisitedPaths is a helper function that collects all visited paths for testing -func collectVisitedPaths(t *testing.T, root config.Root, visitFn func(value dyn.Value, fn VisitFunc) (dyn.Value, error)) []dyn.Path { - var actual []dyn.Path - err := root.Mutate(func(value dyn.Value) (dyn.Value, error) { - return visitFn(value, func(p dyn.Path, mode TranslateMode, v dyn.Value) (dyn.Value, error) { - actual = append(actual, p) - return v, nil - }) +func collectVisitedPaths(t *testing.T, root config.Root, visitFn func(value structvar.View, fn VisitFunc) error) []*structpath.PathNode { + var actual []*structpath.PathNode + err := visitFn(root.View(), func(p *structpath.PathNode, mode TranslateMode, v structvar.View) error { + actual = append(actual, p) + return nil }) require.NoError(t, err) return actual diff --git a/bundle/config/mutator/populate_locations.go b/bundle/config/mutator/populate_locations.go index 8f7a1258b1e..a2a217e0c68 100644 --- a/bundle/config/mutator/populate_locations.go +++ b/bundle/config/mutator/populate_locations.go @@ -2,10 +2,14 @@ package mutator import ( "context" + "fmt" + "strings" "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config/loctable" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn/dynloc" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) type populateLocations struct{} @@ -20,9 +24,53 @@ func (m *populateLocations) Name() string { return "PopulateLocations" } +// locationPatterns are the paths for which locations are recorded. +var locationPatterns = []*structpath.PatternNode{ + structpath.MustParsePattern("*"), // Top level fields + structpath.MustParsePattern("resources.*"), // Resource groups ("resources.jobs") + structpath.MustParsePattern("resources.*.*"), // Resources for all types ("resources.jobs.my_job") + structpath.MustParsePattern("resources.jobs.*.tasks"), // Job tasks ("resources.jobs.my_job.tasks") + structpath.MustParsePattern("resources.jobs.*.tasks[*]"), // Job task items ("resources.jobs.my_job.tasks[2]") +} + +// pathString returns the path with keys separated by dots, whatever characters they contain. +func pathString(p *structpath.PathNode) string { + var b strings.Builder + for i, n := range p.AsSlice() { + if k, ok := n.StringKey(); ok { + if i > 0 { + b.WriteByte('.') + } + b.WriteString(k) + } else if idx, ok := n.Index(); ok { + fmt.Fprintf(&b, "[%d]", idx) + } + } + return b.String() +} + +func gatherLocations(v structvar.View) (map[string][]diag.Location, error) { + locs := map[string][]diag.Location{} + for _, pattern := range locationPatterns { + err := structvar.ForEach(v, pattern, func(p *structpath.PathNode, v structvar.View) error { + locs[pathString(p)] = v.Locations() + return nil + }) + if err != nil { + return nil, err + } + } + return locs, nil +} + func (m *populateLocations) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - locs, err := dynloc.Build( - b.Config.Value(), + pathToLocations, err := gatherLocations(b.Config.View()) + if err != nil { + return diag.FromErr(err) + } + + locs, err := loctable.Build( + pathToLocations, // Make all paths relative to the bundle root. b.BundleRootPath, ) diff --git a/bundle/config/mutator/prepend_workspace_prefix.go b/bundle/config/mutator/prepend_workspace_prefix.go index 3124244c857..05892d50f86 100644 --- a/bundle/config/mutator/prepend_workspace_prefix.go +++ b/bundle/config/mutator/prepend_workspace_prefix.go @@ -7,7 +7,8 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) type prependWorkspacePrefix struct{} @@ -27,45 +28,38 @@ var skipPrefixes = []string{ } func (m *prependWorkspacePrefix) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - patterns := []dyn.Pattern{ - dyn.NewPattern(dyn.Key("workspace"), dyn.Key("root_path")), - dyn.NewPattern(dyn.Key("workspace"), dyn.Key("file_path")), - dyn.NewPattern(dyn.Key("workspace"), dyn.Key("artifact_path")), - dyn.NewPattern(dyn.Key("workspace"), dyn.Key("state_path")), - dyn.NewPattern(dyn.Key("workspace"), dyn.Key("resource_path")), + patterns := []*structpath.PatternNode{ + structpath.MustParsePattern("workspace.root_path"), + structpath.MustParsePattern("workspace.file_path"), + structpath.MustParsePattern("workspace.artifact_path"), + structpath.MustParsePattern("workspace.state_path"), + structpath.MustParsePattern("workspace.resource_path"), } - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { - var err error - for _, pattern := range patterns { - v, err = dyn.MapByPattern(v, pattern, func(p dyn.Path, pv dyn.Value) (dyn.Value, error) { - path, ok := pv.AsString() - if !ok { - return dyn.InvalidValue, fmt.Errorf("expected string, got %s", pv.Kind()) - } + for _, pattern := range patterns { + err := structvar.ForEach(b.Config.View(), pattern, func(p *structpath.PathNode, pv structvar.View) error { + path, ok := pv.AsString() + if !ok { + return fmt.Errorf("expected string, got %s", pv.Kind()) + } - // Skip prefixing if the path does not start with /, it might be variable reference or smth else. - if !strings.HasPrefix(path, "/") { - return pv, nil - } + // Skip prefixing if the path does not start with /, it might be variable reference or smth else. + if !strings.HasPrefix(path, "/") { + return nil + } - for _, prefix := range skipPrefixes { - if strings.HasPrefix(path, prefix) { - return pv, nil - } + for _, prefix := range skipPrefixes { + if strings.HasPrefix(path, prefix) { + return nil } - - // Use pv's locations, not the root v's, so diagnostics point at the original config line. - return dyn.NewValue("/Workspace"+path, pv.Locations()), nil - }) - if err != nil { - return dyn.InvalidValue, err } + + // Set keeps the locations of the value, so diagnostics point at the original config line. + return b.Config.Set(p, "/Workspace"+path) + }) + if err != nil { + return diag.FromErr(err) } - return v, nil - }) - if err != nil { - return diag.FromErr(err) } return nil diff --git a/bundle/config/mutator/python/apply_python_output.go b/bundle/config/mutator/python/apply_python_output.go index fb5452d3b4f..9ed4d7dd56b 100644 --- a/bundle/config/mutator/python/apply_python_output.go +++ b/bundle/config/mutator/python/apply_python_output.go @@ -4,9 +4,8 @@ import ( "fmt" "github.com/databricks/cli/bundle/config/mutator/resourcemutator" - - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/merge" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) // applyPythonOutputResult contains which resources where added, updated, or deleted by Python mutator. @@ -24,162 +23,170 @@ type applyPythonOutputResult struct { // Return value is equivalent to output except for: // - if property is unchanged in output, it's original location will be preserved // - if empty sequence/mapping is deleted in output, it's original value will be preserved -func applyPythonOutput(root, output dyn.Value) (dyn.Value, applyPythonOutputResult, error) { +func applyPythonOutput(root, output structvar.View) (*structvar.OverridePlan, applyPythonOutputResult, error) { result, visitor := createOverrideVisitor(root, output) - merged, err := merge.Override(root, output, visitor) + plan, err := structvar.PlanOverride(root, output, visitor) if err != nil { - return dyn.InvalidValue, result, err + return nil, result, err } - return merged, result, nil + return plan, result, nil +} + +// addResourceKeys adds the keys of the resources in root that match pattern. +func addResourceKeys(set resourcemutator.ResourceKeySet, pattern *structpath.PatternNode, root structvar.View) error { + return structvar.ForEach(root, pattern, func(np *structpath.PathNode, _ structvar.View) error { + set.AddResourceKey(resourcemutator.ResourceKey{Type: np.KeyAt(1), Name: np.KeyAt(2)}) + return nil + }) } -func createOverrideVisitor(leftRoot, rightRoot dyn.Value) (applyPythonOutputResult, merge.OverrideVisitor) { - resourcesPath := dyn.NewPath(dyn.Key("resources")) +func createOverrideVisitor(leftRoot, rightRoot structvar.View) (applyPythonOutputResult, structvar.OverrideVisitor) { + resourcesPath := structpath.MustParsePath("resources") deleted := resourcemutator.NewResourceKeySet() updated := resourcemutator.NewResourceKeySet() added := resourcemutator.NewResourceKeySet() - visitor := merge.OverrideVisitor{ - VisitDelete: func(valuePath dyn.Path, left dyn.Value) error { + visitor := structvar.OverrideVisitor{ + VisitDelete: func(np *structpath.PathNode, left structvar.View) error { if isOmitemptyDelete(left) { - return merge.ErrOverrideUndoDelete + return structvar.ErrOverrideUndoDelete } - if !valuePath.HasPrefix(resourcesPath) { - return fmt.Errorf("unexpected change at %q (delete)", valuePath.String()) + if !np.HasPrefix(resourcesPath) { + return fmt.Errorf("unexpected change at %q (delete)", np.String()) } // use leftRoot below because it contains deleted resources - if len(valuePath) == 1 { + if np.Len() == 1 { // Example: // // valuePath: "resources" // leftRoot: {"bundle": ..., "resources": ...}, // rightRoot: {"bundle": ...} - return deleted.AddPattern( - dyn.NewPatternFromPath(valuePath).Append(dyn.AnyKey(), dyn.AnyKey()), + return addResourceKeys(deleted, + structpath.MustParsePattern("resources.*.*"), leftRoot, ) - } else if len(valuePath) == 2 { + } else if np.Len() == 2 { // Example: // // valuePath: "resources.jobs" // leftRoot: {"resources": { "jobs": ..., "pipeline": ...}}}, // rightRoot: {"resources": { "jobs": ...}}}, - return deleted.AddPattern( - dyn.NewPatternFromPath(valuePath).Append(dyn.AnyKey()), + return addResourceKeys(deleted, + structpath.NewPatternDotStar(structpath.NewPatternStringKey(structpath.MustParsePattern("resources"), np.KeyAt(1))), leftRoot, ) - } else if len(valuePath) == 3 { + } else if np.Len() == 3 { // Example: "resources.jobs.job_0" - resourceKey := resourcemutator.ResourceKey{Type: valuePath[1].Key(), Name: valuePath[2].Key()} + resourceKey := resourcemutator.ResourceKey{Type: np.KeyAt(1), Name: np.KeyAt(2)} deleted.AddResourceKey(resourceKey) return nil } else { // Example: "resources.jobs.job_0.tags" - resourceKey := resourcemutator.ResourceKey{Type: valuePath[1].Key(), Name: valuePath[2].Key()} + resourceKey := resourcemutator.ResourceKey{Type: np.KeyAt(1), Name: np.KeyAt(2)} updated.AddResourceKey(resourceKey) return nil } }, - VisitInsert: func(valuePath dyn.Path, right dyn.Value) (dyn.Value, error) { - if !valuePath.HasPrefix(resourcesPath) { - return dyn.InvalidValue, fmt.Errorf("unexpected change at %q (insert)", valuePath.String()) + VisitInsert: func(np *structpath.PathNode, right structvar.View) error { + if !np.HasPrefix(resourcesPath) { + return fmt.Errorf("unexpected change at %q (insert)", np.String()) } // use rightRoot below because it contains result - if len(valuePath) == 1 { + if np.Len() == 1 { // Example: // // valuePath: "resources" // leftRoot: {"bundle": ..., } // rightRoot: {"bundle": ..., "resources": {...} } - return right, added.AddPattern( - dyn.NewPatternFromPath(valuePath).Append(dyn.AnyKey(), dyn.AnyKey()), + return addResourceKeys(added, + structpath.MustParsePattern("resources.*.*"), rightRoot, ) - } else if len(valuePath) == 2 { + } else if np.Len() == 2 { // Example: // // valuePath: "resources.jobs" // leftRoot: {"resources": { }} // rightRoot: {"resources": { "jobs": {...} }} - return right, added.AddPattern( - dyn.NewPatternFromPath(valuePath).Append(dyn.AnyKey()), + return addResourceKeys(added, + structpath.NewPatternDotStar(structpath.NewPatternStringKey(structpath.MustParsePattern("resources"), np.KeyAt(1))), rightRoot, ) - } else if len(valuePath) == 3 { + } else if np.Len() == 3 { // Example: // // valuePath: "resources.jobs" // leftRoot: {"resources": { "jobs": { }}} // rightRoot: {"resources": { "jobs": {"job_0": {...} }}} - resourceKey := resourcemutator.ResourceKey{Type: valuePath[1].Key(), Name: valuePath[2].Key()} + resourceKey := resourcemutator.ResourceKey{Type: np.KeyAt(1), Name: np.KeyAt(2)} added.AddResourceKey(resourceKey) - return right, nil + return nil } else { // Example: "resources.jobs.job_0.email_notifications" - resourceKey := resourcemutator.ResourceKey{Type: valuePath[1].Key(), Name: valuePath[2].Key()} + resourceKey := resourcemutator.ResourceKey{Type: np.KeyAt(1), Name: np.KeyAt(2)} updated.AddResourceKey(resourceKey) - return right, nil + return nil } }, - VisitUpdate: func(valuePath dyn.Path, _, right dyn.Value) (dyn.Value, error) { - if !valuePath.HasPrefix(resourcesPath) { - return dyn.InvalidValue, fmt.Errorf("unexpected change at %q (update)", valuePath.String()) + VisitUpdate: func(np *structpath.PathNode, _, right structvar.View) error { + if !np.HasPrefix(resourcesPath) { + return fmt.Errorf("unexpected change at %q (update)", np.String()) } // use rightRoot below because it contains result - if len(valuePath) == 1 { + if np.Len() == 1 { // Example: // // valuePath: "resources" // leftRoot: {"bundle": ..., "resources": null } // rightRoot: {"bundle": ..., "resources": {...} } - return right, added.AddPattern( - dyn.NewPatternFromPath(valuePath).Append(dyn.AnyKey(), dyn.AnyKey()), + return addResourceKeys(added, + structpath.MustParsePattern("resources.*.*"), rightRoot, ) - } else if len(valuePath) == 2 { + } else if np.Len() == 2 { // Example: // // valuePath: "resources.jobs" // leftRoot: {"resources": { "jobs": null }} // rightRoot: {"resources": { "jobs": {...} }} - return right, added.AddPattern( - dyn.NewPatternFromPath(valuePath).Append(dyn.AnyKey()), + return addResourceKeys(added, + structpath.NewPatternDotStar(structpath.NewPatternStringKey(structpath.MustParsePattern("resources"), np.KeyAt(1))), rightRoot, ) - } else if len(valuePath) == 3 { + } else if np.Len() == 3 { // Example: // // valuePath: "resources.jobs.job_0" // leftRoot: {"resources": { "jobs": {"job_0": null }}} // rightRoot: {"resources": { "jobs": {"job_0": {...} }}} - resourceKey := resourcemutator.ResourceKey{Type: valuePath[1].Key(), Name: valuePath[2].Key()} + resourceKey := resourcemutator.ResourceKey{Type: np.KeyAt(1), Name: np.KeyAt(2)} added.AddResourceKey(resourceKey) - return right, nil + return nil } else { // Example: "resources.jobs.job_0.name" - resourceKey := resourcemutator.ResourceKey{Type: valuePath[1].Key(), Name: valuePath[2].Key()} + resourceKey := resourcemutator.ResourceKey{Type: np.KeyAt(1), Name: np.KeyAt(2)} updated.AddResourceKey(resourceKey) - return right, nil + return nil } }, } @@ -191,19 +198,25 @@ func createOverrideVisitor(leftRoot, rightRoot dyn.Value) (applyPythonOutputResu }, visitor } -func isOmitemptyDelete(left dyn.Value) bool { +func isOmitemptyDelete(left structvar.View) bool { // Python output can omit empty sequences/mappings, because we don't track them as optional, // there is no semantic difference between empty and missing, so we keep them as they were before // Python mutator deleted them. switch left.Kind() { - case dyn.KindMap: - return left.MustMap().Len() == 0 + case structvar.KindMap: + for range left.MapItems() { + return false + } + return true - case dyn.KindSequence: - return len(left.MustSequence()) == 0 + case structvar.KindSequence: + for range left.Sequence() { + return false + } + return true - case dyn.KindNil: + case structvar.KindNil: // map/sequence can be nil, for instance, bad YAML like: `foo:` return true diff --git a/bundle/config/mutator/python/apply_python_output_test.go b/bundle/config/mutator/python/apply_python_output_test.go index 94d7e7c39cc..fcba01a24ae 100644 --- a/bundle/config/mutator/python/apply_python_output_test.go +++ b/bundle/config/mutator/python/apply_python_output_test.go @@ -1,22 +1,24 @@ package python import ( + "encoding/json" "testing" + "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/mutator/resourcemutator" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn/merge" - - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) type applyPythonOutputTestCase struct { name string - input dyn.Value - output dyn.Value + input any + output any added []resourcemutator.ResourceKey updated []resourcemutator.ResourceKey @@ -24,8 +26,8 @@ type applyPythonOutputTestCase struct { } func TestApplyPythonOutput(t *testing.T) { - job1 := mapOf("name", dyn.V("job 1")) - job2 := mapOf("name", dyn.V("job 2")) + job1 := mapOf("name", "job 1") + job2 := mapOf("name", "job 2") testCases := []applyPythonOutputTestCase{ { @@ -130,10 +132,10 @@ func TestApplyPythonOutput(t *testing.T) { { name: "update job through 'description' insert", input: mapOf("resources", mapOf("jobs", - mapOf("job_1", mapOf("name", dyn.V("name"))), + mapOf("job_1", mapOf("name", "name")), )), output: mapOf("resources", mapOf("jobs", - mapOf("job_1", mapOf2("name", dyn.V("name"), "description", dyn.V("description"))), + mapOf("job_1", mapOf2("name", "name", "description", "description")), )), updated: []resourcemutator.ResourceKey{ { @@ -146,10 +148,13 @@ func TestApplyPythonOutput(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - merged, state, err := applyPythonOutput(tc.input, tc.output) + input := loadRoot(t, tc.input) + output := loadRoot(t, tc.output) + plan, state, err := applyPythonOutput(input.View(), output.View()) + require.NoError(t, err) - assert.NoError(t, err) - assert.Equal(t, tc.output, merged) + require.NoError(t, input.Override(plan)) + assert.Equal(t, output.View().AsAny(), input.View().AsAny()) assert.ElementsMatch(t, tc.added, state.AddedResources.ToArray()) assert.ElementsMatch(t, tc.updated, state.UpdatedResources.ToArray()) @@ -159,36 +164,45 @@ func TestApplyPythonOutput(t *testing.T) { } func TestMergeOutput_disallowDelete(t *testing.T) { - input := mapOf("not_resource", dyn.V("value")) - output := emptyMap() + input := loadRoot(t, mapOf("bundle", mapOf("name", "value"))) + output := loadRoot(t, emptyMap()) - _, _, err := applyPythonOutput(input, output) + _, _, err := applyPythonOutput(input.View(), output.View()) - assert.EqualError(t, err, `unexpected change at "not_resource" (delete)`) + assert.EqualError(t, err, `unexpected change at "bundle" (delete)`) } func TestMergeOutput_disallowInsert(t *testing.T) { - output := mapOf("not_resource", dyn.V("value")) - input := emptyMap() + output := loadRoot(t, mapOf("bundle", mapOf("name", "value"))) + input := loadRoot(t, emptyMap()) - _, _, err := applyPythonOutput(input, output) + _, _, err := applyPythonOutput(input.View(), output.View()) - assert.EqualError(t, err, `unexpected change at "not_resource" (insert)`) + assert.EqualError(t, err, `unexpected change at "bundle" (insert)`) } func TestMergeOutput_disallowUpdate(t *testing.T) { - output := mapOf("not_resource", dyn.V("value")) - input := mapOf("not_resource", dyn.V("new value")) + output := loadRoot(t, mapOf("bundle", mapOf("name", "value"))) + input := loadRoot(t, mapOf("bundle", mapOf("name", "new value"))) + + _, _, err := applyPythonOutput(input.View(), output.View()) - _, _, err := applyPythonOutput(input, output) + assert.EqualError(t, err, `unexpected change at "bundle.name" (update)`) +} - assert.EqualError(t, err, `unexpected change at "not_resource" (update)`) +// loadRoot returns the configuration for the configuration tree v. +func loadRoot(t *testing.T, v any) *config.Root { + raw, err := json.Marshal(v) + require.NoError(t, err) + r, diags := config.LoadFromBytes("output.json", raw) + require.NoError(t, diags.Error()) + return r } type overrideVisitorOmitemptyTestCase struct { name string - path dyn.Path - left dyn.Value + path *structpath.PathNode + left any expectedErr error } @@ -197,76 +211,70 @@ func TestCreateOverrideVisitor_omitempty(t *testing.T) { // there is no semantic difference between empty and missing, so we keep them as they were before // Python code deleted them. - location := diag.Location{ - File: "databricks.yml", - Line: 10, - Column: 20, - } - testCases := []overrideVisitorOmitemptyTestCase{ { name: "undo delete of empty variables", - path: dyn.MustPathFromString("variables"), - left: dyn.NewValue([]dyn.Value{}, []diag.Location{location}), - expectedErr: merge.ErrOverrideUndoDelete, + path: structpath.MustParsePath("variables"), + left: &[]string{}, + expectedErr: structvar.ErrOverrideUndoDelete, }, { name: "undo delete of empty job clusters", - path: dyn.MustPathFromString("resources.jobs.job0.job_clusters"), - left: dyn.NewValue([]dyn.Value{}, []diag.Location{location}), - expectedErr: merge.ErrOverrideUndoDelete, + path: structpath.MustParsePath("resources.jobs.job0.job_clusters"), + left: &[]string{}, + expectedErr: structvar.ErrOverrideUndoDelete, }, { name: "allow delete of non-empty job clusters", - path: dyn.MustPathFromString("resources.jobs.job0.job_clusters"), - left: dyn.NewValue([]dyn.Value{dyn.NewValue("abc", []diag.Location{location})}, []diag.Location{location}), + path: structpath.MustParsePath("resources.jobs.job0.job_clusters"), + left: &[]string{"abc"}, expectedErr: nil, }, { name: "undo delete of empty tags", - path: dyn.MustPathFromString("resources.jobs.job0.tags"), - left: dyn.NewValue(map[string]dyn.Value{}, []diag.Location{location}), - expectedErr: merge.ErrOverrideUndoDelete, + path: structpath.MustParsePath("resources.jobs.job0.tags"), + left: &map[string]string{}, + expectedErr: structvar.ErrOverrideUndoDelete, }, { name: "allow delete of non-empty tags", - path: dyn.MustPathFromString("resources.jobs.job0.tags"), - left: dyn.NewValue(map[string]dyn.Value{"dev": dyn.NewValue("true", []diag.Location{location})}, []diag.Location{location}), + path: structpath.MustParsePath("resources.jobs.job0.tags"), + left: &map[string]string{"dev": "true"}, expectedErr: nil, }, { name: "undo delete of nil", - path: dyn.MustPathFromString("resources.jobs.job0.tags"), - left: dyn.NilValue.WithLocations([]diag.Location{location}), - expectedErr: merge.ErrOverrideUndoDelete, + path: structpath.MustParsePath("resources.jobs.job0.tags"), + left: (*map[string]string)(nil), + expectedErr: structvar.ErrOverrideUndoDelete, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - _, visitor := createOverrideVisitor(dyn.NilValue, dyn.NilValue) + _, visitor := createOverrideVisitor(structvar.View{}, structvar.View{}) - err := visitor.VisitDelete(tc.path, tc.left) + err := visitor.VisitDelete(tc.path, structvar.NewView(tc.left, nil, nil)) assert.Equal(t, tc.expectedErr, err) }) } } -func mapOf(key string, value dyn.Value) dyn.Value { - return dyn.V(map[string]dyn.Value{ +func mapOf(key string, value any) any { + return map[string]any{ key: value, - }) + } } -func mapOf2(key1 string, value1 dyn.Value, key2 string, value2 dyn.Value) dyn.Value { - return dyn.V(map[string]dyn.Value{ +func mapOf2(key1 string, value1 any, key2 string, value2 any) any { + return map[string]any{ key1: value1, key2: value2, - }) + } } -func emptyMap() dyn.Value { - return dyn.V(map[string]dyn.Value{}) +func emptyMap() any { + return map[string]any{} } diff --git a/bundle/config/mutator/python/python_diagnostics.go b/bundle/config/mutator/python/python_diagnostics.go index e47f6fb89ae..0ec85910409 100644 --- a/bundle/config/mutator/python/python_diagnostics.go +++ b/bundle/config/mutator/python/python_diagnostics.go @@ -6,7 +6,6 @@ import ( "io" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/structs/structpath" ) @@ -58,7 +57,7 @@ func parsePythonDiagnostics(input io.Reader) (diag.Diagnostics, error) { } var paths []*structpath.PathNode if path != nil { - paths = dyn.ToStructPaths(path) + paths = []*structpath.PathNode{path} } var locations []diag.Location @@ -81,12 +80,12 @@ func parsePythonDiagnostics(input io.Reader) (diag.Diagnostics, error) { return diags, nil } -func convertPythonPath(path string) (dyn.Path, error) { +func convertPythonPath(path string) (*structpath.PathNode, error) { if path == "" { return nil, nil } - return dyn.NewPathFromString(path) + return structpath.ParsePath(path) } func convertPythonSeverity(severity pythonSeverity) (diag.Severity, error) { diff --git a/bundle/config/mutator/python/python_locations.go b/bundle/config/mutator/python/python_locations.go index 5f85edfc488..430f0f716e9 100644 --- a/bundle/config/mutator/python/python_locations.go +++ b/bundle/config/mutator/python/python_locations.go @@ -8,7 +8,8 @@ import ( "path/filepath" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) // generatedFileName is used as the virtual file name for YAML generated by Python code. @@ -19,7 +20,7 @@ const generatedFileName = "__generated_by_python__.yml" // pythonLocations is data structure for efficient location lookup for a given path // -// Locations form a tree, and we assign locations of the closest ancestor to each dyn.Value based on its path. +// Locations form a tree, and we assign locations of the closest ancestor to each value based on its path. // We implement it as a trie (prefix tree) where keys are components of the path. With that, lookups are O(n) // where n is the number of components in the path. // @@ -57,15 +58,16 @@ type pythonLocationEntry struct { Column int `json:"column"` } -// mergePythonLocations applies locations from Python mutator into given dyn.Value +// mergePythonLocations returns a location mapper that applies locations from the +// Python mutator. // // The primary use-case is to merge locations.json with output.json, so that any // validation errors will point to Python source code instead of generated YAML. -func mergePythonLocations(value dyn.Value, locations *pythonLocations) (dyn.Value, error) { - return dyn.Walk(value, func(path dyn.Path, value dyn.Value) (dyn.Value, error) { +func mergePythonLocations(locations *pythonLocations) structvar.LocationMapper { + return func(path *structpath.PathNode, locs []diag.Location) []diag.Location { newLocation, ok := findPythonLocation(locations, path) if !ok { - return value, nil + return locs } // The first item in the list is the "last" location used for error reporting @@ -75,13 +77,8 @@ func mergePythonLocations(value dyn.Value, locations *pythonLocations) (dyn.Valu // // We preserve any previous non-virtual locations in case when Python function modified // resource defined in YAML. - newLocations := append( - []diag.Location{newLocation}, - removeVirtualLocations(value.Locations())..., - ) - - return value.WithLocations(newLocations), nil - }) + return append([]diag.Location{newLocation}, removeVirtualLocations(locs)...) + } } func removeVirtualLocations(locations []diag.Location) []diag.Location { @@ -113,7 +110,7 @@ func parsePythonLocations(bundleRoot string, input io.Reader) (*pythonLocations, return nil, fmt.Errorf("failed to parse python location: %s", err) } - path, err := dyn.NewPathFromString(entry.Path) + path, err := structpath.ParsePath(entry.Path) if err != nil { return nil, fmt.Errorf("failed to parse python location: %s", err) } @@ -137,18 +134,18 @@ func parsePythonLocations(bundleRoot string, input io.Reader) (*pythonLocations, } // putPythonLocation puts the location to the trie for the given path -func putPythonLocation(trie *pythonLocations, path dyn.Path, location diag.Location) { +func putPythonLocation(trie *pythonLocations, path *structpath.PathNode, location diag.Location) { currentNode := trie - for _, component := range path { - if key := component.Key(); key != "" { + for _, component := range path.AsSlice() { + if key, ok := component.StringKey(); ok { if _, ok := currentNode.keys[key]; !ok { currentNode.keys[key] = newPythonLocations() } currentNode = currentNode.keys[key] } else { - index := component.Index() + index, _ := component.Index() if _, ok := currentNode.indexes[index]; !ok { currentNode.indexes[index] = newPythonLocations() } @@ -171,20 +168,20 @@ func newPythonLocations() *pythonLocations { // findPythonLocation finds the location or closest ancestor location in the trie for the given path // if no ancestor or exact location is found, false is returned. -func findPythonLocation(locations *pythonLocations, path dyn.Path) (diag.Location, bool) { +func findPythonLocation(locations *pythonLocations, path *structpath.PathNode) (diag.Location, bool) { currentNode := locations lastLocation := locations.location exists := locations.exists - for _, component := range path { - if key := component.Key(); key != "" { + for _, component := range path.AsSlice() { + if key, ok := component.StringKey(); ok { if _, ok := currentNode.keys[key]; !ok { break } currentNode = currentNode.keys[key] } else { - index := component.Index() + index, _ := component.Index() if _, ok := currentNode.indexes[index]; !ok { break } diff --git a/bundle/config/mutator/python/python_locations_test.go b/bundle/config/mutator/python/python_locations_test.go index 2bc815e7cdc..f8326c18578 100644 --- a/bundle/config/mutator/python/python_locations_test.go +++ b/bundle/config/mutator/python/python_locations_test.go @@ -6,8 +6,7 @@ import ( "testing" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/dynassert" + "github.com/databricks/cli/libs/structs/structpath" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -18,42 +17,25 @@ func TestMergeLocations(t *testing.T) { yamlLocation := diag.Location{File: "foo.yml", Line: 1, Column: 1} locations := newPythonLocations() - putPythonLocation(locations, dyn.MustPathFromString("foo"), pythonLocation) - - input := dyn.NewValue( - map[string]dyn.Value{ - "foo": dyn.V( - map[string]dyn.Value{ - "baz": dyn.NewValue("baz", []diag.Location{yamlLocation}), - "qux": dyn.NewValue("baz", []diag.Location{generatedLocation, yamlLocation}), - }, - ), - "bar": dyn.NewValue("baz", []diag.Location{generatedLocation}), - }, - []diag.Location{yamlLocation}, - ) - - expected := dyn.NewValue( - map[string]dyn.Value{ - "foo": dyn.NewValue( - map[string]dyn.Value{ - // pythonLocation is appended to the beginning of the list if absent - "baz": dyn.NewValue("baz", []diag.Location{pythonLocation, yamlLocation}), - // generatedLocation is replaced by pythonLocation - "qux": dyn.NewValue("baz", []diag.Location{pythonLocation, yamlLocation}), - }, - []diag.Location{pythonLocation}, - ), - // if location is unknown, we keep it as-is - "bar": dyn.NewValue("baz", []diag.Location{generatedLocation}), - }, - []diag.Location{yamlLocation}, - ) - - actual, err := mergePythonLocations(input, locations) + putPythonLocation(locations, structpath.MustParsePath("foo"), pythonLocation) - assert.NoError(t, err) - dynassert.Equal(t, expected, actual) + mapper := mergePythonLocations(locations) + at := func(path string, locs ...diag.Location) []diag.Location { + var p *structpath.PathNode + if path != "" { + p = structpath.MustParsePath(path) + } + return mapper(p, locs) + } + + // pythonLocation is prepended if absent + assert.Equal(t, []diag.Location{pythonLocation, yamlLocation}, at("foo.baz", yamlLocation)) + // generatedLocation is replaced by pythonLocation + assert.Equal(t, []diag.Location{pythonLocation, yamlLocation}, at("foo.qux", generatedLocation, yamlLocation)) + assert.Equal(t, []diag.Location{pythonLocation}, at("foo")) + // if location is unknown, we keep it as-is + assert.Equal(t, []diag.Location{generatedLocation}, at("bar", generatedLocation)) + assert.Equal(t, []diag.Location{yamlLocation}, at("", yamlLocation)) } func TestFindLocation(t *testing.T) { @@ -61,10 +43,10 @@ func TestFindLocation(t *testing.T) { location1 := diag.Location{File: "foo.py", Line: 2, Column: 1} locations := newPythonLocations() - putPythonLocation(locations, dyn.MustPathFromString("foo"), location0) - putPythonLocation(locations, dyn.MustPathFromString("foo.bar"), location1) + putPythonLocation(locations, structpath.MustParsePath("foo"), location0) + putPythonLocation(locations, structpath.MustParsePath("foo.bar"), location1) - actual, exists := findPythonLocation(locations, dyn.MustPathFromString("foo.bar")) + actual, exists := findPythonLocation(locations, structpath.MustParsePath("foo.bar")) assert.True(t, exists) assert.Equal(t, location1, actual) @@ -76,11 +58,11 @@ func TestFindLocation_indexPathComponent(t *testing.T) { location2 := diag.Location{File: "foo.py", Line: 3, Column: 1} locations := newPythonLocations() - putPythonLocation(locations, dyn.MustPathFromString("foo"), location0) - putPythonLocation(locations, dyn.MustPathFromString("foo.bar"), location1) - putPythonLocation(locations, dyn.MustPathFromString("foo.bar[0]"), location2) + putPythonLocation(locations, structpath.MustParsePath("foo"), location0) + putPythonLocation(locations, structpath.MustParsePath("foo.bar"), location1) + putPythonLocation(locations, structpath.MustParsePath("foo.bar[0]"), location2) - actual, exists := findPythonLocation(locations, dyn.MustPathFromString("foo.bar[0]")) + actual, exists := findPythonLocation(locations, structpath.MustParsePath("foo.bar[0]")) assert.True(t, exists) assert.Equal(t, location2, actual) @@ -91,10 +73,10 @@ func TestFindLocation_closestAncestorLocation(t *testing.T) { location1 := diag.Location{File: "foo.py", Line: 2, Column: 1} locations := newPythonLocations() - putPythonLocation(locations, dyn.MustPathFromString("foo"), location0) - putPythonLocation(locations, dyn.MustPathFromString("foo.bar"), location1) + putPythonLocation(locations, structpath.MustParsePath("foo"), location0) + putPythonLocation(locations, structpath.MustParsePath("foo.bar"), location1) - actual, exists := findPythonLocation(locations, dyn.MustPathFromString("foo.bar.baz")) + actual, exists := findPythonLocation(locations, structpath.MustParsePath("foo.bar.baz")) assert.True(t, exists) assert.Equal(t, location1, actual) @@ -105,10 +87,10 @@ func TestFindLocation_unknownLocation(t *testing.T) { location1 := diag.Location{File: "foo.py", Line: 2, Column: 1} locations := newPythonLocations() - putPythonLocation(locations, dyn.MustPathFromString("foo"), location0) - putPythonLocation(locations, dyn.MustPathFromString("foo.bar"), location1) + putPythonLocation(locations, structpath.MustParsePath("foo"), location0) + putPythonLocation(locations, structpath.MustParsePath("foo.bar"), location1) - _, exists := findPythonLocation(locations, dyn.MustPathFromString("bar")) + _, exists := findPythonLocation(locations, structpath.MustParsePath("bar")) assert.False(t, exists) } @@ -137,21 +119,18 @@ func TestLoadOutput(t *testing.T) { locations := newPythonLocations() putPythonLocation( locations, - dyn.MustPathFromString("resources.jobs.my_job"), + structpath.MustParsePath("resources.jobs.my_job"), location, ) - value, diags := loadOutput( + root, diags := loadOutput( bundleRoot, bytes.NewReader([]byte(output)), locations, ) assert.Equal(t, diag.Diagnostics{}, diags) - - name, err := dyn.Get(value, "resources.jobs.my_job.name") - require.NoError(t, err) - require.Equal(t, []diag.Location{location}, name.Locations()) + require.Equal(t, []diag.Location{location}, root.LocationsAt(structpath.MustParsePath("resources.jobs.my_job.name"))) } func TestParsePythonLocations_absolutePath(t *testing.T) { diff --git a/bundle/config/mutator/python/python_mutator.go b/bundle/config/mutator/python/python_mutator.go index 90d7f9ff287..f05fee37d7b 100644 --- a/bundle/config/mutator/python/python_mutator.go +++ b/bundle/config/mutator/python/python_mutator.go @@ -28,10 +28,9 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/convert" - "github.com/databricks/cli/libs/dyn/yamlloader" "github.com/databricks/cli/libs/process" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) type phase string @@ -160,42 +159,25 @@ func getOpts(b *bundle.Bundle, phase phase) (opts, error) { // Later version of Python package should reject 'experimental.python' unless 'python' section // is set to equivalent value, and after that reject 'experimental.python' altogether. func applyBackwardsCompatibilityFixes(b *bundle.Bundle) error { - return b.Config.Mutate(func(value dyn.Value) (dyn.Value, error) { - outValue := value + root := b.Config.View() - pythonValue, _ := dyn.Get(outValue, "python") - if !pythonValue.IsValid() { - // if 'python' section doesn't exist, nothing to do - return value, nil - } - - experimentalPythonValue, _ := dyn.Get(outValue, "experimental.python") - - if experimentalPythonValue.IsValid() { - // if 'experimental.python' section exists, nothing to do - return value, nil - } - - experimentalValue, _ := dyn.Get(outValue, "experimental") - if !experimentalValue.IsValid() { - updated, err := dyn.Set(outValue, "experimental", dyn.NewValue(map[string]dyn.Value{}, nil)) - if err != nil { - return dyn.InvalidValue, fmt.Errorf("failed to create 'experimental' section: %w", err) - } else { - outValue = updated - } - } + // if 'python' section doesn't exist, nothing to do + pythonValue := root.Get("python") + if !pythonValue.IsValid() { + return nil + } - // move 'python' section to 'experimental.python' - updated, err := dyn.Set(outValue, "experimental.python", pythonValue) - if err != nil { - return dyn.InvalidValue, fmt.Errorf("failed to set 'experimental.python' section: %w", err) - } else { - outValue = updated - } + // if 'experimental.python' section exists, nothing to do + experimentalPythonPath := structpath.MustParsePath("experimental.python") + if root.Lookup(experimentalPythonPath).IsValid() { + return nil + } - return outValue, nil - }) + // move 'python' section to 'experimental.python' (creating 'experimental' if needed) + if err := b.Config.Assign(experimentalPythonPath, pythonValue); err != nil { + return fmt.Errorf("failed to set 'experimental.python' section: %w", err) + } + return nil } func (m *pythonMutator) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { @@ -227,93 +209,82 @@ func (m *pythonMutator) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagno return diag.FromErr(err) } - // mutateDiags is used because Mutate returns 'error' instead of 'diag.Diagnostics' - var mutateDiags diag.Diagnostics - var result applyPythonOutputResult - mutateDiagsHasError := errors.New("unexpected error") - - err = b.Config.Mutate(func(leftRoot dyn.Value) (dyn.Value, error) { - pythonPath, err := detectExecutable(ctx, opts.venvPath) - if err != nil { - return dyn.InvalidValue, fmt.Errorf("failed to get Python interpreter path: %w", err) - } + result, diags := m.run(ctx, b, opts, authEnv) + // we can precisely track resources that are added/updated, so sum doesn't double-count + b.Metrics.PythonUpdatedResourcesCount += int64(result.UpdatedResources.Size()) + b.Metrics.PythonAddedResourcesCount += int64(result.AddedResources.Size()) - cacheDir, cleanup, err := createCacheDir(ctx) - if err != nil { - return dyn.InvalidValue, fmt.Errorf("failed to create cache dir: %w", err) - } - defer cleanup() - - rightRoot, diags := m.runPythonMutator(ctx, leftRoot, runPythonMutatorOpts{ - cacheDir: cacheDir, - bundleRootPath: b.BundleRootPath, - pythonPath: pythonPath, - loadLocations: opts.loadLocations, - authEnv: authEnv, - }) - mutateDiags = diags - if diags.HasError() { - return dyn.InvalidValue, mutateDiagsHasError - } + if diags.HasError() { + return diags + } - newRoot, result0, err := applyPythonOutput(leftRoot, rightRoot) - result = result0 - if err != nil { - return dyn.InvalidValue, fmt.Errorf("internal error when merging output of Python mutator: %w", err) - } + resourcemutator.NormalizeAndInitializeResources(ctx, b, result.AddedResources) + if logdiag.HasError(ctx) { + return diags + } - for _, resourceKey := range result.AddedResources.ToArray() { - log.Debugf(ctx, "added resource at 'resources.%s.%s'", resourceKey.Type, resourceKey.Name) - } + resourcemutator.NormalizeResources(ctx, b, result.UpdatedResources) + return diags +} - for _, resourceKey := range result.UpdatedResources.ToArray() { - log.Debugf(ctx, "updated resource at 'resources.%s.%s'", resourceKey.Type, resourceKey.Name) - } +// run runs the Python mutator and replaces the configuration with its output. +// The configuration is left unchanged if there are errors. +func (m *pythonMutator) run(ctx context.Context, b *bundle.Bundle, opts opts, authEnv map[string]string) (applyPythonOutputResult, diag.Diagnostics) { + var result applyPythonOutputResult - for _, resourceKey := range result.DeletedResources.ToArray() { - log.Debugf(ctx, "deleted resource at 'resources.%s.%s'", resourceKey.Type, resourceKey.Name) - } + pythonPath, err := detectExecutable(ctx, opts.venvPath) + if err != nil { + return result, diag.FromErr(fmt.Errorf("failed to get Python interpreter path: %w", err)) + } - if !result.DeletedResources.IsEmpty() { - return dyn.InvalidValue, fmt.Errorf("unexpected deleted resources: %s", result.DeletedResources.ToArray()) - } + cacheDir, cleanup, err := createCacheDir(ctx) + if err != nil { + return result, diag.FromErr(fmt.Errorf("failed to create cache dir: %w", err)) + } + defer cleanup() - if !result.AddedResources.IsEmpty() && m.phase == PythonMutatorPhaseApplyMutators { - return dyn.InvalidValue, fmt.Errorf("unexpected added resources: %s", result.AddedResources.ToArray()) - } + leftRoot := b.Config.View() + rightRoot, diags := m.runPythonMutator(ctx, leftRoot, runPythonMutatorOpts{ + cacheDir: cacheDir, + bundleRootPath: b.BundleRootPath, + pythonPath: pythonPath, + loadLocations: opts.loadLocations, + authEnv: authEnv, + }) + if diags.HasError() { + return result, diags + } - if !result.UpdatedResources.IsEmpty() && m.phase == PythonMutatorPhaseLoadResources { - return dyn.InvalidValue, fmt.Errorf("unexpected updated resources: %s", result.UpdatedResources.ToArray()) - } + plan, result, err := applyPythonOutput(leftRoot, rightRoot.View()) + if err != nil { + return result, diags.Extend(diag.FromErr(fmt.Errorf("internal error when merging output of Python mutator: %w", err))) + } - return newRoot, nil - }) + for _, resourceKey := range result.AddedResources.ToArray() { + log.Debugf(ctx, "added resource at 'resources.%s.%s'", resourceKey.Type, resourceKey.Name) + } - // we can precisely track resources that are added/updated, so sum doesn't double-count - b.Metrics.PythonUpdatedResourcesCount += int64(result.UpdatedResources.Size()) - b.Metrics.PythonAddedResourcesCount += int64(result.AddedResources.Size()) + for _, resourceKey := range result.UpdatedResources.ToArray() { + log.Debugf(ctx, "updated resource at 'resources.%s.%s'", resourceKey.Type, resourceKey.Name) + } - if err == mutateDiagsHasError { - if !mutateDiags.HasError() { - panic("mutateDiags has no error, but error is expected") - } + for _, resourceKey := range result.DeletedResources.ToArray() { + log.Debugf(ctx, "deleted resource at 'resources.%s.%s'", resourceKey.Type, resourceKey.Name) + } - return mutateDiags - } else { - mutateDiags = mutateDiags.Extend(diag.FromErr(err)) + if !result.DeletedResources.IsEmpty() { + return result, diags.Extend(diag.Errorf("unexpected deleted resources: %s", result.DeletedResources.ToArray())) } - if mutateDiags.HasError() { - return mutateDiags + if !result.AddedResources.IsEmpty() && m.phase == PythonMutatorPhaseApplyMutators { + return result, diags.Extend(diag.Errorf("unexpected added resources: %s", result.AddedResources.ToArray())) } - resourcemutator.NormalizeAndInitializeResources(ctx, b, result.AddedResources) - if logdiag.HasError(ctx) { - return mutateDiags + if !result.UpdatedResources.IsEmpty() && m.phase == PythonMutatorPhaseLoadResources { + return result, diags.Extend(diag.Errorf("unexpected updated resources: %s", result.UpdatedResources.ToArray())) } - resourcemutator.NormalizeResources(ctx, b, result.UpdatedResources) - return mutateDiags + return result, diags.Extend(diag.FromErr(b.Config.Override(plan))) } // createCacheDir returns the directory for input/output files of the Python subprocess, and a cleanup function. @@ -343,7 +314,7 @@ func createCacheDir(ctx context.Context) (string, func(), error) { return cacheDir, func() { _ = os.RemoveAll(cacheDir) }, nil } -func (m *pythonMutator) runPythonMutator(ctx context.Context, root dyn.Value, opts runPythonMutatorOpts) (dyn.Value, diag.Diagnostics) { +func (m *pythonMutator) runPythonMutator(ctx context.Context, root structvar.View, opts runPythonMutatorOpts) (*config.Root, diag.Diagnostics) { inputPath := filepath.Join(opts.cacheDir, "input.json") outputPath := filepath.Join(opts.cacheDir, "output.json") diagnosticsPath := filepath.Join(opts.cacheDir, "diagnostics.json") @@ -368,7 +339,7 @@ func (m *pythonMutator) runPythonMutator(ctx context.Context, root dyn.Value, op } if err := writeInputFile(inputPath, root); err != nil { - return dyn.InvalidValue, diag.Errorf("failed to write input file: %s", err) + return nil, diag.Errorf("failed to write input file: %s", err) } stderrBuf := bytes.Buffer{} @@ -398,7 +369,7 @@ func (m *pythonMutator) runPythonMutator(ctx context.Context, root dyn.Value, op // if diagnostics file exists, it gives the most descriptive errors // if there is any error, we treat it as fatal error, and stop processing if pythonDiagnostics.HasError() { - return dyn.InvalidValue, pythonDiagnostics + return nil, pythonDiagnostics } // process can fail without reporting errors in diagnostics file or creating it, for instance, @@ -410,21 +381,24 @@ func (m *pythonMutator) runPythonMutator(ctx context.Context, root dyn.Value, op Detail: explainProcessErr(ctx, stderrBuf.String()), } - return dyn.InvalidValue, diag.Diagnostics{diagnostic} + return nil, diag.Diagnostics{diagnostic} } // or we can fail to read diagnostics file, that should always be created if pythonDiagnosticsErr != nil { - return dyn.InvalidValue, diag.Errorf("failed to load diagnostics: %s", pythonDiagnosticsErr) + return nil, diag.Errorf("failed to load diagnostics: %s", pythonDiagnosticsErr) } locations, err := loadLocationsFile(opts.bundleRootPath, locationsPath) if err != nil { - return dyn.InvalidValue, diag.Errorf("failed to load locations: %s", err) + return nil, diag.Errorf("failed to load locations: %s", err) } output, outputDiags := loadOutputFile(opts.bundleRootPath, outputPath, locations) pythonDiagnostics = pythonDiagnostics.Extend(outputDiags) + if pythonDiagnostics.HasError() { + return nil, pythonDiagnostics + } // we pass through pythonDiagnostic because it contains warnings return output, pythonDiagnostics @@ -456,9 +430,9 @@ func explainProcessErr(ctx context.Context, stderr string) string { return stderr } -func writeInputFile(inputPath string, input dyn.Value) error { - // we need to marshal dyn.Value instead of bundle.Config to JSON to support - // non-string fields assigned with bundle variables +func writeInputFile(inputPath string, input structvar.View) error { + // we need to marshal the configuration tree instead of bundle.Config to JSON to + // support non-string fields assigned with bundle variables rootConfigJson, err := json.Marshal(input.AsAny()) if err != nil { return fmt.Errorf("failed to marshal input: %w", err) @@ -481,10 +455,10 @@ func loadLocationsFile(bundleRoot, locationsPath string) (*pythonLocations, erro return parsePythonLocations(bundleRoot, locationsFile) } -func loadOutputFile(rootPath, outputPath string, locations *pythonLocations) (dyn.Value, diag.Diagnostics) { +func loadOutputFile(rootPath, outputPath string, locations *pythonLocations) (*config.Root, diag.Diagnostics) { outputFile, err := os.Open(outputPath) if err != nil { - return dyn.InvalidValue, diag.FromErr(fmt.Errorf("failed to open output file: %w", err)) + return nil, diag.FromErr(fmt.Errorf("failed to open output file: %w", err)) } defer outputFile.Close() @@ -492,7 +466,7 @@ func loadOutputFile(rootPath, outputPath string, locations *pythonLocations) (dy return loadOutput(rootPath, outputFile, locations) } -func loadOutput(rootPath string, outputFile io.Reader, locations *pythonLocations) (dyn.Value, diag.Diagnostics) { +func loadOutput(rootPath string, outputFile io.Reader, locations *pythonLocations) (*config.Root, diag.Diagnostics) { // we need absolute path because later parts of pipeline assume all paths are absolute // and this file will be used as location to resolve relative paths. // @@ -503,41 +477,27 @@ func loadOutput(rootPath string, outputFile io.Reader, locations *pythonLocation // for that, we pass virtualPath instead of outputPath as file location virtualPath, err := filepath.Abs(filepath.Join(rootPath, generatedFileName)) if err != nil { - return dyn.InvalidValue, diag.FromErr(fmt.Errorf("failed to get absolute path: %w", err)) + return nil, diag.FromErr(fmt.Errorf("failed to get absolute path: %w", err)) } - generated, err := yamlloader.LoadYAML(virtualPath, outputFile) + // The output has locations as if it comes from the generated YAML file; earlier we + // loaded locations.json with source locations in Python code. + root, diags, err := config.LoadFromReader(virtualPath, outputFile, mergePythonLocations(locations)) if err != nil { - return dyn.InvalidValue, diag.FromErr(fmt.Errorf("failed to parse output file: %w", err)) + return nil, diag.FromErr(fmt.Errorf("failed to parse output file: %w", err)) } - // generated has diag.Location as if it comes from generated YAML file - // earlier we loaded locations.json with source locations in Python code - generatedWithLocations, err := mergePythonLocations(generated, locations) - if err != nil { - return dyn.InvalidValue, diag.FromErr(fmt.Errorf("failed to update locations: %w", err)) - } - - return strictNormalize(config.Root{}, generatedWithLocations) -} - -func strictNormalize(dst any, generated dyn.Value) (dyn.Value, diag.Diagnostics) { - normalized, diags := convert.Normalize(dst, generated) - // warnings shouldn't happen because output should be already normalized // when it happens, it's a bug in the mutator, and should be treated as an error - strictDiags := diag.Diagnostics{} - for _, d := range diags { if d.Severity == diag.Warning { d.Severity = diag.Error } - strictDiags = strictDiags.Append(d) } - return normalized, strictDiags + return root, strictDiags } // loadDiagnosticsFile loads diagnostics from a file. diff --git a/bundle/config/mutator/python/python_mutator_test.go b/bundle/config/mutator/python/python_mutator_test.go index f2c91749e87..bf5ea03eaab 100644 --- a/bundle/config/mutator/python/python_mutator_test.go +++ b/bundle/config/mutator/python/python_mutator_test.go @@ -12,19 +12,24 @@ import ( "testing" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn/convert" - "github.com/databricks/cli/bundle/env" "github.com/stretchr/testify/require" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/internal/testutil" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/process" + "github.com/stretchr/testify/assert" + + "strings" + + "github.com/databricks/cli/libs/structs/structpath" ) func TestPythonMutator_Name_loadResources(t *testing.T) { @@ -120,11 +125,11 @@ workspace: { current_user: { userName: test }}`) assert.Equal(t, "pipeline_0", pipeline0.Name) } - // output of locations.json should be applied to underlying dyn.Value - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { + // output of locations.json should be applied to underlying configuration + { // location is databricks.yml, because output contains resource as-is - jobName0, err := dyn.GetByPath(v, dyn.MustPathFromString("resources.jobs.job0.name")) - require.NoError(t, err) + jobName0 := b.Config.View().Lookup(structpath.MustParsePath("resources.jobs.job0.name")) + require.True(t, jobName0.IsValid()) assert.Equal(t, []diag.Location{ { File: "databricks.yml", @@ -133,8 +138,8 @@ workspace: { current_user: { userName: test }}`) }, }, jobName0.Locations()) - jobName1, err := dyn.GetByPath(v, dyn.MustPathFromString("resources.jobs.job1.name")) - require.NoError(t, err) + jobName1 := b.Config.View().Lookup(structpath.MustParsePath("resources.jobs.job1.name")) + require.True(t, jobName1.IsValid()) assert.Equal(t, []diag.Location{ { File: filepath.Join(rootPath, "src/examples/job1.py"), @@ -143,8 +148,8 @@ workspace: { current_user: { userName: test }}`) }, }, jobName1.Locations()) - pipelineName0, err := dyn.GetByPath(v, dyn.MustPathFromString("resources.pipelines.pipeline0.name")) - require.NoError(t, err) + pipelineName0 := b.Config.View().Lookup(structpath.MustParsePath("resources.pipelines.pipeline0.name")) + require.True(t, pipelineName0.IsValid()) assert.Equal(t, []diag.Location{ { File: filepath.Join(rootPath, "src/examples/pipeline0.py"), @@ -153,9 +158,7 @@ workspace: { current_user: { userName: test }}`) }, }, pipelineName0.Locations()) - return v, nil - }) - assert.NoError(t, err) + } assert.Equal(t, int64(2), b.Metrics.PythonAddedResourcesCount) assert.Equal(t, int64(0), b.Metrics.PythonUpdatedResourcesCount) @@ -219,24 +222,22 @@ resources: assert.Equal(t, "job_0", b.Config.Resources.Jobs["job0"].Name) assert.Equal(t, "my job", b.Config.Resources.Jobs["job0"].Description) - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { + { // 'name' wasn't changed, so it keeps its location - name, err := dyn.GetByPath(v, dyn.MustPathFromString("resources.jobs.job0.name")) - require.NoError(t, err) + name := b.Config.View().Lookup(structpath.MustParsePath("resources.jobs.job0.name")) + require.True(t, name.IsValid()) assert.Equal(t, "databricks.yml", name.Location().File) // 'description' was updated by Python code and has location of generated file until // we implement source maps - description, err := dyn.GetByPath(v, dyn.MustPathFromString("resources.jobs.job0.description")) - require.NoError(t, err) + description := b.Config.View().Lookup(structpath.MustParsePath("resources.jobs.job0.description")) + require.True(t, description.IsValid()) expectedVirtualPath, err := filepath.Abs(generatedFileName) require.NoError(t, err) assert.Equal(t, expectedVirtualPath, description.Location().File) - return v, nil - }) - assert.NoError(t, err) + } assert.Equal(t, int64(0), b.Metrics.PythonAddedResourcesCount) assert.Equal(t, int64(1), b.Metrics.PythonUpdatedResourcesCount) @@ -457,20 +458,14 @@ func TestInterpreterPath(t *testing.T) { } } -func TestStrictNormalize(t *testing.T) { - // NB: there is no way to trigger diag.Error, so we don't test it - - type TestStruct struct { - A int `json:"a"` - } - - value := dyn.NewValue(map[string]dyn.Value{"A": dyn.NewValue("abc", nil)}, nil) +func TestLoadOutputStrict(t *testing.T) { + // Warnings when decoding the output are a bug in the Python code and are errors. + output := `{"resources": {"jobs": {"my_job": {"max_concurrent_runs": "abc"}}}}` - _, diags := convert.Normalize(TestStruct{}, value) - _, strictDiags := strictNormalize(TestStruct{}, value) + _, diags := loadOutput(t.TempDir(), strings.NewReader(output), newPythonLocations()) - assert.False(t, diags.HasError()) - assert.True(t, strictDiags.HasError()) + require.True(t, diags.HasError()) + assert.Equal(t, `cannot parse "abc" as an integer`, diags[0].Summary) } func TestCreateCacheDir(t *testing.T) { diff --git a/bundle/config/mutator/resolve_variable_references.go b/bundle/config/mutator/resolve_variable_references.go index 257309c55f8..bab21234c41 100644 --- a/bundle/config/mutator/resolve_variable_references.go +++ b/bundle/config/mutator/resolve_variable_references.go @@ -4,18 +4,17 @@ import ( "context" "errors" "fmt" + "maps" + "reflect" "slices" "strings" - "github.com/databricks/cli/libs/dyn/merge" - "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/variable" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/convert" - "github.com/databricks/cli/libs/dyn/dynvar" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) /* @@ -45,13 +44,13 @@ var defaultPrefixes = []string{ "variables", } -var artifactPath = dyn.MustPathFromString("artifacts") +var artifactPath = structpath.MustParsePath("artifacts") type resolveVariableReferences struct { prefixes []string - pattern dyn.Pattern - lookupFn func(dyn.Value, dyn.Path, *bundle.Bundle) (dyn.Value, error) - allowPathFn func(dyn.Path) bool + pattern *structpath.PatternNode + lookupFn func(structvar.View, *structpath.PathNode, *bundle.Bundle) (structvar.View, error) + allowPathFn func(*structpath.PathNode) bool extraRounds int // includeResources allows resolving variables in 'resources', otherwise, they are excluded. @@ -76,7 +75,7 @@ func ResolveVariableReferencesOnlyResources(prefixes ...string) bundle.Mutator { prefixes: prefixes, lookupFn: lookup, extraRounds: maxResolutionRounds - 1, - pattern: dyn.NewPattern(dyn.Key("resources")), + pattern: structpath.MustParsePattern("resources"), includeResources: true, } } @@ -95,7 +94,7 @@ func ResolveVariableReferencesWithoutResources(prefixes ...string) bundle.Mutato func ResolveVariableReferencesInLookup() bundle.Mutator { return &resolveVariableReferences{ prefixes: defaultPrefixes, - pattern: dyn.NewPattern(dyn.Key("variables"), dyn.AnyKey(), dyn.Key("lookup")), + pattern: structpath.MustParsePattern("variables.*.lookup"), lookupFn: lookupForVariables, extraRounds: maxResolutionRounds - 1, } @@ -112,36 +111,44 @@ func ResolveVolumePathReferencesOnlyResources() bundle.Mutator { } } -func lookup(v dyn.Value, path dyn.Path, b *bundle.Bundle) (dyn.Value, error) { +func lookup(v structvar.View, path *structpath.PathNode, b *bundle.Bundle) (structvar.View, error) { if config.IsExplicitlyEnabled(b.Config.Presets.SourceLinkedDeployment) { if path.String() == "workspace.file_path" { - return dyn.V(b.SyncRootPath), nil + return structvar.NewView(&b.SyncRootPath, nil, nil), nil } } // Future opportunity: if we lookup this path in both the given root // and the synthesized root, we know if it was explicitly set or implied to be empty. // Then we can emit a warning if it was not explicitly set. - return dyn.GetByPath(v, path) + return lookupValue(v, path) } -func lookupForVariables(v dyn.Value, path dyn.Path, b *bundle.Bundle) (dyn.Value, error) { - if path[0].Key() != "variables" { - return lookup(v, path, b) - } +// lookupValue returns the value at path. Fields that are declared in the type but not +// set resolve to their zero value. This enables users to interpolate variable references +// to fields that haven't been set, e.g. ${bundle.git.origin_url} resolves to an empty +// string if a bundle isn't located in a Git repository (yet). +func lookupValue(x structvar.View, path *structpath.PathNode) (structvar.View, error) { + return x.LookupWithDefaults(path) +} - varV, err := dyn.GetByPath(v, path[:len(path)-1]) - if err != nil { - return dyn.InvalidValue, err +func lookupForVariables(v structvar.View, path *structpath.PathNode, b *bundle.Bundle) (structvar.View, error) { + if path.KeyAt(0) != "variables" { + return lookup(v, path, b) } - var vv variable.Variable - err = convert.ToTyped(&vv, varV) + varV, err := lookupValue(v, path.Parent()) if err != nil { - return dyn.InvalidValue, err + return structvar.View{}, err } - if vv.Lookup != nil && vv.Lookup.String() != "" { - return dyn.InvalidValue, errors.New("lookup variables cannot contain references to another lookup variables") + if lookupV := varV.Get("lookup"); lookupV.IsValid() { + var vl variable.Lookup + if _, err := (&structvar.StructVar{Value: &vl}).Assign(nil, lookupV); err != nil { + return structvar.View{}, err + } + if vl.String() != "" { + return structvar.View{}, errors.New("lookup variables cannot contain references to another lookup variables") + } } return lookup(v, path, b) @@ -160,14 +167,23 @@ func (m *resolveVariableReferences) Validate(ctx context.Context, b *bundle.Bund } func (m *resolveVariableReferences) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - prefixes := make([]dyn.Path, len(m.prefixes)) + prefixes := make([]*structpath.PathNode, len(m.prefixes)) for i, prefix := range m.prefixes { - prefixes[i] = dyn.MustPathFromString(prefix) + prefixes[i] = structpath.MustParsePath(prefix) } // The path ${var.foo} is a shorthand for ${variables.foo.value}. // We rewrite it here to make the resolution logic simpler. - varPath := dyn.NewPath(dyn.Key("var")) + varPath := structpath.MustParsePath("var") + + // Resolution converts the whole configuration on every round; skip it if no + // reference would be resolved. + if !m.hasReferencesToResolve(b, prefixes, varPath) { + if m.artifactsReferenceUsed { + b.Metrics.SetBoolValue("artifacts_reference_used", true) + } + return nil + } var diags diag.Diagnostics maxRounds := 1 + m.extraRounds @@ -185,6 +201,11 @@ func (m *resolveVariableReferences) Apply(ctx context.Context, b *bundle.Bundle) break } + // Another round would only find out that nothing is left to resolve. + if !m.hasReferencesToResolve(b, prefixes, varPath) { + break + } + if round >= maxRounds-1 { diags = diags.Append(diag.Diagnostic{ Severity: diag.Warning, @@ -202,86 +223,195 @@ func (m *resolveVariableReferences) Apply(ctx context.Context, b *bundle.Bundle) return diags } -func (m *resolveVariableReferences) resolveOnce(b *bundle.Bundle, prefixes []dyn.Path, varPath dyn.Path) (bool, diag.Diagnostics) { +// hasReferencesToResolve reports whether the configuration in scope of this mutator +// has a reference with one of the prefixes. It is conservative: the scope is all of +// the configuration outside "resources" (or only "resources"), regardless of the pattern. +// Like resolution itself, it records whether "artifacts" is referenced. +func (m *resolveVariableReferences) hasReferencesToResolve(b *bundle.Bundle, prefixes []*structpath.PathNode, varPath *structpath.PathNode) bool { + onlyResources := m.includeResources && m.pattern != nil + inScope := func(path string) bool { + isResources := path == "resources" || strings.HasPrefix(path, "resources.") + return (m.includeResources || !isResources) && (!onlyResources || isResources) + } + + found := false + check := func(s string) { + ref, ok := structvar.NewRef(s) + if !ok { + return + } + for _, r := range ref.References() { + path, err := structpath.ParsePath(r) + if err != nil { + // Let resolution report it. + found = true + return + } + if path.HasPrefix(varPath) { + path = structpath.Join(structpath.NewStringKey(nil, "variables"), path.SkipPrefix(1).AsSlice()...) + } + if path.HasPrefix(artifactPath) { + m.artifactsReferenceUsed = true + } + if slices.ContainsFunc(prefixes, path.HasPrefix) { + found = true + } + } + } + + root := reflect.ValueOf(&b.Config).Elem() + rootType := root.Type() + for i := range rootType.NumField() { + name, _, _ := strings.Cut(rootType.Field(i).Tag.Get("json"), ",") + if name == "" || name == "-" || !inScope(name) { + continue + } + walkStrings(root.Field(i), check) + } + for path, ref := range b.Config.References() { + if inScope(path.String()) { + check(ref) + } + } + return found +} + +func (m *resolveVariableReferences) resolveOnce(b *bundle.Bundle, prefixes []*structpath.PathNode, varPath *structpath.PathNode) (bool, diag.Diagnostics) { var diags diag.Diagnostics hasUpdates := false - err := m.selectivelyMutate(b, func(root dyn.Value) (dyn.Value, error) { - // Synthesize a copy of the root that has all fields that are present in the type - // but not set in the dynamic value set to their corresponding empty value. - // This enables users to interpolate variable references to fields that haven't - // been explicitly set in the dynamic value. - // - // For example: ${bundle.git.origin_url} should resolve to an empty string - // if a bundle isn't located in a Git repository (yet). - // - // This is consistent with the behavior prior to using the dynamic value system. - // - // We can ignore the diagnostics return value because we know that the dynamic value - // has already been normalized when it was first loaded from the configuration file. - // - normalized, _ := convert.Normalize(b.Config, root, convert.IncludeMissingFields) - - // If the pattern is nil, we resolve references in the entire configuration. - root, err := dyn.MapByPattern(root, m.pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - // Resolve variable references in all values. - return dynvar.Resolve(v, func(path dyn.Path) (dyn.Value, error) { - // Rewrite the shorthand path ${var.foo} into ${variables.foo.value}. - if path.HasPrefix(varPath) { - newPath := dyn.NewPath( - dyn.Key("variables"), - path[1], - dyn.Key("value"), - ) - - if len(path) > 2 { - newPath = newPath.Append(path[2:]...) - } - - path = newPath - } - - // If the path starts with "artifacts", we need to add a metric to track if this reference is used. - if path.HasPrefix(artifactPath) { - m.artifactsReferenceUsed = true - } - - // Perform resolution only if the path starts with one of the specified prefixes. - if slices.ContainsFunc(prefixes, path.HasPrefix) { - if slices.Contains(m.excludePaths, path.String()) { - return dyn.InvalidValue, dynvar.ErrSkipResolution - } - if m.allowPathFn != nil && !m.allowPathFn(path) { - return dyn.InvalidValue, dynvar.ErrSkipResolution - } - value, err := m.lookupFn(normalized, path, b) - hasUpdates = hasUpdates || (err == nil && value.IsValid()) - return value, err - } - - return dyn.InvalidValue, dynvar.ErrSkipResolution - }) - }) + root := b.Config.View() + + lookupFn := func(sp *structpath.PathNode) (structvar.View, error) { + path := sp + // Rewrite the shorthand path ${var.foo} into ${variables.foo.value}. + if path.HasPrefix(varPath) { + path = structpath.Join(structpath.NewStringKeys(nil, "variables", path.KeyAt(1), "value"), path.SkipPrefix(2).AsSlice()...) + } + + // If the path starts with "artifacts", we need to add a metric to track if this reference is used. + if path.HasPrefix(artifactPath) { + m.artifactsReferenceUsed = true + } + + // Perform resolution only if the path starts with one of the specified prefixes. + if slices.ContainsFunc(prefixes, path.HasPrefix) { + if slices.Contains(m.excludePaths, path.String()) { + return structvar.View{}, structvar.ErrSkipResolution + } + if m.allowPathFn != nil && !m.allowPathFn(path) { + return structvar.View{}, structvar.ErrSkipResolution + } + value, err := m.lookupFn(root, path, b) + hasUpdates = hasUpdates || (err == nil && value.IsValid()) + return value, err + } + + return structvar.View{}, structvar.ErrSkipResolution + } + + // Resolve the references in each value matching the pattern (the whole configuration + // if the pattern is nil). Template keys are relative to that value, like the paths + // in cycle errors. The results are applied after all of them are resolved. + type update struct { + path *structpath.PathNode + value structvar.View + } + var updates []update + resolveIn := func(p *structpath.PathNode, refs []referenceString) error { + templates := map[string]structvar.Template{} + paths := map[string]*structpath.PathNode{} + for _, ref := range refs { + key := ref.path.String() + templates[key] = structvar.Template{Value: ref.value, Locations: ref.locs} + paths[key] = structpath.Join(p, ref.path.AsSlice()...) + } + out, err := structvar.Resolve(templates, lookupFn) if err != nil { - return dyn.InvalidValue, err + return err } + for _, key := range slices.Sorted(maps.Keys(out)) { + updates = append(updates, update{path: paths[key], value: out[key]}) + } + return nil + } - // Normalize the result because variable resolution may have been applied to non-string fields. - // For example, a variable reference may have been resolved to a integer. - root, normaliseDiags := convert.Normalize(b.Config, root) - diags = diags.Extend(normaliseDiags) - return root, nil - }) + var err error + if m.pattern == nil { + err = resolveIn(nil, m.referencesInScope(root)) + } else { + err = structvar.ForEach(root, m.pattern, func(np *structpath.PathNode, v structvar.View) error { + if !m.inScope(np) { + return nil + } + return resolveIn(np, collectReferenceStrings(v)) + }) + } if err != nil { - diags = diags.Extend(resolveErrorDiags(err)) + return hasUpdates, diags.Extend(resolveErrorDiags(err)) + } + + // Store the results in the typed configuration, converting them to the type of the + // field (e.g. a variable reference resolved to an integer). + for _, u := range updates { + d, err := b.Config.Decode(u.path, u.value) + diags = diags.Extend(d).Extend(diag.FromErr(err)) } return hasUpdates, diags } +// inScope reports whether path is in the part of the configuration this mutator resolves. +func (m *resolveVariableReferences) inScope(path *structpath.PathNode) bool { + return m.includeResources || path.KeyAt(0) != "resources" +} + +// referencesInScope returns the reference strings in the whole configuration, +// excluding "resources" unless resources are included. +func (m *resolveVariableReferences) referencesInScope(root structvar.View) []referenceString { + if m.includeResources { + return collectReferenceStrings(root) + } + var out []referenceString + for k, v := range root.MapItems() { + if !m.inScope(structpath.NewStringKey(nil, k)) { + continue + } + p := structpath.NewStringKey(nil, k) + for _, ref := range collectReferenceStrings(v) { + ref.path = structpath.Join(p, ref.path.AsSlice()...) + out = append(out, ref) + } + } + return out +} + +// referenceString is a string with a variable reference at path (relative to the +// value being resolved). +type referenceString struct { + path *structpath.PathNode + value string + locs []diag.Location +} + +// collectReferenceStrings returns the strings in v that contain variable references. +// Pure references in fields that cannot hold a string are strings in the view. +func collectReferenceStrings(v structvar.View) []referenceString { + var out []referenceString + _ = structvar.Walk(v, func(p *structpath.PathNode, v structvar.View) error { + if s, ok := v.AsString(); ok { + if _, ok := structvar.NewRef(s); ok { + out = append(out, referenceString{path: p, value: s, locs: v.Locations()}) + } + } + return nil + }) + return out +} + // resolveErrorDiags renders "did you mean" suggestions as a diagnostic Detail so // libs/diag owns the multi-line formatting. func resolveErrorDiags(err error) diag.Diagnostics { - refErr, ok := errors.AsType[*dynvar.ReferenceError](err) + refErr, ok := errors.AsType[*structvar.ReferenceError](err) if !ok || len(refErr.Suggestions) == 0 { return diag.FromErr(err) } @@ -303,66 +433,11 @@ func resolveErrorDiags(err error) diag.Diagnostics { }} } -// selectivelyMutate applies a function to a subset of the configuration -func (m *resolveVariableReferences) selectivelyMutate(b *bundle.Bundle, fn func(value dyn.Value) (dyn.Value, error)) error { - return b.Config.Mutate(func(root dyn.Value) (dyn.Value, error) { - allKeys, err := getAllKeys(root) - if err != nil { - return dyn.InvalidValue, err - } - - var included []string - for _, key := range allKeys { - if key == "resources" { - if m.includeResources { - included = append(included, key) - } - } else { - included = append(included, key) - } - } - - includedRoot, err := merge.Select(root, included) - if err != nil { - return dyn.InvalidValue, err - } - - excludedRoot, err := merge.AntiSelect(root, included) - if err != nil { - return dyn.InvalidValue, err - } - - updatedRoot, err := fn(includedRoot) - if err != nil { - return dyn.InvalidValue, err - } - - // merge is recursive, but it doesn't matter because keys are mutually exclusive - return merge.Merge(updatedRoot, excludedRoot) - }) -} - -func getAllKeys(root dyn.Value) ([]string, error) { - var keys []string - - if mapping, ok := root.AsMap(); ok { - for _, key := range mapping.Keys() { - if keyString, ok := key.AsString(); ok { - keys = append(keys, keyString) - } else { - return nil, fmt.Errorf("key is not a string: %v", key) - } - } - } - - return keys, nil -} - -func isVolumePathReferencePath(path dyn.Path) bool { - if len(path) != 4 { +func isVolumePathReferencePath(path *structpath.PathNode) bool { + if path.Len() != 4 { return false } - return path[0].Key() == "resources" && - path[1].Key() == "volumes" && - path[3].Key() == "volume_path" + return path.KeyAt(0) == "resources" && + path.KeyAt(1) == "volumes" && + path.KeyAt(3) == "volume_path" } diff --git a/bundle/config/mutator/resolve_variable_references_test.go b/bundle/config/mutator/resolve_variable_references_test.go index 284b78846b4..4be7f012af2 100644 --- a/bundle/config/mutator/resolve_variable_references_test.go +++ b/bundle/config/mutator/resolve_variable_references_test.go @@ -6,7 +6,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/resources" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/databricks-sdk-go/service/pipelines" "github.com/stretchr/testify/require" ) @@ -106,7 +106,7 @@ func TestResolveVolumePathReferencesOnlyResources_MissingTarget(t *testing.T) { } func TestIsVolumePathReferencePath(t *testing.T) { - require.True(t, isVolumePathReferencePath(dyn.MustPathFromString("resources.volumes.foo.volume_path"))) - require.False(t, isVolumePathReferencePath(dyn.MustPathFromString("resources.volumes.foo.name"))) - require.False(t, isVolumePathReferencePath(dyn.MustPathFromString("resources.jobs.foo.name"))) + require.True(t, isVolumePathReferencePath(structpath.MustParsePath("resources.volumes.foo.volume_path"))) + require.False(t, isVolumePathReferencePath(structpath.MustParsePath("resources.volumes.foo.name"))) + require.False(t, isVolumePathReferencePath(structpath.MustParsePath("resources.jobs.foo.name"))) } diff --git a/bundle/config/mutator/resourcemutator/apply_bundle_permissions.go b/bundle/config/mutator/resourcemutator/apply_bundle_permissions.go index f2a6d857090..33a09135222 100644 --- a/bundle/config/mutator/resourcemutator/apply_bundle_permissions.go +++ b/bundle/config/mutator/resourcemutator/apply_bundle_permissions.go @@ -3,17 +3,17 @@ package resourcemutator import ( "context" "fmt" + "reflect" "slices" "strings" - "github.com/databricks/cli/bundle/permissions" - "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/permissions" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/convert" "github.com/databricks/cli/libs/logdiag" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" "github.com/databricks/databricks-sdk-go/service/iam" ) @@ -108,60 +108,84 @@ func (m *bundlePermissions) Apply(ctx context.Context, b *bundle.Bundle) diag.Di return diag.FromErr(err) } - patterns := make(map[string]dyn.Pattern, 0) + keys := make([]string, 0, len(levelsMap)) for key := range levelsMap { - patterns[key] = dyn.NewPattern( - dyn.Key("resources"), - dyn.Key(key), - dyn.AnyKey(), - ) + keys = append(keys, key) } + slices.Sort(keys) - err = b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { - for key, pattern := range patterns { - v, err = dyn.MapByPattern(v, pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - var permissions []resources.Permission - pv, err := dyn.Get(v, "permissions") - // If the permissions field is not found, we set to an empty array - if err != nil { - pv = dyn.V([]dyn.Value{}) - } - - err = convert.ToTyped(&permissions, pv) - if err != nil { - return dyn.InvalidValue, fmt.Errorf("failed to convert permissions: %w", err) - } + for _, key := range keys { + pattern := structpath.MustParsePattern("resources." + key + ".*") - permissions = append(permissions, convertPermissions( - ctx, - b.Config.Permissions, - permissions, - key, - levelsMap[key], - )...) + err = structvar.ForEach(b.Config.View(), pattern, func(p *structpath.PathNode, v structvar.View) error { + had := v.Get("permissions").IsValid() + var permissions []resources.Permission + for _, pv := range v.Get("permissions").Sequence() { + level, _ := pv.Get("level").AsString() + userName, _ := pv.Get("user_name").AsString() + groupName, _ := pv.Get("group_name").AsString() + servicePrincipalName, _ := pv.Get("service_principal_name").AsString() + permissions = append(permissions, resources.Permission{ + Level: iam.PermissionLevel(level), + UserName: userName, + GroupName: groupName, + ServicePrincipalName: servicePrincipalName, + }) + } - if len(permissions) == 0 { - permissions = nil + added := convertPermissions( + ctx, + b.Config.Permissions, + permissions, + key, + levelsMap[key], + ) + if len(added) > 0 { + if err := appendPermissions(v, added); err != nil { + return err } + } - pv, err = convert.FromTyped(permissions, dyn.NilValue) - if err != nil { - return dyn.InvalidValue, fmt.Errorf("failed to convert permissions: %w", err) - } + if !had { + return nil + } - return dyn.Set(v, "permissions", pv) - }) - if err != nil { - return dyn.InvalidValue, err + // Empty permissions are dropped. Otherwise they are rebuilt without locations. + permissionsPath := structpath.NewStringKey(p, "permissions") + if len(permissions) == 0 && len(added) == 0 { + return b.Config.Delete(permissionsPath) } + b.Config.SetLocations(permissionsPath, nil) + return nil + }) + if err != nil { + return diag.FromErr(err) } + } - return v, nil - }) - if err != nil { - return diag.FromErr(err) + return nil +} + +// appendPermissions appends permissions to the permissions field of the resource described by v. +// The type of the permissions differs between resources, but all have the same fields. +func appendPermissions(v structvar.View, permissions []resources.Permission) error { + r := v.Reflect() + for r.Kind() == reflect.Pointer || r.Kind() == reflect.Interface { + r = r.Elem() + } + field := r.FieldByName("Permissions") + if !field.IsValid() || !field.CanSet() || field.Kind() != reflect.Slice { + return fmt.Errorf("cannot set permissions of %s", r.Type()) } + for _, p := range permissions { + elem := reflect.New(field.Type().Elem()).Elem() + elem.FieldByName("Level").SetString(string(p.Level)) + elem.FieldByName("UserName").SetString(p.UserName) + elem.FieldByName("GroupName").SetString(p.GroupName) + elem.FieldByName("ServicePrincipalName").SetString(p.ServicePrincipalName) + field.Set(reflect.Append(field, elem)) + } return nil } diff --git a/bundle/config/mutator/resourcemutator/apply_default_task_source.go b/bundle/config/mutator/resourcemutator/apply_default_task_source.go index 30e000b1fb1..a95130597f3 100644 --- a/bundle/config/mutator/resourcemutator/apply_default_task_source.go +++ b/bundle/config/mutator/resourcemutator/apply_default_task_source.go @@ -5,7 +5,6 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/databricks-sdk-go/service/jobs" ) @@ -37,57 +36,53 @@ func (a *applyDefaultTaskSource) Name() string { return "ApplyDefaultTaskSource" } -// sourceAwareTaskTypes are the task types that support the `source` field. -// https://docs.databricks.com/api/workspace/jobs/create -var sourceAwareTaskTypes = []string{ - "dbt_task", - "gen_ai_compute_task", - "notebook_task", - "spark_python_task", - "sql_task.file", -} - func (a *applyDefaultTaskSource) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - jobsPattern := dyn.NewPattern(dyn.Key("resources"), dyn.Key("jobs"), dyn.AnyKey()) + for name, job := range b.Config.Resources.Jobs { + // Only git_source jobs need an explicit source; leave the rest untouched. + // A missing key or an explicit `git_source: null` both count as absent, + // matching the typed nil pointer TranslatePaths keys off; otherwise we would + // set source: GIT on a job whose paths get translated to workspace paths. + // A reference is not absent, although the typed field is nil. + if job == nil || (job.GitSource == nil && !b.Config.IsReference("resources.jobs."+name+".git_source")) { + continue + } - err := b.Config.Mutate(func(root dyn.Value) (dyn.Value, error) { - return dyn.MapByPattern(root, jobsPattern, func(_ dyn.Path, job dyn.Value) (dyn.Value, error) { - // Only git_source jobs need an explicit source; leave the rest untouched. - // A missing key (KindInvalid) or an explicit `git_source: null` (KindNil) - // both count as absent, matching the typed nil pointer TranslatePaths keys - // off; otherwise we would set source: GIT on a job whose paths get - // translated to workspace paths. - gitSource := job.Get("git_source") - if gitSource.Kind() == dyn.KindInvalid || gitSource.Kind() == dyn.KindNil { - return job, nil + for i := range job.Tasks { + task := &job.Tasks[i] + if task.ForEachTask != nil { + setGitTaskSource(&task.ForEachTask.Task) } - - return dyn.Map(job, "tasks", dyn.Foreach(func(_ dyn.Path, task dyn.Value) (dyn.Value, error) { - task, err := dyn.Map(task, "for_each_task.task", func(_ dyn.Path, foreachTask dyn.Value) (dyn.Value, error) { - return setGitTaskSource(foreachTask) - }) - if err != nil { - return dyn.InvalidValue, err - } - return setGitTaskSource(task) - })) - }) - }) - - return diag.FromErr(err) + setGitTaskSource(task) + } + } + return nil } // setGitTaskSource sets source: GIT on the first task-type block present that // supports the field, unless the user already set it. -func setGitTaskSource(task dyn.Value) (dyn.Value, error) { - for _, taskType := range sourceAwareTaskTypes { - t, err := dyn.Get(task, taskType) - if err != nil { - continue - } - if _, err := dyn.Get(t, "source"); err != nil { - return dyn.Set(task, taskType+".source", dyn.V(string(jobs.SourceGit))) +// The task types are the ones that support the `source` field: +// https://docs.databricks.com/api/workspace/jobs/create +func setGitTaskSource(task *jobs.Task) { + var sources []*jobs.Source + if task.DbtTask != nil { + sources = append(sources, &task.DbtTask.Source) + } + if task.GenAiComputeTask != nil { + sources = append(sources, &task.GenAiComputeTask.Source) + } + if task.NotebookTask != nil { + sources = append(sources, &task.NotebookTask.Source) + } + if task.SparkPythonTask != nil { + sources = append(sources, &task.SparkPythonTask.Source) + } + if task.SqlTask != nil && task.SqlTask.File != nil { + sources = append(sources, &task.SqlTask.File.Source) + } + for _, source := range sources { + if *source == "" { + *source = jobs.SourceGit + return } } - return task, nil } diff --git a/bundle/config/mutator/resourcemutator/capture_uc_dependencies.go b/bundle/config/mutator/resourcemutator/capture_uc_dependencies.go index 488e215a2e2..d18714e8842 100644 --- a/bundle/config/mutator/resourcemutator/capture_uc_dependencies.go +++ b/bundle/config/mutator/resourcemutator/capture_uc_dependencies.go @@ -8,7 +8,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn/dynvar" + "github.com/databricks/cli/libs/structs/structvar" ) type captureUCDependencies struct{} @@ -159,7 +159,7 @@ func (m *captureUCDependencies) Apply(ctx context.Context, b *bundle.Bundle) dia } // A name that already contains a reference is left as is: we only rewrite a // fully literal name and do not support a mix of references and literals. - if dynvar.ContainsVariableReference(qm.OutputSchemaName) { + if structvar.ContainsVariableReference(qm.OutputSchemaName) { continue } // OutputSchemaName is a compound "catalog.schema" string. @@ -179,7 +179,7 @@ func (m *captureUCDependencies) Apply(ctx context.Context, b *bundle.Bundle) dia } // A name that already contains a reference is left as is; a mix of // references and literals is not supported. - if dynvar.ContainsVariableReference(idx.Name) { + if structvar.ContainsVariableReference(idx.Name) { continue } // Name is a three-part "catalog.schema.index" UC identifier. @@ -212,7 +212,7 @@ func (m *captureUCDependencies) Apply(ctx context.Context, b *bundle.Bundle) dia } // A parent that already contains a reference is left as is; a mix of // references and literals is not supported. - if dynvar.ContainsVariableReference(ms.Parent) { + if structvar.ContainsVariableReference(ms.Parent) { continue } ms.Parent = resolveAiGatewayParent(b, ms.Parent) diff --git a/bundle/config/mutator/resourcemutator/configure_cluster_policy_definition.go b/bundle/config/mutator/resourcemutator/configure_cluster_policy_definition.go index 1e7a0d6f64b..84dddb367ba 100644 --- a/bundle/config/mutator/resourcemutator/configure_cluster_policy_definition.go +++ b/bundle/config/mutator/resourcemutator/configure_cluster_policy_definition.go @@ -7,7 +7,8 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) // jsonPolicyFields are the JSON-policy fields normalized from inline YAML to a JSON string. @@ -26,44 +27,43 @@ func (c configureClusterPolicyDefinition) Name() string { func (c configureClusterPolicyDefinition) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { var diags diag.Diagnostics - pattern := dyn.NewPattern( - dyn.Key("resources"), - dyn.Key("cluster_policies"), - dyn.AnyKey(), - ) + // Skip converting the configuration if there is nothing to configure. + if len(b.Config.Resources.ClusterPolicies) == 0 { + return nil + } - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { - return dyn.MapByPattern(v, pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - for _, field := range jsonPolicyFields { - def := v.Get(field) + pattern := structpath.MustParsePattern("resources.cluster_policies.*") - // Marshal an inline structured value to a JSON string so both - // config-side and state-side carry the same plain string. Otherwise - // YAML decodes small ints as Go `int` while state JSON round-trip - // decodes them as `float64`, and structdiff reports false drift. - switch def.Kind() { - case dyn.KindInvalid, dyn.KindNil, dyn.KindString: - // KindInvalid means the field is absent; leave it for backend validation. - continue - case dyn.KindMap: - jsonBytes, err := json.Marshal(def.AsAny()) - if err != nil { - return dyn.InvalidValue, fmt.Errorf("failed to marshal inline %s: %w", field, err) - } - v, err = dyn.Set(v, field, dyn.V(string(jsonBytes))) - if err != nil { - return dyn.InvalidValue, err - } - default: - diags = diags.Append(diag.Diagnostic{ - Severity: diag.Error, - Summary: fmt.Sprintf("%s must be a string or map, got %s", field, def.Kind()), - Locations: def.Locations(), - }) + err := structvar.ForEach(b.Config.View(), pattern, func(p *structpath.PathNode, v structvar.View) error { + for _, field := range jsonPolicyFields { + def := v.Get(field) + + // Marshal an inline structured value to a JSON string so both + // config-side and state-side carry the same plain string. Otherwise + // YAML decodes small ints as Go `int` while state JSON round-trip + // decodes them as `float64`, and structdiff reports false drift. + switch def.Kind() { + case structvar.KindInvalid, structvar.KindNil, structvar.KindString: + // KindInvalid means the field is absent; leave it for backend validation. + continue + case structvar.KindMap: + jsonBytes, err := json.Marshal(def.AsAny()) + if err != nil { + return fmt.Errorf("failed to marshal inline %s: %w", field, err) + } + err = b.Config.Set(structpath.NewStringKey(p, field), string(jsonBytes)) + if err != nil { + return err } + default: + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("%s must be a string or map, got %s", field, def.Kind()), + Locations: def.Locations(), + }) } - return v, nil - }) + } + return nil }) diags = diags.Extend(diag.FromErr(err)) diff --git a/bundle/config/mutator/resourcemutator/configure_dashboards_serialized_dashboard.go b/bundle/config/mutator/resourcemutator/configure_dashboards_serialized_dashboard.go index a17e5c6b07c..5251c92716d 100644 --- a/bundle/config/mutator/resourcemutator/configure_dashboards_serialized_dashboard.go +++ b/bundle/config/mutator/resourcemutator/configure_dashboards_serialized_dashboard.go @@ -7,7 +7,8 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) const ( @@ -28,62 +29,62 @@ func (c configureDashboardSerializedDashboard) Name() string { func (c configureDashboardSerializedDashboard) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { var diags diag.Diagnostics - pattern := dyn.NewPattern( - dyn.Key("resources"), - dyn.Key("dashboards"), - dyn.AnyKey(), - ) + // Skip converting the configuration if there is nothing to configure. + if len(b.Config.Resources.Dashboards) == 0 { + return nil + } - // Configure serialized_dashboard field for all dashboards. - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { - return dyn.MapByPattern(v, pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - // Include "serialized_dashboard" field if "file_path" is set. - // Note: the Terraform resource supports "file_path" natively, but we read the contents of the dashboard here - // to be able to read file contents in Databricks Workspace (reading a dashboard file via file system fails there) - filePath, hasFilePath := v.Get(filePathFieldName).AsString() - sd := v.Get(serializedDashboardFieldName) - - if hasFilePath { - // file_path and serialized_dashboard are two ways to provide the - // same content. Accepting both is ambiguous, so reject it instead - // of silently picking one. - if sd.IsValid() && sd.Kind() != dyn.KindNil { - diags = diags.Append(diag.Diagnostic{ - Severity: diag.Error, - Summary: "both file_path and serialized_dashboard are set; specify only one", - Locations: sd.Locations(), - }) - return v, nil - } + pattern := structpath.MustParsePattern("resources.dashboards.*") - contents, err := b.SyncRoot.ReadFile(filePath) - if err != nil { - return dyn.InvalidValue, fmt.Errorf("failed to read serialized dashboard from file_path %s: %w", filePath, err) - } - return dyn.Set(v, serializedDashboardFieldName, dyn.V(string(contents))) - } + // Configure serialized_dashboard field for all dashboards. + err := structvar.ForEach(b.Config.View(), pattern, func(p *structpath.PathNode, v structvar.View) error { + // Include "serialized_dashboard" field if "file_path" is set. + // Note: the Terraform resource supports "file_path" natively, but we read the contents of the dashboard here + // to be able to read file contents in Databricks Workspace (reading a dashboard file via file system fails there) + filePath, hasFilePath := v.Get(filePathFieldName).AsString() + sd := v.Get(serializedDashboardFieldName) + sdPath := structpath.NewStringKey(p, serializedDashboardFieldName) - // Marshal an inline structured serialized_dashboard to a JSON string - switch sd.Kind() { - case dyn.KindInvalid, dyn.KindNil, dyn.KindString: - // KindInvalid means serialized_dashboard is absent (neither it nor - // file_path is set); leave it for backend validation to reject. - return v, nil - case dyn.KindMap: - jsonBytes, err := json.Marshal(sd.AsAny()) - if err != nil { - return dyn.InvalidValue, fmt.Errorf("failed to marshal inline serialized_dashboard: %w", err) - } - return dyn.Set(v, serializedDashboardFieldName, dyn.V(string(jsonBytes))) - default: + if hasFilePath { + // file_path and serialized_dashboard are two ways to provide the + // same content. Accepting both is ambiguous, so reject it instead + // of silently picking one. + if sd.IsValid() && sd.Kind() != structvar.KindNil { diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, - Summary: fmt.Sprintf("serialized_dashboard must be a string or map, got %s", sd.Kind()), + Summary: "both file_path and serialized_dashboard are set; specify only one", Locations: sd.Locations(), }) - return v, nil + return nil + } + + contents, err := b.SyncRoot.ReadFile(filePath) + if err != nil { + return fmt.Errorf("failed to read serialized dashboard from file_path %s: %w", filePath, err) + } + return b.Config.Set(sdPath, string(contents)) + } + + // Marshal an inline structured serialized_dashboard to a JSON string + switch sd.Kind() { + case structvar.KindInvalid, structvar.KindNil, structvar.KindString: + // KindInvalid means serialized_dashboard is absent (neither it nor + // file_path is set); leave it for backend validation to reject. + return nil + case structvar.KindMap: + jsonBytes, err := json.Marshal(sd.AsAny()) + if err != nil { + return fmt.Errorf("failed to marshal inline serialized_dashboard: %w", err) } - }) + return b.Config.Set(sdPath, string(jsonBytes)) + default: + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("serialized_dashboard must be a string or map, got %s", sd.Kind()), + Locations: sd.Locations(), + }) + return nil + } }) diags = diags.Extend(diag.FromErr(err)) diff --git a/bundle/config/mutator/resourcemutator/configure_genie_space_serialized_space.go b/bundle/config/mutator/resourcemutator/configure_genie_space_serialized_space.go index e16bd57625b..0eecb3b2bb9 100644 --- a/bundle/config/mutator/resourcemutator/configure_genie_space_serialized_space.go +++ b/bundle/config/mutator/resourcemutator/configure_genie_space_serialized_space.go @@ -7,7 +7,8 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) const serializedSpaceFieldName = "serialized_space" @@ -25,64 +26,64 @@ func (c configureGenieSpaceSerializedSpace) Name() string { func (c configureGenieSpaceSerializedSpace) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { var diags diag.Diagnostics - pattern := dyn.NewPattern( - dyn.Key("resources"), - dyn.Key("genie_spaces"), - dyn.AnyKey(), - ) + // Skip converting the configuration if there is nothing to configure. + if len(b.Config.Resources.GenieSpaces) == 0 { + return nil + } - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { - return dyn.MapByPattern(v, pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - filePath, hasFilePath := v.Get(filePathFieldName).AsString() - ss := v.Get(serializedSpaceFieldName) + pattern := structpath.MustParsePattern("resources.genie_spaces.*") - if hasFilePath { - // file_path and serialized_space are two ways to provide the same - // content. Accepting both is ambiguous, so reject it instead of - // silently picking one. - if ss.IsValid() && ss.Kind() != dyn.KindNil { - diags = diags.Append(diag.Diagnostic{ - Severity: diag.Error, - Summary: "both file_path and serialized_space are set; specify only one", - Locations: ss.Locations(), - }) - return v, nil - } - contents, err := b.SyncRoot.ReadFile(filePath) - if err != nil { - return dyn.InvalidValue, fmt.Errorf("failed to read serialized genie space from file_path %s: %w", filePath, err) - } - return dyn.Set(v, serializedSpaceFieldName, dyn.V(string(contents))) - } + err := structvar.ForEach(b.Config.View(), pattern, func(p *structpath.PathNode, v structvar.View) error { + filePath, hasFilePath := v.Get(filePathFieldName).AsString() + ss := v.Get(serializedSpaceFieldName) + ssPath := structpath.NewStringKey(p, serializedSpaceFieldName) - // Marshal an inline structured serialized_space to a JSON string so - // both config-side and state-side carry the same plain string. - // Otherwise YAML decodes small ints as Go `int` while state JSON - // round-trip decodes them as `float64`, and structdiff reports - // false drift on every plan. - switch ss.Kind() { - case dyn.KindInvalid, dyn.KindNil, dyn.KindString: - // KindInvalid means serialized_space is absent (neither it nor - // file_path is set); leave it for backend validation to reject. - return v, nil - case dyn.KindMap: - // A top-level sequence would be valid JSON but is meaningless for a - // genie space, so KindSequence is not accepted here and falls through - // to the default rejection below. - jsonBytes, err := json.Marshal(ss.AsAny()) - if err != nil { - return dyn.InvalidValue, fmt.Errorf("failed to marshal inline serialized_space: %w", err) - } - return dyn.Set(v, serializedSpaceFieldName, dyn.V(string(jsonBytes))) - default: + if hasFilePath { + // file_path and serialized_space are two ways to provide the same + // content. Accepting both is ambiguous, so reject it instead of + // silently picking one. + if ss.IsValid() && ss.Kind() != structvar.KindNil { diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, - Summary: fmt.Sprintf("serialized_space must be a string or map, got %s", ss.Kind()), + Summary: "both file_path and serialized_space are set; specify only one", Locations: ss.Locations(), }) - return v, nil + return nil + } + contents, err := b.SyncRoot.ReadFile(filePath) + if err != nil { + return fmt.Errorf("failed to read serialized genie space from file_path %s: %w", filePath, err) + } + return b.Config.Set(ssPath, string(contents)) + } + + // Marshal an inline structured serialized_space to a JSON string so + // both config-side and state-side carry the same plain string. + // Otherwise YAML decodes small ints as Go `int` while state JSON + // round-trip decodes them as `float64`, and structdiff reports + // false drift on every plan. + switch ss.Kind() { + case structvar.KindInvalid, structvar.KindNil, structvar.KindString: + // KindInvalid means serialized_space is absent (neither it nor + // file_path is set); leave it for backend validation to reject. + return nil + case structvar.KindMap: + // A top-level sequence would be valid JSON but is meaningless for a + // genie space, so KindSequence is not accepted here and falls through + // to the default rejection below. + jsonBytes, err := json.Marshal(ss.AsAny()) + if err != nil { + return fmt.Errorf("failed to marshal inline serialized_space: %w", err) } - }) + return b.Config.Set(ssPath, string(jsonBytes)) + default: + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("serialized_space must be a string or map, got %s", ss.Kind()), + Locations: ss.Locations(), + }) + return nil + } }) diags = diags.Extend(diag.FromErr(err)) diff --git a/bundle/config/mutator/resourcemutator/expand_pipeline_glob_paths.go b/bundle/config/mutator/resourcemutator/expand_pipeline_glob_paths.go index 38f835793ea..1f9f2427851 100644 --- a/bundle/config/mutator/resourcemutator/expand_pipeline_glob_paths.go +++ b/bundle/config/mutator/resourcemutator/expand_pipeline_glob_paths.go @@ -8,8 +8,10 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/libraries" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/patchwheel" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" + "github.com/databricks/databricks-sdk-go/service/pipelines" ) type expandPipelineGlobPaths struct{} @@ -18,92 +20,92 @@ func ExpandPipelineGlobPaths() bundle.Mutator { return &expandPipelineGlobPaths{} } -func (m *expandPipelineGlobPaths) expandLibrary(ctx context.Context, dir string, v dyn.Value) ([]dyn.Value, error) { +// expandLibrary returns the libraries lib is replaced with. +func (m *expandPipelineGlobPaths) expandLibrary(ctx context.Context, dir string, lib pipelines.PipelineLibrary) ([]pipelines.PipelineLibrary, error) { // Probe for the path field in the library. - for _, p := range []dyn.Path{ - dyn.NewPath(dyn.Key("notebook"), dyn.Key("path")), - dyn.NewPath(dyn.Key("file"), dyn.Key("path")), - } { - pv, err := dyn.GetByPath(v, p) - if dyn.IsNoSuchKeyError(err) { - continue - } - if err != nil { - return nil, err - } + var path string + switch { + case lib.Notebook != nil && lib.Notebook.Path != "": + path = lib.Notebook.Path + case lib.File != nil && lib.File.Path != "": + path = lib.File.Path + default: + // Neither of the library paths were found (or the path is empty). This is likely an invalid node, + // but it isn't this mutator's job to enforce that. Return the original value. + return []pipelines.PipelineLibrary{lib}, nil + } - // If the path is empty or not a local path, return the original value. - path := pv.MustString() - if path == "" || !libraries.IsLocalPath(path) { - return []dyn.Value{v}, nil - } + // If the path is not a local path, return the original value. + if !libraries.IsLocalPath(path) { + return []pipelines.PipelineLibrary{lib}, nil + } - matches, err := filepath.Glob(filepath.Join(dir, path)) - if err != nil { - return nil, err - } + matches, err := filepath.Glob(filepath.Join(dir, path)) + if err != nil { + return nil, err + } - // If there are no matches, return the original value. - if len(matches) == 0 { - return []dyn.Value{v}, nil - } + // If there are no matches, return the original value. + if len(matches) == 0 { + return []pipelines.PipelineLibrary{lib}, nil + } - matches = patchwheel.FilterLatestWheels(ctx, matches) + matches = patchwheel.FilterLatestWheels(ctx, matches) - // Emit a new value for each match. - var ev []dyn.Value - for _, match := range matches { - m, err := filepath.Rel(dir, match) - if err != nil { - return nil, err - } - nv, err := dyn.SetByPath(v, p, dyn.NewValue(filepath.ToSlash(m), pv.Locations())) - if err != nil { - return nil, err - } - ev = append(ev, nv) + // Emit a new value for each match. + var expanded []pipelines.PipelineLibrary + for _, match := range matches { + m, err := filepath.Rel(dir, match) + if err != nil { + return nil, err } - - return ev, nil + nl := lib + if lib.Notebook != nil { + notebook := *lib.Notebook + notebook.Path = filepath.ToSlash(m) + nl.Notebook = ¬ebook + } else { + file := *lib.File + file.Path = filepath.ToSlash(m) + nl.File = &file + } + expanded = append(expanded, nl) } - // Neither of the library paths were found. This is likely an invalid node, - // but it isn't this mutator's job to enforce that. Return the original value. - return []dyn.Value{v}, nil + return expanded, nil } -func (m *expandPipelineGlobPaths) expandSequence(ctx context.Context, dir string, p dyn.Path, v dyn.Value) (dyn.Value, error) { - s, ok := v.AsSequence() - if !ok { - return dyn.InvalidValue, fmt.Errorf("expected sequence, got %s", v.Kind()) - } +func (m *expandPipelineGlobPaths) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + p := structpath.MustParsePattern("resources.pipelines.*.libraries") - var vs []dyn.Value - for _, sv := range s { - v, err := m.expandLibrary(ctx, dir, sv) - if err != nil { - return dyn.InvalidValue, err + // Visit each pipeline's "libraries" field and expand any glob patterns. + err := structvar.ForEach(b.Config.View(), p, func(path *structpath.PathNode, value structvar.View) error { + if value.Kind() != structvar.KindSequence { + return fmt.Errorf("expected sequence, got %s", value.Kind()) } - vs = append(vs, v...) - } + pipeline := b.Config.Resources.Pipelines[path.KeyAt(2)] + if len(pipeline.Libraries) == 0 { + return nil + } - return dyn.NewValue(vs, v.Locations()), nil -} + var expanded []pipelines.PipelineLibrary + var sources [][]int + for i, lib := range pipeline.Libraries { + libs, err := m.expandLibrary(ctx, b.BundleRootPath, lib) + if err != nil { + return err + } -func (m *expandPipelineGlobPaths) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { - p := dyn.NewPattern( - dyn.Key("resources"), - dyn.Key("pipelines"), - dyn.AnyKey(), - dyn.Key("libraries"), - ) - - // Visit each pipeline's "libraries" field and expand any glob patterns. - return dyn.MapByPattern(v, p, func(path dyn.Path, value dyn.Value) (dyn.Value, error) { - return m.expandSequence(ctx, b.BundleRootPath, path, value) - }) + expanded = append(expanded, libs...) + for range libs { + sources = append(sources, []int{i}) + } + } + + pipeline.Libraries = expanded + b.Config.UpdateSequence(path, sources) + return nil }) return diag.FromErr(err) diff --git a/bundle/config/mutator/resourcemutator/expand_pipeline_glob_paths_test.go b/bundle/config/mutator/resourcemutator/expand_pipeline_glob_paths_test.go index ead2084d835..515d82f81f8 100644 --- a/bundle/config/mutator/resourcemutator/expand_pipeline_glob_paths_test.go +++ b/bundle/config/mutator/resourcemutator/expand_pipeline_glob_paths_test.go @@ -9,7 +9,6 @@ import ( "github.com/databricks/cli/bundle/config/mutator/resourcemutator" "github.com/stretchr/testify/assert" - "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/resources" diff --git a/bundle/config/mutator/resourcemutator/fix_permissions.go b/bundle/config/mutator/resourcemutator/fix_permissions.go index 9e6c785b0d9..6f631fe6dd9 100644 --- a/bundle/config/mutator/resourcemutator/fix_permissions.go +++ b/bundle/config/mutator/resourcemutator/fix_permissions.go @@ -2,13 +2,16 @@ package resourcemutator import ( "context" + "fmt" + "reflect" "strings" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/iamutil" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) const ( @@ -52,165 +55,132 @@ func (m *fixPermissions) Name() string { return "FixPermissions" } -func processPermissions(currentUser string) dyn.MapFunc { - return func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - // Extract resource type from path: resources...permissions - if len(p) != 4 || p[0].Key() != "resources" || p[3].Key() != "permissions" { - return v, nil - } - - resourceType := p[1].Key() - if ignoredResources[resourceType] { - return v, nil - } - - v, err := ensureCurrentUserMgmtPermissions(v, currentUser, resourceType) - if err != nil { - return v, err - } - - return useMaximumLevel(v, resourceType) - } +// permission is a permission entry: a level and the principal it is granted to. +type permission struct { + level string + userName string + servicePrincipalName string + groupName string } -func readUser(v dyn.Value) string { - userName, _ := dyn.GetValue(v, "user_name").AsString() - if userName != "" { - return userName +func (p permission) user() string { + if p.userName != "" { + return p.userName } - servicePrincipalName, _ := dyn.GetValue(v, "service_principal_name").AsString() - return servicePrincipalName + return p.servicePrincipalName } -func readPrincipal(v dyn.Value) string { - value, _ := dyn.GetValue(v, "user_name").AsString() - if value != "" { - return "user_name:" + value - } - value, _ = dyn.GetValue(v, "service_principal_name").AsString() - if value != "" { - return "service_principal_name:" + value - } - value, _ = dyn.GetValue(v, "group_name").AsString() - if value != "" { - return "group_name:" + value +// principal returns the principal in the form ":", or "" if there is none. +func (p permission) principal() string { + switch { + case p.userName != "": + return "user_name:" + p.userName + case p.servicePrincipalName != "": + return "service_principal_name:" + p.servicePrincipalName + case p.groupName != "": + return "group_name:" + p.groupName } return "" } -func ensureCurrentUserMgmtPermissions(permissions dyn.Value, currentUser, resourceType string) (dyn.Value, error) { +func processPermissions(currentUser, resourceType string, permissions []permission) []permission { + return useMaximumLevel(ensureCurrentUserMgmtPermissions(permissions, currentUser, resourceType)) +} + +func ensureCurrentUserMgmtPermissions(permissions []permission, currentUser, resourceType string) []permission { currentUserHasIsOwner := false currentUserIndCanManage := -1 canAddIsOwner := hasIsOwner[resourceType] - permissionArray, ok := permissions.AsSequence() - if !ok { - return permissions, nil - } - - for ind, permission := range permissionArray { - level, ok := dyn.GetValue(permission, "level").AsString() - if !ok { + for ind, p := range permissions { + if p.level == "" { continue } - user := readUser(permission) - if level == isOwner { + user := p.user() + if p.level == isOwner { canAddIsOwner = false if user == currentUser { currentUserHasIsOwner = true } } - if user == currentUser && level == canManage { + if user == currentUser && p.level == canManage { currentUserIndCanManage = ind } } if currentUserHasIsOwner { - return dyn.V(permissionArray), nil + return permissions } if canAddIsOwner { if currentUserIndCanManage >= 0 { // Upgrade current user's CAN_MANAGE to IS_OWNER. We do this because terraform will add IS_OWNER if it does not see one // and that may confuse backend. We can stop doing it when removed terraform. - v, _ := dyn.Set(permissionArray[currentUserIndCanManage], "level", dyn.V(isOwner)) - permissionArray[currentUserIndCanManage] = v + permissions[currentUserIndCanManage].level = isOwner } else { - permissionArray = append(permissionArray, createPermission(currentUser, isOwner)) + permissions = append(permissions, createPermission(currentUser, isOwner)) } - return dyn.V(permissionArray), nil + return permissions } if currentUserIndCanManage < 0 { - permissionArray = append(permissionArray, createPermission(currentUser, canManage)) + permissions = append(permissions, createPermission(currentUser, canManage)) } - return dyn.V(permissionArray), nil + return permissions } -func useMaximumLevel(permissions dyn.Value, resourceType string) (dyn.Value, error) { - permissionArray, ok := permissions.AsSequence() - if !ok { - return permissions, nil - } - +func useMaximumLevel(permissions []permission) []permission { levelPerPrincipal := make(map[string]string) - principalIndex := make(map[string]int) + seen := make(map[string]bool) var principals []string - for _, permission := range permissionArray { - level, _ := dyn.GetValue(permission, "level").AsString() - if level == "" { + for _, p := range permissions { + if p.level == "" { continue } - principal := readPrincipal(permission) + principal := p.principal() if principal == "" { continue } - _, ok = principalIndex[principal] - if !ok { - ind := len(principalIndex) - principalIndex[principal] = ind + if !seen[principal] { + seen[principal] = true principals = append(principals, principal) } - levelPerPrincipal[principal] = resources.GetMaxLevel(levelPerPrincipal[principal], level) + levelPerPrincipal[principal] = resources.GetMaxLevel(levelPerPrincipal[principal], p.level) } - var newPermissions []dyn.Value - + var newPermissions []permission for _, principal := range principals { newPermissions = append(newPermissions, createPermissionFromPrincipal(principal, levelPerPrincipal[principal])) } - return dyn.V(newPermissions), nil + return newPermissions } -func createPermission(user, level string) dyn.Value { - permission := map[string]dyn.Value{ - "level": dyn.V(level), - } - +func createPermission(user, level string) permission { // Determine if currentUser is a service principal or user if iamutil.IsServicePrincipalName(user) { - permission["service_principal_name"] = dyn.V(user) - } else { - permission["user_name"] = dyn.V(user) + return permission{level: level, servicePrincipalName: user} } - - return dyn.V(permission) + return permission{level: level, userName: user} } -func createPermissionFromPrincipal(principal, level string) dyn.Value { - permission := map[string]dyn.Value{ - "level": dyn.V(level), - } - +func createPermissionFromPrincipal(principal, level string) permission { items := strings.SplitN(principal, ":", 2) field := items[0] value := items[1] - permission[field] = dyn.V(value) - return dyn.V(permission) + p := permission{level: level} + switch field { + case "user_name": + p.userName = value + case "service_principal_name": + p.servicePrincipalName = value + case "group_name": + p.groupName = value + } + return p } func (m *fixPermissions) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { @@ -222,15 +192,55 @@ func (m *fixPermissions) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagn } currentUser := b.Config.Workspace.CurrentUser.UserName - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { - // Use MapByPattern to directly process permissions arrays - return dyn.MapByPattern(v, dyn.NewPattern( - dyn.Key("resources"), - dyn.AnyKey(), - dyn.AnyKey(), - dyn.Key("permissions"), - ), processPermissions(currentUser)) + err := structvar.ForEach(b.Config.View(), structpath.MustParsePattern("resources.*.*.permissions"), func(p *structpath.PathNode, v structvar.View) error { + // Extract resource type from path: resources...permissions + resourceType := p.KeyAt(1) + if ignoredResources[resourceType] || v.Kind() != structvar.KindSequence { + return nil + } + + var permissions []permission + for _, pv := range v.Sequence() { + var perm permission + perm.level, _ = pv.Get("level").AsString() + perm.userName, _ = pv.Get("user_name").AsString() + perm.servicePrincipalName, _ = pv.Get("service_principal_name").AsString() + perm.groupName, _ = pv.Get("group_name").AsString() + permissions = append(permissions, perm) + } + + err := setPermissions(v, processPermissions(currentUser, resourceType, permissions)) + if err != nil { + return err + } + + // The permissions are rebuilt without locations. + b.Config.SetLocations(p, nil) + return nil }) return diag.FromErr(err) } + +// setPermissions replaces the permissions described by v. The type of the permissions +// differs between resources, but all have the same fields. +func setPermissions(v structvar.View, permissions []permission) error { + field := v.Reflect() + for field.Kind() == reflect.Pointer || field.Kind() == reflect.Interface { + field = field.Elem() + } + if !field.CanSet() || field.Kind() != reflect.Slice { + return fmt.Errorf("cannot set permissions of type %s", field.Type()) + } + + out := reflect.MakeSlice(field.Type(), len(permissions), len(permissions)) + for i, p := range permissions { + elem := out.Index(i) + elem.FieldByName("Level").SetString(p.level) + elem.FieldByName("UserName").SetString(p.userName) + elem.FieldByName("ServicePrincipalName").SetString(p.servicePrincipalName) + elem.FieldByName("GroupName").SetString(p.groupName) + } + field.Set(out) + return nil +} diff --git a/bundle/config/mutator/resourcemutator/keyed_merge.go b/bundle/config/mutator/resourcemutator/keyed_merge.go new file mode 100644 index 00000000000..baf184f01b5 --- /dev/null +++ b/bundle/config/mutator/resourcemutator/keyed_merge.go @@ -0,0 +1,80 @@ +package resourcemutator + +import ( + "slices" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" +) + +// mergeByKey merges, for every resource in resources (visited in sorted order), the sequence +// at resources... by key, skipping sequences for which the merge +// is a no-op (see [keyedMergeIsNoop]). The key of an element is noopKey(element), which must +// agree with keyFn applied to the key field of the element. +func mergeByKey[R, E any]( + b *bundle.Bundle, + resourceType string, + resources map[string]*R, + field, keyField string, + elems func(*R) []E, + noopKey func(E) string, + keyFn func(structvar.View) string, + sortKeys bool, +) error { + names := make([]string, 0, len(resources)) + for name := range resources { + names = append(names, name) + } + slices.Sort(names) + for _, name := range names { + r := resources[name] + if r == nil { + continue + } + if keyedMergeIsNoop(b, "resources."+resourceType+"."+name+"."+field, elems(r), noopKey, sortKeys) { + continue + } + path := structpath.NewStringKeys(nil, "resources", resourceType, name, field) + if err := b.Config.MergeElementsByKey(path, keyField, keyFn, sortKeys); err != nil { + return err + } + } + return nil +} + +// stringKey returns a key function for sequence elements whose key is a string +// (an absent key is the empty string). +func stringKey(what string) func(structvar.View) string { + return func(v structvar.View) string { + switch v.Kind() { + case structvar.KindInvalid, structvar.KindNil: + return "" + case structvar.KindString: + s, _ := v.AsString() + return s + default: + panic(what + " must be a string") + } + } +} + +// keyedMergeIsNoop reports whether merging the elements of the sequence at path by key +// leaves it unchanged: the sequence is not a reference, and every key is non-empty and +// unique (and sorted, if the merge sorts by key). The merge operates on the dynamic +// configuration, which is built from the whole typed configuration, so mutators skip +// it when it would have no effect. +func keyedMergeIsNoop[T any](b *bundle.Bundle, path string, elems []T, key func(T) string, sorted bool) bool { + if b.Config.IsReference(path) { + return false + } + keys := make([]string, 0, len(elems)) + for _, e := range elems { + k := key(e) + if k == "" || slices.Contains(keys, k) { + return false + } + keys = append(keys, k) + } + return !sorted || slices.IsSorted(keys) +} diff --git a/bundle/config/mutator/resourcemutator/merge_apps.go b/bundle/config/mutator/resourcemutator/merge_apps.go index edbdf7712ce..07396ce7537 100644 --- a/bundle/config/mutator/resourcemutator/merge_apps.go +++ b/bundle/config/mutator/resourcemutator/merge_apps.go @@ -2,11 +2,12 @@ package resourcemutator import ( "context" + "slices" "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/merge" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/databricks-sdk-go/service/apps" ) type mergeApps struct{} @@ -19,27 +20,52 @@ func (m *mergeApps) Name() string { return "MergeApps" } -func (m *mergeApps) resourceName(v dyn.Value) string { - switch v.Kind() { - case dyn.KindInvalid, dyn.KindNil: - return "" - case dyn.KindString: - return v.MustString() - default: - panic("app name must be a string") +func (m *mergeApps) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + names := make([]string, 0, len(b.Config.Resources.Apps)) + for name := range b.Config.Resources.Apps { + names = append(names, name) } -} + slices.Sort(names) -func (m *mergeApps) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { - if v.Kind() == dyn.KindNil { - return v, nil + for _, name := range names { + app := b.Config.Resources.Apps[name] + if app == nil { + continue + } + path := structpath.NewStringKeys(nil, "resources", "apps", name, "resources") + if keyedMergeIsNoop(b, "resources.apps."+name+".resources", app.Resources, func(r apps.AppResource) string { return r.Name }, false) { + continue } - return dyn.Map(v, "resources.apps", dyn.Foreach(func(_ dyn.Path, app dyn.Value) (dyn.Value, error) { - return dyn.Map(app, "resources", merge.ElementsByKeyWithOverride("name", m.resourceName)) - })) - }) + // Elements with the same name are overridden by the later ones. The result + // is ordered by the first appearance of the name. + var order []string + last := map[string]int{} + for i, r := range app.Resources { + if _, ok := last[r.Name]; !ok { + order = append(order, r.Name) + } + last[r.Name] = i + } - return diag.FromErr(err) + merged := make([]apps.AppResource, 0, len(order)) + sources := make([][]int, 0, len(order)) + for _, key := range order { + merged = append(merged, app.Resources[last[key]]) + sources = append(sources, []int{last[key]}) + } + app.Resources = merged + b.Config.UpdateSequence(path, sources) + + // The merge sets the key of every element, even if it is empty. + for i, r := range merged { + if r.Name != "" { + continue + } + if err := b.Config.Set(structpath.NewStringKey(structpath.NewIndex(path, i), "name"), ""); err != nil { + return diag.FromErr(err) + } + } + } + return nil } diff --git a/bundle/config/mutator/resourcemutator/merge_grants.go b/bundle/config/mutator/resourcemutator/merge_grants.go index 3966190a5ee..0702396a9df 100644 --- a/bundle/config/mutator/resourcemutator/merge_grants.go +++ b/bundle/config/mutator/resourcemutator/merge_grants.go @@ -2,24 +2,16 @@ package resourcemutator import ( "context" + "slices" "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/merge" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" + "github.com/databricks/databricks-sdk-go/service/catalog" ) -// Resource types that support grants. -var grantResourceTypes = []string{ - "catalogs", - "schemas", - "external_locations", - "secrets", - "volumes", - "registered_models", - "vector_search_indexes", -} - type mergeGrants struct{} // MergeGrants returns a mutator that deduplicates grant entries. @@ -33,58 +25,83 @@ func (m *mergeGrants) Name() string { } func (m *mergeGrants) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { - if v.Kind() == dyn.KindNil { - return v, nil - } - - for _, resourceType := range grantResourceTypes { - var mapErr error - v, mapErr = dyn.Map(v, "resources."+resourceType, dyn.Foreach(func(_ dyn.Path, resource dyn.Value) (dyn.Value, error) { - // Merge grant entries by principal. This concatenates privileges - // for entries with the same principal via the standard merge semantics. - resource, err := dyn.Map(resource, "grants", merge.ElementsByKey("principal", func(v dyn.Value) string { - s, _ := v.AsString() - return s - })) - if err != nil { - return resource, err - } - - // Deduplicate privileges within each grant entry. - return dyn.Map(resource, "grants", dyn.Foreach(func(_ dyn.Path, grant dyn.Value) (dyn.Value, error) { - return dyn.Map(grant, "privileges", deduplicateSequence) - })) - })) - if mapErr != nil { - return v, mapErr - } - } - - return v, nil - }) + r := &b.Config.Resources + // Resource types that support grants. + err := mergeGrantsOf(b, "catalogs", r.Catalogs, func(x *resources.Catalog) []catalog.PrivilegeAssignment { return x.Grants }) + if err == nil { + err = mergeGrantsOf(b, "schemas", r.Schemas, func(x *resources.Schema) []catalog.PrivilegeAssignment { return x.Grants }) + } + if err == nil { + err = mergeGrantsOf(b, "external_locations", r.ExternalLocations, func(x *resources.ExternalLocation) []catalog.PrivilegeAssignment { return x.Grants }) + } + if err == nil { + err = mergeGrantsOf(b, "secrets", r.Secrets, func(x *resources.Secret) []catalog.PrivilegeAssignment { return x.Grants }) + } + if err == nil { + err = mergeGrantsOf(b, "volumes", r.Volumes, func(x *resources.Volume) []catalog.PrivilegeAssignment { return x.Grants }) + } + if err == nil { + err = mergeGrantsOf(b, "registered_models", r.RegisteredModels, func(x *resources.RegisteredModel) []catalog.PrivilegeAssignment { return x.Grants }) + } + if err == nil { + err = mergeGrantsOf(b, "vector_search_indexes", r.VectorSearchIndexes, func(x *resources.VectorSearchIndex) []catalog.PrivilegeAssignment { return x.Grants }) + } return diag.FromErr(err) } -// deduplicateSequence removes duplicate values from a dyn sequence, -// preserving the order of first appearance. -func deduplicateSequence(_ dyn.Path, v dyn.Value) (dyn.Value, error) { - elements, ok := v.AsSequence() - if !ok { - return v, nil +// mergeGrantsOf merges the grants of every resource of the given type (visited in sorted +// order) by principal, and deduplicates the privileges of each grant. +func mergeGrantsOf[R any](b *bundle.Bundle, resourceType string, rs map[string]*R, grants func(*R) []catalog.PrivilegeAssignment) error { + names := make([]string, 0, len(rs)) + for name := range rs { + names = append(names, name) } + slices.Sort(names) - seen := make(map[string]bool, len(elements)) - out := make([]dyn.Value, 0, len(elements)) - for _, elem := range elements { - key, _ := elem.AsString() - if seen[key] { + for _, name := range names { + if rs[name] == nil || len(grants(rs[name])) == 0 { continue } - seen[key] = true - out = append(out, elem) + + // Merge grant entries by principal. This concatenates privileges + // for entries with the same principal via the standard merge semantics. + path := structpath.NewStringKeys(nil, "resources", resourceType, name, "grants") + err := b.Config.MergeElementsByKey(path, "principal", func(v structvar.View) string { + s, _ := v.AsString() + return s + }, false) + if err != nil { + return err + } + + // Deduplicate privileges within each grant entry. + gs := grants(rs[name]) + for i := range gs { + privileges, sources := deduplicatePrivileges(gs[i].Privileges) + if len(privileges) == len(gs[i].Privileges) { + continue + } + gs[i].Privileges = privileges + b.Config.UpdateSequence(structpath.NewStringKey(structpath.NewIndex(path, i), "privileges"), sources) + } } + return nil +} - return dyn.NewValue(out, v.Locations()), nil +// deduplicatePrivileges removes duplicate privileges, preserving the order of first +// appearance. It also returns, for every remaining privilege, its index in the input. +func deduplicatePrivileges(in []catalog.Privilege) ([]catalog.Privilege, [][]int) { + seen := make(map[catalog.Privilege]bool, len(in)) + out := make([]catalog.Privilege, 0, len(in)) + sources := make([][]int, 0, len(in)) + for i, p := range in { + if seen[p] { + continue + } + seen[p] = true + out = append(out, p) + sources = append(sources, []int{i}) + } + return out, sources } diff --git a/bundle/config/mutator/resourcemutator/merge_job_clusters.go b/bundle/config/mutator/resourcemutator/merge_job_clusters.go index c7c46a3fdef..9d63460dc50 100644 --- a/bundle/config/mutator/resourcemutator/merge_job_clusters.go +++ b/bundle/config/mutator/resourcemutator/merge_job_clusters.go @@ -4,9 +4,9 @@ import ( "context" "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/merge" + "github.com/databricks/databricks-sdk-go/service/jobs" ) type mergeJobClusters struct{} @@ -19,27 +19,10 @@ func (m *mergeJobClusters) Name() string { return "MergeJobClusters" } -func (m *mergeJobClusters) jobClusterKey(v dyn.Value) string { - switch v.Kind() { - case dyn.KindInvalid, dyn.KindNil: - return "" - case dyn.KindString: - return v.MustString() - default: - panic("job cluster key must be a string") - } -} - func (m *mergeJobClusters) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { - if v.Kind() == dyn.KindNil { - return v, nil - } - - return dyn.Map(v, "resources.jobs", dyn.Foreach(func(_ dyn.Path, job dyn.Value) (dyn.Value, error) { - return dyn.Map(job, "job_clusters", merge.ElementsByKey("job_cluster_key", m.jobClusterKey)) - })) - }) - + err := mergeByKey(b, "jobs", b.Config.Resources.Jobs, "job_clusters", "job_cluster_key", + func(r *resources.Job) []jobs.JobCluster { return r.JobClusters }, + func(c jobs.JobCluster) string { return c.JobClusterKey }, + stringKey("job cluster key"), false) return diag.FromErr(err) } diff --git a/bundle/config/mutator/resourcemutator/merge_job_parameters.go b/bundle/config/mutator/resourcemutator/merge_job_parameters.go index 79458b36a60..8b1413e4bf6 100644 --- a/bundle/config/mutator/resourcemutator/merge_job_parameters.go +++ b/bundle/config/mutator/resourcemutator/merge_job_parameters.go @@ -4,9 +4,9 @@ import ( "context" "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/merge" + "github.com/databricks/databricks-sdk-go/service/jobs" ) type mergeJobParameters struct{} @@ -19,27 +19,10 @@ func (m *mergeJobParameters) Name() string { return "MergeJobParameters" } -func (m *mergeJobParameters) parameterNameString(v dyn.Value) string { - switch v.Kind() { - case dyn.KindInvalid, dyn.KindNil: - return "" - case dyn.KindString: - return v.MustString() - default: - panic("task key must be a string") - } -} - func (m *mergeJobParameters) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { - if v.Kind() == dyn.KindNil { - return v, nil - } - - return dyn.Map(v, "resources.jobs", dyn.Foreach(func(_ dyn.Path, job dyn.Value) (dyn.Value, error) { - return dyn.Map(job, "parameters", merge.ElementsByKey("name", m.parameterNameString)) - })) - }) - + err := mergeByKey(b, "jobs", b.Config.Resources.Jobs, "parameters", "name", + func(r *resources.Job) []jobs.JobParameterDefinition { return r.Parameters }, + func(p jobs.JobParameterDefinition) string { return p.Name }, + stringKey("parameter name"), false) return diag.FromErr(err) } diff --git a/bundle/config/mutator/resourcemutator/merge_job_tasks.go b/bundle/config/mutator/resourcemutator/merge_job_tasks.go index b85a863d80f..562157a46a6 100644 --- a/bundle/config/mutator/resourcemutator/merge_job_tasks.go +++ b/bundle/config/mutator/resourcemutator/merge_job_tasks.go @@ -4,9 +4,9 @@ import ( "context" "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/merge" + "github.com/databricks/databricks-sdk-go/service/jobs" ) type mergeJobTasks struct{} @@ -19,31 +19,14 @@ func (m *mergeJobTasks) Name() string { return "MergeJobTasks" } -func (m *mergeJobTasks) taskKeyString(v dyn.Value) string { - switch v.Kind() { - case dyn.KindInvalid, dyn.KindNil: - return "" - case dyn.KindString: - return v.MustString() - default: - panic("task key must be a string") - } -} - func (m *mergeJobTasks) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { - if v.Kind() == dyn.KindNil { - return v, nil - } - - return dyn.Map(v, "resources.jobs", dyn.Foreach(func(_ dyn.Path, job dyn.Value) (dyn.Value, error) { - // Sorting keys here since it'll be sorted by TF anyway - // https://github.com/databricks/terraform-provider-databricks/blob/0a932c2/jobs/resource_job.go#L343 - // However, if we don't sort we have a difference between direct and TF and between configs in - // "bundle validate" and configs sent to backend. - return dyn.Map(job, "tasks", merge.ElementsBySortedKey("task_key", m.taskKeyString)) - })) - }) - + // Sorting keys here since it'll be sorted by TF anyway + // https://github.com/databricks/terraform-provider-databricks/blob/0a932c2/jobs/resource_job.go#L343 + // However, if we don't sort we have a difference between direct and TF and between configs in + // "bundle validate" and configs sent to backend. + err := mergeByKey(b, "jobs", b.Config.Resources.Jobs, "tasks", "task_key", + func(r *resources.Job) []jobs.Task { return r.Tasks }, + func(t jobs.Task) string { return t.TaskKey }, + stringKey("task key"), true) return diag.FromErr(err) } diff --git a/bundle/config/mutator/resourcemutator/merge_pipeline_clusters.go b/bundle/config/mutator/resourcemutator/merge_pipeline_clusters.go index 6496a00d90e..622f15bfec4 100644 --- a/bundle/config/mutator/resourcemutator/merge_pipeline_clusters.go +++ b/bundle/config/mutator/resourcemutator/merge_pipeline_clusters.go @@ -5,9 +5,10 @@ import ( "strings" "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/merge" + "github.com/databricks/cli/libs/structs/structvar" + "github.com/databricks/databricks-sdk-go/service/pipelines" ) type mergePipelineClusters struct{} @@ -20,29 +21,31 @@ func (m *mergePipelineClusters) Name() string { return "MergePipelineClusters" } -func (m *mergePipelineClusters) clusterLabel(v dyn.Value) string { +func (m *mergePipelineClusters) clusterLabel(v structvar.View) string { switch v.Kind() { - case dyn.KindInvalid, dyn.KindNil: + case structvar.KindInvalid, structvar.KindNil: // Note: the cluster label is optional and defaults to 'default'. // We therefore ALSO merge all clusters without a label. return "default" - case dyn.KindString: - return strings.ToLower(v.MustString()) + case structvar.KindString: + s, _ := v.AsString() + return strings.ToLower(s) default: panic("task key must be a string") } } func (m *mergePipelineClusters) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { - if v.Kind() == dyn.KindNil { - return v, nil - } - - return dyn.Map(v, "resources.pipelines", dyn.Foreach(func(_ dyn.Path, pipeline dyn.Value) (dyn.Value, error) { - return dyn.Map(pipeline, "clusters", merge.ElementsByKey("label", m.clusterLabel)) - })) - }) - + // The merge also lowercases labels and sets the "default" label, so only + // lowercase labels are left unchanged. + err := mergeByKey(b, "pipelines", b.Config.Resources.Pipelines, "clusters", "label", + func(r *resources.Pipeline) []pipelines.PipelineCluster { return r.Clusters }, + func(c pipelines.PipelineCluster) string { + if c.Label != strings.ToLower(c.Label) { + return "" + } + return c.Label + }, + m.clusterLabel, false) return diag.FromErr(err) } diff --git a/bundle/config/mutator/resourcemutator/process_static_resources.go b/bundle/config/mutator/resourcemutator/process_static_resources.go index b1f689808f6..7998517328b 100644 --- a/bundle/config/mutator/resourcemutator/process_static_resources.go +++ b/bundle/config/mutator/resourcemutator/process_static_resources.go @@ -2,12 +2,10 @@ package resourcemutator import ( "context" - "fmt" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/logdiag" ) @@ -64,18 +62,10 @@ func (p processStaticResources) Apply(ctx context.Context, b *bundle.Bundle) dia func getAllResources(b *bundle.Bundle) (ResourceKeySet, error) { set := NewResourceKeySet() - err := b.Config.Mutate(func(root dyn.Value) (dyn.Value, error) { - pattern := dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey()) - err := set.AddPattern(pattern, root) - if err != nil { - return dyn.InvalidValue, err + for _, group := range b.Config.Resources.AllResources() { + for name := range group.Resources { + set.AddResourceKey(ResourceKey{Type: group.Description.PluralName, Name: name}) } - - return root, nil - }) - if err != nil { - return nil, fmt.Errorf("failed to collect resources: %s", err) } - return set, nil } diff --git a/bundle/config/mutator/resourcemutator/resource_key.go b/bundle/config/mutator/resourcemutator/resource_key.go index 699867b4f40..f5fe19deddb 100644 --- a/bundle/config/mutator/resourcemutator/resource_key.go +++ b/bundle/config/mutator/resourcemutator/resource_key.go @@ -3,7 +3,8 @@ package resourcemutator import ( "errors" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) // ResourceKey uniquely identifies a resource in configuration. @@ -48,23 +49,21 @@ func (r ResourceKeySet) Size() int { } // AddPattern adds all resource keys that match the pattern. -func (r ResourceKeySet) AddPattern(pattern dyn.Pattern, root dyn.Value) error { - if len(pattern) != 3 { +func (r ResourceKeySet) AddPattern(pattern *structpath.PatternNode, root structvar.View) error { + if pattern.Len() != 3 { return errors.New("pattern must have 3 keys") } - _, err := dyn.MapByPattern(root, pattern, func(path dyn.Path, v dyn.Value) (dyn.Value, error) { + return structvar.ForEach(root, pattern, func(path *structpath.PathNode, v structvar.View) error { parsed, err := getResourceKey(path) if err != nil { - return dyn.InvalidValue, err + return err } r.AddResourceKey(parsed) - return v, nil + return nil }) - - return err } // Types returns the types of all resources in the set. @@ -105,17 +104,13 @@ func (r ResourceKeySet) ToArray() []ResourceKey { return result } -func getResourceKey(path dyn.Path) (ResourceKey, error) { - if len(path) < 3 { - return ResourceKey{}, errors.New("can't parse resource key") - } - - if path[0].Key() != "resources" { +func getResourceKey(path *structpath.PathNode) (ResourceKey, error) { + if path.KeyAt(0) != "resources" { return ResourceKey{}, errors.New("can't parse resource key") } - resourceType := path[1].Key() - resourceName := path[2].Key() + resourceType := path.KeyAt(1) + resourceName := path.KeyAt(2) if resourceType == "" || resourceName == "" { return ResourceKey{}, errors.New("can't parse resource key") diff --git a/bundle/config/mutator/resourcemutator/resource_key_test.go b/bundle/config/mutator/resourcemutator/resource_key_test.go index 1af72a9ba2d..9722359a236 100644 --- a/bundle/config/mutator/resourcemutator/resource_key_test.go +++ b/bundle/config/mutator/resourcemutator/resource_key_test.go @@ -3,9 +3,12 @@ package resourcemutator import ( "testing" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" + "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/stretchr/testify/assert" - - "github.com/databricks/cli/libs/dyn" ) type getResourceKeyTestCase struct { @@ -38,7 +41,7 @@ func TestGetResourceKey(t *testing.T) { for _, tc := range testCases { t.Run(tc.path, func(t *testing.T) { - key, err := getResourceKey(dyn.MustPathFromString(tc.path)) + key, err := getResourceKey(structpath.MustParsePath(tc.path)) if tc.err { assert.Error(t, err) } else { @@ -51,29 +54,23 @@ func TestGetResourceKey(t *testing.T) { type resourceKeySetAddTestCase struct { name string - pattern dyn.Pattern - root dyn.Value + pattern *structpath.PatternNode + root structvar.View expected []ResourceKey } func TestResourceKeySet_AddPattern(t *testing.T) { - root := dyn.V(map[string]dyn.Value{ - "resources": dyn.V(map[string]dyn.Value{ - "jobs": dyn.V(map[string]dyn.Value{ - "job_1": dyn.V(map[string]dyn.Value{ - "name": dyn.V("job_1"), - }), - "job_2": dyn.V(map[string]dyn.Value{ - "name": dyn.V("job_2"), - }), - }), - }), - }) + var cfg config.Root + cfg.Resources.Jobs = map[string]*resources.Job{ + "job_1": {JobSettings: jobs.JobSettings{Name: "job_1"}}, + "job_2": {JobSettings: jobs.JobSettings{Name: "job_2"}}, + } + root := cfg.View() testCases := []resourceKeySetAddTestCase{ { name: "one job pattern", - pattern: dyn.NewPattern(dyn.Key("resources"), dyn.Key("jobs"), dyn.Key("job_1")), + pattern: structpath.MustParsePattern("resources.jobs.job_1"), root: root, expected: []ResourceKey{ { @@ -84,7 +81,7 @@ func TestResourceKeySet_AddPattern(t *testing.T) { }, { name: "all resources pattern", - pattern: dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey()), + pattern: structpath.MustParsePattern("resources.*.*"), root: root, expected: []ResourceKey{ { diff --git a/bundle/config/mutator/resourcemutator/resource_mutator.go b/bundle/config/mutator/resourcemutator/resource_mutator.go index 4d0b3bd47ab..768f1e875f4 100644 --- a/bundle/config/mutator/resourcemutator/resource_mutator.go +++ b/bundle/config/mutator/resourcemutator/resource_mutator.go @@ -2,17 +2,16 @@ package resourcemutator import ( "context" - "errors" "fmt" + "reflect" + "strings" + "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/bundle/config/validate" - - "github.com/databricks/cli/libs/dyn/merge" "github.com/databricks/cli/libs/logdiag" - - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/databricks-sdk-go/service/jobs" ) // When a new resource is added to configuration, we apply bundle @@ -46,68 +45,61 @@ func applyInitializeMutators(ctx context.Context, b *bundle.Bundle) { return } - defaults := []struct { - pattern string - value any - }{ - {"resources.dashboards.*.parent_path", b.Config.Workspace.ResourcePath}, - {"resources.dashboards.*.embed_credentials", false}, - {"resources.genie_spaces.*.parent_path", b.Config.Workspace.ResourcePath}, - {"resources.volumes.*.volume_type", "MANAGED"}, + defaults := []bundle.Default{ + {Pattern: "resources.dashboards.*.parent_path", Value: b.Config.Workspace.ResourcePath}, + {Pattern: "resources.dashboards.*.embed_credentials", Value: false}, + {Pattern: "resources.genie_spaces.*.parent_path", Value: b.Config.Workspace.ResourcePath}, + {Pattern: "resources.volumes.*.volume_type", Value: "MANAGED"}, - {"resources.alerts.*.parent_path", b.Config.Workspace.ResourcePath}, + {Pattern: "resources.alerts.*.parent_path", Value: b.Config.Workspace.ResourcePath}, // Jobs: // The defaults are the same as for terraform provider latest version (v1.75.0) // https://github.com/databricks/terraform-provider-databricks/blob/v1.75.0/jobs/resource_job.go#L532 - {"resources.jobs.*.name", "Untitled"}, - {"resources.jobs.*.max_concurrent_runs", 1}, - {"resources.jobs.*.schedule.pause_status", "UNPAUSED"}, - {"resources.jobs.*.trigger.pause_status", "UNPAUSED"}, - {"resources.jobs.*.continuous.pause_status", "UNPAUSED"}, + {Pattern: "resources.jobs.*.name", Value: "Untitled"}, + {Pattern: "resources.jobs.*.max_concurrent_runs", Value: 1}, + {Pattern: "resources.jobs.*.schedule.pause_status", Value: "UNPAUSED"}, + {Pattern: "resources.jobs.*.trigger.pause_status", Value: "UNPAUSED"}, + {Pattern: "resources.jobs.*.continuous.pause_status", Value: "UNPAUSED"}, // Enable queueing for jobs by default, following the behavior from API 2.2+. // As of 2024-04, we're still using API 2.1 which has queueing disabled by default. - {"resources.jobs.*.queue", map[string]dyn.Value{ - "enabled": dyn.V(true), - }}, + {Pattern: "resources.jobs.*.queue", Value: jobs.QueueSettings{Enabled: true}}, // This is converted from single-task to multi-task - {"resources.jobs.*.task[*].dbt_task.schema", "default"}, - {"resources.jobs.*.task[*].for_each_task.task.dbt_task.schema", "default"}, + {Pattern: "resources.jobs.*.task[*].dbt_task.schema", Value: "default"}, + {Pattern: "resources.jobs.*.task[*].for_each_task.task.dbt_task.schema", Value: "default"}, // https://github.com/databricks/terraform-provider-databricks/blob/v1.75.0/clusters/resource_cluster.go - {"resources.jobs.*.job_clusters[*].new_cluster.workload_type.clients.notebooks", true}, - {"resources.jobs.*.job_clusters[*].new_cluster.workload_type.clients.jobs", true}, + {Pattern: "resources.jobs.*.job_clusters[*].new_cluster.workload_type.clients.notebooks", Value: true}, + {Pattern: "resources.jobs.*.job_clusters[*].new_cluster.workload_type.clients.jobs", Value: true}, // Pipelines (same as terraform) // https://github.com/databricks/terraform-provider-databricks/blob/v1.75.0/pipelines/resource_pipeline.go#L253 - {"resources.pipelines.*.edition", "ADVANCED"}, - {"resources.pipelines.*.channel", "CURRENT"}, + {Pattern: "resources.pipelines.*.edition", Value: "ADVANCED"}, + {Pattern: "resources.pipelines.*.channel", Value: "CURRENT"}, // SqlWarehouses (same as terraform) // https://github.com/databricks/terraform-provider-databricks/blob/v1.75.0/sql/resource_sql_endpoint.go#L59 - {"resources.sql_warehouses.*.auto_stop_mins", 120}, - {"resources.sql_warehouses.*.enable_photon", true}, - {"resources.sql_warehouses.*.max_num_clusters", 1}, - {"resources.sql_warehouses.*.spot_instance_policy", "COST_OPTIMIZED"}, + {Pattern: "resources.sql_warehouses.*.auto_stop_mins", Value: 120}, + {Pattern: "resources.sql_warehouses.*.enable_photon", Value: true}, + {Pattern: "resources.sql_warehouses.*.max_num_clusters", Value: 1}, + {Pattern: "resources.sql_warehouses.*.spot_instance_policy", Value: "COST_OPTIMIZED"}, // Apps: - {"resources.apps.*.description", ""}, + {Pattern: "resources.apps.*.description", Value: ""}, // Clusters (same as terraform) // https://github.com/databricks/terraform-provider-databricks/blob/v1.75.0/clusters/resource_cluster.go#L315 - {"resources.clusters.*.autotermination_minutes", 60}, - {"resources.clusters.*.workload_type.clients.notebooks", true}, - {"resources.clusters.*.workload_type.clients.jobs", true}, + {Pattern: "resources.clusters.*.autotermination_minutes", Value: 60}, + {Pattern: "resources.clusters.*.workload_type.clients.notebooks", Value: true}, + {Pattern: "resources.clusters.*.workload_type.clients.jobs", Value: true}, } - for _, defaultDef := range defaults { - bundle.SetDefault(ctx, b, defaultDef.pattern, defaultDef.value) - if logdiag.HasError(ctx) { - return - } + bundle.SetDefaults(ctx, b, defaults) + if logdiag.HasError(ctx) { + return } bundle.ApplySeqContext(ctx, b, @@ -224,17 +216,7 @@ func NormalizeAndInitializeResources( return } - var snapshot dyn.Value - - err := b.Config.Mutate(func(root dyn.Value) (dyn.Value, error) { - snapshot = root - - return selectResources(root, addedResources) - }) - if err != nil { - logdiag.LogError(ctx, fmt.Errorf("failed to select resources: %s", err)) - return - } + restore := selectResources(b, addedResources) applyNormalizeMutators(ctx, b) if logdiag.HasError(ctx) { @@ -246,12 +228,10 @@ func NormalizeAndInitializeResources( return } - // after mutators, we merge updated resources back to snapshot to preserve non-selected resources - err = b.Config.Mutate(func(root dyn.Value) (dyn.Value, error) { - return mergeResources(root, snapshot) - }) + // after mutators, we merge updated resources back to the snapshot to preserve non-selected resources + err := restore() if err != nil { - logdiag.LogError(ctx, fmt.Errorf("failed to merge resources: %s", err)) + logdiag.LogError(ctx, fmt.Errorf("failed to merge resources: %w", err)) } } @@ -266,17 +246,7 @@ func NormalizeResources( return } - var snapshot dyn.Value - - err := b.Config.Mutate(func(root dyn.Value) (dyn.Value, error) { - snapshot = root - - return selectResources(root, updatedResources) - }) - if err != nil { - logdiag.LogError(ctx, fmt.Errorf("failed to select resources: %s", err)) - return - } + restore := selectResources(b, updatedResources) applyNormalizeMutators(ctx, b) if logdiag.HasError(ctx) { @@ -294,129 +264,82 @@ func NormalizeResources( return } - // after mutators, we merge updated resources back to snapshot to preserve non-selected resources - err = b.Config.Mutate(func(root dyn.Value) (dyn.Value, error) { - return mergeResources(root, snapshot) - }) + // after mutators, we merge updated resources back to the snapshot to preserve non-selected resources + err := restore() if err != nil { - logdiag.LogError(ctx, fmt.Errorf("failed to merge resources: %s", err)) + logdiag.LogError(ctx, fmt.Errorf("failed to merge resources: %w", err)) } } -// selectResources returns bundle configuration with resources only present in resourcePaths. -func selectResources(root dyn.Value, resourcePaths ResourceKeySet) (dyn.Value, error) { - resourcesKeyString := "resources" - resourcesPath := dyn.NewPath(dyn.Key(resourcesKeyString)) - - newRoot := root - var err error - - // remove resource types that are not in resourcePaths - newRoot, err = dyn.MapByPath( - newRoot, - resourcesPath, - func(p dyn.Path, resources dyn.Value) (dyn.Value, error) { - return merge.Select(resources, resourcePaths.Types()) - }, - ) - if err != nil { - return dyn.InvalidValue, err - } - - // for each resource type, remove resources by name - for _, resourceType := range resourcePaths.Types() { - resourceTypePath := resourcesPath.Append(dyn.Key(resourceType)) - - newRoot, err = dyn.MapByPath( - newRoot, - resourceTypePath, - func(p dyn.Path, resource dyn.Value) (dyn.Value, error) { - return merge.Select(resource, resourcePaths.Names(resourceType)) - }, - ) - if err != nil { - return dyn.InvalidValue, err +// resourceTypeFields returns the fields of resources that hold the resources of each type, +// by the name of the type (e.g. "jobs"). +func resourceTypeFields(resources reflect.Value) map[string]reflect.Value { + fields := map[string]reflect.Value{} + t := resources.Type() + for i := range t.NumField() { + name, _, _ := strings.Cut(t.Field(i).Tag.Get("json"), ",") + if name == "" || name == "-" || resources.Field(i).Kind() != reflect.Map { + continue } + fields[name] = resources.Field(i) } - - return newRoot, err + return fields } -// mergeResources returns bundle configuration by merging all resources from src into dst, -// overriding existing resources if they exist. -func mergeResources(src, dst dyn.Value) (dyn.Value, error) { - resourcesKey := dyn.Key("resources") - - newDst := dst - - // merge 'resources..' - _, err := dyn.MapByPattern( - src, - dyn.NewPattern(resourcesKey, dyn.AnyKey(), dyn.AnyKey()), - func(path dyn.Path, v dyn.Value) (dyn.Value, error) { - // if parent 'resources.' doesn't exist, handle it on the next pass - updated, _ := dyn.SetByPath(newDst, path, v) - if !updated.IsValid() { - return v, nil - } else { - newDst = updated - } - - return v, nil - }, - ) - if err != nil { - return newDst, err - } - - // merge 'resources.' - _, err = dyn.MapByPattern( - src, - dyn.NewPattern(resourcesKey, dyn.AnyKey()), - func(path dyn.Path, v dyn.Value) (dyn.Value, error) { - // if already exists, we already handled it in the previous pass - existing, _ := dyn.GetByPath(newDst, path) - if existing.IsValid() { - return v, nil - } +// selectResources removes all resources except the ones in resourceKeys from the bundle +// configuration. It returns a function that restores the removed resources: it puts the +// configuration back to what it was before, except for the selected resources, which take +// the values they have at that point. Any other changes to the configuration are discarded. +func selectResources(b *bundle.Bundle, resourceKeys ResourceKeySet) func() error { + // A shallow copy of the configuration: the resources maps are replaced below, not + // modified, so the copy keeps all resources. + snapshot := b.Config + + for resourceType, field := range resourceTypeFields(reflect.ValueOf(&b.Config.Resources).Elem()) { + if _, ok := resourceKeys[resourceType]; !ok { + field.SetZero() + continue + } - // if parent 'resources' doesn't exist, handle it on the next pass - updated, _ := dyn.SetByPath(newDst, path, v) - if !updated.IsValid() { - return v, nil - } else { - newDst = updated - return v, nil + selected := reflect.MakeMap(field.Type()) + for _, name := range resourceKeys.Names(resourceType) { + v := field.MapIndex(reflect.ValueOf(name)) + if v.IsValid() { + selected.SetMapIndex(reflect.ValueOf(name), v) } - }, - ) - if err != nil { - return newDst, err + } + field.Set(selected) } - // merge 'resources' - _, err = dyn.MapByPattern( - src, - dyn.NewPattern(resourcesKey), - func(path dyn.Path, v dyn.Value) (dyn.Value, error) { - // if already exists, we already handled it in the previous pass - existing, _ := dyn.GetByPath(newDst, path) - if existing.IsValid() { - return v, nil - } - - updated, _ := dyn.SetByPath(newDst, path, v) - if !updated.IsValid() { - return v, errors.New("failed to update resources") - } else { - newDst = updated - return v, nil + return func() error { + updated := b.Config + updatedView := updated.View() + updatedFields := resourceTypeFields(reflect.ValueOf(&updated.Resources).Elem()) + + b.Config = snapshot + fields := resourceTypeFields(reflect.ValueOf(&b.Config.Resources).Elem()) + + for resourceType, names := range resourceKeys { + for name := range names { + v := updatedFields[resourceType].MapIndex(reflect.ValueOf(name)) + if !v.IsValid() { + continue + } + + path := structpath.NewStringKeys(nil, "resources", resourceType, name) + err := b.Config.Assign(path, updatedView.Lookup(path)) + if err != nil { + return err + } + + // Keep the resource that was updated, it may have more than the configuration describes. + field := fields[resourceType] + if field.IsNil() { + field.Set(reflect.MakeMap(field.Type())) + } + field.SetMapIndex(reflect.ValueOf(name), v) } - }, - ) - if err != nil { - return newDst, err + } + return nil } - - return newDst, nil } diff --git a/bundle/config/mutator/resourcemutator/resource_mutator_test.go b/bundle/config/mutator/resourcemutator/resource_mutator_test.go index 42aee5bf883..572b688946b 100644 --- a/bundle/config/mutator/resourcemutator/resource_mutator_test.go +++ b/bundle/config/mutator/resourcemutator/resource_mutator_test.go @@ -3,114 +3,77 @@ package resourcemutator import ( "testing" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/dynassert" + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) -type mergeResourcesTestCase struct { - name string - src dyn.Value - dst dyn.Value - expected dyn.Value +func newJob(name string) *resources.Job { + return &resources.Job{JobSettings: jobs.JobSettings{Name: name}} } -func TestMergeResources(t *testing.T) { - job1 := dyn.V("job_1") - job2 := dyn.V("job_2") - job3 := dyn.V("job_3") - - testCases := []mergeResourcesTestCase{ - { - name: "add resources to empty bundle (1)", - src: mapOf("resources", mapOf("jobs", mapOf("job_1", job1))), - dst: mapOf("resources", emptyMap()), - expected: mapOf("resources", mapOf("jobs", mapOf("job_1", job1))), - }, - { - name: "add resources to empty bundle (2)", - src: mapOf("resources", mapOf("jobs", mapOf("job_1", job1))), - dst: emptyMap(), - expected: mapOf("resources", mapOf("jobs", mapOf("job_1", job1))), - }, - { - name: "add new resource", - src: mapOf("resources", mapOf("jobs", mapOf("job_1", job1))), - dst: mapOf("resources", mapOf("jobs", mapOf("job_2", job2))), - expected: mapOf("resources", mapOf("jobs", mapOf2("job_1", job1, "job_2", job2))), - }, - { - name: "override resource", - src: mapOf("resources", mapOf("jobs", mapOf("job_1", job3))), - dst: mapOf("resources", mapOf("jobs", mapOf2("job_1", job1, "job_2", job2))), - expected: mapOf("resources", mapOf("jobs", mapOf2("job_1", job3, "job_2", job2))), +func TestSelectResources(t *testing.T) { + job1 := newJob("job_1") + job2 := newJob("job_2") + + b := &bundle.Bundle{Config: config.Root{ + Resources: config.Resources{ + Jobs: map[string]*resources.Job{"job_1": job1, "job_2": job2}, + Pipelines: map[string]*resources.Pipeline{"pipeline_1": {}}, }, - } + }} - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - actual, err := mergeResources(tc.src, tc.dst) + keys := NewResourceKeySet() + keys.AddResourceKey(ResourceKey{Type: "jobs", Name: "job_1"}) - assert.NoError(t, err) - dynassert.Equal(t, tc.expected, actual) - }) - } -} + restore := selectResources(b, keys) + assert.Equal(t, map[string]*resources.Job{"job_1": job1}, b.Config.Resources.Jobs) + assert.Nil(t, b.Config.Resources.Pipelines) -type selectResourcesTestCase struct { - name string - config dyn.Value - resourcePaths []ResourceKey - expected dyn.Value + require.NoError(t, restore()) + assert.Equal(t, map[string]*resources.Job{"job_1": job1, "job_2": job2}, b.Config.Resources.Jobs) + assert.Len(t, b.Config.Resources.Pipelines, 1) } -func TestSelectResources(t *testing.T) { - job1 := dyn.V("job_1") - job2 := dyn.V("job_2") - - testCases := []selectResourcesTestCase{ - { - name: "extract resources", - config: mapOf("resources", mapOf("jobs", mapOf2("job_1", job1, "job_2", job2))), - resourcePaths: []ResourceKey{ - { - Type: "jobs", - Name: "job_1", - }, - }, - expected: mapOf("resources", mapOf("jobs", mapOf("job_1", job1))), - }, - { - name: "extract no resources", - config: mapOf("resources", mapOf("jobs", mapOf2("job_1", job1, "job_2", job2))), - resourcePaths: []ResourceKey{}, - expected: mapOf("resources", emptyMap()), +func TestSelectResourcesRestoreMergesUpdates(t *testing.T) { + b := &bundle.Bundle{Config: config.Root{ + Resources: config.Resources{ + Jobs: map[string]*resources.Job{"job_1": newJob("job_1"), "job_2": newJob("job_2")}, }, - } + }} + b.Config.Bundle.Name = "before" - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - resourceKeySet := NewResourceKeySet() - for _, key := range tc.resourcePaths { - resourceKeySet.AddResourceKey(key) - } + keys := NewResourceKeySet() + keys.AddResourceKey(ResourceKey{Type: "jobs", Name: "job_1"}) - actual, err := selectResources(tc.config, resourceKeySet) + restore := selectResources(b, keys) - assert.NoError(t, err) - assert.Equal(t, tc.expected, actual) - }) - } -} + // Changes to selected resources are kept, other changes are discarded. + require.NoError(t, b.Config.Set(structpath.MustParsePath("resources.jobs.job_1.name"), "updated")) + b.Config.Bundle.Name = "after" -func mapOf(key string, value dyn.Value) dyn.Value { - return dyn.V(map[string]dyn.Value{key: value}) + require.NoError(t, restore()) + assert.Equal(t, "updated", b.Config.Resources.Jobs["job_1"].Name) + assert.Equal(t, "job_2", b.Config.Resources.Jobs["job_2"].Name) + assert.Equal(t, "before", b.Config.Bundle.Name) } -func mapOf2(k0 string, v0 dyn.Value, k1 string, v1 dyn.Value) dyn.Value { - return dyn.V(map[string]dyn.Value{k0: v0, k1: v1}) -} +func TestSelectResourcesRestoreKeepsResourceType(t *testing.T) { + b := &bundle.Bundle{Config: config.Root{ + Resources: config.Resources{ + Jobs: map[string]*resources.Job{"job_1": newJob("job_1")}, + }, + }} + + keys := NewResourceKeySet() + keys.AddResourceKey(ResourceKey{Type: "jobs", Name: "job_1"}) -func emptyMap() dyn.Value { - return dyn.V(map[string]dyn.Value{}) + restore := selectResources(b, keys) + require.NoError(t, restore()) + assert.Equal(t, "job_1", b.Config.Resources.Jobs["job_1"].Name) + assert.Nil(t, b.Config.Resources.Pipelines) } diff --git a/bundle/config/mutator/resourcemutator/run_as.go b/bundle/config/mutator/resourcemutator/run_as.go index 65279c7f212..21751e32813 100644 --- a/bundle/config/mutator/resourcemutator/run_as.go +++ b/bundle/config/mutator/resourcemutator/run_as.go @@ -8,7 +8,6 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/databricks/databricks-sdk-go/service/pipelines" @@ -220,17 +219,17 @@ func (m *setRunAs) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { b.Metrics.AddBoolValue("experimental.use_legacy_run_as", b.Config.Experimental != nil && b.Config.Experimental.UseLegacyRunAs) // Track whether top level run_as is set. - b.Metrics.AddBoolValue("run_as_set", b.Config.Value().Get("run_as").Kind() != dyn.KindInvalid) + b.Metrics.AddBoolValue("run_as_set", b.Config.RunAs != nil) // Mutator is a no-op if run_as is not specified in the bundle - if b.Config.Value().Get("run_as").Kind() == dyn.KindInvalid { + if b.Config.RunAs == nil { return nil } // User has opted to use the legacy behavior of run_as with the // experimental.use_legacy_run_as flag. if b.Config.Experimental != nil && b.Config.Experimental.UseLegacyRunAs { - if b.Config.Value().Get("run_as").Get("group_name").Kind() != dyn.KindInvalid { + if b.Config.RunAs.GroupName != "" || slices.Contains(b.Config.RunAs.ForceSendFields, "GroupName") { return diag.Errorf("run_as.group_name is not supported with experimental.use_legacy_run_as; disable experimental.use_legacy_run_as to use a group identity") } setPipelineOwnersToRunAsIdentity(b) diff --git a/bundle/config/mutator/resourcemutator/run_as_test.go b/bundle/config/mutator/resourcemutator/run_as_test.go index 5d1aa602d02..1be8d37957c 100644 --- a/bundle/config/mutator/resourcemutator/run_as_test.go +++ b/bundle/config/mutator/resourcemutator/run_as_test.go @@ -11,8 +11,6 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/convert" "github.com/databricks/databricks-sdk-go/service/iam" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/databricks/databricks-sdk-go/service/pipelines" @@ -23,18 +21,15 @@ import ( func allResourceTypes(t *testing.T) []string { // Compute supported resource types based on the `Resources{}` struct. - r := &config.Resources{} - rv, err := convert.FromTyped(r, dyn.NilValue) - require.NoError(t, err) - normalized, _ := convert.Normalize(r, rv, convert.IncludeMissingFields) var resourceTypes []string - for _, k := range normalized.MustMap().Keys() { - resourceTypes = append(resourceTypes, k.MustString()) + for f := range reflect.TypeFor[config.Resources]().Fields() { + name, _, _ := strings.Cut(f.Tag.Get("json"), ",") + resourceTypes = append(resourceTypes, name) } slices.Sort(resourceTypes) // Assert the total list of resource supported, as a sanity check that using - // the dyn library gives us the correct list of all resources supported. Please + // the struct gives us the correct list of all resources supported. Please // also update this check when adding a new resource require.Equal( t, []string{ @@ -298,14 +293,6 @@ func TestRunAsErrorForUnsupportedResources(t *testing.T) { }, } - v, err := convert.FromTyped(base, dyn.NilValue) - require.NoError(t, err) - - // Define top level resources key in the bundle configuration. - // This is not part of the typed configuration, so we need to add it manually. - v, err = dyn.Set(v, "resources", dyn.V(map[string]dyn.Value{})) - require.NoError(t, err) - for _, rt := range allResourceTypes(t) { // Skip allowed resources if slices.Contains(allowList, rt) { @@ -314,23 +301,16 @@ func TestRunAsErrorForUnsupportedResources(t *testing.T) { // Add an instance of the resource type that is not on the allow list to // the bundle configuration. - nv, err := dyn.SetByPath(v, dyn.NewPath(dyn.Key("resources"), dyn.Key(rt)), dyn.V(map[string]dyn.Value{ - "foo": dyn.V(map[string]dyn.Value{ - "path": dyn.V("bar"), - }), - })) - require.NoError(t, err) - - // Get back typed configuration from the newly created invalid bundle configuration. - r := &config.Root{} - err = convert.ToTyped(r, nv) - require.NoError(t, err) + r, diags := config.LoadFromBytes("databricks.yml", fmt.Appendf(nil, "resources:\n %s:\n foo:\n path: bar\n", rt)) + require.NoError(t, diags.Error()) + r.Workspace = base.Workspace + r.RunAs = base.RunAs // Assert this invalid bundle configuration fails validation. b := &bundle.Bundle{ Config: *r, } - diags := bundle.Apply(t.Context(), b, SetRunAs()) + diags = bundle.Apply(t.Context(), b, SetRunAs()) require.Error(t, diags.Error()) assert.Contains(t, diags.Error().Error(), "do not support a setting a run_as user that is different from the owner.\n"+ "Current identity: alice. Run as identity: bob.\n"+ @@ -352,14 +332,6 @@ func TestRunAsNoErrorForSupportedResources(t *testing.T) { }, } - v, err := convert.FromTyped(base, dyn.NilValue) - require.NoError(t, err) - - // Define top level resources key in the bundle configuration. - // This is not part of the typed configuration, so we need to add it manually. - v, err = dyn.Set(v, "resources", dyn.V(map[string]dyn.Value{})) - require.NoError(t, err) - for _, rt := range allResourceTypes(t) { // Skip unsupported resources if !slices.Contains(allowList, rt) { @@ -368,23 +340,16 @@ func TestRunAsNoErrorForSupportedResources(t *testing.T) { // Add an instance of the resource type that is not on the allow list to // the bundle configuration. - nv, err := dyn.SetByPath(v, dyn.NewPath(dyn.Key("resources"), dyn.Key(rt)), dyn.V(map[string]dyn.Value{ - "foo": dyn.V(map[string]dyn.Value{ - "name": dyn.V("bar"), - }), - })) - require.NoError(t, err) - - // Get back typed configuration from the newly created invalid bundle configuration. - r := &config.Root{} - err = convert.ToTyped(r, nv) - require.NoError(t, err) + r, diags := config.LoadFromBytes("databricks.yml", fmt.Appendf(nil, "resources:\n %s:\n foo:\n name: bar\n", rt)) + require.NoError(t, diags.Error()) + r.Workspace = base.Workspace + r.RunAs = base.RunAs // Assert this configuration passes validation. b := &bundle.Bundle{ Config: *r, } - diags := bundle.Apply(t.Context(), b, SetRunAs()) + diags = bundle.Apply(t.Context(), b, SetRunAs()) require.NoError(t, diags.Error()) } } @@ -394,7 +359,8 @@ func TestRunAsIdentities(t *testing.T) { runAs string wantError bool }{ - {`null`, true}, + // A null run_as is the same as not specifying it. + {`null`, false}, {`{}`, true}, {`{user_name: ""}`, true}, {`{service_principal_name: ""}`, true}, @@ -453,14 +419,14 @@ func TestRunAsGroupResources(t *testing.T) { r, diags := config.LoadFromBytes("databricks.yml", []byte(yaml)) require.NoError(t, diags.Error()) b := &bundle.Bundle{Config: *r} - before := b.Config.Value().Get("resources") + before := b.Config.View().Get("resources").AsAny() diags = bundle.Apply(t.Context(), b, SetRunAs()) if tc.wantError != "" { require.Error(t, diags.Error()) assert.Contains(t, diags.Error().Error(), tc.wantError) } else { require.NoError(t, diags.Error()) - assert.Equal(t, before, b.Config.Value().Get("resources")) + assert.Equal(t, before, b.Config.View().Get("resources").AsAny()) } }) } diff --git a/bundle/config/mutator/resourcemutator/secret_scope_fixups.go b/bundle/config/mutator/resourcemutator/secret_scope_fixups.go index 8404d2444eb..3c7b4a033c3 100644 --- a/bundle/config/mutator/resourcemutator/secret_scope_fixups.go +++ b/bundle/config/mutator/resourcemutator/secret_scope_fixups.go @@ -131,6 +131,7 @@ func (m *secretScopeFixups) Apply(ctx context.Context, b *bundle.Bundle) diag.Di currentUser := b.Config.Workspace.CurrentUser.User addManageForCurrentUser(scope, currentUser) + path := structpath.NewStringKeys(nil, "resources", "secret_scopes", key) err := collapsePermissions(scope) if err != nil { return diag.Diagnostics{ @@ -138,8 +139,8 @@ func (m *secretScopeFixups) Apply(ctx context.Context, b *bundle.Bundle) diag.Di Severity: diag.Error, Summary: "Failed to collapse permissions for secret scope", Detail: err.Error(), - Paths: structpath.NewPathSlice("resources", "secret_scopes", key), - Locations: []diag.Location{b.Config.GetLocationOf(structpath.NewPath(nil, "resources", "secret_scopes", key))}, + Paths: []*structpath.PathNode{path}, + Locations: []diag.Location{b.Config.GetLocation(path.String())}, }, } } diff --git a/bundle/config/mutator/resourcemutator/validate_target_mode.go b/bundle/config/mutator/resourcemutator/validate_target_mode.go index bf6386264ad..c3832ceb090 100644 --- a/bundle/config/mutator/resourcemutator/validate_target_mode.go +++ b/bundle/config/mutator/resourcemutator/validate_target_mode.go @@ -5,12 +5,11 @@ import ( "fmt" "strings" - "github.com/databricks/cli/libs/iamutil" - "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/iamutil" ) type validateTargetMode struct{} diff --git a/bundle/config/mutator/rewrite_sync_paths.go b/bundle/config/mutator/rewrite_sync_paths.go index 228c5484def..c4eefebae2e 100644 --- a/bundle/config/mutator/rewrite_sync_paths.go +++ b/bundle/config/mutator/rewrite_sync_paths.go @@ -8,7 +8,8 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) type rewriteSyncPaths struct{} @@ -21,9 +22,8 @@ func (m *rewriteSyncPaths) Name() string { return "RewriteSyncPaths" } -// makeRelativeTo returns a dyn.MapFunc that joins the relative path -// of the file it was defined in w.r.t. the bundle root path, with -// the contents of the string node. +// makeRelativeTo joins the relative path of the file the string node was +// defined in w.r.t. the bundle root path, with the contents of the string node. // // For example: // - The bundle root is /foo @@ -31,63 +31,46 @@ func (m *rewriteSyncPaths) Name() string { // - The string node contains "somefile.*" // // Then the resulting value will be "bar/somefile.*". -func (m *rewriteSyncPaths) makeRelativeTo(root string) dyn.MapFunc { - return func(_ dyn.Path, v dyn.Value) (dyn.Value, error) { - dir := filepath.Dir(v.Location().File) - rel, err := filepath.Rel(root, dir) - if err != nil { - return dyn.InvalidValue, err - } +func (m *rewriteSyncPaths) makeRelativeTo(root string, v structvar.View) (string, error) { + dir := filepath.Dir(v.Location().File) + rel, err := filepath.Rel(root, dir) + if err != nil { + return "", err + } - return dyn.NewValue(filepath.Join(rel, v.MustString()), v.Locations()), nil + s, ok := v.AsString() + if !ok { + return "", fmt.Errorf("expected string value but got %s", v.Kind()) } + + return filepath.Join(rel, s), nil } func (m *rewriteSyncPaths) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { - return dyn.Map(v, "sync", func(_ dyn.Path, v dyn.Value) (nv dyn.Value, err error) { - v, err = dyn.Map(v, "paths", dyn.Foreach(m.makeRelativeTo(b.BundleRootPath))) - if err != nil { - return dyn.InvalidValue, err - } - - makeRelativeFn := m.makeRelativeTo(b.BundleRootPath) - - // Makes include and exclude paths relative to the bundle root first. - // Then converts them to use Unix-style slashes. - // This is required for the ignore.GitIgnore we use in libs/fileset to work correctly. - v, err = dyn.Map(v, "include", dyn.Foreach(func(p dyn.Path, val dyn.Value) (dyn.Value, error) { - relPath, err := makeRelativeFn(p, val) - if err != nil { - return dyn.InvalidValue, err - } - str, ok := relPath.AsString() - if !ok { - return dyn.InvalidValue, fmt.Errorf("expected string value but got %s", relPath.Kind()) - } - return dyn.NewValue(filepath.ToSlash(str), relPath.Locations()), nil - })) + rewrite := func(field string, toSlash bool) error { + pattern := structpath.MustParsePattern("sync." + field + "[*]") + return structvar.ForEach(b.Config.View(), pattern, func(p *structpath.PathNode, v structvar.View) error { + path, err := m.makeRelativeTo(b.BundleRootPath, v) if err != nil { - return dyn.InvalidValue, err + return err } - - v, err = dyn.Map(v, "exclude", dyn.Foreach(func(p dyn.Path, val dyn.Value) (dyn.Value, error) { - relPath, err := makeRelativeFn(p, val) - if err != nil { - return dyn.InvalidValue, err - } - str, ok := relPath.AsString() - if !ok { - return dyn.InvalidValue, fmt.Errorf("expected string value but got %s", relPath.Kind()) - } - return dyn.NewValue(filepath.ToSlash(str), relPath.Locations()), nil - })) - if err != nil { - return dyn.InvalidValue, err + if toSlash { + path = filepath.ToSlash(path) } - return v, nil + return b.Config.Set(p, path) }) - }) + } + + // Makes include and exclude paths relative to the bundle root first. + // Then converts them to use Unix-style slashes. + // This is required for the ignore.GitIgnore we use in libs/fileset to work correctly. + err := rewrite("paths", false) + if err == nil { + err = rewrite("include", true) + } + if err == nil { + err = rewrite("exclude", true) + } return diag.FromErr(err) } diff --git a/bundle/config/mutator/rewrite_workspace_prefix.go b/bundle/config/mutator/rewrite_workspace_prefix.go index 30914fd149e..390a13bb490 100644 --- a/bundle/config/mutator/rewrite_workspace_prefix.go +++ b/bundle/config/mutator/rewrite_workspace_prefix.go @@ -7,7 +7,8 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) type rewriteWorkspacePrefix struct{} @@ -44,38 +45,36 @@ func (m *rewriteWorkspacePrefix) Apply(ctx context.Context, b *bundle.Bundle) di {"/Workspace${workspace.resource_path}", "${workspace.resource_path}"}, } - err := b.Config.Mutate(func(root dyn.Value) (dyn.Value, error) { - // Walk through the bundle configuration, check all the string leafs and - // see if any of the prefixes are used in the remote path. - return dyn.Walk(root, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - vv, ok := v.AsString() - if !ok { - return v, nil - } - - newPath := vv - for _, rewrite := range paths { - if !strings.Contains(newPath, rewrite.pattern) { - continue - } + // Walk through the bundle configuration, check all the string leafs and + // see if any of the prefixes are used in the remote path. + err := structvar.Walk(b.Config.View(), func(p *structpath.PathNode, v structvar.View) error { + vv, ok := v.AsString() + if !ok { + return nil + } - newPath = strings.ReplaceAll(newPath, rewrite.pattern, rewrite.replacement) - diags = append(diags, diag.Diagnostic{ - Severity: diag.Warning, - Summary: fmt.Sprintf("substring %q found in %q. Please update this to %q.", rewrite.pattern, vv, newPath), - Detail: "For more information, please refer to: https://docs.databricks.com/en/release-notes/dev-tools/bundles.html#workspace-paths", - Locations: v.Locations(), - Paths: dyn.ToStructPaths(p), - }) + newPath := vv + for _, rewrite := range paths { + if !strings.Contains(newPath, rewrite.pattern) { + continue } - if newPath == vv { - return v, nil - } + newPath = strings.ReplaceAll(newPath, rewrite.pattern, rewrite.replacement) + diags = append(diags, diag.Diagnostic{ + Severity: diag.Warning, + Summary: fmt.Sprintf("substring %q found in %q. Please update this to %q.", rewrite.pattern, vv, newPath), + Detail: "For more information, please refer to: https://docs.databricks.com/en/release-notes/dev-tools/bundles.html#workspace-paths", + Locations: v.Locations(), + Paths: []*structpath.PathNode{p}, + }) + } + + if newPath == vv { + return nil + } - // Remove the workspace prefix from the string. - return dyn.NewValue(newPath, v.Locations()), nil - }) + // Remove the workspace prefix from the string. + return b.Config.Set(p, newPath) }) if err != nil { return diag.FromErr(err) diff --git a/bundle/config/mutator/set_variables.go b/bundle/config/mutator/set_variables.go index 3038b05acaf..6060d1d0bc1 100644 --- a/bundle/config/mutator/set_variables.go +++ b/bundle/config/mutator/set_variables.go @@ -7,11 +7,12 @@ import ( "path/filepath" "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/variable" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/jsonloader" "github.com/databricks/cli/libs/env" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) const bundleVarPrefix = "BUNDLE_VAR_" @@ -30,90 +31,99 @@ func getDefaultVariableFilePath(target string) string { return ".databricks/bundle/" + target + "/variable-overrides.json" } -func setVariable(ctx context.Context, v dyn.Value, variable *variable.Variable, name string, fileDefault dyn.Value) (dyn.Value, error) { +func setVariable(ctx context.Context, cfg *config.Root, variable *variable.Variable, name string, fileDefault structvar.View) error { // case: variable already has value initialized, so skip if variable.HasValue() { - return v, nil + return nil } + variablePath := structpath.NewStringKeys(nil, "variables", name) + valuePath := structpath.NewStringKey(variablePath, "value") + // case: read and set variable value from process environment envVarName := bundleVarPrefix + name if val, ok := env.Lookup(ctx, envVarName); ok { if variable.IsComplex() { - return dyn.InvalidValue, fmt.Errorf(`setting via environment variables (%s) is not supported for complex variable %s`, envVarName, name) + return fmt.Errorf(`setting via environment variables (%s) is not supported for complex variable %s`, envVarName, name) } - v, err := dyn.Set(v, "value", dyn.V(val)) + err := cfg.Set(valuePath, val) if err != nil { - return dyn.InvalidValue, fmt.Errorf(`failed to assign value "%s" to variable %s from environment variable %s with error: %v`, val, name, envVarName, err) + return fmt.Errorf(`failed to assign value "%s" to variable %s from environment variable %s with error: %v`, val, name, envVarName, err) } - return v, nil + return nil } // case: Defined a variable for named lookup for a resource // It will be resolved later in ResolveResourceReferences mutator if variable.Lookup != nil { - return v, nil + return nil } // case: Set the variable to the default value from the variable file - if fileDefault.Kind() != dyn.KindInvalid && fileDefault.Kind() != dyn.KindNil { + if fileDefault.Kind() != structvar.KindInvalid && fileDefault.Kind() != structvar.KindNil { hasComplexType := variable.IsComplex() - hasComplexValue := fileDefault.Kind() == dyn.KindMap || fileDefault.Kind() == dyn.KindSequence + hasComplexValue := fileDefault.Kind() == structvar.KindMap || fileDefault.Kind() == structvar.KindSequence if hasComplexType && !hasComplexValue { - return dyn.InvalidValue, fmt.Errorf(`variable %s is of type complex, but the value in the variable file is not a complex type`, name) + return fmt.Errorf(`variable %s is of type complex, but the value in the variable file is not a complex type`, name) } if !hasComplexType && hasComplexValue { - return dyn.InvalidValue, fmt.Errorf(`variable %s is not of type complex, but the value in the variable file is a complex type`, name) + return fmt.Errorf(`variable %s is not of type complex, but the value in the variable file is a complex type`, name) } - v, err := dyn.Set(v, "value", fileDefault) + err := cfg.Assign(valuePath, fileDefault) if err != nil { - return dyn.InvalidValue, fmt.Errorf(`failed to assign default value from variable file to variable %s with error: %v`, name, err) + return fmt.Errorf(`failed to assign default value from variable file to variable %s with error: %v`, name, err) } - return v, nil + return nil } // case: Set the variable to its default value if variable.HasDefault() { - vDefault, err := dyn.Get(v, "default") - if err != nil { - return dyn.InvalidValue, fmt.Errorf(`failed to get default value from config "%s" for variable %s with error: %v`, variable.Default, name, err) + vDefault := cfg.View().Lookup(structpath.NewStringKey(variablePath, "default")) + if !vDefault.IsValid() { + return fmt.Errorf(`failed to get default value from config "%s" for variable %s with error: no such key: default`, variable.Default, name) } - v, err := dyn.Set(v, "value", vDefault) + err := cfg.Assign(valuePath, vDefault) if err != nil { - return dyn.InvalidValue, fmt.Errorf(`failed to assign default value from config "%s" to variable %s with error: %v`, variable.Default, name, err) + return fmt.Errorf(`failed to assign default value from config "%s" to variable %s with error: %v`, variable.Default, name, err) } - return v, nil + return nil } // We should have had a value to set for the variable at this point. - return dyn.InvalidValue, fmt.Errorf(`no value assigned to required variable %s. Variables are usually assigned in databricks.yml, and they can be overridden using "--var", the %s environment variable, or %s`, name, bundleVarPrefix+name, getDefaultVariableFilePath("")) + return fmt.Errorf(`no value assigned to required variable %s. Variables are usually assigned in databricks.yml, and they can be overridden using "--var", the %s environment variable, or %s`, name, bundleVarPrefix+name, getDefaultVariableFilePath("")) } -func readVariablesFromFile(b *bundle.Bundle) (dyn.Value, diag.Diagnostics) { +func readVariablesFromFile(b *bundle.Bundle) (structvar.View, diag.Diagnostics) { var diags diag.Diagnostics filePath := filepath.Join(b.BundleRootPath, getDefaultVariableFilePath(b.Config.Bundle.Target)) if _, err := os.Stat(filePath); err != nil { - return dyn.InvalidValue, nil + return structvar.View{}, nil } f, err := os.ReadFile(filePath) if err != nil { - return dyn.InvalidValue, diag.FromErr(fmt.Errorf("failed to read variables file: %w", err)) + return structvar.View{}, diag.FromErr(fmt.Errorf("failed to read variables file: %w", err)) } - val, err := jsonloader.LoadJSON(f, filePath) + node, err := structvar.ParseJSON(filePath, f) if err != nil { - return dyn.InvalidValue, diag.FromErr(fmt.Errorf("failed to parse variables file %s: %w", filePath, err)) + return structvar.View{}, diag.FromErr(fmt.Errorf("failed to parse variables file %s: %w", filePath, err)) + } + var v any + decoded, _, err := structvar.DecodeYAMLNode(filePath, node, &v, nil) + if err != nil { + return structvar.View{}, diag.FromErr(fmt.Errorf("failed to parse variables file %s: %w", filePath, err)) } - if val.Kind() != dyn.KindMap { - return dyn.InvalidValue, diags.Append(diag.Diagnostic{ + val := decoded.View() + if val.Kind() != structvar.KindMap { + return structvar.View{}, diags.Append(diag.Diagnostic{ Severity: diag.Error, Summary: fmt.Sprintf("failed to parse variables file %s: invalid format", filePath), Detail: "Variables file must be a JSON object with the following format:\n{\"var1\": \"value1\", \"var2\": \"value2\"}", @@ -128,18 +138,18 @@ func (m *setVariables) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnos if diags.HasError() { return diags } - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { - return dyn.Map(v, "variables", dyn.Foreach(func(p dyn.Path, variable dyn.Value) (dyn.Value, error) { - name := p[1].Key() - v, ok := b.Config.Variables[name] - if !ok { - return dyn.InvalidValue, fmt.Errorf(`variable "%s" is not defined`, name) - } - - fileDefault, _ := dyn.Get(defaults, name) - return setVariable(ctx, variable, v, name, fileDefault) - })) - }) - - return diags.Extend(diag.FromErr(err)) + for name := range b.Config.View().Get("variables").MapItems() { + v, ok := b.Config.Variables[name] + if !ok { + return diags.Extend(diag.Errorf(`variable "%s" is not defined`, name)) + } + + fileDefault := defaults.Get(name) + err := setVariable(ctx, &b.Config, v, name, fileDefault) + if err != nil { + return diags.Extend(diag.FromErr(err)) + } + } + + return diags } diff --git a/bundle/config/mutator/set_variables_test.go b/bundle/config/mutator/set_variables_test.go index ecc67548017..977e7e94529 100644 --- a/bundle/config/mutator/set_variables_test.go +++ b/bundle/config/mutator/set_variables_test.go @@ -6,12 +6,16 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/variable" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/convert" + "github.com/databricks/cli/libs/structs/structvar" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) +// configWithVariable returns a configuration that has the variable with name "foo". +func configWithVariable(v *variable.Variable) *config.Root { + return &config.Root{Variables: map[string]*variable.Variable{"foo": v}} +} + func TestSetVariableFromProcessEnvVar(t *testing.T) { defaultVal := "default" variable := variable.Variable{ @@ -21,13 +25,9 @@ func TestSetVariableFromProcessEnvVar(t *testing.T) { // set value for variable as an environment variable t.Setenv("BUNDLE_VAR_foo", "process-env") - v, err := convert.FromTyped(variable, dyn.NilValue) - require.NoError(t, err) - - v, err = setVariable(t.Context(), v, &variable, "foo", dyn.NilValue) - require.NoError(t, err) + cfg := configWithVariable(&variable) - err = convert.ToTyped(&variable, v) + err := setVariable(t.Context(), cfg, &variable, "foo", structvar.View{}) require.NoError(t, err) assert.Equal(t, "process-env", variable.Value) } @@ -39,13 +39,9 @@ func TestSetVariableUsingDefaultValue(t *testing.T) { Default: defaultVal, } - v, err := convert.FromTyped(variable, dyn.NilValue) - require.NoError(t, err) - - v, err = setVariable(t.Context(), v, &variable, "foo", dyn.NilValue) - require.NoError(t, err) + cfg := configWithVariable(&variable) - err = convert.ToTyped(&variable, v) + err := setVariable(t.Context(), cfg, &variable, "foo", structvar.View{}) require.NoError(t, err) assert.Equal(t, "default", variable.Value) } @@ -61,13 +57,9 @@ func TestSetVariableWhenAlreadyAValueIsAssigned(t *testing.T) { // since a value is already assigned to the variable, it would not be overridden // by the default value - v, err := convert.FromTyped(variable, dyn.NilValue) - require.NoError(t, err) - - v, err = setVariable(t.Context(), v, &variable, "foo", dyn.NilValue) - require.NoError(t, err) + cfg := configWithVariable(&variable) - err = convert.ToTyped(&variable, v) + err := setVariable(t.Context(), cfg, &variable, "foo", structvar.View{}) require.NoError(t, err) assert.Equal(t, "assigned-value", variable.Value) } @@ -86,13 +78,9 @@ func TestSetVariableEnvVarValueDoesNotOverridePresetValue(t *testing.T) { // since a value is already assigned to the variable, it would not be overridden // by the value from environment - v, err := convert.FromTyped(variable, dyn.NilValue) - require.NoError(t, err) + cfg := configWithVariable(&variable) - v, err = setVariable(t.Context(), v, &variable, "foo", dyn.NilValue) - require.NoError(t, err) - - err = convert.ToTyped(&variable, v) + err := setVariable(t.Context(), cfg, &variable, "foo", structvar.View{}) require.NoError(t, err) assert.Equal(t, "assigned-value", variable.Value) } @@ -103,10 +91,9 @@ func TestSetVariablesErrorsIfAValueCouldNotBeResolved(t *testing.T) { } // fails because we could not resolve a value for the variable - v, err := convert.FromTyped(variable, dyn.NilValue) - require.NoError(t, err) + cfg := configWithVariable(&variable) - _, err = setVariable(t.Context(), v, &variable, "foo", dyn.NilValue) + err := setVariable(t.Context(), cfg, &variable, "foo", structvar.View{}) assert.ErrorContains(t, err, "no value assigned to required variable foo. Variables are usually assigned in databricks.yml, and they can be overridden using \"--var\", the BUNDLE_VAR_foo environment variable, or .databricks/bundle//variable-overrides.json") } @@ -153,9 +140,8 @@ func TestSetComplexVariablesViaEnvVariablesIsNotAllowed(t *testing.T) { // set value for variable as an environment variable t.Setenv("BUNDLE_VAR_foo", "process-env") - v, err := convert.FromTyped(variable, dyn.NilValue) - require.NoError(t, err) + cfg := configWithVariable(&variable) - _, err = setVariable(t.Context(), v, &variable, "foo", dyn.NilValue) + err := setVariable(t.Context(), cfg, &variable, "foo", structvar.View{}) assert.ErrorContains(t, err, "setting via environment variables (BUNDLE_VAR_foo) is not supported for complex variable foo") } diff --git a/bundle/config/mutator/sync_default_path.go b/bundle/config/mutator/sync_default_path.go index 16d4a4d6179..586d864e98c 100644 --- a/bundle/config/mutator/sync_default_path.go +++ b/bundle/config/mutator/sync_default_path.go @@ -5,7 +5,6 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" ) type syncDefaultPath struct{} @@ -20,24 +19,9 @@ func (m *syncDefaultPath) Name() string { } func (m *syncDefaultPath) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - isset := false - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { - pv, _ := dyn.Get(v, "sync.paths") - - // If the sync paths field is already set, do nothing. - // We know it is set if its value is either a nil or a sequence (empty or not). - if pv.Kind() == dyn.KindNil || pv.Kind() == dyn.KindSequence { - isset = true - } - - return v, nil - }) - if err != nil { - return diag.FromErr(err) - } - // If the sync paths field is already set, do nothing. - if isset { + // We know it is set if it is a sequence (empty or not) or a reference. + if b.Config.Sync.Paths != nil || b.Config.IsReference("sync.paths") { return nil } diff --git a/bundle/config/mutator/sync_default_path_test.go b/bundle/config/mutator/sync_default_path_test.go index c4bd6a65af7..a85821b6cc2 100644 --- a/bundle/config/mutator/sync_default_path_test.go +++ b/bundle/config/mutator/sync_default_path_test.go @@ -7,8 +7,8 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/mutator" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/logdiag" + "github.com/databricks/cli/libs/structs/structpath" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -28,22 +28,22 @@ func TestSyncDefaultPath_DefaultIfUnset(t *testing.T) { func TestSyncDefaultPath_SkipIfSet(t *testing.T) { tcases := []struct { name string - paths dyn.Value + paths any expect []string }{ { name: "nil", - paths: dyn.V(nil), + paths: nil, expect: []string{"."}, }, { name: "empty sequence", - paths: dyn.V([]dyn.Value{}), + paths: []string{}, expect: []string{}, }, { name: "non-empty sequence", - paths: dyn.V([]dyn.Value{dyn.V("something")}), + paths: []string{"something"}, expect: []string{"something"}, }, } @@ -58,17 +58,7 @@ func TestSyncDefaultPath_SkipIfSet(t *testing.T) { ctx := logdiag.InitContext(t.Context()) bundle.ApplyFuncContext(ctx, b, func(ctx context.Context, b *bundle.Bundle) { - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { - v, err := dyn.Set(v, "sync", dyn.V(dyn.NewMapping())) - if err != nil { - return dyn.InvalidValue, err - } - v, err = dyn.Set(v, "sync.paths", tcase.paths) - if err != nil { - return dyn.InvalidValue, err - } - return v, nil - }) + err := b.Config.Set(structpath.MustParsePath("sync.paths"), tcase.paths) require.NoError(t, err) }) require.False(t, logdiag.HasError(ctx)) diff --git a/bundle/config/mutator/sync_infer_root.go b/bundle/config/mutator/sync_infer_root.go index bd1a529380a..75223780081 100644 --- a/bundle/config/mutator/sync_infer_root.go +++ b/bundle/config/mutator/sync_infer_root.go @@ -90,12 +90,11 @@ func (m *syncInferRoot) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagno continue } - syncPath := structpath.NewPath(nil, "sync", "paths", i) diags = append(diags, diag.Diagnostic{ Severity: diag.Error, Summary: fmt.Sprintf("invalid sync path %q", path), - Locations: b.Config.GetLocationsOf(syncPath), - Paths: []*structpath.PathNode{syncPath}, + Locations: b.Config.GetLocations(fmt.Sprintf("sync.paths[%d]", i)), + Paths: []*structpath.PathNode{structpath.NewIndex(structpath.MustParsePath("sync.paths"), i)}, }) } diff --git a/bundle/config/mutator/translate_paths.go b/bundle/config/mutator/translate_paths.go index f44c7235043..9ad6c53dbaf 100644 --- a/bundle/config/mutator/translate_paths.go +++ b/bundle/config/mutator/translate_paths.go @@ -11,15 +11,15 @@ import ( "path/filepath" "strings" - "github.com/databricks/cli/bundle/config/mutator/paths" - "github.com/databricks/cli/bundle/libraries" - "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/mutator/paths" "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/libraries" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/notebook" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) // translateOptions control path translation behavior. @@ -300,33 +300,47 @@ func (t *translateContext) translateLocalRelativeWithPrefixPath(ctx context.Cont return localRelPath, nil } -func (t *translateContext) rewriteValue(ctx context.Context, p dyn.Path, v dyn.Value, dir string, opts translateOptions) (dyn.Value, error) { - out, err := t.rewritePath(ctx, dir, v.MustString(), opts) +// rewriteValue rewrites the path input found at p. It returns an empty string if the path was not rewritten. +func (t *translateContext) rewriteValue(ctx context.Context, p *structpath.PathNode, input, dir string, opts translateOptions) (string, error) { + out, err := t.rewritePath(ctx, dir, input, opts) if err != nil { if target, ok := errors.AsType[ErrIsNotebook](err); ok { - return dyn.InvalidValue, fmt.Errorf(`expected a file for "%s" but got a notebook: %w`, p, target) + return "", fmt.Errorf(`expected a file for "%s" but got a notebook: %w`, p, target) } if target, ok := errors.AsType[ErrIsNotNotebook](err); ok { - return dyn.InvalidValue, fmt.Errorf(`expected a notebook for "%s" but got a file: %w`, p, target) + return "", fmt.Errorf(`expected a notebook for "%s" but got a file: %w`, p, target) } - return dyn.InvalidValue, err + return "", err } - // If the path was not rewritten, return the original value. + return out, nil +} + +// setRewritten stores the rewritten path at p (keeping its locations) unless the path was not rewritten. +func (t *translateContext) setRewritten(p *structpath.PathNode, out string) error { if out == "" { - return v, nil + return nil } + return t.b.Config.Set(p, out) +} - return dyn.NewValue(out, v.Locations()), nil +// rewriteAt rewrites the string value v found at p relative to the bundle root and stores the result. +func (t *translateContext) rewriteAt(ctx context.Context, p *structpath.PathNode, v structvar.View, opts translateOptions) error { + input, _ := v.AsString() + out, err := t.rewriteValue(ctx, p, input, t.b.BundleRootPath, opts) + if err != nil { + return err + } + return t.setRewritten(p, out) } -func applyTranslations(ctx context.Context, b *bundle.Bundle, t *translateContext, translations []func(context.Context, dyn.Value) (dyn.Value, error)) diag.Diagnostics { +func applyTranslations(ctx context.Context, b *bundle.Bundle, t *translateContext, translations []translateFunc) diag.Diagnostics { switch { case b.IsImmutableFolder(): // Reject an explicit workspace.file_path: immutable bundles set it // automatically to the content-addressed snapshot location. A user-supplied // value would be silently discarded during path translation, so we error early. - if loc := b.Config.GetLocation("workspace.file_path"); loc.File != "" { + if loc := b.Config.DefinitionLocation("workspace.file_path"); loc.File != "" { return diag.Diagnostics{{ Severity: diag.Error, Summary: "workspace.file_path cannot be configured when experimental.immutable_folder is enabled", @@ -340,18 +354,13 @@ func applyTranslations(ctx context.Context, b *bundle.Bundle, t *translateContex t.remoteRoot = t.b.Config.Workspace.FilePath } - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { - var err error - for _, fn := range translations { - v, err = fn(ctx, v) - if err != nil { - return dyn.InvalidValue, err - } + for _, fn := range translations { + if err := fn(ctx, b.Config.View()); err != nil { + return diag.FromErr(err) } - return v, nil - }) + } - return diag.FromErr(err) + return nil } func (m *translatePaths) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { @@ -361,7 +370,7 @@ func (m *translatePaths) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagn skipLocalFileValidation: b.SkipLocalFileValidation, } - return applyTranslations(ctx, b, t, []func(context.Context, dyn.Value) (dyn.Value, error){ + return applyTranslations(ctx, b, t, []translateFunc{ t.applyJobTranslations(paths.VisitJobPaths, false), t.applyJobTranslations(paths.VisitJobLibrariesPaths, true), t.applyPipelineTranslations(paths.VisitPipelinePaths, false), @@ -378,7 +387,7 @@ func (m *translatePathsDashboards) Apply(ctx context.Context, b *bundle.Bundle) skipLocalFileValidation: b.SkipLocalFileValidation, } - return applyTranslations(ctx, b, t, []func(context.Context, dyn.Value) (dyn.Value, error){ + return applyTranslations(ctx, b, t, []translateFunc{ t.applyDashboardTranslations, t.applyGenieSpaceTranslations, }) diff --git a/bundle/config/mutator/translate_paths_apps.go b/bundle/config/mutator/translate_paths_apps.go index 8daef7e4857..8c8c688b23c 100644 --- a/bundle/config/mutator/translate_paths_apps.go +++ b/bundle/config/mutator/translate_paths_apps.go @@ -5,18 +5,19 @@ import ( "github.com/databricks/cli/bundle/config/mutator/paths" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) -func (t *translateContext) applyAppsTranslations(ctx context.Context, v dyn.Value) (dyn.Value, error) { +func (t *translateContext) applyAppsTranslations(ctx context.Context, v structvar.View) error { // Convert the `source_code_path` field to a remote absolute path. // We use this path for app deployment to point to the source code. - return paths.VisitAppPaths(v, func(p dyn.Path, mode paths.TranslateMode, v dyn.Value) (dyn.Value, error) { + return paths.VisitAppPaths(v, func(p *structpath.PathNode, mode paths.TranslateMode, v structvar.View) error { opts := translateOptions{ Mode: mode, } - return t.rewriteValue(ctx, p, v, t.b.BundleRootPath, opts) + return t.rewriteAt(ctx, p, v, opts) }) } diff --git a/bundle/config/mutator/translate_paths_artifacts.go b/bundle/config/mutator/translate_paths_artifacts.go index 51ae0d8f8f5..17112c43e66 100644 --- a/bundle/config/mutator/translate_paths_artifacts.go +++ b/bundle/config/mutator/translate_paths_artifacts.go @@ -5,11 +5,12 @@ import ( "github.com/databricks/cli/bundle/config/mutator/paths" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) -func (t *translateContext) applyArtifactTranslations(ctx context.Context, v dyn.Value) (dyn.Value, error) { - return paths.VisitArtifactPaths(v, func(p dyn.Path, mode paths.TranslateMode, v dyn.Value) (dyn.Value, error) { +func (t *translateContext) applyArtifactTranslations(ctx context.Context, v structvar.View) error { + return paths.VisitArtifactPaths(v, func(p *structpath.PathNode, mode paths.TranslateMode, v structvar.View) error { opts := translateOptions{ Mode: mode, @@ -18,6 +19,6 @@ func (t *translateContext) applyArtifactTranslations(ctx context.Context, v dyn. AllowPathOutsideSyncRoot: true, } - return t.rewriteValue(ctx, p, v, t.b.BundleRootPath, opts) + return t.rewriteAt(ctx, p, v, opts) }) } diff --git a/bundle/config/mutator/translate_paths_dashboards.go b/bundle/config/mutator/translate_paths_dashboards.go index 2e93ed3998f..85d4c809022 100644 --- a/bundle/config/mutator/translate_paths_dashboards.go +++ b/bundle/config/mutator/translate_paths_dashboards.go @@ -5,18 +5,19 @@ import ( "github.com/databricks/cli/bundle/config/mutator/paths" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) -func (t *translateContext) applyDashboardTranslations(ctx context.Context, v dyn.Value) (dyn.Value, error) { +func (t *translateContext) applyDashboardTranslations(ctx context.Context, v structvar.View) error { // Convert the `file_path` field to a local absolute path. // We load the file at this path and use its contents for the dashboard contents. - return paths.VisitDashboardPaths(v, func(p dyn.Path, mode paths.TranslateMode, v dyn.Value) (dyn.Value, error) { + return paths.VisitDashboardPaths(v, func(p *structpath.PathNode, mode paths.TranslateMode, v structvar.View) error { opts := translateOptions{ Mode: mode, } - return t.rewriteValue(ctx, p, v, t.b.BundleRootPath, opts) + return t.rewriteAt(ctx, p, v, opts) }) } diff --git a/bundle/config/mutator/translate_paths_genie_spaces.go b/bundle/config/mutator/translate_paths_genie_spaces.go index 4e6d41f1cec..d8f46498492 100644 --- a/bundle/config/mutator/translate_paths_genie_spaces.go +++ b/bundle/config/mutator/translate_paths_genie_spaces.go @@ -4,18 +4,19 @@ import ( "context" "github.com/databricks/cli/bundle/config/mutator/paths" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) -func (t *translateContext) applyGenieSpaceTranslations(ctx context.Context, v dyn.Value) (dyn.Value, error) { +func (t *translateContext) applyGenieSpaceTranslations(ctx context.Context, v structvar.View) error { // Rewrite the `file_path` field to a path relative to the bundle sync root. // We load the file at this path and use its contents for the genie space contents. - return paths.VisitGenieSpacePaths(v, func(p dyn.Path, mode paths.TranslateMode, v dyn.Value) (dyn.Value, error) { + return paths.VisitGenieSpacePaths(v, func(p *structpath.PathNode, mode paths.TranslateMode, v structvar.View) error { opts := translateOptions{ Mode: mode, } - return t.rewriteValue(ctx, p, v, t.b.BundleRootPath, opts) + return t.rewriteAt(ctx, p, v, opts) }) } diff --git a/bundle/config/mutator/translate_paths_jobs.go b/bundle/config/mutator/translate_paths_jobs.go index 2c94e21802b..bcfd00a9e73 100644 --- a/bundle/config/mutator/translate_paths_jobs.go +++ b/bundle/config/mutator/translate_paths_jobs.go @@ -5,17 +5,17 @@ import ( "slices" "github.com/databricks/cli/bundle/config/mutator/paths" - - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) type ( - visitFunc func(dyn.Value, paths.VisitFunc) (dyn.Value, error) - translateFunc func(ctx context.Context, v dyn.Value) (dyn.Value, error) + visitFunc func(structvar.View, paths.VisitFunc) error + translateFunc func(ctx context.Context, v structvar.View) error ) func (t *translateContext) applyJobTranslations(visitor visitFunc, allowOutsideSyncRoot bool) translateFunc { - return func(ctx context.Context, v dyn.Value) (dyn.Value, error) { + return func(ctx context.Context, v structvar.View) error { // Do not translate job task paths if using Git source var ignore []string for key, job := range t.b.Config.Resources.Jobs { @@ -24,12 +24,13 @@ func (t *translateContext) applyJobTranslations(visitor visitFunc, allowOutsideS } } - return visitor(v, func(p dyn.Path, mode paths.TranslateMode, v dyn.Value) (dyn.Value, error) { - key := p[2].Key() + return visitor(v, func(p *structpath.PathNode, mode paths.TranslateMode, v structvar.View) error { + key := p.KeyAt(2) + input, _ := v.AsString() // Skip path translation if the job is using git source. if slices.Contains(ignore, key) { - return v, nil + return nil } opts := translateOptions{ @@ -38,12 +39,12 @@ func (t *translateContext) applyJobTranslations(visitor visitFunc, allowOutsideS } // Handle path as if it's relative to the bundle root - nv, err := t.rewriteValue(ctx, p, v, t.b.BundleRootPath, opts) + nv, err := t.rewriteValue(ctx, p, input, t.b.BundleRootPath, opts) if err == nil { - return nv, nil + return t.setRewritten(p, nv) } - return dyn.InvalidValue, err + return err }) } } diff --git a/bundle/config/mutator/translate_paths_pipelines.go b/bundle/config/mutator/translate_paths_pipelines.go index 07c2e58cd93..7b6f0c2e52b 100644 --- a/bundle/config/mutator/translate_paths_pipelines.go +++ b/bundle/config/mutator/translate_paths_pipelines.go @@ -4,25 +4,26 @@ import ( "context" "github.com/databricks/cli/bundle/config/mutator/paths" - - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) func (t *translateContext) applyPipelineTranslations(visitor visitFunc, allowOutsideSyncRoot bool) translateFunc { - return func(ctx context.Context, v dyn.Value) (dyn.Value, error) { - return visitor(v, func(p dyn.Path, mode paths.TranslateMode, v dyn.Value) (dyn.Value, error) { + return func(ctx context.Context, v structvar.View) error { + return visitor(v, func(p *structpath.PathNode, mode paths.TranslateMode, v structvar.View) error { + input, _ := v.AsString() opts := translateOptions{ Mode: mode, AllowPathOutsideSyncRoot: allowOutsideSyncRoot, } // Handle path as if it's relative to the bundle root - nv, err := t.rewriteValue(ctx, p, v, t.b.BundleRootPath, opts) + nv, err := t.rewriteValue(ctx, p, input, t.b.BundleRootPath, opts) if err == nil { - return nv, nil + return t.setRewritten(p, nv) } - return dyn.InvalidValue, err + return err }) } } diff --git a/bundle/config/mutator/translate_paths_test.go b/bundle/config/mutator/translate_paths_test.go index 8aaf9ce9c71..63569c0f747 100644 --- a/bundle/config/mutator/translate_paths_test.go +++ b/bundle/config/mutator/translate_paths_test.go @@ -15,7 +15,7 @@ import ( "github.com/databricks/cli/bundle/config/variable" "github.com/databricks/cli/bundle/internal/bundletest" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/cli/libs/vfs" "github.com/databricks/databricks-sdk-go/service/compute" "github.com/databricks/databricks-sdk-go/service/jobs" @@ -855,10 +855,8 @@ func TestTranslatePathWithComplexVariables(t *testing.T) { ctx := t.Context() // Assign the variables to the dynamic configuration. bundle.ApplyFuncContext(ctx, b, func(ctx context.Context, b *bundle.Bundle) { - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { - p := dyn.MustPathFromString("resources.jobs.job.tasks[0]") - return dyn.SetByPath(v, p.Append(dyn.Key("libraries")), dyn.V("${var.cluster_libraries}")) - }) + p := structpath.MustParsePath("resources.jobs.job.tasks[0]") + err := b.Config.SetReference(structpath.NewStringKey(p, "libraries"), "${var.cluster_libraries}") require.NoError(t, err) }) diff --git a/bundle/config/mutator/validate_secret_value_is_variable.go b/bundle/config/mutator/validate_secret_value_is_variable.go index 1f979e15f74..70770022221 100644 --- a/bundle/config/mutator/validate_secret_value_is_variable.go +++ b/bundle/config/mutator/validate_secret_value_is_variable.go @@ -6,8 +6,8 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/dynvar" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) type validateSecretValueIsVariable struct{} @@ -25,21 +25,18 @@ func (v *validateSecretValueIsVariable) Apply(ctx context.Context, b *bundle.Bun // Iterate over all secrets in the bundle for key := range b.Config.Resources.Secrets { - p := dyn.NewPath(dyn.Key("resources"), dyn.Key("secrets"), dyn.Key(key), dyn.Key("value")) - val, err := dyn.GetByPath(b.Config.Value(), p) - if dyn.IsNoSuchKeyError(err) { + p := structpath.NewStringKeys(nil, "resources", "secrets", key, "value") + val := b.Config.View().Lookup(p) + if !val.IsValid() { diags = append(diags, diag.Diagnostic{ Severity: diag.Error, Summary: "Secret value must be a string", Detail: fmt.Sprintf(`The secret value for "%s" must be a string.`, key), Locations: val.Locations(), - Paths: dyn.ToStructPaths(p), + Paths: []*structpath.PathNode{p}, }) continue } - if err != nil { - return diag.FromErr(err) - } valueStr, ok := val.AsString() if !ok { @@ -48,13 +45,13 @@ func (v *validateSecretValueIsVariable) Apply(ctx context.Context, b *bundle.Bun Summary: "Secret value must be a string", Detail: fmt.Sprintf(`The secret value for "%s" must be a string.`, key), Locations: val.Locations(), - Paths: dyn.ToStructPaths(p), + Paths: []*structpath.PathNode{p}, }) continue } // Value must be a variable reference to prevent leaking secrets in config files - if !dynvar.IsPureVariableReference(valueStr) { + if !structvar.IsPureVariableReference(valueStr) { diags = append(diags, diag.Diagnostic{ Severity: diag.Error, Summary: "Secret value must be a variable reference", @@ -62,7 +59,7 @@ func (v *validateSecretValueIsVariable) Apply(ctx context.Context, b *bundle.Bun Plain text secret values are not allowed to prevent leaking secrets in configuration files. Use bundle variables to pass secret values at deployment time.`, key), Locations: val.Locations(), - Paths: dyn.ToStructPaths(p), + Paths: []*structpath.PathNode{p}, }) continue } @@ -79,25 +76,21 @@ Use bundle variables to pass secret values at deployment time.`, key), // checkVariableDefault emits an error if valueStr is a pure ${var.} reference // and the referenced variable has a default value set. -func (v *validateSecretValueIsVariable) checkVariableDefault(b *bundle.Bundle, secretKey, valueStr string, p dyn.Path, val dyn.Value) diag.Diagnostics { - refPath, ok := dynvar.PureReferenceToPath(valueStr) - if !ok || len(refPath) < 2 || refPath[0].Key() != "var" { +func (v *validateSecretValueIsVariable) checkVariableDefault(b *bundle.Bundle, secretKey, valueStr string, p *structpath.PathNode, val structvar.View) diag.Diagnostics { + refPath, ok := structvar.PureReferenceToPath(valueStr) + if !ok || refPath.Len() < 2 || refPath.KeyAt(0) != "var" { return nil } - varName := refPath[1].Key() + varName := refPath.KeyAt(1) variable, exists := b.Config.Variables[varName] if !exists || variable == nil || !variable.HasDefault() { return nil } // The default path in the dynamic config for the variable's default field. - defaultPath := dyn.NewPath(dyn.Key("variables"), dyn.Key(varName), dyn.Key("default")) - defaultVal, err := dyn.GetByPath(b.Config.Value(), defaultPath) - locations := val.Locations() - if err == nil { - locations = append(defaultVal.Locations(), locations...) - } + defaultPath := structpath.NewStringKeys(nil, "variables", varName, "default") + locations := append(b.Config.LocationsAt(defaultPath), val.Locations()...) return diag.Diagnostics{{ Severity: diag.Error, @@ -106,6 +99,6 @@ func (v *validateSecretValueIsVariable) checkVariableDefault(b *bundle.Bundle, s A default value is stored in plain text in the configuration file, which defeats the purpose of using a variable reference for a secret. Remove the default value and pass the secret value at deployment time using "--var", the BUNDLE_VAR_%s environment variable, or a variable overrides file.`, varName, secretKey, varName), Locations: locations, - Paths: dyn.ToStructPaths(p), + Paths: []*structpath.PathNode{p}, }} } diff --git a/bundle/config/resources_types_test.go b/bundle/config/resources_types_test.go index 7a698604612..af53fe0ef04 100644 --- a/bundle/config/resources_types_test.go +++ b/bundle/config/resources_types_test.go @@ -3,20 +3,19 @@ package config import ( "encoding/json" "fmt" + "maps" "reflect" "slices" "strings" "testing" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/databricks/cli/bundle/config/resources" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/convert" "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/cli/libs/structs/structtag" + "github.com/databricks/cli/libs/structs/structvar" "github.com/databricks/cli/libs/structs/structwalk" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestResourcesTypesMap(t *testing.T) { @@ -32,14 +31,14 @@ func TestResourcesTypesMap(t *testing.T) { } // TestResourceTypesZeroValueFieldsSerialize guards against the ForceSendFields -// routing bug fixed in libs/dyn/convert: a field declared in a struct embedded +// routing bug: a field declared in a struct embedded // more than one level deep (e.g. PostgresProject -> PostgresProjectConfig -> // ProjectSpec) had its zero value recorded in the wrong struct's ForceSendFields, // which the SDK marshaler rejects with "field X cannot be found in struct Y". // The direct engine hits this path when it serializes planned state to JSON. // // For every registered resource type it sets every omitempty scalar field (at any -// depth) to its zero value, converts via ToTyped, and marshals - the same round +// depth) to its zero value, decodes it into the typed value, and marshals - the same round // trip the direct engine performs. Any newly added resource whose wrapper embeds // an SDK spec is covered automatically. func TestResourceTypesZeroValueFieldsSerialize(t *testing.T) { @@ -53,41 +52,38 @@ func TestResourceTypesZeroValueFieldsSerialize(t *testing.T) { t.Run(name, func(t *testing.T) { typ := ResourcesTypes[name] zeros := zeroValueScalars(typ, 0, map[reflect.Type]bool{}) - if zeros.Kind() != dyn.KindMap { + if zeros == nil { return } ptr := reflect.New(typ) - require.NoError(t, convert.ToTyped(ptr.Interface(), zeros)) + _, err := (&structvar.StructVar{Value: ptr.Interface()}).Assign(nil, structvar.NewView(&zeros, nil, nil)) + require.NoError(t, err) - _, err := json.Marshal(ptr.Interface()) + _, err = json.Marshal(ptr.Interface()) require.NoError(t, err) }) } } -// zeroValueScalars builds a [dyn.Value] map that sets every omitempty scalar field +// zeroValueScalars builds a map that sets every omitempty scalar field // reachable through embedded anonymous structs to its zero value. Those are exactly // the fields the convert layer records in ForceSendFields, so they exercise the // routing logic. depth and seen bound recursion against deep or recursive types. -func zeroValueScalars(t reflect.Type, depth int, seen map[reflect.Type]bool) dyn.Value { +func zeroValueScalars(t reflect.Type, depth int, seen map[reflect.Type]bool) map[string]any { for t.Kind() == reflect.Pointer { t = t.Elem() } if t.Kind() != reflect.Struct || depth > 6 || seen[t] { - return dyn.NilValue + return nil } seen[t] = true defer delete(seen, t) - m := dyn.NewMapping() + m := map[string]any{} for f := range t.Fields() { if f.Anonymous { - if sub := zeroValueScalars(f.Type, depth+1, seen); sub.Kind() == dyn.KindMap { - for _, p := range sub.MustMap().Pairs() { - m.SetLoc(p.Key.MustString(), nil, p.Value) - } - } + maps.Copy(m, zeroValueScalars(f.Type, depth+1, seen)) continue } @@ -99,18 +95,18 @@ func zeroValueScalars(t reflect.Type, depth int, seen map[reflect.Type]bool) dyn switch f.Type.Kind() { case reflect.Bool: - m.SetLoc(name, nil, dyn.V(false)) + m[name] = false case reflect.String: - m.SetLoc(name, nil, dyn.V("")) + m[name] = "" case reflect.Int, reflect.Int32, reflect.Int64: - m.SetLoc(name, nil, dyn.V(int64(0))) + m[name] = int64(0) case reflect.Float32, reflect.Float64: - m.SetLoc(name, nil, dyn.V(float64(0))) + m[name] = float64(0) default: // Only basic types are eligible for ForceSendFields; skip the rest. } } - return dyn.V(m) + return m } // TestNoSameDepthJSONShadows uses structwalk.WalkType — the dumb walker that diff --git a/bundle/config/root.go b/bundle/config/root.go index 3aa6fdd8b87..34c2c9dee63 100644 --- a/bundle/config/root.go +++ b/bundle/config/root.go @@ -2,24 +2,24 @@ package config import ( "bytes" - "context" "errors" "fmt" + "io" + "iter" + "maps" "os" "reflect" + "slices" "strings" + "github.com/databricks/cli/bundle/config/loctable" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/config/variable" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/convert" - "github.com/databricks/cli/libs/dyn/dynloc" - "github.com/databricks/cli/libs/dyn/merge" - "github.com/databricks/cli/libs/dyn/yamlloader" - "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" "github.com/databricks/databricks-sdk-go/service/jobs" + "go.yaml.in/yaml/v3" ) type Script struct { @@ -35,8 +35,12 @@ type Script struct { } type Root struct { //nolint:recvcheck // value receivers for read-only accessors, pointer for mutators - value dyn.Value - depth int + // refs holds the pure variable references in fields that cannot hold a string + // (e.g. `max_retries: ${var.n}`, where the typed field is zero), keyed by path. + refs map[string]string + + // locations holds the source locations of the values. + locations *structvar.Locations // Contains user defined variables Variables map[string]*variable.Variable `json:"variables,omitempty"` @@ -88,7 +92,7 @@ type Root struct { //nolint:recvcheck // value receivers for read-only accessors // Locations is an output-only field that holds configuration location // information for every path in the configuration tree. - Locations *dynloc.Locations `json:"__locations,omitempty" bundle:"internal"` + Locations *loctable.Locations `json:"__locations,omitempty" bundle:"internal"` Scripts map[string]Script `json:"scripts,omitempty"` @@ -110,161 +114,55 @@ func Load(path string) (*Root, diag.Diagnostics) { } func LoadFromBytes(path string, raw []byte) (*Root, diag.Diagnostics) { - r := Root{} + node, err := structvar.ParseYAML(bytes.NewReader(raw)) + if err != nil { + return nil, diag.Errorf("failed to load %s: %v", path, err) + } - // Load configuration tree from YAML. - v, err := yamlloader.LoadYAML(path, bytes.NewBuffer(raw)) + // Rewrite configuration tree where necessary. + rewriteShorthands(node) + + var r Root + sv, diags, err := structvar.DecodeYAMLNode(path, node, &r, nil) if err != nil { - if le, ok := errors.AsType[*yamlloader.LocationError](err); ok { + if le, ok := errors.AsType[*structvar.LocationError](err); ok { return nil, diag.Diagnostics{{ Severity: diag.Error, Summary: le.Summary, Locations: []diag.Location{le.Loc}, }} } - return nil, diag.Errorf("failed to load %s: %v", path, err) - } - - // Rewrite configuration tree where necessary. - v, err = rewriteShorthands(v) - if err != nil { - return nil, diag.Errorf("failed to rewrite %s: %v", path, err) - } - - // Normalize dynamic configuration tree according to configuration type. - v, diags := convert.Normalize(r, v) - - // Convert normalized configuration tree to typed configuration. - err = r.updateWithDynamicValue(v) - if err != nil { - diags = diags.Extend(diag.Errorf("failed to load %s: %v", path, err)) - return nil, diags + return nil, diags.Extend(diag.Errorf("failed to load %s: %v", path, err)) } + r.store(sv) return &r, diags } -func (r *Root) initializeDynamicValue() error { - // Many test cases initialize a config as a Go struct literal. - // The value will be invalid and we need to populate it from the typed configuration. - if r.value.IsValid() { - return nil - } - - nv, err := convert.FromTyped(r, dyn.NilValue) +// LoadFromReader decodes the configuration in r, recording the locations mapLocations +// returns (if not nil) instead of the locations in the file. +func LoadFromReader(path string, r io.Reader, mapLocations structvar.LocationMapper) (*Root, diag.Diagnostics, error) { + node, err := structvar.ParseYAML(r) if err != nil { - return err + return nil, nil, err } - - r.value = nv - return nil -} - -func (r *Root) updateWithDynamicValue(nv dyn.Value) error { - // Hack: restore state; it may be cleared by [ToTyped] if - // the configuration equals nil (happens in tests). - depth := r.depth - - defer func() { - r.depth = depth - }() - - // Convert normalized configuration tree to typed configuration. - err := convert.ToTyped(r, nv) + var root Root + sv, diags, err := structvar.DecodeYAMLNode(path, node, &root, mapLocations) if err != nil { - return err + return nil, diags, err } - - // Assign the normalized configuration tree. - r.value = nv - return nil + root.store(sv) + return &root, diags, nil } -// Mutate applies a transformation to the dynamic configuration value of a Root object. -// -// Parameters: -// - fn: A function that mutates a dyn.Value object -// -// Example usage, setting bundle.deployment.lock.enabled to false: -// -// err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { -// return dyn.Map(v, "bundle.deployment.lock", func(_ dyn.Path, v dyn.Value) (dyn.Value, error) { -// return dyn.Set(v, "enabled", dyn.V(false)) -// }) -// }) -func (r *Root) Mutate(fn func(dyn.Value) (dyn.Value, error)) error { - err := r.initializeDynamicValue() - if err != nil { - return err - } - nv, err := fn(r.value) - if err != nil { - return err - } - err = r.updateWithDynamicValue(nv) - if err != nil { - return err - } - return nil +// vars returns the configuration as a [structvar.StructVar]; use [Root.store] to +// keep changes made to its references and locations. +func (r *Root) vars() *structvar.StructVar { + return &structvar.StructVar{Value: r, Refs: r.refs, Locations: r.locations} } -func (r *Root) MarkMutatorEntry(ctx context.Context) error { - err := r.initializeDynamicValue() - if err != nil { - return err - } - - r.depth++ - - // If we are entering a mutator at depth 1, we need to convert - // the dynamic configuration tree to typed configuration. - if r.depth == 1 { - // Always run ToTyped upon entering a mutator. - // Convert normalized configuration tree to typed configuration. - err := r.updateWithDynamicValue(r.value) - if err != nil { - log.Warnf(ctx, "unable to convert dynamic configuration to typed configuration: %v", err) - return err - } - - } else { - nv, err := convert.FromTyped(r, r.value) - if err != nil { - log.Warnf(ctx, "unable to convert typed configuration to dynamic configuration: %v", err) - return err - } - - // Re-run ToTyped to ensure that no state is piggybacked - err = r.updateWithDynamicValue(nv) - if err != nil { - log.Warnf(ctx, "unable to convert dynamic configuration to typed configuration: %v", err) - return err - } - } - - return nil -} - -func (r *Root) MarkMutatorExit(ctx context.Context) error { - r.depth-- - - // If we are exiting a mutator at depth 0, we need to convert - // the typed configuration to a dynamic configuration tree. - if r.depth == 0 { - nv, err := convert.FromTyped(r, r.value) - if err != nil { - log.Warnf(ctx, "unable to convert typed configuration to dynamic configuration: %v", err) - return err - } - - // Re-run ToTyped to ensure that no state is piggybacked - err = r.updateWithDynamicValue(nv) - if err != nil { - log.Warnf(ctx, "unable to convert dynamic configuration to typed configuration: %v", err) - return err - } - } - - return nil +func (r *Root) store(sv *structvar.StructVar) { + r.refs = sv.Refs + r.locations = sv.Locations } // Initializes variables using values passed from the command line flag @@ -295,52 +193,34 @@ func (r *Root) InitializeVariables(vars []string) error { return nil } -func (r *Root) Merge(other *Root) error { - // Merge dynamic configuration values. - return r.Mutate(func(root dyn.Value) (dyn.Value, error) { - return merge.Merge(root, other.value) - }) -} - -func mergeField(rv, ov dyn.Value, name string) (dyn.Value, error) { - path := dyn.NewPath(dyn.Key(name)) - reference, _ := dyn.GetByPath(rv, path) - override, _ := dyn.GetByPath(ov, path) - - // Merge the override into the reference. - var out dyn.Value - var err error - if reference.IsValid() && override.IsValid() { - out, err = merge.Merge(reference, override) - if err != nil { - return dyn.InvalidValue, err +// Merge merges the other configurations into this one, in order. +func (r *Root) Merge(others ...*Root) error { + for _, other := range others { + if err := r.MergeAt(nil, other.View()); err != nil { + return err } - } else if reference.IsValid() { - out = reference - } else if override.IsValid() { - out = override - } else { - return rv, nil } - - return dyn.SetByPath(rv, path, out) + return nil } +var bundleGitPath = structpath.MustParsePath("bundle.git") + func (r *Root) MergeTargetOverrides(name string) error { - root := r.value - target, err := dyn.GetByPath(root, dyn.NewPath(dyn.Key("targets"), dyn.Key(name))) - if err != nil { - return err + targetPath := structpath.NewStringKeys(nil, "targets", name) + target := r.View().Lookup(targetPath) + if !target.IsValid() { + return fmt.Errorf("target %s not found", name) } // Confirm validity of variable overrides. - err = validateVariableOverrides(root, target) + err := validateVariableOverrides(r.Variables, r.Targets[name]) if err != nil { return err } - // Merge fields that can be merged 1:1. - for _, f := range []string{ + // Merge fields that can be merged 1:1. Check all of them first so that a failed + // merge leaves the configuration unchanged. + fields := []string{ "bundle", "workspace", "artifacts", @@ -348,106 +228,73 @@ func (r *Root) MergeTargetOverrides(name string) error { "sync", "permissions", "presets", - } { - if root, err = mergeField(root, target, f); err != nil { + } + for _, f := range fields { + if err := structvar.CheckMerge(r.View().Get(f), target.Get(f)); err != nil { + return fmt.Errorf("failed to merge target=%s field=%s: %w", name, f, err) + } + } + if err := structvar.CheckMerge(r.View().Lookup(bundleGitPath), target.Get("git")); err != nil { + return err + } + for _, f := range fields { + if err := r.MergeAt(structpath.NewStringKey(nil, f), target.Get(f)); err != nil { return fmt.Errorf("failed to merge target=%s field=%s: %w", name, f, err) } } // Merge `variables`. This field must be overwritten if set, not merged. - if v := target.Get("variables"); v.Kind() != dyn.KindInvalid { - _, err = dyn.Map(v, ".", dyn.Foreach(func(p dyn.Path, variable dyn.Value) (dyn.Value, error) { - varPath := dyn.MustPathFromString("variables").Append(p...) - - vDefault := variable.Get("default") - if vDefault.Kind() != dyn.KindInvalid { - defaultPath := varPath.Append(dyn.Key("default")) - root, err = dyn.SetByPath(root, defaultPath, vDefault) - if err != nil { - return root, err - } + for varName, variable := range target.Get("variables").MapItems() { + varPath := structpath.NewStringKeys(nil, "variables", varName) - // If the target explicitly sets a default value, drop any lookup from the - // root variable definition so SetVariables can assign this default. - lookupPath := varPath.Append(dyn.Key("lookup")) - root, err = dyn.SetByPath(root, lookupPath, dyn.NilValue) + if vDefault := variable.Get("default"); vDefault.IsValid() { + if err := r.Assign(structpath.NewStringKey(varPath, "default"), vDefault); err != nil { + return err } - vLookup := variable.Get("lookup") - if vLookup.Kind() != dyn.KindInvalid { - lookupPath := varPath.Append(dyn.Key("lookup")) - root, err = dyn.SetByPath(root, lookupPath, vLookup) - if err != nil { - return root, err - } + // If the target explicitly sets a default value, drop any lookup from the + // root variable definition so SetVariables can assign this default. + if err := r.Delete(structpath.NewStringKey(varPath, "lookup")); err != nil { + return err + } + } - // If the target explicitly sets a lookup, drop any default value from the - // root variable definition so lookup resolution remains authoritative. - defaultPath := varPath.Append(dyn.Key("default")) - root, err = dyn.SetByPath(root, defaultPath, dyn.NilValue) + if vLookup := variable.Get("lookup"); vLookup.IsValid() { + if err := r.Assign(structpath.NewStringKey(varPath, "lookup"), vLookup); err != nil { + return err } - return root, err - })) - if err != nil { - return err + // If the target explicitly sets a lookup, drop any default value from the + // root variable definition so lookup resolution remains authoritative. + if err := r.Delete(structpath.NewStringKey(varPath, "default")); err != nil { + return err + } } } // Merge `run_as`. This field must be overwritten if set, not merged. - if v := target.Get("run_as"); v.Kind() != dyn.KindInvalid { - root, err = dyn.Set(root, "run_as", v) - if err != nil { - return err - } - } - - // Below, we're setting fields on the bundle key, so make sure it exists. - if root.Get("bundle").Kind() == dyn.KindInvalid { - root, err = dyn.Set(root, "bundle", dyn.V(map[string]dyn.Value{})) - if err != nil { + if v := target.Get("run_as"); v.IsValid() { + if err := r.Assign(structpath.NewStringKey(nil, "run_as"), v); err != nil { return err } } // Merge `mode`. This field must be overwritten if set, not merged. - if v := target.Get("mode"); v.Kind() != dyn.KindInvalid { - root, err = dyn.SetByPath(root, dyn.NewPath(dyn.Key("bundle"), dyn.Key("mode")), v) - if err != nil { + if v := target.Get("mode"); v.IsValid() { + if err := r.Assign(structpath.MustParsePath("bundle.mode"), v); err != nil { return err } } // Merge `cluster_id`. This field must be overwritten if set, not merged. - if v := target.Get("cluster_id"); v.Kind() != dyn.KindInvalid { - root, err = dyn.SetByPath(root, dyn.NewPath(dyn.Key("bundle"), dyn.Key("cluster_id")), v) - if err != nil { + if v := target.Get("cluster_id"); v.IsValid() { + if err := r.Assign(structpath.MustParsePath("bundle.cluster_id"), v); err != nil { return err } } // Merge `git`. - if v := target.Get("git"); v.Kind() != dyn.KindInvalid { - ref, err := dyn.GetByPath(root, dyn.NewPath(dyn.Key("bundle"), dyn.Key("git"))) - if err != nil { - ref = dyn.V(map[string]dyn.Value{}) - } - - // Merge the override into the reference. - out, err := merge.Merge(ref, v) - if err != nil { - return err - } - - // Set the merged value. - root, err = dyn.SetByPath(root, dyn.NewPath(dyn.Key("bundle"), dyn.Key("git")), out) - if err != nil { - return err - } - } - - // Convert normalized configuration tree to typed configuration. - return r.updateWithDynamicValue(root) + return r.MergeAt(bundleGitPath, target.Get("git")) } var allowedVariableDefinitions = []([]string){ @@ -459,30 +306,25 @@ var allowedVariableDefinitions = []([]string){ {"lookup"}, } -// isFullVariableOverrideDef checks if the given value is a full syntax variable override. -// A full syntax variable override is a map with either 1 of 2 keys. +// isFullVariableOverrideDef checks if a mapping with the given keys is a full syntax +// variable override. A full syntax variable override is a map with either 1 of 2 keys. // If it's 2 keys, the keys should be "default" and "type". // If it's 1 key, the key should be one of the following keys: "default", "lookup". -func isFullVariableOverrideDef(v dyn.Value) bool { - mv, ok := v.AsMap() - if !ok { - return false - } - +func isFullVariableOverrideDef(keys []string) bool { // If the map has more than 3 keys, it is not a full variable override. - if mv.Len() > 3 { + if len(keys) > 3 { return false } - for _, keys := range allowedVariableDefinitions { - if len(keys) != mv.Len() { + for _, allowed := range allowedVariableDefinitions { + if len(allowed) != len(keys) { continue } // Check if the keys are the same. match := true - for _, key := range keys { - if _, ok := mv.GetByString(key); !ok { + for _, key := range allowed { + if !slices.Contains(keys, key) { match = false break } @@ -496,114 +338,234 @@ func isFullVariableOverrideDef(v dyn.Value) bool { return false } +// dealias returns the node an alias refers to, or node itself. +func dealias(node *yaml.Node) *yaml.Node { + for node != nil && node.Kind == yaml.AliasNode { + node = node.Alias + } + return node +} + +// mappingValue returns the value of key in the YAML mapping node (following aliases). +func mappingValue(node *yaml.Node, key string) *yaml.Node { + node = dealias(node) + if node == nil || node.Kind != yaml.MappingNode { + return nil + } + for i := 0; i+1 < len(node.Content); i += 2 { + if node.Content[i].Value == key { + return dealias(node.Content[i+1]) + } + } + return nil +} + +// expandMergeKeys returns a copy of the mapping node with "<<" merge keys replaced +// by the pairs they merge in (keys set explicitly take precedence), so the pairs +// can be rewritten individually. +func expandMergeKeys(node *yaml.Node) *yaml.Node { + var merged, explicit []*yaml.Node + for i := 0; i+1 < len(node.Content); i += 2 { + if node.Content[i].Value != "<<" || node.Content[i].ShortTag() != "!!merge" { + explicit = append(explicit, node.Content[i], node.Content[i+1]) + continue + } + sources := []*yaml.Node{dealias(node.Content[i+1])} + if sources[0].Kind == yaml.SequenceNode { + sources = sources[0].Content + } + for _, src := range sources { + if src = dealias(src); src.Kind == yaml.MappingNode { + merged = append(merged, expandMergeKeys(src).Content...) + } + } + } + if merged == nil { + return node + } + out := *node + out.Content = explicit + for i := 0; i+1 < len(merged); i += 2 { + if mappingValue(&out, merged[i].Value) == nil { + out.Content = append(out.Content, merged[i], merged[i+1]) + } + } + return &out +} + +func mappingKeys(node *yaml.Node) []string { + var keys []string + for i := 0; i+1 < len(node.Content); i += 2 { + keys = append(keys, node.Content[i].Value) + } + return keys +} + +// mappingNode returns a mapping node with the given keys and values, at the location of at. +func mappingNode(at *yaml.Node, kvs ...any) *yaml.Node { + out := &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map", Line: at.Line, Column: at.Column} + for i := 0; i < len(kvs); i += 2 { + out.Content = append(out.Content, &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: kvs[i].(string)}, kvs[i+1].(*yaml.Node)) + } + return out +} + // rewriteShorthands performs lightweight rewriting of the configuration // tree where we allow users to write a shorthand and must rewrite to the full form. -func rewriteShorthands(v dyn.Value) (dyn.Value, error) { - if v.Kind() != dyn.KindMap { - return v, nil +func rewriteShorthands(root *yaml.Node) { + targets := mappingValue(root, "targets") + if targets == nil || targets.Kind != yaml.MappingNode { + return } // For each target, rewrite the variables block. - return dyn.Map(v, "targets", dyn.Foreach(func(_ dyn.Path, target dyn.Value) (dyn.Value, error) { - // Confirm it has a variables block. - if target.Get("variables").Kind() == dyn.KindInvalid { - return target, nil + for i := 1; i < len(targets.Content); i += 2 { + variables := mappingValue(targets.Content[i], "variables") + if variables == nil || variables.Kind != yaml.MappingNode { + continue } + *variables = *expandMergeKeys(variables) - // For each variable, normalize its contents if it is a single string. - return dyn.Map(target, "variables", dyn.Foreach(func(p dyn.Path, variable dyn.Value) (dyn.Value, error) { - switch variable.Kind() { - - case dyn.KindString, dyn.KindBool, dyn.KindFloat, dyn.KindInt: + for j := 0; j+1 < len(variables.Content); j += 2 { + name := variables.Content[j].Value + variable := dealias(variables.Content[j+1]) + switch { + case variable.Kind == yaml.ScalarNode && variable.ShortTag() != "!!null": // Rewrite the variable to a map with a single key called "default". // This conforms to the variable type. Normalization back to the typed // configuration will convert this to a string if necessary. - return dyn.NewValue(map[string]dyn.Value{ - "default": variable, - }, variable.Locations()), nil + variables.Content[j+1] = mappingNode(variable, "default", variable) - case dyn.KindMap, dyn.KindSequence: + case variable.Kind == yaml.MappingNode || variable.Kind == yaml.SequenceNode: // If it's a full variable definition, leave it as is. - if isFullVariableOverrideDef(variable) { - return variable, nil + if variable.Kind == yaml.MappingNode && isFullVariableOverrideDef(mappingKeys(variable)) { + continue } // Check if the original definition of variable has a type field. // If it has a type field, it means the shorthand is a value of a complex type. // Type might not be found if the variable overridden in a separate file // and configuration is not merged yet. - typeV, err := dyn.GetByPath(v, p.Append(dyn.Key("type"))) - if err == nil && typeV.MustString() == "complex" { - return dyn.NewValue(map[string]dyn.Value{ - "type": typeV, - "default": variable, - }, variable.Locations()), nil + typeV := mappingValue(mappingValue(mappingValue(root, "variables"), name), "type") + if typeV != nil && typeV.Value == "complex" { + variables.Content[j+1] = mappingNode(variable, "type", typeV, "default", variable) + continue } // If it's a shorthand, rewrite it to a full variable definition. - return dyn.NewValue(map[string]dyn.Value{ - "default": variable, - }, variable.Locations()), nil - + variables.Content[j+1] = mappingNode(variable, "default", variable) default: - return variable, nil } - })) - })) + } + } } // validateVariableOverrides checks that all variables specified // in the target override are also defined in the root. -func validateVariableOverrides(root, target dyn.Value) (err error) { - var rv map[string]variable.Variable - var tv map[string]variable.Variable - - // Collect variables from the root. - if v := root.Get("variables"); v.Kind() != dyn.KindInvalid { - err = convert.ToTyped(&rv, v) - if err != nil { - return fmt.Errorf("unable to collect variables from root: %w", err) - } - } - - // Collect variables from the target. - if v := target.Get("variables"); v.Kind() != dyn.KindInvalid { - err = convert.ToTyped(&tv, v) - if err != nil { - return fmt.Errorf("unable to collect variables from target: %w", err) - } +func validateVariableOverrides(root map[string]*variable.Variable, target *Target) error { + if target == nil { + return nil } - - // Check that all variables in the target exist in the root. - for k := range tv { - if _, ok := rv[k]; !ok { + for k := range target.Variables { + if _, ok := root[k]; !ok { return fmt.Errorf("variable %s is not defined but is assigned a value", k) } } - return nil } +// Set sets value at path (see [structvar.StructVar.Set]). +func (r *Root) Set(path *structpath.PathNode, value any) error { + sv := r.vars() + defer r.store(sv) + return sv.Set(path, value) +} + +// SetReference records the pure reference ref at path, a field that cannot hold a string. +func (r *Root) SetReference(path *structpath.PathNode, ref string) error { + sv := r.vars() + defer r.store(sv) + return sv.SetReference(path, ref) +} + +// Assign sets the value described by v at path, with its locations and references. +func (r *Root) Assign(path *structpath.PathNode, v structvar.View) error { + _, err := r.Decode(path, v) + return err +} + +// Decode sets the value described by v at path, converting it to the type at path; +// the diagnostics explain values that could not be converted and were dropped. +func (r *Root) Decode(path *structpath.PathNode, v structvar.View) (diag.Diagnostics, error) { + sv := r.vars() + defer r.store(sv) + return sv.Assign(path, v) +} + +// Delete removes the value at path (see [structvar.StructVar.Delete]). +func (r *Root) Delete(path *structpath.PathNode) error { + sv := r.vars() + defer r.store(sv) + return sv.Delete(path) +} + +// MergeAt merges the value described by v into the value at path (see [structvar.StructVar.Merge]). +func (r *Root) MergeAt(path *structpath.PathNode, v structvar.View) error { + sv := r.vars() + defer r.store(sv) + return sv.Merge(path, v) +} + +// MergeElementsByKey merges the elements of the sequence at path that have the same +// key (see [structvar.StructVar.MergeElementsByKey]). +func (r *Root) MergeElementsByKey(path *structpath.PathNode, keyField string, keyFn func(structvar.View) string, sortKeys bool) error { + sv := r.vars() + defer r.store(sv) + return sv.MergeElementsByKey(path, keyField, keyFn, sortKeys) +} + +// SetLocations sets the locations of the value at path and all values below it. +func (r *Root) SetLocations(path *structpath.PathNode, locs []diag.Location) { + sv := r.vars() + sv.SetLocations(path, locs) + r.store(sv) +} + +// UpdateSequence records that the elements of the sequence at path were rebuilt from +// the old ones (see [structvar.StructVar.UpdateSequence]). +func (r *Root) UpdateSequence(path *structpath.PathNode, sources [][]int) { + sv := r.vars() + sv.UpdateSequence(path, sources) + r.store(sv) +} + +// LocationsAt returns all locations of the configuration value at the specified path. +func (r Root) LocationsAt(path *structpath.PathNode) []diag.Location { + return r.locations.At(path) +} + // Best effort to get the location of configuration value at the specified path. // This function is useful to annotate error messages with the location, because // we don't want to fail with a different error message if we cannot retrieve the location. func (r Root) GetLocation(path string) diag.Location { - v, err := dyn.Get(r.value, path) - if err != nil { + locs := r.GetLocations(path) + if len(locs) == 0 { return diag.Location{} } - return v.Location() + return locs[0] } // Get all locations of the configuration value at the specified path. We need both // this function and it's singular version (GetLocation) because some diagnostics just need // the primary location and some need all locations associated with a configuration value. +// A value without locations (e.g. set by a mutator) gets those of its closest ancestor +// that has some; use [Root.DefinitionLocation] to find where a value is defined. func (r Root) GetLocations(path string) []diag.Location { - v, err := dyn.Get(r.value, path) + p, err := structpath.ParsePath(path) if err != nil { return nil } - return v.Locations() + return r.locations.Nearest(p) } // GetLocationOf is [Root.GetLocation] for a path node. @@ -633,38 +595,52 @@ func (r Root) valueOf(path *structpath.PathNode) (dyn.Value, bool) { return v, err == nil } +// DefinitionLocation returns the primary location the value at path is defined at, +// or an empty location if it has none (e.g. it was set by a mutator). +func (r Root) DefinitionLocation(path string) diag.Location { + p, err := structpath.ParsePath(path) + if err != nil { + return diag.Location{} + } + locs := r.locations.At(p) + if len(locs) == 0 { + return diag.Location{} + } + return locs[0] +} + // GetNodeAndType and returns parent resource node and type of the resource in direct backend. // Examples: // // "resources.jobs.foo.name" -> ("resources.jobs.foo", "jobs") // "resources.jobs.foo.permissions[0].level -> ("resources.jobs.foo.permissions", "jobs.permissions") -func GetNodeAndType(path dyn.Path) (dyn.Path, string) { - if len(path) < 3 { +func GetNodeAndType(path *structpath.PathNode) (*structpath.PathNode, string) { + if path.Len() < 3 { return nil, "" } - if path[0].Key() != "resources" { + if path.KeyAt(0) != "resources" { return nil, "" } - if len(path) >= 4 { - if path[3].Key() == "permissions" || path[3].Key() == "grants" { - return path[:4], path[1].Key() + "." + path[3].Key() + if path.Len() >= 4 { + if k := path.KeyAt(3); k == "permissions" || k == "grants" { + return path.Prefix(4), path.KeyAt(1) + "." + k } } - return path[:3], path[1].Key() + return path.Prefix(3), path.KeyAt(1) } // GetResourceTypeFromKey extracts the resource group from a resource path. // For example, "resources.jobs.foo" returns "jobs". // Returns empty string if the path is not in the expected format. func GetResourceTypeFromKey(path string) string { - dp, err := dyn.NewPathFromString(path) + p, err := structpath.ParsePath(path) if err != nil { return "" } - _, rType := GetNodeAndType(dp) + _, rType := GetNodeAndType(p) return rType } @@ -673,13 +649,13 @@ func GetResourceTypeFromKey(path string) string { // The returned value is a pointer to the concrete struct that represents that resource type. // When the path is invalid or resource is not found, the second return value is false. func (r *Root) GetResourceConfig(path string) (any, error) { - dynPath, err := dyn.NewPathFromString(path) + p, err := structpath.ParsePath(path) if err != nil { return nil, err } // Extract and validate the resource group from the path - node, resourceType := GetNodeAndType(dynPath) + node, resourceType := GetNodeAndType(p) if resourceType == "" { return nil, fmt.Errorf("path does not correspond to resource: %q", path) } @@ -690,18 +666,15 @@ func (r *Root) GetResourceConfig(path string) (any, error) { return nil, fmt.Errorf("no such resource type in the config: %q", resourceType) } - // Fetch the raw value from the dynamic representation of the bundle config. - v, err := dyn.GetByPath(r.Value(), node) - if err != nil { - if dyn.IsNoSuchKeyError(err) { - return nil, nil - } - return nil, fmt.Errorf("cannot fetch config for %s: %w", node, err) + // Copy the value, so that the caller can't change the configuration through it. + v := r.View().Lookup(node) + if !v.IsValid() { + return nil, nil } typedConfigPtr := reflect.New(typ) - err = convert.ToTyped(typedConfigPtr.Interface(), v) + _, err = (&structvar.StructVar{Value: typedConfigPtr.Interface()}).Assign(nil, v) if err != nil { return nil, fmt.Errorf("cannot convert config to %s: %w", typ.String(), err) } @@ -709,8 +682,38 @@ func (r *Root) GetResourceConfig(path string) (any, error) { return typedConfigPtr.Interface(), nil } -// Value returns the dynamic configuration value of the root object. This value -// is the source of truth and is kept in sync with values in the typed configuration. -func (r Root) Value() dyn.Value { - return r.value +// IsReference reports whether the value at path is a pure variable reference in a +// field that cannot hold a string (its typed value is the zero value). +func (r Root) IsReference(path string) bool { + p, err := structpath.ParsePath(path) + if err != nil { + return false + } + _, ok := r.refs[p.String()] + return ok +} + +// References returns all pure variable references in fields that cannot hold a string, +// keyed by path. References in string fields are part of their value. +func (r Root) References() iter.Seq2[*structpath.PathNode, string] { + return func(yield func(*structpath.PathNode, string) bool) { + for _, k := range slices.Sorted(maps.Keys(r.refs)) { + p, err := structpath.ParsePath(k) + if err == nil && !yield(p, r.refs[k]) { + return + } + } + } +} + +// View returns the read-only view of the configuration tree. +func (r *Root) View() structvar.View { + return structvar.NewView(r, r.refs, r.locations) +} + +// Override replaces the configuration with the result of plan (see [structvar.PlanOverride]). +func (r *Root) Override(plan *structvar.OverridePlan) error { + sv := r.vars() + defer r.store(sv) + return sv.Override(plan) } diff --git a/bundle/config/root_test.go b/bundle/config/root_test.go index f735c44405e..a0bc32ab031 100644 --- a/bundle/config/root_test.go +++ b/bundle/config/root_test.go @@ -2,11 +2,13 @@ package config import ( "encoding/json" + "maps" "reflect" + "slices" "testing" "github.com/databricks/cli/bundle/config/variable" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -100,7 +102,6 @@ func TestRootMergeTargetOverridesWithMode(t *testing.T) { }, }, } - require.NoError(t, root.initializeDynamicValue()) require.NoError(t, root.MergeTargetOverrides("development")) assert.Equal(t, Development, root.Bundle.Mode) } @@ -156,7 +157,6 @@ func TestRootMergeTargetOverridesWithVariables(t *testing.T) { }, }, } - require.NoError(t, root.initializeDynamicValue()) require.NoError(t, root.MergeTargetOverrides("development")) assert.Equal(t, "bar", root.Variables["foo"].Default) assert.Equal(t, "foo var", root.Variables["foo"].Description) @@ -172,83 +172,138 @@ func TestRootMergeTargetOverridesWithVariables(t *testing.T) { func TestIsFullVariableOverrideDef(t *testing.T) { testCases := []struct { - value dyn.Value + value map[string]any expected bool }{ { - value: dyn.V(map[string]dyn.Value{ - "type": dyn.V("string"), - "default": dyn.V("foo"), - "description": dyn.V("foo var"), - }), + value: map[string]any{ + "type": "string", + "default": "foo", + "description": "foo var", + }, expected: true, }, { - value: dyn.V(map[string]dyn.Value{ - "type": dyn.V("string"), - "lookup": dyn.V("foo"), - "description": dyn.V("foo var"), - }), + value: map[string]any{ + "type": "string", + "lookup": "foo", + "description": "foo var", + }, expected: false, }, { - value: dyn.V(map[string]dyn.Value{ - "type": dyn.V("string"), - "default": dyn.V("foo"), - }), + value: map[string]any{ + "type": "string", + "default": "foo", + }, expected: true, }, { - value: dyn.V(map[string]dyn.Value{ - "type": dyn.V("string"), - "lookup": dyn.V("foo"), - }), + value: map[string]any{ + "type": "string", + "lookup": "foo", + }, expected: false, }, { - value: dyn.V(map[string]dyn.Value{ - "description": dyn.V("string"), - "default": dyn.V("foo"), - }), + value: map[string]any{ + "description": "string", + "default": "foo", + }, expected: true, }, { - value: dyn.V(map[string]dyn.Value{ - "description": dyn.V("string"), - "lookup": dyn.V("foo"), - }), + value: map[string]any{ + "description": "string", + "lookup": "foo", + }, expected: true, }, { - value: dyn.V(map[string]dyn.Value{ - "default": dyn.V("foo"), - }), + value: map[string]any{ + "default": "foo", + }, expected: true, }, { - value: dyn.V(map[string]dyn.Value{ - "lookup": dyn.V("foo"), - }), + value: map[string]any{ + "lookup": "foo", + }, expected: true, }, { - value: dyn.V(map[string]dyn.Value{ - "type": dyn.V("string"), - }), + value: map[string]any{ + "type": "string", + }, expected: false, }, { - value: dyn.V(map[string]dyn.Value{ - "type": dyn.V("string"), - "default": dyn.V("foo"), - "description": dyn.V("foo var"), - "lookup": dyn.V("foo"), - }), + value: map[string]any{ + "type": "string", + "default": "foo", + "description": "foo var", + "lookup": "foo", + }, expected: false, }, } for i, tc := range testCases { - assert.Equal(t, tc.expected, isFullVariableOverrideDef(tc.value), "test case %d", i) + keys := slices.Collect(maps.Keys(tc.value)) + assert.Equal(t, tc.expected, isFullVariableOverrideDef(keys), "test case %d", i) + } +} + +func TestLoadFromBytesNotAMap(t *testing.T) { + for _, content := range []string{"hello\n", "- a\n- b\n"} { + r, diags := LoadFromBytes("databricks.yml", []byte(content)) + assert.Nil(t, r) + assert.ErrorContains(t, diags.Error(), "failed to load databricks.yml") } } + +func TestMergeComplexVariableSequenceDefaults(t *testing.T) { + a, diags := LoadFromBytes("a.yml", []byte("variables:\n v:\n type: complex\n default: [1, 2]\n")) + require.NoError(t, diags.Error()) + b, diags := LoadFromBytes("b.yml", []byte("variables:\n v:\n type: complex\n default: [3]\n")) + require.NoError(t, diags.Error()) + require.NoError(t, a.Merge(b)) + assert.Equal(t, []any{1, 2, 3}, a.Variables["v"].Default) +} + +func TestTargetVariableShorthandThroughAliases(t *testing.T) { + r, diags := LoadFromBytes("a.yml", []byte(` +x: &dev dev_value +y: &common {b: from_anchor, c: from_anchor} +variables: + a: {default: d} + b: {default: d} + c: {default: d} +targets: + dev: + variables: + <<: *common + a: *dev + c: explicit +`)) + require.NoError(t, diags.Error()) + require.NoError(t, r.MergeTargetOverrides("dev")) + assert.Equal(t, "dev_value", r.Variables["a"].Default) + assert.Equal(t, "from_anchor", r.Variables["b"].Default) + assert.Equal(t, "explicit", r.Variables["c"].Default) +} + +func TestNestedYAMLAnchorCycle(t *testing.T) { + _, diags := LoadFromBytes("a.yml", []byte("variables:\n v:\n default: &x {a: [*x]}\n")) + assert.ErrorContains(t, diags.Error(), `cyclic reference to anchor "x"`) +} + +func TestSetReferenceAtPointerField(t *testing.T) { + r, diags := LoadFromBytes("a.yml", []byte("resources:\n jobs:\n j:\n name: j\n")) + require.NoError(t, diags.Error()) + path := structpath.MustParsePath("resources.jobs.j.trigger") + require.NoError(t, r.SetReference(path, "${var.t}")) + s, ok := r.View().Lookup(path).AsString() + require.True(t, ok) + assert.Equal(t, "${var.t}", s) +} diff --git a/bundle/config/validate/all_resources_have_values.go b/bundle/config/validate/all_resources_have_values.go index 713ec0f83af..166115d5557 100644 --- a/bundle/config/validate/all_resources_have_values.go +++ b/bundle/config/validate/all_resources_have_values.go @@ -3,11 +3,14 @@ package validate import ( "context" "fmt" + "maps" + "reflect" + "slices" "strings" "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" ) func AllResourcesHaveValues() bundle.Mutator { @@ -23,33 +26,25 @@ func (m *allResourcesHaveValues) Name() string { func (m *allResourcesHaveValues) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { diags := diag.Diagnostics{} - _, err := dyn.MapByPattern( - b.Config.Value(), - dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey()), - func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - if v.Kind() != dyn.KindNil { - return v, nil + for _, group := range b.Config.Resources.AllResources() { + for _, rName := range slices.Sorted(maps.Keys(group.Resources)) { + // A resource declared without a body is a nil pointer. + if !reflect.ValueOf(group.Resources[rName]).IsNil() { + continue } // Type of the resource, stripped of the trailing 's' to make it // singular. - rType := strings.TrimSuffix(p[1].Key(), "s") - - // Name of the resource. Eg: "foo" in "jobs.foo". - rName := p[2].Key() + rType := strings.TrimSuffix(group.Description.PluralName, "s") + p := structpath.NewStringKeys(nil, "resources", group.Description.PluralName, rName) diags = append(diags, diag.Diagnostic{ Severity: diag.Error, Summary: fmt.Sprintf("%s %s is not defined", rType, rName), - Locations: v.Locations(), - Paths: dyn.ToStructPaths(p), + Locations: b.Config.LocationsAt(p), + Paths: []*structpath.PathNode{p}, }) - - return v, nil - }, - ) - if err != nil { - diags = append(diags, diag.FromErr(err)...) + } } return diags diff --git a/bundle/config/validate/enum.go b/bundle/config/validate/enum.go index 2ce99ea1d19..392b431c50c 100644 --- a/bundle/config/validate/enum.go +++ b/bundle/config/validate/enum.go @@ -4,13 +4,14 @@ import ( "cmp" "context" "fmt" + "maps" "slices" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/internal/validation/generated" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/dynvar" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) type enum struct{} @@ -27,23 +28,15 @@ func (f *enum) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { diags := diag.Diagnostics{} // Generate prefix tree for all enum fields. - trie := &dyn.TrieNode{} - for k := range generated.EnumFields { - pattern, err := dyn.NewPatternFromString(k) - if err != nil { - return diag.FromErr(fmt.Errorf("invalid pattern %q for enum field validation: %w", k, err)) - } - - err = trie.Insert(pattern) - if err != nil { - return diag.FromErr(fmt.Errorf("failed to insert pattern %q into trie: %w", k, err)) - } + patterns, err := newPatternSet(slices.Collect(maps.Keys(generated.EnumFields))) + if err != nil { + return diag.FromErr(fmt.Errorf("enum field validation: %w", err)) } - err := dyn.WalkReadOnly(b.Config.Value(), func(p dyn.Path, v dyn.Value) error { - // If the path is not found in the prefix tree, we do not need to validate any enum + err = structvar.Walk(b.Config.View(), func(np *structpath.PathNode, v structvar.View) error { + // If the path matches no pattern, we do not need to validate any enum // fields in it. - pattern, ok := trie.SearchPath(p) + pattern, ok := patterns.find(np) if !ok { return nil } @@ -56,25 +49,22 @@ func (f *enum) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { // Skip validation for values containing variable references (e.g. // ${resources.jobs.my_job.id}) since they are not yet resolved. - if dynvar.ContainsVariableReference(strValue) { + if structvar.ContainsVariableReference(strValue) { return nil } // Get valid values for this pattern - validValues := generated.EnumFields[pattern.String()] + validValues := generated.EnumFields[pattern] // Check if the value is in the list of valid enum values validValue := slices.Contains(validValues, strValue) if !validValue { - // p is a slice of path components. We need to clone it before using it in diagnostics - // since the WalkReadOnly function will mutate it while walking the config tree. - diags = diags.Append(diag.Diagnostic{ Severity: diag.Warning, Summary: fmt.Sprintf("invalid value %q for enum field. Valid values are %v", strValue, validValues), Locations: v.Locations(), - Paths: dyn.ToStructPaths(p), + Paths: []*structpath.PathNode{np}, }) } diff --git a/bundle/config/validate/files_to_sync.go b/bundle/config/validate/files_to_sync.go index 14b032e8da1..d1b811e17a7 100644 --- a/bundle/config/validate/files_to_sync.go +++ b/bundle/config/validate/files_to_sync.go @@ -56,14 +56,14 @@ func (v *filesToSync) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnost Summary: "There are no files to sync, please check your .gitignore", }) } else { - path := structpath.NewPath(nil, "sync", "exclude") + path := "sync.exclude" diags = diags.Append(diag.Diagnostic{ Severity: diag.Warning, Summary: "There are no files to sync, please check your .gitignore and sync.exclude configuration", // Show all locations where sync.exclude is defined, since merging // sync.exclude is additive. - Locations: b.Config.GetLocationsOf(path), - Paths: []*structpath.PathNode{path}, + Locations: b.Config.GetLocations(path), + Paths: structpath.MustParsePaths(path), }) } diff --git a/bundle/config/validate/interpolation_in_auth_config.go b/bundle/config/validate/interpolation_in_auth_config.go index f749f9e0613..e1b9d6fba18 100644 --- a/bundle/config/validate/interpolation_in_auth_config.go +++ b/bundle/config/validate/interpolation_in_auth_config.go @@ -7,8 +7,8 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/auth" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/dynvar" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) type noInterpolationInAuthConfig struct{} @@ -51,14 +51,11 @@ func (f *noInterpolationInAuthConfig) Apply(ctx context.Context, b *bundle.Bundl diags := diag.Diagnostics{} for _, fieldName := range authFields { - p := dyn.NewPath(dyn.Key("workspace"), dyn.Key(fieldName)) - v, err := dyn.GetByPath(b.Config.Value(), p) - if dyn.IsNoSuchKeyError(err) { + p := structpath.NewStringKeys(nil, "workspace", fieldName) + v := b.Config.View().Lookup(p) + if !v.IsValid() { continue } - if err != nil { - return diag.FromErr(err) - } vv, ok := v.AsString() if !ok { @@ -66,7 +63,7 @@ func (f *noInterpolationInAuthConfig) Apply(ctx context.Context, b *bundle.Bundl } // Check if the field contains interpolation. - if dynvar.ContainsVariableReference(vv) { + if structvar.ContainsVariableReference(vv) { envVar, ok := auth.GetEnvFor(fieldName) if !ok { continue @@ -78,7 +75,7 @@ func (f *noInterpolationInAuthConfig) Apply(ctx context.Context, b *bundle.Bundl Detail: fmt.Sprintf(`Interpolation is not supported for the field %s. Please set the %s environment variable if you wish to configure this field at runtime.`, p.String(), envVar), Locations: v.Locations(), - Paths: dyn.ToStructPaths(p), + Paths: []*structpath.PathNode{p}, }) } } diff --git a/bundle/config/validate/job_cluster_key_defined.go b/bundle/config/validate/job_cluster_key_defined.go index eb9bea4df70..f3641a658ae 100644 --- a/bundle/config/validate/job_cluster_key_defined.go +++ b/bundle/config/validate/job_cluster_key_defined.go @@ -31,13 +31,14 @@ func (v *jobClusterKeyDefined) Apply(ctx context.Context, b *bundle.Bundle) diag } for index, task := range job.Tasks { + taskPath := structpath.NewIndex(structpath.NewStringKeys(nil, "resources", "jobs", k, "tasks"), index) diags = diags.Extend(checkJobClusterKey(b, jobClusterKeys, task.JobClusterKey, - structpath.NewPath(nil, "resources", "jobs", k, "tasks", index, "job_cluster_key"))) + structpath.NewStringKeys(taskPath, "job_cluster_key"))) // The Jobs API rejects nested for_each_task, so one level is sufficient. if task.ForEachTask != nil { diags = diags.Extend(checkJobClusterKey(b, jobClusterKeys, task.ForEachTask.Task.JobClusterKey, - structpath.NewPath(nil, "resources", "jobs", k, "tasks", index, "for_each_task", "task", "job_cluster_key"))) + structpath.NewStringKeys(taskPath, "for_each_task", "task", "job_cluster_key"))) } } } @@ -60,7 +61,7 @@ func checkJobClusterKey(b *bundle.Bundle, jobClusterKeys map[string]bool, jobClu // Show only the location where the job_cluster_key is defined. // Other associated locations are not relevant since they are // overridden during merging. - Locations: b.Config.GetLocationsOf(path), + Locations: b.Config.GetLocations(path.String()), Paths: []*structpath.PathNode{path}, }} } diff --git a/bundle/config/validate/job_task_cluster_spec.go b/bundle/config/validate/job_task_cluster_spec.go index e387d83f6fb..66f7bf8fc3d 100644 --- a/bundle/config/validate/job_task_cluster_spec.go +++ b/bundle/config/validate/job_task_cluster_spec.go @@ -7,7 +7,6 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/databricks-sdk-go/service/jobs" ) @@ -27,13 +26,13 @@ func (v *jobTaskClusterSpec) Name() string { func (v *jobTaskClusterSpec) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { diags := diag.Diagnostics{} - jobsPath := dyn.NewPath(dyn.Key("resources"), dyn.Key("jobs")) + jobsPath := structpath.NewStringKeys(nil, "resources", "jobs") for resourceName, job := range b.Config.Resources.Jobs { - resourcePath := jobsPath.Append(dyn.Key(resourceName)) + resourcePath := structpath.NewStringKeys(jobsPath, resourceName) for taskIndex, task := range job.Tasks { - taskPath := resourcePath.Append(dyn.Key("tasks"), dyn.Index(taskIndex)) + taskPath := structpath.NewIndex(structpath.NewStringKeys(resourcePath, "tasks"), taskIndex) diags = diags.Extend(validateJobTask(b, task, taskPath)) } @@ -42,7 +41,7 @@ func (v *jobTaskClusterSpec) Apply(ctx context.Context, b *bundle.Bundle) diag.D return diags } -func validateJobTask(b *bundle.Bundle, task jobs.Task, taskPath dyn.Path) diag.Diagnostics { +func validateJobTask(b *bundle.Bundle, task jobs.Task, taskPath *structpath.PathNode) diag.Diagnostics { diags := diag.Diagnostics{} var specified []string @@ -73,7 +72,7 @@ func validateJobTask(b *bundle.Bundle, task jobs.Task, taskPath dyn.Path) diag.D } if task.ForEachTask != nil { - forEachTaskPath := taskPath.Append(dyn.Key("for_each_task"), dyn.Key("task")) + forEachTaskPath := structpath.NewStringKeys(taskPath, "for_each_task", "task") diags = diags.Extend(validateJobTask(b, task.ForEachTask.Task, forEachTaskPath)) } @@ -89,13 +88,12 @@ func validateJobTask(b *bundle.Bundle, task jobs.Task, taskPath dyn.Path) diag.D strings.Join(unspecified, ", "), ) - taskNode := dyn.ToStructPath(taskPath) diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, Summary: "Missing required cluster or environment settings", Detail: detail, - Locations: b.Config.GetLocationsOf(taskNode), - Paths: []*structpath.PathNode{taskNode}, + Locations: b.Config.GetLocations(taskPath.String()), + Paths: []*structpath.PathNode{taskPath}, }) } } diff --git a/bundle/config/validate/no_interpolation_in_bundle_name.go b/bundle/config/validate/no_interpolation_in_bundle_name.go index 8fc547b174e..0850f27ac15 100644 --- a/bundle/config/validate/no_interpolation_in_bundle_name.go +++ b/bundle/config/validate/no_interpolation_in_bundle_name.go @@ -5,9 +5,9 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn/dynvar" "github.com/databricks/cli/libs/logdiag" "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) type noInterpolationInBundleName struct{} @@ -21,7 +21,7 @@ func (m *noInterpolationInBundleName) Name() string { } func (m *noInterpolationInBundleName) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - if dynvar.ContainsVariableReference(b.Config.Bundle.Name) { + if structvar.ContainsVariableReference(b.Config.Bundle.Name) { logdiag.LogDiag(ctx, diag.Diagnostic{ Severity: diag.Warning, Summary: `Please do not use variable interpolation in the name of your bundle. The name of your bundle diff --git a/bundle/config/validate/no_variable_reference_in_resource_key.go b/bundle/config/validate/no_variable_reference_in_resource_key.go index 75b8361ce97..743d47bd7fd 100644 --- a/bundle/config/validate/no_variable_reference_in_resource_key.go +++ b/bundle/config/validate/no_variable_reference_in_resource_key.go @@ -3,11 +3,14 @@ package validate import ( "context" "fmt" + "maps" + "slices" "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/dynvar" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) type noVariableReferenceInResourceKey struct{} @@ -25,30 +28,27 @@ func (m *noVariableReferenceInResourceKey) Name() string { func (m *noVariableReferenceInResourceKey) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { var diags diag.Diagnostics - patterns := []dyn.Pattern{ - dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey()), - dyn.NewPattern(dyn.Key("targets"), dyn.AnyKey(), dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey()), + check := func(prefix *structpath.PathNode, r *config.Resources) { + for _, group := range r.AllResources() { + for _, key := range slices.Sorted(maps.Keys(group.Resources)) { + if !structvar.ContainsVariableReference(key) { + continue + } + p := structpath.NewStringKeys(prefix, group.Description.PluralName, key) + diags = append(diags, diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("resource key %q must not contain variable references", key), + Locations: b.Config.LocationsAt(p), + Paths: []*structpath.PathNode{p}, + }) + } + } } - for _, pattern := range patterns { - _, err := dyn.MapByPattern( - b.Config.Value(), - pattern, - func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - key := p[len(p)-1].Key() - if dynvar.ContainsVariableReference(key) { - diags = append(diags, diag.Diagnostic{ - Severity: diag.Error, - Summary: fmt.Sprintf("resource key %q must not contain variable references", key), - Locations: v.Locations(), - Paths: dyn.ToStructPaths(p), - }) - } - return v, nil - }, - ) - if err != nil { - diags = append(diags, diag.FromErr(err)...) + check(structpath.NewStringKeys(nil, "resources"), &b.Config.Resources) + for _, name := range slices.Sorted(maps.Keys(b.Config.Targets)) { + if t := b.Config.Targets[name]; t != nil && t.Resources != nil { + check(structpath.NewStringKeys(nil, "targets", name, "resources"), t.Resources) } } diff --git a/bundle/config/validate/pattern_set.go b/bundle/config/validate/pattern_set.go new file mode 100644 index 00000000000..07d68f88212 --- /dev/null +++ b/bundle/config/validate/pattern_set.go @@ -0,0 +1,37 @@ +package validate + +import ( + "fmt" + + "github.com/databricks/cli/libs/structs/structpath" +) + +type patternEntry struct { + key string + pattern *structpath.PatternNode +} + +// patternSet finds the pattern that matches a concrete path exactly, where wildcards match a single component. +type patternSet map[int][]patternEntry + +func newPatternSet(keys []string) (patternSet, error) { + s := patternSet{} + for _, k := range keys { + pattern, err := structpath.ParsePattern(k) + if err != nil { + return nil, fmt.Errorf("invalid pattern %q: %w", k, err) + } + s[pattern.Len()] = append(s[pattern.Len()], patternEntry{k, pattern}) + } + return s, nil +} + +// find returns the key of the pattern matching p. +func (s patternSet) find(p *structpath.PathNode) (string, bool) { + for _, e := range s[p.Len()] { + if p.HasPatternPrefix(e.pattern) { + return e.key, true + } + } + return "", false +} diff --git a/bundle/config/validate/pipeline_development_deprecated.go b/bundle/config/validate/pipeline_development_deprecated.go index 3eb3c77c5f0..d837daa5908 100644 --- a/bundle/config/validate/pipeline_development_deprecated.go +++ b/bundle/config/validate/pipeline_development_deprecated.go @@ -2,11 +2,13 @@ package validate import ( "context" + "maps" + "slices" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" ) const ( @@ -32,35 +34,40 @@ func (v *pipelineDevelopmentDeprecated) Apply(_ context.Context, b *bundle.Bundl presetEnabled := config.IsExplicitlyEnabled(b.Config.Presets.PipelinesDevelopment) - pattern := dyn.NewPattern(dyn.Key("resources"), dyn.Key("pipelines"), dyn.AnyKey(), dyn.Key("development")) - _, err := dyn.MapByPattern(b.Config.Value(), pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { + pipelines := b.Config.Resources.Pipelines + for _, key := range slices.Sorted(maps.Keys(pipelines)) { + pipeline := pipelines[key] + //nolint:staticcheck // SA1019: pipeline development is deprecated in the SDK but remains a supported bundle config field + if pipeline == nil || (!pipeline.Development && !slices.Contains(pipeline.ForceSendFields, "Development")) { + continue + } + + p := structpath.NewStringKeys(nil, "resources", "pipelines", key, "development") + // Only user-written values have a location; the value set by "mode: development" does not. - if len(v.Locations()) == 0 { - return v, nil + locs := b.Config.LocationsAt(p) + if len(locs) == 0 { + continue } diags = append(diags, diag.Diagnostic{ Severity: diag.Warning, Summary: pipelineDevelopmentDeprecatedSummary, - Locations: v.Locations(), - Paths: dyn.ToStructPaths(p), + Locations: locs, + Paths: []*structpath.PathNode{p}, }) // The preset overwrites YAML values with true, so a false here was set by a Python // mutator after the preset ran. "bundle run" still sends development: true. - if development, ok := v.AsBool(); ok && presetEnabled && !development { + if presetEnabled && !pipeline.Development { //nolint:staticcheck // SA1019: see above diags = append(diags, diag.Diagnostic{ Severity: diag.Warning, Summary: pipelineDevelopmentIgnoredSummary, Detail: pipelineDevelopmentIgnoredDetail, - Locations: v.Locations(), - Paths: dyn.ToStructPaths(p), + Locations: locs, + Paths: []*structpath.PathNode{p}, }) } - return v, nil - }) - if err != nil { - return diag.FromErr(err) } return diags diff --git a/bundle/config/validate/required.go b/bundle/config/validate/required.go index 9e3274b6f00..b8bb148b8e4 100644 --- a/bundle/config/validate/required.go +++ b/bundle/config/validate/required.go @@ -4,14 +4,15 @@ import ( "cmp" "context" "fmt" + "maps" "slices" "strings" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/internal/validation/generated" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) type required struct{} @@ -29,36 +30,28 @@ func warnForMissingFields(ctx context.Context, b *bundle.Bundle) diag.Diagnostic diags := diag.Diagnostics{} // Generate prefix tree for all required fields. - trie := &dyn.TrieNode{} - for k := range generated.RequiredFields { - pattern, err := dyn.NewPatternFromString(k) - if err != nil { - return diag.FromErr(fmt.Errorf("invalid pattern %q for required field validation: %w", k, err)) - } - - err = trie.Insert(pattern) - if err != nil { - return diag.FromErr(fmt.Errorf("failed to insert pattern %q into trie: %w", k, err)) - } + patterns, err := newPatternSet(slices.Collect(maps.Keys(generated.RequiredFields))) + if err != nil { + return diag.FromErr(fmt.Errorf("required field validation: %w", err)) } - err := dyn.WalkReadOnly(b.Config.Value(), func(p dyn.Path, v dyn.Value) error { - // If the path is not found in the prefix tree, we do not need to validate any required + err = structvar.Walk(b.Config.View(), func(np *structpath.PathNode, v structvar.View) error { + // If the path matches no pattern, we do not need to validate any required // fields in it. - pattern, ok := trie.SearchPath(p) + pattern, ok := patterns.find(np) if !ok { return nil } - fields := generated.RequiredFields[pattern.String()] + fields := generated.RequiredFields[pattern] for _, field := range fields { vv := v.Get(field) - if vv.Kind() == dyn.KindInvalid || vv.Kind() == dyn.KindNil { + if vv.Kind() == structvar.KindInvalid || vv.Kind() == structvar.KindNil { diags = diags.Append(diag.Diagnostic{ Severity: diag.Warning, Summary: fmt.Sprintf("required field %q is not set", field), Locations: v.Locations(), - Paths: dyn.ToStructPaths(p), + Paths: []*structpath.PathNode{np}, }) } } @@ -103,14 +96,12 @@ func errorForMissingFields(ctx context.Context, b *bundle.Bundle) diag.Diagnosti diags := diag.Diagnostics{} for key, dashboard := range b.Config.Resources.Dashboards { if dashboard.DisplayName == "" { - path := structpath.NewPath(nil, "resources", "dashboards", key) - nameLocations = append(nameLocations, b.Config.GetLocationsOf(path)...) - namePaths = append(namePaths, path) + nameLocations = append(nameLocations, b.Config.GetLocations("resources.dashboards."+key)...) + namePaths = append(namePaths, structpath.NewStringKeys(nil, "resources", "dashboards", key)) } if dashboard.WarehouseId == "" { - path := structpath.NewPath(nil, "resources", "dashboards", key) - warehouseIdLocations = append(warehouseIdLocations, b.Config.GetLocationsOf(path)...) - warehouseIdPaths = append(warehouseIdPaths, path) + warehouseIdLocations = append(warehouseIdLocations, b.Config.GetLocations("resources.dashboards."+key)...) + warehouseIdPaths = append(warehouseIdPaths, structpath.NewStringKeys(nil, "resources", "dashboards", key)) } } @@ -135,11 +126,11 @@ func errorForMissingFields(ctx context.Context, b *bundle.Bundle) diag.Diagnosti // by the backend, which rejects whitespace-only names (name.trim.nonEmpty). for key, warehouse := range b.Config.Resources.SqlWarehouses { if strings.TrimSpace(warehouse.Name) == "" { - path := structpath.NewPath(nil, "resources", "sql_warehouses", key) + path := structpath.NewStringKeys(nil, "resources", "sql_warehouses", key) diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, Summary: "sql_warehouse name is required", - Locations: b.Config.GetLocationsOf(path), + Locations: b.Config.GetLocations(path.String()), Paths: []*structpath.PathNode{path}, }) } @@ -158,16 +149,16 @@ func errorForMissingFields(ctx context.Context, b *bundle.Bundle) diag.Diagnosti func errorForInvalidGrants(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { diags := diag.Diagnostics{} - _, err := dyn.MapByPattern( - b.Config.Value(), - dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey(), dyn.Key("grants"), dyn.AnyIndex()), - func(p dyn.Path, v dyn.Value) (dyn.Value, error) { + err := structvar.ForEach( + b.Config.View(), + structpath.MustParsePattern("resources.*.*.grants[*]"), + func(np *structpath.PathNode, v structvar.View) error { if isMissingOrEmptyString(v.Get("principal")) { diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, Summary: "grant principal is required", Locations: v.Locations(), - Paths: dyn.ToStructPaths(p), + Paths: []*structpath.PathNode{np}, }) } if isMissingOrEmptySequence(v.Get("privileges")) { @@ -175,10 +166,10 @@ func errorForInvalidGrants(ctx context.Context, b *bundle.Bundle) diag.Diagnosti Severity: diag.Error, Summary: "grant privileges is required", Locations: v.Locations(), - Paths: dyn.ToStructPaths(p), + Paths: []*structpath.PathNode{np}, }) } - return v, nil + return nil }, ) if err != nil { @@ -200,14 +191,16 @@ func errorForInvalidSecretScopePermissions(ctx context.Context, b *bundle.Bundle if perm.UserName != "" || perm.GroupName != "" || perm.ServicePrincipalName != "" { continue } + scopePath := structpath.NewStringKeys(nil, "resources", "secret_scopes", key) + path := structpath.NewIndex(structpath.NewStringKeys(scopePath, "permissions"), i) // ApplyBundlePermissions rebuilds permissions via convert.FromTyped and drops // per-entry locations, so point at the scope. diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, Summary: "secret scope permission principal is required", Detail: "Set one of user_name, group_name or service_principal_name", - Locations: b.Config.GetLocationsOf(structpath.NewPath(nil, "resources", "secret_scopes", key)), - Paths: structpath.NewPathSlice("resources", "secret_scopes", key, "permissions", i), + Locations: b.Config.GetLocations(scopePath.String()), + Paths: []*structpath.PathNode{path}, }) } } @@ -218,24 +211,28 @@ func errorForInvalidSecretScopePermissions(ctx context.Context, b *bundle.Bundle } // isMissingOrEmptyString reports whether v is unset, null, or an empty string. -func isMissingOrEmptyString(v dyn.Value) bool { +func isMissingOrEmptyString(v structvar.View) bool { switch v.Kind() { - case dyn.KindInvalid, dyn.KindNil: + case structvar.KindInvalid, structvar.KindNil: return true - case dyn.KindString: - return v.MustString() == "" + case structvar.KindString: + s, _ := v.AsString() + return s == "" default: return false } } // isMissingOrEmptySequence reports whether v is unset, null, or an empty sequence. -func isMissingOrEmptySequence(v dyn.Value) bool { +func isMissingOrEmptySequence(v structvar.View) bool { switch v.Kind() { - case dyn.KindInvalid, dyn.KindNil: + case structvar.KindInvalid, structvar.KindNil: + return true + case structvar.KindSequence: + for range v.Sequence() { + return false + } return true - case dyn.KindSequence: - return len(v.MustSequence()) == 0 default: return false } diff --git a/bundle/config/validate/scripts.go b/bundle/config/validate/scripts.go index 323db769d23..f7e5586d73e 100644 --- a/bundle/config/validate/scripts.go +++ b/bundle/config/validate/scripts.go @@ -9,8 +9,8 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/dynvar" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) type validateScripts struct{} @@ -38,13 +38,13 @@ func (f *validateScripts) Apply(ctx context.Context, b *bundle.Bundle) diag.Diag for _, k := range scriptKeys { script := b.Config.Scripts[k] - contentPath := dyn.NewPath(dyn.Key("scripts"), dyn.Key(k), dyn.Key("content")) + contentPath := structpath.NewStringKeys(nil, "scripts", k, "content") if script.Content == "" { diags = append(diags, diag.Diagnostic{ Severity: diag.Error, Summary: fmt.Sprintf("Script %s has no content", k), - Paths: dyn.ToStructPaths(contentPath), + Paths: []*structpath.PathNode{contentPath}, }) continue } @@ -59,14 +59,13 @@ func (f *validateScripts) Apply(ctx context.Context, b *bundle.Bundle) diag.Diag // validateScriptContent rejects any ${...} reference in a script's content. // Content is passed to the shell as-is, so ${...} would be ambiguous with a // bundle reference; reference a declared env entry with $NAME instead. -func validateScriptContent(b *bundle.Bundle, key, content string, p dyn.Path) diag.Diagnostics { - ref, ok := dynvar.NewRef(dyn.V(content)) +func validateScriptContent(b *bundle.Bundle, key, content string, p *structpath.PathNode) diag.Diagnostics { + ref, ok := structvar.NewRef(content) if !ok { return nil } first := ref.Matches[0][0] - v, _ := dyn.GetByPath(b.Config.Value(), p) return diag.Diagnostics{{ Severity: diag.Error, Summary: fmt.Sprintf("Found %s in script %s.content. Interpolation syntax ${...} is not supported in script content", first, key), @@ -79,8 +78,8 @@ from "content" with $NAME: env: MY_VAR: ${var.foo} content: echo "$MY_VAR"`, - Locations: v.Locations(), - Paths: dyn.ToStructPaths(p), + Locations: b.Config.LocationsAt(p), + Paths: []*structpath.PathNode{p}, }} } @@ -88,13 +87,12 @@ func validateScriptEnv(b *bundle.Bundle, key string, env map[string]string) diag var diags diag.Diagnostics for _, name := range slices.Sorted(maps.Keys(env)) { - ref, ok := dynvar.NewRef(dyn.V(env[name])) + ref, ok := structvar.NewRef(env[name]) if !ok { continue } - envValuePath := dyn.NewPath(dyn.Key("scripts"), dyn.Key(key), dyn.Key("env"), dyn.Key(name)) - v, _ := dyn.GetByPath(b.Config.Value(), envValuePath) + envValuePath := structpath.NewStringKeys(nil, "scripts", key, "env", name) for _, refPath := range ref.References() { prefix, _, _ := strings.Cut(refPath, ".") @@ -104,8 +102,8 @@ func validateScriptEnv(b *bundle.Bundle, key string, env map[string]string) diag diags = append(diags, diag.Diagnostic{ Severity: diag.Error, Summary: fmt.Sprintf("${%s} cannot be used in scripts.%s.env.%s; only ${bundle.*}, ${workspace.*}, and ${var.*} are resolved before scripts execute", refPath, key, name), - Locations: v.Locations(), - Paths: dyn.ToStructPaths(envValuePath), + Locations: b.Config.LocationsAt(envValuePath), + Paths: []*structpath.PathNode{envValuePath}, }) } } diff --git a/bundle/config/validate/single_node_cluster.go b/bundle/config/validate/single_node_cluster.go index 88b7536f80b..b5a1355030b 100644 --- a/bundle/config/validate/single_node_cluster.go +++ b/bundle/config/validate/single_node_cluster.go @@ -6,9 +6,9 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/convert" "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) // Validates that any single node clusters defined in the bundle are correctly configured. @@ -36,7 +36,27 @@ are correctly set in the cluster specification: const singleNodeWarningSummary = `Single node cluster is not correctly configured` -func showSingleNodeClusterWarning(ctx context.Context, v dyn.Value) bool { +// stringMap reads a view of a map of strings. It returns false if v is not a map. +func stringMap(v structvar.View) (map[string]string, bool) { + switch v.Kind() { + case structvar.KindInvalid, structvar.KindNil: + return nil, true + case structvar.KindMap: + default: + return nil, false + } + out := map[string]string{} + for k, c := range v.MapItems() { + s, ok := c.AsString() + if !ok { + return nil, false + } + out[k] = s + } + return out, true +} + +func showSingleNodeClusterWarning(ctx context.Context, v structvar.View) bool { // Check if the user has explicitly set the num_workers to 0. Skip the warning // if that's not the case. numWorkers, ok := v.Get("num_workers").AsInt() @@ -51,28 +71,26 @@ func showSingleNodeClusterWarning(ctx context.Context, v dyn.Value) bool { return false } - // Convenient type that contains the common fields from compute.ClusterSpec and - // pipelines.PipelineCluster that we are interested in. - type ClusterConf struct { - SparkConf map[string]string `json:"spark_conf"` - CustomTags map[string]string `json:"custom_tags"` - PolicyId string `json:"policy_id"` + // Read the common fields from compute.ClusterSpec and pipelines.PipelineCluster + // that we are interested in. + sparkConf, ok := stringMap(v.Get("spark_conf")) + if !ok { + return false } - - conf := &ClusterConf{} - err := convert.ToTyped(conf, v) - if err != nil { + customTags, ok := stringMap(v.Get("custom_tags")) + if !ok { return false } + policyId, _ := v.Get("policy_id").AsString() // If the policy id is set, we don't want to show the warning. This is because // the user might have configured `spark_conf` and `custom_tags` correctly // in their cluster policy. - if conf.PolicyId != "" { + if policyId != "" { return false } - profile, ok := conf.SparkConf["spark.databricks.cluster.profile"] + profile, ok := sparkConf["spark.databricks.cluster.profile"] if !ok { log.Debugf(ctx, "spark_conf spark.databricks.cluster.profile not found in single-node cluster spec") return true @@ -82,7 +100,7 @@ func showSingleNodeClusterWarning(ctx context.Context, v dyn.Value) bool { return true } - master, ok := conf.SparkConf["spark.master"] + master, ok := sparkConf["spark.master"] if !ok { log.Debugf(ctx, "spark_conf spark.master not found in single-node cluster spec") return true @@ -92,7 +110,7 @@ func showSingleNodeClusterWarning(ctx context.Context, v dyn.Value) bool { return true } - resourceClass, ok := conf.CustomTags["ResourceClass"] + resourceClass, ok := customTags["ResourceClass"] if !ok { log.Debugf(ctx, "custom_tag ResourceClass not found in single-node cluster spec") return true @@ -108,33 +126,34 @@ func showSingleNodeClusterWarning(ctx context.Context, v dyn.Value) bool { func (m *singleNodeCluster) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { diags := diag.Diagnostics{} - patterns := []dyn.Pattern{ + patterns := []*structpath.PatternNode{ // Interactive clusters - dyn.NewPattern(dyn.Key("resources"), dyn.Key("clusters"), dyn.AnyKey()), + structpath.MustParsePattern("resources.clusters.*"), // Job clusters - dyn.NewPattern(dyn.Key("resources"), dyn.Key("jobs"), dyn.AnyKey(), dyn.Key("job_clusters"), dyn.AnyIndex(), dyn.Key("new_cluster")), + structpath.MustParsePattern("resources.jobs.*.job_clusters[*].new_cluster"), // Job task clusters - dyn.NewPattern(dyn.Key("resources"), dyn.Key("jobs"), dyn.AnyKey(), dyn.Key("tasks"), dyn.AnyIndex(), dyn.Key("new_cluster")), + structpath.MustParsePattern("resources.jobs.*.tasks[*].new_cluster"), // Job for each task clusters - dyn.NewPattern(dyn.Key("resources"), dyn.Key("jobs"), dyn.AnyKey(), dyn.Key("tasks"), dyn.AnyIndex(), dyn.Key("for_each_task"), dyn.Key("task"), dyn.Key("new_cluster")), + structpath.MustParsePattern("resources.jobs.*.tasks[*].for_each_task.task.new_cluster"), // Pipeline clusters - dyn.NewPattern(dyn.Key("resources"), dyn.Key("pipelines"), dyn.AnyKey(), dyn.Key("clusters"), dyn.AnyIndex()), + structpath.MustParsePattern("resources.pipelines.*.clusters[*]"), } + root := b.Config.View() for _, p := range patterns { - _, err := dyn.MapByPattern(b.Config.Value(), p, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { + err := structvar.ForEach(root, p, func(np *structpath.PathNode, v structvar.View) error { warning := diag.Diagnostic{ Severity: diag.Warning, Summary: singleNodeWarningSummary, Detail: singleNodeWarningDetail, Locations: v.Locations(), - Paths: dyn.ToStructPaths(p), + Paths: []*structpath.PathNode{np}, } if showSingleNodeClusterWarning(ctx, v) { diags = append(diags, warning) } - return v, nil + return nil }) if err != nil { log.Debugf(ctx, "Error while applying single node cluster validation: %s", err) diff --git a/bundle/config/validate/single_node_cluster_test.go b/bundle/config/validate/single_node_cluster_test.go index 997d39ea0ac..fc32796b9a1 100644 --- a/bundle/config/validate/single_node_cluster_test.go +++ b/bundle/config/validate/single_node_cluster_test.go @@ -8,12 +8,12 @@ import ( "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/internal/bundletest" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/databricks-sdk-go/service/compute" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/databricks/databricks-sdk-go/service/pipelines" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func failCases() []struct { @@ -112,10 +112,8 @@ func TestValidateSingleNodeClusterFailForInteractiveClusters(t *testing.T) { bundletest.SetLocation(b, "resources.clusters.foo", []diag.Location{{File: "a.yml", Line: 1, Column: 1}}) // We can't set num_workers to 0 explicitly in the typed configuration. - // Do it on the dyn.Value directly. - bundletest.Mutate(t, b, func(v dyn.Value) (dyn.Value, error) { - return dyn.Set(v, "resources.clusters.foo.num_workers", dyn.V(0)) - }) + // Set it explicitly, since zero values are otherwise treated as unset. + require.NoError(t, b.Config.Set(structpath.MustParsePath("resources.clusters.foo.num_workers"), 0)) diags := SingleNodeCluster().Apply(ctx, b) assert.Equal(t, diag.Diagnostics{ { @@ -123,7 +121,7 @@ func TestValidateSingleNodeClusterFailForInteractiveClusters(t *testing.T) { Summary: singleNodeWarningSummary, Detail: singleNodeWarningDetail, Locations: []diag.Location{{File: "a.yml", Line: 1, Column: 1}}, - Paths: structpath.NewPathSlice("resources", "clusters", "foo"), + Paths: []*structpath.PathNode{structpath.NewStringKeys(nil, "resources", "clusters", "foo")}, }, }, diags) }) @@ -160,10 +158,8 @@ func TestValidateSingleNodeClusterFailForJobClusters(t *testing.T) { bundletest.SetLocation(b, "resources.jobs.foo.job_clusters[0].new_cluster", []diag.Location{{File: "b.yml", Line: 1, Column: 1}}) // We can't set num_workers to 0 explicitly in the typed configuration. - // Do it on the dyn.Value directly. - bundletest.Mutate(t, b, func(v dyn.Value) (dyn.Value, error) { - return dyn.Set(v, "resources.jobs.foo.job_clusters[0].new_cluster.num_workers", dyn.V(0)) - }) + // Set it explicitly, since zero values are otherwise treated as unset. + require.NoError(t, b.Config.Set(structpath.MustParsePath("resources.jobs.foo.job_clusters[0].new_cluster.num_workers"), 0)) diags := SingleNodeCluster().Apply(ctx, b) assert.Equal(t, diag.Diagnostics{ @@ -209,10 +205,8 @@ func TestValidateSingleNodeClusterFailForJobTaskClusters(t *testing.T) { bundletest.SetLocation(b, "resources.jobs.foo.tasks[0].new_cluster", []diag.Location{{File: "c.yml", Line: 1, Column: 1}}) // We can't set num_workers to 0 explicitly in the typed configuration. - // Do it on the dyn.Value directly. - bundletest.Mutate(t, b, func(v dyn.Value) (dyn.Value, error) { - return dyn.Set(v, "resources.jobs.foo.tasks[0].new_cluster.num_workers", dyn.V(0)) - }) + // Set it explicitly, since zero values are otherwise treated as unset. + require.NoError(t, b.Config.Set(structpath.MustParsePath("resources.jobs.foo.tasks[0].new_cluster.num_workers"), 0)) diags := bundle.Apply(ctx, b, SingleNodeCluster()) assert.Equal(t, diag.Diagnostics{ @@ -255,10 +249,8 @@ func TestValidateSingleNodeClusterFailForPipelineClusters(t *testing.T) { bundletest.SetLocation(b, "resources.pipelines.foo.clusters[0]", []diag.Location{{File: "d.yml", Line: 1, Column: 1}}) // We can't set num_workers to 0 explicitly in the typed configuration. - // Do it on the dyn.Value directly. - bundletest.Mutate(t, b, func(v dyn.Value) (dyn.Value, error) { - return dyn.Set(v, "resources.pipelines.foo.clusters[0].num_workers", dyn.V(0)) - }) + // Set it explicitly, since zero values are otherwise treated as unset. + require.NoError(t, b.Config.Set(structpath.MustParsePath("resources.pipelines.foo.clusters[0].num_workers"), 0)) diags := bundle.Apply(ctx, b, SingleNodeCluster()) assert.Equal(t, diag.Diagnostics{ @@ -308,10 +300,8 @@ func TestValidateSingleNodeClusterFailForJobForEachTaskCluster(t *testing.T) { bundletest.SetLocation(b, "resources.jobs.foo.tasks[0].for_each_task.task.new_cluster", []diag.Location{{File: "e.yml", Line: 1, Column: 1}}) // We can't set num_workers to 0 explicitly in the typed configuration. - // Do it on the dyn.Value directly. - bundletest.Mutate(t, b, func(v dyn.Value) (dyn.Value, error) { - return dyn.Set(v, "resources.jobs.foo.tasks[0].for_each_task.task.new_cluster.num_workers", dyn.V(0)) - }) + // Set it explicitly, since zero values are otherwise treated as unset. + require.NoError(t, b.Config.Set(structpath.MustParsePath("resources.jobs.foo.tasks[0].for_each_task.task.new_cluster.num_workers"), 0)) diags := bundle.Apply(ctx, b, SingleNodeCluster()) assert.Equal(t, diag.Diagnostics{ @@ -392,9 +382,7 @@ func TestValidateSingleNodeClusterPassInteractiveClusters(t *testing.T) { } if tc.numWorkers != nil { - bundletest.Mutate(t, b, func(v dyn.Value) (dyn.Value, error) { - return dyn.Set(v, "resources.clusters.foo.num_workers", dyn.V(*tc.numWorkers)) - }) + require.NoError(t, b.Config.Set(structpath.MustParsePath("resources.clusters.foo.num_workers"), *tc.numWorkers)) } diags := bundle.Apply(ctx, b, SingleNodeCluster()) @@ -432,9 +420,7 @@ func TestValidateSingleNodeClusterPassJobClusters(t *testing.T) { } if tc.numWorkers != nil { - bundletest.Mutate(t, b, func(v dyn.Value) (dyn.Value, error) { - return dyn.Set(v, "resources.jobs.foo.job_clusters[0].new_cluster.num_workers", dyn.V(*tc.numWorkers)) - }) + require.NoError(t, b.Config.Set(structpath.MustParsePath("resources.jobs.foo.job_clusters[0].new_cluster.num_workers"), *tc.numWorkers)) } diags := bundle.Apply(ctx, b, SingleNodeCluster()) @@ -472,9 +458,7 @@ func TestValidateSingleNodeClusterPassJobTaskClusters(t *testing.T) { } if tc.numWorkers != nil { - bundletest.Mutate(t, b, func(v dyn.Value) (dyn.Value, error) { - return dyn.Set(v, "resources.jobs.foo.tasks[0].new_cluster.num_workers", dyn.V(*tc.numWorkers)) - }) + require.NoError(t, b.Config.Set(structpath.MustParsePath("resources.jobs.foo.tasks[0].new_cluster.num_workers"), *tc.numWorkers)) } diags := bundle.Apply(ctx, b, SingleNodeCluster()) @@ -509,9 +493,7 @@ func TestValidateSingleNodeClusterPassPipelineClusters(t *testing.T) { } if tc.numWorkers != nil { - bundletest.Mutate(t, b, func(v dyn.Value) (dyn.Value, error) { - return dyn.Set(v, "resources.pipelines.foo.clusters[0].num_workers", dyn.V(*tc.numWorkers)) - }) + require.NoError(t, b.Config.Set(structpath.MustParsePath("resources.pipelines.foo.clusters[0].num_workers"), *tc.numWorkers)) } diags := bundle.Apply(ctx, b, SingleNodeCluster()) @@ -553,9 +535,7 @@ func TestValidateSingleNodeClusterPassJobForEachTaskCluster(t *testing.T) { } if tc.numWorkers != nil { - bundletest.Mutate(t, b, func(v dyn.Value) (dyn.Value, error) { - return dyn.Set(v, "resources.jobs.foo.tasks[0].for_each_task.task.new_cluster.num_workers", dyn.V(*tc.numWorkers)) - }) + require.NoError(t, b.Config.Set(structpath.MustParsePath("resources.jobs.foo.tasks[0].for_each_task.task.new_cluster.num_workers"), *tc.numWorkers)) } diags := bundle.Apply(ctx, b, SingleNodeCluster()) @@ -590,13 +570,8 @@ func TestValidateSingleNodeClusterWithIsSingleNode(t *testing.T) { } // Set num_workers to 0 and is_single_node to true - bundletest.Mutate(t, b, func(v dyn.Value) (dyn.Value, error) { - v, err := dyn.Set(v, "resources.jobs.foo.job_clusters[0].new_cluster.num_workers", dyn.V(0)) - if err != nil { - return v, err - } - return dyn.Set(v, "resources.jobs.foo.job_clusters[0].new_cluster.is_single_node", dyn.V(true)) - }) + require.NoError(t, b.Config.Set(structpath.MustParsePath("resources.jobs.foo.job_clusters[0].new_cluster.num_workers"), 0)) + require.NoError(t, b.Config.Set(structpath.MustParsePath("resources.jobs.foo.job_clusters[0].new_cluster.is_single_node"), true)) diags := bundle.Apply(ctx, b, SingleNodeCluster()) assert.Empty(t, diags) diff --git a/bundle/config/validate/tf_only_references.go b/bundle/config/validate/tf_only_references.go index 3603b6f4a7f..ecbc3d42047 100644 --- a/bundle/config/validate/tf_only_references.go +++ b/bundle/config/validate/tf_only_references.go @@ -8,9 +8,8 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/terraform_dabs_map" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/dynvar" "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) type tfOnlyReferences struct{} @@ -30,8 +29,12 @@ func (m *tfOnlyReferences) Apply(_ context.Context, b *bundle.Bundle) diag.Diagn var diags diag.Diagnostics // Walk the entire config looking for ${resources.*} references. - _ = dyn.WalkReadOnly(b.Config.Value(), func(_ dyn.Path, v dyn.Value) error { - ref, ok := dynvar.NewRef(v) + _ = structvar.Walk(b.Config.View(), func(_ *structpath.PathNode, v structvar.View) error { + s, ok := v.AsString() + if !ok { + return nil + } + ref, ok := structvar.NewRef(s) if !ok { return nil } @@ -57,25 +60,20 @@ func (m *tfOnlyReferences) Apply(_ context.Context, b *bundle.Bundle) diag.Diagn // "resources.jobs.src.always_running" and returns a diagnostic when it refers // to a TF-only field, or nil otherwise. func checkTFOnlyReference(ref string, loc diag.Location) *diag.Diagnostic { - p, err := dyn.NewPathFromString(ref) + p, err := structpath.ParsePath(ref) // Need at least resources... - if err != nil || len(p) < 4 || p[0].Key() != "resources" { + if err != nil || p.Len() < 4 || p.KeyAt(0) != "resources" { return nil } - group := p[1].Key() + group := p.KeyAt(1) tfOnlyFields, ok := terraform_dabs_map.TerraformOnlyFields[group] if !ok || len(tfOnlyFields) == 0 { return nil } // Field path is everything after resources... - fieldNode, err := structpath.ParsePath(p[3:].String()) - if err != nil { - return nil - } - - if !tfOnlyFields.Contains(fieldNode) { + if !tfOnlyFields.Contains(p.SkipPrefix(3)) { return nil } diff --git a/bundle/config/validate/tf_only_references_test.go b/bundle/config/validate/tf_only_references_test.go index 7b994cf864e..a4f5850f8c8 100644 --- a/bundle/config/validate/tf_only_references_test.go +++ b/bundle/config/validate/tf_only_references_test.go @@ -6,9 +6,8 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/resources" - "github.com/databricks/cli/bundle/internal/bundletest" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -31,9 +30,7 @@ func makeBundle(t *testing.T) *bundle.Bundle { func TestTFOnlyReferences_Error(t *testing.T) { b := makeBundle(t) - bundletest.Mutate(t, b, func(v dyn.Value) (dyn.Value, error) { - return dyn.Set(v, "resources.jobs.dst.name", dyn.V("${resources.jobs.src.always_running}")) - }) + require.NoError(t, b.Config.Set(structpath.MustParsePath("resources.jobs.dst.name"), "${resources.jobs.src.always_running}")) diags := TFOnlyReferences().Apply(t.Context(), b) require.Len(t, diags, 1) @@ -44,10 +41,8 @@ func TestTFOnlyReferences_Error(t *testing.T) { func TestTFOnlyReferences_NormalReference(t *testing.T) { b := makeBundle(t) - bundletest.Mutate(t, b, func(v dyn.Value) (dyn.Value, error) { - // "name" is not a TF-only field; no diagnostic expected. - return dyn.Set(v, "resources.jobs.dst.name", dyn.V("${resources.jobs.src.name}")) - }) + // "name" is not a TF-only field; no diagnostic expected. + require.NoError(t, b.Config.Set(structpath.MustParsePath("resources.jobs.dst.name"), "${resources.jobs.src.name}")) diags := TFOnlyReferences().Apply(t.Context(), b) assert.Empty(t, diags) @@ -55,10 +50,8 @@ func TestTFOnlyReferences_NormalReference(t *testing.T) { func TestTFOnlyReferences_RenamedField(t *testing.T) { b := makeBundle(t) - bundletest.Mutate(t, b, func(v dyn.Value) (dyn.Value, error) { - // "git_source[0].branch" is a TF rename (not TF-only), should not error. - return dyn.Set(v, "resources.jobs.dst.name", dyn.V("${resources.jobs.src.git_source[0].branch}")) - }) + // "git_source[0].branch" is a TF rename (not TF-only), should not error. + require.NoError(t, b.Config.Set(structpath.MustParsePath("resources.jobs.dst.name"), "${resources.jobs.src.git_source[0].branch}")) diags := TFOnlyReferences().Apply(t.Context(), b) assert.Empty(t, diags) diff --git a/bundle/config/validate/unique_resource_keys.go b/bundle/config/validate/unique_resource_keys.go index 7d2e49569f7..488edd4e8a6 100644 --- a/bundle/config/validate/unique_resource_keys.go +++ b/bundle/config/validate/unique_resource_keys.go @@ -7,7 +7,6 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/structs/structpath" ) @@ -42,7 +41,7 @@ func (m *uniqueResourceKeys) Apply(ctx context.Context, b *bundle.Bundle) diag.D // Maps of key to the paths and locations the resource / script is defined at. resourceAndScriptMetadata := map[string]*metadata{} - addLocationToMetadata := func(k, prefix string, p dyn.Path, v dyn.Value) { + addLocationToMetadata := func(k string, fullPath *structpath.PathNode) { mv, ok := resourceAndScriptMetadata[k] if !ok { mv = &metadata{ @@ -51,46 +50,22 @@ func (m *uniqueResourceKeys) Apply(ctx context.Context, b *bundle.Bundle) diag.D } } - mv.paths = append(mv.paths, dyn.ToStructPath(dyn.NewPath(dyn.Key(prefix)).Append(p...))) - mv.locations = append(mv.locations, v.Locations()...) + mv.paths = append(mv.paths, fullPath) + mv.locations = append(mv.locations, b.Config.LocationsAt(fullPath)...) resourceAndScriptMetadata[k] = mv } // Gather the paths and locations of all resources - rv := b.Config.Value().Get("resources") - if rv.Kind() != dyn.KindInvalid && rv.Kind() != dyn.KindNil { - _, err := dyn.MapByPattern( - rv, - dyn.NewPattern(dyn.AnyKey(), dyn.AnyKey()), - func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - // The key for the resource. Eg: "my_job" for jobs.my_job. - k := p[1].Key() - addLocationToMetadata(k, "resources", p, v) - return v, nil - }, - ) - if err != nil { - return diag.FromErr(err) + for _, group := range b.Config.Resources.AllResources() { + for k := range group.Resources { + addLocationToMetadata(k, structpath.NewStringKeys(nil, "resources", group.Description.PluralName, k)) } } // track locations for all scripts. - sv := b.Config.Value().Get("scripts") - if sv.Kind() != dyn.KindInvalid && sv.Kind() != dyn.KindNil { - _, err := dyn.MapByPattern( - sv, - dyn.NewPattern(dyn.AnyKey()), - func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - // The key for the script. Eg: "my_script" for scripts.my_script. - k := p[0].Key() - addLocationToMetadata(k, "scripts", p, v) - return v, nil - }, - ) - if err != nil { - return diag.FromErr(err) - } + for k := range b.Config.Scripts { + addLocationToMetadata(k, structpath.NewStringKeys(nil, "scripts", k)) } // If duplicate keys are found, report an error. diff --git a/bundle/config/validate/validate_artifact_path.go b/bundle/config/validate/validate_artifact_path.go index 36494cf7d77..76d7fdc52f4 100644 --- a/bundle/config/validate/validate_artifact_path.go +++ b/bundle/config/validate/validate_artifact_path.go @@ -11,9 +11,8 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/libraries" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/dynvar" "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" "github.com/databricks/databricks-sdk-go/apierr" ) @@ -52,7 +51,7 @@ func extractVolumeFromPath(artifactPath string) (string, string, string, error) return catalogName, schemaName, volumeName, nil } -func findVolumeInBundle(r config.Root, catalogName, schemaName, volumeName string) (dyn.Path, []diag.Location, bool) { +func findVolumeInBundle(r config.Root, catalogName, schemaName, volumeName string) (*structpath.PathNode, []diag.Location, bool) { volumes := r.Resources.Volumes for k, v := range volumes { if v.CatalogName != catalogName || v.Name != volumeName { @@ -64,12 +63,13 @@ func findVolumeInBundle(r config.Root, catalogName, schemaName, volumeName strin // schema name is interpolated. // We only have to check for ${resources.schemas...} references because any // other valid reference (like ${var.foo}) would have been interpolated by this point. - p, ok := dynvar.PureReferenceToPath(v.SchemaName) - isSchemaDefinedInBundle := ok && p.HasPrefix(dyn.Path{dyn.Key("resources"), dyn.Key("schemas")}) + p, ok := structvar.PureReferenceToPath(v.SchemaName) + isSchemaDefinedInBundle := ok && p.HasPrefix(structpath.MustParsePath("resources.schemas")) if v.SchemaName != schemaName && !isSchemaDefinedInBundle { continue } - return dyn.Path{dyn.Key("resources"), dyn.Key("volumes"), dyn.Key(k)}, r.GetLocationsOf(structpath.NewPath(nil, "resources", "volumes", k)), true + pathString := "resources.volumes." + k + return structpath.MustParsePath(pathString), r.GetLocations(pathString), true } return nil, nil, false } @@ -86,7 +86,7 @@ func (v *validateArtifactPath) Apply(ctx context.Context, b *bundle.Bundle) diag Summary: s, Severity: diag.Error, Locations: b.Config.GetLocations("workspace.artifact_path"), - Paths: structpath.NewPathSlice("workspace", "artifact_path"), + Paths: structpath.MustParsePaths("workspace.artifact_path"), }, } } @@ -118,7 +118,7 @@ this bundle but which has not been deployed yet. Please first deploy the volume using 'bundle deploy' and then switch over to using it in the artifact_path.`, Locations: slices.Concat(b.Config.GetLocations("workspace.artifact_path"), locations), - Paths: append(structpath.NewPathSlice("workspace", "artifact_path"), dyn.ToStructPath(path)), + Paths: []*structpath.PathNode{structpath.MustParsePath("workspace.artifact_path"), path}, }} } diff --git a/bundle/config/validate/validate_artifact_path_test.go b/bundle/config/validate/validate_artifact_path_test.go index 97a08ab6e30..778326430e0 100644 --- a/bundle/config/validate/validate_artifact_path_test.go +++ b/bundle/config/validate/validate_artifact_path_test.go @@ -8,7 +8,6 @@ import ( "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/internal/bundletest" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/experimental/mocks" @@ -209,7 +208,7 @@ func TestFindVolumeInBundle(t *testing.T) { Line: 1, Column: 2, }}, locations) - assert.Equal(t, dyn.MustPathFromString("resources.volumes.foo"), path) + assert.Equal(t, structpath.MustParsePath("resources.volumes.foo"), path) // wrong volume name _, _, ok = findVolumeInBundle(b.Config, "main", "my_schema", "doesnotexist") @@ -238,5 +237,5 @@ func TestFindVolumeInBundle(t *testing.T) { Line: 1, Column: 2, }}, locations) - assert.Equal(t, dyn.MustPathFromString("resources.volumes.foo"), path) + assert.Equal(t, structpath.MustParsePath("resources.volumes.foo"), path) } diff --git a/bundle/config/validate/validate_dashboard_etags.go b/bundle/config/validate/validate_dashboard_etags.go index 1715f22cb84..2510f52e78f 100644 --- a/bundle/config/validate/validate_dashboard_etags.go +++ b/bundle/config/validate/validate_dashboard_etags.go @@ -27,7 +27,7 @@ func (v *validateDashboardEtags) Apply(ctx context.Context, b *bundle.Bundle) di { Severity: diag.Error, Summary: fmt.Sprintf("dashboard %q has an etag set. Etags must not be set in bundle configuration", dashboard.DisplayName), - Paths: structpath.NewPathSlice("resources", "dashboards", k), + Paths: []*structpath.PathNode{structpath.NewStringKeys(nil, "resources", "dashboards", k)}, Locations: b.Config.GetLocationsOf(structpath.NewPath(nil, "resources", "dashboards", k)), }, } diff --git a/bundle/config/validate/validate_deployment_fields.go b/bundle/config/validate/validate_deployment_fields.go index d1ca337f4ed..f824329adbf 100644 --- a/bundle/config/validate/validate_deployment_fields.go +++ b/bundle/config/validate/validate_deployment_fields.go @@ -26,29 +26,29 @@ func (v *validateDeploymentFields) Apply(_ context.Context, b *bundle.Bundle) di // deployment_id and version_id identify the bundle deployment and its version // in the Deployment Metadata Service. The CLI sets them on every deploy, so a // value provided by hand would be overwritten; reject it up front. - reject := func(resourceType, name, field, value string) { + reject := func(resource *structpath.PathNode, field, value string) { if value == "" { return } - path := structpath.NewPath(nil, "resources", resourceType, name, "deployment", field) + path := structpath.NewStringKeys(resource, "deployment", field) diags = append(diags, diag.Diagnostic{ Severity: diag.Error, Summary: field + " must not be set in bundle configuration; it is managed by Declarative Automation Bundles", Paths: []*structpath.PathNode{path}, - Locations: b.Config.GetLocationsOf(path), + Locations: b.Config.GetLocations(path.String()), }) } for name, job := range b.Config.Resources.Jobs { if d := job.Deployment; d != nil { - reject("jobs", name, "deployment_id", d.DeploymentId) - reject("jobs", name, "version_id", d.VersionId) + reject(structpath.NewStringKeys(nil, "resources", "jobs", name), "deployment_id", d.DeploymentId) + reject(structpath.NewStringKeys(nil, "resources", "jobs", name), "version_id", d.VersionId) } } for name, pipeline := range b.Config.Resources.Pipelines { if d := pipeline.Deployment; d != nil { - reject("pipelines", name, "deployment_id", d.DeploymentId) - reject("pipelines", name, "version_id", d.VersionId) + reject(structpath.NewStringKeys(nil, "resources", "pipelines", name), "deployment_id", d.DeploymentId) + reject(structpath.NewStringKeys(nil, "resources", "pipelines", name), "version_id", d.VersionId) } } diff --git a/bundle/config/validate/validate_engine.go b/bundle/config/validate/validate_engine.go index f19c2270b26..11f7b106b87 100644 --- a/bundle/config/validate/validate_engine.go +++ b/bundle/config/validate/validate_engine.go @@ -7,7 +7,6 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config/engine" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/structs/structpath" ) @@ -26,7 +25,8 @@ func (v *validateEngine) Name() string { func (v *validateEngine) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { var diags diag.Diagnostics - if tf := dyn.GetValue(b.Config.Value(), "bundle.terraform"); tf.IsValid() { + tf := b.Config.View().Lookup(structpath.MustParsePath("bundle.terraform")) + if tf.IsValid() { diags = diags.Append(diag.Diagnostic{ Severity: diag.Warning, Summary: "bundle.terraform is deprecated and has no effect: " + engine.TerraformRemovedSummary, @@ -40,7 +40,7 @@ func (v *validateEngine) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnos return diags } - loc := dyn.GetValue(b.Config.Value(), "bundle.engine").Location() + loc := b.Config.GetLocation("bundle.engine") parsed, ok := engine.Parse(string(configEngine)) if !ok { diff --git a/bundle/config/validate/validate_genie_space_etags.go b/bundle/config/validate/validate_genie_space_etags.go index 963e572346f..53252c720a3 100644 --- a/bundle/config/validate/validate_genie_space_etags.go +++ b/bundle/config/validate/validate_genie_space_etags.go @@ -29,7 +29,7 @@ func (v *validateGenieSpaceEtags) Apply(ctx context.Context, b *bundle.Bundle) d { Severity: diag.Error, Summary: fmt.Sprintf("genie space %q has an etag set. Etags must not be set in bundle configuration", genieSpace.Title), - Paths: structpath.NewPathSlice("resources", "genie_spaces", k), + Paths: []*structpath.PathNode{structpath.NewStringKeys(nil, "resources", "genie_spaces", k)}, Locations: b.Config.GetLocationsOf(structpath.NewPath(nil, "resources", "genie_spaces", k)), }, } diff --git a/bundle/config/validate/validate_job_run_idempotency_token.go b/bundle/config/validate/validate_job_run_idempotency_token.go index 61e07e302ca..c211e5a6364 100644 --- a/bundle/config/validate/validate_job_run_idempotency_token.go +++ b/bundle/config/validate/validate_job_run_idempotency_token.go @@ -32,12 +32,12 @@ func (v *validateJobRunIdempotencyToken) Apply(_ context.Context, b *bundle.Bund } // The CLI mints the token; a configured one would also remain reserved after // the run is deleted and break the next deploy. - path := structpath.NewPath(nil, "resources", "job_runs", name, "idempotency_token") + path := structpath.NewStringKeys(nil, "resources", "job_runs", name, "idempotency_token") diags = append(diags, diag.Diagnostic{ Severity: diag.Error, Summary: "idempotency_token must not be set in bundle configuration; the CLI sets it on each run-now request", Paths: []*structpath.PathNode{path}, - Locations: b.Config.GetLocationsOf(path), + Locations: b.Config.GetLocations(path.String()), }) } diff --git a/bundle/configsync/diff.go b/bundle/configsync/diff.go index e42f0a1d83a..aeb593afe50 100644 --- a/bundle/configsync/diff.go +++ b/bundle/configsync/diff.go @@ -8,10 +8,9 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/deployplan" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/convert" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/structs/structdiff" + "github.com/databricks/cli/libs/structs/structvar" ) type OperationType string @@ -39,13 +38,8 @@ type ResourceChanges map[string]*ConfigChangeDesc type Changes map[string]ResourceChanges -func normalizeValue(v any) (any, error) { - dynValue, err := convert.FromTyped(v, dyn.NilValue) - if err != nil { - return nil, fmt.Errorf("failed to convert value of type %T: %w", v, err) - } - - return dynValue.AsAny(), nil +func normalizeValue(v any) any { + return structvar.NewView(v, nil, nil).AsAny() } func filterEntityDefaults(basePath string, value any) any { @@ -92,10 +86,7 @@ func convertChangeDesc(path string, cd *deployplan.ChangeDesc) (*ConfigChangeDes // cd.Old in this check would classify the change as Replace and fail later in // resolveSelectors because the old key no longer exists in the YAML. hasConfigValue := cd.New != nil - normalizedValue, err := normalizeValue(cd.Remote) - if err != nil { - return nil, fmt.Errorf("failed to normalize remote value: %w", err) - } + normalizedValue := normalizeValue(cd.Remote) if shouldSkipField(path, normalizedValue, hasConfigValue) { return &ConfigChangeDesc{ @@ -129,11 +120,7 @@ func convertChangeDesc(path string, cd *deployplan.ChangeDesc) (*ConfigChangeDes // which keys on the path component (index 3), so a resource literally named // "permissions" ("resources.jobs.permissions") is not misclassified. func isPermissionsOrGrantsSubResource(resourceKey string) bool { - path, err := dyn.NewPathFromString(resourceKey) - if err != nil { - return false - } - _, nodeType := config.GetNodeAndType(path) + nodeType := config.GetResourceTypeFromKey(resourceKey) return strings.HasSuffix(nodeType, ".permissions") || strings.HasSuffix(nodeType, ".grants") } diff --git a/bundle/configsync/resolve.go b/bundle/configsync/resolve.go index cdad5182336..abd74295216 100644 --- a/bundle/configsync/resolve.go +++ b/bundle/configsync/resolve.go @@ -12,10 +12,10 @@ import ( "strings" "github.com/databricks/cli/bundle" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/notebook" "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) type FieldChange struct { @@ -42,7 +42,7 @@ type resolvedChange struct { path *structpath.PatternNode steps []sequenceStep // leaf is the merged value the change addresses, invalid for a new field. - leaf dyn.Value + leaf structvar.View // operation decides how many destinations the change needs: a removal has to // reach every definition, an edit only the one that wins the merge. operation OperationType @@ -62,28 +62,23 @@ func resolveSelectors(pathStr string, b *bundle.Bundle, operation OperationType) nodes := node.AsSlice() var result *structpath.PatternNode var steps []sequenceStep - var currentPath dyn.Path - currentValue := b.Config.Value() + var currentPath *structpath.PathNode + currentValue := b.Config.View() for component, n := range nodes { if key, ok := n.StringKey(); ok { result = structpath.NewPatternStringKey(result, key) - currentPath = append(currentPath, dyn.Key(key)) - if currentValue.IsValid() { - currentValue, _ = dyn.GetByPath(currentValue, dyn.Path{dyn.Key(key)}) - } + currentPath = structpath.NewStringKey(currentPath, key) + currentValue = currentValue.Get(key) continue } if idx, ok := n.Index(); ok { - sequencePath := slices.Clone(currentPath) + sequencePath := currentPath result = structpath.NewPatternIndex(result, idx) - currentPath = append(currentPath, dyn.Index(idx)) - var element dyn.Value - if currentValue.IsValid() { - element, _ = dyn.GetByPath(currentValue, dyn.Path{dyn.Index(idx)}) - currentValue = element - } + currentPath = structpath.NewIndex(currentPath, idx) + element := currentValue.Index(idx) + currentValue = element steps = append(steps, sequenceStep{ component: component, sequencePath: sequencePath, @@ -98,26 +93,22 @@ func resolveSelectors(pathStr string, b *bundle.Bundle, operation OperationType) // Check for key-value selector: [key='value'] if key, value, ok := n.KeyValue(); ok { - if !currentValue.IsValid() || currentValue.Kind() != dyn.KindSequence { + if !currentValue.IsValid() || currentValue.Kind() != structvar.KindSequence { return resolvedChange{}, fmt.Errorf("cannot apply [%s='%s'] selector to non-array value in path %s", key, value, pathStr) } - seq, _ := currentValue.AsSequence() foundIndex := -1 + var foundElement structvar.View - for i, elem := range seq { - keyValue, err := dyn.GetByPath(elem, dyn.Path{dyn.Key(key)}) - if err != nil { - continue - } - - if keyValue.Kind() == dyn.KindString && keyValue.MustString() == value { + for i, elem := range currentValue.Sequence() { + if keyValue, ok := elem.Get(key).AsString(); ok && keyValue == value { foundIndex = i + foundElement = elem break } } - sequencePath := slices.Clone(currentPath) + sequencePath := currentPath if foundIndex == -1 { if operation == OperationAdd { @@ -128,20 +119,20 @@ func resolveSelectors(pathStr string, b *bundle.Bundle, operation OperationType) newElement: true, }) // Can't navigate further into non-existent element - currentValue = dyn.Value{} + currentValue = structvar.View{} continue } return resolvedChange{}, fmt.Errorf("no array element found with %s='%s' in path %s", key, value, pathStr) } result = structpath.NewPatternIndex(result, foundIndex) - currentPath = append(currentPath, dyn.Index(foundIndex)) + currentPath = structpath.NewIndex(currentPath, foundIndex) steps = append(steps, sequenceStep{ component: component, sequencePath: sequencePath, - element: seq[foundIndex], + element: foundElement, }) - currentValue = seq[foundIndex] + currentValue = foundElement continue } } @@ -200,7 +191,7 @@ func adjustArrayIndex(path *structpath.PatternNode, scope string, operations map // disappears looks identical to one that was written. preResolved is the merged // config with ${...} references still literal, used to detect variable-reference // parents. -func ResolveChanges(ctx context.Context, b *bundle.Bundle, configChanges Changes, preResolved dyn.Value) ([]FieldChange, int, error) { +func ResolveChanges(ctx context.Context, b *bundle.Bundle, configChanges Changes, preResolved structvar.View) ([]FieldChange, int, error) { var result []FieldChange skipped := 0 targetName := b.Config.Bundle.Target @@ -360,7 +351,7 @@ func ResolveChanges(ctx context.Context, b *bundle.Bundle, configChanges Changes // on it belongs in that same block. filePath = block.file } else { - resourceLocation := b.Config.GetLocation(resourceKey) + resourceLocation := b.Config.DefinitionLocation(resourceKey) filePath = resourceLocation.File } if filePath == "" { diff --git a/bundle/configsync/sequences.go b/bundle/configsync/sequences.go index 4ce0274fdb3..1be8c0d484b 100644 --- a/bundle/configsync/sequences.go +++ b/bundle/configsync/sequences.go @@ -12,9 +12,9 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) // Write-back has to turn a position in the merged configuration into a position in a @@ -56,13 +56,13 @@ func (b sourceBlock) scopeKey() string { } // blockResolver answers which physical block a merged value came from, by location: -// merging accumulates them (libs/dyn/merge), so a value assembled from two blocks +// merging accumulates them, so a value assembled from two blocks // reports a location in each. Selecting a target folds its overrides into resources and // drops the targets subtree, so the blocks are recovered by parsing the files again. type blockResolver struct { // blocks holds one parsed file per contributing file, keyed by block, so a // path relative to a block can be looked up inside it. - blocks map[sourceBlock]dyn.Value + blocks map[sourceBlock]structvar.View target string byLocation map[diag.Location]sourceBlock } @@ -74,13 +74,10 @@ type blockResolver struct { // The contributing files are parsed directly rather than reloaded through the mutator // pipeline, which would run the bundle's preinit script a second time. func newBlockResolver(ctx context.Context, b *bundle.Bundle) *blockResolver { - root := b.Config.Value() - if !root.IsValid() { - return nil - } + root := b.Config.View() r := &blockResolver{ - blocks: make(map[sourceBlock]dyn.Value), + blocks: make(map[sourceBlock]structvar.View), target: b.Config.Bundle.Target, byLocation: make(map[diag.Location]sourceBlock), } @@ -92,14 +89,15 @@ func newBlockResolver(ctx context.Context, b *bundle.Bundle) *blockResolver { log.Debugf(ctx, "config-remote-sync: cannot read %s, treating its sequences as unsplit: %v", file, err) continue } - parsed, diags := config.LoadFromBytes(file, contents) + loaded, diags := config.LoadFromBytes(file, contents) if diags.HasError() { log.Debugf(ctx, "config-remote-sync: cannot parse %s, treating its sequences as unsplit: %v", file, diags.Error()) continue } - r.registerBlock(parsed.Value(), sourceBlock{file: file}) + parsed := loaded.View() + r.registerBlock(parsed, sourceBlock{file: file}) if r.target != "" { - r.registerBlock(parsed.Value(), sourceBlock{override: true, file: file}) + r.registerBlock(parsed, sourceBlock{override: true, file: file}) } } @@ -109,9 +107,9 @@ func newBlockResolver(ctx context.Context, b *bundle.Bundle) *blockResolver { return r } -func referencedFiles(root dyn.Value) map[string]struct{} { +func referencedFiles(root structvar.View) map[string]struct{} { files := map[string]struct{}{} - _ = dyn.WalkReadOnly(root, func(_ dyn.Path, v dyn.Value) error { + _ = structvar.Walk(root, func(_ *structpath.PathNode, v structvar.View) error { for _, location := range v.Locations() { if location.File != "" { files[location.File] = struct{}{} @@ -126,16 +124,16 @@ func referencedFiles(root dyn.Value) map[string]struct{} { // resources subtree back to it, so a merged value can later be traced to the // region it was written in. Does nothing when the file has no such region, which // is why r.blocks doubles as the set of blocks that exist. -func (r *blockResolver) registerBlock(parsed dyn.Value, block sourceBlock) { - subtree, err := dyn.GetByPath(parsed, r.regionPath(block, dyn.NewPath(dyn.Key("resources")))) - if err != nil { +func (r *blockResolver) registerBlock(parsed structvar.View, block sourceBlock) { + subtree := parsed.Lookup(r.regionPath(block, structpath.NewStringKey(nil, "resources"))) + if !subtree.IsValid() { return } // Keep the parsed file: resolving a path inside this block needs the tree it // came from, and the entry also marks the block as present for sortedBlocks. r.blocks[block] = parsed - _ = dyn.WalkReadOnly(subtree, func(_ dyn.Path, v dyn.Value) error { + _ = structvar.Walk(subtree, func(_ *structpath.PathNode, v structvar.View) error { for _, location := range v.Locations() { // A file only carries locations of its own, so a location seen here // belongs to this block. First writer wins: an outer node accumulates @@ -153,11 +151,11 @@ func (r *blockResolver) registerBlock(parsed dyn.Value, block sourceBlock) { } // regionPath prefixes a resources-relative path with the region it belongs to. -func (r *blockResolver) regionPath(block sourceBlock, path dyn.Path) dyn.Path { +func (r *blockResolver) regionPath(block sourceBlock, path *structpath.PathNode) *structpath.PathNode { if !block.override { return path } - return append(dyn.NewPath(dyn.Key("targets"), dyn.Key(r.target)), path...) + return structpath.Join(structpath.NewStringKeys(nil, "targets", r.target), path.AsSlice()...) } // candidatePath renders a resolved path the way the patch layer addresses it, @@ -211,7 +209,7 @@ func compareBlocks(a, b sourceBlock) int { // blocksOf returns the distinct blocks that contributed to value, sorted with the // top-level block first. More than one result means the value is assembled from // several regions and has no single source location. -func (r *blockResolver) blocksOf(value dyn.Value) []sourceBlock { +func (r *blockResolver) blocksOf(value structvar.View) []sourceBlock { var blocks []sourceBlock for _, location := range value.Locations() { block, ok := r.byLocation[location] @@ -233,7 +231,7 @@ func (r *blockResolver) blocksOf(value dyn.Value) []sourceBlock { // Locations accumulate in merge order, so the first one that maps to a block is the // winner and writing any other copy would leave the effective value unchanged. Load // order is the only thing that distinguishes two blocks in the same scope. -func (r *blockResolver) winningBlock(value dyn.Value) (sourceBlock, bool) { +func (r *blockResolver) winningBlock(value structvar.View) (sourceBlock, bool) { for _, location := range value.Locations() { if block, ok := r.byLocation[location]; ok { return block, true @@ -245,17 +243,13 @@ func (r *blockResolver) winningBlock(value dyn.Value) (sourceBlock, bool) { // indexWithinBlock returns the position of element inside block, where sequencePath is // relative to the block. A block is one parsed file, so the sequence read here holds // only that file's entries and a plain index into it is block-local. -func (r *blockResolver) indexWithinBlock(block sourceBlock, sequencePath dyn.Path, element dyn.Value) (int, bool) { +func (r *blockResolver) indexWithinBlock(block sourceBlock, sequencePath *structpath.PathNode, element structvar.View) (int, bool) { parsed, ok := r.blocks[block] if !ok { return 0, false } - sequence, err := dyn.GetByPath(parsed, r.regionPath(block, sequencePath)) - if err != nil { - return 0, false - } - entries, ok := sequence.AsSequence() - if !ok { + sequence := parsed.Lookup(r.regionPath(block, sequencePath)) + if sequence.Kind() != structvar.KindSequence { return 0, false } @@ -264,7 +258,7 @@ func (r *blockResolver) indexWithinBlock(block sourceBlock, sequencePath dyn.Pat locations[location] = struct{}{} } - for local, entry := range entries { + for local, entry := range sequence.Sequence() { for _, location := range entry.Locations() { if _, ok := locations[location]; ok { return local, true @@ -323,7 +317,7 @@ func (d routeDestination) scopeKey() string { func (r *blockResolver) routeDestinations(change resolvedChange) ([]routeDestination, error) { // The value whose definitions have to be reached: the element itself when the // change addresses one, otherwise the field being removed. - var target dyn.Value + var target structvar.View if addressesWholeElement(change) { target = change.steps[len(change.steps)-1].element } else if change.operation == OperationRemove { @@ -489,15 +483,15 @@ func (r *blockResolver) blockForNewElement(change resolvedChange) (sourceBlock, // blocksDefiningSequence returns the blocks that write the sequence at sequencePath. // This is for a value that does not exist yet, where only the receiving sequence is // known; an existing value is traced through its own locations instead. -func (r *blockResolver) blocksDefiningSequence(change resolvedChange, sequencePath dyn.Path) []sourceBlock { +func (r *blockResolver) blocksDefiningSequence(change resolvedChange, sequencePath *structpath.PathNode) []sourceBlock { var blocks []sourceBlock for _, block := range r.sortedBlocks() { blockPath, ok := r.sequencePathWithinBlock(block, change, sequencePath) if !ok { continue } - sequence, err := dyn.GetByPath(r.blocks[block], r.regionPath(block, blockPath)) - if err != nil { + sequence := r.blocks[block].Lookup(r.regionPath(block, blockPath)) + if !sequence.IsValid() { continue } if slices.ContainsFunc(sequence.Locations(), func(l diag.Location) bool { return l.File == block.file }) { @@ -513,22 +507,22 @@ func (r *blockResolver) blocksDefiningSequence(change resolvedChange, sequencePa // // change.steps is ordered outermost first, so each translated index is already known // by the time a deeper step needs it. -func (r *blockResolver) sequencePathWithinBlock(block sourceBlock, change resolvedChange, sequencePath dyn.Path) (dyn.Path, bool) { - result := slices.Clone(sequencePath) +func (r *blockResolver) sequencePathWithinBlock(block sourceBlock, change resolvedChange, sequencePath *structpath.PathNode) (*structpath.PathNode, bool) { + result := sequencePath.AsSlice() for _, step := range change.steps { - at := len(step.sequencePath) - if step.newElement || at >= len(sequencePath) { + at := step.sequencePath.Len() + if step.newElement || at >= len(result) { continue } // The step's own path is a prefix of sequencePath, so the indices translated // so far already apply to it. - index, ok := r.indexWithinBlock(block, result[:at], step.element) + index, ok := r.indexWithinBlock(block, structpath.Join(nil, result[:at]...), step.element) if !ok { return nil, false } - result[at] = dyn.Index(index) + result[at] = structpath.NewIndex(nil, index) } - return result, true + return structpath.Join(nil, result...), true } // pathWithinBlock rewrites a change's path so every sequence index addresses the @@ -582,8 +576,8 @@ type sequenceStep struct { // component is how many path components precede this sequence's index. component int // sequencePath is the sequence itself, e.g. resources.jobs.j.tasks. - sequencePath dyn.Path - element dyn.Value + sequencePath *structpath.PathNode + element structvar.View // newElement marks an Add whose key is not in the merged sequence yet. newElement bool } diff --git a/bundle/configsync/telemetry_test.go b/bundle/configsync/telemetry_test.go index d2a3da537f8..001ecb2f9d2 100644 --- a/bundle/configsync/telemetry_test.go +++ b/bundle/configsync/telemetry_test.go @@ -3,7 +3,6 @@ package configsync import ( "testing" - "github.com/databricks/cli/libs/dyn" "github.com/stretchr/testify/assert" ) @@ -63,37 +62,30 @@ func TestResourceTypeFromKey(t *testing.T) { } func TestRestoreStatsCounters(t *testing.T) { - resolved := dyn.V(map[string]dyn.Value{ - "variables": dyn.V(map[string]dyn.Value{ - "region": dyn.V(map[string]dyn.Value{"value": dyn.V("us-east-1")}), - "other": dyn.V(map[string]dyn.Value{"value": dyn.V("eu-west-1")}), - }), - }) + resolved := variablesConfig(t, map[string]any{"region": "us-east-1", "other": "eu-west-1"}) // Original pure ref still matching: restored but not counted (safe path). var kept RestoreStats - result := restoreOriginalRefs("us-east-1", dyn.V("${var.region}"), resolved, &kept) + result := restoreOriginalRefs("us-east-1", scalarView(t, "${var.region}"), resolved, &kept) assert.Equal(t, "${var.region}", result) assert.Equal(t, RestoreStats{}, kept) // Pure ref whose value changed to another variable's value: re-targeted. var retargeted RestoreStats - result = restoreOriginalRefs("eu-west-1", dyn.V("${var.region}"), resolved, &retargeted) + result = restoreOriginalRefs("eu-west-1", scalarView(t, "${var.region}"), resolved, &retargeted) assert.Equal(t, "${var.other}", result) assert.Equal(t, RestoreStats{Retargeted: 1}, retargeted) // New sequence element leaf restored from a sibling reference. - siblings := []dyn.Value{ - dyn.V(map[string]dyn.Value{"region": dyn.V("${var.region}")}), - } + siblings := tasksView(t, `task_key: "${var.region}"`) var fromSiblings RestoreStats - resultMap := restoreFromSiblings(map[string]any{"region": "us-east-1"}, siblings, resolved, &fromSiblings).(map[string]any) - assert.Equal(t, "${var.region}", resultMap["region"]) + resultMap := restoreFromSiblings(map[string]any{"task_key": "us-east-1"}, siblings, resolved, &fromSiblings).(map[string]any) + assert.Equal(t, "${var.region}", resultMap["task_key"]) assert.Equal(t, RestoreStats{FromSiblings: 1}, fromSiblings) // Hardcoded value: nothing restored, nothing counted. var none RestoreStats - result = restoreOriginalRefs("hardcoded", dyn.V("hardcoded"), resolved, &none) + result = restoreOriginalRefs("hardcoded", scalarView(t, "hardcoded"), resolved, &none) assert.Equal(t, "hardcoded", result) assert.Equal(t, RestoreStats{}, none) } diff --git a/bundle/configsync/variables.go b/bundle/configsync/variables.go index 442224158f5..5382ea1da1f 100644 --- a/bundle/configsync/variables.go +++ b/bundle/configsync/variables.go @@ -9,20 +9,19 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/bundle/config/mutator/resourcemutator" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/dynvar" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) -// varPrefix is the dyn.Path prefix for the ${var.X} shorthand. -var varPrefix = dyn.NewPath(dyn.Key("var")) +// varPrefix is the path prefix for the ${var.X} shorthand. +var varPrefix = structpath.NewStringKey(nil, "var") // RestoreVariableReferences replaces hardcoded change values with variable // references (${var.foo}, ${bundle.target}, ${resources.X.Y.id}) when the // value can be traced back to a reference in the original YAML. Resource IDs // are injected from state since they aren't materialized into the resolved -// config's dyn.Value tree. +// config's tree. // // For Replace operations, restoration consults the pre-resolved YAML at the // exact field position and tries three steps in order: @@ -52,11 +51,11 @@ var varPrefix = dyn.NewPath(dyn.Key("var")) // Restoration counts by mechanism are accumulated into stats (used for // telemetry); pass nil when counters are not needed (the counter methods are // nil-safe). -func RestoreVariableReferences(ctx context.Context, b *bundle.Bundle, fieldChanges []FieldChange, preResolved dyn.Value, stats *RestoreStats) error { +func RestoreVariableReferences(ctx context.Context, b *bundle.Bundle, fieldChanges []FieldChange, preResolved structvar.View, stats *RestoreStats) error { if !preResolved.IsValid() { return errors.New("pre-resolved config unavailable; variable-backed fields will be hardcoded") } - resolved := b.Config.Value() + resolved := resolvedConfig{view: b.Config.View(), overrides: map[string]any{}} // Mirror mutator.lookup's source-linked deployment override: when enabled, // ${workspace.file_path} resolves to b.SyncRootPath rather than the typed @@ -65,21 +64,18 @@ func RestoreVariableReferences(ctx context.Context, b *bundle.Bundle, fieldChang // actual deployed path and variables are lost on Replace. Keep this in // sync with mutator.lookup if new overrides are added there. if config.IsExplicitlyEnabled(b.Config.Presets.SourceLinkedDeployment) { - fpPath := dyn.NewPath(dyn.Key("workspace"), dyn.Key("file_path")) - if updated, err := dyn.SetByPath(resolved, fpPath, dyn.V(b.SyncRootPath)); err == nil { - resolved = updated - } + resolved.overrides["workspace.file_path"] = b.SyncRootPath } // Augment resolved with resource IDs from state — only when the config // actually uses ${resources.X.Y.id} references. The IDs aren't materialized - // into b.Config.Value() (they live in the StateDB), so we inject them here + // into b.Config (they live in the StateDB), so we inject them here // to enable sibling-based restoration. Skipped entirely for bundles with // no resource refs to avoid opening state DB files unnecessarily. resourceRefs := collectResourceIDRefs(preResolved) if len(resourceRefs) > 0 { if lookup := resourceIDLookup(b); lookup != nil { - resolved = injectResourceIDs(ctx, resolved, resourceRefs, lookup) + injectResourceIDs(ctx, resolved, resourceRefs, lookup) } else { log.Debugf(ctx, "variable restoration: state DB unavailable, skipping resource ID injection for %d refs", len(resourceRefs)) } @@ -116,10 +112,10 @@ func RestoreVariableReferences(ctx context.Context, b *bundle.Bundle, fieldChang // LoadPreResolvedConfig loads the bundle's configuration through the standard // loader mutators (entry point, includes, target overrides) but without -// variable resolution. The resulting dyn.Value is fully merged across files +// variable resolution. The resulting configuration is fully merged across files // and targets, yet retains ${...} references as literal strings. Returns -// InvalidValue if loading fails (restoration is then skipped). -func LoadPreResolvedConfig(ctx context.Context, b *bundle.Bundle) dyn.Value { +// an invalid value if loading fails (restoration is then skipped). +func LoadPreResolvedConfig(ctx context.Context, b *bundle.Bundle) structvar.View { fresh := &bundle.Bundle{ BundleRootPath: b.BundleRootPath, BundleRoot: b.BundleRoot, @@ -141,7 +137,26 @@ func LoadPreResolvedConfig(ctx context.Context, b *bundle.Bundle) dyn.Value { resourcemutator.MergePipelineClusters(), resourcemutator.MergeApps(), ) - return fresh.Config.Value() + return fresh.Config.View() +} + +// resolvedConfig is the bundle's resolved configuration plus values that are known +// only outside of it (the source-linked file path, resource IDs from state). +type resolvedConfig struct { + view structvar.View + overrides map[string]any // by path +} + +// lookup returns the value at path as a Go value, and whether it exists. +func (c resolvedConfig) lookup(path *structpath.PathNode) (any, bool) { + if v, ok := c.overrides[path.String()]; ok { + return v, true + } + v := c.view.Lookup(path) + if !v.IsValid() { + return nil, false + } + return v.AsAny(), true } // resourceIDLookup returns a function that resolves resource keys to their @@ -158,20 +173,17 @@ func resourceIDLookup(b *bundle.Bundle) func(string) string { // ${resources...id} references. Returns the unique set of paths // so the caller can inject IDs at those positions; returns nil if no such // references exist. -func collectResourceIDRefs(preResolved dyn.Value) []dyn.Path { +func collectResourceIDRefs(preResolved structvar.View) []*structpath.PathNode { seen := map[string]bool{} - var paths []dyn.Path - _ = dyn.WalkReadOnly(preResolved, func(_ dyn.Path, v dyn.Value) error { + var paths []*structpath.PathNode + _ = structvar.Walk(preResolved, func(_ *structpath.PathNode, v structvar.View) error { s, ok := v.AsString() - if !ok || !dynvar.IsPureVariableReference(s) || seen[s] { + if !ok || !structvar.IsPureVariableReference(s) || seen[s] { return nil } seen[s] = true - p, ok := dynvar.PureReferenceToPath(s) - if !ok { - return nil - } - if len(p) != 4 || p[0].Key() != "resources" || p[3].Key() != "id" { + p, ok := structvar.PureReferenceToPath(s) + if !ok || p.Len() != 4 || p.KeyAt(0) != "resources" || p.KeyAt(3) != "id" { return nil } paths = append(paths, p) @@ -180,47 +192,33 @@ func collectResourceIDRefs(preResolved dyn.Value) []dyn.Path { return paths } -// injectResourceIDs populates the resolved dyn.Value with IDs from state for -// the given resource reference paths. Skips references whose IDs aren't in -// state or that can't be written back into the dyn.Value tree. -func injectResourceIDs(ctx context.Context, resolved dyn.Value, paths []dyn.Path, lookupID func(string) string) dyn.Value { +// injectResourceIDs records IDs from state for the given resource reference paths in +// resolved. Skips references whose IDs aren't in state. +func injectResourceIDs(ctx context.Context, resolved resolvedConfig, paths []*structpath.PathNode, lookupID func(string) string) { for _, p := range paths { - resourceKey := p[:3].String() + resourceKey := p.Prefix(3).String() id := lookupID(resourceKey) if id == "" { log.Debugf(ctx, "variable restoration: no state entry for resource %q", resourceKey) continue } - updated, err := dyn.SetByPath(resolved, p, dyn.V(id)) - if err != nil { - log.Debugf(ctx, "variable restoration: SetByPath failed for %s: %v", p, err) - continue - } - resolved = updated + resolved.overrides[p.String()] = id } - return resolved } -// resolveReferencePath converts a variable reference string to the dyn.Path +// resolveReferencePath converts a variable reference string to the path // where its resolved value can be found in the bundle config. It applies the // same ${var.X} → variables.X.value shorthand rewriting as the variable // resolution mutator. -func resolveReferencePath(refStr string) (dyn.Path, bool) { - p, ok := dynvar.PureReferenceToPath(refStr) +func resolveReferencePath(refStr string) (*structpath.PathNode, bool) { + p, ok := structvar.PureReferenceToPath(refStr) if !ok { return nil, false } - if p.HasPrefix(varPrefix) && len(p) >= 2 { - newPath := dyn.NewPath( - dyn.Key("variables"), - p[1], - dyn.Key("value"), - ) - if len(p) > 2 { - newPath = newPath.Append(p[2:]...) - } - return newPath, true + if p.HasPrefix(varPrefix) && p.Len() >= 2 { + newPath := structpath.NewStringKeys(nil, "variables", p.KeyAt(1), "value") + return structpath.Join(newPath, p.AsSlice()[2:]...), true } return p, true @@ -234,7 +232,7 @@ func resolveReferencePath(refStr string) (dyn.Path, bool) { // new value, falls back to a global lookup: if the new value uniquely matches // a different variable, that variable is used instead. The field's prior use // of a variable is the false-positive guard. -func restoreOriginalRefs(value any, preResolved, resolved dyn.Value, stats *RestoreStats) any { +func restoreOriginalRefs(value any, preResolved structvar.View, resolved resolvedConfig, stats *RestoreStats) any { switch v := value.(type) { case string, bool, int64: if ref, ok := matchOriginalRef(value, preResolved, resolved); ok { @@ -254,26 +252,14 @@ func restoreOriginalRefs(value any, preResolved, resolved dyn.Value, stats *Rest return value case map[string]any: - preMap, _ := preResolved.AsMap() for key, val := range v { - var childPre dyn.Value - if preMap.Len() > 0 { - if p, ok := preMap.GetPairByString(key); ok { - childPre = p.Value - } - } - v[key] = restoreOriginalRefs(val, childPre, resolved, stats) + v[key] = restoreOriginalRefs(val, preResolved.Get(key), resolved, stats) } return v case []any: - preSeq, _ := preResolved.AsSequence() for i, val := range v { - var childPre dyn.Value - if i < len(preSeq) { - childPre = preSeq[i] - } - v[i] = restoreOriginalRefs(val, childPre, resolved, stats) + v[i] = restoreOriginalRefs(val, preResolved.Index(i), resolved, stats) } return v @@ -287,37 +273,34 @@ func restoreOriginalRefs(value any, preResolved, resolved dyn.Value, stats *Rest // relative path: if exactly one unique pure variable reference across siblings // resolves to the leaf value, that reference is substituted. Multiple // different matching references are treated as ambiguous and skipped. -func restoreFromSiblings(value any, siblings []dyn.Value, resolved dyn.Value, stats *RestoreStats) any { - return restoreFromSiblingsAt(value, siblings, resolved, dyn.EmptyPath, stats) +func restoreFromSiblings(value any, siblings []structvar.View, resolved resolvedConfig, stats *RestoreStats) any { + return restoreFromSiblingsAt(value, siblings, resolved, nil, stats) } -func restoreFromSiblingsAt(value any, siblings []dyn.Value, resolved dyn.Value, relPath dyn.Path, stats *RestoreStats) any { +func restoreFromSiblingsAt(value any, siblings []structvar.View, resolved resolvedConfig, relPath *structpath.PathNode, stats *RestoreStats) any { switch v := value.(type) { case string, bool, int64: refs := map[string]struct{}{} strVal, isStr := value.(string) for _, sib := range siblings { - sv, err := dyn.GetByPath(sib, relPath) - if err != nil { - continue - } + sv := sib.Lookup(relPath) s, ok := sv.AsString() if !ok { continue } - if dynvar.IsPureVariableReference(s) { + if structvar.IsPureVariableReference(s) { rp, ok := resolveReferencePath(s) if !ok { continue } - rv, getErr := dyn.GetByPath(resolved, rp) - if getErr != nil { + rv, found := resolved.lookup(rp) + if !found { continue } - if rv.AsAny() == value { + if rv == value { refs[s] = struct{}{} } - } else if isStr && dynvar.ContainsVariableReference(s) { + } else if isStr && structvar.ContainsVariableReference(s) { // Compound interpolation in sibling: try to align the new // value against the sibling's template. If all variables // match at their positions, the template (possibly with @@ -337,13 +320,13 @@ func restoreFromSiblingsAt(value any, siblings []dyn.Value, resolved dyn.Value, case map[string]any: for key, val := range v { - v[key] = restoreFromSiblingsAt(val, siblings, resolved, relPath.Append(dyn.Key(key)), stats) + v[key] = restoreFromSiblingsAt(val, siblings, resolved, structpath.NewStringKey(relPath, key), stats) } return v case []any: for i, val := range v { - v[i] = restoreFromSiblingsAt(val, siblings, resolved, relPath.Append(dyn.Index(i)), stats) + v[i] = restoreFromSiblingsAt(val, siblings, resolved, structpath.NewIndex(relPath, i), stats) } return v @@ -355,51 +338,33 @@ func restoreFromSiblingsAt(value any, siblings []dyn.Value, resolved dyn.Value, // isPureVarRef reports whether the pre-resolved value at the field is a pure // ${var.X} reference. Used to gate the fallback substitution: only fields that // already used a variable can be re-targeted to a different variable. -func isPureVarRef(preResolved dyn.Value) bool { +func isPureVarRef(preResolved structvar.View) bool { if !preResolved.IsValid() { return false } s, ok := preResolved.AsString() - if !ok || !dynvar.IsPureVariableReference(s) { + if !ok || !structvar.IsPureVariableReference(s) { return false } - p, ok := dynvar.PureReferenceToPath(s) - if !ok { - return false - } - return p.HasPrefix(varPrefix) + p, ok := structvar.PureReferenceToPath(s) + return ok && p.HasPrefix(varPrefix) } // matchAnyVariable searches all bundle variables for a unique scalar value that // equals remoteValue. Returns the ${var.X} reference on a unique match, "" // otherwise. Multiple matches are treated as ambiguous and skipped. -func matchAnyVariable(remoteValue any, resolved dyn.Value) (string, bool) { - variables, err := dyn.GetByPath(resolved, dyn.NewPath(dyn.Key("variables"))) - if err != nil { - return "", false - } - vmap, ok := variables.AsMap() - if !ok { - return "", false - } +func matchAnyVariable(remoteValue any, resolved resolvedConfig) (string, bool) { var match string count := 0 - for _, pair := range vmap.Pairs() { - name, ok := pair.Key.AsString() - if !ok { - continue - } - v, getErr := dyn.GetByPath(pair.Value, dyn.NewPath(dyn.Key("value"))) - if getErr != nil { - continue - } + for name, variable := range resolved.view.Get("variables").MapItems() { + v := variable.Get("value") switch v.Kind() { - case dyn.KindString, dyn.KindInt, dyn.KindBool: + case structvar.KindString, structvar.KindInt, structvar.KindBool: if v.AsAny() == remoteValue { - match = pathToRef(varPrefix.Append(dyn.Key(name))) + match = pathToRef(structpath.NewStringKey(varPrefix, name)) count++ } - case dyn.KindInvalid, dyn.KindMap, dyn.KindSequence, dyn.KindFloat, dyn.KindTime, dyn.KindNil: + case structvar.KindInvalid, structvar.KindMap, structvar.KindSequence, structvar.KindFloat, structvar.KindTime, structvar.KindNil: // Skip non-scalar / unsupported variable values. } } @@ -409,19 +374,19 @@ func matchAnyVariable(remoteValue any, resolved dyn.Value) (string, bool) { return "", false } -// pathToRef formats a dyn.Path as a "${...}" interpolation reference. -func pathToRef(p dyn.Path) string { +// pathToRef formats a path as a "${...}" interpolation reference. +func pathToRef(p *structpath.PathNode) string { return "${" + p.String() + "}" } // matchOriginalRef checks if the pre-resolved config value at this position // was a pure variable reference whose resolved value equals remoteValue. -func matchOriginalRef(remoteValue any, preResolved, resolved dyn.Value) (string, bool) { +func matchOriginalRef(remoteValue any, preResolved structvar.View, resolved resolvedConfig) (string, bool) { if !preResolved.IsValid() { return "", false } s, ok := preResolved.AsString() - if !ok || !dynvar.IsPureVariableReference(s) { + if !ok || !structvar.IsPureVariableReference(s) { return "", false } @@ -430,12 +395,12 @@ func matchOriginalRef(remoteValue any, preResolved, resolved dyn.Value) (string, return "", false } - resolvedV, err := dyn.GetByPath(resolved, resolvedPath) - if err != nil { + resolvedV, found := resolved.lookup(resolvedPath) + if !found { return "", false } - if resolvedV.AsAny() == remoteValue { + if resolvedV == remoteValue { return s, true } return "", false @@ -455,12 +420,12 @@ func matchOriginalRef(remoteValue any, preResolved, resolved dyn.Value) (string, // new value contains "in" inside an unrelated word, that occurrence is still // rewritten to ${var.X}. Variables in the template are processed in order of // appearance, which is usually what the user expects. -func restoreCompoundInterpolation(remoteValue string, preResolved, resolved dyn.Value) (string, bool) { +func restoreCompoundInterpolation(remoteValue string, preResolved structvar.View, resolved resolvedConfig) (string, bool) { if !preResolved.IsValid() { return "", false } template, ok := preResolved.AsString() - if !ok || !dynvar.ContainsVariableReference(template) || dynvar.IsPureVariableReference(template) { + if !ok || !structvar.ContainsVariableReference(template) || structvar.IsPureVariableReference(template) { return "", false } @@ -481,7 +446,7 @@ func restoreCompoundInterpolation(remoteValue string, preResolved, resolved dyn. result = result[:idx] + seg.raw + result[idx+len(seg.resolvedValue):] } - if !dynvar.ContainsVariableReference(result) { + if !structvar.ContainsVariableReference(result) { return "", false } return result, true @@ -498,8 +463,8 @@ type templateSegment struct { // parseTemplateSegments splits a template string like "/mnt/${var.X}/raw" // into alternating literal and variable segments, resolving each variable. // Returns nil if any variable can't be resolved. -func parseTemplateSegments(template string, resolved dyn.Value) []templateSegment { - ref, ok := dynvar.NewRef(dyn.V(template)) +func parseTemplateSegments(template string, resolved resolvedConfig) []templateSegment { + ref, ok := structvar.NewRef(template) if !ok { return nil } @@ -526,12 +491,12 @@ func parseTemplateSegments(template string, resolved dyn.Value) []templateSegmen return nil } - resolvedV, err := dyn.GetByPath(resolved, resolvedPath) - if err != nil { + resolvedV, found := resolved.lookup(resolvedPath) + if !found { return nil } - resolvedStr, ok := resolvedV.AsString() + resolvedStr, ok := resolvedV.(string) if !ok { return nil } @@ -554,18 +519,15 @@ func parseTemplateSegments(template string, resolved dyn.Value) []templateSegmen return segments } -// preResolvedValueAt returns the pre-resolved dyn.Value at the field path, +// preResolvedValueAt returns the pre-resolved value at the field path, // if the field exists in the merged pre-resolved config. -func preResolvedValueAt(preResolved dyn.Value, fieldPath string) (dyn.Value, bool) { - p, err := dyn.NewPathFromString(fieldPath) +func preResolvedValueAt(preResolved structvar.View, fieldPath string) (structvar.View, bool) { + p, err := structpath.ParsePath(fieldPath) if err != nil { - return dyn.InvalidValue, false + return structvar.View{}, false } - v, err := dyn.GetByPath(preResolved, p) - if err != nil { - return dyn.InvalidValue, false - } - return v, true + v := preResolved.Lookup(p) + return v, v.IsValid() } // parentIsVariableReference reports whether the parent of fieldPath resolves to a @@ -573,7 +535,7 @@ func preResolvedValueAt(preResolved dyn.Value, fieldPath string) (dyn.Value, boo // ${var.spark_conf}). A nested key or index cannot be written into such a scalar, // so config-remote-sync skips the change. fieldPath is a merged-index path, the // same space as FieldChange.originalPath. -func parentIsVariableReference(preResolved dyn.Value, fieldPath string) bool { +func parentIsVariableReference(preResolved structvar.View, fieldPath string) bool { node, err := structpath.ParsePattern(fieldPath) if err != nil { return false @@ -587,7 +549,7 @@ func parentIsVariableReference(preResolved dyn.Value, fieldPath string) bool { return false } s, ok := v.AsString() - return ok && dynvar.ContainsVariableReference(s) + return ok && structvar.ContainsVariableReference(s) } // sequenceSiblings returns the sibling elements of the parent sequence when @@ -595,7 +557,7 @@ func parentIsVariableReference(preResolved dyn.Value, fieldPath string) bool { // last component must be an index ([*] or [N]) and the parent must resolve // to a sequence in the pre-resolved config. Returns false for non-sequence // Adds (e.g., new map fields). -func sequenceSiblings(preResolved dyn.Value, fieldPath string) ([]dyn.Value, bool) { +func sequenceSiblings(preResolved structvar.View, fieldPath string) ([]structvar.View, bool) { node, err := structpath.ParsePattern(fieldPath) if err != nil { return nil, false @@ -604,17 +566,17 @@ func sequenceSiblings(preResolved dyn.Value, fieldPath string) ([]dyn.Value, boo if !hasIndex && !node.BracketStar() { return nil, false } - p, err := dyn.NewPathFromString(node.Parent().String()) + p, err := structpath.ParsePath(node.Parent().String()) if err != nil { return nil, false } - parentValue, err := dyn.GetByPath(preResolved, p) - if err != nil { + parentValue := preResolved.Lookup(p) + if parentValue.Kind() != structvar.KindSequence { return nil, false } - seq, ok := parentValue.AsSequence() - if !ok { - return nil, false + var seq []structvar.View + for _, elem := range parentValue.Sequence() { + seq = append(seq, elem) } return seq, true } diff --git a/bundle/configsync/variables_test.go b/bundle/configsync/variables_test.go index 15f11d5eb3c..e25917c2f9e 100644 --- a/bundle/configsync/variables_test.go +++ b/bundle/configsync/variables_test.go @@ -1,22 +1,64 @@ package configsync import ( + "fmt" + "strings" "testing" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) +func loadTestConfig(t *testing.T, yaml string) *config.Root { + t.Helper() + root, diags := config.LoadFromBytes("test.yml", []byte(yaml)) + require.NoError(t, diags.Error()) + return root +} + +// scalarView returns a view of the string s. +func scalarView(t *testing.T, s string) structvar.View { + t.Helper() + root := loadTestConfig(t, fmt.Sprintf("bundle:\n name: %q\n", s)) + return root.View().Lookup(structpath.MustParsePath("bundle.name")) +} + +// tasksView returns views of job tasks, each given as the YAML of its fields in flow style. +func tasksView(t *testing.T, tasks ...string) []structvar.View { + t.Helper() + var sb strings.Builder + sb.WriteString("resources:\n jobs:\n j:\n tasks:\n") + for _, task := range tasks { + fmt.Fprintf(&sb, " - {%s}\n", task) + } + root := loadTestConfig(t, sb.String()) + var out []structvar.View + for _, elem := range root.View().Lookup(structpath.MustParsePath("resources.jobs.j.tasks")).Sequence() { + out = append(out, elem) + } + return out +} + +// variablesConfig returns a resolved configuration with the given variable values. +func variablesConfig(t *testing.T, values map[string]any) resolvedConfig { + t.Helper() + var sb strings.Builder + sb.WriteString("variables:\n") + for name, value := range values { + fmt.Fprintf(&sb, " %s:\n value: %#v\n", name, value) + } + return resolvedConfig{view: loadTestConfig(t, sb.String()).View(), overrides: map[string]any{}} +} + // TestRestoreOriginalRefs_HardcodedFieldNotRewritten fences the Replace safety // invariant: a hardcoded leaf must never be rewritten to a variable reference // just because the remote value coincidentally matches a variable elsewhere. func TestRestoreOriginalRefs_HardcodedFieldNotRewritten(t *testing.T) { - preResolved := dyn.V("us-east-1") - resolved := dyn.V(map[string]dyn.Value{ - "variables": dyn.V(map[string]dyn.Value{ - "region": dyn.V(map[string]dyn.Value{"value": dyn.V("main")}), - }), - }) + preResolved := scalarView(t, "us-east-1") + resolved := variablesConfig(t, map[string]any{"region": "main"}) // Even though "main" matches ${var.region}, restoreOriginalRefs must NOT // rewrite it — the original was hardcoded. result := restoreOriginalRefs("main", preResolved, resolved, &RestoreStats{}) @@ -29,17 +71,8 @@ func TestRestoreOriginalRefs_HardcodedFieldNotRewritten(t *testing.T) { func TestRestoreFromSiblings_ValueMatchesVariableButDifferentPath(t *testing.T) { // Sibling uses ${var.retry_count}=5 at .max_retries. New element has // .min_retry_interval=5 — coincidental match at a DIFFERENT relative path. - siblings := []dyn.Value{ - dyn.V(map[string]dyn.Value{ - "task_key": dyn.V("main"), - "max_retries": dyn.V("${var.retry_count}"), - }), - } - resolved := dyn.V(map[string]dyn.Value{ - "variables": dyn.V(map[string]dyn.Value{ - "retry_count": dyn.V(map[string]dyn.Value{"value": dyn.V(int64(5))}), - }), - }) + siblings := tasksView(t, `task_key: main, max_retries: "${var.retry_count}"`) + resolved := variablesConfig(t, map[string]any{"retry_count": 5}) value := map[string]any{ "task_key": "other", "min_retry_interval": int64(5), @@ -53,32 +86,22 @@ func TestRestoreFromSiblings_ValueMatchesVariableButDifferentPath(t *testing.T) // same-value rule: when two siblings use different variables at the same // relative path that both resolve to the same value, restoration is skipped. func TestRestoreFromSiblings_AmbiguousAcrossSiblings(t *testing.T) { - siblings := []dyn.Value{ - dyn.V(map[string]dyn.Value{"default": dyn.V("${var.landing_schema}")}), - dyn.V(map[string]dyn.Value{"default": dyn.V("${var.curated_schema}")}), - } - resolved := dyn.V(map[string]dyn.Value{ - "variables": dyn.V(map[string]dyn.Value{ - "landing_schema": dyn.V(map[string]dyn.Value{"value": dyn.V("raw_data")}), - "curated_schema": dyn.V(map[string]dyn.Value{"value": dyn.V("raw_data")}), - }), - }) - value := map[string]any{"default": "raw_data"} + siblings := tasksView(t, `task_key: "${var.landing_schema}"`, `task_key: "${var.curated_schema}"`) + resolved := variablesConfig(t, map[string]any{"landing_schema": "raw_data", "curated_schema": "raw_data"}) + value := map[string]any{"task_key": "raw_data"} result := restoreFromSiblings(value, siblings, resolved, &RestoreStats{}).(map[string]any) - assert.Equal(t, "raw_data", result["default"]) + assert.Equal(t, "raw_data", result["task_key"]) } // TestRestoreCompoundInterpolation covers the template alignment algorithm. // End-to-end coverage (pure ref match, sibling match, non-sequence skip, etc.) // lives in acceptance/bundle/config-remote-sync/resolve_variables. func TestRestoreCompoundInterpolation(t *testing.T) { - resolved := dyn.V(map[string]dyn.Value{ - "variables": dyn.V(map[string]dyn.Value{ - "host": dyn.V(map[string]dyn.Value{"value": dyn.V("dev-sql.example.com")}), - "port": dyn.V(map[string]dyn.Value{"value": dyn.V("1433")}), - "db": dyn.V(map[string]dyn.Value{"value": dyn.V("analytics_dev")}), - "acct": dyn.V(map[string]dyn.Value{"value": dyn.V("acct")}), - }), + resolved := variablesConfig(t, map[string]any{ + "host": "dev-sql.example.com", + "port": "1433", + "db": "analytics_dev", + "acct": "acct", }) tests := []struct { @@ -108,7 +131,7 @@ func TestRestoreCompoundInterpolation(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - result := restoreOriginalRefs(tt.remote, dyn.V(tt.template), resolved, &RestoreStats{}) + result := restoreOriginalRefs(tt.remote, scalarView(t, tt.template), resolved, &RestoreStats{}) assert.Equal(t, tt.want, result) }) } diff --git a/bundle/deploy/check_dashboards_modified_remotely.go b/bundle/deploy/check_dashboards_modified_remotely.go index d685f7ebeb6..97288de568a 100644 --- a/bundle/deploy/check_dashboards_modified_remotely.go +++ b/bundle/deploy/check_dashboards_modified_remotely.go @@ -72,8 +72,8 @@ func (l *checkDashboardsModifiedRemotely) Apply(ctx context.Context, b *bundle.B continue } - path := structpath.NewPath(nil, "resources", "dashboards", dashboard.Name) - loc := b.Config.GetLocationOf(path) + path := structpath.NewStringKeys(nil, "resources", "dashboards", dashboard.Name) + loc := b.Config.GetLocation(path.String()) actual, err := b.WorkspaceClient(ctx).Lakeview.GetByDashboardId(ctx, dashboard.ID) if err != nil { diags = diags.Append(diag.Diagnostic{ diff --git a/bundle/deploy/metadata/compute.go b/bundle/deploy/metadata/compute.go index 1fa4c3bf2d9..c8775748151 100644 --- a/bundle/deploy/metadata/compute.go +++ b/bundle/deploy/metadata/compute.go @@ -46,7 +46,7 @@ func (m *compute) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics for name, job := range b.Config.Resources.Jobs { // Compute config file path the job is defined in, relative to the bundle // root - l := b.Config.GetLocation("resources.jobs." + name) + l := b.Config.DefinitionLocation("resources.jobs." + name) if l.File == "" { // Skip resources that exist only in the deployment state: statemgmt.Load, // which runs before this mutator, injects them into the config without a @@ -73,7 +73,7 @@ func (m *compute) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics for name, pipeline := range b.Config.Resources.Pipelines { // Compute config file path the pipeline is defined in, relative to the bundle // root - l := b.Config.GetLocation("resources.pipelines." + name) + l := b.Config.DefinitionLocation("resources.pipelines." + name) if l.File == "" { // Skip resources that exist only in the deployment state: statemgmt.Load, // which runs before this mutator, injects them into the config without a @@ -98,7 +98,7 @@ func (m *compute) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics for name, dashboard := range b.Config.Resources.Dashboards { // Compute config file path the dashboard is defined in, relative to the bundle // root - l := b.Config.GetLocation("resources.dashboards." + name) + l := b.Config.DefinitionLocation("resources.dashboards." + name) if l.File == "" { // Skip resources that exist only in the deployment state: statemgmt.Load, // which runs before this mutator, injects them into the config without a diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 98da4f1376d..6503ba402b8 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -15,8 +15,6 @@ import ( "github.com/databricks/cli/bundle/direct/dresources" "github.com/databricks/cli/bundle/direct/dstate" "github.com/databricks/cli/bundle/terraform_dabs_map" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/dynvar" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/logdiag" "github.com/databricks/cli/libs/retry" @@ -885,7 +883,7 @@ func isEmpty(rv reflect.Value) bool { // Empty slices and maps cannot be represented in proto and because of that they cannot be represented // by SDK's JSON encoder. However, they can be provided by users in the config and can be represented in - // Bundle struct (currently libs/structs and libs/dyn use ForceSendFields for maps and slices, unlike SDK). + // Bundle struct (currently libs/structs uses ForceSendFields for maps and slices, unlike SDK). // Thus we get permanent drift because we see that new config is [] but in the state it is omitted. if rv.Kind() == reflect.Slice { @@ -1086,7 +1084,7 @@ func (b *DeploymentBundle) resolveReferences(ctx context.Context, resourceKey st var resolved bool for fieldPathStr, refString := range sv.Refs { - refs, ok := dynvar.NewRef(dyn.V(refString)) + refs, ok := structvar.NewRef(refString) if !ok { logdiag.LogError(ctx, fmt.Errorf("%s: cannot parse %q", errorPrefix, refString)) return false @@ -1160,31 +1158,36 @@ func (b *DeploymentBundle) makePlan(ctx context.Context, configRoot *config.Root existingKeys := maps.Clone(db.State) - patterns := []dyn.Pattern{ - dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey()), - dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey(), dyn.Key("permissions")), - dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey(), dyn.Key("grants")), + patterns := []*structpath.PatternNode{ + structpath.MustParsePattern("resources.*.*"), + structpath.MustParsePattern("resources.*.*.permissions"), + structpath.MustParsePattern("resources.*.*.grants"), + } + + var rootView structvar.View + if configRoot != nil { + rootView = configRoot.View() } // Walk? if configRoot != nil { for _, pat := range patterns { - _, err := dyn.MapByPattern( - configRoot.Value(), + err := structvar.ForEach( + rootView, pat, - func(p dyn.Path, v dyn.Value) (dyn.Value, error) { + func(p *structpath.PathNode, _ structvar.View) error { s := p.String() resourceType := config.GetResourceTypeFromKey(s) if resourceType == "" { - return v, fmt.Errorf("cannot parse resource key: %q", s) + return fmt.Errorf("cannot parse resource key: %q", s) } _, ok := dresources.SupportedResources[resourceType] if !ok { - return v, fmt.Errorf("unsupported resource type: %s", resourceType) + return fmt.Errorf("unsupported resource type: %s", resourceType) } nodes = append(nodes, s) - return dyn.InvalidValue, nil + return nil }, ) if err != nil { @@ -1244,7 +1247,7 @@ func (b *DeploymentBundle) makePlan(ctx context.Context, configRoot *config.Root // This means input and state must be compatible: input can have more fields, but existing fields should not be moved // This means one cannot refer to fields not present in state (e.g. ${resources.jobs.foo.permissions}) - refs, err := extractReferences(configRoot.Value(), node, adapter.StateType()) + refs, err := extractReferences(rootView, node, adapter.StateType()) if err != nil { return nil, fmt.Errorf("failed to read references from config for %s: %w", node, err) } @@ -1253,13 +1256,13 @@ func (b *DeploymentBundle) makePlan(ctx context.Context, configRoot *config.Root var dependsOn []deployplan.DependsOnEntry for _, reference := range refs { - ref, ok := dynvar.NewRef(dyn.V(reference)) + ref, ok := structvar.NewRef(reference) if !ok { continue } for _, targetPath := range ref.References() { - targetPathParsed, err := dyn.NewPathFromString(targetPath) + targetPathParsed, err := structpath.ParsePath(targetPath) if err != nil { return nil, fmt.Errorf("parsing %q: %w", targetPath, err) } @@ -1348,26 +1351,26 @@ func (b *DeploymentBundle) makePlan(ctx context.Context, configRoot *config.Root // keeping only those whose field path exists in stateType (references in input-only or // bundle:"readonly" fields, such as volumes' computed volume_path, are skipped). // Returns a map from structpath string (field path within the resource) to template string. -func ExtractReferences(root dyn.Value, node string, stateType reflect.Type) (map[string]string, error) { +func ExtractReferences(root structvar.View, node string, stateType reflect.Type) (map[string]string, error) { return extractReferences(root, node, stateType) } -func extractReferences(root dyn.Value, node string, stateType reflect.Type) (map[string]string, error) { +func extractReferences(root structvar.View, node string, stateType reflect.Type) (map[string]string, error) { nodeType := config.GetResourceTypeFromKey(node) refs := make(map[string]string) - path, err := dyn.NewPathFromString(node) + path, err := structpath.ParsePath(node) if err != nil { return nil, fmt.Errorf("internal error: bad node key: %q: %w", node, err) } - val, err := dyn.GetByPath(root, path) - if err != nil { - return nil, err + val := root.Lookup(path) + if !val.IsValid() { + return nil, fmt.Errorf("no value at %s", node) } - err = dyn.WalkReadOnly(val, func(p dyn.Path, v dyn.Value) error { - fullPath := append(path, p...) + err = structvar.Walk(val, func(p *structpath.PathNode, v structvar.View) error { + fullPath := structpath.Join(path, p.AsSlice()...) targetType := config.GetResourceTypeFromKey(fullPath.String()) if targetType != nodeType { // Make sure these are associated with different nodes: @@ -1376,15 +1379,19 @@ func extractReferences(root dyn.Value, node string, stateType reflect.Type) (map // resources.jobs.foo.grants... return nil } - ref, ok := dynvar.NewRef(v) + str, ok := v.AsString() + if !ok { + return nil + } + ref, ok := structvar.NewRef(str) if !ok { return nil } // ValidatePath and the refs keys both operate on structpath (keys are // re-parsed and applied to the typed state in structvar.ResolveRef). // structpath's bracket notation (['key.with.dots']) also round-trips - // keys with dots, which dyn.Path.String()'s dot notation cannot. - fieldPath := dynPathToStructPath(p) + // keys with dots. + fieldPath := p // References resolve against the state type, not the input config (see PlanResources // and dresources.TestInputSubset). A field in input but not in state — e.g. a @@ -1403,19 +1410,6 @@ func extractReferences(root dyn.Value, node string, stateType reflect.Type) (map return refs, nil } -// dynPathToStructPath converts a dyn.Path to a structpath.PathNode. -func dynPathToStructPath(p dyn.Path) *structpath.PathNode { - var node *structpath.PathNode - for _, c := range p { - if key := c.Key(); key != "" { - node = structpath.NewStringKey(node, key) - } else { - node = structpath.NewIndex(node, c.Index()) - } - } - return node -} - func (b *DeploymentBundle) getAdapterForKey(resourceKey string) (*dresources.Adapter, error) { group := config.GetResourceTypeFromKey(resourceKey) if group == "" { diff --git a/bundle/direct/bundle_plan_test.go b/bundle/direct/bundle_plan_test.go index 2d051dc21b9..95c2e4470a2 100644 --- a/bundle/direct/bundle_plan_test.go +++ b/bundle/direct/bundle_plan_test.go @@ -1,15 +1,13 @@ package direct import ( - "bytes" "slices" "testing" + "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/bundle/direct/dresources" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/yamlloader" "github.com/databricks/cli/libs/structs/structdiff" "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/cli/libs/structs/structvar" @@ -20,35 +18,6 @@ import ( "github.com/stretchr/testify/require" ) -func TestDynPathToStructPath(t *testing.T) { - tests := []struct { - path dyn.Path - expected string - }{ - { - path: dyn.NewPath(dyn.Key("foo"), dyn.Key("bar")), - expected: "foo.bar", - }, - { - path: dyn.NewPath(dyn.Key("foo"), dyn.Index(1), dyn.Key("bar")), - expected: "foo[1].bar", - }, - { - path: dyn.NewPath(dyn.Key("configuration"), dyn.Key("europris.swipe.egress_streaming_schema")), - expected: "configuration['europris.swipe.egress_streaming_schema']", - }, - { - path: dyn.NewPath(dyn.Key("tags"), dyn.Key("it's.here")), - expected: "tags['it''s.here']", - }, - } - - for _, tc := range tests { - node := dynPathToStructPath(tc.path) - assert.Equal(t, tc.expected, node.String()) - } -} - // extractReferences gates references on the state type: a reference in an input-only field // (e.g. a bundle:"readonly" field like volumes' volume_path) must not become a dependency, // while references in state fields (e.g. comment) are still extracted. @@ -69,10 +38,10 @@ resources: comment: "${resources.schemas.kept.name}" volume_path: "/Volumes/main/${resources.schemas.dropped.name}/myvol" ` - root, err := yamlloader.LoadYAML("test", bytes.NewBufferString(yml)) - require.NoError(t, err) + root, diags := config.LoadFromBytes("test", []byte(yml)) + require.NoError(t, diags.Error()) - refs, err := extractReferences(root, "resources.volumes.v", stateType) + refs, err := extractReferences(root.View(), "resources.volumes.v", stateType) require.NoError(t, err) assert.Equal(t, map[string]string{ diff --git a/bundle/direct/dresources/adapter.go b/bundle/direct/dresources/adapter.go index 80ece24b2be..3f2da957804 100644 --- a/bundle/direct/dresources/adapter.go +++ b/bundle/direct/dresources/adapter.go @@ -512,7 +512,7 @@ func (a *Adapter) FieldIsStableOutput(path *structpath.PathNode) bool { // references needed to complete it. Resources without PrepareInputConfig pass their config through. func (a *Adapter) PrepareInputConfig(inputConfig any, resourceKey string) (*structvar.StructVar, error) { if a.prepareInputConfig == nil { - return &structvar.StructVar{Value: inputConfig, Refs: nil}, nil + return &structvar.StructVar{Value: inputConfig, Refs: nil, Locations: nil}, nil } outs, err := a.prepareInputConfig.Call(inputConfig, resourceKey) diff --git a/bundle/direct/dresources/grants.go b/bundle/direct/dresources/grants.go index 3aa48c58279..3fac2e0800b 100644 --- a/bundle/direct/dresources/grants.go +++ b/bundle/direct/dresources/grants.go @@ -75,6 +75,7 @@ func (r *ResourceGrants) PrepareInputConfig(inputConfig *[]catalog.PrivilegeAssi Refs: map[string]string{ "full_name": "${" + baseNode + ".id}", }, + Locations: nil, }, nil } diff --git a/bundle/direct/dresources/permissions.go b/bundle/direct/dresources/permissions.go index c2aebb4d0ac..b09e9cfe2ff 100644 --- a/bundle/direct/dresources/permissions.go +++ b/bundle/direct/dresources/permissions.go @@ -116,6 +116,7 @@ func (r *ResourcePermissions) PrepareInputConfig(inputConfig any, resourceKey st Refs: map[string]string{ "object_id": r.objectType + "${" + baseNode + "." + r.idField + "}", }, + Locations: nil, }, nil } diff --git a/bundle/direct/dresources/secret_scope_acls.go b/bundle/direct/dresources/secret_scope_acls.go index 1de301bde2c..61269ce4097 100644 --- a/bundle/direct/dresources/secret_scope_acls.go +++ b/bundle/direct/dresources/secret_scope_acls.go @@ -60,6 +60,7 @@ func (*ResourceSecretScopeAcls) PrepareInputConfig(inputConfig *[]resources.Secr Refs: map[string]string{ "scope_name": "${" + baseNode + ".name}", }, + Locations: nil, }, nil } diff --git a/bundle/direct/dresources/serialize_test.go b/bundle/direct/dresources/serialize_test.go index 76078460193..e1bd30579c6 100644 --- a/bundle/direct/dresources/serialize_test.go +++ b/bundle/direct/dresources/serialize_test.go @@ -130,7 +130,7 @@ func TestRoundtripAllFieldsRemoteType(t *testing.T) { // TestRoundtripAllFieldsInputConfigType verifies InputConfigType, the typed // bundle config a resource is loaded into, survives a JSON round-trip with every -// field populated. Bundle config is normally read and written through libs/dyn, +// field populated. Bundle config is normally read and written through libs/structs/structvar, // which walks the struct itself and never calls these marshalers, so this is a // latent trap rather than live corruption. It is guarded anyway because it is the // same trap as StateType and RemoteType: a resource that embeds a member with its diff --git a/bundle/direct/resolve_config.go b/bundle/direct/resolve_config.go index 8ce28d710e1..45ab83ef1fb 100644 --- a/bundle/direct/resolve_config.go +++ b/bundle/direct/resolve_config.go @@ -1,14 +1,15 @@ package direct import ( + "fmt" + "github.com/databricks/cli/bundle/config" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/convert" - "github.com/databricks/cli/libs/dyn/dynvar" - "github.com/databricks/cli/libs/dyn/jsonloader" + "github.com/databricks/cli/libs/structs/structaccess" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) -var resourcesPrefix = dyn.MustPathFromString("resources") +var resourcesPrefix = structpath.MustParsePath("resources") // ResolveConfigAgainstState resolves ${resources.*} references within the resource at // target so its runner ("bundle run") sees concrete values rather than references. For a @@ -24,48 +25,97 @@ var resourcesPrefix = dyn.MustPathFromString("resources") // // The state DB holds fields like the immutable snapshot's full_path that never reach the // config, so it must be open. -func (b *DeploymentBundle) ResolveConfigAgainstState(cfg *config.Root, target dyn.Path) error { - return cfg.Mutate(func(root dyn.Value) (dyn.Value, error) { - // Fall back to the fully-normalized config so references to fields that are - // implied (not explicitly set) still resolve. - normalized, _ := convert.Normalize(cfg, root, convert.IncludeMissingFields) +func (b *DeploymentBundle) ResolveConfigAgainstState(cfg *config.Root, target *structpath.PathNode) error { + view := cfg.View() + resource := view.Lookup(target) + if !resource.IsValid() { + return fmt.Errorf("resource %s not found in configuration", target) + } + + // Each string is resolved on its own (references only point at other paths), then + // written back; collect first because writing changes the view. + type update struct { + path *structpath.PathNode + value any + } + var updates []update + + lookup := func(path *structpath.PathNode) (structvar.View, error) { + if !path.HasPrefix(resourcesPrefix) { + return structvar.View{}, structvar.ErrSkipResolution + } + if v, ok := b.lookupStateField(path); ok { + return v, nil + } + // Fall back to the config, including fields that are implied (not explicitly set). + if v := view.Lookup(path); v.IsValid() { + return v, nil + } + got, err := structaccess.Get(cfg, path) + if err != nil { + return structvar.View{}, structvar.ErrSkipResolution + } + return structvar.NewView(&got, nil, nil), nil + } - return dyn.MapByPath(root, target, func(_ dyn.Path, resource dyn.Value) (dyn.Value, error) { - return dynvar.Resolve(resource, func(path dyn.Path) (dyn.Value, error) { - if !path.HasPrefix(resourcesPrefix) { - return dyn.InvalidValue, dynvar.ErrSkipResolution - } - if v, ok := b.lookupStateField(path); ok { - return v, nil - } - v, err := dyn.GetByPath(normalized, path) - if err != nil { - return dyn.InvalidValue, dynvar.ErrSkipResolution - } - return v, nil - }) - }) + err := structvar.Walk(resource, func(p *structpath.PathNode, v structvar.View) error { + s, ok := v.AsString() + if !ok { + return nil + } + if _, ok := structvar.NewRef(s); !ok { + return nil + } + key := p.String() + out, err := structvar.Resolve(map[string]structvar.Template{key: {Value: s}}, lookup) + if err != nil { + return err + } + if resolved, ok := out[key]; ok { + updates = append(updates, update{structpath.Join(target, p.AsSlice()...), resolved.AsAny()}) + } + return nil }) + if err != nil { + return err + } + + for _, u := range updates { + // Decode converts like loading did, e.g. a string id into an int job_id. + diags, err := cfg.Decode(u.path, structvar.NewView(&u.value, nil, nil)) + if err != nil { + return err + } + if diags.HasError() { + return diags.Error() + } + } + return nil } // lookupStateField returns the value at resources... from the // resource's persisted state, if that resource is in state and holds the field. -func (b *DeploymentBundle) lookupStateField(path dyn.Path) (dyn.Value, bool) { - if len(path) < 4 || path[0].Key() != "resources" { - return dyn.InvalidValue, false +func (b *DeploymentBundle) lookupStateField(path *structpath.PathNode) (structvar.View, bool) { + if path.Len() < 4 || path.KeyAt(0) != "resources" { + return structvar.View{}, false } - resourceKey := "resources." + path[1].Key() + "." + path[2].Key() + resourceKey := "resources." + path.KeyAt(1) + "." + path.KeyAt(2) entry, ok := b.StateDB.GetResourceEntry(resourceKey) if !ok || len(entry.State) == 0 { - return dyn.InvalidValue, false + return structvar.View{}, false } - stateVal, err := jsonloader.LoadJSON(entry.State, resourceKey) + // ParseJSON keeps large ids exact. + node, err := structvar.ParseJSON(resourceKey, entry.State) if err != nil { - return dyn.InvalidValue, false + return structvar.View{}, false + } + var state any + if _, _, err := structvar.DecodeYAMLNode(resourceKey, node, &state, nil); err != nil { + return structvar.View{}, false } - fieldVal, err := dyn.GetByPath(stateVal, path[3:]) - if err != nil || !fieldVal.IsValid() { - return dyn.InvalidValue, false + v := structvar.NewView(&state, nil, nil).Lookup(path.SkipPrefix(3)) + if !v.IsValid() { + return structvar.View{}, false } - return fieldVal, true + return v, true } diff --git a/bundle/direct/unescape.go b/bundle/direct/unescape.go index 31f9925e098..40ec5d02bed 100644 --- a/bundle/direct/unescape.go +++ b/bundle/direct/unescape.go @@ -5,9 +5,9 @@ import ( "reflect" "strings" - "github.com/databricks/cli/libs/dyn/dynvar" "github.com/databricks/cli/libs/structs/structaccess" "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" "github.com/databricks/cli/libs/structs/structwalk" ) @@ -24,7 +24,7 @@ func unescapeRefs(state any) error { return } paths = append(paths, path) - values = append(values, dynvar.Unescape(s)) + values = append(values, structvar.Unescape(s)) }) if err != nil { return err diff --git a/bundle/generate/alert.go b/bundle/generate/alert.go index eba68126dd8..d3765205d17 100644 --- a/bundle/generate/alert.go +++ b/bundle/generate/alert.go @@ -1,19 +1,16 @@ package generate import ( - "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structyaml" "github.com/databricks/databricks-sdk-go/service/sql" ) -func ConvertAlertToValue(alert *sql.AlertV2, filePath string) (dyn.Value, error) { +func ConvertAlertToValue(alert *sql.AlertV2, filePath string) (structyaml.Map, error) { // The majority of fields of the alert struct are present in .dbalert.json file. // We copy the relevant fields manually. - dv := map[string]dyn.Value{ - "display_name": dyn.NewValue(alert.DisplayName, []diag.Location{{Line: 1}}), - "warehouse_id": dyn.NewValue(alert.WarehouseId, []diag.Location{{Line: 2}}), - "file_path": dyn.NewValue(filePath, []diag.Location{{Line: 3}}), - } - - return dyn.V(dv), nil + return structyaml.M( + "display_name", alert.DisplayName, + "warehouse_id", alert.WarehouseId, + "file_path", filePath, + ), nil } diff --git a/bundle/generate/app.go b/bundle/generate/app.go index 0714fc9b324..a0f40a5a4ef 100644 --- a/bundle/generate/app.go +++ b/bundle/generate/app.go @@ -1,54 +1,47 @@ package generate import ( - "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/convert" + "github.com/databricks/cli/libs/structs/structyaml" "github.com/databricks/databricks-sdk-go/service/apps" ) -func ConvertAppToValue(app *apps.App, sourceCodePath string) (dyn.Value, error) { - ar, err := convert.FromTyped(app.Resources, dyn.NilValue) - if err != nil { - return dyn.NilValue, err - } - +func ConvertAppToValue(app *apps.App, sourceCodePath string) (structyaml.Map, error) { // The majority of fields of the app struct are read-only. // We copy the relevant fields manually. - dv := map[string]dyn.Value{ - "name": dyn.NewValue(app.Name, []diag.Location{{Line: 1}}), - "description": dyn.NewValue(app.Description, []diag.Location{{Line: 2}}), - } + dv := structyaml.M( + "name", app.Name, + "description", app.Description, + ) // For a git-backed app, emit git_repository + git_source instead of a // workspace source_code_path. Otherwise the generated bundle would silently // down-convert the app to workspace source and point source_code_path at a // local directory that has nothing downloaded into it. if app.GitRepository != nil { - dv["git_repository"] = gitRepositoryValue(app.GitRepository) - if gs := gitSourceValue(app); gs.Kind() != dyn.KindNil { - dv["git_source"] = gs + dv.Add("git_repository", gitRepositoryValue(app.GitRepository)) + if gs := gitSourceValue(app); len(gs) > 0 { + dv.Add("git_source", gs) } } else { - dv["source_code_path"] = dyn.NewValue(sourceCodePath, []diag.Location{{Line: 4}}) + dv.Add("source_code_path", sourceCodePath) } - if ar.Kind() != dyn.KindNil { - dv["resources"] = ar.WithLocations([]diag.Location{{Line: 5}}) + if ar := structyaml.Value(app.Resources); ar != nil { + dv.Add("resources", ar) } - return dyn.V(dv), nil + return dv, nil } -func gitRepositoryValue(r *apps.GitRepository) dyn.Value { - m := map[string]dyn.Value{ - "url": dyn.NewValue(r.Url, []diag.Location{{Line: 1}}), - "provider": dyn.NewValue(r.Provider, []diag.Location{{Line: 2}}), - } +func gitRepositoryValue(r *apps.GitRepository) structyaml.Map { + m := structyaml.M( + "url", r.Url, + "provider", r.Provider, + ) if r.AutoDeploy { - m["auto_deploy"] = dyn.NewValue(r.AutoDeploy, []diag.Location{{Line: 3}}) + m.Add("auto_deploy", r.AutoDeploy) } - return dyn.NewValue(m, []diag.Location{{Line: 3}}) + return m } // gitSourceValue returns the reference the app deploys from (branch, tag, or @@ -56,29 +49,26 @@ func gitRepositoryValue(r *apps.GitRepository) dyn.Value { // configured git_source and falls back to the default source of the app's most // recent deployment. System-populated fields (resolved_commit and the nested // git_repository) are intentionally omitted. -func gitSourceValue(app *apps.App) dyn.Value { +func gitSourceValue(app *apps.App) structyaml.Map { src := app.GitSource if src == nil { src = app.DefaultGitSource } if src == nil { - return dyn.NilValue + return nil } - m := map[string]dyn.Value{} + var m structyaml.Map switch { case src.Branch != "": - m["branch"] = dyn.NewValue(src.Branch, []diag.Location{{Line: 1}}) + m.Add("branch", src.Branch) case src.Tag != "": - m["tag"] = dyn.NewValue(src.Tag, []diag.Location{{Line: 1}}) + m.Add("tag", src.Tag) case src.Commit != "": - m["commit"] = dyn.NewValue(src.Commit, []diag.Location{{Line: 1}}) + m.Add("commit", src.Commit) } if src.SourceCodePath != "" { - m["source_code_path"] = dyn.NewValue(src.SourceCodePath, []diag.Location{{Line: 2}}) - } - if len(m) == 0 { - return dyn.NilValue + m.Add("source_code_path", src.SourceCodePath) } - return dyn.NewValue(m, []diag.Location{{Line: 4}}) + return m } diff --git a/bundle/generate/dashboard.go b/bundle/generate/dashboard.go index 6f045a39cb2..a12e8b57f5d 100644 --- a/bundle/generate/dashboard.go +++ b/bundle/generate/dashboard.go @@ -1,19 +1,16 @@ package generate import ( - "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structyaml" "github.com/databricks/databricks-sdk-go/service/dashboards" ) -func ConvertDashboardToValue(dashboard *dashboards.Dashboard, filePath string) (dyn.Value, error) { +func ConvertDashboardToValue(dashboard *dashboards.Dashboard, filePath string) (structyaml.Map, error) { // The majority of fields of the dashboard struct are read-only. // We copy the relevant fields manually. - dv := map[string]dyn.Value{ - "display_name": dyn.NewValue(dashboard.DisplayName, []diag.Location{{Line: 1}}), - "warehouse_id": dyn.NewValue(dashboard.WarehouseId, []diag.Location{{Line: 2}}), - "file_path": dyn.NewValue(filePath, []diag.Location{{Line: 3}}), - } - - return dyn.V(dv), nil + return structyaml.M( + "display_name", dashboard.DisplayName, + "warehouse_id", dashboard.WarehouseId, + "file_path", filePath, + ), nil } diff --git a/bundle/generate/genie_space.go b/bundle/generate/genie_space.go index 639c65f314b..88cbe8b05d9 100644 --- a/bundle/generate/genie_space.go +++ b/bundle/generate/genie_space.go @@ -1,29 +1,28 @@ package generate import ( - "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structyaml" "github.com/databricks/databricks-sdk-go/service/dashboards" ) -func ConvertGenieSpaceToValue(genieSpace *dashboards.GenieSpace, filePath string) (dyn.Value, error) { +func ConvertGenieSpaceToValue(genieSpace *dashboards.GenieSpace, filePath string) (structyaml.Map, error) { // Emit only the fields a user authors in a bundle. serialized_space is // written to a separate file and referenced via file_path, and output-only // fields (e.g. space_id, etag) must not appear in the generated config, so // we build the value field by field rather than marshaling the struct. - dv := map[string]dyn.Value{ - "title": dyn.NewValue(genieSpace.Title, []diag.Location{{Line: 1}}), - "warehouse_id": dyn.NewValue(genieSpace.WarehouseId, []diag.Location{{Line: 2}}), - "file_path": dyn.NewValue(filePath, []diag.Location{{Line: 3}}), - } + dv := structyaml.M( + "title", genieSpace.Title, + "warehouse_id", genieSpace.WarehouseId, + "file_path", filePath, + ) if genieSpace.Description != "" { - dv["description"] = dyn.NewValue(genieSpace.Description, []diag.Location{{Line: 4}}) + dv.Add("description", genieSpace.Description) } if genieSpace.ParentPath != "" { - dv["parent_path"] = dyn.NewValue(ensureWorkspacePrefix(genieSpace.ParentPath), []diag.Location{{Line: 5}}) + dv.Add("parent_path", ensureWorkspacePrefix(genieSpace.ParentPath)) } - return dyn.V(dv), nil + return dv, nil } diff --git a/bundle/generate/job.go b/bundle/generate/job.go index 801eee1f5bd..b1784258e1c 100644 --- a/bundle/generate/job.go +++ b/bundle/generate/job.go @@ -1,50 +1,51 @@ package generate import ( - "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/yamlsaver" + "github.com/databricks/cli/libs/structs/structyaml" "github.com/databricks/databricks-sdk-go/service/jobs" ) var ( - jobOrder = yamlsaver.NewOrder([]string{"name", "job_clusters", "compute", "tasks"}) - taskOrder = yamlsaver.NewOrder([]string{"task_key", "depends_on", "existing_cluster_id", "new_cluster", "job_cluster_key"}) + jobOrder = []string{"name", "job_clusters", "compute", "tasks", "parameters"} + taskOrder = []string{"task_key", "depends_on", "existing_cluster_id", "new_cluster", "job_cluster_key"} ) -func ConvertJobToValue(job *jobs.Job) (dyn.Value, error) { - value := make(map[string]dyn.Value) +func ConvertJobToValue(job *jobs.Job) (structyaml.Map, error) { + // Tasks and parameters are processed separately. + value, err := structyaml.Struct(job.Settings, "format", "new_cluster", "existing_cluster_id", "tasks", "parameters") + if err != nil { + return nil, err + } + if job.Settings.Tasks != nil { - var tasks []dyn.Value + var tasks []any for _, task := range job.Settings.Tasks { - v, err := convertTaskToValue(task, taskOrder) + v, err := convertTaskToValue(task) if err != nil { - return dyn.InvalidValue, err + return nil, err } tasks = append(tasks, v) } - // We're using location lines to define the order of keys in exported YAML. - value["tasks"] = dyn.NewValue(tasks, []diag.Location{{Line: jobOrder.Get("tasks")}}) + value.Add("tasks", tasks) } // We're processing job.Settings.Parameters separately to retain empty default values. if len(job.Settings.Parameters) > 0 { - var params []dyn.Value + var params []any for _, parameter := range job.Settings.Parameters { - p := map[string]dyn.Value{ - "name": dyn.NewValue(parameter.Name, []diag.Location{{Line: 0}}), // We use Line: 0 to ensure that the name goes first. - "default": dyn.NewValue(parameter.Default, []diag.Location{{Line: 1}}), - } - params = append(params, dyn.V(p)) + params = append(params, structyaml.M("name", parameter.Name, "default", parameter.Default)) } - value["parameters"] = dyn.NewValue(params, []diag.Location{{Line: jobOrder.Get("parameters")}}) + value.Add("parameters", params) } - return yamlsaver.ConvertToMapValue(job.Settings, jobOrder, []string{"format", "new_cluster", "existing_cluster_id"}, value) + return value.Order(jobOrder...), nil } -func convertTaskToValue(task jobs.Task, order *yamlsaver.Order) (dyn.Value, error) { - dst := make(map[string]dyn.Value) - return yamlsaver.ConvertToMapValue(task, order, []string{"format"}, dst) +func convertTaskToValue(task jobs.Task) (structyaml.Map, error) { + value, err := structyaml.Struct(task, "format") + if err != nil { + return nil, err + } + return value.Order(taskOrder...), nil } diff --git a/bundle/generate/pipeline.go b/bundle/generate/pipeline.go index 115330ea0f6..beb8bf2e5ad 100644 --- a/bundle/generate/pipeline.go +++ b/bundle/generate/pipeline.go @@ -3,14 +3,13 @@ package generate import ( "strings" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/yamlsaver" + "github.com/databricks/cli/libs/structs/structyaml" "github.com/databricks/databricks-sdk-go/service/pipelines" ) -var pipelineOrder = yamlsaver.NewOrder([]string{"name", "clusters", "configuration", "libraries"}) +var pipelineOrder = []string{"name", "clusters", "configuration", "libraries"} -func ConvertPipelineToValue(pipeline *pipelines.PipelineSpec, rootPath, remoteRootPath string) (dyn.Value, error) { +func ConvertPipelineToValue(pipeline *pipelines.PipelineSpec, rootPath, remoteRootPath string) (structyaml.Map, error) { if pipeline.RootPath != "" { pipeline.RootPath = rootPath } @@ -24,11 +23,14 @@ func ConvertPipelineToValue(pipeline *pipelines.PipelineSpec, rootPath, remoteRo } } - value := make(map[string]dyn.Value) // We ignore the following fields: // - id: this is a read-only field // - storage: changes to this field are rare because changing the storage recreates pipeline-related resources // - edition: this field is rarely changed // - development: this field is specific to the mode where it's used and does not need to be saved to the bundle configuration. - return yamlsaver.ConvertToMapValue(pipeline, pipelineOrder, []string{"id", "storage", "edition", "development"}, value) + value, err := structyaml.Struct(pipeline, "id", "storage", "edition", "development") + if err != nil { + return nil, err + } + return value.Order(pipelineOrder...), nil } diff --git a/bundle/internal/bundletest/benchmark.go b/bundle/internal/bundletest/benchmark.go index 59647d814fd..2cb4b738652 100644 --- a/bundle/internal/bundletest/benchmark.go +++ b/bundle/internal/bundletest/benchmark.go @@ -9,7 +9,6 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/resources" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/stretchr/testify/require" ) @@ -210,32 +209,6 @@ const jobExample = ` } }` -func BundleV(b *testing.B, numJobs int) dyn.Value { - allJobs := map[string]*resources.Job{} - for i := range numJobs { - job := jobs.JobSettings{} - err := json.Unmarshal([]byte(jobExample), &job) - require.NoError(b, err) - - allJobs[strconv.Itoa(i)] = &resources.Job{ - JobSettings: job, - } - } - - myBundle := bundle.Bundle{ - Config: config.Root{ - Resources: config.Resources{ - Jobs: allJobs, - }, - }, - } - - // Apply noop mutator to initialize the bundle value. - bundle.ApplyFuncContext(b.Context(), &myBundle, func(ctx context.Context, b *bundle.Bundle) {}) - - return myBundle.Config.Value() -} - func Bundle(b *testing.B, numJobs int) *bundle.Bundle { allJobs := map[string]*resources.Job{} for i := range numJobs { diff --git a/bundle/internal/bundletest/benchmark_test.go b/bundle/internal/bundletest/benchmark_test.go index 75b121e095b..df8d3659a4b 100644 --- a/bundle/internal/bundletest/benchmark_test.go +++ b/bundle/internal/bundletest/benchmark_test.go @@ -3,61 +3,18 @@ package bundletest import ( "testing" - "github.com/databricks/cli/bundle/internal/validation/generated" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" "github.com/stretchr/testify/assert" ) -// This took 40ms to run on 18th June 2025. func BenchmarkWalkReadOnly(b *testing.B) { - input := BundleV(b, 10000) + input := Bundle(b, 10000).Config.View() for b.Loop() { - err := dyn.WalkReadOnly(input, func(p dyn.Path, v dyn.Value) error { + err := structvar.Walk(input, func(p *structpath.PathNode, v structvar.View) error { return nil }) assert.NoError(b, err) } } - -// This took 160ms to run on 18th June 2025. -func BenchmarkWalk(b *testing.B) { - input := BundleV(b, 10000) - - for b.Loop() { - _, err := dyn.Walk(input, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - return v, nil - }) - assert.NoError(b, err) - } -} - -// This took 49 microseconds to run on 6th Aug 2025. -func BenchmarkEnumPrefixTree(b *testing.B) { - for b.Loop() { - // Generate prefix tree for all enum fields. - trie := &dyn.TrieNode{} - for k := range generated.EnumFields { - pattern, err := dyn.NewPatternFromString(k) - assert.NoError(b, err) - - err = trie.Insert(pattern) - assert.NoError(b, err) - } - } -} - -// This took 15 microseconds to run on 6th Aug 2025. -func BenchmarkRequiredPrefixTree(b *testing.B) { - for b.Loop() { - // Generate prefix tree for all required fields. - trie := &dyn.TrieNode{} - for k := range generated.RequiredFields { - pattern, err := dyn.NewPatternFromString(k) - assert.NoError(b, err) - - err = trie.Insert(pattern) - assert.NoError(b, err) - } - } -} diff --git a/bundle/internal/bundletest/location.go b/bundle/internal/bundletest/location.go index b96cab0907d..7290a8ae2a9 100644 --- a/bundle/internal/bundletest/location.go +++ b/bundle/internal/bundletest/location.go @@ -3,32 +3,17 @@ package bundletest import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" ) // SetLocation sets the location of all values in the bundle to the given path. // This is useful for testing where we need to associate configuration // with the path it is loaded from. func SetLocation(b *bundle.Bundle, prefix string, locations []diag.Location) { - start := dyn.MustPathFromString(prefix) - err := b.Config.Mutate(func(root dyn.Value) (dyn.Value, error) { - return dyn.Walk(root, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - // If the path has the given prefix, set the location. - if p.HasPrefix(start) { - return v.WithLocations(locations), nil - } - - // The path is not nested under the given prefix. - // If the path is a prefix of the prefix, keep traversing and return the node verbatim. - if start.HasPrefix(p) { - return v, nil - } - - // Return verbatim, but skip traversal. - return v, dyn.ErrSkip - }) - }) - if err != nil { - panic("Mutate() failed: " + err.Error()) + // "." is the root of the configuration. + var path *structpath.PathNode + if prefix != "." { + path = structpath.MustParsePath(prefix) } + b.Config.SetLocations(path, locations) } diff --git a/bundle/internal/bundletest/mutate.go b/bundle/internal/bundletest/mutate.go deleted file mode 100644 index 00adc7d29e0..00000000000 --- a/bundle/internal/bundletest/mutate.go +++ /dev/null @@ -1,17 +0,0 @@ -package bundletest - -import ( - "context" - "testing" - - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/libs/dyn" - "github.com/stretchr/testify/require" -) - -func Mutate(t *testing.T, b *bundle.Bundle, f func(v dyn.Value) (dyn.Value, error)) { - bundle.ApplyFuncContext(t.Context(), b, func(ctx context.Context, b *bundle.Bundle) { - err := b.Config.Mutate(f) - require.NoError(t, err) - }) -} diff --git a/bundle/internal/bundletest/mutator_benchmark_test.go b/bundle/internal/bundletest/mutator_benchmark_test.go index 03d63dcd072..5e60f0cfab0 100644 --- a/bundle/internal/bundletest/mutator_benchmark_test.go +++ b/bundle/internal/bundletest/mutator_benchmark_test.go @@ -7,7 +7,8 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config/validate" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" "github.com/stretchr/testify/assert" ) @@ -35,9 +36,9 @@ func benchmarkWalkReadOnlyBaseline(b *testing.B, numJobs int) { myBundle := Bundle(b, numJobs) for b.Loop() { - var paths []dyn.Path + var paths []*structpath.PathNode bundle.ApplyFuncContext(b.Context(), myBundle, func(ctx context.Context, b *bundle.Bundle) { - _ = dyn.WalkReadOnly(b.Config.Value(), func(p dyn.Path, v dyn.Value) error { + _ = structvar.Walk(b.Config.View(), func(p *structpath.PathNode, v structvar.View) error { paths = append(paths, p) return nil }) diff --git a/bundle/internal/schema/annotations.go b/bundle/internal/schema/annotations.go index 548cf5ae95f..7cc5776ff4c 100644 --- a/bundle/internal/schema/annotations.go +++ b/bundle/internal/schema/annotations.go @@ -8,9 +8,6 @@ import ( "github.com/databricks/cli/bundle/internal/annotation" "github.com/databricks/cli/internal/clijson" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/convert" - "github.com/databricks/cli/libs/dyn/merge" "github.com/databricks/cli/libs/jsonschema" ) @@ -27,12 +24,8 @@ type annotationHandler struct { // Adds annotations to the JSON schema reading from the annotation files. // More details https://json-schema.org/understanding-json-schema/reference/annotations func newAnnotationHandler(extracted, fromFile annotation.File) (*annotationHandler, error) { - merged, err := mergeAnnotationFiles(extracted, fromFile) - if err != nil { - return nil, err - } return &annotationHandler{ - parsedAnnotations: merged, + parsedAnnotations: mergeAnnotationFiles(extracted, fromFile), fileAnnotations: fromFile, missingAnnotations: annotation.File{}, }, nil @@ -79,25 +72,42 @@ func isEmptyDescriptor(d annotation.Descriptor) bool { // mergeAnnotationFiles merges later layers over earlier ones with the same // semantics the on-disk annotation files used to be merged with: maps merge // recursively, scalars take the later value, sequences concatenate. -func mergeAnnotationFiles(files ...annotation.File) (annotation.File, error) { - prev := dyn.NilValue +func mergeAnnotationFiles(files ...annotation.File) annotation.File { + merged := annotation.File{} for _, f := range files { - v, err := convert.FromTyped(f, dyn.NilValue) - if err != nil { - return nil, err - } - prev, err = merge.Merge(prev, v) - if err != nil { - return nil, err + for typeKey, ta := range f { + merged.SetSelf(typeKey, mergeDescriptor(merged[typeKey].Self, ta.Self)) + for name, d := range ta.Fields { + merged.SetField(typeKey, name, mergeDescriptor(merged[typeKey].Fields[name], d)) + } } } + return merged +} - var data annotation.File - err := convert.ToTyped(&data, prev) - if err != nil { - return nil, err +// mergeDescriptor merges the set fields of b over a: maps merge by key, sequences +// concatenate and other values are replaced. +func mergeDescriptor(a, b annotation.Descriptor) annotation.Descriptor { + dst, src := reflect.ValueOf(&a).Elem(), reflect.ValueOf(b) + for i := range dst.NumField() { + f, g := dst.Field(i), src.Field(i) + switch { + case g.IsZero(): + case f.Kind() == reflect.Slice: + f.Set(reflect.AppendSlice(reflect.AppendSlice(reflect.Zero(f.Type()), f), g)) + case f.Kind() == reflect.Map: + m := reflect.MakeMap(f.Type()) + for _, v := range []reflect.Value{f, g} { + for iter := v.MapRange(); iter.Next(); { + m.SetMapIndex(iter.Key(), iter.Value()) + } + } + f.Set(m) + default: + f.Set(g) + } } - return data, nil + return a } func (d *annotationHandler) addAnnotations(typ reflect.Type, s jsonschema.Schema) jsonschema.Schema { @@ -125,10 +135,7 @@ func (d *annotationHandler) addAnnotations(typ reflect.Type, s jsonschema.Schema // descriptions for fields that have no documentation anywhere. Entries for // fields that no longer exist in the config are dropped with a warning. func (d *annotationHandler) syncWithMissingAnnotations(outputPath string, g *typeGraph) error { - updated, err := mergeAnnotationFiles(d.fileAnnotations, d.missingAnnotations) - if err != nil { - return err - } + updated := mergeAnnotationFiles(d.fileAnnotations, d.missingAnnotations) detached, err := saveAnnotationsFile(outputPath, updated, g) if err != nil { diff --git a/bundle/internal/schema/annotations_file.go b/bundle/internal/schema/annotations_file.go index 78e279f4476..5d9f96b55bc 100644 --- a/bundle/internal/schema/annotations_file.go +++ b/bundle/internal/schema/annotations_file.go @@ -2,20 +2,19 @@ package main import ( "bytes" + "cmp" + "errors" "fmt" "os" + "path/filepath" "reflect" "slices" + "strconv" "strings" - yaml3 "go.yaml.in/yaml/v3" - "github.com/databricks/cli/bundle/internal/annotation" - "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/convert" - "github.com/databricks/cli/libs/dyn/yamlloader" - "github.com/databricks/cli/libs/dyn/yamlsaver" + "github.com/databricks/cli/libs/structs/structvar" + "go.yaml.in/yaml/v3" ) // fieldsKey nests a type's block of field nodes inside the node of a field @@ -34,14 +33,6 @@ const ( typeDocKey = "$type" ) -// lineTypeDoc and lineFields sort the "$type" and "$fields" keys after a -// node's inline descriptor keys, which the saver orders with small line -// numbers (see descriptorKeyOrder). -const ( - lineTypeDoc = 9999 - lineFields = 10000 -) - const annotationsFileHeader = `# This file contains the documentation the CLI owns for the bundle # configuration JSON schema: docs for fields that do not exist in the upstream # API spec (.codegen/cli.json), and overrides of upstream docs. Documentation @@ -82,26 +73,71 @@ var descriptorKeys = func() map[string]bool { // follow alphabetically. var descriptorKeyOrder = []string{"description", "markdown_description", "title", "default", "enum"} -// descriptorToMap serializes d into dst with its keys ordered. It returns the -// nil value (writing nothing) when d carries no content. -// -// The empty check relies on the dyn.NilValue reference: with a nil reference, -// FromTyped omits zero-valued fields, so an all-zero descriptor collapses to -// KindNil rather than an empty map. Do not pass a map reference here, or empty -// descriptors would start serializing as "{}". -func descriptorToMap(d annotation.Descriptor, dst map[string]dyn.Value) (dyn.Value, error) { - v, err := convert.FromTyped(d, dyn.NilValue) - if err != nil || v.Kind() == dyn.KindNil { - return dyn.NilValue, err - } - return yamlsaver.ConvertToMapValue(d, yamlsaver.NewOrder(descriptorKeyOrder), []string{}, dst) +// descriptorNodes serializes d into the key and value nodes of a mapping, with its +// keys ordered. It returns nothing when d carries no content. +func descriptorNodes(d annotation.Descriptor, style yaml.Style) []*yaml.Node { + var pairs [][2]*yaml.Node + for key, v := range structvar.NewView(d, nil, nil).MapItems() { + pairs = append(pairs, [2]*yaml.Node{{Kind: yaml.ScalarNode, Value: key, Style: style}, valueNode(v, style)}) + } + // Keys in descriptorKeyOrder come first, the others follow alphabetically. + rank := func(key string) int { + if i := slices.Index(descriptorKeyOrder, key); i >= 0 { + return i + } + return len(descriptorKeyOrder) + } + slices.SortFunc(pairs, func(a, b [2]*yaml.Node) int { + return cmp.Or(cmp.Compare(rank(a[0].Value), rank(b[0].Value)), cmp.Compare(a[0].Value, b[0].Value)) + }) + var out []*yaml.Node + for _, p := range pairs { + out = append(out, p[:]...) + } + return out } -// descriptorEmpty reports whether d carries no documentation. See -// descriptorToMap for why the nil reference is what makes this work. +// valueNode converts a value of a descriptor to a YAML node. +func valueNode(x structvar.View, style yaml.Style) *yaml.Node { + switch x.Kind() { + case structvar.KindMap: + n := &yaml.Node{Kind: yaml.MappingNode, Style: style} + for k, c := range x.MapItems() { + n.Content = append(n.Content, &yaml.Node{Kind: yaml.ScalarNode, Value: k, Style: style}, valueNode(c, style)) + } + return n + case structvar.KindSequence: + n := &yaml.Node{Kind: yaml.SequenceNode, Style: style} + for _, c := range x.Sequence() { + n.Content = append(n.Content, valueNode(c, style)) + } + return n + case structvar.KindString: + s, _ := x.AsString() + // A string that reads as another scalar (bool, number) is quoted to stay a string. + if isScalarValueInString(s) { + style = yaml.DoubleQuotedStyle + } + return &yaml.Node{Kind: yaml.ScalarNode, Value: s, Style: style} + default: + return &yaml.Node{Kind: yaml.ScalarNode, Value: fmt.Sprint(x.AsAny()), Style: style} + } +} + +func isScalarValueInString(s string) bool { + if s == "true" || s == "false" || s == "" { + return true + } + if _, err := strconv.ParseInt(s, 0, 64); err == nil { + return true + } + _, err := strconv.ParseFloat(s, 64) + return err == nil +} + +// descriptorEmpty reports whether d carries no documentation. func descriptorEmpty(d annotation.Descriptor) bool { - v, err := convert.FromTyped(d, dyn.NilValue) - return err == nil && v.Kind() == dyn.KindNil + return structvar.NewView(d, nil, nil).Kind() == structvar.KindNil } // loadAnnotationsFile reads the tree-format annotations file and flattens it @@ -113,7 +149,7 @@ func loadAnnotationsFile(path string, g *typeGraph) (annotation.File, []string, if err != nil { return nil, nil, err } - v, err := yamlloader.LoadYAML(path, bytes.NewBuffer(b)) + v, err := structvar.ParseYAML(bytes.NewReader(b)) if err != nil { return nil, nil, err } @@ -133,17 +169,14 @@ type fileLoader struct { } // block loads one type's block of field nodes. -func (l *fileLoader) block(v dyn.Value, typeKey, where string) error { - if v.Kind() == dyn.KindNil { - return nil - } - m, ok := v.AsMap() - if !ok { - return fmt.Errorf("%s: expected a mapping, got %s", where, v.Kind()) +func (l *fileLoader) block(v *yaml.Node, typeKey, where string) error { + pairs, err := mappingPairs(v, where) + if err != nil { + return err } - for _, pair := range m.Pairs() { - key := pair.Key.MustString() + for i := 0; i < len(pairs); i += 2 { + key := pairs[i].Value child := where + "." + key if where == "" { child = key @@ -154,7 +187,7 @@ func (l *fileLoader) block(v dyn.Value, typeKey, where string) error { l.unknown = append(l.unknown, child) continue } - err := l.node(pair.Value, typeKey, edge, child) + err := l.node(pairs[i+1], typeKey, edge, child) if err != nil { return err } @@ -165,26 +198,23 @@ func (l *fileLoader) block(v dyn.Value, typeKey, where string) error { // node loads one field's node: the inline descriptor for the field, the // "$type" docs for the type it resolves to, and the "$fields" block of that // type's fields. -func (l *fileLoader) node(v dyn.Value, typeKey string, edge fieldEdge, where string) error { - if v.Kind() == dyn.KindNil { - return nil - } - m, ok := v.AsMap() - if !ok { - return fmt.Errorf("%s: expected a mapping, got %s", where, v.Kind()) +func (l *fileLoader) node(v *yaml.Node, typeKey string, edge fieldEdge, where string) error { + pairs, err := mappingPairs(v, where) + if err != nil { + return err } - desc := dyn.NewMapping() - for _, pair := range m.Pairs() { - key := pair.Key.MustString() + var desc []*yaml.Node + for i := 0; i < len(pairs); i += 2 { + key, value := pairs[i].Value, pairs[i+1] switch { case key == fieldsKey && edge.typ != "": - err := l.block(pair.Value, edge.typ, where+"."+fieldsKey) + err := l.block(value, edge.typ, where+"."+fieldsKey) if err != nil { return err } case key == typeDocKey && edge.typ != "": - d, ok, err := l.descriptor(pair.Value, where+"."+typeDocKey) + d, ok, err := l.descriptor(value, where+"."+typeDocKey) if err != nil { return err } @@ -192,13 +222,13 @@ func (l *fileLoader) node(v dyn.Value, typeKey string, edge fieldEdge, where str l.data.SetSelf(edge.typ, d) } case descriptorKeys[key]: - desc.SetLoc(key, nil, pair.Value) + desc = append(desc, pairs[i], value) default: l.unknown = append(l.unknown, where+"."+key) } } - if desc.Len() > 0 { + if len(desc) > 0 { d, err := toDescriptor(desc, where) if err != nil { return err @@ -211,21 +241,20 @@ func (l *fileLoader) node(v dyn.Value, typeKey string, edge fieldEdge, where str // descriptor parses a mapping of descriptor keys (the value of a "$type" key). // Non-descriptor keys are flagged as unknown. The second return is false when // the mapping carries no descriptor keys. -func (l *fileLoader) descriptor(v dyn.Value, where string) (annotation.Descriptor, bool, error) { - m, ok := v.AsMap() - if !ok { - return annotation.Descriptor{}, false, fmt.Errorf("%s: expected a mapping, got %s", where, v.Kind()) - } - desc := dyn.NewMapping() - for _, pair := range m.Pairs() { - key := pair.Key.MustString() +func (l *fileLoader) descriptor(v *yaml.Node, where string) (annotation.Descriptor, bool, error) { + if v.Kind != yaml.MappingNode { + return annotation.Descriptor{}, false, fmt.Errorf("%s: expected a mapping, got %s", where, nodeKind(v)) + } + var desc []*yaml.Node + for i := 0; i < len(v.Content); i += 2 { + key := v.Content[i].Value if descriptorKeys[key] { - desc.SetLoc(key, nil, pair.Value) + desc = append(desc, v.Content[i], v.Content[i+1]) } else { l.unknown = append(l.unknown, where+"."+key) } } - if desc.Len() == 0 { + if len(desc) == 0 { return annotation.Descriptor{}, false, nil } d, err := toDescriptor(desc, where) @@ -233,15 +262,40 @@ func (l *fileLoader) descriptor(v dyn.Value, where string) (annotation.Descripto } // toDescriptor converts a mapping of descriptor keys to a typed descriptor. -func toDescriptor(desc dyn.Mapping, where string) (annotation.Descriptor, error) { +func toDescriptor(desc []*yaml.Node, where string) (annotation.Descriptor, error) { var d annotation.Descriptor - err := convert.ToTyped(&d, dyn.V(desc)) + _, diags, err := structvar.DecodeYAMLNode(where, &yaml.Node{Kind: yaml.MappingNode, Content: desc}, &d, nil) + if err == nil { + err = diags.Error() + } if err != nil { return annotation.Descriptor{}, fmt.Errorf("%s: %w", where, err) } return d, nil } +// mappingPairs returns the alternating key and value nodes of a mapping. A null node has none. +func mappingPairs(v *yaml.Node, where string) ([]*yaml.Node, error) { + switch { + case v.ShortTag() == "!!null": + return nil, nil + case v.Kind != yaml.MappingNode: + return nil, fmt.Errorf("%s: expected a mapping, got %s", where, nodeKind(v)) + } + return v.Content, nil +} + +func nodeKind(v *yaml.Node) string { + switch v.Kind { + case yaml.MappingNode: + return "map" + case yaml.SequenceNode: + return "sequence" + default: + return strings.TrimPrefix(v.ShortTag(), "!!") + } +} + // saveAnnotationsFile writes data to path in the canonical tree layout: a // depth-first walk over the config type graph in struct declaration order // expands every type at its first occurrence; keys are emitted alphabetically. @@ -258,18 +312,10 @@ func saveAnnotationsFile(path string, data annotation.File, g *typeGraph) ([]str } s.assignCanonical(g.root) - root, err := s.block(g.root) - if err != nil { - return nil, err - } - - // Style every top-level key so all nested scalars render in literal block - // style, matching the formatting of the previous annotation files. - style := map[string]yaml3.Style{} - for k := range root { - style[k] = yaml3.LiteralStyle - } - err = yamlsaver.NewSaverWithStyle(style).SaveAsYAML(root, path, true) + // Everything below the top-level keys renders in literal block style, matching + // the formatting of the previous annotation files. + root := &yaml.Node{Kind: yaml.MappingNode, Content: s.block(g.root, 0)} + err := writeYAML(path, root) if err != nil { return nil, err } @@ -280,6 +326,20 @@ func saveAnnotationsFile(path string, data annotation.File, g *typeGraph) ([]str return s.detached(), nil } +func writeYAML(path string, node *yaml.Node) error { + err := os.MkdirAll(filepath.Dir(path), 0o755) + if err != nil { + return err + } + f, err := os.Create(path) + if err != nil { + return err + } + enc := yaml.NewEncoder(f) + enc.SetIndent(2) + return errors.Join(enc.Encode(node), f.Close()) +} + type edgeKey struct { typ string name string @@ -307,61 +367,52 @@ func (s *fileSaver) assignCanonical(typeKey string) { } } -// block renders one type's block of field nodes, emitted alphabetically. -// Lines in the value locations encode the output order for the YAML saver. -func (s *fileSaver) block(typeKey string) (map[string]dyn.Value, error) { - out := map[string]dyn.Value{} - line := 0 +// block renders one type's block of field nodes as the content of a mapping, emitted +// alphabetically, with the keys in keyStyle. +func (s *fileSaver) block(typeKey string, keyStyle yaml.Style) []*yaml.Node { + var out []*yaml.Node edges := slices.Clone(s.graph.fields[typeKey]) slices.SortFunc(edges, func(a, b fieldEdge) int { return strings.Compare(a.name, b.name) }) for _, edge := range edges { - node, err := s.node(typeKey, edge) - if err != nil { - return nil, err - } + node := s.node(typeKey, edge) if len(node) > 0 { - out[edge.name] = dyn.NewValue(node, []diag.Location{{Line: line}}) - line++ + out = append(out, &yaml.Node{Kind: yaml.ScalarNode, Value: edge.name, Style: keyStyle}, mapping(node)) } } - return out, nil + return out +} + +func mapping(content []*yaml.Node) *yaml.Node { + return &yaml.Node{Kind: yaml.MappingNode, Content: content, Style: yaml.LiteralStyle} +} + +func literalKey(name string) *yaml.Node { + return &yaml.Node{Kind: yaml.ScalarNode, Value: name, Style: yaml.LiteralStyle} } // node renders one field's node: the inline field descriptor plus, at the // field's canonical position, the resolved type's "$type" docs and the // "fields" block of its fields. -func (s *fileSaver) node(typeKey string, edge fieldEdge) (map[string]dyn.Value, error) { - out := map[string]dyn.Value{} +func (s *fileSaver) node(typeKey string, edge fieldEdge) []*yaml.Node { + var out []*yaml.Node - // The inline descriptor keys are written directly into the node, sharing - // it with the "$type" and "$fields" keys added below. if d, ok := s.takeField(typeKey, edge.name); ok { - if _, err := descriptorToMap(d, out); err != nil { - return nil, err - } + out = descriptorNodes(d, yaml.LiteralStyle) } if s.expandAt[edgeKey{typeKey, edge.name}] { - v, err := descriptorToMap(s.takeSelf(edge.typ), map[string]dyn.Value{}) - if err != nil { - return nil, err - } - if v.Kind() != dyn.KindNil { - out[typeDocKey] = v.WithLocations([]diag.Location{{Line: lineTypeDoc}}) + if doc := descriptorNodes(s.takeSelf(edge.typ), yaml.LiteralStyle); len(doc) > 0 { + out = append(out, literalKey(typeDocKey), mapping(doc)) } - child, err := s.block(edge.typ) - if err != nil { - return nil, err - } - if len(child) > 0 { - out[fieldsKey] = dyn.NewValue(child, []diag.Location{{Line: lineFields}}) + if child := s.block(edge.typ, yaml.LiteralStyle); len(child) > 0 { + out = append(out, literalKey(fieldsKey), mapping(child)) } } - return out, nil + return out } // takeField returns the descriptor for a field and marks it consumed for the diff --git a/bundle/internal/schema/main.go b/bundle/internal/schema/main.go index 01a743933b2..75d36dd4824 100644 --- a/bundle/internal/schema/main.go +++ b/bundle/internal/schema/main.go @@ -13,17 +13,17 @@ import ( "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/config/variable" "github.com/databricks/cli/internal/clijson" - "github.com/databricks/cli/libs/dyn/dynvar" "github.com/databricks/cli/libs/jsonschema" + "github.com/databricks/cli/libs/structs/structvar" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/databricks/databricks-sdk-go/service/pipelines" ) // interpolationPattern builds a JSON Schema regex for ${prefix.path...} references. -// Path segments use [dynvar.BaseVarDef]; unlike the runtime matcher in ref.go, this +// Path segments use [structvar.BaseVarDef]; unlike the runtime matcher in ref.go, this // requires a fixed prefix (var, resources, ...) and at least one ".segment" after it. func interpolationPattern(s string) string { - return fmt.Sprintf(`\$\{(%s(\.%s(\[[0-9]+\])*)+)\}`, s, dynvar.BaseVarDef) + return fmt.Sprintf(`\$\{(%s(\.%s(\[[0-9]+\])*)+)\}`, s, structvar.BaseVarDef) } func addInterpolationPatterns(typ reflect.Type, s jsonschema.Schema) jsonschema.Schema { diff --git a/bundle/internal/schema/main_test.go b/bundle/internal/schema/main_test.go index 92184a2d48f..4f6a4c741d8 100644 --- a/bundle/internal/schema/main_test.go +++ b/bundle/internal/schema/main_test.go @@ -1,21 +1,20 @@ package main import ( - "bytes" "io" + "maps" "os" "path" "reflect" + "slices" "strings" "testing" "github.com/databricks/cli/bundle/config/resources" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/merge" - "github.com/databricks/cli/libs/dyn/yamlloader" "github.com/databricks/cli/libs/jsonschema" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.yaml.in/yaml/v3" ) const cliJSONPath = "../../../.codegen/cli.json" @@ -73,36 +72,45 @@ func TestRequiredAnnotationsForNewFields(t *testing.T) { require.NoError(t, err) currentFile, err := os.ReadFile(annotationsPath) require.NoError(t, err) - original, err := yamlloader.LoadYAML("", bytes.NewBuffer(originalFile)) - require.NoError(t, err) - current, err := yamlloader.LoadYAML("", bytes.NewBuffer(currentFile)) - require.NoError(t, err) + var original, current any + require.NoError(t, yaml.Unmarshal(originalFile, &original)) + require.NoError(t, yaml.Unmarshal(currentFile, ¤t)) // Regenerating from the committed file must be a no-op: no new placeholders // (a new undocumented config field) and no deletes/updates (stale - // placeholders not yet pruned). VisitDelete/VisitUpdate must be set or - // Override panics on any change. + // placeholders not yet pruned). var addedFieldPaths []string var changedFieldPaths []string - _, err = merge.Override(original, current, merge.OverrideVisitor{ - VisitInsert: func(basePath dyn.Path, right dyn.Value) (dyn.Value, error) { - addedFieldPaths = append(addedFieldPaths, basePath.String()) - return right, nil - }, - VisitDelete: func(basePath dyn.Path, left dyn.Value) error { - changedFieldPaths = append(changedFieldPaths, basePath.String()) - return nil - }, - VisitUpdate: func(basePath dyn.Path, left, right dyn.Value) (dyn.Value, error) { - changedFieldPaths = append(changedFieldPaths, basePath.String()) - return right, nil - }, - }) - assert.NoError(t, err) + diffYAML("", original, current, &addedFieldPaths, &changedFieldPaths) assert.Empty(t, addedFieldPaths, "Missing JSON-schema descriptions for new config fields in bundle/internal/schema/annotations.yml:\n%s", strings.Join(addedFieldPaths, "\n")) assert.Empty(t, changedFieldPaths, "annotations.yml is out of sync; run `./task generate-schema` and commit the result:\n%s", strings.Join(changedFieldPaths, "\n")) } +// diffYAML records the paths of the values added to, and removed from or changed in, left to get right. +func diffYAML(path string, left, right any, added, changed *[]string) { + l, lok := left.(map[string]any) + r, rok := right.(map[string]any) + if !lok || !rok { + if !reflect.DeepEqual(left, right) { + *changed = append(*changed, path) + } + return + } + for _, k := range slices.Sorted(maps.Keys(l)) { + if _, ok := r[k]; !ok { + *changed = append(*changed, strings.TrimPrefix(path+"."+k, ".")) + } + } + for _, k := range slices.Sorted(maps.Keys(r)) { + p := strings.TrimPrefix(path+"."+k, ".") + if lv, ok := l[k]; ok { + diffYAML(p, lv, r[k], added, changed) + } else { + *added = append(*added, p) + } + } +} + // Checks that the annotations file only contains entries that match the // current bundle configuration structure. func TestNoDetachedAnnotations(t *testing.T) { diff --git a/bundle/libraries/expand_glob_references.go b/bundle/libraries/expand_glob_references.go index df089f880a7..632bf1be60e 100644 --- a/bundle/libraries/expand_glob_references.go +++ b/bundle/libraries/expand_glob_references.go @@ -4,35 +4,35 @@ import ( "context" "fmt" "path/filepath" + "reflect" "strings" "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/patchwheel" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" + "github.com/databricks/databricks-sdk-go/service/compute" ) type expand struct{} -func matchError(p dyn.Path, l []diag.Location, message string) diag.Diagnostic { +func matchError(p *structpath.PathNode, l []diag.Location, message string) diag.Diagnostic { return diag.Diagnostic{ Severity: diag.Error, Summary: message, Locations: l, - Paths: dyn.ToStructPaths(p), + Paths: []*structpath.PathNode{p}, } } -func getLibDetails(v dyn.Value) (string, string, bool) { - m := v.MustMap() - whl, ok := m.GetByString("whl") - if ok { - return whl.MustString(), "whl", true +func getLibDetails(lib compute.Library) (string, string, bool) { + if lib.Whl != "" { + return lib.Whl, "whl", true } - jar, ok := m.GetByString("jar") - if ok { - return jar.MustString(), "jar", true + if lib.Jar != "" { + return lib.Jar, "jar", true } return "", "", false @@ -72,48 +72,92 @@ func isGlobPattern(path string) bool { return strings.ContainsAny(path, "*?[") } -func expandLibraries(ctx context.Context, b *bundle.Bundle, p dyn.Path, v dyn.Value) (diag.Diagnostics, []dyn.Value) { - var output []dyn.Value +// expandFunc expands the item at path p into the items it is replaced with. +// If relocate is not empty, it names the field of each new item that takes the locations of the item. +type expandFunc[T any] func(ctx context.Context, b *bundle.Bundle, p *structpath.PathNode, item T) (output []T, relocate string, diags diag.Diagnostics) + +// expandSequence replaces the items of the sequence at path p with the items expandFunc returns for them. +// New items keep the locations of the item they were expanded from. +func expandSequence[T any](ctx context.Context, b *bundle.Bundle, p *structpath.PathNode, lv structvar.View, fn expandFunc[T]) diag.Diagnostics { + items, ok := sequencePointer[T](lv) + if !ok { + return nil + } + var diags diag.Diagnostics + output := make([]T, 0, len(*items)) + var sources [][]int + var relocates []string + var locations [][]diag.Location + for i, item := range *items { + ip := structpath.NewIndex(p, i) + expanded, relocate, d := fn(ctx, b, ip, item) + diags = diags.Extend(d) + locs := b.Config.LocationsAt(ip) + for _, e := range expanded { + output = append(output, e) + sources = append(sources, []int{i}) + relocates = append(relocates, relocate) + locations = append(locations, locs) + } + } + + *items = output + b.Config.UpdateSequence(p, sources) + for i, relocate := range relocates { + if relocate != "" { + b.Config.SetLocations(structpath.NewStringKey(structpath.NewIndex(p, i), relocate), locations[i]) + } + } + + return diags +} - libs := v.MustSequence() - for i, lib := range libs { - lp := p.Append(dyn.Index(i)) +// sequencePointer returns a pointer to the typed sequence the view is based on. +func sequencePointer[T any](lv structvar.View) (*[]T, bool) { + if lv.Kind() != structvar.KindSequence { + return nil, false + } + + v := lv.Reflect() + for v.Kind() == reflect.Pointer || v.Kind() == reflect.Interface { + v = v.Elem() + } + if !v.CanAddr() { + return nil, false + } + + return reflect.TypeAssert[*[]T](v.Addr()) +} + +func expandLibraries(ctx context.Context, lv structvar.View, p *structpath.PathNode, b *bundle.Bundle) diag.Diagnostics { + return expandSequence(ctx, b, p, lv, func(ctx context.Context, b *bundle.Bundle, ip *structpath.PathNode, lib compute.Library) ([]compute.Library, string, diag.Diagnostics) { path, libType, supported := getLibDetails(lib) if !supported || !IsLibraryLocal(path) { - output = append(output, lib) - continue + return []compute.Library{lib}, "", nil } - lp = lp.Append(dyn.Key(libType)) - matches, err := findMatches(ctx, b, path) if err != nil { - diags = diags.Append(matchError(lp, lib.Locations(), err.Error())) - continue + return nil, "", diag.Diagnostics{matchError(structpath.NewStringKey(ip, libType), b.Config.LocationsAt(ip), err.Error())} } + var output []compute.Library for _, match := range matches { - output = append(output, dyn.NewValue(map[string]dyn.Value{ - libType: dyn.NewValue(match, lib.Locations()), - }, lib.Locations())) + if libType == "whl" { + output = append(output, compute.Library{Whl: match}) + } else { + output = append(output, compute.Library{Jar: match}) + } } - } - - return diags, output + return output, libType, nil + }) } -func expandEnvironmentDeps(ctx context.Context, b *bundle.Bundle, p dyn.Path, v dyn.Value) (diag.Diagnostics, []dyn.Value) { - var output []dyn.Value - var diags diag.Diagnostics - - deps := v.MustSequence() - for i, dep := range deps { - lp := p.Append(dyn.Index(i)) - path := dep.MustString() +func expandEnvironmentDeps(ctx context.Context, lv structvar.View, p *structpath.PathNode, b *bundle.Bundle) diag.Diagnostics { + return expandSequence(ctx, b, p, lv, func(ctx context.Context, b *bundle.Bundle, ip *structpath.PathNode, path string) ([]string, string, diag.Diagnostics) { if !IsLibraryLocal(path) { - output = append(output, dep) - continue + return []string{path}, "", nil } // Strip extras before globbing so "[...]" isn't read as a glob class, then re-append. @@ -121,88 +165,30 @@ func expandEnvironmentDeps(ctx context.Context, b *bundle.Bundle, p dyn.Path, v matches, err := findMatches(ctx, b, path) if err != nil { - diags = diags.Append(matchError(lp, dep.Locations(), err.Error())) - continue + return nil, "", diag.Diagnostics{matchError(ip, b.Config.LocationsAt(ip), err.Error())} } + var output []string for _, match := range matches { - output = append(output, dyn.NewValue(match+extras, dep.Locations())) + output = append(output, match+extras) } - } - - return diags, output + return output, "", nil + }) } type expandPattern struct { - pattern dyn.Pattern - fn func(ctx context.Context, b *bundle.Bundle, p dyn.Path, v dyn.Value) (diag.Diagnostics, []dyn.Value) + pattern *structpath.PatternNode + fn func(ctx context.Context, lv structvar.View, p *structpath.PathNode, b *bundle.Bundle) diag.Diagnostics } -var taskLibrariesPattern = dyn.NewPattern( - dyn.Key("resources"), - dyn.Key("jobs"), - dyn.AnyKey(), - dyn.Key("tasks"), - dyn.AnyIndex(), - dyn.Key("libraries"), -) - -var forEachTaskLibrariesPattern = dyn.NewPattern( - dyn.Key("resources"), - dyn.Key("jobs"), - dyn.AnyKey(), - dyn.Key("tasks"), - dyn.AnyIndex(), - dyn.Key("for_each_task"), - dyn.Key("task"), - dyn.Key("libraries"), -) - -var aiRuntimeCodeSourcePattern = dyn.NewPattern( - dyn.Key("resources"), - dyn.Key("jobs"), - dyn.AnyKey(), - dyn.Key("tasks"), - dyn.AnyIndex(), - dyn.Key("ai_runtime_task"), - dyn.Key("code_source_path"), -) - -var forEachAiRuntimeCodeSourcePattern = dyn.NewPattern( - dyn.Key("resources"), - dyn.Key("jobs"), - dyn.AnyKey(), - dyn.Key("tasks"), - dyn.AnyIndex(), - dyn.Key("for_each_task"), - dyn.Key("task"), - dyn.Key("ai_runtime_task"), - dyn.Key("code_source_path"), -) - -var envDepsPattern = dyn.NewPattern( - dyn.Key("resources"), - dyn.Key("jobs"), - dyn.AnyKey(), - dyn.Key("environments"), - dyn.AnyIndex(), - dyn.Key("spec"), - dyn.Key("dependencies"), -) - -var pipelineEnvDepsPattern = dyn.NewPattern( - dyn.Key("resources"), - dyn.Key("pipelines"), - dyn.AnyKey(), - dyn.Key("environment"), - dyn.Key("dependencies"), -) - -var clusterLibrariesPattern = dyn.NewPattern( - dyn.Key("resources"), - dyn.Key("clusters"), - dyn.AnyKey(), - dyn.Key("libraries"), +var ( + taskLibrariesPattern = structpath.MustParsePattern("resources.jobs.*.tasks[*].libraries") + forEachTaskLibrariesPattern = structpath.MustParsePattern("resources.jobs.*.tasks[*].for_each_task.task.libraries") + aiRuntimeCodeSourcePattern = structpath.MustParsePattern("resources.jobs.*.tasks[*].ai_runtime_task.code_source_path") + forEachAiRuntimeCodeSourcePattern = structpath.MustParsePattern("resources.jobs.*.tasks[*].for_each_task.task.ai_runtime_task.code_source_path") + envDepsPattern = structpath.MustParsePattern("resources.jobs.*.environments[*].spec.dependencies") + pipelineEnvDepsPattern = structpath.MustParsePattern("resources.pipelines.*.environment.dependencies") + clusterLibrariesPattern = structpath.MustParsePattern("resources.clusters.*.libraries") ) func (e *expand) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { @@ -231,23 +217,15 @@ func (e *expand) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { var diags diag.Diagnostics - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { - var err error - for _, expander := range expanders { - v, err = dyn.MapByPattern(v, expander.pattern, func(p dyn.Path, lv dyn.Value) (dyn.Value, error) { - d, output := expander.fn(ctx, b, p, lv) - diags = diags.Extend(d) - return dyn.NewValue(output, lv.Locations()), nil - }) - if err != nil { - return dyn.InvalidValue, err - } + for _, expander := range expanders { + err := structvar.ForEach(b.Config.View(), expander.pattern, func(p *structpath.PathNode, lv structvar.View) error { + diags = diags.Extend(expander.fn(ctx, lv, p, b)) + return nil + }) + if err != nil { + diags = diags.Extend(diag.FromErr(err)) + break } - - return v, nil - }) - if err != nil { - diags = diags.Extend(diag.FromErr(err)) } return diags diff --git a/bundle/libraries/expand_glob_references_test.go b/bundle/libraries/expand_glob_references_test.go index 331fbf2e7cb..5bb4a49fff8 100644 --- a/bundle/libraries/expand_glob_references_test.go +++ b/bundle/libraries/expand_glob_references_test.go @@ -10,7 +10,7 @@ import ( "github.com/databricks/cli/bundle/internal/bundletest" "github.com/databricks/cli/internal/testutil" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/databricks-sdk-go/service/compute" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/databricks/databricks-sdk-go/service/pipelines" @@ -283,11 +283,19 @@ func TestExpandGlobReferencesPreservesLocations(t *testing.T) { diags := bundle.Apply(t.Context(), b, ExpandGlobReferences()) require.Empty(t, diags) - libs, err := dyn.GetByPath(b.Config.Value(), dyn.MustPathFromString("resources.jobs.job.tasks[0].libraries")) - require.NoError(t, err) - assert.Equal(t, loc.File, libs.Location().File) + libs := b.Config.LocationsAt(structpath.MustParsePath("resources.jobs.job.tasks[0].libraries")) + require.NotEmpty(t, libs) + assert.Equal(t, loc.File, libs[0].File) - deps, err := dyn.GetByPath(b.Config.Value(), dyn.MustPathFromString("resources.pipelines.pipeline.environment.dependencies")) - require.NoError(t, err) - assert.Equal(t, loc.File, deps.Location().File) + lib := b.Config.LocationsAt(structpath.MustParsePath("resources.jobs.job.tasks[0].libraries[0].whl")) + require.NotEmpty(t, lib) + assert.Equal(t, loc.File, lib[0].File) + + deps := b.Config.LocationsAt(structpath.MustParsePath("resources.pipelines.pipeline.environment.dependencies")) + require.NotEmpty(t, deps) + assert.Equal(t, loc.File, deps[0].File) + + dep := b.Config.LocationsAt(structpath.MustParsePath("resources.pipelines.pipeline.environment.dependencies[0]")) + require.NotEmpty(t, dep) + assert.Equal(t, loc.File, dep[0].File) } diff --git a/bundle/libraries/remote_path.go b/bundle/libraries/remote_path.go index 70e577a1a2a..d28e21c10c1 100644 --- a/bundle/libraries/remote_path.go +++ b/bundle/libraries/remote_path.go @@ -11,8 +11,9 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/patchwheel" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) // LocalLibraryPaths returns the local file paths of all libraries in the bundle @@ -101,25 +102,20 @@ func ReplaceWithRemotePath(ctx context.Context, b *bundle.Bundle) (map[string][] sources := slices.Sorted(maps.Keys(libs)) // Update all the config paths to point to the uploaded location - err = b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { - for _, source := range sources { - locations := libs[source] - remotePath := path.Join(uploadPath, filepath.Base(source)) - - for _, location := range locations { - // Re-append the extras suffix that was stripped before upload. - remotePathWithExtras := remotePath + location.extras - v, err = dyn.SetByPath(v, location.configPath, dyn.NewValue(remotePathWithExtras, []diag.Location{location.location})) - if err != nil { - return v, fmt.Errorf("internal error: failed to update path %#v to %#v: %w", source, remotePathWithExtras, err) - } + for _, source := range sources { + locations := libs[source] + remotePath := path.Join(uploadPath, filepath.Base(source)) + + for _, location := range locations { + // Re-append the extras suffix that was stripped before upload. + remotePathWithExtras := remotePath + location.extras + err = b.Config.Set(location.configPath, remotePathWithExtras) + if err != nil { + diags = diags.Extend(diag.FromErr(fmt.Errorf("internal error: failed to update path %#v to %#v: %w", source, remotePathWithExtras, err))) + return libs, diags } + b.Config.SetLocations(location.configPath, []diag.Location{location.location}) } - - return v, nil - }) - if err != nil { - diags = diags.Extend(diag.FromErr(err)) } return libs, diags @@ -134,15 +130,15 @@ func ReplaceWithRemotePath(ctx context.Context, b *bundle.Bundle) (map[string][] func collectLocalLibraries(b *bundle.Bundle) (map[string][]LocationToUpdate, error) { libs := make(map[string]([]LocationToUpdate)) - patterns := []dyn.Pattern{ - taskLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("whl")), - taskLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("jar")), - forEachTaskLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("whl")), - forEachTaskLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("jar")), - clusterLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("whl")), - clusterLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("jar")), - envDepsPattern.Append(dyn.AnyIndex()), - pipelineEnvDepsPattern.Append(dyn.AnyIndex()), + patterns := []*structpath.PatternNode{ + structpath.MustParsePattern(taskLibrariesPattern.String() + "[*].whl"), + structpath.MustParsePattern(taskLibrariesPattern.String() + "[*].jar"), + structpath.MustParsePattern(forEachTaskLibrariesPattern.String() + "[*].whl"), + structpath.MustParsePattern(forEachTaskLibrariesPattern.String() + "[*].jar"), + structpath.MustParsePattern(clusterLibrariesPattern.String() + "[*].whl"), + structpath.MustParsePattern(clusterLibrariesPattern.String() + "[*].jar"), + structpath.MustParsePattern(envDepsPattern.String() + "[*]"), + structpath.MustParsePattern(pipelineEnvDepsPattern.String() + "[*]"), // The AI Runtime task's code_source_path is a local archive (typically an // artifact-built .tar.gz) that must be uploaded and referenced by its remote // path, exactly like a wheel or jar library. @@ -150,75 +146,63 @@ func collectLocalLibraries(b *bundle.Bundle) (map[string][]LocationToUpdate, err forEachAiRuntimeCodeSourcePattern, } + root := b.Config.View() for _, pattern := range patterns { - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { - return dyn.MapByPattern(v, pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - source, ok := v.AsString() - if !ok { - return v, fmt.Errorf("expected string, got %s", v.Kind()) - } - - if !IsLibraryLocal(source) { - return v, nil - } + err := structvar.ForEach(root, pattern, func(p *structpath.PathNode, v structvar.View) error { + source, ok := v.AsString() + if !ok { + return fmt.Errorf("expected string, got %s", v.Kind()) + } - // Split off any pip extras suffix so the upload targets the real - // file; the suffix is re-appended to the remote path afterwards. - source, extras := patchwheel.SplitWheelExtras(source) + if !IsLibraryLocal(source) { + return nil + } - source = filepath.Join(b.SyncRootPath, source) - libs[source] = append(libs[source], LocationToUpdate{ - configPath: p, - location: v.Location(), - extras: extras, - }) + // Split off any pip extras suffix so the upload targets the real + // file; the suffix is re-appended to the remote path afterwards. + source, extras := patchwheel.SplitWheelExtras(source) - return v, nil + source = filepath.Join(b.SyncRootPath, source) + libs[source] = append(libs[source], LocationToUpdate{ + configPath: p, + location: v.Location(), + extras: extras, }) + + return nil }) if err != nil { return nil, err } } - artifactPattern := dyn.NewPattern( - dyn.Key("artifacts"), - dyn.AnyKey(), - dyn.Key("files"), - dyn.AnyIndex(), - ) - - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { - return dyn.MapByPattern(v, artifactPattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - file, ok := v.AsMap() - if !ok { - return v, fmt.Errorf("expected map, got %s", v.Kind()) - } + artifactPattern := structpath.MustParsePattern("artifacts.*.files[*]") - sv, ok := file.GetByString("source") - if !ok { - return v, nil - } + err := structvar.ForEach(root, artifactPattern, func(p *structpath.PathNode, v structvar.View) error { + if v.Kind() != structvar.KindMap { + return fmt.Errorf("expected map, got %s", v.Kind()) + } - source, ok := sv.AsString() - if !ok { - return v, fmt.Errorf("expected string, got %s", v.Kind()) - } + sv := v.Get("source") + if !sv.IsValid() { + return nil + } - if sv, ok = file.GetByString("patched"); ok { - patched, ok := sv.AsString() - if ok && patched != "" { - source = patched - } - } + source, ok := sv.AsString() + if !ok { + return fmt.Errorf("expected string, got %s", v.Kind()) + } - libs[source] = append(libs[source], LocationToUpdate{ - configPath: p.Append(dyn.Key("remote_path")), - location: v.Location(), - }) + if patched, ok := v.Get("patched").AsString(); ok && patched != "" { + source = patched + } - return v, nil + libs[source] = append(libs[source], LocationToUpdate{ + configPath: structpath.NewStringKey(p, "remote_path"), + location: v.Location(), }) + + return nil }) if err != nil { return nil, err diff --git a/bundle/libraries/same_name_libraries.go b/bundle/libraries/same_name_libraries.go index 722d42c7f64..a2ba9591a4f 100644 --- a/bundle/libraries/same_name_libraries.go +++ b/bundle/libraries/same_name_libraries.go @@ -7,21 +7,21 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) type checkForSameNameLibraries struct{} -var patterns = []dyn.Pattern{ - taskLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("whl")), - taskLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("jar")), - forEachTaskLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("whl")), - forEachTaskLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("jar")), - clusterLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("whl")), - clusterLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("jar")), - envDepsPattern.Append(dyn.AnyIndex()), - pipelineEnvDepsPattern.Append(dyn.AnyIndex()), +var patterns = []*structpath.PatternNode{ + structpath.MustParsePattern(taskLibrariesPattern.String() + "[*].whl"), + structpath.MustParsePattern(taskLibrariesPattern.String() + "[*].jar"), + structpath.MustParsePattern(forEachTaskLibrariesPattern.String() + "[*].whl"), + structpath.MustParsePattern(forEachTaskLibrariesPattern.String() + "[*].jar"), + structpath.MustParsePattern(clusterLibrariesPattern.String() + "[*].whl"), + structpath.MustParsePattern(clusterLibrariesPattern.String() + "[*].jar"), + structpath.MustParsePattern(envDepsPattern.String() + "[*]"), + structpath.MustParsePattern(pipelineEnvDepsPattern.String() + "[*]"), } type libData struct { @@ -35,50 +35,43 @@ func (c checkForSameNameLibraries) Apply(ctx context.Context, b *bundle.Bundle) var diags diag.Diagnostics libs := make(map[string]*libData) - err := b.Config.Mutate(func(rootConfig dyn.Value) (dyn.Value, error) { - var err error - for _, pattern := range patterns { - rootConfig, err = dyn.MapByPattern(rootConfig, pattern, func(p dyn.Path, libraryValue dyn.Value) (dyn.Value, error) { - libPath, ok := libraryValue.AsString() - if !ok { - return libraryValue, nil - } + root := b.Config.View() + var err error + for _, pattern := range patterns { + err = structvar.ForEach(root, pattern, func(p *structpath.PathNode, libraryValue structvar.View) error { + libPath, ok := libraryValue.AsString() + if !ok { + return nil + } - // If not local library, skip the check - if !IsLibraryLocal(libPath) { - return libraryValue, nil - } + // If not local library, skip the check + if !IsLibraryLocal(libPath) { + return nil + } - lib := filepath.Base(libPath) - // If the same basename was seen already but full path is different - // then it's a duplicate. Add the location to the location list. - lp, ok := libs[lib] - if !ok { - libs[lib] = &libData{ - fullPath: libPath, - locations: []diag.Location{libraryValue.Location()}, - paths: dyn.ToStructPaths(p), - otherPaths: []string{}, - } - } else if lp.fullPath != libPath { - lp.locations = append(lp.locations, libraryValue.Location()) - lp.paths = append(lp.paths, dyn.ToStructPath(p)) - lp.otherPaths = append(lp.otherPaths, libPath) + lib := filepath.Base(libPath) + // If the same basename was seen already but full path is different + // then it's a duplicate. Add the location to the location list. + lp, ok := libs[lib] + if !ok { + libs[lib] = &libData{ + fullPath: libPath, + locations: []diag.Location{libraryValue.Location()}, + paths: []*structpath.PathNode{p}, + otherPaths: []string{}, } - - return libraryValue, nil - }) - if err != nil { - return dyn.InvalidValue, err + } else if lp.fullPath != libPath { + lp.locations = append(lp.locations, libraryValue.Location()) + lp.paths = append(lp.paths, p) + lp.otherPaths = append(lp.otherPaths, libPath) } - } + return nil + }) if err != nil { - return dyn.InvalidValue, err + break } - - return rootConfig, nil - }) + } // Iterate over all the libraries and check if there are any duplicates. // Duplicates will have more than one location. diff --git a/bundle/libraries/upload.go b/bundle/libraries/upload.go index eeb71cfb9e8..52e34de1915 100644 --- a/bundle/libraries/upload.go +++ b/bundle/libraries/upload.go @@ -12,11 +12,10 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/filer" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/retry" - + "github.com/databricks/cli/libs/structs/structpath" "golang.org/x/sync/errgroup" ) @@ -37,7 +36,7 @@ type upload struct { } type LocationToUpdate struct { - configPath dyn.Path + configPath *structpath.PathNode location diag.Location // extras is the pip extras suffix (e.g. "[train]") to re-append to the // rewritten remote path. Empty for libraries that carry no extras. diff --git a/bundle/migrate/build_state.go b/bundle/migrate/build_state.go index 0f0776152eb..4bc28889fc5 100644 --- a/bundle/migrate/build_state.go +++ b/bundle/migrate/build_state.go @@ -13,8 +13,6 @@ import ( "github.com/databricks/cli/bundle/direct" "github.com/databricks/cli/bundle/direct/dresources" "github.com/databricks/cli/bundle/direct/dstate" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/dynvar" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/structs/structaccess" "github.com/databricks/cli/libs/structs/structpath" @@ -58,18 +56,20 @@ func BuildStateFromTF( warningsSeen := false // Collect all resource nodes (same patterns as makePlan). var nodes []string - patterns := []dyn.Pattern{ - dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey()), - dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey(), dyn.Key("permissions")), - dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey(), dyn.Key("grants")), + rootView := configRoot.View() + + patterns := []*structpath.PatternNode{ + structpath.MustParsePattern("resources.*.*"), + structpath.MustParsePattern("resources.*.*.permissions"), + structpath.MustParsePattern("resources.*.*.grants"), } for _, pat := range patterns { - _, err := dyn.MapByPattern( - configRoot.Value(), + err := structvar.ForEach( + rootView, pat, - func(p dyn.Path, v dyn.Value) (dyn.Value, error) { + func(p *structpath.PathNode, _ structvar.View) error { nodes = append(nodes, p.String()) - return dyn.InvalidValue, nil + return nil }, ) if err != nil { @@ -112,7 +112,7 @@ func BuildStateFromTF( return warningsSeen, fmt.Errorf("%s: PrepareState: %w", node, err) } - refs, err := direct.ExtractReferences(configRoot.Value(), node, adapter.StateType()) + refs, err := direct.ExtractReferences(rootView, node, adapter.StateType()) if err != nil { return warningsSeen, fmt.Errorf("%s: extracting references: %w", node, err) } @@ -125,12 +125,12 @@ func BuildStateFromTF( // Same logic as makePlan in bundle/direct/bundle_plan.go. var dependsOn []deployplan.DependsOnEntry //nolint:prealloc for _, refTemplate := range refs { - ref, ok := dynvar.NewRef(dyn.V(refTemplate)) + ref, ok := structvar.NewRef(refTemplate) if !ok { continue } for _, targetPath := range ref.References() { - targetPathParsed, err := dyn.NewPathFromString(targetPath) + targetPathParsed, err := structpath.ParsePath(targetPath) if err != nil { continue } diff --git a/bundle/migrate/build_state_test.go b/bundle/migrate/build_state_test.go index a495436008c..c73ac2131b9 100644 --- a/bundle/migrate/build_state_test.go +++ b/bundle/migrate/build_state_test.go @@ -1,7 +1,6 @@ package migrate_test import ( - "bytes" "encoding/json" "os" "path/filepath" @@ -12,24 +11,18 @@ import ( "github.com/databricks/cli/bundle/direct/dresources" "github.com/databricks/cli/bundle/direct/dstate" "github.com/databricks/cli/bundle/migrate" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/convert" - "github.com/databricks/cli/libs/dyn/yamlloader" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) // rootFromYAML builds a config.Root from a YAML snippet. // Template strings like "${resources.jobs.src.name}" are preserved in the -// internal dyn.Value so BuildStateFromTF can find them via ExtractReferences. +// configuration so BuildStateFromTF can find them via ExtractReferences. func rootFromYAML(t *testing.T, yaml string) config.Root { t.Helper() - v, err := yamlloader.LoadYAML("test", bytes.NewBufferString(yaml)) - require.NoError(t, err) - var root config.Root - require.NoError(t, convert.ToTyped(&root, v)) - require.NoError(t, root.Mutate(func(_ dyn.Value) (dyn.Value, error) { return v, nil })) - return root + root, diags := config.LoadFromBytes("test", []byte(yaml)) + require.NoError(t, diags.Error()) + return *root } func runBuildStateFromTF( diff --git a/bundle/migrate/resolve.go b/bundle/migrate/resolve.go index 23d0180e3c6..41e09176e47 100644 --- a/bundle/migrate/resolve.go +++ b/bundle/migrate/resolve.go @@ -5,16 +5,15 @@ import ( "fmt" "strings" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/dynvar" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) // evaluateTemplate evaluates a template string like "${resources.pipelines.bar.cluster[0].label}" // by looking up each ${...} reference from TF state. func evaluateTemplate(state TFStateAttrs, template string) (string, error) { - ref, ok := dynvar.NewRef(dyn.V(template)) + ref, ok := structvar.NewRef(template) if !ok { return template, nil } diff --git a/bundle/mutator.go b/bundle/mutator.go index 41ef8fda48c..ed9bd5e56e5 100644 --- a/bundle/mutator.go +++ b/bundle/mutator.go @@ -75,19 +75,6 @@ func ApplyContext(ctx context.Context, b *Bundle, m Mutator) { log.Debugf(ctx, "Apply") - err := b.Config.MarkMutatorEntry(ctx) - if err != nil { - logdiag.LogError(ctx, fmt.Errorf("entry error: %w", err)) - return - } - - defer func() { - err := b.Config.MarkMutatorExit(ctx) - if err != nil { - logdiag.LogError(ctx, fmt.Errorf("exit error: %w", err)) - } - }() - diags := m.Apply(ctx, b) for _, d := range diags { @@ -104,34 +91,6 @@ func ApplySeqContext(ctx context.Context, b *Bundle, mutators ...Mutator) { } } -// ApplySeqInScopeContext applies mutators without opening a mutator scope per mutator, -// reusing the caller's scope instead. -// -// [ApplyContext] converts the whole configuration tree between its typed and dynamic -// representations on entry and exit (see [config.Root.MarkMutatorEntry]). That cost is -// proportional to the size of the accumulated configuration, so applying N mutators this -// way is quadratic in N. For a bundle with thousands of included files that dominates -// load time, hence this variant. -// -// Only use it for mutators that modify the configuration through [config.Root.Mutate] -// (which keeps both representations in sync). A mutator that assigns to a typed field -// directly relies on the scope entry to carry that value into the dynamic tree, and -// would lose it here. -func ApplySeqInScopeContext(ctx context.Context, b *Bundle, mutators ...Mutator) { - for _, m := range mutators { - mctx := log.NewContext(ctx, log.GetLogger(ctx).With("mutator", m.Name())) - log.Debugf(mctx, "Apply") - - for _, d := range m.Apply(mctx, b) { - logdiag.LogDiag(mctx, d) - } - - if logdiag.HasError(ctx) { - break - } - } -} - type funcMutator struct { fn func(context.Context, *Bundle) } diff --git a/bundle/mutator_test.go b/bundle/mutator_test.go index 45473634119..c8bddb24eed 100644 --- a/bundle/mutator_test.go +++ b/bundle/mutator_test.go @@ -5,10 +5,7 @@ import ( "testing" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/logdiag" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" ) type testMutator struct { @@ -52,79 +49,6 @@ func TestMutator(t *testing.T) { assert.Equal(t, 1, nested[1].applyCalled) } -// ApplySeqInScopeContext applies each mutator once, in order. -func TestApplySeqInScopeContext(t *testing.T) { - var order []string - makeMutator := func(name string) *testMutator { - return &testMutator{fn: func(ctx context.Context, b *Bundle) diag.Diagnostics { - order = append(order, name) - return nil - }} - } - first := makeMutator("first") - second := makeMutator("second") - - b := &Bundle{} - ctx := logdiag.InitContext(t.Context()) - logdiag.SetCollect(ctx, true) - ApplySeqInScopeContext(ctx, b, first, second) - - assert.Equal(t, 1, first.applyCalled) - assert.Equal(t, 1, second.applyCalled) - assert.Equal(t, []string{"first", "second"}, order) - assert.Empty(t, logdiag.FlushCollected(ctx)) -} - -// ApplySeqInScopeContext collects the diagnostics returned by each mutator and -// stops at the first one that logs an error, without applying later mutators. -func TestApplySeqInScopeContextStopsOnError(t *testing.T) { - failing := &testMutator{fn: func(ctx context.Context, b *Bundle) diag.Diagnostics { - return diag.Diagnostics{diag.Diagnostic{Severity: diag.Error, Summary: "boom"}} - }} - later := &testMutator{} - - b := &Bundle{} - ctx := logdiag.InitContext(t.Context()) - logdiag.SetCollect(ctx, true) - ApplySeqInScopeContext(ctx, b, failing, later) - - assert.Equal(t, 1, failing.applyCalled) - assert.Equal(t, 0, later.applyCalled, "mutator after an error must not run") - - diags := logdiag.FlushCollected(ctx) - require.Len(t, diags, 1) - assert.Equal(t, "boom", diags[0].Summary) -} - -// Unlike ApplySeqContext, ApplySeqInScopeContext does not open a mutator scope per -// mutator, so it must be called from within one. Changes made through Root.Mutate -// keep the typed and dynamic configuration in sync and survive without a per-mutator -// scope. This is the property ProcessRootIncludes relies on. -func TestApplySeqInScopeContextPreservesMutateChanges(t *testing.T) { - setHost := &testMutator{fn: func(ctx context.Context, b *Bundle) diag.Diagnostics { - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { - return dyn.Set(v, "bundle", dyn.V(map[string]dyn.Value{"name": dyn.V("set-in-scope")})) - }) - require.NoError(t, err) - return nil - }} - - // Enclosing mutator that applies setHost in its own scope, mirroring how - // ProcessRootIncludes applies the per-file includes. - outer := &testMutator{fn: func(ctx context.Context, b *Bundle) diag.Diagnostics { - ApplySeqInScopeContext(ctx, b, setHost) - return nil - }} - - b := &Bundle{} - diags := Apply(t.Context(), b, outer) - require.NoError(t, diags.Error()) - - // Visible in both representations once the enclosing scope exits. - assert.Equal(t, "set-in-scope", b.Config.Bundle.Name) - assert.Equal(t, "set-in-scope", b.Config.Value().Get("bundle").Get("name").MustString()) -} - func TestSafeMutatorName(t *testing.T) { tests := []struct { name string @@ -137,9 +61,9 @@ func TestSafeMutatorName(t *testing.T) { expected: "bundle.(funcMutator)", }, { - name: "setDefault mutator", - mutator: SetDefaultMutator(dyn.NewPattern(dyn.Key("test")), "key", "value"), - expected: "bundle.(setDefault)", + name: "setDefaults mutator", + mutator: &setDefaults{}, + expected: "bundle.(setDefaults)", }, { name: "funcMutator as pointer", diff --git a/bundle/phases/plan.go b/bundle/phases/plan.go index 4be81a102d2..25516d2777d 100644 --- a/bundle/phases/plan.go +++ b/bundle/phases/plan.go @@ -13,7 +13,7 @@ import ( "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/bundle/direct/dresources" "github.com/databricks/cli/bundle/statemgmt" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" ) // PreDeployChecks is common set of mutators between "bundle plan" and "bundle deploy". @@ -52,22 +52,22 @@ func pipelineDeletionCascades(b *bundle.Bundle, action deployplan.Action) (bool, // checkForPreventDestroy checks if the resource has lifecycle.prevent_destroy set, but the plan calls for this resource to be recreated or destroyed. // If it does, it returns an error. func checkForPreventDestroy(b *bundle.Bundle, actions []deployplan.Action) error { - root := b.Config.Value() + root := b.Config.View() var errs []error for _, action := range actions { if action.ActionType != deployplan.Recreate && action.ActionType != deployplan.Delete { continue } - path, err := dyn.NewPathFromString(action.ResourceKey) + path, err := structpath.ParsePath(action.ResourceKey) if err != nil { return fmt.Errorf("failed to parse %q", action.ResourceKey) } - path = append(path, dyn.Key("lifecycle"), dyn.Key("prevent_destroy")) + path = structpath.NewStringKeys(path, "lifecycle", "prevent_destroy") - preventDestroyV, err := dyn.GetByPath(root, path) - if err != nil { + preventDestroyV := root.Lookup(path) + if !preventDestroyV.IsValid() { continue } diff --git a/bundle/phases/plan_test.go b/bundle/phases/plan_test.go index 772f7401f9b..6063cc6a0bd 100644 --- a/bundle/phases/plan_test.go +++ b/bundle/phases/plan_test.go @@ -7,9 +7,10 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/bundle/direct/dstate" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -77,20 +78,14 @@ func TestPipelineDeletionCascades(t *testing.T) { func TestCheckPreventDestroyForAllResources(t *testing.T) { for resourceType := range config.SupportedResources() { + // Snapshots are internal and their lifecycle is not part of the configuration (json:"-"). + if resourceType == "internal_immutable_snapshots" { + continue + } t.Run(resourceType, func(t *testing.T) { b := &bundle.Bundle{} - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { - return dyn.Set(v, "resources", dyn.NewValue(map[string]dyn.Value{ - resourceType: dyn.NewValue(map[string]dyn.Value{ - "test_resource": dyn.NewValue(map[string]dyn.Value{ - "lifecycle": dyn.NewValue(map[string]dyn.Value{ - "prevent_destroy": dyn.NewValue(true, nil), - }, nil), - }, nil), - }, nil), - }, nil)) - }) + err := b.Config.Set(structpath.MustParsePath("resources."+resourceType+".test_resource.lifecycle.prevent_destroy"), true) require.NoError(t, err) actions := []deployplan.Action{ @@ -111,17 +106,7 @@ func TestCheckPreventDestroyForAllResources(t *testing.T) { func TestCheckPreventDestroyForJob(t *testing.T) { b := &bundle.Bundle{} - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { - return dyn.Set(v, "resources", dyn.NewValue(map[string]dyn.Value{ - "jobs": dyn.NewValue(map[string]dyn.Value{ - "test_resource": dyn.NewValue(map[string]dyn.Value{ - "lifecycle": dyn.NewValue(map[string]dyn.Value{ - "prevent_destroy": dyn.NewValue(true, nil), - }, nil), - }, nil), - }, nil), - }, nil)) - }) + err := b.Config.Set(structpath.MustParsePath("resources.jobs.test_resource.lifecycle.prevent_destroy"), true) require.NoError(t, err) actions := []deployplan.Action{ @@ -140,17 +125,7 @@ func TestCheckPreventDestroyForJob(t *testing.T) { func TestCheckPreventDestroyForApp(t *testing.T) { b := &bundle.Bundle{} - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { - return dyn.Set(v, "resources", dyn.NewValue(map[string]dyn.Value{ - "apps": dyn.NewValue(map[string]dyn.Value{ - "test_resource": dyn.NewValue(map[string]dyn.Value{ - "lifecycle": dyn.NewValue(map[string]dyn.Value{ - "prevent_destroy": dyn.NewValue(true, nil), - }, nil), - }, nil), - }, nil), - }, nil)) - }) + err := b.Config.Set(structpath.MustParsePath("resources.apps.test_resource.lifecycle.prevent_destroy"), true) require.NoError(t, err) actions := []deployplan.Action{ @@ -167,14 +142,7 @@ func TestCheckPreventDestroyForApp(t *testing.T) { func TestCheckPreventDestroyNoError(t *testing.T) { b := &bundle.Bundle{} - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { - return dyn.Set(v, "resources", dyn.NewValue(map[string]dyn.Value{ - "jobs": dyn.NewValue(map[string]dyn.Value{ - "test_resource": dyn.NewValue(map[string]dyn.Value{}, nil), - }, nil), - }, nil)) - }) - require.NoError(t, err) + b.Config.Resources.Jobs = map[string]*resources.Job{"test_resource": {}} actions := []deployplan.Action{ { @@ -183,26 +151,14 @@ func TestCheckPreventDestroyNoError(t *testing.T) { }, } - err = checkForPreventDestroy(b, actions) + err := checkForPreventDestroy(b, actions) require.NoError(t, err) } func TestCheckForPreventDestroyWhenFirstHasNoPreventDestroy(t *testing.T) { b := &bundle.Bundle{} - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { - return dyn.Set(v, "resources", dyn.NewValue(map[string]dyn.Value{ - "jobs": dyn.NewValue(map[string]dyn.Value{ - "test_job": dyn.NewValue(map[string]dyn.Value{}, nil), - }, nil), - "apps": dyn.NewValue(map[string]dyn.Value{ - "test_app": dyn.NewValue(map[string]dyn.Value{ - "lifecycle": dyn.NewValue(map[string]dyn.Value{ - "prevent_destroy": dyn.NewValue(true, nil), - }, nil), - }, nil), - }, nil), - }, nil)) - }) + b.Config.Resources.Jobs = map[string]*resources.Job{"test_job": {}} + err := b.Config.Set(structpath.MustParsePath("resources.apps.test_app.lifecycle.prevent_destroy"), true) require.NoError(t, err) actions := []deployplan.Action{ diff --git a/bundle/phases/telemetry.go b/bundle/phases/telemetry.go index 99464f3ef24..159afeb9560 100644 --- a/bundle/phases/telemetry.go +++ b/bundle/phases/telemetry.go @@ -14,8 +14,9 @@ import ( "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/libraries" "github.com/databricks/cli/bundle/metrics" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" "github.com/databricks/cli/libs/telemetry" "github.com/databricks/cli/libs/telemetry/protos" "github.com/databricks/databricks-sdk-go/service/jobs" @@ -201,9 +202,9 @@ func LogDeployTelemetry(ctx context.Context, b *bundle.Bundle, errMsg string) { b.Metrics.SetBoolValue(metrics.DeploymentHistoryEnabled, b.DeploymentBundle.StateDB.IsDeploymentMetadataService()) resourcesCount := int64(0) - _, err := dyn.MapByPattern(b.Config.Value(), dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey()), func(p dyn.Path, v dyn.Value) (dyn.Value, error) { + err := structvar.ForEach(b.Config.View(), structpath.MustParsePattern("resources.*.*"), func(p *structpath.PathNode, v structvar.View) error { resourcesCount++ - return v, nil + return nil }) if err != nil { log.Debugf(ctx, "failed to count resources: %s", err) diff --git a/bundle/set_default.go b/bundle/set_default.go index 3100f67d57c..5c4543eb1f6 100644 --- a/bundle/set_default.go +++ b/bundle/set_default.go @@ -5,60 +5,66 @@ import ( "fmt" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/logdiag" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) -type setDefault struct { - pattern dyn.Pattern - key dyn.Path +// Default is a value to set at every path matching Pattern where it is not set yet. +type Default struct { + Pattern string + Value any +} + +type defaultValue struct { + pattern *structpath.PatternNode + key string value any } -func SetDefaultMutator(pattern dyn.Pattern, key string, value any) Mutator { - return &setDefault{ - pattern: pattern, - key: dyn.NewPath(dyn.Key(key)), - value: value, - } +type setDefaults struct { + defaults []defaultValue } -func (m *setDefault) Name() string { - return fmt.Sprintf("SetDefaultMutator(%v, %v, %v)", m.pattern, m.key, m.value) +func (m *setDefaults) Name() string { + return "SetDefaults" } -func (m *setDefault) Apply(ctx context.Context, b *Bundle) diag.Diagnostics { - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { - return dyn.MapByPattern(v, m.pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - _, err := dyn.GetByPath(v, m.key) - switch { - case dyn.IsNoSuchKeyError(err): - return dyn.SetByPath(v, m.key, dyn.V(m.value)) - default: - return v, err +func (m *setDefaults) Apply(ctx context.Context, b *Bundle) diag.Diagnostics { + for _, d := range m.defaults { + err := structvar.ForEach(b.Config.View(), d.pattern, func(p *structpath.PathNode, v structvar.View) error { + if v.Get(d.key).IsValid() { + return nil } + return b.Config.Set(structpath.NewStringKey(p, d.key), d.value) }) - }) - if err != nil { - return diag.FromErr(err) + if err != nil { + return diag.FromErr(err) + } } return nil } -func SetDefault(ctx context.Context, b *Bundle, pattern string, value any) { - pat, err := dyn.NewPatternFromString(pattern) - if err != nil { - logdiag.LogError(ctx, fmt.Errorf("internal error: invalid pattern: %s: %w", pattern, err)) - return - } +// SetDefaults sets each default value at every path matching its pattern where no value is set. +// Defaults are applied in order. +func SetDefaults(ctx context.Context, b *Bundle, defaults []Default) { + m := &setDefaults{} + for _, d := range defaults { + pat, err := structpath.ParsePattern(d.Pattern) + if err != nil { + logdiag.LogError(ctx, fmt.Errorf("internal error: invalid pattern: %s: %w", d.Pattern, err)) + return + } + + key, ok := pat.StringKey() + if !ok || key == "" { + logdiag.LogError(ctx, fmt.Errorf("internal error: invalid pattern: %s", d.Pattern)) + return + } - pat, key := pat.SplitKey() - if pat == nil || key == "" { - logdiag.LogError(ctx, fmt.Errorf("internal error: invalid pattern: %s", pattern)) - return + m.defaults = append(m.defaults, defaultValue{pattern: pat.Parent(), key: key, value: d.Value}) } - m := SetDefaultMutator(pat, key, value) ApplyContext(ctx, b, m) } diff --git a/bundle/statemgmt/direct_migration.go b/bundle/statemgmt/direct_migration.go index f3d632f138f..7ad17b780ab 100644 --- a/bundle/statemgmt/direct_migration.go +++ b/bundle/statemgmt/direct_migration.go @@ -21,7 +21,6 @@ import ( "github.com/databricks/cli/bundle/metrics" "github.com/databricks/cli/bundle/migrate" "github.com/databricks/cli/libs/cmdio" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/env" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/logdiag" @@ -269,19 +268,11 @@ func convertTFStateToDirect(ctx context.Context, b *bundle.Bundle, tfState *migr // The config may use terraform-style references (${databricks_pipeline.x.id}). // BuildStateFromTF expects ${resources.*} references, so rewrite them first. - uninterpolatedRoot, err := reverseInterpolate(b.Config.Value()) + uninterpolatedConfig, err := reverseInterpolateConfig(&b.Config) if err != nil { return tempStatePath, false, nil, fmt.Errorf("failed to reverse interpolation: %w", err) } - var uninterpolatedConfig config.Root - err = uninterpolatedConfig.Mutate(func(_ dyn.Value) (dyn.Value, error) { - return uninterpolatedRoot, nil - }) - if err != nil { - return tempStatePath, false, nil, fmt.Errorf("failed to create uninterpolated config: %w", err) - } - adapters, err := dresources.InitAll(nil) if err != nil { return tempStatePath, false, nil, err @@ -296,7 +287,7 @@ func convertTFStateToDirect(ctx context.Context, b *bundle.Bundle, tfState *migr } // warnPrefix labels the conversion's warnings as coming from the background dry run. - hasWarnings, err := migrate.BuildStateFromTF(ctx, &uninterpolatedConfig, adapters, &stateDB, tfState.Attrs, tfState.IDs, warnPrefix) + hasWarnings, err := migrate.BuildStateFromTF(ctx, uninterpolatedConfig, adapters, &stateDB, tfState.Attrs, tfState.IDs, warnPrefix) if err != nil { return tempStatePath, hasWarnings, nil, err } @@ -310,5 +301,5 @@ func convertTFStateToDirect(ctx context.Context, b *bundle.Bundle, tfState *migr return tempStatePath, hasWarnings, nil, errors.New("state conversion failed") } - return tempStatePath, hasWarnings, &uninterpolatedConfig, nil + return tempStatePath, hasWarnings, uninterpolatedConfig, nil } diff --git a/bundle/statemgmt/reverse_interpolate.go b/bundle/statemgmt/reverse_interpolate.go index d133c0e0f7d..b498f85eafb 100644 --- a/bundle/statemgmt/reverse_interpolate.go +++ b/bundle/statemgmt/reverse_interpolate.go @@ -3,34 +3,85 @@ package statemgmt import ( "fmt" + "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/deploy/terraform" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/dynvar" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) -// reverseInterpolate converts terraform-style resource references to bundle-style. +// reverseInterpolate converts terraform-style resource references in s to bundle-style. // Example: ${databricks_pipeline.my_etl.id} → ${resources.pipelines.my_etl.id} -func reverseInterpolate(root dyn.Value) (dyn.Value, error) { - return dynvar.Resolve(root, func(path dyn.Path) (dyn.Value, error) { +func reverseInterpolate(s string) (string, error) { + out, err := structvar.Resolve(map[string]structvar.Template{"": {Value: s}}, func(sp *structpath.PathNode) (structvar.View, error) { // Need at least 2 components: resource_type.resource_name - if len(path) < 2 { - return dyn.InvalidValue, dynvar.ErrSkipResolution + if sp.Len() < 2 { + return structvar.View{}, structvar.ErrSkipResolution } - resourceType := path[0].Key() + resourceType := sp.KeyAt(0) isAlreadyBundleFormat := resourceType == "resources" if isAlreadyBundleFormat { - return dyn.InvalidValue, dynvar.ErrSkipResolution + return structvar.View{}, structvar.ErrSkipResolution } bundleGroup, ok := terraform.TerraformToGroupName[resourceType] if !ok { - return dyn.InvalidValue, dynvar.ErrSkipResolution + return structvar.View{}, structvar.ErrSkipResolution } // Reconstruct path in bundle format: // databricks_pipeline.my_pipeline.id → resources.pipelines.my_pipeline.id - bundlePath := dyn.NewPath(dyn.Key("resources"), dyn.Key(bundleGroup)).Append(path[1:]...) - return dyn.V(fmt.Sprintf("${%s}", bundlePath.String())), nil + ref := fmt.Sprintf("${resources.%s.%s}", bundleGroup, sp.SkipPrefix(1)) + return structvar.NewView(&ref, nil, nil), nil }) + if err != nil { + return "", err + } + if v, ok := out[""]; ok { + rs, _ := v.AsString() + return rs, nil + } + return s, nil +} + +// reverseInterpolateConfig returns a copy of cfg in which terraform-style references +// are rewritten to bundle-style ones (see [reverseInterpolate]). cfg is not changed. +func reverseInterpolateConfig(cfg *config.Root) (*config.Root, error) { + out := &config.Root{} + if err := out.Assign(nil, cfg.View()); err != nil { + return nil, err + } + + type update struct { + path *structpath.PathNode + s string + } + var updates []update + err := structvar.Walk(cfg.View(), func(p *structpath.PathNode, v structvar.View) error { + s, ok := v.AsString() + if !ok { + return nil + } + if _, ok := structvar.NewRef(s); !ok { + return nil + } + rs, err := reverseInterpolate(s) + if err != nil { + return err + } + if rs != s { + updates = append(updates, update{p, rs}) + } + return nil + }) + if err != nil { + return nil, err + } + + for _, u := range updates { + if err := out.SetReference(u.path, u.s); err != nil { + return nil, err + } + } + return out, nil } diff --git a/bundle/statemgmt/reverse_interpolate_test.go b/bundle/statemgmt/reverse_interpolate_test.go index fe1779cd65b..ef6a40ee5da 100644 --- a/bundle/statemgmt/reverse_interpolate_test.go +++ b/bundle/statemgmt/reverse_interpolate_test.go @@ -6,129 +6,86 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestReverseInterpolatePreservesBConfigValue(t *testing.T) { - // This test verifies that our approach of getting b.Config.Value(), - // reverse interpolating it, and wrapping in a new config.Root - // does NOT mutate b.Config + // This test verifies that reverse interpolating a config returns a new + // config.Root and does NOT mutate the original. - b := &bundle.Bundle{ - Config: config.Root{ - Bundle: config.Bundle{ - Name: "test", - }, - }, - } + root, diags := config.LoadFromBytes("test.yml", []byte(` +bundle: + name: test +resources: + jobs: + my_job: + name: My Job + description: ${databricks_pipeline.my_pipeline.id} + max_concurrent_runs: ${databricks_job.other.id} +`)) + require.NoError(t, diags.Error()) + b := &bundle.Bundle{Config: *root} - err := b.Config.Mutate(func(_ dyn.Value) (dyn.Value, error) { - return dyn.V(map[string]dyn.Value{ - "bundle": dyn.V(map[string]dyn.Value{ - "name": dyn.V("test"), - }), - "resources": dyn.V(map[string]dyn.Value{ - "jobs": dyn.V(map[string]dyn.Value{ - "my_job": dyn.V(map[string]dyn.Value{ - "name": dyn.V("My Job"), - "depends_on": dyn.V("${databricks_pipeline.my_pipeline.id}"), - }), - }), - }), - }), nil - }) + originalJSON, err := json.Marshal(b.Config.View().AsAny()) require.NoError(t, err) - originalValue := b.Config.Value() - originalJSON, err := json.Marshal(originalValue.AsAny()) + uninterpolatedConfig, err := reverseInterpolateConfig(&b.Config) require.NoError(t, err) - interpolatedRoot := b.Config.Value() - - uninterpolatedRoot, err := reverseInterpolate(interpolatedRoot) - require.NoError(t, err) + uninterpolated := uninterpolatedConfig.View() + description, ok := uninterpolated.Lookup(structpath.MustParsePath("resources.jobs.my_job.description")).AsString() + require.True(t, ok) + assert.Equal(t, "${resources.pipelines.my_pipeline.id}", description, "should be bundle-style after reverse interpolation") - dependsOn, err := dyn.GetByPath(uninterpolatedRoot, dyn.MustPathFromString("resources.jobs.my_job.depends_on")) - require.NoError(t, err) - dependsOnStr, ok := dependsOn.AsString() + // References in fields that cannot hold a string are rewritten too. + maxRuns, ok := uninterpolated.Lookup(structpath.MustParsePath("resources.jobs.my_job.max_concurrent_runs")).AsString() require.True(t, ok) - assert.Equal(t, "${resources.pipelines.my_pipeline.id}", dependsOnStr, "should be bundle-style after reverse interpolation") + assert.Equal(t, "${resources.jobs.other.id}", maxRuns) - var uninterpolatedConfig config.Root - err = uninterpolatedConfig.Mutate(func(_ dyn.Value) (dyn.Value, error) { - return uninterpolatedRoot, nil - }) - require.NoError(t, err) + // Locations are kept. + assert.Equal(t, b.Config.GetLocations("resources.jobs.my_job.description"), uninterpolatedConfig.GetLocations("resources.jobs.my_job.description")) - afterValue := b.Config.Value() - afterJSON, err := json.Marshal(afterValue.AsAny()) + afterJSON, err := json.Marshal(b.Config.View().AsAny()) require.NoError(t, err) + assert.Equal(t, string(originalJSON), string(afterJSON), "b.Config should not change") - assert.Equal(t, string(originalJSON), string(afterJSON), "b.Config.Value() should not change") - - originalDependsOn, err := dyn.GetByPath(afterValue, dyn.MustPathFromString("resources.jobs.my_job.depends_on")) - require.NoError(t, err) - originalDependsOnStr, ok := originalDependsOn.AsString() + originalDescription, ok := b.Config.View().Lookup(structpath.MustParsePath("resources.jobs.my_job.description")).AsString() require.True(t, ok) - assert.Equal(t, "${databricks_pipeline.my_pipeline.id}", originalDependsOnStr, "terraform-style reference should be preserved in b.Config") + assert.Equal(t, "${databricks_pipeline.my_pipeline.id}", originalDescription, "terraform-style reference should be preserved in b.Config") } func TestReverseInterpolate(t *testing.T) { tests := []struct { name string - input dyn.Value - expected dyn.Value + input string + expected string }{ { - name: "converts terraform-style job reference to bundle-style", - input: dyn.V(map[string]dyn.Value{ - "job_id": dyn.V("${databricks_job.my_job.id}"), - }), - expected: dyn.V(map[string]dyn.Value{ - "job_id": dyn.V("${resources.jobs.my_job.id}"), - }), + name: "converts terraform-style job reference to bundle-style", + input: "${databricks_job.my_job.id}", + expected: "${resources.jobs.my_job.id}", }, { - name: "leaves bundle-style references unchanged", - input: dyn.V(map[string]dyn.Value{ - "pipeline_id": dyn.V("${resources.pipelines.my_pipeline.id}"), - }), - expected: dyn.V(map[string]dyn.Value{ - "pipeline_id": dyn.V("${resources.pipelines.my_pipeline.id}"), - }), + name: "leaves bundle-style references unchanged", + input: "${resources.pipelines.my_pipeline.id}", + expected: "${resources.pipelines.my_pipeline.id}", }, { - name: "handles nested paths", - input: dyn.V(map[string]dyn.Value{ - "config": dyn.V(map[string]dyn.Value{ - "source": dyn.V("${databricks_pipeline.my_pipeline.url}"), - }), - }), - expected: dyn.V(map[string]dyn.Value{ - "config": dyn.V(map[string]dyn.Value{ - "source": dyn.V("${resources.pipelines.my_pipeline.url}"), - }), - }), + name: "handles nested paths", + input: "${databricks_pipeline.my_pipeline.url}", + expected: "${resources.pipelines.my_pipeline.url}", }, { - name: "skips unknown terraform resource types", - input: dyn.V(map[string]dyn.Value{ - "unknown": dyn.V("${unknown_resource.my_resource.id}"), - }), - expected: dyn.V(map[string]dyn.Value{ - "unknown": dyn.V("${unknown_resource.my_resource.id}"), - }), + name: "skips unknown terraform resource types", + input: "${unknown_resource.my_resource.id}", + expected: "${unknown_resource.my_resource.id}", }, { - name: "handles multiple references in one value", - input: dyn.V(map[string]dyn.Value{ - "combined": dyn.V("${databricks_job.job1.id}/${databricks_pipeline.pipeline1.id}"), - }), - expected: dyn.V(map[string]dyn.Value{ - "combined": dyn.V("${resources.jobs.job1.id}/${resources.pipelines.pipeline1.id}"), - }), + name: "handles multiple references in one value", + input: "${databricks_job.job1.id}/${databricks_pipeline.pipeline1.id}", + expected: "${resources.jobs.job1.id}/${resources.pipelines.pipeline1.id}", }, } @@ -136,7 +93,7 @@ func TestReverseInterpolate(t *testing.T) { t.Run(tt.name, func(t *testing.T) { result, err := reverseInterpolate(tt.input) require.NoError(t, err) - assert.Equal(t, tt.expected.AsAny(), result.AsAny()) + assert.Equal(t, tt.expected, result) }) } } diff --git a/bundle/statemgmt/state_load.go b/bundle/statemgmt/state_load.go index abe260d4d4a..a3327cf6688 100644 --- a/bundle/statemgmt/state_load.go +++ b/bundle/statemgmt/state_load.go @@ -3,7 +3,6 @@ package statemgmt import ( "context" "errors" - "fmt" "slices" "strings" @@ -12,7 +11,8 @@ import ( "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/statemgmt/resourcestate" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) type ( @@ -93,80 +93,64 @@ func applyState(ctx context.Context, b *bundle.Bundle, state ExportedResourcesMa return nil } -func ensureMap(v dyn.Value, path dyn.Path) (dyn.Value, error) { - item, _ := dyn.GetByPath(v, path) - if !item.IsValid() { - var err error - v, err = dyn.SetByPath(v, path, dyn.V(dyn.NewMapping())) - if err != nil { - return dyn.InvalidValue, fmt.Errorf("internal error: failed to create %s: %s", path, err) +func StateToBundle(ctx context.Context, state ExportedResourcesMap, cfg *config.Root) error { + resourcesPath := structpath.NewStringKey(nil, "resources") + if !cfg.View().Lookup(resourcesPath).IsValid() { + if err := cfg.Set(resourcesPath, config.Resources{}); err != nil { + return err } } - return v, nil -} -func StateToBundle(ctx context.Context, state ExportedResourcesMap, config *config.Root) error { - return config.Mutate(func(v dyn.Value) (dyn.Value, error) { - var err error - v, err = ensureMap(v, dyn.Path{dyn.Key("resources")}) - if err != nil { - return v, err + for resourceKey, attrs := range state { + // Parse resource key like "resources.jobs.foo" or "resources.jobs.foo.permissions" + parts := strings.Split(resourceKey, ".") + if len(parts) < 3 || parts[0] != "resources" { + continue // Skip invalid resource keys } - for resourceKey, attrs := range state { - // Parse resource key like "resources.jobs.foo" or "resources.jobs.foo.permissions" - parts := strings.Split(resourceKey, ".") - if len(parts) < 3 || parts[0] != "resources" { - continue // Skip invalid resource keys - } + groupName := parts[1] + resourceName := parts[2] - groupName := parts[1] - resourceName := parts[2] - - // Skip permissions for now as they are sub-resources - if len(parts) > 3 { - continue - } + // Skip permissions for now as they are sub-resources + if len(parts) > 3 { + continue + } - var err error - v, err = ensureMap(v, dyn.Path{dyn.Key("resources"), dyn.Key(groupName)}) - if err != nil { - return v, err - } + if !hasID(groupName) { + continue + } - path := dyn.Path{dyn.Key("resources"), dyn.Key(groupName), dyn.Key(resourceName)} - resource, err := dyn.GetByPath(v, path) - if !resource.IsValid() { - m := dyn.NewMapping() - m.SetLoc("id", nil, dyn.V(attrs.ID)) - m.SetLoc("modified_status", nil, dyn.V(resources.ModifiedStatusDeleted)) - v, err = dyn.SetByPath(v, path, dyn.V(m)) - if err != nil { - return dyn.InvalidValue, err - } - } else if err != nil { - return dyn.InvalidValue, err - } else { - v, err = dyn.SetByPath(v, dyn.Path{dyn.Key("resources"), dyn.Key(groupName), dyn.Key(resourceName), dyn.Key("id")}, dyn.V(attrs.ID)) - if err != nil { - return dyn.InvalidValue, err - } + path := structpath.NewStringKeys(resourcesPath, groupName, resourceName) + if !cfg.View().Lookup(path).IsValid() { + if err := cfg.Set(structpath.NewStringKey(path, "modified_status"), resources.ModifiedStatusDeleted); err != nil { + return err } } + if err := cfg.Set(structpath.NewStringKey(path, "id"), attrs.ID); err != nil { + return err + } + } - return dyn.MapByPattern(v, dyn.Pattern{dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey()}, func(p dyn.Path, inner dyn.Value) (dyn.Value, error) { - idPath := dyn.Path{dyn.Key("id")} - statusPath := dyn.Path{dyn.Key("modified_status")} - id, _ := dyn.GetByPath(inner, idPath) - status, _ := dyn.GetByPath(inner, statusPath) - if !id.IsValid() && !status.IsValid() { - return dyn.SetByPath(inner, statusPath, dyn.V(resources.ModifiedStatusCreated)) - } - return inner, nil - }) + return structvar.ForEach(cfg.View(), structpath.MustParsePattern("resources.*.*"), func(p *structpath.PathNode, inner structvar.View) error { + group, _ := p.Parent().StringKey() + if !hasID(group) || inner.Get("id").IsValid() || inner.Get("modified_status").IsValid() { + return nil + } + return cfg.Set(structpath.NewStringKey(p, "modified_status"), resources.ModifiedStatusCreated) }) } +// hasID reports whether resources of group can hold the deployed id and status. Groups +// that are not part of the configuration, and internal snapshots, cannot. +func hasID(group string) bool { + typ, ok := config.ResourcesTypes[group] + if !ok { + return false + } + _, ok = typ.FieldByName("ID") + return ok +} + func validateLoadedState(state ExportedResourcesMap, modes []LoadMode) error { if len(state) == 0 && slices.Contains(modes, ErrorOnEmptyState) { return errors.New("resource not found or not yet deployed. Did you forget to run 'databricks bundle deploy'?") diff --git a/bundle/trampoline/python_dbr_warning.go b/bundle/trampoline/python_dbr_warning.go index 4cc7a67dc8c..bfcd885d1b8 100644 --- a/bundle/trampoline/python_dbr_warning.go +++ b/bundle/trampoline/python_dbr_warning.go @@ -10,8 +10,8 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/libraries" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn/dynvar" "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/structs/structvar" "github.com/databricks/databricks-sdk-go" "golang.org/x/mod/semver" ) @@ -82,18 +82,18 @@ func hasIncompatibleWheelTasks(ctx context.Context, b *bundle.Bundle) diag.Diagn // So we can get the version from the cluster definition. // It's defined in a form of resources.clusters..id if strings.HasPrefix(task.ExistingClusterId, "${") { - p, ok := dynvar.PureReferenceToPath(task.ExistingClusterId) - if !ok || len(p) < 3 { + p, ok := structvar.PureReferenceToPath(task.ExistingClusterId) + if !ok || p.Len() < 3 { log.Warnf(ctx, "unable to parse cluster key from %s", task.ExistingClusterId) continue } - if p[0].Key() != "resources" || p[1].Key() != "clusters" { + if p.KeyAt(0) != "resources" || p.KeyAt(1) != "clusters" { log.Warnf(ctx, "incorrect variable reference for cluster id %s", task.ExistingClusterId) continue } - clusterKey := p[2].Key() + clusterKey := p.KeyAt(2) cluster, ok := b.Config.Resources.Clusters[clusterKey] if !ok { log.Warnf(ctx, "unable to find cluster with key %s", clusterKey) diff --git a/cmd/air/convert_to_dabs.go b/cmd/air/convert_to_dabs.go index 02b29a4d02f..910835a036a 100644 --- a/cmd/air/convert_to_dabs.go +++ b/cmd/air/convert_to_dabs.go @@ -13,9 +13,7 @@ import ( "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/cmdio" - "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/yamlsaver" + "github.com/databricks/cli/libs/structs/structyaml" "github.com/spf13/cobra" ) @@ -112,10 +110,10 @@ does not contact the workspace.`, // convertToDabs builds the DABs bundle value and the loose launch artifacts for a // run config. It reads only what the run path's buildArtifacts reads, so the // mapping is unit-testable in isolation. Returns the bundle root as a -// map[string]dyn.Value (ready for yamlsaver) and the loose artifacts (command.sh + +// structyaml.Map (ready for structyaml.Save) and the loose artifacts (command.sh + // env/secret/param sidecars) to write under generated_artifacts/. It does not touch the // code_source; the emitted `tgz` artifact packages it at deploy. -func convertToDabs(ctx context.Context, cfg *runConfig, configPath, bundleDir string) (map[string]dyn.Value, []uploadItem, error) { +func convertToDabs(ctx context.Context, cfg *runConfig, configPath, bundleDir string) (structyaml.Map, []uploadItem, error) { // idempotency_token is intentionally not mapped: it dedups a single runs/submit // call, which has no analogue for a persistent, repeatedly-runnable bundle job. // @@ -259,14 +257,6 @@ func bundleCodeSourcePath(ctx context.Context, cfg *runConfig, configPath, bundl return localBundlePath(filepath.ToSlash(rel)), nil } -// nv builds a dyn.Value at "position" n: yamlsaver orders a map's keys by their -// Location line, so assigning ascending n values fixes the emitted key order. -// It routes through dyn.V so nested Go maps/slices are converted recursively, -// then stamps the ordering location. -func nv(v any, n int) dyn.Value { - return dyn.V(v).WithLocations([]diag.Location{{Line: n}}) -} - // localBundlePath renders a bundle-relative path with a leading "./" so bundle // deploy classifies it as a local artifact to upload (see IsLibraryLocal). It is // built from path.Join (forward slashes) so the emitted YAML is identical across @@ -275,10 +265,10 @@ func localBundlePath(p string) string { return "./" + p } -// buildBundleValue assembles the bundle root as an ordered map[string]dyn.Value. +// buildBundleValue assembles the bundle root as an ordered structyaml.Map. // codeSourcePath is the "./"-prefixed code_source dir relative to the bundle (empty // when the config has no code_source); command.sh is a bundle-local artifact. -func buildBundleValue(ctx context.Context, cfg *runConfig, configPath, codeSourcePath string, art *codeArtifact) map[string]dyn.Value { +func buildBundleValue(ctx context.Context, cfg *runConfig, configPath, codeSourcePath string, art *codeArtifact) structyaml.Map { name := cfg.ExperimentName // ai_runtime_task: experiment + one deployment (command_path + compute) + @@ -288,67 +278,57 @@ func buildBundleValue(ctx context.Context, cfg *runConfig, configPath, codeSourc // command_path is "./"-prefixed so bundle deploy treats it as LOCAL and uploads // it: libraries.IsLibraryLocal classifies a bare, extensionless path as a PyPI // package name, which would deploy a path the backend can't resolve. - compute := map[string]dyn.Value{ - "accelerator_type": nv(cfg.Compute.AcceleratorType, 1), - "accelerator_count": nv(cfg.Compute.NumAccelerators, 2), - } + compute := structyaml.M( + "accelerator_type", cfg.Compute.AcceleratorType, + "accelerator_count", cfg.Compute.NumAccelerators, + ) if cfg.Compute.PoolID != nil { - compute["provisioned_capacity_id"] = nv(*cfg.Compute.PoolID, 3) - } - deployment := map[string]dyn.Value{ - "command_path": nv(localBundlePath(path.Join(generatedArtifactsDir, commandScriptName)), 1), - "compute": nv(compute, 2), + compute.Add("provisioned_capacity_id", *cfg.Compute.PoolID) } + deployment := structyaml.M( + "command_path", localBundlePath(path.Join(generatedArtifactsDir, commandScriptName)), + "compute", compute, + ) - aiRuntimeTask := map[string]dyn.Value{ - "experiment": nv(name, 1), - "deployments": nv([]dyn.Value{dyn.V(deployment)}, 2), - } - line := 3 + aiRuntimeTask := structyaml.M( + "experiment", name, + "deployments", []any{deployment}, + ) if codeSourcePath != "" { // Points at the `tgz` artifact's built tarball; deploy uploads it and rewrites // this to the uploaded workspace path. - aiRuntimeTask["code_source_path"] = nv(codeSourcePath, line) - line++ + aiRuntimeTask.Add("code_source_path", codeSourcePath) } if cfg.MLflowRunName != nil { - aiRuntimeTask["mlflow_run"] = nv(*cfg.MLflowRunName, line) - line++ + aiRuntimeTask.Add("mlflow_run", *cfg.MLflowRunName) } if cfg.MLflowExperimentDirectory != nil { - aiRuntimeTask["mlflow_experiment_directory"] = nv(*cfg.MLflowExperimentDirectory, line) - line++ + aiRuntimeTask.Add("mlflow_experiment_directory", *cfg.MLflowExperimentDirectory) } if cfg.MLflowArtifactLocation != nil { - aiRuntimeTask["mlflow_artifact_location"] = nv(*cfg.MLflowArtifactLocation, line) - line++ + aiRuntimeTask.Add("mlflow_artifact_location", *cfg.MLflowArtifactLocation) } if cfg.Compute.PriorityClass != nil { - aiRuntimeTask["priority_class"] = nv(*cfg.Compute.PriorityClass, line) - line++ + aiRuntimeTask.Add("priority_class", *cfg.Compute.PriorityClass) } if cfg.Environment != nil && cfg.Environment.UnityCatalogImage != "" { - aiRuntimeTask["unity_catalog_image_path"] = nv(cfg.Environment.UnityCatalogImage, line) + aiRuntimeTask.Add("unity_catalog_image_path", cfg.Environment.UnityCatalogImage) } // Task wrapper: task_key + framework fields (retries/timeout) + env key + // the ai_runtime_task. Framework fields live here per the schema, not inside // ai_runtime_task. - task := map[string]dyn.Value{ - "task_key": nv(name, 1), - "environment_key": nv(aiRuntimeEnvironmentKey, 2), - } - taskLine := 3 maxRetries := cfg.maxRetries() - task["max_retries"] = nv(maxRetries, taskLine) - taskLine++ - task["retry_on_timeout"] = nv(maxRetries > 0, taskLine) - taskLine++ + task := structyaml.M( + "task_key", name, + "environment_key", aiRuntimeEnvironmentKey, + "max_retries", maxRetries, + "retry_on_timeout", maxRetries > 0, + ) if cfg.TimeoutMinutes != nil { - task["timeout_seconds"] = nv(cfg.timeoutSeconds(), taskLine) - taskLine++ + task.Add("timeout_seconds", cfg.timeoutSeconds()) } - task["ai_runtime_task"] = nv(aiRuntimeTask, taskLine) + task.Add("ai_runtime_task", aiRuntimeTask) // environments[]: version + the dependency set. The runtime installs deps from this // spec directly, so the full dependency set (whether authored inline or in a @@ -358,65 +338,57 @@ func buildBundleValue(ctx context.Context, cfg *runConfig, configPath, codeSourc // default channel) so a config without an explicit version still pins the version // the workload would have run with — not an empty spec. envVersion, deps := bundleEnvironmentDeps(ctx, cfg) - envSpec := map[string]dyn.Value{} + var envSpec structyaml.Map if strings.HasPrefix(envVersion, databricksAIPrefix) { - envSpec["base_environment"] = nv("workspace-base-environments/"+envVersion, 1) + envSpec.Add("base_environment", "workspace-base-environments/"+envVersion) } else { - envSpec["environment_version"] = nv(envVersion, 1) + envSpec.Add("environment_version", envVersion) } if len(deps) > 0 { - depVals := make([]dyn.Value, len(deps)) - for i, d := range deps { - depVals[i] = dyn.V(d) - } - envSpec["dependencies"] = nv(depVals, 2) - } - environment := map[string]dyn.Value{ - "environment_key": nv(aiRuntimeEnvironmentKey, 1), - "spec": nv(envSpec, 2), + envSpec.Add("dependencies", deps) } + environment := structyaml.M( + "environment_key", aiRuntimeEnvironmentKey, + "spec", envSpec, + ) - job := map[string]dyn.Value{ - "name": nv(name, 1), - "tasks": nv([]dyn.Value{dyn.V(task)}, 2), - "environments": nv([]dyn.Value{dyn.V(environment)}, 3), - } + job := structyaml.M( + "name", name, + "tasks", []any{task}, + "environments", []any{environment}, + ) // usage_policy_id is an already-resolved budget policy id, so it maps directly // to the job's budget_policy_id. (usage_policy_name needs server-side resolution // and is rejected in convertToDabs.) if cfg.UsagePolicyID != nil { - job["budget_policy_id"] = nv(*cfg.UsagePolicyID, 4) + job.Add("budget_policy_id", *cfg.UsagePolicyID) } - if perms := buildPermissionsValue(cfg.Permissions); perms.Kind() != dyn.KindInvalid { - job["permissions"] = nv(perms.MustSequence(), 5) + if perms := buildPermissionsValue(cfg.Permissions); perms != nil { + job.Add("permissions", perms) } - rootValue := map[string]dyn.Value{ - "bundle": nv(map[string]dyn.Value{ - "name": nv(name, 1), - }, 1), + rootValue := structyaml.M( + "bundle", structyaml.M("name", name), // sync.paths replaces the default of syncing the whole bundle root. The code // directory is omitted deliberately: the `tgz` artifact packages it, so syncing // it too would upload the tree twice. - "sync": nv(map[string]dyn.Value{ - "paths": nv([]dyn.Value{nv(generatedArtifactsDir, 1)}, 1), - }, 2), - "targets": nv(map[string]dyn.Value{ - dabsTargetName: nv(map[string]dyn.Value{ - "mode": nv("development", 1), - "default": nv(true, 2), - }, 1), - }, 4), - "resources": nv(map[string]dyn.Value{ - "jobs": nv(map[string]dyn.Value{ - bundleResourceKey(name): nv(job, 1), - }, 1), - }, 5), - } + "sync", structyaml.M("paths", []any{generatedArtifactsDir}), + ) // The `tgz` artifact that packages the code_source (nil only when there is none). if art != nil { - rootValue["artifacts"] = nv(buildArtifactsValue(art), 3) - } + rootValue.Add("artifacts", buildArtifactsValue(art)) + } + rootValue.Add("targets", structyaml.M( + dabsTargetName, structyaml.M( + "mode", "development", + "default", true, + ), + )) + rootValue.Add("resources", structyaml.M( + "jobs", structyaml.M( + bundleResourceKey(name), job, + ), + )) return rootValue } @@ -424,34 +396,23 @@ func buildBundleValue(ctx context.Context, cfg *runConfig, configPath, codeSourc // single `tgz` artifact whose `path` is the code-source root, carrying the git ref // and/or include subpaths, and whose `files` output is the tarball code_source_path // points at. -func buildArtifactsValue(art *codeArtifact) map[string]dyn.Value { - a := map[string]dyn.Value{ - "type": nv("tgz", 1), - "path": nv(art.path, 2), - } - fileLine := 3 +func buildArtifactsValue(art *codeArtifact) structyaml.Map { + a := structyaml.M( + "type", "tgz", + "path", art.path, + ) if art.gitCommit != nil || art.gitBranch != nil { - g := map[string]dyn.Value{} // commit wins over branch, matching the artifact builder. if art.gitCommit != nil { - g["commit"] = nv(*art.gitCommit, 1) + a.Add("git", structyaml.M("commit", *art.gitCommit)) } else { - g["branch"] = nv(*art.gitBranch, 1) + a.Add("git", structyaml.M("branch", *art.gitBranch)) } - a["git"] = nv(g, fileLine) - fileLine++ } // include is always set: the basename (whole dir) or basename-prefixed subpaths. - vals := make([]dyn.Value, len(art.include)) - for i, p := range art.include { - vals[i] = dyn.V(p) - } - a["include"] = nv(vals, fileLine) - fileLine++ - a["files"] = nv([]dyn.Value{ - dyn.V(map[string]dyn.Value{"source": nv(art.tgzPath, 1)}), - }, fileLine) - return map[string]dyn.Value{codeSourceArtifactKey: nv(a, 1)} + a.Add("include", art.include) + a.Add("files", []any{structyaml.M("source", art.tgzPath)}) + return structyaml.M(codeSourceArtifactKey, a) } // bundleEnvironmentDeps resolves the runtime version and the inline dependency @@ -476,7 +437,7 @@ func bundleResourceKey(name string) string { } // quoteJobKey rewrites the emitted job resource key as a quoted YAML key when the -// name would otherwise load as a non-string scalar. yamlsaver emits map keys +// name would otherwise load as a non-string scalar. structyaml emits map keys // unquoted, and the bundle loader rejects a key that types as something other than // a string ("12345" -> !!int, "true" -> !!bool) with "invalid key tag". Only the // job key needs this: every other key convert emits is a fixed schema field name. @@ -516,25 +477,22 @@ func yamlKeyNeedsQuoting(key string) bool { } // buildPermissionsValue maps run-config permissions to DABs job permissions -// (level → principal). Returns an invalid value when there are none. -func buildPermissionsValue(perms []permission) dyn.Value { - if len(perms) == 0 { - return dyn.InvalidValue - } - out := make([]dyn.Value, 0, len(perms)) +// (level → principal). Returns nil when there are none. +func buildPermissionsValue(perms []permission) []any { + var out []any for _, p := range perms { - m := map[string]dyn.Value{"level": nv(p.Level, 1)} + m := structyaml.M("level", p.Level) switch { case p.UserName != nil: - m["user_name"] = nv(*p.UserName, 2) + m.Add("user_name", *p.UserName) case p.GroupName != nil: - m["group_name"] = nv(*p.GroupName, 2) + m.Add("group_name", *p.GroupName) case p.ServicePrincipalName != nil: - m["service_principal_name"] = nv(*p.ServicePrincipalName, 2) + m.Add("service_principal_name", *p.ServicePrincipalName) } - out = append(out, dyn.V(m)) + out = append(out, m) } - return dyn.V(out) + return out } // writeBundle writes the bundle into dir: databricks.yml plus the loose launch @@ -580,9 +538,9 @@ func writeBundle(ctx context.Context, cfg *runConfig, configPath, dir string, fo return nil, fmt.Errorf("databricks.yml already exists in %s; pass --force to overwrite or remove it", dir) } } - // SaveAsYAML's force arg is passed true unconditionally: the collision check + // Save's force arg is passed true unconditionally: the collision check // above already decided whether overwriting is allowed. - if err := yamlsaver.NewSaver().SaveAsYAML(root, bundlePath, true); err != nil { + if err := structyaml.Save(bundlePath, root, true, nil); err != nil { return nil, err } if err := quoteJobKey(bundlePath, bundleResourceKey(cfg.ExperimentName)); err != nil { diff --git a/cmd/air/convert_to_dabs_test.go b/cmd/air/convert_to_dabs_test.go index cbffa6b224c..ac07cac17d5 100644 --- a/cmd/air/convert_to_dabs_test.go +++ b/cmd/air/convert_to_dabs_test.go @@ -3,10 +3,11 @@ package aircmd import ( "os" "path/filepath" + "strconv" "strings" "testing" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structyaml" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.yaml.in/yaml/v3" @@ -86,17 +87,71 @@ func TestConvertToDabsCommandShape(t *testing.T) { assert.Error(t, cmd.Args(cmd, []string{"a", "b"})) } +// val is a value of the emitted bundle root. +type val struct{ v any } + +func (v val) MustString() string { return v.v.(string) } +func (v val) MustInt() int64 { return int64(v.v.(int)) } +func (v val) MustBool() bool { return v.v.(bool) } + +func (v val) MustSequence() []val { + if ss, ok := v.v.([]string); ok { + out := make([]val, len(ss)) + for i, s := range ss { + out[i] = val{s} + } + return out + } + var out []val + for _, e := range v.v.([]any) { + out = append(out, val{e}) + } + return out +} + +func (v val) Get(key string) val { + for _, p := range v.v.(structyaml.Map) { + if p.Key == key { + return val{p.Value} + } + } + return val{} +} + +// lookup reads a dotted path, with optional [index] suffixes, out of the emitted bundle root. +func lookup(root structyaml.Map, path string) (val, bool) { + v := val{root} + for part := range strings.SplitSeq(path, ".") { + key, idx, hasIdx := strings.Cut(strings.TrimSuffix(part, "]"), "[") + if _, ok := v.v.(structyaml.Map); !ok { + return val{}, false + } + if v = v.Get(key); v.v == nil { + return val{}, false + } + if hasIdx { + i, err := strconv.Atoi(idx) + seq, ok := v.v.([]any) + if err != nil || !ok || i >= len(seq) { + return val{}, false + } + v = val{seq[i]} + } + } + return v, true +} + // get is a small helper: read a dotted path out of the emitted bundle root. -func get(t *testing.T, root map[string]dyn.Value, path string) dyn.Value { +func get(t *testing.T, root structyaml.Map, path string) val { t.Helper() - v, err := dyn.GetByPath(dyn.V(root), dyn.MustPathFromString(path)) - require.NoError(t, err, "path %q should exist", path) + v, ok := lookup(root, path) + require.True(t, ok, "path %q should exist", path) return v } -func has(root map[string]dyn.Value, path string) bool { - _, err := dyn.GetByPath(dyn.V(root), dyn.MustPathFromString(path)) - return err == nil +func has(root structyaml.Map, path string) bool { + _, ok := lookup(root, path) + return ok } // A full config maps onto a schema-shaped bundle: bundle name, job/task keys, the @@ -543,11 +598,7 @@ func TestConvertToDabsSafeJobKey(t *testing.T) { root, _, err := convertToDabs(t.Context(), loaded, path, dir) require.NoError(t, err) // The key is the raw name; name/experiment keep the same value. - jobs, err := dyn.GetByPath(dyn.V(root), dyn.MustPathFromString("resources.jobs")) - require.NoError(t, err) - job := jobs.Get(name) - require.True(t, job.IsValid(), "job must be keyed by %q", name) - assert.Equal(t, name, job.Get("name").MustString()) + assert.Equal(t, name, get(t, root, "resources.jobs."+name+".name").MustString()) // The emitted YAML must load back with the key still a string. _, err = writeBundle(t.Context(), loaded, path, dir, true) diff --git a/cmd/apps/import.go b/cmd/apps/import.go index 355c51c690e..dc679fdce58 100644 --- a/cmd/apps/import.go +++ b/cmd/apps/import.go @@ -25,10 +25,6 @@ import ( "github.com/databricks/cli/libs/apps/prompt" "github.com/databricks/cli/libs/cmdctx" "github.com/databricks/cli/libs/cmdio" - "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/convert" - "github.com/databricks/cli/libs/dyn/yamlsaver" "github.com/databricks/cli/libs/env" "github.com/databricks/cli/libs/logdiag" "github.com/databricks/cli/libs/textutil" @@ -37,6 +33,7 @@ import ( "github.com/databricks/databricks-sdk-go/service/iam" "github.com/databricks/databricks-sdk-go/service/workspace" "github.com/spf13/cobra" + "github.com/databricks/cli/libs/structs/structyaml" ) func newImportCommand() *cobra.Command { @@ -456,22 +453,13 @@ func generateAppBundle(ctx context.Context, w *databricks.WorkspaceClient, app * } } - // Create the bundle configuration with explicit line numbers to control ordering // Use the app name for the bundle name bundleName := textutil.NormalizeString(app.Name) - bundleConfig := map[string]dyn.Value{ - "bundle": dyn.NewValue(map[string]dyn.Value{ - "name": dyn.NewValue(bundleName, []diag.Location{{Line: 1}}), - }, []diag.Location{{Line: 1}}), - "workspace": dyn.NewValue(map[string]dyn.Value{ - "host": dyn.NewValue(w.Config.Host, []diag.Location{{Line: 2}}), - }, []diag.Location{{Line: 10}}), - "resources": dyn.NewValue(map[string]dyn.Value{ - "apps": dyn.V(map[string]dyn.Value{ - appKey: v, - }), - }, []diag.Location{{Line: 20}}), - } + bundleConfig := structyaml.M( + "bundle", structyaml.M("name", bundleName), + "workspace", structyaml.M("host", w.Config.Host), + "resources", structyaml.M("apps", structyaml.M(appKey, v)), + ) // Download the app source files err = downloader.FlushToDisk(ctx, false) @@ -481,8 +469,7 @@ func generateAppBundle(ctx context.Context, w *databricks.WorkspaceClient, app * // Save databricks.yml databricksYml := filepath.Join(".", "databricks.yml") - saver := yamlsaver.NewSaver() - err = saver.SaveAsYAML(bundleConfig, databricksYml, false) + err = structyaml.Save(databricksYml, bundleConfig, false, nil) if err != nil { return "", fmt.Errorf("failed to save databricks.yml: %w", err) } @@ -545,7 +532,7 @@ func addBlankLinesBetweenTopLevelKeys(filename string) error { } // inlineAppConfigFile reads app.yml or app.yaml, inlines it into the app value, and returns the filename -func inlineAppConfigFile(appValue *dyn.Value) (string, error) { +func inlineAppConfigFile(appValue *structyaml.Map) (string, error) { // Check for app.yml first, then app.yaml var appConfigFile string var appConfigData []byte @@ -574,62 +561,33 @@ func inlineAppConfigFile(appValue *dyn.Value) (string, error) { return "", fmt.Errorf("failed to parse %s: %w", appConfigFile, err) } - // Get the current app value as a map - appMap, ok := appValue.AsMap() - if !ok { - return "", errors.New("app value is not a map") - } - - // Build the new app map with the config section - newPairs := make([]dyn.Pair, 0, len(appMap.Pairs())+2) - - // Copy existing pairs - newPairs = append(newPairs, appMap.Pairs()...) - // Create config section - configMap := make(map[string]dyn.Value) + var config structyaml.Map // Add command if present if cmd, ok := appConfig["command"]; ok { - cmdValue, err := convert.FromTyped(cmd, dyn.NilValue) - if err != nil { - return "", fmt.Errorf("failed to convert command: %w", err) - } - configMap["command"] = cmdValue + config.Add("command", structyaml.Value(cmd)) } // Add env if present if env, ok := appConfig["env"]; ok { - envValue, err := convert.FromTyped(env, dyn.NilValue) - if err != nil { - return "", fmt.Errorf("failed to convert env: %w", err) - } - configMap["env"] = envValue + config.Add("env", structyaml.Value(env)) } + // The config section and the top-level resources go before the existing fields. + var added structyaml.Map + // Add the config section if we have any items - if len(configMap) > 0 { - newPairs = append(newPairs, dyn.Pair{ - Key: dyn.V("config"), - Value: dyn.V(configMap), - }) + if len(config) > 0 { + added.Add("config", config) } // Add resources at top level if present if resources, ok := appConfig["resources"]; ok { - resourcesValue, err := convert.FromTyped(resources, dyn.NilValue) - if err != nil { - return "", fmt.Errorf("failed to convert resources: %w", err) - } - newPairs = append(newPairs, dyn.Pair{ - Key: dyn.V("resources"), - Value: resourcesValue, - }) + added.Add("resources", structyaml.Value(resources)) } - // Create the new app value with the config section - newMapping := dyn.NewMappingFromPairs(newPairs) - *appValue = dyn.NewValue(newMapping, appValue.Locations()) + *appValue = append(added, *appValue...) return appConfigFile, nil } diff --git a/cmd/apps/import_test.go b/cmd/apps/import_test.go index b50ec937991..30e9c8d239f 100644 --- a/cmd/apps/import_test.go +++ b/cmd/apps/import_test.go @@ -7,7 +7,7 @@ import ( "strings" "testing" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structyaml" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -16,17 +16,15 @@ func TestInlineAppConfigFile(t *testing.T) { tests := []struct { name string setupFiles map[string]string - inputValue dyn.Value + inputValue structyaml.Map expectedFile string expectedConfig map[string]any expectError bool }{ { - name: "no app config file", - setupFiles: map[string]string{}, - inputValue: dyn.V(map[string]dyn.Value{ - "name": dyn.V("test-app"), - }), + name: "no app config file", + setupFiles: map[string]string{}, + inputValue: structyaml.M("name", "test-app"), expectedFile: "", expectedConfig: map[string]any{"name": "test-app"}, }, @@ -38,9 +36,7 @@ env: - name: FOO value: bar`, }, - inputValue: dyn.V(map[string]dyn.Value{ - "name": dyn.V("test-app"), - }), + inputValue: structyaml.M("name", "test-app"), expectedFile: "app.yml", expectedConfig: nil, // Will check manually }, @@ -53,9 +49,7 @@ env: - name: TEST value: value`, }, - inputValue: dyn.V(map[string]dyn.Value{ - "name": dyn.V("test-app"), - }), + inputValue: structyaml.M("name", "test-app"), expectedFile: "app.yml", expectedConfig: nil, // Will check manually }, @@ -68,9 +62,7 @@ resources: serving_endpoint: name: my-endpoint`, }, - inputValue: dyn.V(map[string]dyn.Value{ - "name": dyn.V("test-app"), - }), + inputValue: structyaml.M("name", "test-app"), expectedFile: "app.yml", expectedConfig: nil, // Will check manually }, @@ -80,9 +72,7 @@ resources: "app.yml": `resources: - name: SERVING_ENDPOINT`, }, - inputValue: dyn.V(map[string]dyn.Value{ - "name": dyn.V("test-app"), - }), + inputValue: structyaml.M("name", "test-app"), expectedFile: "app.yml", expectedConfig: nil, // Will check manually }, @@ -92,9 +82,7 @@ resources: "app.yml": `command: ["python", "app.py"] env: []`, }, - inputValue: dyn.V(map[string]dyn.Value{ - "name": dyn.V("test-app"), - }), + inputValue: structyaml.M("name", "test-app"), expectedFile: "app.yml", expectedConfig: nil, // Will check manually }, @@ -125,20 +113,17 @@ env: []`, // Verify the structure if expectedConfig is set if tt.expectedConfig != nil { - appMap := appValue.MustMap() result := make(map[string]any) - for _, pair := range appMap.Pairs() { - key := pair.Key.MustString() - result[key] = pair.Value.AsAny() + for _, pair := range appValue { + result[pair.Key] = pair.Value } assert.Equal(t, tt.expectedConfig, result) } else if tt.expectedFile != "" { // Just verify that config or resources were added - appMap := appValue.MustMap() var hasConfigOrResources bool - for _, pair := range appMap.Pairs() { - key := pair.Key.MustString() + for _, pair := range appValue { + key := pair.Key if key == "config" || key == "resources" { hasConfigOrResources = true break @@ -158,24 +143,12 @@ func TestInlineAppConfigFileErrors(t *testing.T) { err := os.WriteFile("app.yml", []byte("invalid: yaml: content:\n - broken"), 0o644) require.NoError(t, err) - appValue := dyn.V(map[string]dyn.Value{"name": dyn.V("test")}) + appValue := structyaml.M("name", "test") _, err = inlineAppConfigFile(&appValue) assert.Error(t, err) assert.Contains(t, err.Error(), "failed to parse") }) - t.Run("app value not a map", func(t *testing.T) { - t.Chdir(t.TempDir()) - - err := os.WriteFile("app.yml", []byte("command: [\"test\"]"), 0o644) - require.NoError(t, err) - - appValue := dyn.V("not a map") - _, err = inlineAppConfigFile(&appValue) - assert.Error(t, err) - assert.Contains(t, err.Error(), "app value is not a map") - }) - t.Run("unreadable app.yml", func(t *testing.T) { t.Chdir(t.TempDir()) @@ -202,7 +175,7 @@ func TestInlineAppConfigFileErrors(t *testing.T) { require.NoError(t, err) } - appValue := dyn.V(map[string]dyn.Value{"name": dyn.V("test")}) + appValue := structyaml.M("name", "test") _, err = inlineAppConfigFile(&appValue) assert.Error(t, err) assert.Contains(t, err.Error(), "failed to read") @@ -223,21 +196,19 @@ resources: name: test`), 0o644) require.NoError(t, err) - appValue := dyn.V(map[string]dyn.Value{ - "name": dyn.V("test-app"), - "description": dyn.V("existing description"), - }) + appValue := structyaml.M( + "name", "test-app", + "description", "existing description", + ) filename, err := inlineAppConfigFile(&appValue) require.NoError(t, err) assert.Equal(t, "app.yml", filename) // Verify structure - appMap := appValue.MustMap() result := make(map[string]any) - for _, pair := range appMap.Pairs() { - key := pair.Key.MustString() - result[key] = pair.Value.AsAny() + for _, pair := range appValue { + result[pair.Key] = pair.Value } // Should have original fields plus config and resources diff --git a/cmd/bundle/generate/alert.go b/cmd/bundle/generate/alert.go index cb78b24de1e..2e4c5150370 100644 --- a/cmd/bundle/generate/alert.go +++ b/cmd/bundle/generate/alert.go @@ -12,9 +12,8 @@ import ( "github.com/databricks/cli/bundle/generate" "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/cmdio" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/yamlsaver" "github.com/databricks/cli/libs/logdiag" + "github.com/databricks/cli/libs/structs/structyaml" "github.com/databricks/cli/libs/textutil" "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/service/sql" @@ -141,13 +140,7 @@ After generation, you can deploy this alert to other targets using: return err } - result := map[string]dyn.Value{ - "resources": dyn.V(map[string]dyn.Value{ - "alerts": dyn.V(map[string]dyn.Value{ - alertKey: v, - }), - }), - } + result := structyaml.M("resources", structyaml.M("alerts", structyaml.M(alertKey, v))) // Create config directory if needed if err := os.MkdirAll(configDir, 0o755); err != nil { @@ -156,11 +149,11 @@ After generation, you can deploy this alert to other targets using: // Save configuration file configPath := filepath.Join(configDir, alertKey+".alert.yml") - saver := yamlsaver.NewSaverWithStyle(map[string]yaml.Style{ + styles := map[string]yaml.Style{ "display_name": yaml.DoubleQuotedStyle, - }) + } - err = saver.SaveAsYAML(result, configPath, force) + err = structyaml.Save(configPath, result, force, styles) if err != nil { return err } diff --git a/cmd/bundle/generate/app.go b/cmd/bundle/generate/app.go index bec005408cd..0705dc3f0c2 100644 --- a/cmd/bundle/generate/app.go +++ b/cmd/bundle/generate/app.go @@ -8,9 +8,8 @@ import ( "github.com/databricks/cli/cmd/bundle/deployment" "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/cmdio" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/yamlsaver" "github.com/databricks/cli/libs/logdiag" + "github.com/databricks/cli/libs/structs/structyaml" "github.com/databricks/cli/libs/textutil" "github.com/databricks/databricks-sdk-go/service/apps" "github.com/spf13/cobra" @@ -107,13 +106,7 @@ per target environment.`, appKey = textutil.NormalizeString(app.Name) } - result := map[string]dyn.Value{ - "resources": dyn.V(map[string]dyn.Value{ - "apps": dyn.V(map[string]dyn.Value{ - appKey: v, - }), - }), - } + result := structyaml.M("resources", structyaml.M("apps", structyaml.M(appKey, v))) err = downloader.FlushToDisk(ctx, force) if err != nil { @@ -122,8 +115,7 @@ per target environment.`, filename := filepath.Join(configDir, appKey+".app.yml") - saver := yamlsaver.NewSaver() - err = saver.SaveAsYAML(result, filename, force) + err = structyaml.Save(filename, result, force, nil) if err != nil { return err } diff --git a/cmd/bundle/generate/dashboard.go b/cmd/bundle/generate/dashboard.go index 9c4be81b391..d5058e60a5e 100644 --- a/cmd/bundle/generate/dashboard.go +++ b/cmd/bundle/generate/dashboard.go @@ -22,9 +22,8 @@ import ( "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/yamlsaver" "github.com/databricks/cli/libs/logdiag" + "github.com/databricks/cli/libs/structs/structyaml" "github.com/databricks/cli/libs/textutil" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/apierr" @@ -215,13 +214,7 @@ func (d *dashboard) saveConfiguration(ctx context.Context, b *bundle.Bundle, das return err } - result := map[string]dyn.Value{ - "resources": dyn.V(map[string]dyn.Value{ - "dashboards": dyn.V(map[string]dyn.Value{ - key: v, - }), - }), - } + result := structyaml.M("resources", structyaml.M("dashboards", structyaml.M(key, v))) // Make sure the output directory exists. if err := os.MkdirAll(d.resourceDir, 0o755); err != nil { @@ -230,9 +223,9 @@ func (d *dashboard) saveConfiguration(ctx context.Context, b *bundle.Bundle, das // Save the configuration to the resource directory. resourcePath := filepath.Join(d.resourceDir, key+".dashboard.yml") - saver := yamlsaver.NewSaverWithStyle(map[string]yaml.Style{ + styles := map[string]yaml.Style{ "display_name": yaml.DoubleQuotedStyle, - }) + } // Attempt to make the path relative to the bundle root. rel, err := filepath.Rel(b.BundleRootPath, resourcePath) @@ -241,7 +234,7 @@ func (d *dashboard) saveConfiguration(ctx context.Context, b *bundle.Bundle, das } cmdio.LogString(ctx, "Writing configuration to "+filepath.ToSlash(rel)) - err = saver.SaveAsYAML(result, resourcePath, d.force) + err = structyaml.Save(resourcePath, result, d.force, styles) if err != nil { return err } diff --git a/cmd/bundle/generate/genie_space.go b/cmd/bundle/generate/genie_space.go index 7c520abc320..2107d62ffd8 100644 --- a/cmd/bundle/generate/genie_space.go +++ b/cmd/bundle/generate/genie_space.go @@ -18,9 +18,8 @@ import ( "github.com/databricks/cli/cmd/bundle/deployment" "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/cmdio" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/yamlsaver" "github.com/databricks/cli/libs/logdiag" + "github.com/databricks/cli/libs/structs/structyaml" "github.com/databricks/cli/libs/textutil" "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/service/dashboards" @@ -130,13 +129,7 @@ func (g *genieSpace) saveConfiguration(ctx context.Context, b *bundle.Bundle, ge return err } - result := map[string]dyn.Value{ - "resources": dyn.V(map[string]dyn.Value{ - "genie_spaces": dyn.V(map[string]dyn.Value{ - key: v, - }), - }), - } + result := structyaml.M("resources", structyaml.M("genie_spaces", structyaml.M(key, v))) // Make sure the output directory exists. if err := os.MkdirAll(g.resourceDir, 0o755); err != nil { @@ -145,9 +138,9 @@ func (g *genieSpace) saveConfiguration(ctx context.Context, b *bundle.Bundle, ge // Save the configuration to the resource directory. resourcePath := filepath.Join(g.resourceDir, key+".genie_space.yml") - saver := yamlsaver.NewSaverWithStyle(map[string]yaml.Style{ + styles := map[string]yaml.Style{ "title": yaml.DoubleQuotedStyle, - }) + } // Attempt to make the path relative to the bundle root. rel, err := filepath.Rel(b.BundleRootPath, resourcePath) @@ -156,7 +149,7 @@ func (g *genieSpace) saveConfiguration(ctx context.Context, b *bundle.Bundle, ge } cmdio.LogString(ctx, "Writing configuration to "+filepath.ToSlash(rel)) - err = saver.SaveAsYAML(result, resourcePath, g.force) + err = structyaml.Save(resourcePath, result, g.force, styles) if err != nil { return err } diff --git a/cmd/bundle/generate/job.go b/cmd/bundle/generate/job.go index db6839ade98..07066c0ec8a 100644 --- a/cmd/bundle/generate/job.go +++ b/cmd/bundle/generate/job.go @@ -12,9 +12,8 @@ import ( "github.com/databricks/cli/cmd/bundle/deployment" "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/cmdio" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/yamlsaver" "github.com/databricks/cli/libs/logdiag" + "github.com/databricks/cli/libs/structs/structyaml" "github.com/databricks/cli/libs/textutil" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/spf13/cobra" @@ -115,13 +114,7 @@ After generation, you can deploy this job to other targets using: jobKey = textutil.NormalizeString(job.Settings.Name) } - result := map[string]dyn.Value{ - "resources": dyn.V(map[string]dyn.Value{ - "jobs": dyn.V(map[string]dyn.Value{ - jobKey: v, - }), - }), - } + result := structyaml.M("resources", structyaml.M("jobs", structyaml.M(jobKey, v))) err = downloader.FlushToDisk(ctx, force) if err != nil { @@ -141,13 +134,13 @@ After generation, you can deploy this job to other targets using: return fmt.Errorf("failed to rename file %s. DABs uses the resource type as a sub-extension for generated content, please rename it to %s, err: %w", oldFilename, filename, err) } - saver := yamlsaver.NewSaverWithStyle(map[string]yaml.Style{ + styles := map[string]yaml.Style{ // Including all JobSettings and nested fields which are map[string]string type "spark_conf": yaml.DoubleQuotedStyle, "custom_tags": yaml.DoubleQuotedStyle, "tags": yaml.DoubleQuotedStyle, - }) - err = saver.SaveAsYAML(result, filename, force) + } + err = structyaml.Save(filename, result, force, styles) if err != nil { return err } diff --git a/cmd/bundle/generate/pipeline.go b/cmd/bundle/generate/pipeline.go index cdde5984d40..d6de9ab9cc4 100644 --- a/cmd/bundle/generate/pipeline.go +++ b/cmd/bundle/generate/pipeline.go @@ -11,9 +11,8 @@ import ( "github.com/databricks/cli/cmd/bundle/deployment" "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/cmdio" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/yamlsaver" "github.com/databricks/cli/libs/logdiag" + "github.com/databricks/cli/libs/structs/structyaml" "github.com/databricks/cli/libs/textutil" "github.com/databricks/databricks-sdk-go/service/pipelines" "github.com/spf13/cobra" @@ -112,13 +111,7 @@ like catalogs, schemas, and compute configurations per target.`, pipelineKey = textutil.NormalizeString(pipeline.Name) } - result := map[string]dyn.Value{ - "resources": dyn.V(map[string]dyn.Value{ - "pipelines": dyn.V(map[string]dyn.Value{ - pipelineKey: v, - }), - }), - } + result := structyaml.M("resources", structyaml.M("pipelines", structyaml.M(pipelineKey, v))) err = downloader.FlushToDisk(ctx, force) if err != nil { @@ -136,15 +129,13 @@ like catalogs, schemas, and compute configurations per target.`, return fmt.Errorf("failed to rename file %s. DABs uses the resource type as a sub-extension for generated content, please rename it to %s, err: %w", oldFilename, filename, err) } - saver := yamlsaver.NewSaverWithStyle( - // Including all CreatePipeline and nested fields which are map[string]string type - map[string]yaml.Style{ - "spark_conf": yaml.DoubleQuotedStyle, - "custom_tags": yaml.DoubleQuotedStyle, - "configuration": yaml.DoubleQuotedStyle, - }, - ) - err = saver.SaveAsYAML(result, filename, force) + // Including all CreatePipeline and nested fields which are map[string]string type + styles := map[string]yaml.Style{ + "spark_conf": yaml.DoubleQuotedStyle, + "custom_tags": yaml.DoubleQuotedStyle, + "configuration": yaml.DoubleQuotedStyle, + } + err = structyaml.Save(filename, result, force, styles) if err != nil { return err } diff --git a/cmd/bundle/run.go b/cmd/bundle/run.go index 17199a6cabf..b6ce23c18f3 100644 --- a/cmd/bundle/run.go +++ b/cmd/bundle/run.go @@ -21,10 +21,10 @@ import ( "github.com/databricks/cli/libs/auth" "github.com/databricks/cli/libs/cmdctx" "github.com/databricks/cli/libs/cmdio" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/execv" "github.com/databricks/cli/libs/flags" "github.com/databricks/cli/libs/logdiag" + "github.com/databricks/cli/libs/structs/structpath" "github.com/spf13/cobra" ) @@ -189,7 +189,7 @@ Example usage: // runner sees concrete values (e.g. an app's env vars referencing another // resource, or its source_code_path pointing at the immutable snapshot's // full_path, which lives only in the deployed state). - target := dyn.NewPath(dyn.Key("resources"), dyn.Key(ref.Description.PluralName), dyn.Key(ref.Key)) + target := structpath.NewStringKeys(nil, "resources", ref.Description.PluralName, ref.Key) if err := b.DeploymentBundle.ResolveConfigAgainstState(&b.Config, target); err != nil { return err } diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index 6ccb3367db7..6332da33089 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -28,7 +28,6 @@ import ( "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dms" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/logdiag" "github.com/databricks/cli/libs/sync" @@ -715,8 +714,7 @@ func isNewerVersion(stateVersion, currentVersion string) bool { func rejectDefinitions(ctx context.Context, b *bundle.Bundle) { if b.Config.Definitions != nil { - v := dyn.GetValue(b.Config.Value(), "definitions") - loc := v.Locations() + loc := b.Config.GetLocations("definitions") filename := "input yaml" if len(loc) > 0 { filename = filepath.ToSlash(loc[0].File) diff --git a/cmd/bundle/validate.go b/cmd/bundle/validate.go index 9f8257731a5..52c17d99c09 100644 --- a/cmd/bundle/validate.go +++ b/cmd/bundle/validate.go @@ -9,7 +9,6 @@ import ( "github.com/databricks/cli/bundle/render" "github.com/databricks/cli/cmd/bundle/utils" "github.com/databricks/cli/cmd/root" - "github.com/databricks/cli/libs/dyn/convert" "github.com/databricks/cli/libs/flags" "github.com/databricks/cli/libs/logdiag" "github.com/spf13/cobra" @@ -23,11 +22,7 @@ func renderJsonOutput(cmd *cobra.Command, b *bundle.Bundle) error { if err != nil { return err } - converted, err := convert.FromTyped(redactedRoot, b.Config.Value()) - if err != nil { - return err - } - buf, err := json.MarshalIndent(converted.AsAny(), "", " ") + buf, err := json.MarshalIndent(redactedRoot.View().AsAny(), "", " ") if err != nil { return err } diff --git a/cmd/pipelines/generate.go b/cmd/pipelines/generate.go index c865213c36a..cf362bd89f1 100644 --- a/cmd/pipelines/generate.go +++ b/cmd/pipelines/generate.go @@ -10,12 +10,9 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/libs/cmdio" - "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/convert" - "github.com/databricks/cli/libs/dyn/yamlloader" - "github.com/databricks/cli/libs/dyn/yamlsaver" "github.com/databricks/cli/libs/logdiag" + "github.com/databricks/cli/libs/structs/structvar" + "github.com/databricks/cli/libs/structs/structyaml" "github.com/databricks/databricks-sdk-go/service/pipelines" "github.com/spf13/cobra" ) @@ -83,8 +80,7 @@ Use --existing-pipeline-dir to generate pipeline configuration from spark-pipeli return fmt.Errorf("failed to construct .pipeline.yml: %w", err) } - saver := yamlsaver.NewSaver() - err = saver.SaveAsYAML(resourcesMap, outputFile, force) + err = structyaml.Save(outputFile, resourcesMap, force, nil) if err != nil { return err } @@ -208,31 +204,21 @@ func parseSparkPipelineYAML(ctx context.Context, filePath string) (*sdpPipeline, } defer file.Close() - dv, err := yamlloader.LoadYAML(filePath, file) + out := sdpPipeline{} + _, diags, err := structvar.DecodeYAML(filePath, file, &out) if err != nil { return nil, fmt.Errorf("failed to load %s: %w", filePath, err) } - out := sdpPipeline{} - normalized, diags := convert.Normalize(&out, dv) - if diags.HasError() { - return nil, fmt.Errorf("failed to parse %s: %w", filePath, diags.Error()) - } - for _, diag := range diags { logdiag.LogDiag(ctx, diag) } - err = convert.ToTyped(&out, normalized) - if err != nil { - return nil, fmt.Errorf("failed to parse %s: %w", filePath, diags.Error()) - } - return &out, nil } // convertToResources converts a spark-pipeline.yml spec to DABs YAML format with "resources" property -func convertToResources(spec *sdpPipeline, resourceName, srcFolder string) (map[string]dyn.Value, error) { +func convertToResources(spec *sdpPipeline, resourceName, srcFolder string) (structyaml.Map, error) { // YAML paths are relative to directory containing YAML file, in this case: // DABs YAML is in "./resources/.pipeline.yml" // SDP YAML is in ".//spark-pipeline.yml" @@ -256,55 +242,29 @@ func convertToResources(spec *sdpPipeline, resourceName, srcFolder string) (map[ }, } - environmentDyn, err := convert.FromTyped(environment, dyn.NilValue) - if err != nil { - return nil, fmt.Errorf("failed to convert environments into dyn.Value: %w", err) - } - - librariesDyn, err := convertLibraries(relativePath, spec.Libraries) - if err != nil { - return nil, fmt.Errorf("failed to convert libraries into dyn.Value: %w", err) - } - - // maps are unordered, and saver is sorting keys by diag.Location - // this is helper function to monotonically assign locations as keys are created - var line int - nextLocation := func() []diag.Location { - line += 1 - return []diag.Location{{Line: line}} - } - - pipelineMap := map[string]dyn.Value{ - "name": dyn.V(spec.Name).WithLocations(nextLocation()), - "catalog": dyn.V(catalog).WithLocations(nextLocation()), - "schema": dyn.V(schema).WithLocations(nextLocation()), - "root_path": dyn.V(relativePath).WithLocations(nextLocation()), - "serverless": dyn.V(true).WithLocations(nextLocation()), - "libraries": librariesDyn.WithLocations(nextLocation()), - } + // Keys are written in the order they are added. + pipelineMap := structyaml.M( + "name", spec.Name, + "catalog", catalog, + "schema", schema, + "root_path", relativePath, + "serverless", true, + "libraries", convertLibraries(relativePath, spec.Libraries), + ) // configuration is optional field, skip if empty if spec.Configuration != nil { - dv, err := convert.FromTyped(spec.Configuration, dyn.NilValue) - if err != nil { - return nil, fmt.Errorf("failed to convert configuration into dyn.Value: %w", err) - } - - // NB: golang maps are unordered, and currently we don't preserve the order - pipelineMap["configuration"] = dv.WithLocations(nextLocation()) + pipelineMap.Add("configuration", structyaml.Value(spec.Configuration)) } - pipelineMap["environment"] = environmentDyn.WithLocations(nextLocation()) + pipelineMap.Add("environment", structyaml.Value(environment)) - resourcesMap := map[string]dyn.Value{ - "resources": dyn.V(map[string]dyn.Value{ - "pipelines": dyn.V(map[string]dyn.Value{ - resourceName: dyn.V(pipelineMap), - }), - }), - } + resourcesMap := structyaml.M("resources", structyaml.M("pipelines", structyaml.M(resourceName, pipelineMap))) - _, diag := convert.Normalize(&config.Root{}, dyn.V(resourcesMap)) + _, diag, err := structvar.DecodeYAMLNode("", structyaml.Node(resourcesMap, nil), &config.Root{}, nil) + if err != nil { + return nil, err + } if len(diag) > 0 { return nil, fmt.Errorf("generated output doesn't match expected schema: %v", diag) } @@ -316,7 +276,7 @@ func convertToResources(spec *sdpPipeline, resourceName, srcFolder string) (map[ // // relativePath contains a path to append into SDP libraries path to make // them relative to generated DABs YAML -func convertLibraries(relativePath string, specLibraries []sdpPipelineLibrary) (dyn.Value, error) { +func convertLibraries(relativePath string, specLibraries []sdpPipelineLibrary) any { var libraries []pipelines.PipelineLibrary for _, lib := range specLibraries { @@ -329,16 +289,11 @@ func convertLibraries(relativePath string, specLibraries []sdpPipelineLibrary) ( } } - librariesDyn, err := convert.FromTyped(libraries, dyn.NilValue) - if err != nil { - return dyn.InvalidValue, fmt.Errorf("failed to convert libraries into dyn.Value: %w", err) - } - - // FromTyped returns NilValue if libraries is an empty array - if librariesDyn.Kind() == dyn.KindNil { - // we always want to leave empty array as a placeholder in generated YAML - return dyn.V([]dyn.Value{}), nil + // Value returns nil if libraries is an empty array + if v := structyaml.Value(libraries); v != nil { + return v } - return librariesDyn, nil + // we always want to leave empty array as a placeholder in generated YAML + return []any{} } diff --git a/cmd/pipelines/generate_test.go b/cmd/pipelines/generate_test.go index 7135031ccbb..c795d463164 100644 --- a/cmd/pipelines/generate_test.go +++ b/cmd/pipelines/generate_test.go @@ -5,9 +5,7 @@ import ( "path/filepath" "testing" - "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/dynassert" + "github.com/databricks/cli/libs/structs/structyaml" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -174,38 +172,27 @@ func TestConvertToResources(t *testing.T) { }, } - expected := map[string]dyn.Value{ - "resources": dyn.V(map[string]dyn.Value{ - "pipelines": dyn.V(map[string]dyn.Value{ - "my_pipeline": dyn.V(map[string]dyn.Value{ - "name": dyn.V("My Pipeline").WithLocations([]diag.Location{{Line: 1}}), - "catalog": dyn.V("${var.catalog}").WithLocations([]diag.Location{{Line: 2}}), - "schema": dyn.V("${var.schema}").WithLocations([]diag.Location{{Line: 3}}), - "root_path": dyn.V("../src/my_pipeline").WithLocations([]diag.Location{{Line: 4}}), - "serverless": dyn.V(true).WithLocations([]diag.Location{{Line: 5}}), - "libraries": dyn.V([]dyn.Value{ - dyn.V(map[string]dyn.Value{ - "glob": dyn.V(map[string]dyn.Value{ - "include": dyn.V("../src/my_pipeline/transformations/**"), - }), - }), - }).WithLocations([]diag.Location{{Line: 6}}), - "configuration": dyn.V(map[string]dyn.Value{ - "key0": dyn.V("value0"), - "key1": dyn.V("value1"), - }).WithLocations([]diag.Location{{Line: 7}}), - "environment": dyn.V(map[string]dyn.Value{ - "dependencies": dyn.V([]dyn.Value{ - dyn.V("--editable ${workspace.file_path}"), - }), - }).WithLocations([]diag.Location{{Line: 8}}), - }), - }), - }), - } + expected := structyaml.M( + "resources", structyaml.M( + "pipelines", structyaml.M( + "my_pipeline", structyaml.M( + "name", "My Pipeline", + "catalog", "${var.catalog}", + "schema", "${var.schema}", + "root_path", "../src/my_pipeline", + "serverless", true, + "libraries", []any{ + structyaml.M("glob", structyaml.M("include", "../src/my_pipeline/transformations/**")), + }, + "configuration", structyaml.M("key0", "value0", "key1", "value1"), + "environment", structyaml.M("dependencies", []any{"--editable ${workspace.file_path}"}), + ), + ), + ), + ) actual, err := convertToResources(&input, "my_pipeline", "src/my_pipeline") require.NoError(t, err) - dynassert.Equal(t, dyn.V(expected), dyn.V(actual)) + assert.Equal(t, expected, actual) } diff --git a/integration/assumptions/dashboard_assumptions_test.go b/integration/assumptions/dashboard_assumptions_test.go index 111482a6f81..be991dd081d 100644 --- a/integration/assumptions/dashboard_assumptions_test.go +++ b/integration/assumptions/dashboard_assumptions_test.go @@ -6,13 +6,11 @@ import ( "github.com/databricks/cli/integration/internal/acc" "github.com/databricks/cli/internal/testutil" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/convert" - "github.com/databricks/cli/libs/dyn/merge" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/service/dashboards" "github.com/databricks/databricks-sdk-go/service/workspace" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -86,27 +84,21 @@ func TestDashboardAssumptions_WorkspaceImport(t *testing.T) { }) require.NoError(t, err) - // Convert the dashboard object to a [dyn.Value] to make comparison easier. - previous, err := convert.FromTyped(previousDashboard, dyn.NilValue) - require.NoError(t, err) - current, err := convert.FromTyped(currentDashboard, dyn.NilValue) - require.NoError(t, err) - // Collect updated and deleted paths. var updatedFieldPaths []string var deletedFieldPaths []string - _, err = merge.Override(previous, current, merge.OverrideVisitor{ - VisitDelete: func(basePath dyn.Path, left dyn.Value) error { - deletedFieldPaths = append(deletedFieldPaths, basePath.String()) + _, err = structvar.PlanOverride(structvar.NewView(previousDashboard, nil, nil), structvar.NewView(currentDashboard, nil, nil), structvar.OverrideVisitor{ + VisitDelete: func(path *structpath.PathNode, left structvar.View) error { + deletedFieldPaths = append(deletedFieldPaths, path.String()) return nil }, - VisitInsert: func(basePath dyn.Path, right dyn.Value) (dyn.Value, error) { + VisitInsert: func(path *structpath.PathNode, right structvar.View) error { assert.Fail(t, "unexpected insert operation") - return right, nil + return nil }, - VisitUpdate: func(basePath dyn.Path, left, right dyn.Value) (dyn.Value, error) { - updatedFieldPaths = append(updatedFieldPaths, basePath.String()) - return right, nil + VisitUpdate: func(path *structpath.PathNode, left, right structvar.View) error { + updatedFieldPaths = append(updatedFieldPaths, path.String()) + return nil }, }) require.NoError(t, err) diff --git a/libs/dyn/convert/end_to_end_test.go b/libs/dyn/convert/end_to_end_test.go deleted file mode 100644 index e3ea273cb04..00000000000 --- a/libs/dyn/convert/end_to_end_test.go +++ /dev/null @@ -1,171 +0,0 @@ -package convert - -import ( - "testing" - - "github.com/databricks/cli/libs/dyn" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func assertFromTypedToTypedEqual[T any](t *testing.T, src T) { - nv, err := FromTyped(src, dyn.NilValue) - require.NoError(t, err) - - var dst T - err = ToTyped(&dst, nv) - require.NoError(t, err) - assert.Equal(t, src, dst) -} - -func TestAdditional(t *testing.T) { - type StructType struct { - Str string `json:"str"` - } - - type Tmp struct { - MapToPointer map[string]*string `json:"map_to_pointer"` - SliceOfPointer []*string `json:"slice_of_pointer"` - NestedStruct StructType `json:"nested_struct"` - } - - t.Run("nil", func(t *testing.T) { - assertFromTypedToTypedEqual(t, Tmp{}) - }) - - t.Run("empty map", func(t *testing.T) { - assertFromTypedToTypedEqual(t, Tmp{ - MapToPointer: map[string]*string{}, - }) - }) - - t.Run("map with empty string value", func(t *testing.T) { - s := "" - assertFromTypedToTypedEqual(t, Tmp{ - MapToPointer: map[string]*string{ - "key": &s, - }, - }) - }) - - t.Run("map with nil value", func(t *testing.T) { - assertFromTypedToTypedEqual(t, Tmp{ - MapToPointer: map[string]*string{ - "key": nil, - }, - }) - }) - - t.Run("empty slice", func(t *testing.T) { - assertFromTypedToTypedEqual(t, Tmp{ - SliceOfPointer: []*string{}, - }) - }) - - t.Run("slice with nil value", func(t *testing.T) { - assertFromTypedToTypedEqual(t, Tmp{ - SliceOfPointer: []*string{nil}, - }) - }) - - t.Run("pointer to a empty string", func(t *testing.T) { - s := "" - assertFromTypedToTypedEqual(t, &s) - }) - - t.Run("nil pointer", func(t *testing.T) { - var s *string - assertFromTypedToTypedEqual(t, s) - }) - - t.Run("pointer to struct with scalar values", func(t *testing.T) { - s := "" - type foo struct { - A string `json:"a"` - B int `json:"b"` - C bool `json:"c"` - D *string `json:"d"` - } - assertFromTypedToTypedEqual(t, &foo{ - A: "a", - B: 1, - C: true, - D: &s, - }) - assertFromTypedToTypedEqual(t, &foo{ - A: "", - B: 0, - C: false, - D: nil, - }) - }) - - t.Run("map with scalar values", func(t *testing.T) { - assertFromTypedToTypedEqual(t, map[string]string{ - "a": "a", - "b": "b", - "c": "", - }) - assertFromTypedToTypedEqual(t, map[string]int{ - "a": 1, - "b": 0, - "c": 2, - }) - }) -} - -func TestEndToEndForceSendFields(t *testing.T) { - type Inner struct { - InnerField string `json:"inner_field"` - ForceSendFields []string `json:"-"` - } - type Outer struct { - OuterField string `json:"outer_field"` - Inner - } - - // Test with zero value in embedded struct - src := Outer{ - OuterField: "outer_value", - Inner: Inner{ - InnerField: "", // Zero value - ForceSendFields: []string{"InnerField"}, // Should be preserved - }, - } - - assertFromTypedToTypedEqual(t, src) -} - -func TestEndToEndPointerForceSendFields(t *testing.T) { - type NewCluster struct { - NumWorkers int `json:"num_workers"` - SparkVersion string `json:"spark_version"` - ForceSendFields []string `json:"-"` - } - type JobCluster struct { - JobClusterKey string `json:"job_cluster_key"` - NewCluster *NewCluster `json:"new_cluster"` - } - type JobSettings struct { - JobClusters []JobCluster `json:"job_clusters"` - Name string `json:"name"` - ForceSendFields []string `json:"-"` - } - - // Test with zero value in pointer embedded struct (like acceptance test) - src := JobSettings{ - Name: "test-job", - JobClusters: []JobCluster{ - { - JobClusterKey: "key", - NewCluster: &NewCluster{ - NumWorkers: 0, // Zero value - SparkVersion: "13.3.x-scala2.12", - ForceSendFields: []string{"NumWorkers"}, // Should be preserved - }, - }, - }, - } - - assertFromTypedToTypedEqual(t, src) -} diff --git a/libs/dyn/convert/error.go b/libs/dyn/convert/error.go deleted file mode 100644 index d3770d82d74..00000000000 --- a/libs/dyn/convert/error.go +++ /dev/null @@ -1,16 +0,0 @@ -package convert - -import ( - "fmt" - - "github.com/databricks/cli/libs/dyn" -) - -type TypeError struct { - value dyn.Value - msg string -} - -func (e TypeError) Error() string { - return fmt.Sprintf("%s: %s", e.value.Location(), e.msg) -} diff --git a/libs/dyn/convert/from_typed.go b/libs/dyn/convert/from_typed.go deleted file mode 100644 index 66451124293..00000000000 --- a/libs/dyn/convert/from_typed.go +++ /dev/null @@ -1,362 +0,0 @@ -package convert - -import ( - "fmt" - "reflect" - "slices" - - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/dynvar" -) - -type fromTypedOptions int - -const ( - // If this flag is set, zero values in the typed representation are resolved to - // the equivalent zero value in the dynamic representation. - // If it is not set, zero values resolve to [dyn.NilValue]. - // - // This flag exists to reconcile default values in Go being zero values with values - // being intentionally set to their zero value. We capture zero values in the dynamic - // configuration if they are 1) behind a pointer, 2) a map value, 3) a slice element, - // in the typed configuration. - includeZeroValues fromTypedOptions = 1 << iota -) - -// FromTyped converts changes made in the typed structure w.r.t. the configuration value -// back to the configuration value, retaining existing location information where possible. -// -// It uses the reference value both for location information and to determine if the typed -// value was changed or not. For example, if a struct-by-value field is nil in the reference -// it will be zero-valued in the typed configuration. If it remains zero-valued, this -// function will still emit a nil value in the dynamic representation. -func FromTyped(src any, ref dyn.Value) (dyn.Value, error) { - return fromTyped(src, ref) -} - -// Private implementation of FromTyped that allows for additional options not exposed -// in the public API. -func fromTyped(src any, ref dyn.Value, options ...fromTypedOptions) (dyn.Value, error) { - srcv := reflect.ValueOf(src) - - // Dereference pointer if necessary - for srcv.Kind() == reflect.Pointer { - if srcv.IsNil() { - return dyn.NilValue.WithLocations(ref.Locations()), nil - } - srcv = srcv.Elem() - - // If a pointer to a type points to a zero value, we should include - // that zero value in the dynamic representation. - // This is because by default a pointer is nil in Go, and it not being nil - // indicates its value was intentionally set to zero. - if !slices.Contains(options, includeZeroValues) { - options = append(options, includeZeroValues) - } - } - - var v dyn.Value - var err error - switch srcv.Kind() { - case reflect.Struct: - // Handle SDK native types using JSON marshaling. - if slices.Contains(sdkNativeTypes, srcv.Type()) { - v, err = fromTypedSDKNative(srcv, ref, options...) - } else { - v, err = fromTypedStruct(srcv, ref, options...) - } - case reflect.Map: - v, err = fromTypedMap(srcv, ref) - case reflect.Slice: - v, err = fromTypedSlice(srcv, ref) - case reflect.String: - v, err = fromTypedString(srcv, ref, options...) - case reflect.Bool: - v, err = fromTypedBool(srcv, ref, options...) - case reflect.Int, reflect.Int32, reflect.Int64: - v, err = fromTypedInt(srcv, ref, options...) - case reflect.Float32, reflect.Float64: - v, err = fromTypedFloat(srcv, ref, options...) - case reflect.Invalid: - // If the value is untyped and not set (e.g. any type with nil value), we return nil. - v, err = dyn.NilValue, nil - default: - return dyn.InvalidValue, fmt.Errorf("unsupported type: %s", srcv.Kind()) - } - - // Ensure the location metadata is retained. - if err != nil { - return dyn.InvalidValue, err - } - return v.WithLocations(ref.Locations()), err -} - -func fromTypedStruct(src reflect.Value, ref dyn.Value, options ...fromTypedOptions) (dyn.Value, error) { - // Check that the reference value is compatible or nil. - switch ref.Kind() { - case dyn.KindString: - // Ignore pure variable references (e.g. ${var.foo}). - if dynvar.IsPureVariableReference(ref.MustString()) { - return ref, nil - } - case dyn.KindMap, dyn.KindNil: - default: - return dyn.InvalidValue, fmt.Errorf("cannot convert struct field to dynamic type %#v: src=%#v ref=%#v", ref.Kind().String(), src, ref.AsAny()) - } - - refm, _ := ref.AsMap() - out := dyn.NewMapping() - info := getStructInfo(src.Type()) - - for _, fieldval := range info.FieldValues(src) { - k := fieldval.Key - v := fieldval.Value - isForced := fieldval.IsForced - pair, ok := refm.GetPairByString(k) - refloc := pair.Key.Locations() - refv := pair.Value - - // Use nil reference if there is no reference for this key - if !ok { - refloc = nil - refv = dyn.NilValue - } - - var options []fromTypedOptions - if v.Kind() == reflect.Interface { - options = append(options, includeZeroValues) - } - - // Convert the field taking into account the reference value (may be equal to config.NilValue). - nv, err := fromTyped(v.Interface(), refv, options...) - if err != nil { - return dyn.InvalidValue, err - } - - // Either if the key was set in the reference, the field is not zero-valued, OR it's forced - if ok || nv.Kind() != dyn.KindNil || isForced { - // If v isZero, it could be because it's a variable reference; so we check that nv is zero as well - // BUT: always include if it's forced - if v.Kind() != reflect.Struct && v.IsZero() && nv.IsZero() && !info.ForceEmpty[k] && !isForced { - continue - } - - // If the field is forced but nv is nil, convert it to the appropriate zero value - if isForced && nv.Kind() == dyn.KindNil { - // Convert the zero value using proper recursive conversion instead of dyn.V() directly - // This prevents "not handled" panics for complex types like structs, slices, maps, etc. - // Use refv to preserve location information from the original reference - var err error - nv, err = fromTyped(v.Interface(), refv, includeZeroValues) - if err != nil { - return dyn.InvalidValue, err - } - } - - out.SetLoc(k, refloc, nv) - } - } - - // Return the new mapping if: - // 1. The mapping has entries (i.e. the struct was not empty). - // 2. The reference is a map (i.e. the struct was and still is empty). - // 3. The "includeZeroValues" option is set (i.e. the struct is a non-nil pointer). - if out.Len() > 0 || ref.Kind() == dyn.KindMap || slices.Contains(options, includeZeroValues) { - return dyn.V(out), nil - } - - // Otherwise, return nil. - return dyn.NilValue, nil -} - -func fromTypedMap(src reflect.Value, ref dyn.Value) (dyn.Value, error) { - // Check that the reference value is compatible or nil. - switch ref.Kind() { - case dyn.KindString: - // Ignore pure variable references (e.g. ${var.foo}). - if dynvar.IsPureVariableReference(ref.MustString()) { - return ref, nil - } - case dyn.KindMap, dyn.KindNil: - default: - return dyn.InvalidValue, fmt.Errorf("cannot convert map field to dynamic type %#v: src=%#v ref=%#v", ref.Kind().String(), src, ref.AsAny()) - } - - // Return nil if the map is nil. - if src.IsNil() { - return dyn.NilValue, nil - } - - refm, _ := ref.AsMap() - out := dyn.NewMapping() - iter := src.MapRange() - for iter.Next() { - k := iter.Key().String() - v := iter.Value() - pair, ok := refm.GetPairByString(k) - refloc := pair.Key.Locations() - refv := pair.Value - - // Use nil reference if there is no reference for this key - if !ok { - refloc = nil - refv = dyn.NilValue - } - - // Convert entry taking into account the reference value (may be equal to dyn.NilValue). - nv, err := fromTyped(v.Interface(), refv, includeZeroValues) - if err != nil { - return dyn.InvalidValue, err - } - - // Every entry is represented, even if it is a nil. - // Otherwise, a map with zero-valued structs would yield a nil as well. - out.SetLoc(k, refloc, nv) - } - - return dyn.V(out), nil -} - -func fromTypedSlice(src reflect.Value, ref dyn.Value) (dyn.Value, error) { - // Check that the reference value is compatible or nil. - switch ref.Kind() { - case dyn.KindString: - // Ignore pure variable references (e.g. ${var.foo}). - if dynvar.IsPureVariableReference(ref.MustString()) { - return ref, nil - } - case dyn.KindSequence, dyn.KindNil: - default: - return dyn.InvalidValue, fmt.Errorf("cannot convert slice field to dynamic type %#v: src=%#v ref=%#v", ref.Kind().String(), src, ref.AsAny()) - } - - // Return nil if the slice is nil. - if src.IsNil() { - return dyn.NilValue, nil - } - - out := make([]dyn.Value, src.Len()) - for i := range src.Len() { - v := src.Index(i) - refv := ref.Index(i) - - // Use nil reference if there is no reference for this index. - if refv.Kind() == dyn.KindInvalid { - refv = dyn.NilValue - } - - // Convert entry taking into account the reference value (may be equal to dyn.NilValue). - nv, err := fromTyped(v.Interface(), refv, includeZeroValues) - if err != nil { - return dyn.InvalidValue, err - } - - out[i] = nv - } - - return dyn.V(out), nil -} - -func fromTypedString(src reflect.Value, ref dyn.Value, options ...fromTypedOptions) (dyn.Value, error) { - switch ref.Kind() { - case dyn.KindString: - value := src.String() - if value == ref.MustString() { - return ref, nil - } - - return dyn.V(value), nil - case dyn.KindNil: - // This field is not set in the reference. We set it to nil if it's zero - // valued in the typed representation and the includeZeroValues option is not set. - if src.IsZero() && !slices.Contains(options, includeZeroValues) { - return dyn.NilValue, nil - } - return dyn.V(src.String()), nil - default: - // Fall through to the error case. - } - - return dyn.InvalidValue, fmt.Errorf("cannot convert string field to dynamic type %#v: src=%#v ref=%#v", ref.Kind().String(), src, ref.AsAny()) -} - -func fromTypedBool(src reflect.Value, ref dyn.Value, options ...fromTypedOptions) (dyn.Value, error) { - switch ref.Kind() { - case dyn.KindBool: - value := src.Bool() - if value == ref.MustBool() { - return ref, nil - } - return dyn.V(value), nil - case dyn.KindNil: - // This field is not set in the reference. We set it to nil if it's zero - // valued in the typed representation and the includeZeroValues option is not set. - if src.IsZero() && !slices.Contains(options, includeZeroValues) { - return dyn.NilValue, nil - } - return dyn.V(src.Bool()), nil - case dyn.KindString: - // Ignore pure variable references (e.g. ${var.foo}). - if dynvar.IsPureVariableReference(ref.MustString()) { - return ref, nil - } - default: - // Fall through to the error case. - } - - return dyn.InvalidValue, fmt.Errorf("cannot convert bool field to dynamic type %#v: src=%#v ref=%#v", ref.Kind().String(), src, ref.AsAny()) -} - -func fromTypedInt(src reflect.Value, ref dyn.Value, options ...fromTypedOptions) (dyn.Value, error) { - switch ref.Kind() { - case dyn.KindInt: - value := src.Int() - if value == ref.MustInt() { - return ref, nil - } - return dyn.V(value), nil - case dyn.KindNil: - // This field is not set in the reference. We set it to nil if it's zero - // valued in the typed representation and the includeZeroValues option is not set. - if src.IsZero() && !slices.Contains(options, includeZeroValues) { - return dyn.NilValue, nil - } - return dyn.V(src.Int()), nil - case dyn.KindString: - // Ignore pure variable references (e.g. ${var.foo}). - if dynvar.IsPureVariableReference(ref.MustString()) { - return ref, nil - } - default: - // Fall through to the error case. - } - - return dyn.InvalidValue, fmt.Errorf("cannot convert int field to dynamic type %#v: src=%#v ref=%#v", ref.Kind().String(), src, ref.AsAny()) -} - -func fromTypedFloat(src reflect.Value, ref dyn.Value, options ...fromTypedOptions) (dyn.Value, error) { - switch ref.Kind() { - case dyn.KindFloat: - value := src.Float() - if value == ref.MustFloat() { - return ref, nil - } - return dyn.V(value), nil - case dyn.KindNil: - // This field is not set in the reference. We set it to nil if it's zero - // valued in the typed representation and the includeZeroValues option is not set. - if src.IsZero() && !slices.Contains(options, includeZeroValues) { - return dyn.NilValue, nil - } - return dyn.V(src.Float()), nil - case dyn.KindString: - // Ignore pure variable references (e.g. ${var.foo}). - if dynvar.IsPureVariableReference(ref.MustString()) { - return ref, nil - } - default: - // Fall through to the error case. - } - - return dyn.InvalidValue, fmt.Errorf("cannot convert float field to dynamic type %#v: src=%#v ref=%#v", ref.Kind().String(), src, ref.AsAny()) -} diff --git a/libs/dyn/convert/from_typed_test.go b/libs/dyn/convert/from_typed_test.go deleted file mode 100644 index 3420c46cbd7..00000000000 --- a/libs/dyn/convert/from_typed_test.go +++ /dev/null @@ -1,927 +0,0 @@ -package convert - -import ( - "testing" - - "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/dynassert" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestFromTypedStructZeroFields(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - Bar string `json:"bar"` - } - - src := Tmp{} - - // For an empty struct with a nil reference we expect a nil. - nv, err := FromTyped(src, dyn.NilValue) - require.NoError(t, err) - assert.Equal(t, dyn.NilValue, nv) - - // For an empty struct with a non-nil reference we expect an empty map. - nv, err = FromTyped(src, dyn.V(map[string]dyn.Value{})) - require.NoError(t, err) - assert.Equal(t, dyn.V(map[string]dyn.Value{}), nv) -} - -func TestFromTypedStructPointerZeroFields(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - Bar string `json:"bar"` - } - - var src *Tmp - var nv dyn.Value - var err error - - // For a nil pointer with a nil reference we expect a nil. - src = nil - nv, err = FromTyped(src, dyn.NilValue) - require.NoError(t, err) - assert.Equal(t, dyn.NilValue, nv) - - // For a nil pointer with a non-nil reference we expect a nil. - src = nil - nv, err = FromTyped(src, dyn.V(map[string]dyn.Value{})) - require.NoError(t, err) - assert.Equal(t, dyn.NilValue, nv) - - // For an initialized pointer with a nil reference we expect an empty map. - src = &Tmp{} - nv, err = FromTyped(src, dyn.NilValue) - require.NoError(t, err) - assert.Equal(t, dyn.V(map[string]dyn.Value{}), nv) - - // For an initialized pointer with a non-nil reference we expect an empty map. - src = &Tmp{} - nv, err = FromTyped(src, dyn.V(map[string]dyn.Value{})) - require.NoError(t, err) - assert.Equal(t, dyn.V(map[string]dyn.Value{}), nv) -} - -func TestFromTypedStructNilFields(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - Bar string `json:"bar"` - } - - // For a zero value struct with a reference containing nil fields we expect the nils to be retained. - src := Tmp{} - ref := dyn.V(map[string]dyn.Value{ - "foo": dyn.NilValue, - "bar": dyn.NilValue, - }) - - nv, err := FromTyped(src, ref) - require.NoError(t, err) - dynassert.Equal(t, dyn.V(map[string]dyn.Value{ - "foo": dyn.NilValue, - "bar": dyn.NilValue, - }), nv) -} - -func TestFromTypedStructSetFields(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - Bar string `json:"bar"` - } - - src := Tmp{ - Foo: "foo", - Bar: "bar", - } - - ref := dyn.NilValue - nv, err := FromTyped(src, ref) - require.NoError(t, err) - dynassert.Equal(t, dyn.V(map[string]dyn.Value{ - "foo": dyn.V("foo"), - "bar": dyn.V("bar"), - }), nv) -} - -func TestFromTypedStructSetFieldsIntoExistingFields(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - Bar string `json:"bar"` - } - - src := Tmp{ - Foo: "foo", - Bar: "bar", - } - - ref := dyn.V(map[string]dyn.Value{ - "foo": dyn.V("foo_old"), - "bar": dyn.V("bar_old"), - }) - - nv, err := FromTyped(src, ref) - require.NoError(t, err) - dynassert.Equal(t, dyn.V(map[string]dyn.Value{ - "foo": dyn.V("foo"), - "bar": dyn.V("bar"), - }), nv) -} - -func TestFromTypedStructZeroFieldsIntoExistingFields(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - Bar string `json:"bar,omitempty"` - BazBugz string `json:"baz_bugs,omitempty"` - ForceSendFields []string `json:"-"` - } - - src := Tmp{ForceSendFields: []string{"BazBugz"}} - - ref := dyn.V(map[string]dyn.Value{ - "foo": dyn.V("foo_old"), - "bar": dyn.V("bar_old"), - "baz_bugs": dyn.V("baz_old"), - }) - - nv, err := FromTyped(src, ref) - require.NoError(t, err) - dynassert.Equal(t, dyn.V(map[string]dyn.Value{ - "foo": dyn.V(""), - "baz_bugs": dyn.V(""), - }), nv) -} - -func TestFromTypedStructZeroFieldsIntoExistingFieldsFloat64(t *testing.T) { - type Tmp struct { - Foo float64 `json:"foo"` - Bar float64 `json:"bar,omitempty"` - BazBugz float64 `json:"baz_bugs,omitempty"` - ForceSendFields []string `json:"-"` - } - - src := Tmp{ForceSendFields: []string{"BazBugz"}} - - ref := dyn.V(map[string]dyn.Value{ - "foo": dyn.V(1.2), - "bar": dyn.V(1.2), - "baz_bugs": dyn.V(1.2), - }) - - nv, err := FromTyped(src, ref) - require.NoError(t, err) - dynassert.Equal(t, dyn.V(map[string]dyn.Value{ - "foo": dyn.V(0.0), - "baz_bugs": dyn.V(0.0), - }), nv) -} - -func TestFromTypedStructSetFieldsRetainLocation(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - Bar string `json:"bar"` - } - - src := Tmp{ - Foo: "bar", - Bar: "qux", - } - - ref := dyn.V(map[string]dyn.Value{ - "foo": dyn.NewValue("bar", []diag.Location{{File: "foo"}}), - "bar": dyn.NewValue("baz", []diag.Location{{File: "bar"}}), - }) - - nv, err := FromTyped(src, ref) - require.NoError(t, err) - - // Assert foo and bar have retained their location. - assert.Equal(t, dyn.NewValue("bar", []diag.Location{{File: "foo"}}), nv.Get("foo")) - assert.Equal(t, dyn.NewValue("qux", []diag.Location{{File: "bar"}}), nv.Get("bar")) -} - -func TestFromTypedStringMapWithZeroValue(t *testing.T) { - ref := dyn.NilValue - src := map[string]string{ - "foo": "", - "bar": "fuzz", - } - - nv, err := FromTyped(src, ref) - require.NoError(t, err) - dynassert.Equal(t, dyn.V(map[string]dyn.Value{ - "foo": dyn.V(""), - "bar": dyn.V("fuzz"), - }), nv) -} - -func TestFromTypedStringSliceWithZeroValue(t *testing.T) { - ref := dyn.NilValue - src := []string{"a", "", "c"} - - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.V([]dyn.Value{ - dyn.V("a"), dyn.V(""), dyn.V("c"), - }), nv) -} - -func TestFromTypedStringStructWithZeroValue(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - Bar string `json:"bar"` - } - - ref := dyn.NilValue - src := Tmp{ - Foo: "foo", - Bar: "", - } - - // Note, the zero value is not included in the output. - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.V(map[string]dyn.Value{ - "foo": dyn.V("foo"), - }), nv) -} - -func TestFromTypedBoolMapWithZeroValue(t *testing.T) { - ref := dyn.NilValue - src := map[string]bool{ - "foo": false, - "bar": true, - } - - nv, err := FromTyped(src, ref) - require.NoError(t, err) - dynassert.Equal(t, dyn.V(map[string]dyn.Value{ - "foo": dyn.V(false), - "bar": dyn.V(true), - }), nv) -} - -func TestFromTypedBoolSliceWithZeroValue(t *testing.T) { - ref := dyn.NilValue - src := []bool{true, false, true} - - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.V([]dyn.Value{ - dyn.V(true), dyn.V(false), dyn.V(true), - }), nv) -} - -func TestFromTypedBoolStructWithZeroValue(t *testing.T) { - type Tmp struct { - Foo bool `json:"foo"` - Bar bool `json:"bar"` - } - - ref := dyn.NilValue - src := Tmp{ - Foo: true, - Bar: false, - } - - // Note, the zero value is not included in the output. - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.V(map[string]dyn.Value{ - "foo": dyn.V(true), - }), nv) -} - -func TestFromTypedIntMapWithZeroValue(t *testing.T) { - ref := dyn.NilValue - src := map[string]int{ - "foo": 0, - "bar": 1, - } - - nv, err := FromTyped(src, ref) - require.NoError(t, err) - dynassert.Equal(t, dyn.V(map[string]dyn.Value{ - "foo": dyn.V(int64(0)), - "bar": dyn.V(int64(1)), - }), nv) -} - -func TestFromTypedIntSliceWithZeroValue(t *testing.T) { - ref := dyn.NilValue - src := []int{1, 0, 2} - - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.V([]dyn.Value{ - dyn.V(int64(1)), dyn.V(int64(0)), dyn.V(int64(2)), - }), nv) -} - -func TestFromTypedIntStructWithZeroValue(t *testing.T) { - type Tmp struct { - Foo int `json:"foo"` - Bar int `json:"bar"` - } - - ref := dyn.NilValue - src := Tmp{ - Foo: 1, - Bar: 0, - } - - // Note, the zero value is not included in the output. - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.V(map[string]dyn.Value{ - "foo": dyn.V(int64(1)), - }), nv) -} - -func TestFromTypedFloatMapWithZeroValue(t *testing.T) { - ref := dyn.NilValue - src := map[string]float64{ - "foo": 0.0, - "bar": 1.0, - } - - nv, err := FromTyped(src, ref) - require.NoError(t, err) - dynassert.Equal(t, dyn.V(map[string]dyn.Value{ - "foo": dyn.V(0.0), - "bar": dyn.V(1.0), - }), nv) -} - -func TestFromTypedFloatSliceWithZeroValue(t *testing.T) { - ref := dyn.NilValue - src := []float64{1.0, 0.0, 2.0} - - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.V([]dyn.Value{ - dyn.V(1.0), dyn.V(0.0), dyn.V(2.0), - }), nv) -} - -func TestFromTypedFloatStructWithZeroValue(t *testing.T) { - type Tmp struct { - Foo float64 `json:"foo"` - Bar float64 `json:"bar"` - } - - ref := dyn.NilValue - src := Tmp{ - Foo: 1.0, - Bar: 0.0, - } - - // Note, the zero value is not included in the output. - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.V(map[string]dyn.Value{ - "foo": dyn.V(1.0), - }), nv) -} - -func TestFromTypedMapNil(t *testing.T) { - var src map[string]string = nil - - ref := dyn.V(map[string]dyn.Value{ - "foo": dyn.V("bar"), - "bar": dyn.V("baz"), - }) - - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.NilValue, nv) -} - -func TestFromTypedMapEmpty(t *testing.T) { - src := map[string]string{} - - ref := dyn.V(map[string]dyn.Value{ - "foo": dyn.V("bar"), - "bar": dyn.V("baz"), - }) - - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.V(map[string]dyn.Value{}), nv) -} - -func TestFromTypedMapNonEmpty(t *testing.T) { - src := map[string]string{ - "foo": "foo", - "bar": "bar", - } - - ref := dyn.NilValue - nv, err := FromTyped(src, ref) - require.NoError(t, err) - dynassert.Equal(t, dyn.V(map[string]dyn.Value{ - "foo": dyn.V("foo"), - "bar": dyn.V("bar"), - }), nv) -} - -func TestFromTypedMapNonEmptyRetainLocation(t *testing.T) { - src := map[string]string{ - "foo": "bar", - "bar": "qux", - } - - ref := dyn.V(map[string]dyn.Value{ - "foo": dyn.NewValue("bar", []diag.Location{{File: "foo"}}), - "bar": dyn.NewValue("baz", []diag.Location{{File: "bar"}}), - }) - - nv, err := FromTyped(src, ref) - require.NoError(t, err) - - // Assert foo and bar have retained their locations. - assert.Equal(t, dyn.NewValue("bar", []diag.Location{{File: "foo"}}), nv.Get("foo")) - assert.Equal(t, dyn.NewValue("qux", []diag.Location{{File: "bar"}}), nv.Get("bar")) -} - -func TestFromTypedMapFieldWithZeroValue(t *testing.T) { - src := map[string]string{ - "foo": "", - } - - ref := dyn.NilValue - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.V(map[string]dyn.Value{ - "foo": dyn.V(""), - }), nv) -} - -func TestFromTypedSliceNil(t *testing.T) { - var src []string = nil - - ref := dyn.V([]dyn.Value{ - dyn.V("bar"), - dyn.V("baz"), - }) - - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.NilValue, nv) -} - -func TestFromTypedSliceEmpty(t *testing.T) { - //nolint:gocritic - src := []string{} - - ref := dyn.V([]dyn.Value{ - dyn.V("bar"), - dyn.V("baz"), - }) - - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.V([]dyn.Value{}), nv) -} - -func TestFromTypedSliceNonEmpty(t *testing.T) { - src := []string{ - "foo", - "bar", - } - - ref := dyn.NilValue - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.V([]dyn.Value{ - dyn.V("foo"), - dyn.V("bar"), - }), nv) -} - -func TestFromTypedSliceNonEmptyRetainLocation(t *testing.T) { - src := []string{ - "foo", - "bar", - } - - ref := dyn.V([]dyn.Value{ - dyn.NewValue("foo", []diag.Location{{File: "foo"}}), - dyn.NewValue("bar", []diag.Location{{File: "bar"}}), - }) - - nv, err := FromTyped(src, ref) - require.NoError(t, err) - - // Assert foo and bar have retained their locations. - assert.Equal(t, dyn.NewValue("foo", []diag.Location{{File: "foo"}}), nv.Index(0)) - assert.Equal(t, dyn.NewValue("bar", []diag.Location{{File: "bar"}}), nv.Index(1)) -} - -func TestFromTypedStringEmpty(t *testing.T) { - var src string - ref := dyn.NilValue - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.NilValue, nv) -} - -func TestFromTypedStringEmptyOverwrite(t *testing.T) { - var src string - ref := dyn.V("old") - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.V(""), nv) -} - -func TestFromTypedStringNonEmpty(t *testing.T) { - src := "new" - ref := dyn.NilValue - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.V("new"), nv) -} - -func TestFromTypedStringNonEmptyOverwrite(t *testing.T) { - src := "new" - ref := dyn.V("old") - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.V("new"), nv) -} - -func TestFromTypedStringRetainsLocations(t *testing.T) { - ref := dyn.NewValue("foo", []diag.Location{{File: "foo"}}) - - // case: value has not been changed - src := "foo" - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.NewValue("foo", []diag.Location{{File: "foo"}}), nv) - - // case: value has been changed - src = "bar" - nv, err = FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.NewValue("bar", []diag.Location{{File: "foo"}}), nv) -} - -func TestFromTypedStringTypeError(t *testing.T) { - src := "foo" - ref := dyn.V(1234) - _, err := FromTyped(src, ref) - require.Error(t, err) -} - -func TestFromTypedBoolEmpty(t *testing.T) { - var src bool - ref := dyn.NilValue - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.NilValue, nv) -} - -func TestFromTypedBoolEmptyOverwrite(t *testing.T) { - var src bool - ref := dyn.V(true) - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.V(false), nv) -} - -func TestFromTypedBoolNonEmpty(t *testing.T) { - src := true - ref := dyn.NilValue - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.V(true), nv) -} - -func TestFromTypedBoolNonEmptyOverwrite(t *testing.T) { - src := true - ref := dyn.V(false) - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.V(true), nv) -} - -func TestFromTypedBoolRetainsLocations(t *testing.T) { - ref := dyn.NewValue(true, []diag.Location{{File: "foo"}}) - - // case: value has not been changed - src := true - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.NewValue(true, []diag.Location{{File: "foo"}}), nv) - - // case: value has been changed - src = false - nv, err = FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.NewValue(false, []diag.Location{{File: "foo"}}), nv) -} - -func TestFromTypedBoolVariableReference(t *testing.T) { - src := true - ref := dyn.V("${var.foo}") - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.V("${var.foo}"), nv) -} - -func TestFromTypedBoolTypeError(t *testing.T) { - src := true - ref := dyn.V("string") - _, err := FromTyped(src, ref) - require.Error(t, err) -} - -func TestFromTypedIntEmpty(t *testing.T) { - var src int - ref := dyn.NilValue - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.NilValue, nv) -} - -func TestFromTypedIntEmptyOverwrite(t *testing.T) { - var src int - ref := dyn.V(1234) - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.V(int64(0)), nv) -} - -func TestFromTypedIntNonEmpty(t *testing.T) { - src := 1234 - ref := dyn.NilValue - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.V(int64(1234)), nv) -} - -func TestFromTypedIntNonEmptyOverwrite(t *testing.T) { - src := 1234 - ref := dyn.V(1233) - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.V(int64(1234)), nv) -} - -func TestFromTypedIntRetainsLocations(t *testing.T) { - ref := dyn.NewValue(1234, []diag.Location{{File: "foo"}}) - - // case: value has not been changed - src := 1234 - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.NewValue(1234, []diag.Location{{File: "foo"}}), nv) - - // case: value has been changed - src = 1235 - nv, err = FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.NewValue(int64(1235), []diag.Location{{File: "foo"}}), nv) -} - -func TestFromTypedIntVariableReference(t *testing.T) { - src := 1234 - ref := dyn.V("${var.foo}") - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.V("${var.foo}"), nv) -} - -func TestFromTypedIntTypeError(t *testing.T) { - src := 1234 - ref := dyn.V("string") - _, err := FromTyped(src, ref) - require.Error(t, err) -} - -func TestFromTypedFloatEmpty(t *testing.T) { - var src float64 - ref := dyn.NilValue - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.NilValue, nv) -} - -func TestFromTypedFloatEmptyOverwrite(t *testing.T) { - var src float64 - ref := dyn.V(1.23) - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.V(0.0), nv) -} - -func TestFromTypedFloatNonEmpty(t *testing.T) { - src := 1.23 - ref := dyn.NilValue - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.V(1.23), nv) -} - -func TestFromTypedFloatNonEmptyOverwrite(t *testing.T) { - src := 1.23 - ref := dyn.V(1.24) - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.V(1.23), nv) -} - -func TestFromTypedFloatRetainsLocations(t *testing.T) { - var src float64 - ref := dyn.NewValue(1.23, []diag.Location{{File: "foo"}}) - - // case: value has not been changed - src = 1.23 - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.NewValue(1.23, []diag.Location{{File: "foo"}}), nv) - - // case: value has been changed - src = 1.24 - nv, err = FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.NewValue(1.24, []diag.Location{{File: "foo"}}), nv) -} - -func TestFromTypedFloatVariableReference(t *testing.T) { - src := 1.23 - ref := dyn.V("${var.foo}") - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.V("${var.foo}"), nv) -} - -func TestFromTypedFloatTypeError(t *testing.T) { - src := 1.23 - ref := dyn.V("string") - _, err := FromTyped(src, ref) - require.Error(t, err) -} - -func TestFromTypedAny(t *testing.T) { - type Tmp struct { - Foo any `json:"foo"` - Bar any `json:"bar"` - Foz any `json:"foz"` - Baz any `json:"baz"` - } - - src := Tmp{ - Foo: "foo", - Bar: false, - Foz: 0, - Baz: map[string]any{ - "foo": "foo", - "bar": 1234, - "qux": 0, - "nil": nil, - }, - } - - ref := dyn.NilValue - nv, err := FromTyped(src, ref) - require.NoError(t, err) - dynassert.Equal(t, dyn.V(map[string]dyn.Value{ - "foo": dyn.V("foo"), - "bar": dyn.V(false), - "foz": dyn.V(int64(0)), - "baz": dyn.V(map[string]dyn.Value{ - "foo": dyn.V("foo"), - "bar": dyn.V(int64(1234)), - "qux": dyn.V(int64(0)), - "nil": dyn.V(nil), - }), - }), nv) -} - -func TestFromTypedAnyNil(t *testing.T) { - var src any = nil - ref := dyn.NilValue - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.NilValue, nv) -} - -func TestFromTypedNilPointerRetainsLocations(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - Bar string `json:"bar"` - } - - var src *Tmp - ref := dyn.NewValue(nil, []diag.Location{{File: "foobar"}}) - - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.NewValue(nil, []diag.Location{{File: "foobar"}}), nv) -} - -func TestFromTypedNilMapRetainsLocation(t *testing.T) { - var src map[string]string - ref := dyn.NewValue(nil, []diag.Location{{File: "foobar"}}) - - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.NewValue(nil, []diag.Location{{File: "foobar"}}), nv) -} - -func TestFromTypedNilSliceRetainsLocation(t *testing.T) { - var src []string - ref := dyn.NewValue(nil, []diag.Location{{File: "foobar"}}) - - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.NewValue(nil, []diag.Location{{File: "foobar"}}), nv) -} - -func TestFromTypedForceSendFieldsComplexTypes(t *testing.T) { - type Inner struct { - Value string `json:"value"` - } - - tests := []struct { - name string - src any - }{ - { - name: "struct_pointer", - src: struct { - Field *Inner `json:"field"` - ForceSendFields []string `json:"-"` - }{Field: nil, ForceSendFields: []string{"Field"}}, - }, - { - name: "struct_value", - src: struct { - Field Inner `json:"field"` - ForceSendFields []string `json:"-"` - }{Field: Inner{}, ForceSendFields: []string{"Field"}}, - }, - { - name: "slice", - src: struct { - Field []string `json:"field"` - ForceSendFields []string `json:"-"` - }{Field: nil, ForceSendFields: []string{"Field"}}, - }, - { - name: "map", - src: struct { - Field map[string]string `json:"field"` - ForceSendFields []string `json:"-"` - }{Field: nil, ForceSendFields: []string{"Field"}}, - }, - { - name: "interface", - src: struct { - Field any `json:"field"` - ForceSendFields []string `json:"-"` - }{Field: (*string)(nil), ForceSendFields: []string{"Field"}}, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - nv, err := FromTyped(tt.src, dyn.NilValue) - require.NoError(t, err) - - // All should include the field because it's in ForceSendFields - field := nv.Get("field") - assert.True(t, field.IsValid(), "field should be present due to ForceSendFields") - }) - } -} - -// Test embedded structs with ForceSendFields (separate test due to different structure) -func TestFromTypedForceSendFieldsEmbedded(t *testing.T) { - type Inner struct { - Field *string `json:"field"` - ForceSendFields []string `json:"-"` - } - type Outer struct { - Inner - Other string `json:"other"` - } - - src := Outer{ - Field: nil, - ForceSendFields: []string{"Field"}, - Other: "value", - } - - nv, err := FromTyped(src, dyn.NilValue) - require.NoError(t, err) - - // Both fields should be present - field := nv.Get("field") - other := nv.Get("other") - assert.True(t, field.IsValid(), "embedded field should be present due to ForceSendFields") - assert.Equal(t, dyn.KindNil, field.Kind(), "embedded field should be present due to ForceSendFields") - assert.Equal(t, dyn.V("value"), other) -} diff --git a/libs/dyn/convert/normalize.go b/libs/dyn/convert/normalize.go deleted file mode 100644 index e41ed4a0a87..00000000000 --- a/libs/dyn/convert/normalize.go +++ /dev/null @@ -1,507 +0,0 @@ -package convert - -import ( - "fmt" - "reflect" - "slices" - "strconv" - - "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/dynvar" -) - -// NormalizeOption is the type for options that can be passed to Normalize. -type NormalizeOption int - -const ( - // IncludeMissingFields causes the normalization to include fields that defined on the given - // type but are missing in the source value. They are included with their zero values. - IncludeMissingFields NormalizeOption = iota - - // DropEmptyStrings drops struct fields whose value is an empty string, unless - // the field is tagged without omitempty (i.e. its zero value must be sent). - // This mirrors JSON serialization, where an omitempty string field set to "" - // is omitted from the request. Without this, an explicitly-set "" reaches - // ToTyped, which force-sends it, and the backend rejects it (e.g. - // "'' is not a valid cluster policy ID"). - DropEmptyStrings -) - -type normalizeOptions struct { - includeMissingFields bool - dropEmptyStrings bool -} - -func Normalize(dst any, src dyn.Value, opts ...NormalizeOption) (dyn.Value, diag.Diagnostics) { - var n normalizeOptions - for _, opt := range opts { - switch opt { - case IncludeMissingFields: - n.includeMissingFields = true - case DropEmptyStrings: - n.dropEmptyStrings = true - } - } - - return n.normalizeType(reflect.TypeOf(dst), src, []reflect.Type{}, dyn.EmptyPath) -} - -func (n normalizeOptions) normalizeType(typ reflect.Type, src dyn.Value, seen []reflect.Type, path dyn.Path) (dyn.Value, diag.Diagnostics) { - for typ.Kind() == reflect.Pointer { - typ = typ.Elem() - } - - switch typ.Kind() { - case reflect.Struct: - // Handle SDK native types as strings since they use custom JSON marshaling. - if slices.Contains(sdkNativeTypes, typ) { - return n.normalizeString(reflect.TypeFor[string](), src, path) - } - return n.normalizeStruct(typ, src, append(seen, typ), path) - case reflect.Map: - return n.normalizeMap(typ, src, append(seen, typ), path) - case reflect.Slice: - return n.normalizeSlice(typ, src, append(seen, typ), path) - case reflect.String: - return n.normalizeString(typ, src, path) - case reflect.Bool: - return n.normalizeBool(typ, src, path) - case reflect.Int, reflect.Int32, reflect.Int64: - return n.normalizeInt(typ, src, path) - case reflect.Float32, reflect.Float64: - return n.normalizeFloat(typ, src, path) - case reflect.Interface: - return n.normalizeInterface(typ, src, path) - default: - // Fall through to the error case. - } - - return dyn.InvalidValue, diag.Errorf("unsupported type: %s", typ.Kind()) -} - -func nullWarning(expected dyn.Kind, src dyn.Value, path dyn.Path) diag.Diagnostic { - return diag.Diagnostic{ - Severity: diag.Warning, - Summary: fmt.Sprintf("expected a %s value, found null", expected), - Locations: []diag.Location{src.Location()}, - Paths: dyn.ToStructPaths(path), - } -} - -func typeMismatch(expected dyn.Kind, src dyn.Value, path dyn.Path) diag.Diagnostic { - return diag.Diagnostic{ - Severity: diag.Warning, - Summary: fmt.Sprintf("expected %s, found %s", expected, src.Kind()), - Locations: []diag.Location{src.Location()}, - Paths: dyn.ToStructPaths(path), - } -} - -// isAnchorContainer reports whether v is a YAML anchor or a non-empty -// sequence/map composed entirely of anchor containers. Anchors define reusable -// blocks and must not trigger "unknown field" warnings, including when nested -// inside a container. -func isAnchorContainer(v dyn.Value) bool { - if v.IsAnchor() { - return true - } - - var elements []dyn.Value - switch v.Kind() { - case dyn.KindSequence: - elements = v.MustSequence() - case dyn.KindMap: - elements = v.MustMap().Values() - default: - return false - } - - if len(elements) == 0 { - return false - } - for _, e := range elements { - if !isAnchorContainer(e) { - return false - } - } - return true -} - -func (n normalizeOptions) normalizeStruct(typ reflect.Type, src dyn.Value, seen []reflect.Type, path dyn.Path) (dyn.Value, diag.Diagnostics) { - var diags diag.Diagnostics - - switch src.Kind() { - case dyn.KindMap: - out := dyn.NewMapping() - info := getStructInfo(typ) - for _, pair := range src.MustMap().Pairs() { - pk := pair.Key - pv := pair.Value - - fieldName := pk.MustString() - index, ok := info.Fields[fieldName] - if !ok { - if !isAnchorContainer(pv) { - // Special case: provide a more helpful message for "valueFrom" vs "value_from" - if fieldName == "valueFrom" { - if _, hasValueFrom := info.Fields["value_from"]; hasValueFrom { - diags = diags.Append(diag.Diagnostic{ - Severity: diag.Warning, - Summary: "Use 'value_from' instead of 'valueFrom'", - Detail: "The field 'valueFrom' should be 'value_from' (snake_case). The 'valueFrom' field will be ignored.", - Locations: pk.Locations(), - Paths: dyn.ToStructPaths(path), - }) - continue - } - } - - diags = diags.Append(diag.Diagnostic{ - Severity: diag.Warning, - Summary: "unknown field: " + fieldName, - Locations: pk.Locations(), - Paths: dyn.ToStructPaths(path), - }) - } - continue - } - - // Normalize the value according to the field type. - nv, err := n.normalizeType(typ.FieldByIndex(index).Type, pv, seen, path.Append(dyn.Key(pk.MustString()))) - if err != nil { - diags = diags.Extend(err) - // Skip the element if it cannot be normalized. - if !nv.IsValid() { - continue - } - } - - // Drop an empty string on an omitempty field so it is not force-sent - // to the backend (see DropEmptyStrings). ForceEmpty marks fields whose - // zero value must be kept, so those are left in place. - if n.dropEmptyStrings && !info.ForceEmpty[fieldName] { - if s, ok := nv.AsString(); ok && s == "" { - continue - } - } - - out.SetLoc(pk.MustString(), pk.Locations(), nv) - } - - // Return the normalized value if missing fields are not included. - if !n.includeMissingFields { - return dyn.NewValue(out, src.Locations()), diags - } - - // Populate missing fields with their zero values. - for k, index := range info.Fields { - if _, ok := out.GetByString(k); ok { - continue - } - - // Optionally dereference pointers to get the underlying field type. - ftyp := typ.FieldByIndex(index).Type - for ftyp.Kind() == reflect.Pointer { - ftyp = ftyp.Elem() - } - - // Skip field if we have already seen its type to avoid infinite recursion - // when filling in the zero value of a recursive type. - if slices.Contains(seen, ftyp) { - continue - } - - var v dyn.Value - switch ftyp.Kind() { - case reflect.Struct, reflect.Map: - v, _ = n.normalizeType(ftyp, dyn.V(map[string]dyn.Value{}), seen, path.Append(dyn.Key(k))) - case reflect.Slice: - v, _ = n.normalizeType(ftyp, dyn.V([]dyn.Value{}), seen, path.Append(dyn.Key(k))) - case reflect.String: - v, _ = n.normalizeType(ftyp, dyn.V(""), seen, path.Append(dyn.Key(k))) - case reflect.Bool: - v, _ = n.normalizeType(ftyp, dyn.V(false), seen, path.Append(dyn.Key(k))) - case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: - v, _ = n.normalizeType(ftyp, dyn.V(int64(0)), seen, path.Append(dyn.Key(k))) - case reflect.Float32, reflect.Float64: - v, _ = n.normalizeType(ftyp, dyn.V(float64(0)), seen, path.Append(dyn.Key(k))) - default: - // Skip fields for which we do not have a natural [dyn.Value] equivalent. - // For example, we don't handle reflect.Complex* and reflect.Uint* types. - continue - } - if v.IsValid() { - out.SetLoc(k, nil, v) - } - } - - return dyn.NewValue(out, src.Locations()), diags - case dyn.KindNil: - return src, diags - - case dyn.KindString: - // Return verbatim if it's a pure variable reference. - if dynvar.IsPureVariableReference(src.MustString()) { - return src, nil - } - default: - // Fall through to the error case. - } - - // Cannot interpret as a struct. - return dyn.InvalidValue, diags.Append(typeMismatch(dyn.KindMap, src, path)) -} - -func (n normalizeOptions) normalizeMap(typ reflect.Type, src dyn.Value, seen []reflect.Type, path dyn.Path) (dyn.Value, diag.Diagnostics) { - var diags diag.Diagnostics - - switch src.Kind() { - case dyn.KindMap: - out := dyn.NewMapping() - for _, pair := range src.MustMap().Pairs() { - pk := pair.Key - pv := pair.Value - - // Normalize the value according to the map element type. - nv, err := n.normalizeType(typ.Elem(), pv, seen, path.Append(dyn.Key(pk.MustString()))) - if err != nil { - diags = diags.Extend(err) - // Skip the element if it cannot be normalized. - if !nv.IsValid() { - continue - } - } - - out.SetLoc(pk.MustString(), pk.Locations(), nv) - } - - return dyn.NewValue(out, src.Locations()), diags - case dyn.KindNil: - return src, diags - - case dyn.KindString: - // Return verbatim if it's a pure variable reference. - if dynvar.IsPureVariableReference(src.MustString()) { - return src, nil - } - default: - // Fall through to the error case. - } - - // Cannot interpret as a map. - return dyn.InvalidValue, diags.Append(typeMismatch(dyn.KindMap, src, path)) -} - -func (n normalizeOptions) normalizeSlice(typ reflect.Type, src dyn.Value, seen []reflect.Type, path dyn.Path) (dyn.Value, diag.Diagnostics) { - var diags diag.Diagnostics - - switch src.Kind() { - case dyn.KindSequence: - out := make([]dyn.Value, 0, len(src.MustSequence())) - for _, v := range src.MustSequence() { - // Normalize the value according to the slice element type. - v, err := n.normalizeType(typ.Elem(), v, seen, path.Append(dyn.Index(len(out)))) - if err != nil { - diags = diags.Extend(err) - // Skip the element if it cannot be normalized. - if !v.IsValid() { - continue - } - } - - out = append(out, v) - } - - return dyn.NewValue(out, src.Locations()), diags - case dyn.KindNil: - return src, diags - - case dyn.KindString: - // Return verbatim if it's a pure variable reference. - if dynvar.IsPureVariableReference(src.MustString()) { - return src, nil - } - default: - // Fall through to the error case. - } - - // Cannot interpret as a slice. - return dyn.InvalidValue, diags.Append(typeMismatch(dyn.KindSequence, src, path)) -} - -func (n normalizeOptions) normalizeString(typ reflect.Type, src dyn.Value, path dyn.Path) (dyn.Value, diag.Diagnostics) { - var diags diag.Diagnostics - - switch src.Kind() { - case dyn.KindString: - return dyn.NewValue(src.MustString(), src.Locations()), nil - case dyn.KindBool: - return dyn.NewValue(strconv.FormatBool(src.MustBool()), src.Locations()), nil - case dyn.KindInt: - return dyn.NewValue(strconv.FormatInt(src.MustInt(), 10), src.Locations()), nil - case dyn.KindFloat: - return dyn.NewValue(strconv.FormatFloat(src.MustFloat(), 'f', -1, 64), src.Locations()), nil - case dyn.KindTime: - return dyn.NewValue(src.MustTime().String(), src.Locations()), nil - case dyn.KindNil: - // Return a warning if the field is present but has a null value. - return dyn.InvalidValue, diags.Append(nullWarning(dyn.KindString, src, path)) - default: - // Fall through to the error case. - } - - return dyn.InvalidValue, diags.Append(typeMismatch(dyn.KindString, src, path)) -} - -func (n normalizeOptions) normalizeBool(typ reflect.Type, src dyn.Value, path dyn.Path) (dyn.Value, diag.Diagnostics) { - var diags diag.Diagnostics - - switch src.Kind() { - case dyn.KindBool: - return dyn.NewValue(src.MustBool(), src.Locations()), nil - case dyn.KindString: - // See https://yaml.org/type/bool.html. - switch src.MustString() { - case "true", "True", "TRUE", "y", "Y", "yes", "Yes", "YES", "on", "On", "ON": - return dyn.NewValue(true, src.Locations()), nil - case "false", "False", "FALSE", "n", "N", "no", "No", "NO", "off", "Off", "OFF": - return dyn.NewValue(false, src.Locations()), nil - default: - // Return verbatim if it's a pure variable reference. - if dynvar.IsPureVariableReference(src.MustString()) { - return src, nil - } - - // Cannot interpret as a boolean. - return dyn.InvalidValue, diags.Append(typeMismatch(dyn.KindBool, src, path)) - } - case dyn.KindNil: - // Return a warning if the field is present but has a null value. - return dyn.InvalidValue, diags.Append(nullWarning(dyn.KindBool, src, path)) - default: - // Fall through to the error case. - } - - return dyn.InvalidValue, diags.Append(typeMismatch(dyn.KindBool, src, path)) -} - -func (n normalizeOptions) normalizeInt(typ reflect.Type, src dyn.Value, path dyn.Path) (dyn.Value, diag.Diagnostics) { - var diags diag.Diagnostics - - switch src.Kind() { - case dyn.KindInt: - return dyn.NewValue(src.MustInt(), src.Locations()), nil - case dyn.KindFloat: - out := int64(src.MustFloat()) - if src.MustFloat() != float64(out) { - return dyn.InvalidValue, diags.Append(diag.Diagnostic{ - Severity: diag.Warning, - Summary: fmt.Sprintf(`cannot accurately represent "%g" as integer due to precision loss`, src.MustFloat()), - Locations: []diag.Location{src.Location()}, - Paths: dyn.ToStructPaths(path), - }) - } - return dyn.NewValue(out, src.Locations()), nil - case dyn.KindString: - var err error - out, err := strconv.ParseInt(src.MustString(), 10, 64) - if err != nil { - // Return verbatim if it's a pure variable reference. - if dynvar.IsPureVariableReference(src.MustString()) { - return src, nil - } - - return dyn.InvalidValue, diags.Append(diag.Diagnostic{ - Severity: diag.Warning, - Summary: fmt.Sprintf("cannot parse %q as an integer", src.MustString()), - Locations: []diag.Location{src.Location()}, - Paths: dyn.ToStructPaths(path), - }) - } - return dyn.NewValue(out, src.Locations()), nil - case dyn.KindNil: - // Return a warning if the field is present but has a null value. - return dyn.InvalidValue, diags.Append(nullWarning(dyn.KindInt, src, path)) - default: - // Fall through to the error case. - } - - return dyn.InvalidValue, diags.Append(typeMismatch(dyn.KindInt, src, path)) -} - -func (n normalizeOptions) normalizeFloat(typ reflect.Type, src dyn.Value, path dyn.Path) (dyn.Value, diag.Diagnostics) { - var diags diag.Diagnostics - - switch src.Kind() { - case dyn.KindFloat: - return dyn.NewValue(src.MustFloat(), src.Locations()), nil - case dyn.KindInt: - out := float64(src.MustInt()) - if src.MustInt() != int64(out) { - return dyn.InvalidValue, diags.Append(diag.Diagnostic{ - Severity: diag.Warning, - Summary: fmt.Sprintf(`cannot accurately represent "%d" as floating point number due to precision loss`, src.MustInt()), - Locations: []diag.Location{src.Location()}, - Paths: dyn.ToStructPaths(path), - }) - } - return dyn.NewValue(out, src.Locations()), nil - case dyn.KindString: - var err error - out, err := strconv.ParseFloat(src.MustString(), 64) - if err != nil { - // Return verbatim if it's a pure variable reference. - if dynvar.IsPureVariableReference(src.MustString()) { - return src, nil - } - - return dyn.InvalidValue, diags.Append(diag.Diagnostic{ - Severity: diag.Warning, - Summary: fmt.Sprintf("cannot parse %q as a floating point number", src.MustString()), - Locations: []diag.Location{src.Location()}, - Paths: dyn.ToStructPaths(path), - }) - } - return dyn.NewValue(out, src.Locations()), nil - case dyn.KindNil: - // Return a warning if the field is present but has a null value. - return dyn.InvalidValue, diags.Append(nullWarning(dyn.KindFloat, src, path)) - default: - // Fall through to the error case. - } - - return dyn.InvalidValue, diags.Append(typeMismatch(dyn.KindFloat, src, path)) -} - -func (n normalizeOptions) normalizeInterface(_ reflect.Type, src dyn.Value, path dyn.Path) (dyn.Value, diag.Diagnostics) { - // Deal with every [dyn.Kind] here to ensure completeness. - switch src.Kind() { - case dyn.KindMap: - // Fall through - case dyn.KindSequence: - // Fall through - case dyn.KindString: - // Fall through - case dyn.KindBool: - // Fall through - case dyn.KindInt: - // Fall through - case dyn.KindFloat: - // Fall through - case dyn.KindTime: - // Conversion of a time value to an interface{}. - // The [dyn.Value.AsAny] equivalent for this kind is the [time.Time] struct. - // If we convert to a typed representation and back again, we cannot distinguish - // a [time.Time] struct from any other struct. - // - // Therefore, we normalize the time value to a string. - return dyn.NewValue(src.MustTime().String(), src.Locations()), nil - case dyn.KindNil: - // Fall through - default: - return dyn.InvalidValue, diag.Errorf("unsupported kind: %s", src.Kind()) - } - - return src, nil -} diff --git a/libs/dyn/convert/normalize_test.go b/libs/dyn/convert/normalize_test.go deleted file mode 100644 index ac95aaafac6..00000000000 --- a/libs/dyn/convert/normalize_test.go +++ /dev/null @@ -1,1018 +0,0 @@ -package convert - -import ( - "testing" - - "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/dynassert" - "github.com/databricks/cli/libs/structs/structpath" - "github.com/stretchr/testify/assert" -) - -func TestNormalizeStruct(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - Bar string `json:"bar"` - } - - var typ Tmp - vin := dyn.V(map[string]dyn.Value{ - "foo": dyn.V("bar"), - "bar": dyn.V("baz"), - }) - - vout, diags := Normalize(typ, vin) - assert.Empty(t, diags) - assert.Equal(t, vin, vout) -} - -func TestNormalizeStructElementDiagnostic(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - Bar string `json:"bar"` - } - - var typ Tmp - vin := dyn.V(map[string]dyn.Value{ - "foo": dyn.V("bar"), - "bar": dyn.V(map[string]dyn.Value{"an": dyn.V("error")}), - }) - - vout, diags := Normalize(typ, vin) - assert.Len(t, diags, 1) - assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected string, found map`, - Locations: []diag.Location{{}}, - Paths: structpath.NewPathSlice("bar"), - }, diags[0]) - - // Elements that encounter an error during normalization are dropped. - assert.Equal(t, map[string]any{ - "foo": "bar", - }, vout.AsAny()) -} - -func TestNormalizeStructUnknownField(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - } - - var typ Tmp - - m := dyn.NewMapping() - m.SetLoc("foo", nil, dyn.V("val-foo")) - - // Set the unknown field, with location information. - m.SetLoc("bar", []diag.Location{ - {File: "hello.yaml", Line: 1, Column: 1}, - {File: "world.yaml", Line: 2, Column: 2}, - }, dyn.V("var-bar")) - - vin := dyn.V(m) - - vout, diags := Normalize(typ, vin) - assert.Len(t, diags, 1) - assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `unknown field: bar`, - // Assert location of the unknown field is included in the diagnostic. - Locations: []diag.Location{ - {File: "hello.yaml", Line: 1, Column: 1}, - {File: "world.yaml", Line: 2, Column: 2}, - }, - Paths: dyn.ToStructPaths(dyn.EmptyPath), - }, diags[0]) - - // The field that can be mapped to the struct field is retained. - assert.Equal(t, map[string]any{ - "foo": "val-foo", - }, vout.AsAny()) -} - -func TestNormalizeStructNil(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - } - - var typ Tmp - vin := dyn.NilValue - vout, err := Normalize(typ, vin) - assert.Empty(t, err) - assert.Equal(t, vin, vout) -} - -func TestNormalizeStructError(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - } - - var typ Tmp - vin := dyn.V("string") - _, err := Normalize(typ, vin) - assert.Len(t, err, 1) - assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected map, found string`, - Locations: []diag.Location{vin.Get("foo").Location()}, - Paths: dyn.ToStructPaths(dyn.EmptyPath), - }, err[0]) -} - -func TestNormalizeStructNestedError(t *testing.T) { - type Nested struct { - F1 int `json:"f1"` - F2 int `json:"f2"` - } - type Tmp struct { - Foo Nested `json:"foo"` - Bar Nested `json:"bar"` - } - - var typ Tmp - vin := dyn.V(map[string]dyn.Value{ - "foo": dyn.V(map[string]dyn.Value{ - "f1": dyn.V("error"), - "f2": dyn.V(1), - }), - "bar": dyn.V(map[string]dyn.Value{ - "f1": dyn.V(1), - "f2": dyn.V("error"), - }), - }) - vout, err := Normalize(typ, vin) - assert.Len(t, err, 2) - - // Verify that valid fields are retained. - dynassert.Equal(t, - dyn.V(map[string]dyn.Value{ - "foo": dyn.V(map[string]dyn.Value{ - "f2": dyn.V(int64(1)), - }), - "bar": dyn.V(map[string]dyn.Value{ - "f1": dyn.V(int64(1)), - }), - }), - vout, - ) -} - -func TestNormalizeStructIncludeMissingFields(t *testing.T) { - type Nested struct { - String string `json:"string"` - } - - type Tmp struct { - // Verify that fields that are already set in the dynamic value are not overridden. - Existing string `json:"existing"` - - // Verify that structs are recursively normalized if not set. - Nested Nested `json:"nested"` - Ptr *Nested `json:"ptr"` - - // Verify that containers are also zero-initialized if not set. - Map map[string]string `json:"map"` - Slice []string `json:"slice"` - - // Verify that primitive types are zero-initialized if not set. - String string `json:"string"` - Bool bool `json:"bool"` - Int int `json:"int"` - Float float64 `json:"float"` - } - - var typ Tmp - vin := dyn.V(map[string]dyn.Value{ - "existing": dyn.V("already set"), - }) - vout, err := Normalize(typ, vin, IncludeMissingFields) - assert.Empty(t, err) - dynassert.Equal(t, dyn.V(map[string]dyn.Value{ - "existing": dyn.V("already set"), - "nested": dyn.V(map[string]dyn.Value{ - "string": dyn.V(""), - }), - "ptr": dyn.V(map[string]dyn.Value{ - "string": dyn.V(""), - }), - "map": dyn.V(map[string]dyn.Value{}), - "slice": dyn.V([]dyn.Value{}), - "string": dyn.V(""), - "bool": dyn.V(false), - "int": dyn.V(int64(0)), - "float": dyn.V(float64(0)), - }), vout) -} - -func TestNormalizeStructIncludeMissingFieldsOnRecursiveType(t *testing.T) { - type Tmp struct { - // Verify that structs are recursively normalized if not set. - Ptr *Tmp `json:"ptr"` - - // Verify that primitive types are zero-initialized if not set. - String string `json:"string"` - } - - var typ Tmp - vin := dyn.V(map[string]dyn.Value{ - "ptr": dyn.V(map[string]dyn.Value{ - "ptr": dyn.V(map[string]dyn.Value{ - "string": dyn.V("already set"), - }), - }), - }) - vout, err := Normalize(typ, vin, IncludeMissingFields) - assert.Empty(t, err) - dynassert.Equal(t, dyn.V(map[string]dyn.Value{ - "ptr": dyn.V(map[string]dyn.Value{ - "ptr": dyn.V(map[string]dyn.Value{ - // Note: the ptr field is not zero-initialized because that would recurse. - "string": dyn.V("already set"), - }), - "string": dyn.V(""), - }), - "string": dyn.V(""), - }), vout) -} - -func TestNormalizeStructVariableReference(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - } - - var typ Tmp - vin := dyn.NewValue("${var.foo}", []diag.Location{{File: "file", Line: 1, Column: 1}}) - vout, err := Normalize(typ, vin) - assert.Empty(t, err) - assert.Equal(t, vin, vout) -} - -func TestNormalizeStructRandomStringError(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - } - - var typ Tmp - vin := dyn.NewValue("var foo", []diag.Location{{File: "file", Line: 1, Column: 1}}) - _, err := Normalize(typ, vin) - assert.Len(t, err, 1) - assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected map, found string`, - Locations: []diag.Location{vin.Location()}, - Paths: dyn.ToStructPaths(dyn.EmptyPath), - }, err[0]) -} - -func TestNormalizeStructIntError(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - } - - var typ Tmp - vin := dyn.NewValue(1, []diag.Location{{File: "file", Line: 1, Column: 1}}) - _, err := Normalize(typ, vin) - assert.Len(t, err, 1) - assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected map, found int`, - Locations: []diag.Location{vin.Location()}, - Paths: dyn.ToStructPaths(dyn.EmptyPath), - }, err[0]) -} - -func TestNormalizeMap(t *testing.T) { - var typ map[string]string - vin := dyn.V(map[string]dyn.Value{ - "foo": dyn.V("bar"), - "bar": dyn.V("baz"), - }) - - vout, err := Normalize(typ, vin) - assert.Empty(t, err) - assert.Equal(t, vin, vout) -} - -func TestNormalizeMapElementDiagnostic(t *testing.T) { - var typ map[string]string - vin := dyn.V(map[string]dyn.Value{ - "foo": dyn.V("bar"), - "bar": dyn.V(map[string]dyn.Value{"an": dyn.V("error")}), - }) - - vout, err := Normalize(typ, vin) - assert.Len(t, err, 1) - assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected string, found map`, - Locations: []diag.Location{{}}, - Paths: structpath.NewPathSlice("bar"), - }, err[0]) - - // Elements that encounter an error during normalization are dropped. - assert.Equal(t, map[string]any{ - "foo": "bar", - }, vout.AsAny()) -} - -func TestNormalizeMapNil(t *testing.T) { - var typ map[string]string - vin := dyn.NilValue - vout, err := Normalize(typ, vin) - assert.Empty(t, err) - assert.Equal(t, vin, vout) -} - -func TestNormalizeMapError(t *testing.T) { - var typ map[string]string - vin := dyn.V("string") - _, err := Normalize(typ, vin) - assert.Len(t, err, 1) - assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected map, found string`, - Locations: []diag.Location{vin.Location()}, - Paths: dyn.ToStructPaths(dyn.EmptyPath), - }, err[0]) -} - -func TestNormalizeMapNestedError(t *testing.T) { - type Nested struct { - F1 int `json:"f1"` - F2 int `json:"f2"` - } - - var typ map[string]Nested - vin := dyn.V(map[string]dyn.Value{ - "foo": dyn.V(map[string]dyn.Value{ - "f1": dyn.V("error"), - "f2": dyn.V(1), - }), - "bar": dyn.V(map[string]dyn.Value{ - "f1": dyn.V(1), - "f2": dyn.V("error"), - }), - }) - vout, err := Normalize(typ, vin) - assert.Len(t, err, 2) - - // Verify that valid fields are retained. - dynassert.Equal(t, - dyn.V(map[string]dyn.Value{ - "foo": dyn.V(map[string]dyn.Value{ - "f2": dyn.V(int64(1)), - }), - "bar": dyn.V(map[string]dyn.Value{ - "f1": dyn.V(int64(1)), - }), - }), - vout, - ) -} - -func TestNormalizeMapVariableReference(t *testing.T) { - var typ map[string]string - vin := dyn.NewValue("${var.foo}", []diag.Location{{File: "file", Line: 1, Column: 1}}) - vout, err := Normalize(typ, vin) - assert.Empty(t, err) - assert.Equal(t, vin, vout) -} - -func TestNormalizeMapRandomStringError(t *testing.T) { - var typ map[string]string - vin := dyn.NewValue("var foo", []diag.Location{{File: "file", Line: 1, Column: 1}}) - _, err := Normalize(typ, vin) - assert.Len(t, err, 1) - assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected map, found string`, - Locations: []diag.Location{vin.Location()}, - Paths: dyn.ToStructPaths(dyn.EmptyPath), - }, err[0]) -} - -func TestNormalizeMapIntError(t *testing.T) { - var typ map[string]string - vin := dyn.NewValue(1, []diag.Location{{File: "file", Line: 1, Column: 1}}) - _, err := Normalize(typ, vin) - assert.Len(t, err, 1) - assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected map, found int`, - Locations: []diag.Location{vin.Location()}, - Paths: dyn.ToStructPaths(dyn.EmptyPath), - }, err[0]) -} - -func TestNormalizeSlice(t *testing.T) { - var typ []string - vin := dyn.V([]dyn.Value{ - dyn.V("foo"), - dyn.V("bar"), - }) - - vout, err := Normalize(typ, vin) - assert.Empty(t, err) - assert.Equal(t, vin, vout) -} - -func TestNormalizeSliceElementDiagnostic(t *testing.T) { - var typ []string - vin := dyn.V([]dyn.Value{ - dyn.V("foo"), - dyn.V("bar"), - dyn.V(map[string]dyn.Value{"an": dyn.V("error")}), - }) - - vout, err := Normalize(typ, vin) - assert.Len(t, err, 1) - assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected string, found map`, - Locations: []diag.Location{{}}, - Paths: []*structpath.PathNode{structpath.NewIndex(nil, 2)}, - }, err[0]) - - // Elements that encounter an error during normalization are dropped. - assert.Equal(t, []any{"foo", "bar"}, vout.AsAny()) -} - -func TestNormalizeSliceNil(t *testing.T) { - var typ []string - vin := dyn.NilValue - vout, err := Normalize(typ, vin) - assert.Empty(t, err) - assert.Equal(t, vin, vout) -} - -func TestNormalizeSliceError(t *testing.T) { - var typ []string - vin := dyn.V("string") - _, err := Normalize(typ, vin) - assert.Len(t, err, 1) - assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected sequence, found string`, - Locations: []diag.Location{vin.Location()}, - Paths: dyn.ToStructPaths(dyn.EmptyPath), - }, err[0]) -} - -func TestNormalizeSliceNestedError(t *testing.T) { - type Nested struct { - F1 int `json:"f1"` - F2 int `json:"f2"` - } - - var typ []Nested - vin := dyn.V([]dyn.Value{ - dyn.V(map[string]dyn.Value{ - "f1": dyn.V("error"), - "f2": dyn.V(1), - }), - dyn.V(map[string]dyn.Value{ - "f1": dyn.V(1), - "f2": dyn.V("error"), - }), - }) - vout, err := Normalize(typ, vin) - assert.Len(t, err, 2) - - // Verify that valid fields are retained. - dynassert.Equal(t, - dyn.V([]dyn.Value{ - dyn.V(map[string]dyn.Value{ - "f2": dyn.V(int64(1)), - }), - dyn.V(map[string]dyn.Value{ - "f1": dyn.V(int64(1)), - }), - }), - vout, - ) -} - -func TestNormalizeSliceVariableReference(t *testing.T) { - var typ []string - vin := dyn.NewValue("${var.foo}", []diag.Location{{File: "file", Line: 1, Column: 1}}) - vout, err := Normalize(typ, vin) - assert.Empty(t, err) - assert.Equal(t, vin, vout) -} - -func TestNormalizeSliceRandomStringError(t *testing.T) { - var typ []string - vin := dyn.NewValue("var foo", []diag.Location{{File: "file", Line: 1, Column: 1}}) - _, err := Normalize(typ, vin) - assert.Len(t, err, 1) - assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected sequence, found string`, - Locations: []diag.Location{vin.Location()}, - Paths: dyn.ToStructPaths(dyn.EmptyPath), - }, err[0]) -} - -func TestNormalizeSliceIntError(t *testing.T) { - var typ []string - vin := dyn.NewValue(1, []diag.Location{{File: "file", Line: 1, Column: 1}}) - _, err := Normalize(typ, vin) - assert.Len(t, err, 1) - assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected sequence, found int`, - Locations: []diag.Location{vin.Location()}, - Paths: dyn.ToStructPaths(dyn.EmptyPath), - }, err[0]) -} - -func TestNormalizeString(t *testing.T) { - var typ string - vin := dyn.V("string") - vout, err := Normalize(&typ, vin) - assert.Empty(t, err) - assert.Equal(t, vin, vout) -} - -func TestNormalizeStringNil(t *testing.T) { - var typ string - vin := dyn.NewValue(nil, []diag.Location{{File: "file", Line: 1, Column: 1}}) - _, err := Normalize(&typ, vin) - assert.Len(t, err, 1) - assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected a string value, found null`, - Locations: []diag.Location{vin.Location()}, - Paths: dyn.ToStructPaths(dyn.EmptyPath), - }, err[0]) -} - -func TestNormalizeStringFromBool(t *testing.T) { - var typ string - vin := dyn.NewValue(true, []diag.Location{{File: "file", Line: 1, Column: 1}}) - vout, err := Normalize(&typ, vin) - assert.Empty(t, err) - assert.Equal(t, dyn.NewValue("true", vin.Locations()), vout) -} - -func TestNormalizeStringFromInt(t *testing.T) { - var typ string - vin := dyn.NewValue(123, []diag.Location{{File: "file", Line: 1, Column: 1}}) - vout, err := Normalize(&typ, vin) - assert.Empty(t, err) - assert.Equal(t, dyn.NewValue("123", vin.Locations()), vout) -} - -func TestNormalizeStringFromFloat(t *testing.T) { - var typ string - vin := dyn.NewValue(1.20, []diag.Location{{File: "file", Line: 1, Column: 1}}) - vout, err := Normalize(&typ, vin) - assert.Empty(t, err) - assert.Equal(t, dyn.NewValue("1.2", vin.Locations()), vout) -} - -func TestNormalizeStringFromTime(t *testing.T) { - var typ string - vin := dyn.NewValue(dyn.MustTime("2024-08-29"), []diag.Location{{File: "file", Line: 1, Column: 1}}) - vout, err := Normalize(&typ, vin) - assert.Empty(t, err) - assert.Equal(t, dyn.NewValue("2024-08-29", vin.Locations()), vout) -} - -func TestNormalizeStringError(t *testing.T) { - var typ string - vin := dyn.V(map[string]dyn.Value{"an": dyn.V("error")}) - _, err := Normalize(&typ, vin) - assert.Len(t, err, 1) - assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected string, found map`, - Locations: []diag.Location{{}}, - Paths: dyn.ToStructPaths(dyn.EmptyPath), - }, err[0]) -} - -func TestNormalizeBool(t *testing.T) { - var typ bool - vin := dyn.V(true) - vout, err := Normalize(&typ, vin) - assert.Empty(t, err) - assert.Equal(t, dyn.V(true), vout) -} - -func TestNormalizeBoolNil(t *testing.T) { - var typ bool - vin := dyn.NewValue(nil, []diag.Location{{File: "file", Line: 1, Column: 1}}) - _, err := Normalize(&typ, vin) - assert.Len(t, err, 1) - assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected a bool value, found null`, - Locations: []diag.Location{vin.Location()}, - Paths: dyn.ToStructPaths(dyn.EmptyPath), - }, err[0]) -} - -func TestNormalizeBoolFromString(t *testing.T) { - var typ bool - - for _, c := range []struct { - Input string - Output bool - }{ - {"true", true}, - {"false", false}, - {"True", true}, - {"False", false}, - {"TRUE", true}, - {"FALSE", false}, - {"Y", true}, - {"N", false}, - {"on", true}, - {"off", false}, - } { - vin := dyn.V(c.Input) - vout, err := Normalize(&typ, vin) - assert.Empty(t, err) - assert.Equal(t, dyn.V(c.Output), vout) - } -} - -func TestNormalizeBoolFromStringVariableReference(t *testing.T) { - var typ bool - vin := dyn.V("${var.foo}") - vout, err := Normalize(&typ, vin) - assert.Empty(t, err) - assert.Equal(t, vin, vout) -} - -func TestNormalizeBoolFromStringError(t *testing.T) { - var typ bool - vin := dyn.V("abc") - _, err := Normalize(&typ, vin) - assert.Len(t, err, 1) - assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected bool, found string`, - Locations: []diag.Location{vin.Location()}, - Paths: dyn.ToStructPaths(dyn.EmptyPath), - }, err[0]) -} - -func TestNormalizeBoolError(t *testing.T) { - var typ bool - vin := dyn.V(map[string]dyn.Value{"an": dyn.V("error")}) - _, err := Normalize(&typ, vin) - assert.Len(t, err, 1) - assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected bool, found map`, - Locations: []diag.Location{{}}, - Paths: dyn.ToStructPaths(dyn.EmptyPath), - }, err[0]) -} - -func TestNormalizeInt(t *testing.T) { - var typ int - vin := dyn.V(123) - vout, err := Normalize(&typ, vin) - assert.Empty(t, err) - assert.Equal(t, dyn.V(int64(123)), vout) -} - -func TestNormalizeIntNil(t *testing.T) { - var typ int - vin := dyn.NewValue(nil, []diag.Location{{File: "file", Line: 1, Column: 1}}) - _, err := Normalize(&typ, vin) - assert.Len(t, err, 1) - assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected a int value, found null`, - Locations: []diag.Location{vin.Location()}, - Paths: dyn.ToStructPaths(dyn.EmptyPath), - }, err[0]) -} - -func TestNormalizeIntFromFloat(t *testing.T) { - var typ int - vin := dyn.V(float64(1.0)) - vout, err := Normalize(&typ, vin) - assert.Empty(t, err) - assert.Equal(t, dyn.V(int64(1)), vout) -} - -func TestNormalizeIntFromFloatError(t *testing.T) { - var typ int - vin := dyn.V(1.5) - _, err := Normalize(&typ, vin) - assert.Len(t, err, 1) - assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `cannot accurately represent "1.5" as integer due to precision loss`, - Locations: []diag.Location{vin.Location()}, - Paths: dyn.ToStructPaths(dyn.EmptyPath), - }, err[0]) -} - -func TestNormalizeIntFromString(t *testing.T) { - var typ int - vin := dyn.V("123") - vout, err := Normalize(&typ, vin) - assert.Empty(t, err) - assert.Equal(t, dyn.V(int64(123)), vout) -} - -func TestNormalizeIntFromStringVariableReference(t *testing.T) { - var typ int - vin := dyn.V("${var.foo}") - vout, err := Normalize(&typ, vin) - assert.Empty(t, err) - assert.Equal(t, vin, vout) -} - -func TestNormalizeIntFromStringError(t *testing.T) { - var typ int - vin := dyn.V("abc") - _, err := Normalize(&typ, vin) - assert.Len(t, err, 1) - assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `cannot parse "abc" as an integer`, - Locations: []diag.Location{vin.Location()}, - Paths: dyn.ToStructPaths(dyn.EmptyPath), - }, err[0]) -} - -func TestNormalizeIntError(t *testing.T) { - var typ int - vin := dyn.V(map[string]dyn.Value{"an": dyn.V("error")}) - _, err := Normalize(&typ, vin) - assert.Len(t, err, 1) - assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected int, found map`, - Locations: []diag.Location{{}}, - Paths: dyn.ToStructPaths(dyn.EmptyPath), - }, err[0]) -} - -func TestNormalizeFloat(t *testing.T) { - var typ float64 - vin := dyn.V(1.2) - vout, err := Normalize(&typ, vin) - assert.Empty(t, err) - assert.Equal(t, dyn.V(1.2), vout) -} - -func TestNormalizeFloatNil(t *testing.T) { - var typ float64 - vin := dyn.NewValue(nil, []diag.Location{{File: "file", Line: 1, Column: 1}}) - _, err := Normalize(&typ, vin) - assert.Len(t, err, 1) - assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected a float value, found null`, - Locations: []diag.Location{vin.Location()}, - Paths: dyn.ToStructPaths(dyn.EmptyPath), - }, err[0]) -} - -func TestNormalizeFloatFromInt(t *testing.T) { - var typ float64 - - // Maximum safe integer that can be accurately represented as a float. - vin := dyn.V(int64(9007199254740992)) - vout, err := Normalize(&typ, vin) - assert.Empty(t, err) - assert.Equal(t, dyn.V(float64(9007199254740992)), vout) -} - -func TestNormalizeFloatFromIntError(t *testing.T) { - var typ float64 - - // Minimum integer that cannot be accurately represented as a float. - vin := dyn.V(9007199254740992 + 1) - _, err := Normalize(&typ, vin) - assert.Len(t, err, 1) - assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `cannot accurately represent "9007199254740993" as floating point number due to precision loss`, - Locations: []diag.Location{vin.Location()}, - Paths: dyn.ToStructPaths(dyn.EmptyPath), - }, err[0]) -} - -func TestNormalizeFloatFromString(t *testing.T) { - var typ float64 - vin := dyn.V("1.2") - vout, err := Normalize(&typ, vin) - assert.Empty(t, err) - assert.Equal(t, dyn.V(1.2), vout) -} - -func TestNormalizeFloatFromStringVariableReference(t *testing.T) { - var typ float64 - vin := dyn.V("${var.foo}") - vout, err := Normalize(&typ, vin) - assert.Empty(t, err) - assert.Equal(t, vin, vout) -} - -func TestNormalizeFloatFromStringError(t *testing.T) { - var typ float64 - vin := dyn.V("abc") - _, err := Normalize(&typ, vin) - assert.Len(t, err, 1) - assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `cannot parse "abc" as a floating point number`, - Locations: []diag.Location{vin.Location()}, - Paths: dyn.ToStructPaths(dyn.EmptyPath), - }, err[0]) -} - -func TestNormalizeFloatError(t *testing.T) { - var typ float64 - vin := dyn.V(map[string]dyn.Value{"an": dyn.V("error")}) - _, err := Normalize(&typ, vin) - assert.Len(t, err, 1) - assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected float, found map`, - Locations: []diag.Location{{}}, - Paths: dyn.ToStructPaths(dyn.EmptyPath), - }, err[0]) -} - -func TestNormalizeAnchors(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - } - - var typ Tmp - vin := dyn.V(map[string]dyn.Value{ - "foo": dyn.V("bar"), - "anchor": dyn.V("anchor").MarkAnchor(), - }) - - vout, err := Normalize(typ, vin) - assert.Empty(t, err) - - // The field that can be mapped to the struct field is retained. - assert.Equal(t, map[string]any{ - "foo": "bar", - }, vout.AsAny()) -} - -func TestNormalizeAnchorContainers(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - } - - anchor := func() dyn.Value { - return dyn.V(map[string]dyn.Value{"name": dyn.V("x")}).MarkAnchor() - } - - tcases := []struct { - name string - value dyn.Value - wantWarn bool - }{ - { - name: "list of anchors", - value: dyn.V([]dyn.Value{anchor(), anchor()}), - wantWarn: false, - }, - { - name: "map of anchors", - value: dyn.V(map[string]dyn.Value{ - "a": anchor(), - "b": anchor(), - }), - wantWarn: false, - }, - { - name: "nested list of anchors", - value: dyn.V([]dyn.Value{dyn.V([]dyn.Value{anchor()})}), - wantWarn: false, - }, - { - name: "list with a non-anchor element", - value: dyn.V([]dyn.Value{anchor(), dyn.V(map[string]dyn.Value{"name": dyn.V("x")})}), - wantWarn: true, - }, - { - name: "empty list", - value: dyn.V([]dyn.Value{}), - wantWarn: true, - }, - { - name: "empty map", - value: dyn.V(map[string]dyn.Value{}), - wantWarn: true, - }, - } - - for _, tc := range tcases { - t.Run(tc.name, func(t *testing.T) { - var typ Tmp - vin := dyn.V(map[string]dyn.Value{ - "foo": dyn.V("bar"), - "thing": tc.value, - }) - - vout, diags := Normalize(typ, vin) - if tc.wantWarn { - assert.Len(t, diags, 1) - assert.Equal(t, "unknown field: thing", diags[0].Summary) - } else { - assert.Empty(t, diags) - } - - // The unknown field is never retained regardless of the warning. - assert.Equal(t, map[string]any{ - "foo": "bar", - }, vout.AsAny()) - }) - } -} - -func TestNormalizeAnyFromSlice(t *testing.T) { - var typ any - v1 := dyn.NewValue(1, []diag.Location{{File: "file", Line: 1, Column: 1}}) - v2 := dyn.NewValue(2, []diag.Location{{File: "file", Line: 1, Column: 1}}) - vin := dyn.NewValue([]dyn.Value{v1, v2}, []diag.Location{{File: "file", Line: 1, Column: 1}}) - vout, err := Normalize(&typ, vin) - assert.Empty(t, err) - assert.Equal(t, dyn.NewValue([]dyn.Value{v1, v2}, []diag.Location{{File: "file", Line: 1, Column: 1}}), vout) -} - -func TestNormalizeAnyFromString(t *testing.T) { - var typ any - vin := dyn.NewValue("string", []diag.Location{{File: "file", Line: 1, Column: 1}}) - vout, err := Normalize(&typ, vin) - assert.Empty(t, err) - assert.Equal(t, dyn.NewValue("string", []diag.Location{{File: "file", Line: 1, Column: 1}}), vout) -} - -func TestNormalizeAnyFromBool(t *testing.T) { - var typ any - vin := dyn.NewValue(false, []diag.Location{{File: "file", Line: 1, Column: 1}}) - vout, err := Normalize(&typ, vin) - assert.Empty(t, err) - assert.Equal(t, dyn.NewValue(false, []diag.Location{{File: "file", Line: 1, Column: 1}}), vout) -} - -func TestNormalizeAnyFromInt(t *testing.T) { - var typ any - vin := dyn.NewValue(10, []diag.Location{{File: "file", Line: 1, Column: 1}}) - vout, err := Normalize(&typ, vin) - assert.Empty(t, err) - assert.Equal(t, dyn.NewValue(10, []diag.Location{{File: "file", Line: 1, Column: 1}}), vout) -} - -func TestNormalizeAnyFromTime(t *testing.T) { - var typ any - vin := dyn.NewValue(dyn.MustTime("2024-08-29"), []diag.Location{{File: "file", Line: 1, Column: 1}}) - vout, err := Normalize(&typ, vin) - assert.Empty(t, err) - assert.Equal(t, dyn.NewValue("2024-08-29", vin.Locations()), vout) -} - -func TestNormalizeStructDropEmptyStrings(t *testing.T) { - type Tmp struct { - // Optional: dropped when empty. - Opt string `json:"opt,omitempty"` - // Required (no omitempty): kept even when empty. - Req string `json:"req"` - } - - vin := dyn.V(map[string]dyn.Value{ - "opt": dyn.V(""), - "req": dyn.V(""), - }) - - vout, diags := Normalize(Tmp{}, vin, DropEmptyStrings) - assert.Empty(t, diags) - - _, hasOpt := vout.MustMap().Get(dyn.V("opt")) - assert.False(t, hasOpt, "empty omitempty string should be dropped") - req, hasReq := vout.MustMap().Get(dyn.V("req")) - assert.True(t, hasReq, "empty non-omitempty string should be kept") - assert.Empty(t, req.MustString()) -} - -func TestNormalizeStructDropEmptyStringsKeepsNonEmpty(t *testing.T) { - type Tmp struct { - Opt string `json:"opt,omitempty"` - } - - vin := dyn.V(map[string]dyn.Value{"opt": dyn.V("value")}) - - vout, diags := Normalize(Tmp{}, vin, DropEmptyStrings) - assert.Empty(t, diags) - opt, ok := vout.MustMap().Get(dyn.V("opt")) - assert.True(t, ok) - assert.Equal(t, "value", opt.MustString()) -} diff --git a/libs/dyn/convert/sdk_native_types.go b/libs/dyn/convert/sdk_native_types.go deleted file mode 100644 index b67d9b2b1e3..00000000000 --- a/libs/dyn/convert/sdk_native_types.go +++ /dev/null @@ -1,97 +0,0 @@ -package convert - -import ( - "encoding/json" - "fmt" - "reflect" - "slices" - - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/dynvar" - sdkduration "github.com/databricks/databricks-sdk-go/common/types/duration" - sdkfieldmask "github.com/databricks/databricks-sdk-go/common/types/fieldmask" - sdktime "github.com/databricks/databricks-sdk-go/common/types/time" -) - -// sdkNativeTypes is a list of SDK native types that use custom JSON marshaling -// and should be treated as strings in dyn.Value. These types all implement -// json.Marshaler and json.Unmarshaler interfaces. -var sdkNativeTypes = []reflect.Type{ - reflect.TypeFor[sdkduration.Duration](), // Protobuf duration format (e.g., "300s") - reflect.TypeFor[sdktime.Time](), // RFC3339 timestamp format (e.g., "2023-12-25T10:30:00Z") - reflect.TypeFor[sdkfieldmask.FieldMask](), // Comma-separated paths (e.g., "name,age,email") -} - -// fromTypedSDKNative converts SDK native types to dyn.Value. -// SDK native types (duration.Duration, time.Time, fieldmask.FieldMask) use -// custom JSON marshaling with string representations. -func fromTypedSDKNative(src reflect.Value, ref dyn.Value, options ...fromTypedOptions) (dyn.Value, error) { - // Check that the reference value is compatible or nil. - switch ref.Kind() { - case dyn.KindString: - // Ignore pure variable references (e.g. ${var.foo}). - if dynvar.IsPureVariableReference(ref.MustString()) { - return ref, nil - } - case dyn.KindNil: - // Allow nil reference. - default: - return dyn.InvalidValue, fmt.Errorf("cannot convert SDK native type to dynamic type %#v", ref.Kind().String()) - } - - // Check for zero value first. - if src.IsZero() && !slices.Contains(options, includeZeroValues) { - return dyn.NilValue, nil - } - - // Use JSON marshaling since SDK native types implement json.Marshaler. - jsonBytes, err := json.Marshal(src.Interface()) - if err != nil { - return dyn.InvalidValue, err - } - - // All SDK native types marshal to JSON strings. Unmarshal to get the raw string value. - // For example: duration.Duration(300s) -> JSON "300s" -> string "300s" - var str string - if err := json.Unmarshal(jsonBytes, &str); err != nil { - return dyn.InvalidValue, err - } - - // Handle empty string as zero value. - if str == "" && !slices.Contains(options, includeZeroValues) { - return dyn.NilValue, nil - } - - return dyn.V(str), nil -} - -// toTypedSDKNative converts a dyn.Value to an SDK native type. -// SDK native types (duration.Duration, time.Time, fieldmask.FieldMask) use -// custom JSON marshaling with string representations. -func toTypedSDKNative(dst reflect.Value, src dyn.Value) error { - switch src.Kind() { - case dyn.KindString: - // Ignore pure variable references (e.g. ${var.foo}). - if dynvar.IsPureVariableReference(src.MustString()) { - dst.SetZero() - return nil - } - // Use JSON unmarshaling since SDK native types implement json.Unmarshaler. - // Marshal the string to create a valid JSON string literal for unmarshaling. - jsonBytes, err := json.Marshal(src.MustString()) - if err != nil { - return err - } - return json.Unmarshal(jsonBytes, dst.Addr().Interface()) - case dyn.KindNil: - dst.SetZero() - return nil - default: - // Fall through to the error case. - } - - return TypeError{ - value: src, - msg: fmt.Sprintf("expected a string, found a %s", src.Kind()), - } -} diff --git a/libs/dyn/convert/sdk_native_types_test.go b/libs/dyn/convert/sdk_native_types_test.go deleted file mode 100644 index cd3d14fa129..00000000000 --- a/libs/dyn/convert/sdk_native_types_test.go +++ /dev/null @@ -1,417 +0,0 @@ -package convert - -import ( - "reflect" - "testing" - "time" - - "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - sdkduration "github.com/databricks/databricks-sdk-go/common/types/duration" - sdkfieldmask "github.com/databricks/databricks-sdk-go/common/types/fieldmask" - sdktime "github.com/databricks/databricks-sdk-go/common/types/time" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// Roundtrip tests - verify SDK native types convert to dyn.Value and back for both value and pointer types - -func TestDurationRoundtrip(t *testing.T) { - tests := []struct { - name string - duration time.Duration - expectedString string - }{ - {"5min", 5 * time.Minute, "300s"}, - {"7days", 7 * 24 * time.Hour, "604800s"}, - {"1hour", 1 * time.Hour, "3600s"}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - // Test value type - t.Run("value", func(t *testing.T) { - src := *sdkduration.New(tt.duration) - dynValue, err := FromTyped(src, dyn.NilValue) - require.NoError(t, err) - assert.Equal(t, tt.expectedString, dynValue.MustString()) - - var out sdkduration.Duration - err = ToTyped(&out, dynValue) - require.NoError(t, err) - assert.Equal(t, src.AsDuration(), out.AsDuration()) - }) - - // Test pointer type - t.Run("pointer", func(t *testing.T) { - src := *sdkduration.New(tt.duration) - dynValue, err := FromTyped(&src, dyn.NilValue) - require.NoError(t, err) - assert.Equal(t, tt.expectedString, dynValue.MustString()) - - var out *sdkduration.Duration - err = ToTyped(&out, dynValue) - require.NoError(t, err) - require.NotNil(t, out) - assert.Equal(t, src.AsDuration(), out.AsDuration()) - }) - }) - } -} - -func TestTimeRoundtrip(t *testing.T) { - tests := []struct { - name string - time time.Time - expectedString string - }{ - { - "no_nanos", - time.Date(2023, 12, 25, 10, 30, 0, 0, time.UTC), - "2023-12-25T10:30:00Z", - }, - { - "with_nanos", - time.Date(2023, 12, 25, 10, 30, 0, 123456789, time.UTC), - "2023-12-25T10:30:00.123456789Z", - }, - { - "epoch", - time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC), - "1970-01-01T00:00:00Z", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - // Test value type - t.Run("value", func(t *testing.T) { - src := *sdktime.New(tt.time) - dynValue, err := FromTyped(src, dyn.NilValue) - require.NoError(t, err) - assert.Equal(t, tt.expectedString, dynValue.MustString()) - - var out sdktime.Time - err = ToTyped(&out, dynValue) - require.NoError(t, err) - assert.Equal(t, src.AsTime(), out.AsTime()) - }) - - // Test pointer type - t.Run("pointer", func(t *testing.T) { - src := *sdktime.New(tt.time) - dynValue, err := FromTyped(&src, dyn.NilValue) - require.NoError(t, err) - assert.Equal(t, tt.expectedString, dynValue.MustString()) - - var out *sdktime.Time - err = ToTyped(&out, dynValue) - require.NoError(t, err) - require.NotNil(t, out) - assert.Equal(t, src.AsTime(), out.AsTime()) - }) - }) - } -} - -func TestFieldMaskRoundtrip(t *testing.T) { - tests := []struct { - name string - paths []string - expectedString string - }{ - {"single", []string{"name"}, "name"}, - {"multiple", []string{"name", "age", "email"}, "name,age,email"}, - {"nested", []string{"user.name", "user.email"}, "user.name,user.email"}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - // Test value type - t.Run("value", func(t *testing.T) { - src := *sdkfieldmask.New(tt.paths) - dynValue, err := FromTyped(src, dyn.NilValue) - require.NoError(t, err) - assert.Equal(t, tt.expectedString, dynValue.MustString()) - - var out sdkfieldmask.FieldMask - err = ToTyped(&out, dynValue) - require.NoError(t, err) - assert.Equal(t, src.Paths, out.Paths) - }) - - // Test pointer type - t.Run("pointer", func(t *testing.T) { - src := *sdkfieldmask.New(tt.paths) - dynValue, err := FromTyped(&src, dyn.NilValue) - require.NoError(t, err) - assert.Equal(t, tt.expectedString, dynValue.MustString()) - - var out *sdkfieldmask.FieldMask - err = ToTyped(&out, dynValue) - require.NoError(t, err) - require.NotNil(t, out) - assert.Equal(t, src.Paths, out.Paths) - }) - }) - } -} - -// Edge case tests - -func TestNilValues(t *testing.T) { - tests := []struct { - name string - typ any - }{ - {"duration", (*sdkduration.Duration)(nil)}, - {"time", (*sdktime.Time)(nil)}, - {"fieldmask", (*sdkfieldmask.FieldMask)(nil)}, - } - - t.Run("from_typed", func(t *testing.T) { - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - nv, err := FromTyped(tt.typ, dyn.NilValue) - require.NoError(t, err) - assert.Equal(t, dyn.NilValue, nv) - }) - } - }) - - t.Run("normalize", func(t *testing.T) { - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - vout, diags := Normalize(tt.typ, dyn.NilValue) - assert.Len(t, diags, 1) - assert.Equal(t, diag.Warning, diags[0].Severity) - assert.Equal(t, `expected a string value, found null`, diags[0].Summary) - assert.Equal(t, dyn.InvalidValue, vout) - }) - } - }) -} - -func TestToTypedErrors(t *testing.T) { - wrongTypeInput := dyn.V(map[string]dyn.Value{"foo": dyn.V("bar")}) - - tests := []struct { - name string - destValue any - input dyn.Value - expectedError string - }{ - { - name: "duration_wrong_type", - destValue: new(sdkduration.Duration), - input: wrongTypeInput, - expectedError: "expected a string", - }, - { - name: "duration_invalid_format", - destValue: new(sdkduration.Duration), - input: dyn.V("7d"), - expectedError: "invalid google.protobuf.Duration value", - }, - { - name: "time_wrong_type", - destValue: new(sdktime.Time), - input: wrongTypeInput, - expectedError: "expected a string", - }, - { - name: "time_invalid_format", - destValue: new(sdktime.Time), - input: dyn.V("not-a-time"), - expectedError: "invalid google.protobuf.Timestamp value", - }, - { - name: "fieldmask_wrong_type", - destValue: new(sdkfieldmask.FieldMask), - input: wrongTypeInput, - expectedError: "expected a string", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - err := ToTyped(tt.destValue, tt.input) - require.Error(t, err) - assert.Contains(t, err.Error(), tt.expectedError) - }) - } -} - -func TestFromTypedVariableReferences(t *testing.T) { - tests := []struct { - name string - src any - }{ - { - name: "duration", - src: *sdkduration.New(5 * time.Minute), - }, - { - name: "time", - src: *sdktime.New(time.Date(2023, 12, 25, 10, 30, 0, 0, time.UTC)), - }, - { - name: "fieldmask", - src: *sdkfieldmask.New([]string{"name", "age"}), - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - ref := dyn.V("${var.foo}") - nv, err := FromTyped(tt.src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.V("${var.foo}"), nv) - }) - } -} - -func TestToTypedVariableReferences(t *testing.T) { - tests := []struct { - name string - dst any - zero any - }{ - { - name: "duration", - dst: sdkduration.New(5 * time.Minute), - zero: sdkduration.Duration{}, - }, - { - name: "time", - dst: sdktime.New(time.Date(2023, 12, 25, 10, 30, 0, 0, time.UTC)), - zero: sdktime.Time{}, - }, - { - name: "fieldmask", - dst: sdkfieldmask.New([]string{"name", "age"}), - zero: sdkfieldmask.FieldMask{}, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - err := ToTyped(tt.dst, dyn.V("${var.foo}")) - require.NoError(t, err) - assert.Equal(t, tt.zero, reflect.ValueOf(tt.dst).Elem().Interface()) - }) - } -} - -func TestSpecialCases(t *testing.T) { - t.Run("duration_zero", func(t *testing.T) { - var src sdkduration.Duration - nv, err := FromTyped(src, dyn.NilValue) - require.NoError(t, err) - assert.Equal(t, dyn.NilValue, nv) - }) - - t.Run("fieldmask_empty_fromtyped", func(t *testing.T) { - src := sdkfieldmask.New([]string{}) - nv, err := FromTyped(src, dyn.NilValue) - require.NoError(t, err) - // Empty field mask marshals to empty string - assert.Equal(t, dyn.V(""), nv) - }) - - t.Run("fieldmask_empty_totyped", func(t *testing.T) { - var out sdkfieldmask.FieldMask - err := ToTyped(&out, dyn.V("")) - require.NoError(t, err) - assert.Empty(t, out.Paths) - }) -} - -// End-to-end tests with structs containing SDK native types - -func TestSDKTypesRoundTripWithPostgresBranchSpec(t *testing.T) { - type BranchSpec struct { - ExpireTime *sdktime.Time `json:"expire_time,omitempty"` - SourceBranchTime *sdktime.Time `json:"source_branch_time,omitempty"` - Ttl *sdkduration.Duration `json:"ttl,omitempty"` - IsProtected bool `json:"is_protected,omitempty"` - } - - original := BranchSpec{ - ExpireTime: sdktime.New(time.Date(2024, 12, 31, 23, 59, 59, 0, time.UTC)), - SourceBranchTime: sdktime.New(time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)), - Ttl: sdkduration.New(7 * 24 * time.Hour), - IsProtected: true, - } - - dynValue, err := FromTyped(original, dyn.NilValue) - require.NoError(t, err) - assert.Equal(t, "2024-12-31T23:59:59Z", dynValue.Get("expire_time").MustString()) - assert.Equal(t, "2024-01-01T00:00:00Z", dynValue.Get("source_branch_time").MustString()) - assert.Equal(t, "604800s", dynValue.Get("ttl").MustString()) - assert.True(t, dynValue.Get("is_protected").MustBool()) - - var roundtrip BranchSpec - err = ToTyped(&roundtrip, dynValue) - require.NoError(t, err) - require.NotNil(t, roundtrip.ExpireTime) - require.NotNil(t, roundtrip.SourceBranchTime) - require.NotNil(t, roundtrip.Ttl) - assert.Equal(t, original.ExpireTime.AsTime(), roundtrip.ExpireTime.AsTime()) - assert.Equal(t, original.SourceBranchTime.AsTime(), roundtrip.SourceBranchTime.AsTime()) - assert.Equal(t, original.Ttl.AsDuration(), roundtrip.Ttl.AsDuration()) - assert.Equal(t, original.IsProtected, roundtrip.IsProtected) -} - -func TestSDKTypesRoundTripWithUpdateRequest(t *testing.T) { - type UpdateRequest struct { - Name string `json:"name"` - UpdateMask sdkfieldmask.FieldMask `json:"update_mask"` - } - - original := UpdateRequest{ - Name: "projects/123/branches/456", - UpdateMask: *sdkfieldmask.New([]string{"spec.ttl", "spec.is_protected"}), - } - - dynValue, err := FromTyped(original, dyn.NilValue) - require.NoError(t, err) - assert.Equal(t, "projects/123/branches/456", dynValue.Get("name").MustString()) - assert.Equal(t, "spec.ttl,spec.is_protected", dynValue.Get("update_mask").MustString()) - - var roundtrip UpdateRequest - err = ToTyped(&roundtrip, dynValue) - require.NoError(t, err) - assert.Equal(t, original.Name, roundtrip.Name) - assert.Equal(t, []string{"spec.ttl", "spec.is_protected"}, roundtrip.UpdateMask.Paths) -} - -func TestSDKTypesNormalizeWithStruct(t *testing.T) { - type BranchSpec struct { - ExpireTime *sdktime.Time `json:"expire_time,omitempty"` - Ttl *sdkduration.Duration `json:"ttl,omitempty"` - IsProtected bool `json:"is_protected,omitempty"` - } - - var typ BranchSpec - vin := dyn.V(map[string]dyn.Value{ - "expire_time": dyn.V("2024-12-31T23:59:59Z"), - "ttl": dyn.V("604800s"), - "is_protected": dyn.V(true), - }) - - vout, diags := Normalize(typ, vin) - assert.Empty(t, diags) - assert.Equal(t, "2024-12-31T23:59:59Z", vout.Get("expire_time").MustString()) - assert.Equal(t, "604800s", vout.Get("ttl").MustString()) - assert.True(t, vout.Get("is_protected").MustBool()) - - var out BranchSpec - err := ToTyped(&out, vout) - require.NoError(t, err) - require.NotNil(t, out.ExpireTime) - require.NotNil(t, out.Ttl) - assert.Equal(t, time.Date(2024, 12, 31, 23, 59, 59, 0, time.UTC), out.ExpireTime.AsTime()) - assert.Equal(t, 7*24*time.Hour, out.Ttl.AsDuration()) - assert.True(t, out.IsProtected) -} diff --git a/libs/dyn/convert/struct_info_test.go b/libs/dyn/convert/struct_info_test.go deleted file mode 100644 index f921523c391..00000000000 --- a/libs/dyn/convert/struct_info_test.go +++ /dev/null @@ -1,228 +0,0 @@ -package convert - -import ( - "reflect" - "testing" - - "github.com/databricks/cli/libs/dyn" - "github.com/stretchr/testify/assert" -) - -func TestStructInfoPlain(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - Bar string `json:"bar,omitempty"` - - // Baz must be skipped. - Baz string `json:""` - - // Qux must be skipped. - Qux string `json:"-"` - } - - si := getStructInfo(reflect.TypeFor[Tmp]()) - assert.Len(t, si.Fields, 2) - assert.Equal(t, []int{0}, si.Fields["foo"]) - assert.Equal(t, []int{1}, si.Fields["bar"]) -} - -func TestStructInfoAnonymousByValue(t *testing.T) { - type Bar struct { - Bar string `json:"bar"` - } - - type Foo struct { - Foo string `json:"foo"` - Bar - } - - type Tmp struct { - Foo - } - - si := getStructInfo(reflect.TypeFor[Tmp]()) - assert.Len(t, si.Fields, 2) - assert.Equal(t, []int{0, 0}, si.Fields["foo"]) - assert.Equal(t, []int{0, 1, 0}, si.Fields["bar"]) -} - -func TestStructInfoAnonymousByValuePrecedence(t *testing.T) { - type Bar struct { - Bar string `json:"bar"` - } - - type Foo struct { - Foo string `json:"foo"` - Bar - } - - type Tmp struct { - // "foo" comes from [Foo]. - Foo - // "bar" comes from [Bar] directly, not through [Foo]. - Bar - } - - si := getStructInfo(reflect.TypeFor[Tmp]()) - assert.Len(t, si.Fields, 2) - assert.Equal(t, []int{0, 0}, si.Fields["foo"]) - assert.Equal(t, []int{1, 0}, si.Fields["bar"]) -} - -func TestStructInfoAnonymousByPointer(t *testing.T) { - type Bar struct { - Bar string `json:"bar"` - } - - type Foo struct { - Foo string `json:"foo"` - *Bar - } - - type Tmp struct { - *Foo - } - - si := getStructInfo(reflect.TypeFor[Tmp]()) - assert.Len(t, si.Fields, 2) - assert.Equal(t, []int{0, 0}, si.Fields["foo"]) - assert.Equal(t, []int{0, 1, 0}, si.Fields["bar"]) -} - -func TestStructInfoFieldValues(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - Bar string `json:"bar"` - } - - src := Tmp{ - Foo: "foo", - Bar: "bar", - } - - si := getStructInfo(reflect.TypeFor[Tmp]()) - fv := si.FieldValues(reflect.ValueOf(src)) - assert.Len(t, fv, 2) - assert.Equal(t, "foo", fv[0].Key) - assert.True(t, reflect.ValueOf("foo").Equal(fv[0].Value)) - assert.Equal(t, "bar", fv[1].Key) - assert.True(t, reflect.ValueOf("bar").Equal(fv[1].Value)) -} - -func TestStructInfoFieldValuesAnonymousByValue(t *testing.T) { - type Bar struct { - Bar string `json:"bar"` - } - - type Foo struct { - Foo string `json:"foo"` - Bar - } - - type Tmp struct { - Foo - } - - src := Tmp{ - Foo: Foo{ - Foo: "foo", - Bar: Bar{ - Bar: "bar", - }, - }, - } - - si := getStructInfo(reflect.TypeFor[Tmp]()) - fv := si.FieldValues(reflect.ValueOf(src)) - assert.Len(t, fv, 2) - assert.Equal(t, "foo", fv[0].Key) - assert.Equal(t, "bar", fv[1].Key) -} - -func TestStructInfoFieldValuesAnonymousByPointer(t *testing.T) { - type Bar struct { - Bar string `json:"bar"` - } - - type Foo struct { - Foo string `json:"foo"` - *Bar - } - - type Tmp struct { - *Foo - } - - // Test that the embedded fields are dereferenced properly. - t.Run("all are set", func(t *testing.T) { - src := Tmp{ - Foo: &Foo{ - Foo: "foo", - Bar: &Bar{ - Bar: "bar", - }, - }, - } - - si := getStructInfo(reflect.TypeFor[Tmp]()) - fv := si.FieldValues(reflect.ValueOf(src)) - assert.Len(t, fv, 2) - assert.Equal(t, "foo", fv[0].Key) - assert.Equal(t, "bar", fv[1].Key) - }) - - // Test that fields of embedded types are skipped if the embedded type is nil. - t.Run("top level is set", func(t *testing.T) { - src := Tmp{ - Foo: &Foo{ - Foo: "foo", - Bar: nil, - }, - } - - si := getStructInfo(reflect.TypeFor[Tmp]()) - fv := si.FieldValues(reflect.ValueOf(src)) - assert.Len(t, fv, 1) - assert.Equal(t, "foo", fv[0].Key) - }) - - // Test that fields of embedded types are skipped if the embedded type is nil. - t.Run("none are set", func(t *testing.T) { - src := Tmp{ - Foo: nil, - } - - si := getStructInfo(reflect.TypeFor[Tmp]()) - fv := si.FieldValues(reflect.ValueOf(src)) - assert.Empty(t, fv) - }) -} - -func TestStructInfoValueFieldAbsent(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - } - - si := getStructInfo(reflect.TypeFor[Tmp]()) - assert.Nil(t, si.ValueField) -} - -func TestStructInfoValueFieldPresent(t *testing.T) { - type Tmp struct { - Foo dyn.Value - } - - si := getStructInfo(reflect.TypeFor[Tmp]()) - assert.NotNil(t, si.ValueField) -} - -func TestStructInfoValueFieldMultiple(t *testing.T) { - type Tmp struct { - Foo dyn.Value - Bar dyn.Value - } - - assert.Panics(t, func() { - getStructInfo(reflect.TypeFor[Tmp]()) - }) -} diff --git a/libs/dyn/convert/to_typed.go b/libs/dyn/convert/to_typed.go deleted file mode 100644 index 87b0fe7d8ca..00000000000 --- a/libs/dyn/convert/to_typed.go +++ /dev/null @@ -1,350 +0,0 @@ -package convert - -import ( - "fmt" - "reflect" - "slices" - "strconv" - - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/dynvar" -) - -// Populate a destination typed value from a source dynamic value. -// -// At any point while walking the destination type tree using -// reflection, if this function sees an exported field with type dyn.Value it -// will populate that field with the appropriate source dynamic value. -// see PR: https://github.com/databricks/cli/pull/1010 -func ToTyped(dst any, src dyn.Value) error { - dstv := reflect.ValueOf(dst) - - // Dereference pointer if necessary - for dstv.Kind() == reflect.Pointer { - // If the source value is nil and the destination is a settable pointer, - // set the destination to nil. Also see `end_to_end_test.go`. - if dstv.CanSet() && src.Kind() == dyn.KindNil { - dstv.SetZero() - return nil - } - if dstv.IsNil() { - dstv.Set(reflect.New(dstv.Type().Elem())) - } - dstv = dstv.Elem() - } - - // Verify that vv is settable. - if !dstv.CanSet() { - panic("cannot set destination value") - } - - switch dstv.Kind() { - case reflect.Struct: - // Handle SDK native types using JSON unmarshaling. - if slices.Contains(sdkNativeTypes, dstv.Type()) { - return toTypedSDKNative(dstv, src) - } - return toTypedStruct(dstv, src) - case reflect.Map: - return toTypedMap(dstv, src) - case reflect.Slice: - return toTypedSlice(dstv, src) - case reflect.String: - return toTypedString(dstv, src) - case reflect.Bool: - return toTypedBool(dstv, src) - case reflect.Int, reflect.Int32, reflect.Int64: - return toTypedInt(dstv, src) - case reflect.Float32, reflect.Float64: - return toTypedFloat(dstv, src) - case reflect.Interface: - return toTypedInterface(dstv, src) - default: - // Fall through to the error case. - } - - return fmt.Errorf("unsupported type: %s", dstv.Kind()) -} - -func toTypedStruct(dst reflect.Value, src dyn.Value) error { - switch src.Kind() { - case dyn.KindMap: - // Zero the destination struct such that fields - // that aren't present in [src] are cleared. - dst.SetZero() - - info := getStructInfo(dst.Type()) - - for _, pair := range src.MustMap().Pairs() { - pk := pair.Key - pv := pair.Value - jsonKey := pk.MustString() - - index, ok := info.Fields[jsonKey] - if !ok { - // Ignore unknown fields. - // A warning will be printed later. See PR #904. - continue - } - - f := getOrNewFieldByIndex(dst, index) - - err := ToTyped(f.Addr().Interface(), pv) - if err != nil { - return err - } - - if pv.IsZero() { - // The field's zero value must still serialize, so append its Go name - // to the ForceSendFields of the struct that declares it. That struct - // shares a prefix with the field's index path, so it is already - // allocated by the walk above. - if fsfIndex, ok := info.ForceSendFieldsIndex[jsonKey]; ok { - fsf := getOrNewFieldByIndex(dst, fsfIndex) - fsf.Set(reflect.Append(fsf, reflect.ValueOf(info.GolangNames[jsonKey]))) - } - } - } - - // Populate field(s) for [dyn.Value], if any. - if info.ValueField != nil { - vv := dst.FieldByIndex(info.ValueField) - vv.Set(reflect.ValueOf(src)) - } - - return nil - case dyn.KindNil: - dst.SetZero() - return nil - case dyn.KindString: - // Ignore pure variable references (e.g. ${var.foo}). - if dynvar.IsPureVariableReference(src.MustString()) { - dst.SetZero() - return nil - } - default: - // Fall through to the error case. - } - - return TypeError{ - value: src, - msg: fmt.Sprintf("expected a map, found a %s", src.Kind()), - } -} - -// getOrNewFieldByIndex resolves the value at the given index path within an addressable -// struct, allocating intermediate structs embedded as pointer types along the way. -// Code inspired by [reflect.FieldByIndex] implementation. -func getOrNewFieldByIndex(v reflect.Value, index []int) reflect.Value { - for i, x := range index { - if i > 0 { - if v.Kind() == reflect.Pointer { - if v.IsNil() { - v.Set(reflect.New(v.Type().Elem())) - } - v = v.Elem() - } - } - v = v.Field(x) - } - return v -} - -func toTypedMap(dst reflect.Value, src dyn.Value) error { - switch src.Kind() { - case dyn.KindMap: - m := src.MustMap() - - // Always overwrite. - dst.Set(reflect.MakeMapWithSize(dst.Type(), m.Len())) - for _, pair := range m.Pairs() { - pk := pair.Key - pv := pair.Value - kv := reflect.ValueOf(pk.MustString()) - kt := dst.Type().Key() - vv := reflect.New(dst.Type().Elem()) - err := ToTyped(vv.Interface(), pv) - if err != nil { - return err - } - dst.SetMapIndex(kv.Convert(kt), vv.Elem()) - } - return nil - case dyn.KindNil: - dst.SetZero() - return nil - case dyn.KindString: - // Ignore pure variable references (e.g. ${var.foo}). - if dynvar.IsPureVariableReference(src.MustString()) { - dst.SetZero() - return nil - } - default: - // Fall through to the error case. - } - - return TypeError{ - value: src, - msg: fmt.Sprintf("expected a map, found a %s", src.Kind()), - } -} - -func toTypedSlice(dst reflect.Value, src dyn.Value) error { - switch src.Kind() { - case dyn.KindSequence: - seq := src.MustSequence() - - // Always overwrite. - dst.Set(reflect.MakeSlice(dst.Type(), len(seq), len(seq))) - for i := range seq { - err := ToTyped(dst.Index(i).Addr().Interface(), seq[i]) - if err != nil { - return err - } - } - return nil - case dyn.KindNil: - dst.SetZero() - return nil - case dyn.KindString: - // Ignore pure variable references (e.g. ${var.foo}). - if dynvar.IsPureVariableReference(src.MustString()) { - dst.SetZero() - return nil - } - default: - // Fall through to the error case. - } - - return TypeError{ - value: src, - msg: fmt.Sprintf("expected a sequence, found a %s", src.Kind()), - } -} - -func toTypedString(dst reflect.Value, src dyn.Value) error { - switch src.Kind() { - case dyn.KindString: - dst.SetString(src.MustString()) - return nil - case dyn.KindBool: - dst.SetString(strconv.FormatBool(src.MustBool())) - return nil - case dyn.KindInt: - dst.SetString(strconv.FormatInt(src.MustInt(), 10)) - return nil - case dyn.KindFloat: - dst.SetString(strconv.FormatFloat(src.MustFloat(), 'f', -1, 64)) - return nil - default: - // Fall through to the error case. - } - - return TypeError{ - value: src, - msg: fmt.Sprintf("expected a string, found a %s", src.Kind()), - } -} - -func toTypedBool(dst reflect.Value, src dyn.Value) error { - switch src.Kind() { - case dyn.KindBool: - dst.SetBool(src.MustBool()) - return nil - case dyn.KindString: - // See https://yaml.org/type/bool.html. - switch src.MustString() { - case "y", "Y", "yes", "Yes", "YES", "on", "On", "ON", "true", "True", "TRUE": - dst.SetBool(true) - return nil - case "n", "N", "no", "No", "NO", "off", "Off", "OFF", "false", "False", "FALSE": - dst.SetBool(false) - return nil - } - // Ignore pure variable references (e.g. ${var.foo}). - if dynvar.IsPureVariableReference(src.MustString()) { - dst.SetZero() - return nil - } - default: - // Fall through to the error case. - } - - return TypeError{ - value: src, - msg: fmt.Sprintf("expected a boolean, found a %s", src.Kind()), - } -} - -func toTypedInt(dst reflect.Value, src dyn.Value) error { - switch src.Kind() { - case dyn.KindInt: - dst.SetInt(src.MustInt()) - return nil - case dyn.KindFloat: - v := src.MustFloat() - if v == float64(int64(v)) { - // If the destination is smaller than int64, but the value to set is bigger - // then destination overflows and is set to -1 - dst.SetInt(int64(src.MustFloat())) - return nil - } - - return TypeError{ - value: src, - msg: fmt.Sprintf("expected an int, found a %s", src.Kind()), - } - case dyn.KindString: - if i64, err := strconv.ParseInt(src.MustString(), 10, 64); err == nil { - dst.SetInt(i64) - return nil - } - // Ignore pure variable references (e.g. ${var.foo}). - if dynvar.IsPureVariableReference(src.MustString()) { - dst.SetZero() - return nil - } - default: - // Fall through to the error case. - } - - return TypeError{ - value: src, - msg: fmt.Sprintf("expected an int, found a %s", src.Kind()), - } -} - -func toTypedFloat(dst reflect.Value, src dyn.Value) error { - switch src.Kind() { - case dyn.KindFloat: - dst.SetFloat(src.MustFloat()) - return nil - case dyn.KindString: - if f64, err := strconv.ParseFloat(src.MustString(), 64); err == nil { - dst.SetFloat(f64) - return nil - } - // Ignore pure variable references (e.g. ${var.foo}). - if dynvar.IsPureVariableReference(src.MustString()) { - dst.SetZero() - return nil - } - default: - // Fall through to the error case. - } - - return TypeError{ - value: src, - msg: fmt.Sprintf("expected a float, found a %s", src.Kind()), - } -} - -func toTypedInterface(dst reflect.Value, src dyn.Value) error { - if src.Kind() == dyn.KindNil { - dst.Set(reflect.Zero(dst.Type())) - return nil - } - - dst.Set(reflect.ValueOf(src.AsAny())) - return nil -} diff --git a/libs/dyn/convert/to_typed_test.go b/libs/dyn/convert/to_typed_test.go deleted file mode 100644 index a838dfab33f..00000000000 --- a/libs/dyn/convert/to_typed_test.go +++ /dev/null @@ -1,790 +0,0 @@ -package convert - -import ( - "testing" - - "github.com/databricks/cli/libs/dyn" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestToTypedStruct(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - Bar string `json:"bar,omitempty"` - - // Baz must be skipped. - Baz string `json:""` - - // Qux must be skipped. - Qux string `json:"-"` - } - - var out Tmp - v := dyn.V(map[string]dyn.Value{ - "foo": dyn.V("bar"), - "bar": dyn.V("baz"), - }) - - err := ToTyped(&out, v) - require.NoError(t, err) - assert.Equal(t, "bar", out.Foo) - assert.Equal(t, "baz", out.Bar) -} - -func TestToTypedStructOverwrite(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - Bar string `json:"bar,omitempty"` - - // Baz must be skipped. - Baz string `json:""` - - // Qux must be skipped. - Qux string `json:"-"` - } - - out := Tmp{ - Foo: "baz", - Bar: "qux", - } - v := dyn.V(map[string]dyn.Value{ - "foo": dyn.V("bar"), - "bar": dyn.V("baz"), - }) - - err := ToTyped(&out, v) - require.NoError(t, err) - assert.Equal(t, "bar", out.Foo) - assert.Equal(t, "baz", out.Bar) -} - -func TestToTypedStructClearFields(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - Bar string `json:"bar,omitempty"` - } - - // Struct value with non-empty fields. - out := Tmp{ - Foo: "baz", - Bar: "qux", - } - - // Value is an empty map. - v := dyn.V(map[string]dyn.Value{}) - - // The previously set fields should be cleared. - err := ToTyped(&out, v) - require.NoError(t, err) - assert.Equal(t, Tmp{}, out) -} - -func TestToTypedStructClearFieldsForceSend(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - Bar string `json:"bar,omitempty"` - ForceSendFields []string `json:"-"` - } - - // Struct value with non-empty fields. - out := Tmp{ - Foo: "baz", - Bar: "qux", - } - - // Value is an empty map. - v := dyn.V(map[string]dyn.Value{}) - - // The previously set fields should be cleared. - err := ToTyped(&out, v) - require.NoError(t, err) - assert.Equal(t, Tmp{}, out) -} - -func TestToTypedStructZeroFields(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - Bar string `json:"bar,omitempty"` - } - - // Struct value with non-empty fields. - out := Tmp{ - Foo: "baz", - Bar: "qux", - } - - // Value is an empty map. - v := dyn.V(map[string]dyn.Value{ - "foo": dyn.V(""), - "bar": dyn.V(""), - }) - - // The previously set fields should be cleared. - err := ToTyped(&out, v) - require.NoError(t, err) - assert.Equal(t, Tmp{}, out) -} - -func TestToTypedStructZeroFieldsForceSend(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - Bar string `json:"bar,omitempty"` - NumWorkers int `json:"num_workers,omitempty"` - ForceSendFields []string `json:"-"` - } - - // Struct value with non-empty fields. - out := Tmp{ - Foo: "baz", - Bar: "qux", - } - - // Value is an empty map. - m := dyn.Mapping{} - m.SetLoc("foo", nil, dyn.V("")) - m.SetLoc("bar", nil, dyn.V("")) - m.SetLoc("num_workers", nil, dyn.V(int64(0))) - v := dyn.V(m) - - // The previously set fields should be cleared. - err := ToTyped(&out, v) - require.NoError(t, err) - assert.Equal(t, Tmp{ForceSendFields: []string{"Foo", "Bar", "NumWorkers"}}, out) -} - -func TestToTypedStructAnonymousByValue(t *testing.T) { - type Bar struct { - Bar string `json:"bar"` - } - - type Foo struct { - Foo string `json:"foo"` - Bar - } - - type Tmp struct { - Foo - } - - var out Tmp - v := dyn.V(map[string]dyn.Value{ - "foo": dyn.V("bar"), - "bar": dyn.V("baz"), - }) - - err := ToTyped(&out, v) - require.NoError(t, err) - assert.Equal(t, "bar", out.Foo.Foo) - assert.Equal(t, "baz", out.Bar.Bar) -} - -func TestToTypedStructAnonymousByPointer(t *testing.T) { - type Bar struct { - Bar string `json:"bar"` - } - - type Foo struct { - Foo string `json:"foo"` - *Bar - } - - type Tmp struct { - *Foo - } - - var out Tmp - v := dyn.V(map[string]dyn.Value{ - "foo": dyn.V("bar"), - "bar": dyn.V("baz"), - }) - - err := ToTyped(&out, v) - require.NoError(t, err) - assert.Equal(t, "bar", out.Foo.Foo) - assert.Equal(t, "baz", out.Bar.Bar) -} - -func TestToTypedStructNil(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - } - - out := Tmp{} - err := ToTyped(&out, dyn.NilValue) - require.NoError(t, err) - assert.Equal(t, Tmp{}, out) -} - -func TestToTypedStructNilOverwrite(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - } - - out := Tmp{"bar"} - err := ToTyped(&out, dyn.NilValue) - require.NoError(t, err) - assert.Equal(t, Tmp{}, out) -} - -func TestToTypedStructWithValueField(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - - ConfigValue dyn.Value - } - - var out Tmp - v := dyn.V(map[string]dyn.Value{ - "foo": dyn.V("bar"), - }) - - err := ToTyped(&out, v) - require.NoError(t, err) - assert.Equal(t, "bar", out.Foo) - assert.Equal(t, v, out.ConfigValue) -} - -func TestToTypedMap(t *testing.T) { - out := map[string]string{} - - v := dyn.V(map[string]dyn.Value{ - "key": dyn.V("value"), - }) - - err := ToTyped(&out, v) - require.NoError(t, err) - assert.Len(t, out, 1) - assert.Equal(t, "value", out["key"]) -} - -func TestToTypedMapOverwrite(t *testing.T) { - out := map[string]string{ - "foo": "bar", - } - - v := dyn.V(map[string]dyn.Value{ - "bar": dyn.V("qux"), - }) - - err := ToTyped(&out, v) - require.NoError(t, err) - assert.Len(t, out, 1) - assert.Equal(t, "qux", out["bar"]) -} - -func TestToTypedMapWithPointerElement(t *testing.T) { - var out map[string]*string - - v := dyn.V(map[string]dyn.Value{ - "key": dyn.V("value"), - }) - - err := ToTyped(&out, v) - require.NoError(t, err) - assert.Len(t, out, 1) - assert.Equal(t, "value", *out["key"]) -} - -func TestToTypedMapNil(t *testing.T) { - out := map[string]string{} - err := ToTyped(&out, dyn.NilValue) - require.NoError(t, err) - assert.Nil(t, out) -} - -func TestToTypedMapNilOverwrite(t *testing.T) { - out := map[string]string{ - "foo": "bar", - } - err := ToTyped(&out, dyn.NilValue) - require.NoError(t, err) - assert.Nil(t, out) -} - -func TestToTypedSlice(t *testing.T) { - var out []string - - v := dyn.V([]dyn.Value{ - dyn.V("foo"), - dyn.V("bar"), - }) - - err := ToTyped(&out, v) - require.NoError(t, err) - assert.Len(t, out, 2) - assert.Equal(t, "foo", out[0]) - assert.Equal(t, "bar", out[1]) -} - -func TestToTypedSliceOverwrite(t *testing.T) { - out := []string{"qux"} - - v := dyn.V([]dyn.Value{ - dyn.V("foo"), - dyn.V("bar"), - }) - - err := ToTyped(&out, v) - require.NoError(t, err) - assert.Len(t, out, 2) - assert.Equal(t, "foo", out[0]) - assert.Equal(t, "bar", out[1]) -} - -func TestToTypedSliceWithPointerElement(t *testing.T) { - var out []*string - - v := dyn.V([]dyn.Value{ - dyn.V("foo"), - dyn.V("bar"), - }) - - err := ToTyped(&out, v) - require.NoError(t, err) - assert.Len(t, out, 2) - assert.Equal(t, "foo", *out[0]) - assert.Equal(t, "bar", *out[1]) -} - -func TestToTypedSliceNil(t *testing.T) { - var out []string - err := ToTyped(&out, dyn.NilValue) - require.NoError(t, err) - assert.Nil(t, out) -} - -func TestToTypedSliceNilOverwrite(t *testing.T) { - out := []string{"foo"} - err := ToTyped(&out, dyn.NilValue) - require.NoError(t, err) - assert.Nil(t, out) -} - -func TestToTypedString(t *testing.T) { - var out string - err := ToTyped(&out, dyn.V("foo")) - require.NoError(t, err) - assert.Equal(t, "foo", out) -} - -func TestToTypedStringOverwrite(t *testing.T) { - out := "bar" - err := ToTyped(&out, dyn.V("foo")) - require.NoError(t, err) - assert.Equal(t, "foo", out) -} - -func TestToTypedStringFromBool(t *testing.T) { - var out string - err := ToTyped(&out, dyn.V(true)) - require.NoError(t, err) - assert.Equal(t, "true", out) -} - -func TestToTypedStringFromInt(t *testing.T) { - var out string - err := ToTyped(&out, dyn.V(123)) - require.NoError(t, err) - assert.Equal(t, "123", out) -} - -func TestToTypedStringFromFloat(t *testing.T) { - var out string - err := ToTyped(&out, dyn.V(1.2)) - require.NoError(t, err) - assert.Equal(t, "1.2", out) -} - -func TestToTypedBool(t *testing.T) { - var out bool - err := ToTyped(&out, dyn.V(true)) - require.NoError(t, err) - assert.True(t, out) -} - -func TestToTypedBoolOverwrite(t *testing.T) { - out := true - err := ToTyped(&out, dyn.V(false)) - require.NoError(t, err) - assert.False(t, out) -} - -func TestToTypedBoolFromString(t *testing.T) { - var out bool - - // True-ish - for _, v := range []string{"y", "yes", "on", "true", "True", "TRUE"} { - err := ToTyped(&out, dyn.V(v)) - require.NoError(t, err) - assert.True(t, out) - } - - // False-ish - for _, v := range []string{"n", "no", "off", "false", "False", "FALSE"} { - err := ToTyped(&out, dyn.V(v)) - require.NoError(t, err) - assert.False(t, out) - } - - // Other - err := ToTyped(&out, dyn.V("some other string")) - require.Error(t, err) -} - -func TestToTypedBoolFromStringVariableReference(t *testing.T) { - out := true - err := ToTyped(&out, dyn.V("${var.foo}")) - require.NoError(t, err) - assert.False(t, out) -} - -func TestToTypedInt(t *testing.T) { - var out int - err := ToTyped(&out, dyn.V(1234)) - require.NoError(t, err) - assert.Equal(t, int(1234), out) -} - -func TestToTypedInt32(t *testing.T) { - var out32 int32 - err := ToTyped(&out32, dyn.V(1235)) - require.NoError(t, err) - assert.Equal(t, int32(1235), out32) -} - -func TestToTypedInt64(t *testing.T) { - var out64 int64 - err := ToTyped(&out64, dyn.V(1236)) - require.NoError(t, err) - assert.Equal(t, int64(1236), out64) -} - -func TestToTypedIntOverwrite(t *testing.T) { - out := 123 - err := ToTyped(&out, dyn.V(1234)) - require.NoError(t, err) - assert.Equal(t, int(1234), out) -} - -func TestToTypedInt32Overwrite(t *testing.T) { - var out32 int32 = 123 - err := ToTyped(&out32, dyn.V(1234)) - require.NoError(t, err) - assert.Equal(t, int32(1234), out32) -} - -func TestToTypedInt64Overwrite(t *testing.T) { - var out64 int64 = 123 - err := ToTyped(&out64, dyn.V(1234)) - require.NoError(t, err) - assert.Equal(t, int64(1234), out64) -} - -func TestToTypedIntFromStringError(t *testing.T) { - var out int - err := ToTyped(&out, dyn.V("abc")) - require.Error(t, err) -} - -func TestToTypedIntFromStringInt(t *testing.T) { - var out int - err := ToTyped(&out, dyn.V("123")) - require.NoError(t, err) - assert.Equal(t, int(123), out) -} - -func TestToTypedIntFromStringVariableReference(t *testing.T) { - out := 123 - err := ToTyped(&out, dyn.V("${var.foo}")) - require.NoError(t, err) - assert.Equal(t, int(0), out) -} - -func TestToTypedIntFromFloat(t *testing.T) { - var out int - err := ToTyped(&out, dyn.V(1.0)) - require.NoError(t, err) - assert.Equal(t, int(1), out) -} - -func TestToTypedIntFromFloatError(t *testing.T) { - var out int - err := ToTyped(&out, dyn.V(1.2)) - require.ErrorContains(t, err, "expected an int, found a float") -} - -func TestToTypedFloat32(t *testing.T) { - var out float32 - err := ToTyped(&out, dyn.V(float32(1.0))) - require.NoError(t, err) - assert.Zero(t, 1.0-out) -} - -func TestToTypedFloat64(t *testing.T) { - var out float64 - err := ToTyped(&out, dyn.V(float64(1.0))) - require.NoError(t, err) - assert.Zero(t, 1.0-out) -} - -func TestToTypedFloat32Overwrite(t *testing.T) { - var out float32 = 1.0 - err := ToTyped(&out, dyn.V(float32(2.0))) - require.NoError(t, err) - assert.Zero(t, 2.0-out) -} - -func TestToTypedFloat64Overwrite(t *testing.T) { - out := 1.0 - err := ToTyped(&out, dyn.V(float64(2.0))) - require.NoError(t, err) - assert.Zero(t, 2.0-out) -} - -func TestToTypedFloat32FromStringError(t *testing.T) { - var out float32 - err := ToTyped(&out, dyn.V("abc")) - require.Error(t, err) -} - -func TestToTypedFloat64FromStringError(t *testing.T) { - var out float64 - err := ToTyped(&out, dyn.V("abc")) - require.Error(t, err) -} - -func TestToTypedFloat32FromString(t *testing.T) { - var out float32 - err := ToTyped(&out, dyn.V("1.2")) - require.NoError(t, err) - assert.Zero(t, 1.2-out) -} - -func TestToTypedFloat64FromString(t *testing.T) { - var out float64 - err := ToTyped(&out, dyn.V("1.2")) - require.NoError(t, err) - assert.Zero(t, 1.2-out) -} - -func TestToTypedFloat32FromStringVariableReference(t *testing.T) { - var out float32 = 1.0 - err := ToTyped(&out, dyn.V("${var.foo}")) - require.NoError(t, err) - assert.Zero(t, out) -} - -func TestToTypedFloat64FromStringVariableReference(t *testing.T) { - out := 1.0 - err := ToTyped(&out, dyn.V("${var.foo}")) - require.NoError(t, err) - assert.Zero(t, out) -} - -func TestToTypedWithAliasKeyType(t *testing.T) { - type custom string - - var out map[custom]string - v := dyn.V(map[string]dyn.Value{ - "foo": dyn.V("bar"), - "bar": dyn.V("baz"), - }) - - err := ToTyped(&out, v) - require.NoError(t, err) - assert.Len(t, out, 2) - assert.Equal(t, "bar", out["foo"]) - assert.Equal(t, "baz", out["bar"]) -} - -func TestToTypedAnyWithBool(t *testing.T) { - var out any - err := ToTyped(&out, dyn.V(false)) - require.NoError(t, err) - assert.Equal(t, false, out) - - err = ToTyped(&out, dyn.V(true)) - require.NoError(t, err) - assert.Equal(t, true, out) -} - -func TestToTypedAnyWithMap(t *testing.T) { - var out any - v := dyn.V(map[string]dyn.Value{ - "foo": dyn.V("bar"), - "bar": dyn.V("baz"), - }) - err := ToTyped(&out, v) - require.NoError(t, err) - assert.Equal(t, map[string]any{"foo": "bar", "bar": "baz"}, out) -} - -func TestToTypedAnyWithNil(t *testing.T) { - var out any - err := ToTyped(&out, dyn.NilValue) - require.NoError(t, err) - assert.Nil(t, out) -} - -func TestToTypedEmbeddedStructForceSendFields(t *testing.T) { - type Inner struct { - InnerField string `json:"inner_field"` - ForceSendFields []string `json:"-"` - } - - type Outer struct { - OuterField string `json:"outer_field"` - Inner - } - - var out Outer - m := dyn.Mapping{} - m.SetLoc("outer_field", nil, dyn.V("")) - m.SetLoc("inner_field", nil, dyn.V("")) - v := dyn.V(m) - - err := ToTyped(&out, v) - require.NoError(t, err) - - // Bug: ForceSendFields contains "OuterField" which belongs to outer struct - // Expected: ForceSendFields should only contain "InnerField" - assert.Equal(t, []string{"InnerField"}, out.ForceSendFields) -} - -func TestToTypedMultipleEmbeddedStructsForceSendFields(t *testing.T) { - type First struct { - FirstField string `json:"first_field"` - ForceSendFields []string `json:"-"` - } - - type Second struct { - SecondField string `json:"second_field"` - ForceSendFields []string `json:"-"` - } - - type Outer struct { - OuterField string `json:"outer_field"` - First - Second - } - - var out Outer - m := dyn.Mapping{} - m.SetLoc("outer_field", nil, dyn.V("")) - m.SetLoc("first_field", nil, dyn.V("")) - m.SetLoc("second_field", nil, dyn.V("")) - v := dyn.V(m) - - err := ToTyped(&out, v) - require.NoError(t, err) - - // Each embedded struct should only get its own fields in ForceSendFields - assert.Equal(t, []string{"FirstField"}, out.First.ForceSendFields) - assert.Equal(t, []string{"SecondField"}, out.Second.ForceSendFields) -} - -func TestToTypedMixedForceSendFields(t *testing.T) { - type First struct { - FirstField string `json:"first_field"` - // No ForceSendFields - } - - type Second struct { - SecondField string `json:"second_field"` - ForceSendFields []string `json:"-"` - } - - type Outer struct { - OuterField string `json:"outer_field"` - ForceSendFields []string `json:"-"` - First - Second - } - - var out Outer - m := dyn.Mapping{} - m.SetLoc("outer_field", nil, dyn.V("")) - m.SetLoc("first_field", nil, dyn.V("")) - m.SetLoc("second_field", nil, dyn.V("")) - v := dyn.V(m) - - err := ToTyped(&out, v) - require.NoError(t, err) - - // Outer should get its own direct fields - assert.Equal(t, []string{"OuterField"}, out.ForceSendFields) - // First has no ForceSendFields field, so nothing to check - // Second should get its own field - assert.Equal(t, []string{"SecondField"}, out.Second.ForceSendFields) -} - -func TestToTypedFieldByNameBugRegressionTest(t *testing.T) { - // This test reproduces the EXACT bug: only direct field is zero, embedded fields are non-zero - // The bug occurred because dst.FieldByName("ForceSendFields") finds embedded ForceSendFields - // instead of direct ForceSendFields (which doesn't exist) - type BaseResource struct { - ID string `json:"id,omitempty"` - } - - type JobSettings struct { - Name string `json:"name"` - ForceSendFields []string `json:"-"` - } - - type Job struct { - BaseResource // embedded at [0] - JobSettings // embedded at [1], has ForceSendFields - Permissions []string `json:"permissions,omitempty"` // direct at [2] - } - - var out Job - m := dyn.Mapping{} - // Non-zero JobSettings fields - these should NOT go to ForceSendFields - m.SetLoc("name", nil, dyn.V("test-job")) - // Zero direct field - this SHOULD trigger the bug with old code - m.SetLoc("permissions", nil, dyn.V([]dyn.Value{})) - v := dyn.V(m) - - err := ToTyped(&out, v) - require.NoError(t, err) - - // "Permissions" should be dropped, JobSettings.ForceSendFields stays empty - assert.Empty(t, out.ForceSendFields) - assert.Equal(t, "test-job", out.Name) - assert.Empty(t, out.Permissions) -} - -func TestToTypedDeeplyEmbeddedStructForceSendFields(t *testing.T) { - // Mirrors resources.PostgresProject -> PostgresProjectConfig -> ProjectSpec: - // Spec is embedded two levels down and Wrapper shadows ForceSendFields to keep - // its own direct field out of Spec's ForceSendFields. A zero-value spec field - // must route to Spec.ForceSendFields, not Wrapper's, otherwise the SDK marshaler - // fails with "field ... cannot be found in struct". - type Spec struct { - SpecField bool `json:"spec_field,omitempty"` - ForceSendFields []string `json:"-"` - } - - type Wrapper struct { - Spec - WrapperField string `json:"wrapper_field,omitempty"` - ForceSendFields []string `json:"-"` - } - - type Outer struct { - Wrapper - } - - var out Outer - m := dyn.Mapping{} - m.SetLoc("spec_field", nil, dyn.V(false)) - m.SetLoc("wrapper_field", nil, dyn.V("")) - v := dyn.V(m) - - err := ToTyped(&out, v) - require.NoError(t, err) - - assert.Equal(t, []string{"SpecField"}, out.Spec.ForceSendFields) - assert.Equal(t, []string{"WrapperField"}, out.ForceSendFields) -} diff --git a/libs/dyn/drop_keys.go b/libs/dyn/drop_keys.go deleted file mode 100644 index 494f9b9cd88..00000000000 --- a/libs/dyn/drop_keys.go +++ /dev/null @@ -1,27 +0,0 @@ -package dyn - -func DropKeys(v Value, drop []string) (Value, error) { - var err error - nv, err := Walk(v, func(p Path, v Value) (Value, error) { - if len(p) == 0 { - return v, nil - } - - // Check if this key should be dropped. - for _, key := range drop { - if p[0].Key() != key { - continue - } - - return InvalidValue, ErrDrop - } - - // Pass through all other values. - return v, ErrSkip - }) - if err != nil { - return InvalidValue, err - } - - return nv, nil -} diff --git a/libs/dyn/drop_keys_test.go b/libs/dyn/drop_keys_test.go deleted file mode 100644 index 83a9744ca65..00000000000 --- a/libs/dyn/drop_keys_test.go +++ /dev/null @@ -1,24 +0,0 @@ -package dyn - -import ( - "testing" - - "github.com/stretchr/testify/require" -) - -func TestDropKeysTest(t *testing.T) { - v := V(map[string]Value{ - "key1": V("value1"), - "key2": V("value2"), - "key3": V("value3"), - }) - - vout, err := DropKeys(v, []string{"key1", "key3"}) - require.NoError(t, err) - - mv := vout.MustMap() - require.Equal(t, 1, mv.Len()) - v, ok := mv.GetByString("key2") - require.True(t, ok) - require.Equal(t, "value2", v.MustString()) -} diff --git a/libs/dyn/dynassert/assert.go b/libs/dyn/dynassert/assert.go deleted file mode 100644 index 9e02eeff39c..00000000000 --- a/libs/dyn/dynassert/assert.go +++ /dev/null @@ -1,36 +0,0 @@ -package dynassert - -import ( - "github.com/databricks/cli/libs/dyn" - "github.com/stretchr/testify/assert" -) - -// Equal is a variant of assert.Equal that first converts dyn.Value to any. The practical consequence is that maps are compared -// without order. Another difference here is that locations are compared via Location() function, which drops all -// but first and get rids of nils. -func Equal(t assert.TestingT, expected, actual any, msgAndArgs ...any) bool { - ev, eok := expected.(dyn.Value) - av, aok := actual.(dyn.Value) - if eok && aok && ev.IsValid() && av.IsValid() { - if !assert.Equal(t, ev.AsAny(), av.AsAny(), msgAndArgs...) { - return false - } - - // The values are equal on contents. Now compare the locations. - if !assert.Equal(t, ev.Location(), av.Location(), msgAndArgs...) { - return false - } - - // Walk ev and av and compare the locations of each element. - _, err := dyn.Walk(ev, func(p dyn.Path, evv dyn.Value) (dyn.Value, error) { - avv, err := dyn.GetByPath(av, p) - if assert.NoError(t, err, "unable to get value from actual value at path %v", p.String()) { - assert.Equal(t, evv.Location(), avv.Location()) - } - return evv, nil - }) - return assert.NoError(t, err) - } - - return assert.Equal(t, expected, actual, msgAndArgs...) -} diff --git a/libs/dyn/dynvar/lookup.go b/libs/dyn/dynvar/lookup.go deleted file mode 100644 index 2bc08f47d47..00000000000 --- a/libs/dyn/dynvar/lookup.go +++ /dev/null @@ -1,21 +0,0 @@ -package dynvar - -import ( - "errors" - - "github.com/databricks/cli/libs/dyn" -) - -// Lookup is the type of lookup functions that can be used with [Resolve]. -type Lookup func(path dyn.Path) (dyn.Value, error) - -// ErrSkipResolution is returned by a lookup function to indicate that the -// resolution of a variable reference should be skipped. -var ErrSkipResolution = errors.New("skip resolution") - -// DefaultLookup is the default lookup function used by [Resolve]. -func DefaultLookup(in dyn.Value) Lookup { - return func(path dyn.Path) (dyn.Value, error) { - return dyn.GetByPath(in, path) - } -} diff --git a/libs/dyn/dynvar/lookup_test.go b/libs/dyn/dynvar/lookup_test.go deleted file mode 100644 index 2341d72084b..00000000000 --- a/libs/dyn/dynvar/lookup_test.go +++ /dev/null @@ -1,27 +0,0 @@ -package dynvar_test - -import ( - "testing" - - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/dynvar" - "github.com/stretchr/testify/assert" -) - -func TestDefaultLookup(t *testing.T) { - lookup := dynvar.DefaultLookup(dyn.V(map[string]dyn.Value{ - "a": dyn.V("a"), - "b": dyn.V("b"), - })) - - v1, err := lookup(dyn.NewPath(dyn.Key("a"))) - assert.NoError(t, err) - assert.Equal(t, dyn.V("a"), v1) - - v2, err := lookup(dyn.NewPath(dyn.Key("b"))) - assert.NoError(t, err) - assert.Equal(t, dyn.V("b"), v2) - - _, err = lookup(dyn.NewPath(dyn.Key("c"))) - assert.True(t, dyn.IsNoSuchKeyError(err)) -} diff --git a/libs/dyn/dynvar/resolve.go b/libs/dyn/dynvar/resolve.go deleted file mode 100644 index 99981a2e51b..00000000000 --- a/libs/dyn/dynvar/resolve.go +++ /dev/null @@ -1,255 +0,0 @@ -package dynvar - -import ( - "errors" - "fmt" - "maps" - "slices" - "strings" - - "github.com/databricks/cli/libs/dyn" -) - -// Resolve resolves variable references in the given input value using the provided lookup function. -// It returns the resolved output value and any error encountered during the resolution process. -// -// For example, given the input value: -// -// { -// "a": "a", -// "b": "${a}", -// "c": "${b}${b}", -// } -// -// The output value will be: -// -// { -// "a": "a", -// "b": "a", -// "c": "aa", -// } -// -// If the input value contains a variable reference that cannot be resolved, an error is returned. -// If a cycle is detected in the variable references, an error is returned. -// If for some path the resolution function returns [ErrSkipResolution], the variable reference is left in place. -// This is useful when some variable references are not yet ready to be interpolated. -func Resolve(in dyn.Value, fn Lookup) (out dyn.Value, err error) { - return resolver{in: in, fn: fn}.run() -} - -// ReferenceError is returned for an unresolved variable reference. Suggestions -// are carried as data so callers (which can import libs/diag) format them. -type ReferenceError struct { - Reference string // original reference text, e.g. "var.hst" - Suggestions []string // corrected references, e.g. ["var.host", "var.hosts"] -} - -func (e *ReferenceError) Error() string { - return fmt.Sprintf("reference does not exist: ${%s}", e.Reference) -} - -type lookupResult struct { - v dyn.Value - err error -} - -type resolver struct { //nolint:recvcheck // value receiver for run(), pointer for mutation methods - in dyn.Value - fn Lookup - - refs map[string]Ref - resolved map[string]dyn.Value - - // Memoization for lookups. - lookups map[string]lookupResult -} - -func (r resolver) run() (out dyn.Value, err error) { - err = r.collectVariableReferences() - if err != nil { - return dyn.InvalidValue, err - } - - err = r.resolveVariableReferences() - if err != nil { - return dyn.InvalidValue, err - } - - out, err = r.replaceVariableReferences() - if err != nil { - return dyn.InvalidValue, err - } - - return out, nil -} - -func (r *resolver) collectVariableReferences() (err error) { - r.refs = make(map[string]Ref) - - // First walk the input to gather all values with a variable reference. - _, err = dyn.Walk(r.in, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - ref, ok := NewRef(v) - if !ok { - // Skip values without variable references. - return v, nil - } - - r.refs[p.String()] = ref - return v, nil - }) - - return err -} - -func (r *resolver) resolveVariableReferences() (err error) { - // Initialize cache for lookups. - r.lookups = make(map[string]lookupResult) - - // Initialize cache for resolved variable references. - r.resolved = make(map[string]dyn.Value) - - // Resolve each variable reference (in order). - // We sort the keys here to ensure that we always resolve the same variable reference first. - // This is done such that the cycle detection error is deterministic. If we did not do this, - // we could enter the cycle at any point in the cycle and return varying errors. - keys := slices.Sorted(maps.Keys(r.refs)) - for _, key := range keys { - v, err := r.resolveRef(r.refs[key], []string{key}) - if err != nil { - return err - } - r.resolved[key] = v - } - - return nil -} - -func (r *resolver) resolveRef(ref Ref, seen []string) (dyn.Value, error) { - // This is an unresolved variable reference. - deps := ref.References() - - // Resolve each of the dependencies, then interpolate them in the ref. - resolved := make([]dyn.Value, len(deps)) - complete := true - - for j, dep := range deps { - // Cycle detection. - if slices.Contains(seen, dep) { - return dyn.InvalidValue, fmt.Errorf( - "cycle detected in field resolution: %s", - strings.Join(append(seen, dep), " -> "), - ) - } - - v, err := r.resolveKey(dep, append(seen, dep)) - - // If we should skip resolution of this key, index j will hold an invalid [dyn.Value]. - if errors.Is(err, ErrSkipResolution) { - complete = false - continue - } else if err != nil { - // Otherwise, propagate the error. - return dyn.InvalidValue, err - } - - resolved[j] = v - } - - // Interpolate the resolved values. - if ref.IsPure() && complete { - // If the variable reference is pure, we can substitute it. - // This is useful for interpolating values of non-string types. - // - // Note: we use the location of the variable reference to preserve the information - // of where it is used. This also means that relative path resolution is done - // relative to where a variable is used, not where it is defined. - // - return dyn.NewValue(resolved[0].Value(), ref.Value.Locations()), nil - } - - // Not pure; perform string interpolation. - // - // Substitute by byte offset rather than by searching for the match text: the same - // reference may also appear escaped ("$${foo} ${foo}"), and a search would replace - // that occurrence instead, corrupting the literal and leaving the real one unresolved. - var sb strings.Builder - consumed := 0 - for j := range ref.Matches { - // The value is invalid if resolution returned [ErrSkipResolution]. - // We must skip those and leave the original variable reference in place, - // which happens naturally by not advancing past its span. - if !resolved[j].IsValid() { - continue - } - - // Try to turn the resolved value into a string. - s, ok := resolved[j].AsString() - if !ok { - // Only allow primitive types to be converted to string. - switch resolved[j].Kind() { - case dyn.KindString, dyn.KindBool, dyn.KindInt, dyn.KindFloat, dyn.KindTime, dyn.KindNil: - s = fmt.Sprint(resolved[j].AsAny()) - default: - return dyn.InvalidValue, fmt.Errorf("cannot interpolate non-primitive value of type %s into string", resolved[j].Kind()) - } - } - - start, end := ref.Spans[j][0], ref.Spans[j][1] - sb.WriteString(ref.Str[consumed:start]) - sb.WriteString(s) - consumed = end - } - sb.WriteString(ref.Str[consumed:]) - - return dyn.NewValue(sb.String(), ref.Value.Locations()), nil -} - -func (r *resolver) resolveKey(key string, seen []string) (dyn.Value, error) { - // Check if we have already looked up this key. - if v, ok := r.lookups[key]; ok { - return v.v, v.err - } - - // Parse the key into a path. - p, err := dyn.NewPathFromString(key) - if err != nil { - return dyn.InvalidValue, err - } - - // Look up the value for the given key. - v, err := r.fn(p) - if err != nil { - if dyn.IsNoSuchKeyError(err) { - // Carry suggestions as data; the caller formats them via libs/diag. - err = &ReferenceError{Reference: key, Suggestions: dyn.SuggestedReferences(err, key)} - } - - // Cache the return value and return to the caller. - r.lookups[key] = lookupResult{v: dyn.InvalidValue, err: err} - return dyn.InvalidValue, err - } - - // If the returned value is a valid variable reference, resolve it. - ref, ok := NewRef(v) - if ok { - v, err = r.resolveRef(ref, seen) - } - - // Cache the return value and return to the caller. - r.lookups[key] = lookupResult{v: v, err: err} - return v, err -} - -func (r *resolver) replaceVariableReferences() (dyn.Value, error) { - // Walk the input and replace all variable references. - return dyn.Walk(r.in, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - nv, ok := r.resolved[p.String()] - if !ok { - // No variable reference; return the original value. - return v, nil - } - - // We have a variable reference; return the resolved value. - return nv, nil - }) -} diff --git a/libs/dyn/dynvar/resolve_test.go b/libs/dyn/dynvar/resolve_test.go deleted file mode 100644 index c4b28e88d3f..00000000000 --- a/libs/dyn/dynvar/resolve_test.go +++ /dev/null @@ -1,430 +0,0 @@ -package dynvar_test - -import ( - "testing" - - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/dynvar" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func getByPath(t *testing.T, v dyn.Value, path string) dyn.Value { - v, err := dyn.Get(v, path) - require.NoError(t, err) - return v -} - -func TestResolve(t *testing.T) { - in := dyn.V(map[string]dyn.Value{ - "a": dyn.V("a"), - "b": dyn.V("${a}"), - "c": dyn.V("${a}"), - }) - - out, err := dynvar.Resolve(in, dynvar.DefaultLookup(in)) - require.NoError(t, err) - - assert.Equal(t, "a", getByPath(t, out, "a").MustString()) - assert.Equal(t, "a", getByPath(t, out, "b").MustString()) - assert.Equal(t, "a", getByPath(t, out, "c").MustString()) -} - -func TestResolveNotFound(t *testing.T) { - in := dyn.V(map[string]dyn.Value{ - "b": dyn.V("${a}"), - }) - - _, err := dynvar.Resolve(in, dynvar.DefaultLookup(in)) - require.ErrorContains(t, err, `reference does not exist: ${a}`) -} - -func TestResolveNotFoundSuggestsCloseKey(t *testing.T) { - in := dyn.V(map[string]dyn.Value{ - "host": dyn.V("example.com"), - "b": dyn.V("${hst}"), - }) - - _, err := dynvar.Resolve(in, dynvar.DefaultLookup(in)) - require.ErrorContains(t, err, "reference does not exist: ${hst}") - - var refErr *dynvar.ReferenceError - require.ErrorAs(t, err, &refErr) - assert.Equal(t, []string{"host"}, refErr.Suggestions) -} - -func TestResolveWithNesting(t *testing.T) { - in := dyn.V(map[string]dyn.Value{ - "a": dyn.V("${f.a}"), - "f": dyn.V(map[string]dyn.Value{ - "a": dyn.V("a"), - "b": dyn.V("${f.a}"), - }), - }) - - out, err := dynvar.Resolve(in, dynvar.DefaultLookup(in)) - require.NoError(t, err) - - assert.Equal(t, "a", getByPath(t, out, "a").MustString()) - assert.Equal(t, "a", getByPath(t, out, "f.a").MustString()) - assert.Equal(t, "a", getByPath(t, out, "f.b").MustString()) -} - -func TestResolveWithRecursion(t *testing.T) { - in := dyn.V(map[string]dyn.Value{ - "a": dyn.V("a"), - "b": dyn.V("${a}"), - "c": dyn.V("${b}"), - }) - - out, err := dynvar.Resolve(in, dynvar.DefaultLookup(in)) - require.NoError(t, err) - - assert.Equal(t, "a", getByPath(t, out, "a").MustString()) - assert.Equal(t, "a", getByPath(t, out, "b").MustString()) - assert.Equal(t, "a", getByPath(t, out, "c").MustString()) -} - -func TestResolveWithRecursionLoop(t *testing.T) { - in := dyn.V(map[string]dyn.Value{ - "a": dyn.V("a"), - "b": dyn.V("${c}"), - "c": dyn.V("${d}"), - "d": dyn.V("${b}"), - }) - - _, err := dynvar.Resolve(in, dynvar.DefaultLookup(in)) - assert.ErrorContains(t, err, "cycle detected in field resolution: b -> c -> d -> b") -} - -func TestResolveWithRecursionLoopSelf(t *testing.T) { - in := dyn.V(map[string]dyn.Value{ - "a": dyn.V("${a}"), - }) - - _, err := dynvar.Resolve(in, dynvar.DefaultLookup(in)) - assert.ErrorContains(t, err, "cycle detected in field resolution: a -> a") -} - -func TestResolveWithStringConcatenation(t *testing.T) { - in := dyn.V(map[string]dyn.Value{ - "a": dyn.V("a"), - "b": dyn.V("b"), - "c": dyn.V("${a}${b}${a}"), - }) - - out, err := dynvar.Resolve(in, dynvar.DefaultLookup(in)) - require.NoError(t, err) - - assert.Equal(t, "a", getByPath(t, out, "a").MustString()) - assert.Equal(t, "b", getByPath(t, out, "b").MustString()) - assert.Equal(t, "aba", getByPath(t, out, "c").MustString()) -} - -func TestResolveWithEscapedReference(t *testing.T) { - in := dyn.V(map[string]dyn.Value{ - "a": dyn.V("A"), - "escaped_only": dyn.V("$${a}"), - "escaped_first": dyn.V("$${a} ${a}"), - "real_first": dyn.V("${a} $${a}"), - "surrounded": dyn.V("$${a} ${a} $${a}"), - "other_name": dyn.V("$${zz} ${a}"), - }) - - out, err := dynvar.Resolve(in, dynvar.DefaultLookup(in)) - require.NoError(t, err) - - // The escape is preserved here; it is removed when the API payload is built. - assert.Equal(t, "$${a}", getByPath(t, out, "escaped_only").MustString()) - assert.Equal(t, "$${a} A", getByPath(t, out, "escaped_first").MustString()) - assert.Equal(t, "A $${a}", getByPath(t, out, "real_first").MustString()) - assert.Equal(t, "$${a} A $${a}", getByPath(t, out, "surrounded").MustString()) - assert.Equal(t, "$${zz} A", getByPath(t, out, "other_name").MustString()) -} - -func TestResolveWithTypeInterpolation(t *testing.T) { - in := dyn.V(map[string]dyn.Value{ - "a": dyn.V(1), - "b": dyn.V(2), - "c": dyn.V("${a} ${b}"), - "float_val": dyn.V(3.14), - "bool_true": dyn.V(true), - "bool_false": dyn.V(false), - "time_val": dyn.V(dyn.MustTime("2024-01-01")), - "nil_val": dyn.NilValue, - // Test interpolation of different types in string templates - "float_interp": dyn.V("Value: ${float_val}"), - "bool_true_interp": dyn.V("Enabled: ${bool_true}"), - "bool_false_interp": dyn.V("Disabled: ${bool_false}"), - "time_interp": dyn.V("Date: ${time_val}"), - "nil_interp": dyn.V("Null: ${nil_val}"), - }) - - out, err := dynvar.Resolve(in, dynvar.DefaultLookup(in)) - require.NoError(t, err) - - // Integer interpolation - assert.Equal(t, "1 2", getByPath(t, out, "c").MustString()) - - // Float interpolation - assert.Equal(t, "Value: 3.14", getByPath(t, out, "float_interp").MustString()) - - // Bool interpolation - assert.Equal(t, "Enabled: true", getByPath(t, out, "bool_true_interp").MustString()) - assert.Equal(t, "Disabled: false", getByPath(t, out, "bool_false_interp").MustString()) - - // Time interpolation should convert to string representation of time.Time - // Note: time.Time string representation includes timezone info - timeResult := getByPath(t, out, "time_interp").MustString() - assert.Contains(t, timeResult, "Date: 2024-01-01") - assert.Contains(t, timeResult, "00:00:00") - - // Nil interpolation - assert.Equal(t, "Null: ", getByPath(t, out, "nil_interp").MustString()) -} - -func TestResolveWithTypeRetentionFailure(t *testing.T) { - // Test that mapping interpolation fails with an error - mappingTest := dyn.V(map[string]dyn.Value{ - "mapping": dyn.V(map[string]dyn.Value{"key": dyn.V("value")}), - "interp": dyn.V("Config: ${mapping}"), - }) - _, err := dynvar.Resolve(mappingTest, dynvar.DefaultLookup(mappingTest)) - require.Error(t, err) - assert.Equal(t, "cannot interpolate non-primitive value of type map into string", err.Error()) -} - -func TestResolveWithTypeRetention(t *testing.T) { - in := dyn.V(map[string]dyn.Value{ - "int": dyn.V(1), - "int_var": dyn.V("${int}"), - "bool_true": dyn.V(true), - "bool_true_var": dyn.V("${bool_true}"), - "bool_false": dyn.V(false), - "bool_false_var": dyn.V("${bool_false}"), - "float": dyn.V(1.0), - "float_var": dyn.V("${float}"), - "string": dyn.V("a"), - "string_var": dyn.V("${string}"), - }) - - out, err := dynvar.Resolve(in, dynvar.DefaultLookup(in)) - require.NoError(t, err) - - assert.EqualValues(t, 1, getByPath(t, out, "int").MustInt()) - assert.EqualValues(t, 1, getByPath(t, out, "int_var").MustInt()) - - assert.True(t, getByPath(t, out, "bool_true").MustBool()) - assert.True(t, getByPath(t, out, "bool_true_var").MustBool()) - - assert.False(t, getByPath(t, out, "bool_false").MustBool()) - assert.False(t, getByPath(t, out, "bool_false_var").MustBool()) - - assert.Zero(t, 1.0-getByPath(t, out, "float").MustFloat()) - assert.Zero(t, 1.0-getByPath(t, out, "float_var").MustFloat()) - - assert.Equal(t, "a", getByPath(t, out, "string").MustString()) - assert.Equal(t, "a", getByPath(t, out, "string_var").MustString()) -} - -func TestResolveWithSkip(t *testing.T) { - in := dyn.V(map[string]dyn.Value{ - "a": dyn.V("a"), - "b": dyn.V("b"), - "c": dyn.V("${a}"), - "d": dyn.V("${b}"), - "e": dyn.V("${a} ${b}"), - "f": dyn.V("${b} ${a} ${a} ${b}"), - }) - - fallback := dynvar.DefaultLookup(in) - ignore := func(path dyn.Path) (dyn.Value, error) { - // If the variable reference to look up starts with "b", skip it. - if path.HasPrefix(dyn.NewPath(dyn.Key("b"))) { - return dyn.InvalidValue, dynvar.ErrSkipResolution - } - return fallback(path) - } - - out, err := dynvar.Resolve(in, ignore) - require.NoError(t, err) - - assert.Equal(t, "a", getByPath(t, out, "a").MustString()) - assert.Equal(t, "b", getByPath(t, out, "b").MustString()) - assert.Equal(t, "a", getByPath(t, out, "c").MustString()) - - // Check that the skipped variable references are not interpolated. - assert.Equal(t, "${b}", getByPath(t, out, "d").MustString()) - assert.Equal(t, "a ${b}", getByPath(t, out, "e").MustString()) - assert.Equal(t, "${b} a a ${b}", getByPath(t, out, "f").MustString()) //nolint:dupword -} - -func TestResolveWithSkipEverything(t *testing.T) { - in := dyn.V(map[string]dyn.Value{ - "a": dyn.V("a"), - "b": dyn.V("b"), - "c": dyn.V("${a}"), - "d": dyn.V("${b}"), - "e": dyn.V("${a} ${b}"), - "f": dyn.V("${b} ${a} ${a} ${b}"), - "g": dyn.V("${d} ${c} ${c} ${d}"), - }) - - // The call must not replace anything if the lookup function returns ErrSkipResolution. - out, err := dynvar.Resolve(in, func(path dyn.Path) (dyn.Value, error) { - return dyn.InvalidValue, dynvar.ErrSkipResolution - }) - require.NoError(t, err) - assert.Equal(t, "a", getByPath(t, out, "a").MustString()) - assert.Equal(t, "b", getByPath(t, out, "b").MustString()) - assert.Equal(t, "${a}", getByPath(t, out, "c").MustString()) - assert.Equal(t, "${b}", getByPath(t, out, "d").MustString()) - assert.Equal(t, "${a} ${b}", getByPath(t, out, "e").MustString()) - assert.Equal(t, "${b} ${a} ${a} ${b}", getByPath(t, out, "f").MustString()) - assert.Equal(t, "${d} ${c} ${c} ${d}", getByPath(t, out, "g").MustString()) -} - -func TestResolveWithInterpolateNewRef(t *testing.T) { - in := dyn.V(map[string]dyn.Value{ - "a": dyn.V("a"), - "b": dyn.V("${a}"), - }) - - // The call replaces ${a} with ${foobar} and skips everything else. - out, err := dynvar.Resolve(in, func(path dyn.Path) (dyn.Value, error) { - if path.String() == "a" { - return dyn.V("${foobar}"), nil - } - return dyn.InvalidValue, dynvar.ErrSkipResolution - }) - - require.NoError(t, err) - assert.Equal(t, "a", getByPath(t, out, "a").MustString()) - assert.Equal(t, "${foobar}", getByPath(t, out, "b").MustString()) -} - -func TestResolveWithInterpolateAliasedRef(t *testing.T) { - in := dyn.V(map[string]dyn.Value{ - "a": dyn.V("a"), - "b": dyn.V("${a}"), - "c": dyn.V("${x}"), - }) - - // The call replaces ${x} with ${b} and skips everything else. - out, err := dynvar.Resolve(in, func(path dyn.Path) (dyn.Value, error) { - if path.String() == "x" { - return dyn.V("${b}"), nil - } - return dyn.GetByPath(in, path) - }) - - require.NoError(t, err) - assert.Equal(t, "a", getByPath(t, out, "a").MustString()) - assert.Equal(t, "a", getByPath(t, out, "b").MustString()) - assert.Equal(t, "a", getByPath(t, out, "c").MustString()) -} - -func TestResolveIndexedRefs(t *testing.T) { - in := dyn.V(map[string]dyn.Value{ - "slice": dyn.V([]dyn.Value{dyn.V("a"), dyn.V("b")}), - "a": dyn.V("a: ${slice[0]}"), - }) - - out, err := dynvar.Resolve(in, dynvar.DefaultLookup(in)) - require.NoError(t, err) - - assert.Equal(t, "a: a", getByPath(t, out, "a").MustString()) -} - -func TestResolveIndexedRefsFromMap(t *testing.T) { - in := dyn.V(map[string]dyn.Value{ - "map": dyn.V( - map[string]dyn.Value{ - "slice": dyn.V([]dyn.Value{dyn.V("a")}), - }), - "a": dyn.V("a: ${map.slice[0]}"), - }) - - out, err := dynvar.Resolve(in, dynvar.DefaultLookup(in)) - require.NoError(t, err) - - assert.Equal(t, "a: a", getByPath(t, out, "a").MustString()) -} - -func TestResolveMapFieldFromIndexedRefs(t *testing.T) { - in := dyn.V(map[string]dyn.Value{ - "map": dyn.V( - map[string]dyn.Value{ - "slice": dyn.V([]dyn.Value{ - dyn.V(map[string]dyn.Value{ - "value": dyn.V("a"), - }), - }), - }), - "a": dyn.V("a: ${map.slice[0].value}"), - }) - - out, err := dynvar.Resolve(in, dynvar.DefaultLookup(in)) - require.NoError(t, err) - - assert.Equal(t, "a: a", getByPath(t, out, "a").MustString()) -} - -func TestResolveNestedIndexedRefs(t *testing.T) { - in := dyn.V(map[string]dyn.Value{ - "slice": dyn.V([]dyn.Value{ - dyn.V([]dyn.Value{dyn.V("a")}), - }), - "a": dyn.V("a: ${slice[0][0]}"), - }) - - out, err := dynvar.Resolve(in, dynvar.DefaultLookup(in)) - require.NoError(t, err) - - assert.Equal(t, "a: a", getByPath(t, out, "a").MustString()) -} - -func TestResolveMapVariable(t *testing.T) { - in := dyn.V(map[string]dyn.Value{ - "map": dyn.V(map[string]dyn.Value{ - "key1": dyn.V("value1"), - "key2": dyn.V("value2"), - }), - "var": dyn.V("${map}"), - }) - - out, err := dynvar.Resolve(in, dynvar.DefaultLookup(in)) - require.NoError(t, err) - - // Verify that the map variable was interpolated correctly - mapVal := getByPath(t, out, "var") - _, ok := mapVal.AsMap() - require.True(t, ok, "expected map value") - - // Verify the map contents - assert.Equal(t, "value1", getByPath(t, mapVal, "key1").MustString()) - assert.Equal(t, "value2", getByPath(t, mapVal, "key2").MustString()) -} - -func TestResolveSequenceVariable(t *testing.T) { - in := dyn.V(map[string]dyn.Value{ - "seq": dyn.V([]dyn.Value{ - dyn.V("value1"), - dyn.V("value2"), - }), - "var": dyn.V("${seq}"), - }) - - out, err := dynvar.Resolve(in, dynvar.DefaultLookup(in)) - require.NoError(t, err) - - // Verify that the sequence variable was interpolated correctly - seqVal := getByPath(t, out, "var") - seq, ok := seqVal.AsSequence() - require.True(t, ok, "expected sequence value") - require.Len(t, seq, 2) - - // Verify the sequence contents - assert.Equal(t, "value1", seq[0].MustString()) - assert.Equal(t, "value2", seq[1].MustString()) -} diff --git a/libs/dyn/jsonloader/json.go b/libs/dyn/jsonloader/json.go deleted file mode 100644 index 8db526bd97d..00000000000 --- a/libs/dyn/jsonloader/json.go +++ /dev/null @@ -1,134 +0,0 @@ -package jsonloader - -import ( - "bytes" - "encoding/json" - "errors" - "fmt" - "io" - - "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" -) - -func LoadJSON(data []byte, source string) (dyn.Value, error) { - offset := BuildLineOffsets(data) - offset.SetSource(source) - - reader := bytes.NewReader(data) - decoder := json.NewDecoder(reader) - - // Use json.Number to avoid losing precision on int64 values above 2^53 (e.g. job and run IDs). - decoder.UseNumber() - - // Start decoding from the top-level value - value, err := decodeValue(decoder, &offset) - if err != nil { - if err == io.EOF { - err = errors.New("unexpected end of JSON input") - } - // Get the current offset for error reporting - errorOffset := decoder.InputOffset() - errorLocation := offset.GetPosition(errorOffset) - return dyn.InvalidValue, fmt.Errorf("error decoding JSON at %s: %v", errorLocation, err) - } - - // Check if there are any remaining tokens (should not be valid JSON) - if decoder.More() { - errorOffset := decoder.InputOffset() - errorLocation := offset.GetPosition(errorOffset) - return dyn.InvalidValue, fmt.Errorf("error decoding JSON at %s: unexpected additional content", errorLocation) - } - - return value, nil -} - -func decodeValue(decoder *json.Decoder, o *Offset) (dyn.Value, error) { - // Read the next JSON token - token, err := decoder.Token() - if err != nil { - return dyn.InvalidValue, err - } - - // Get the current byte offset and the location. - // We will later use this location to store the location of the value in the file - // For objects and arrays, we will store the location of the opening '{' or '[' - // For primitive types, we will store the location of the value itself (end of the value) - // We can't reliably calculate the beginning of the value for primitive types because - // the decoder doesn't provide the offset of the beginning of the value and the value might or might not be quoted. - offset := decoder.InputOffset() - location := o.GetPosition(offset) - - switch tok := token.(type) { - case json.Delim: - switch tok { - case '{': - location = o.GetPosition(offset - 1) - // Decode JSON object - obj := dyn.NewMapping() - for decoder.More() { - // Decode the key - keyToken, err := decoder.Token() - if err != nil { - return invalidValueWithLocation(decoder, o), err - } - key, ok := keyToken.(string) - if !ok { - return invalidValueWithLocation(decoder, o), errors.New("expected string for object key") - } - - // Get the offset of the key by subtracting the length of the key and the '"' character - keyOffset := decoder.InputOffset() - int64(len(key)+1) - loc := []diag.Location{o.GetPosition(keyOffset)} - - // Decode the value recursively - val, err := decodeValue(decoder, o) - if err != nil { - return invalidValueWithLocation(decoder, o), err - } - - obj.SetLoc(key, loc, val) - } - // Consume the closing '}' - if _, err := decoder.Token(); err != nil { - return invalidValueWithLocation(decoder, o), err - } - return dyn.NewValue(obj, []diag.Location{location}), nil - case '[': - location = o.GetPosition(offset - 1) - // Decode JSON array - var arr []dyn.Value - for decoder.More() { - val, err := decodeValue(decoder, o) - if err != nil { - return invalidValueWithLocation(decoder, o), err - } - arr = append(arr, val) - } - // Consume the closing ']' - if _, err := decoder.Token(); err != nil { - return invalidValueWithLocation(decoder, o), err - } - return dyn.NewValue(arr, []diag.Location{location}), nil - } - case json.Number: - // Integers that overflow int64 fall back to float64, matching the decoder's behavior without UseNumber. - if i64, err := tok.Int64(); err == nil { - return dyn.NewValue(i64, []diag.Location{location}), nil - } - f64, err := tok.Float64() - if err != nil { - return invalidValueWithLocation(decoder, o), fmt.Errorf("invalid number %q: %w", tok.String(), err) - } - return dyn.NewValue(f64, []diag.Location{location}), nil - default: - return dyn.NewValue(tok, []diag.Location{location}), nil - } - - return invalidValueWithLocation(decoder, o), fmt.Errorf("unexpected token: %v", token) -} - -func invalidValueWithLocation(decoder *json.Decoder, o *Offset) dyn.Value { - location := o.GetPosition(decoder.InputOffset()) - return dyn.InvalidValue.WithLocations([]diag.Location{location}) -} diff --git a/libs/dyn/jsonloader/json_test.go b/libs/dyn/jsonloader/json_test.go deleted file mode 100644 index e274de559ce..00000000000 --- a/libs/dyn/jsonloader/json_test.go +++ /dev/null @@ -1,161 +0,0 @@ -package jsonloader - -import ( - "testing" - - "github.com/databricks/cli/libs/dyn/convert" - "github.com/databricks/databricks-sdk-go/service/jobs" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -const jsonData = ` -{ - "job_id": 123, - "new_settings": { - "name": "xxx", - "email_notifications": { - "on_start": [], - "on_success": [], - "on_failure": [] - }, - "webhook_notifications": { - "on_start": [], - "on_failure": [] - }, - "notification_settings": { - "no_alert_for_skipped_runs": true, - "no_alert_for_canceled_runs": true - }, - "timeout_seconds": 0, - "max_concurrent_runs": 1, - "tasks": [ - { - "task_key": "xxx", - "email_notifications": {}, - "notification_settings": {}, - "timeout_seconds": 0, - "max_retries": 0, - "min_retry_interval_millis": 0, - "retry_on_timeout": "true" - } - ] - } -} -` - -func TestJsonLoader(t *testing.T) { - v, err := LoadJSON([]byte(jsonData), "(inline)") - assert.NoError(t, err) - - var r jobs.ResetJob - err = convert.ToTyped(&r, v) - assert.NoError(t, err) -} - -const malformedMap = ` -{ - "job_id": 123, - "new_settings": { - "name": "xxx", - "wrong", - } -} -` - -func TestJsonLoaderMalformedMap(t *testing.T) { - _, err := LoadJSON([]byte(malformedMap), "(inline)") - assert.ErrorContains(t, err, "error decoding JSON at (inline):6:16: invalid character ',' after object key") -} - -const malformedArray = ` -{ - "job_id": 123, - "new_settings": { - "name": "xxx", - "tasks": [1, "asd",] - } -}` - -func TestJsonLoaderMalformedArray(t *testing.T) { - _, err := LoadJSON([]byte(malformedArray), "path/to/file.json") - assert.ErrorContains(t, err, "error decoding JSON at path/to/file.json:6:27: invalid character ',' looking for beginning of value") -} - -const eofData = ` -{ - "job_id": 123, - "new_settings": { - "name": "xxx",` - -func TestJsonLoaderEOF(t *testing.T) { - _, err := LoadJSON([]byte(eofData), "path/to/file.json") - assert.ErrorContains(t, err, "unexpected end of JSON input") -} - -const mapWithNoBraces = ` -"job_id": 123, -"new_settings": { - "name": "xxx", - "wrong": "xxx", -} -` - -func TestJsonMapWithoutBraces(t *testing.T) { - _, err := LoadJSON([]byte(mapWithNoBraces), "path/to/file.json") - assert.ErrorContains(t, err, "error decoding JSON at") -} - -const validInline = `["job_id", 123]` - -func TestJsonValidInline(t *testing.T) { - _, err := LoadJSON([]byte(validInline), "path/to/file.json") - assert.NoError(t, err) -} - -func TestJsonLoaderNumbers(t *testing.T) { - for _, tc := range []struct { - input string - expected any - }{ - {`123`, int64(123)}, - {`-1`, int64(-1)}, - {`123456789012345678`, int64(123456789012345678)}, - {`-123456789012345678`, int64(-123456789012345678)}, - {`9223372036854775807`, int64(9223372036854775807)}, - {`2.0`, 2.0}, - {`2.5`, 2.5}, - {`1e3`, 1000.0}, - {`18446744073709551615`, 1.8446744073709552e+19}, - } { - v, err := LoadJSON([]byte(tc.input), "(inline)") - assert.NoError(t, err, tc.input) - assert.Equal(t, tc.expected, v.AsAny(), tc.input) - } -} - -func TestJsonLoaderNumberOutOfRange(t *testing.T) { - _, err := LoadJSON([]byte(`1e400`), "(inline)") - assert.ErrorContains(t, err, "value out of range") -} - -const mixedNumbersData = ` -{ - "job_id": 123456789012345678, - "new_settings": { - "name": "xxx", - "timeout_seconds": 100 - } -} -` - -func TestJsonLoaderMixedNumbersToTyped(t *testing.T) { - v, err := LoadJSON([]byte(mixedNumbersData), "(inline)") - require.NoError(t, err) - - var r jobs.ResetJob - err = convert.ToTyped(&r, v) - require.NoError(t, err) - assert.Equal(t, int64(123456789012345678), r.JobId) - assert.Equal(t, 100, r.NewSettings.TimeoutSeconds) -} diff --git a/libs/dyn/jsonloader/locations.go b/libs/dyn/jsonloader/locations.go deleted file mode 100644 index 3dbe2da6630..00000000000 --- a/libs/dyn/jsonloader/locations.go +++ /dev/null @@ -1,49 +0,0 @@ -package jsonloader - -import ( - "sort" - - "github.com/databricks/cli/libs/diag" -) - -type LineOffset struct { - Line int - Start int64 -} - -type Offset struct { //nolint:recvcheck // value receiver for read-only GetPosition, pointer for SetSource - offsets []LineOffset - source string -} - -// buildLineOffsets scans the input data and records the starting byte offset of each line. -func BuildLineOffsets(data []byte) Offset { - offsets := []LineOffset{{Line: 1, Start: 0}} - line := 1 - for i, b := range data { - if b == '\n' { - line++ - offsets = append(offsets, LineOffset{Line: line, Start: int64(i + 1)}) - } - } - return Offset{offsets: offsets} -} - -// GetPosition maps a byte offset to its corresponding line and column numbers. -func (o Offset) GetPosition(offset int64) diag.Location { - // Binary search to find the line - idx := max(sort.Search(len(o.offsets), func(i int) bool { - return o.offsets[i].Start > offset - })-1, 0) - - lineOffset := o.offsets[idx] - return diag.Location{ - File: o.source, - Line: lineOffset.Line, - Column: int(offset-lineOffset.Start) + 1, - } -} - -func (o *Offset) SetSource(source string) { - o.source = source -} diff --git a/libs/dyn/kind_test.go b/libs/dyn/kind_test.go deleted file mode 100644 index 84c90713fb8..00000000000 --- a/libs/dyn/kind_test.go +++ /dev/null @@ -1,38 +0,0 @@ -package dyn_test - -import ( - "testing" - - "github.com/databricks/cli/libs/dyn" - "github.com/stretchr/testify/assert" -) - -func TestKindZeroValue(t *testing.T) { - // Assert that the zero value of [dyn.Kind] is the invalid kind. - var k dyn.Kind - assert.Equal(t, dyn.KindInvalid, k) -} - -func TestKindToString(t *testing.T) { - for _, tt := range []struct { - k dyn.Kind - s string - }{ - {dyn.KindInvalid, "invalid"}, - {dyn.KindMap, "map"}, - {dyn.KindSequence, "sequence"}, - {dyn.KindString, "string"}, - {dyn.KindBool, "bool"}, - {dyn.KindInt, "int"}, - {dyn.KindFloat, "float"}, - {dyn.KindTime, "time"}, - {dyn.KindNil, "nil"}, - } { - assert.Equal(t, tt.s, tt.k.String()) - } - - // Panic on unknown kind. - assert.PanicsWithValue(t, "invalid kind value: 100", func() { - _ = dyn.Kind(100).String() - }) -} diff --git a/libs/dyn/mapping.go b/libs/dyn/mapping.go deleted file mode 100644 index 8b29181e40f..00000000000 --- a/libs/dyn/mapping.go +++ /dev/null @@ -1,158 +0,0 @@ -package dyn - -import ( - "maps" - "slices" - - "github.com/databricks/cli/libs/diag" -) - -// Pair represents a single key-value pair in a Mapping. -type Pair struct { - Key Value - Value Value -} - -// Mapping represents a key-value map of dynamic values. -// It exists because plain Go maps cannot use dynamic values for keys. -// We need to use dynamic values for keys because it lets us associate metadata -// with keys (i.e. their definition location). Keys must be strings. -type Mapping struct { //nolint:recvcheck // value receivers for read-only accessors, pointer for mutators - pairs []Pair - index map[string]int -} - -// NewMapping creates a new empty Mapping. -func NewMapping() Mapping { - return Mapping{ - pairs: make([]Pair, 0), - index: make(map[string]int), - } -} - -// NewMappingFromPairs computes a [Mapping] from a list of [Pair]s. The index -// map does not need to be provided since that will be computed from the -// key-value pairs provided. -func NewMappingFromPairs(pairs []Pair) Mapping { - index := make(map[string]int) - for i, p := range pairs { - index[p.Key.MustString()] = i - } - - return Mapping{ - pairs: pairs, - index: index, - } -} - -// newMappingWithSize creates a new Mapping preallocated to the specified size. -func newMappingWithSize(size int) Mapping { - return Mapping{ - pairs: make([]Pair, 0, size), - index: make(map[string]int, size), - } -} - -// newMappingFromGoMap creates a new Mapping from a Go map of string keys and dynamic values. -func newMappingFromGoMap(vin map[string]Value) Mapping { - m := newMappingWithSize(len(vin)) - for k, v := range vin { - m.SetLoc(k, nil, v) - } - return m -} - -// Pairs returns all the key-value pairs in the Mapping. The pairs are sorted by -// their key in lexicographic order. -func (m Mapping) Pairs() []Pair { - return m.pairs -} - -// Len returns the number of key-value pairs in the Mapping. -func (m Mapping) Len() int { - return len(m.pairs) -} - -// GetPair returns the key-value pair with the specified key. -// It also returns a boolean indicating whether the pair was found. -func (m Mapping) GetPair(key Value) (Pair, bool) { - skey, ok := key.AsString() - if !ok { - return Pair{}, false - } - return m.GetPairByString(skey) -} - -// GetPairByString returns the key-value pair with the specified string key. -// It also returns a boolean indicating whether the pair was found. -func (m Mapping) GetPairByString(skey string) (Pair, bool) { - if i, ok := m.index[skey]; ok { - return m.pairs[i], true - } - return Pair{}, false -} - -// Get returns the value associated with the specified key. -// It also returns a boolean indicating whether the value was found. -func (m Mapping) Get(key Value) (Value, bool) { - p, ok := m.GetPair(key) - return p.Value, ok -} - -// GetByString returns the value associated with the specified string key. -// It also returns a boolean indicating whether the value was found. -func (m *Mapping) GetByString(skey string) (Value, bool) { - p, ok := m.GetPairByString(skey) - return p.Value, ok -} - -// Set sets the value for the given key in the mapping. -// If the key already exists, the value is updated. The location loc is ignored. -// If the key does not exist, a new key-value pair is added. -func (m *Mapping) SetLoc(skey string, loc []diag.Location, value Value) { - // If the key already exists, update the value. - if i, ok := m.index[skey]; ok { - m.pairs[i].Value = value - return - } - - // Otherwise, add a new pair. - m.pairs = append(m.pairs, Pair{NewValue(skey, loc), value}) - if m.index == nil { - m.index = make(map[string]int) - } - m.index[skey] = len(m.pairs) - 1 -} - -// Keys returns all the keys in the Mapping. -func (m Mapping) Keys() []Value { - keys := make([]Value, 0, len(m.pairs)) - for _, p := range m.pairs { - keys = append(keys, p.Key) - } - return keys -} - -// Values returns all the values in the Mapping. -func (m Mapping) Values() []Value { - values := make([]Value, 0, len(m.pairs)) - for _, p := range m.pairs { - values = append(values, p.Value) - } - return values -} - -// Clone creates a shallow copy of the Mapping. -func (m Mapping) Clone() Mapping { - return Mapping{ - pairs: slices.Clone(m.pairs), - index: maps.Clone(m.index), - } -} - -// Merge merges the key-value pairs from another Mapping into the current Mapping. -func (m *Mapping) Merge(n Mapping) { - for _, p := range n.pairs { - m.SetLoc(p.Key.MustString(), p.Key.Locations(), p.Value) - } -} diff --git a/libs/dyn/mapping_test.go b/libs/dyn/mapping_test.go deleted file mode 100644 index f646ab7c1d1..00000000000 --- a/libs/dyn/mapping_test.go +++ /dev/null @@ -1,182 +0,0 @@ -package dyn_test - -import ( - "strconv" - "testing" - - "github.com/databricks/cli/libs/dyn" - "github.com/stretchr/testify/assert" -) - -func TestNewMapping(t *testing.T) { - m := dyn.NewMapping() - assert.Equal(t, 0, m.Len()) -} - -func TestMappingZeroValue(t *testing.T) { - var m dyn.Mapping - assert.Equal(t, 0, m.Len()) - - value, ok := m.Get(dyn.V("key")) - assert.Equal(t, dyn.InvalidValue, value) - assert.False(t, ok) - assert.Empty(t, m.Keys()) - assert.Empty(t, m.Values()) -} - -func TestMappingGet(t *testing.T) { - var m dyn.Mapping - m.SetLoc("key", nil, dyn.V("value")) - assert.Equal(t, 1, m.Len()) - - // Call GetPair - p, ok := m.GetPair(dyn.V("key")) - assert.True(t, ok) - assert.Equal(t, dyn.V("key"), p.Key) - assert.Equal(t, dyn.V("value"), p.Value) - - // Modify the value to make sure we're not getting a reference - p.Value = dyn.V("newvalue") - - // Call GetPair with invalid key - p, ok = m.GetPair(dyn.V(1234)) - assert.False(t, ok) - assert.Equal(t, dyn.InvalidValue, p.Key) - assert.Equal(t, dyn.InvalidValue, p.Value) - - // Call GetPair with non-existent key - p, ok = m.GetPair(dyn.V("enoexist")) - assert.False(t, ok) - assert.Equal(t, dyn.InvalidValue, p.Key) - assert.Equal(t, dyn.InvalidValue, p.Value) - - // Call GetPairByString - p, ok = m.GetPairByString("key") - assert.True(t, ok) - assert.Equal(t, dyn.V("key"), p.Key) - assert.Equal(t, dyn.V("value"), p.Value) - - // Modify the value to make sure we're not getting a reference - p.Value = dyn.V("newvalue") - - // Call GetPairByString with non-existent key - p, ok = m.GetPairByString("enoexist") - assert.False(t, ok) - assert.Equal(t, dyn.InvalidValue, p.Key) - assert.Equal(t, dyn.InvalidValue, p.Value) - - // Call Get - value, ok := m.Get(dyn.V("key")) - assert.True(t, ok) - assert.Equal(t, dyn.V("value"), value) - - // Call Get with invalid key - value, ok = m.Get(dyn.V(1234)) - assert.False(t, ok) - assert.Equal(t, dyn.InvalidValue, value) - - // Call Get with non-existent key - value, ok = m.Get(dyn.V("enoexist")) - assert.False(t, ok) - assert.Equal(t, dyn.InvalidValue, value) - - // Call GetByString - value, ok = m.GetByString("key") - assert.True(t, ok) - assert.Equal(t, dyn.V("value"), value) - - // Call GetByString with non-existent key - value, ok = m.GetByString("enoexist") - assert.False(t, ok) - assert.Equal(t, dyn.InvalidValue, value) -} - -func TestMappingSetLoc(t *testing.T) { - var m dyn.Mapping - - // Set a value - m.SetLoc("key1", nil, dyn.V("foo")) - assert.Equal(t, 1, m.Len()) - - // Confirm the value - value, ok := m.GetByString("key1") - assert.True(t, ok) - assert.Equal(t, dyn.V("foo"), value) - - // Set another value - m.SetLoc("key2", nil, dyn.V("bar")) - assert.Equal(t, 2, m.Len()) - - // Confirm the value - value, ok = m.Get(dyn.V("key2")) - assert.True(t, ok) - assert.Equal(t, dyn.V("bar"), value) - - // Overwrite first value - m.SetLoc("key1", nil, dyn.V("qux")) - assert.Equal(t, 2, m.Len()) - - // Confirm the value - value, ok = m.Get(dyn.V("key1")) - assert.True(t, ok) - assert.Equal(t, dyn.V("qux"), value) -} - -func TestMappingKeysValues(t *testing.T) { - // Configure mapping - var m dyn.Mapping - m.SetLoc("key1", nil, dyn.V("foo")) - m.SetLoc("key2", nil, dyn.V("bar")) - - // Confirm keys - keys := m.Keys() - assert.Len(t, keys, 2) - assert.Contains(t, keys, dyn.V("key1")) - assert.Contains(t, keys, dyn.V("key2")) - - // Confirm values - values := m.Values() - assert.Len(t, values, 2) - assert.Contains(t, values, dyn.V("foo")) - assert.Contains(t, values, dyn.V("bar")) -} - -func TestMappingClone(t *testing.T) { - // Configure mapping - var m1 dyn.Mapping - m1.SetLoc("key1", nil, dyn.V("foo")) - m1.SetLoc("key2", nil, dyn.V("bar")) - - // Clone mapping - m2 := m1.Clone() - assert.Equal(t, m1.Len(), m2.Len()) - - // Modify original mapping - m1.SetLoc("key1", nil, dyn.V("qux")) - - // Confirm values - value, ok := m1.Get(dyn.V("key1")) - assert.True(t, ok) - assert.Equal(t, dyn.V("qux"), value) - value, ok = m2.Get(dyn.V("key1")) - assert.True(t, ok) - assert.Equal(t, dyn.V("foo"), value) -} - -func TestMappingMerge(t *testing.T) { - var m1 dyn.Mapping - for i := range 10 { - m1.SetLoc(strconv.Itoa(i), nil, dyn.V(i)) - } - - var m2 dyn.Mapping - for i := 5; i < 15; i++ { - m2.SetLoc(strconv.Itoa(i), nil, dyn.V(i)) - } - - var out dyn.Mapping - out.Merge(m1) - assert.Equal(t, 10, out.Len()) - out.Merge(m2) - assert.Equal(t, 15, out.Len()) -} diff --git a/libs/dyn/merge/elements_by_key.go b/libs/dyn/merge/elements_by_key.go deleted file mode 100644 index 6bf71d41ae5..00000000000 --- a/libs/dyn/merge/elements_by_key.go +++ /dev/null @@ -1,107 +0,0 @@ -package merge - -import ( - "slices" - - "github.com/databricks/cli/libs/dyn" -) - -type elementsByKey struct { - key string - keyFunc func(dyn.Value) string - sortKeys bool -} - -func (e elementsByKey) doMap(_ dyn.Path, v dyn.Value, mergeFunc func(a, b dyn.Value) (dyn.Value, error)) (dyn.Value, error) { - // We know the type of this value is a sequence. - // For additional defence, return self if it is not. - elements, ok := v.AsSequence() - if !ok { - return v, nil - } - - seen := make(map[string]dyn.Value, len(elements)) - keys := make([]string, 0, len(elements)) - - // Iterate in natural order. For a given key, we first see the - // base definition and merge instances that come after it. - for i := range elements { - kv := elements[i].Get(e.key) - key := e.keyFunc(kv) - - // Register element with key if not yet seen before. - ref, ok := seen[key] - if !ok { - keys = append(keys, key) - seen[key] = elements[i] - continue - } - - // Merge this instance into the reference. - nv, err := mergeFunc(ref, elements[i]) - if err != nil { - return v, err - } - - // Overwrite reference. - seen[key] = nv - } - - if e.sortKeys { - slices.Sort(keys) - } - - // Gather resulting elements in natural order. - out := make([]dyn.Value, 0, len(keys)) - for _, key := range keys { - // Preserve the location from the original key value so that - // downstream code can tell this field was defined in the YAML. - keyLoc := seen[key].Get(e.key).Locations() - nv, err := dyn.Set(seen[key], e.key, dyn.NewValue(key, keyLoc)) - if err != nil { - return dyn.InvalidValue, err - } - out = append(out, nv) - } - - return dyn.NewValue(out, v.Locations()), nil -} - -func (e elementsByKey) Map(_ dyn.Path, v dyn.Value) (dyn.Value, error) { - return e.doMap(nil, v, Merge) -} - -func (e elementsByKey) MapWithOverride(p dyn.Path, v dyn.Value) (dyn.Value, error) { - return e.doMap(nil, v, func(a, b dyn.Value) (dyn.Value, error) { - return Override(a, b, OverrideVisitor{ - VisitInsert: func(_ dyn.Path, v dyn.Value) (dyn.Value, error) { - return v, nil - }, - VisitDelete: func(valuePath dyn.Path, left dyn.Value) error { - return nil - }, - VisitUpdate: func(_ dyn.Path, a, b dyn.Value) (dyn.Value, error) { - return b, nil - }, - }) - }) -} - -// ElementsByKey returns a [dyn.MapFunc] that operates on a sequence -// where each element is a map. It groups elements by a key and merges -// elements with the same key. -// -// The function that extracts the key from an element is provided as -// a parameter. The resulting elements get their key field overwritten -// with the value as returned by the key function. -func ElementsByKey(key string, keyFunc func(dyn.Value) string) dyn.MapFunc { - return elementsByKey{key, keyFunc, false}.Map -} - -func ElementsBySortedKey(key string, keyFunc func(dyn.Value) string) dyn.MapFunc { - return elementsByKey{key, keyFunc, true}.Map -} - -func ElementsByKeyWithOverride(key string, keyFunc func(dyn.Value) string) dyn.MapFunc { - return elementsByKey{key, keyFunc, false}.MapWithOverride -} diff --git a/libs/dyn/merge/elements_by_key_test.go b/libs/dyn/merge/elements_by_key_test.go deleted file mode 100644 index 2aa19306820..00000000000 --- a/libs/dyn/merge/elements_by_key_test.go +++ /dev/null @@ -1,109 +0,0 @@ -package merge - -import ( - "strings" - "testing" - - "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestElementByKey(t *testing.T) { - vin := dyn.V([]dyn.Value{ - dyn.V(map[string]dyn.Value{ - "key": dyn.V("foo"), - "value": dyn.V(42), - }), - dyn.V(map[string]dyn.Value{ - "key": dyn.V("bar"), - "value": dyn.V(43), - }), - dyn.V(map[string]dyn.Value{ - // Use upper case key to test that the resulting element has its - // key field assigned to the output of the key function. - // The key function in this test returns the lower case version of the key. - "key": dyn.V("FOO"), - "value": dyn.V(44), - }), - }) - - keyFunc := func(v dyn.Value) string { - return strings.ToLower(v.MustString()) - } - - vout, err := dyn.MapByPath(vin, dyn.EmptyPath, ElementsByKey("key", keyFunc)) - require.NoError(t, err) - assert.Len(t, vout.MustSequence(), 2) - assert.Equal(t, - map[string]any{ - "key": "foo", - "value": 44, - }, vout.Index(0).AsAny(), - ) - assert.Equal(t, - map[string]any{ - "key": "bar", - "value": 43, - }, vout.Index(1).AsAny(), - ) -} - -func TestElementByKeyPreservesLocations(t *testing.T) { - loc := diag.Location{File: "config.yml", Line: 10, Column: 5} - vin := dyn.V([]dyn.Value{ - dyn.V(map[string]dyn.Value{ - "key": dyn.NewValue("foo", []diag.Location{loc}), - "value": dyn.V(42), - }), - }) - - keyFunc := func(v dyn.Value) string { - return v.MustString() - } - - vout, err := dyn.MapByPath(vin, dyn.EmptyPath, ElementsByKey("key", keyFunc)) - require.NoError(t, err) - - // Verify the key field retains its original location. - keyValue := vout.Index(0).Get("key") - assert.Equal(t, loc, keyValue.Location()) -} - -func TestElementByKeyWithOverride(t *testing.T) { - vin := dyn.V([]dyn.Value{ - dyn.V(map[string]dyn.Value{ - "key": dyn.V("foo"), - "value": dyn.V(42), - }), - dyn.V(map[string]dyn.Value{ - "key": dyn.V("bar"), - "value": dyn.V(43), - }), - dyn.V(map[string]dyn.Value{ - "key": dyn.V("foo"), - "othervalue": dyn.V(44), - }), - }) - - keyFunc := func(v dyn.Value) string { - return strings.ToLower(v.MustString()) - } - - vout, err := dyn.MapByPath(vin, dyn.EmptyPath, ElementsByKeyWithOverride("key", keyFunc)) - require.NoError(t, err) - assert.Len(t, vout.MustSequence(), 2) - assert.Equal(t, - map[string]any{ - "key": "foo", - "othervalue": 44, - }, vout.Index(0).AsAny(), - ) - assert.Equal(t, - map[string]any{ - "key": "bar", - "value": 43, - }, vout.Index(1).AsAny(), - ) -} diff --git a/libs/dyn/merge/merge.go b/libs/dyn/merge/merge.go deleted file mode 100644 index 374450ef240..00000000000 --- a/libs/dyn/merge/merge.go +++ /dev/null @@ -1,119 +0,0 @@ -package merge - -import ( - "fmt" - - "github.com/databricks/cli/libs/dyn" -) - -// Merge recursively merges the specified values. -// -// Semantics are as follows: -// * Merging x with nil or nil with x always yields x. -// * Merging maps a and b means entries from map b take precedence. -// * Merging sequences a and b means concatenating them. -// -// Merging retains and accumulates the locations metadata associated with the values. -// This allows users of the module to track the provenance of values across merging of -// configuration trees, which is useful for reporting errors and warnings. -// -// Semantics for location metadata in the merged value are similar to the semantics -// for the values themselves: -// -// - When merging x with nil or nil with x, the location of x is retained. -// -// - When merging maps or sequences, the combined value retains the location of a and -// accumulates the location of b. The individual elements of the map or sequence retain -// their original locations, i.e., whether they were originally defined in a or b. -// -// The rationale for retaining location of a is that we would like to return -// the first location a bit of configuration showed up when reporting errors and warnings. -// -// - Merging primitive values means using the incoming value `b`. The location of the -// incoming value is retained and the location of the existing value `a` is accumulated. -// This is because the incoming value overwrites the existing value. -func Merge(a, b dyn.Value) (dyn.Value, error) { - return merge(a, b) -} - -func merge(a, b dyn.Value) (dyn.Value, error) { - ak := a.Kind() - bk := b.Kind() - - // If a is nil, return b. - if ak == dyn.KindNil { - return b.AppendLocationsFromValue(a), nil - } - - // If b is nil, return a. - if bk == dyn.KindNil { - return a.AppendLocationsFromValue(b), nil - } - - // Call the appropriate merge function based on the kind of a and b. - switch ak { - case dyn.KindMap: - if bk != dyn.KindMap { - return dyn.InvalidValue, fmt.Errorf("cannot merge map with %s", bk) - } - return mergeMap(a, b) - case dyn.KindSequence: - if bk != dyn.KindSequence { - return dyn.InvalidValue, fmt.Errorf("cannot merge sequence with %s", bk) - } - return mergeSequence(a, b) - default: - if ak != bk { - return dyn.InvalidValue, fmt.Errorf("cannot merge %s with %s", ak, bk) - } - return mergePrimitive(a, b) - } -} - -func mergeMap(a, b dyn.Value) (dyn.Value, error) { - out := dyn.NewMapping() - am := a.MustMap() - bm := b.MustMap() - - // Add the values from a into the output map. - out.Merge(am) - - // Merge the values from b into the output map. - for _, pair := range bm.Pairs() { - pk := pair.Key - key := pk.MustString() - pv := pair.Value - if ov, ok := out.Get(pk); ok { - // If the key already exists, merge the values. - merged, err := merge(ov, pv) - if err != nil { - return dyn.InvalidValue, err - } - out.SetLoc(key, pair.Key.Locations(), merged) - } else { - // Otherwise, just set the value. - out.SetLoc(key, pair.Key.Locations(), pv) - } - } - - // Preserve the location of the first value. Accumulate the locations of the second value. - return dyn.NewValue(out, a.Locations()).AppendLocationsFromValue(b), nil -} - -func mergeSequence(a, b dyn.Value) (dyn.Value, error) { - as := a.MustSequence() - bs := b.MustSequence() - - // Merging sequences means concatenating them. - out := make([]dyn.Value, len(as)+len(bs)) - copy(out[:], as) - copy(out[len(as):], bs) - - // Preserve the location of the first value. Accumulate the locations of the second value. - return dyn.NewValue(out, a.Locations()).AppendLocationsFromValue(b), nil -} - -func mergePrimitive(a, b dyn.Value) (dyn.Value, error) { - // Merging primitive values means using the incoming value. - return b.AppendLocationsFromValue(a), nil -} diff --git a/libs/dyn/merge/merge_test.go b/libs/dyn/merge/merge_test.go deleted file mode 100644 index 9545c752258..00000000000 --- a/libs/dyn/merge/merge_test.go +++ /dev/null @@ -1,301 +0,0 @@ -package merge - -import ( - "testing" - - "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/stretchr/testify/assert" -) - -func TestMergeMaps(t *testing.T) { - l1 := diag.Location{File: "file1", Line: 1, Column: 2} - v1 := dyn.NewValue(map[string]dyn.Value{ - "foo": dyn.NewValue("bar", []diag.Location{l1}), - "bar": dyn.NewValue("baz", []diag.Location{l1}), - }, []diag.Location{l1}) - - l2 := diag.Location{File: "file2", Line: 3, Column: 4} - v2 := dyn.NewValue(map[string]dyn.Value{ - "bar": dyn.NewValue("qux", []diag.Location{l2}), - "qux": dyn.NewValue("foo", []diag.Location{l2}), - }, []diag.Location{l2}) - - // Merge v2 into v1. - { - out, err := Merge(v1, v2) - assert.NoError(t, err) - assert.Equal(t, map[string]any{ - "foo": "bar", - "bar": "qux", - "qux": "foo", - }, out.AsAny()) - - // Locations of both values should be preserved. - assert.Equal(t, []diag.Location{l1, l2}, out.Locations()) - assert.Equal(t, []diag.Location{l2, l1}, out.Get("bar").Locations()) - assert.Equal(t, []diag.Location{l1}, out.Get("foo").Locations()) - assert.Equal(t, []diag.Location{l2}, out.Get("qux").Locations()) - - // Location of the merged value should be the location of v1. - assert.Equal(t, l1, out.Location()) - - // Value of bar is "qux" which comes from v2. This .Location() should - // return the location of v2. - assert.Equal(t, l2, out.Get("bar").Location()) - - // Original locations of keys that were not overwritten should be preserved. - assert.Equal(t, l1, out.Get("foo").Location()) - assert.Equal(t, l2, out.Get("qux").Location()) - } - - // Merge v1 into v2. - { - out, err := Merge(v2, v1) - assert.NoError(t, err) - assert.Equal(t, map[string]any{ - "foo": "bar", - "bar": "baz", - "qux": "foo", - }, out.AsAny()) - - // Locations of both values should be preserved. - assert.Equal(t, []diag.Location{l2, l1}, out.Locations()) - assert.Equal(t, []diag.Location{l1, l2}, out.Get("bar").Locations()) - assert.Equal(t, []diag.Location{l1}, out.Get("foo").Locations()) - assert.Equal(t, []diag.Location{l2}, out.Get("qux").Locations()) - - // Location of the merged value should be the location of v2. - assert.Equal(t, l2, out.Location()) - - // Value of bar is "baz" which comes from v1. This .Location() should - // return the location of v1. - assert.Equal(t, l1, out.Get("bar").Location()) - - // Original locations of keys that were not overwritten should be preserved. - assert.Equal(t, l1, out.Get("foo").Location()) - assert.Equal(t, l2, out.Get("qux").Location()) - } -} - -func TestMergeMapsNil(t *testing.T) { - l := diag.Location{File: "file", Line: 1, Column: 2} - v := dyn.NewValue(map[string]dyn.Value{ - "foo": dyn.V("bar"), - }, []diag.Location{l}) - - nilL := diag.Location{File: "file", Line: 3, Column: 4} - nilV := dyn.NewValue(nil, []diag.Location{nilL}) - - // Merge nil into v. - { - out, err := Merge(v, nilV) - assert.NoError(t, err) - assert.Equal(t, map[string]any{ - "foo": "bar", - }, out.AsAny()) - - // Locations of both values should be preserved. - assert.Equal(t, []diag.Location{l, nilL}, out.Locations()) - - // Location of the non-nil value should be returned by .Location(). - assert.Equal(t, l, out.Location()) - } - - // Merge v into nil. - { - out, err := Merge(nilV, v) - assert.NoError(t, err) - assert.Equal(t, map[string]any{ - "foo": "bar", - }, out.AsAny()) - - // Locations of both values should be preserved. - assert.Equal(t, []diag.Location{l, nilL}, out.Locations()) - - // Location of the non-nil value should be returned by .Location(). - assert.Equal(t, l, out.Location()) - } -} - -func TestMergeMapsError(t *testing.T) { - v := dyn.V(map[string]dyn.Value{ - "foo": dyn.V("bar"), - }) - - other := dyn.V("string") - - // Merge a string into v. - { - out, err := Merge(v, other) - assert.EqualError(t, err, "cannot merge map with string") - assert.Equal(t, dyn.InvalidValue, out) - } -} - -func TestMergeSequences(t *testing.T) { - l1 := diag.Location{File: "file1", Line: 1, Column: 2} - v1 := dyn.NewValue([]dyn.Value{ - dyn.NewValue("bar", []diag.Location{l1}), - dyn.NewValue("baz", []diag.Location{l1}), - }, []diag.Location{l1}) - - l2 := diag.Location{File: "file2", Line: 3, Column: 4} - l3 := diag.Location{File: "file3", Line: 5, Column: 6} - v2 := dyn.NewValue([]dyn.Value{ - dyn.NewValue("qux", []diag.Location{l2}), - dyn.NewValue("foo", []diag.Location{l3}), - }, []diag.Location{l2, l3}) - - // Merge v2 into v1. - { - out, err := Merge(v1, v2) - assert.NoError(t, err) - assert.Equal(t, []any{ - "bar", - "baz", - "qux", - "foo", - }, out.AsAny()) - - // Locations of both values should be preserved. - assert.Equal(t, []diag.Location{l1, l2, l3}, out.Locations()) - - // Location of the merged value should be the location of v1. - assert.Equal(t, l1, out.Location()) - - // Location of the individual values should be preserved. - assert.Equal(t, l1, out.Index(0).Location()) // "bar" - assert.Equal(t, l1, out.Index(1).Location()) // "baz" - assert.Equal(t, l2, out.Index(2).Location()) // "qux" - assert.Equal(t, l3, out.Index(3).Location()) // "foo" - } - - // Merge v1 into v2. - { - out, err := Merge(v2, v1) - assert.NoError(t, err) - assert.Equal(t, []any{ - "qux", - "foo", - "bar", - "baz", - }, out.AsAny()) - - // Locations of both values should be preserved. - assert.Equal(t, []diag.Location{l2, l3, l1}, out.Locations()) - - // Location of the merged value should be the location of v2. - assert.Equal(t, l2, out.Location()) - - // Location of the individual values should be preserved. - assert.Equal(t, l2, out.Index(0).Location()) // "qux" - assert.Equal(t, l3, out.Index(1).Location()) // "foo" - assert.Equal(t, l1, out.Index(2).Location()) // "bar" - assert.Equal(t, l1, out.Index(3).Location()) // "baz" - } -} - -func TestMergeSequencesNil(t *testing.T) { - v := dyn.V([]dyn.Value{ - dyn.V("bar"), - }) - - // Merge nil into v. - { - out, err := Merge(v, dyn.NilValue) - assert.NoError(t, err) - assert.Equal(t, []any{ - "bar", - }, out.AsAny()) - } - - // Merge v into nil. - { - out, err := Merge(dyn.NilValue, v) - assert.NoError(t, err) - assert.Equal(t, []any{ - "bar", - }, out.AsAny()) - } -} - -func TestMergeSequencesError(t *testing.T) { - v := dyn.V([]dyn.Value{ - dyn.V("bar"), - }) - - other := dyn.V("string") - - // Merge a string into v. - { - out, err := Merge(v, other) - assert.EqualError(t, err, "cannot merge sequence with string") - assert.Equal(t, dyn.InvalidValue, out) - } -} - -func TestMergePrimitives(t *testing.T) { - l1 := diag.Location{File: "file1", Line: 1, Column: 2} - l2 := diag.Location{File: "file2", Line: 3, Column: 4} - v1 := dyn.NewValue("bar", []diag.Location{l1}) - v2 := dyn.NewValue("baz", []diag.Location{l2}) - - // Merge v2 into v1. - { - out, err := Merge(v1, v2) - assert.NoError(t, err) - assert.Equal(t, "baz", out.AsAny()) - - // Locations of both values should be preserved. - assert.Equal(t, []diag.Location{l2, l1}, out.Locations()) - - // Location of the merged value should be the location of v2, the second value. - assert.Equal(t, l2, out.Location()) - } - - // Merge v1 into v2. - { - out, err := Merge(v2, v1) - assert.NoError(t, err) - assert.Equal(t, "bar", out.AsAny()) - - // Locations of both values should be preserved. - assert.Equal(t, []diag.Location{l1, l2}, out.Locations()) - - // Location of the merged value should be the location of v1, the second value. - assert.Equal(t, l1, out.Location()) - } -} - -func TestMergePrimitivesNil(t *testing.T) { - v := dyn.V("bar") - - // Merge nil into v. - { - out, err := Merge(v, dyn.NilValue) - assert.NoError(t, err) - assert.Equal(t, "bar", out.AsAny()) - } - - // Merge v into nil. - { - out, err := Merge(dyn.NilValue, v) - assert.NoError(t, err) - assert.Equal(t, "bar", out.AsAny()) - } -} - -func TestMergePrimitivesError(t *testing.T) { - v := dyn.V("bar") - other := dyn.V(map[string]dyn.Value{ - "foo": dyn.V("bar"), - }) - - // Merge a map into v. - { - out, err := Merge(v, other) - assert.EqualError(t, err, "cannot merge string with map") - assert.Equal(t, dyn.InvalidValue, out) - } -} diff --git a/libs/dyn/merge/override.go b/libs/dyn/merge/override.go deleted file mode 100644 index 873c38eead1..00000000000 --- a/libs/dyn/merge/override.go +++ /dev/null @@ -1,200 +0,0 @@ -package merge - -import ( - "errors" - "fmt" - - "github.com/databricks/cli/libs/dyn" -) - -// OverrideVisitor is visiting the changes during the override process -// and allows to control what changes are allowed, or update the effective -// value. -// -// For instance, it can disallow changes outside the specific path(s), or update -// the location of the effective value. -// -// Values returned by 'VisitInsert' and 'VisitUpdate' are used as the final value -// of the node. 'VisitDelete' can return ErrOverrideUndoDelete to undo delete. -// -// 'VisitDelete' is called when a value is removed from mapping or sequence -// 'VisitInsert' is called when a new value is added to mapping or sequence -// 'VisitUpdate' is called when a leaf value is updated -type OverrideVisitor struct { - VisitDelete func(valuePath dyn.Path, left dyn.Value) error - VisitInsert func(valuePath dyn.Path, right dyn.Value) (dyn.Value, error) - VisitUpdate func(valuePath dyn.Path, left, right dyn.Value) (dyn.Value, error) -} - -var ErrOverrideUndoDelete = errors.New("undo delete operation") - -// Override overrides value 'leftRoot' with 'rightRoot', keeping 'location' if values -// haven't changed. Preserving 'location' is important to preserve the original source of the value -// for error reporting. -func Override(leftRoot, rightRoot dyn.Value, visitor OverrideVisitor) (dyn.Value, error) { - return override(dyn.EmptyPath, leftRoot, rightRoot, visitor) -} - -func override(basePath dyn.Path, left, right dyn.Value, visitor OverrideVisitor) (dyn.Value, error) { - if left.Kind() != right.Kind() { - return visitor.VisitUpdate(basePath, left, right) - } - - // NB: we only call 'VisitUpdate' on leaf values, and for sequences and mappings - // we don't know if value was updated or not - - switch left.Kind() { - case dyn.KindMap: - merged, err := overrideMapping(basePath, left.MustMap(), right.MustMap(), visitor) - if err != nil { - return dyn.InvalidValue, err - } - - return dyn.NewValue(merged, left.Locations()), nil - - case dyn.KindSequence: - // some sequences are keyed, and we can detect which elements are added/removed/updated, - // but we don't have this information - merged, err := overrideSequence(basePath, left.MustSequence(), right.MustSequence(), visitor) - if err != nil { - return dyn.InvalidValue, err - } - - return dyn.NewValue(merged, left.Locations()), nil - - case dyn.KindString: - if left.MustString() == right.MustString() { - return left, nil - } else { - return visitor.VisitUpdate(basePath, left, right) - } - - case dyn.KindFloat: - // TODO consider comparison with epsilon if normalization doesn't help, where do we use floats? - - if left.MustFloat() == right.MustFloat() { - return left, nil - } else { - return visitor.VisitUpdate(basePath, left, right) - } - - case dyn.KindBool: - if left.MustBool() == right.MustBool() { - return left, nil - } else { - return visitor.VisitUpdate(basePath, left, right) - } - - case dyn.KindTime: - if left.MustTime() == right.MustTime() { - return left, nil - } else { - return visitor.VisitUpdate(basePath, left, right) - } - - case dyn.KindInt: - if left.MustInt() == right.MustInt() { - return left, nil - } else { - return visitor.VisitUpdate(basePath, left, right) - } - case dyn.KindNil: - return left, nil - default: - // Fall through to the error case. - } - - return dyn.InvalidValue, fmt.Errorf("unexpected kind %s at %s", left.Kind(), basePath.String()) -} - -func overrideMapping(basePath dyn.Path, leftMapping, rightMapping dyn.Mapping, visitor OverrideVisitor) (dyn.Mapping, error) { - out := dyn.NewMapping() - - for _, leftPair := range leftMapping.Pairs() { - // detect if key was removed - if _, ok := rightMapping.GetPair(leftPair.Key); !ok { - key := leftPair.Key.MustString() - keyLoc := leftPair.Key.Locations() - path := basePath.Append(dyn.Key(key)) - - err := visitor.VisitDelete(path, leftPair.Value) - - // if 'delete' was undone, add it back - if errors.Is(err, ErrOverrideUndoDelete) { - out.SetLoc(key, keyLoc, leftPair.Value) - } else if err != nil { - return dyn.NewMapping(), err - } - } - } - - // iterating only right mapping will remove keys not present anymore - // and insert new keys - - for _, rightPair := range rightMapping.Pairs() { - key := rightPair.Key.MustString() - keyLoc := rightPair.Key.Locations() - if leftPair, ok := leftMapping.GetPair(rightPair.Key); ok { - path := basePath.Append(dyn.Key(key)) - newValue, err := override(path, leftPair.Value, rightPair.Value, visitor) - if err != nil { - return dyn.NewMapping(), err - } - - // key was there before, so keep its location - out.SetLoc(key, keyLoc, newValue) - } else { - path := basePath.Append(dyn.Key(rightPair.Key.MustString())) - - newValue, err := visitor.VisitInsert(path, rightPair.Value) - if err != nil { - return dyn.NewMapping(), err - } - - out.SetLoc(key, keyLoc, newValue) - } - } - - return out, nil -} - -func overrideSequence(basePath dyn.Path, left, right []dyn.Value, visitor OverrideVisitor) ([]dyn.Value, error) { - minLen := min(len(left), len(right)) - var values []dyn.Value - - for i := range minLen { - path := basePath.Append(dyn.Index(i)) - merged, err := override(path, left[i], right[i], visitor) - if err != nil { - return nil, err - } - - values = append(values, merged) - } - - if len(right) > len(left) { - for i := minLen; i < len(right); i++ { - path := basePath.Append(dyn.Index(i)) - newValue, err := visitor.VisitInsert(path, right[i]) - if err != nil { - return nil, err - } - - values = append(values, newValue) - } - } else if len(left) > len(right) { - for i := minLen; i < len(left); i++ { - path := basePath.Append(dyn.Index(i)) - err := visitor.VisitDelete(path, left[i]) - - // if 'delete' was undone, add it back - if errors.Is(err, ErrOverrideUndoDelete) { - values = append(values, left[i]) - } else if err != nil { - return nil, err - } - } - } - - return values, nil -} diff --git a/libs/dyn/merge/override_test.go b/libs/dyn/merge/override_test.go deleted file mode 100644 index 17602f57ad7..00000000000 --- a/libs/dyn/merge/override_test.go +++ /dev/null @@ -1,520 +0,0 @@ -package merge - -import ( - "errors" - "testing" - "time" - - "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/dynassert" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -type overrideTestCase struct { - name string - left dyn.Value - right dyn.Value - state visitorState - expected dyn.Value -} - -func TestOverride_Primitive(t *testing.T) { - leftLocation := diag.Location{File: "left.yml", Line: 1, Column: 1} - rightLocation := diag.Location{File: "right.yml", Line: 1, Column: 1} - - modifiedTestCases := []overrideTestCase{ - { - name: "string (updated)", - state: visitorState{updated: []string{"root"}}, - left: dyn.NewValue("a", []diag.Location{leftLocation}), - right: dyn.NewValue("b", []diag.Location{rightLocation}), - expected: dyn.NewValue("b", []diag.Location{rightLocation}), - }, - { - name: "string (not updated)", - state: visitorState{}, - left: dyn.NewValue("a", []diag.Location{leftLocation}), - right: dyn.NewValue("a", []diag.Location{rightLocation}), - expected: dyn.NewValue("a", []diag.Location{leftLocation}), - }, - { - name: "bool (updated)", - state: visitorState{updated: []string{"root"}}, - left: dyn.NewValue(true, []diag.Location{leftLocation}), - right: dyn.NewValue(false, []diag.Location{rightLocation}), - expected: dyn.NewValue(false, []diag.Location{rightLocation}), - }, - { - name: "bool (not updated)", - state: visitorState{}, - left: dyn.NewValue(true, []diag.Location{leftLocation}), - right: dyn.NewValue(true, []diag.Location{rightLocation}), - expected: dyn.NewValue(true, []diag.Location{leftLocation}), - }, - { - name: "int (updated)", - state: visitorState{updated: []string{"root"}}, - left: dyn.NewValue(1, []diag.Location{leftLocation}), - right: dyn.NewValue(2, []diag.Location{rightLocation}), - expected: dyn.NewValue(2, []diag.Location{rightLocation}), - }, - { - name: "int (not updated)", - state: visitorState{}, - left: dyn.NewValue(int32(1), []diag.Location{leftLocation}), - right: dyn.NewValue(int64(1), []diag.Location{rightLocation}), - expected: dyn.NewValue(int32(1), []diag.Location{leftLocation}), - }, - { - name: "float (updated)", - state: visitorState{updated: []string{"root"}}, - left: dyn.NewValue(1.0, []diag.Location{leftLocation}), - right: dyn.NewValue(2.0, []diag.Location{rightLocation}), - expected: dyn.NewValue(2.0, []diag.Location{rightLocation}), - }, - { - name: "float (not updated)", - state: visitorState{}, - left: dyn.NewValue(float32(1.0), []diag.Location{leftLocation}), - right: dyn.NewValue(float64(1.0), []diag.Location{rightLocation}), - expected: dyn.NewValue(float32(1.0), []diag.Location{leftLocation}), - }, - { - name: "time (updated)", - state: visitorState{updated: []string{"root"}}, - left: dyn.NewValue(dyn.FromTime(time.UnixMilli(10000)), []diag.Location{leftLocation}), - right: dyn.NewValue(dyn.FromTime(time.UnixMilli(10001)), []diag.Location{rightLocation}), - expected: dyn.NewValue(dyn.FromTime(time.UnixMilli(10001)), []diag.Location{rightLocation}), - }, - { - name: "time (not updated)", - state: visitorState{}, - left: dyn.NewValue(dyn.FromTime(time.UnixMilli(10000)), []diag.Location{leftLocation}), - right: dyn.NewValue(dyn.FromTime(time.UnixMilli(10000)), []diag.Location{rightLocation}), - expected: dyn.NewValue(dyn.FromTime(time.UnixMilli(10000)), []diag.Location{leftLocation}), - }, - { - name: "different types (updated)", - state: visitorState{updated: []string{"root"}}, - left: dyn.NewValue("a", []diag.Location{leftLocation}), - right: dyn.NewValue(42, []diag.Location{rightLocation}), - expected: dyn.NewValue(42, []diag.Location{rightLocation}), - }, - { - name: "map - remove 'a', update 'b'", - state: visitorState{ - removed: []string{"root.a"}, - updated: []string{"root.b"}, - }, - left: dyn.NewValue( - map[string]dyn.Value{ - "a": dyn.NewValue(42, []diag.Location{leftLocation}), - "b": dyn.NewValue(10, []diag.Location{leftLocation}), - }, - []diag.Location{leftLocation}), - - right: dyn.NewValue( - map[string]dyn.Value{ - "b": dyn.NewValue(20, []diag.Location{rightLocation}), - }, - []diag.Location{rightLocation}), - - expected: dyn.NewValue( - map[string]dyn.Value{ - "b": dyn.NewValue(20, []diag.Location{rightLocation}), - }, - []diag.Location{leftLocation}), - }, - { - name: "map - add 'a'", - state: visitorState{ - added: []string{"root.a"}, - }, - left: dyn.NewValue( - map[string]dyn.Value{ - "b": dyn.NewValue(10, []diag.Location{leftLocation}), - }, - []diag.Location{leftLocation}, - ), - - right: dyn.NewValue( - map[string]dyn.Value{ - "a": dyn.NewValue(42, []diag.Location{rightLocation}), - "b": dyn.NewValue(10, []diag.Location{rightLocation}), - }, - []diag.Location{leftLocation}, - ), - - expected: dyn.NewValue( - map[string]dyn.Value{ - "a": dyn.NewValue(42, []diag.Location{rightLocation}), - // location hasn't changed because value hasn't changed - "b": dyn.NewValue(10, []diag.Location{leftLocation}), - }, - []diag.Location{leftLocation}, - ), - }, - { - name: "map - remove 'a'", - state: visitorState{ - removed: []string{"root.a"}, - }, - left: dyn.NewValue( - map[string]dyn.Value{ - "a": dyn.NewValue(42, []diag.Location{leftLocation}), - "b": dyn.NewValue(10, []diag.Location{leftLocation}), - }, - []diag.Location{leftLocation}, - ), - - right: dyn.NewValue( - map[string]dyn.Value{ - "b": dyn.NewValue(10, []diag.Location{rightLocation}), - }, - []diag.Location{leftLocation}, - ), - - expected: dyn.NewValue( - map[string]dyn.Value{ - // location hasn't changed because value hasn't changed - "b": dyn.NewValue(10, []diag.Location{leftLocation}), - }, - []diag.Location{leftLocation}, - ), - }, - { - name: "map - add 'jobs.job_1'", - state: visitorState{ - added: []string{"root.jobs.job_1"}, - }, - left: dyn.NewValue( - map[string]dyn.Value{ - "jobs": dyn.NewValue( - map[string]dyn.Value{ - "job_0": dyn.NewValue(42, []diag.Location{leftLocation}), - }, - []diag.Location{leftLocation}, - ), - }, - []diag.Location{leftLocation}, - ), - - right: dyn.NewValue( - map[string]dyn.Value{ - "jobs": dyn.NewValue( - map[string]dyn.Value{ - "job_0": dyn.NewValue(42, []diag.Location{rightLocation}), - "job_1": dyn.NewValue(1337, []diag.Location{rightLocation}), - }, - []diag.Location{rightLocation}, - ), - }, - []diag.Location{rightLocation}, - ), - - expected: dyn.NewValue( - map[string]dyn.Value{ - "jobs": dyn.NewValue( - map[string]dyn.Value{ - "job_0": dyn.NewValue(42, []diag.Location{leftLocation}), - "job_1": dyn.NewValue(1337, []diag.Location{rightLocation}), - }, - []diag.Location{leftLocation}, - ), - }, - []diag.Location{leftLocation}, - ), - }, - { - name: "map - remove nested key", - state: visitorState{removed: []string{"root.jobs.job_1"}}, - left: dyn.NewValue( - map[string]dyn.Value{ - "jobs": dyn.NewValue( - map[string]dyn.Value{ - "job_0": dyn.NewValue(42, []diag.Location{leftLocation}), - "job_1": dyn.NewValue(1337, []diag.Location{rightLocation}), - }, - []diag.Location{leftLocation}, - ), - }, - []diag.Location{leftLocation}, - ), - right: dyn.NewValue( - map[string]dyn.Value{ - "jobs": dyn.NewValue( - map[string]dyn.Value{ - "job_0": dyn.NewValue(42, []diag.Location{rightLocation}), - }, - []diag.Location{rightLocation}, - ), - }, - []diag.Location{rightLocation}, - ), - expected: dyn.NewValue( - map[string]dyn.Value{ - "jobs": dyn.NewValue( - map[string]dyn.Value{ - "job_0": dyn.NewValue(42, []diag.Location{leftLocation}), - }, - []diag.Location{leftLocation}, - ), - }, - []diag.Location{leftLocation}, - ), - }, - { - name: "sequence - add", - state: visitorState{added: []string{"root[1]"}}, - left: dyn.NewValue( - []dyn.Value{ - dyn.NewValue(42, []diag.Location{leftLocation}), - }, - []diag.Location{leftLocation}, - ), - right: dyn.NewValue( - []dyn.Value{ - dyn.NewValue(42, []diag.Location{rightLocation}), - dyn.NewValue(10, []diag.Location{rightLocation}), - }, - []diag.Location{rightLocation}, - ), - expected: dyn.NewValue( - []dyn.Value{ - dyn.NewValue(42, []diag.Location{leftLocation}), - dyn.NewValue(10, []diag.Location{rightLocation}), - }, - []diag.Location{leftLocation}, - ), - }, - { - name: "sequence - remove", - state: visitorState{removed: []string{"root[1]"}}, - left: dyn.NewValue( - []dyn.Value{ - dyn.NewValue(42, []diag.Location{leftLocation}), - dyn.NewValue(10, []diag.Location{leftLocation}), - }, - []diag.Location{leftLocation}, - ), - right: dyn.NewValue( - []dyn.Value{ - dyn.NewValue(42, []diag.Location{rightLocation}), - }, - []diag.Location{rightLocation}, - ), - expected: dyn.NewValue( - []dyn.Value{ - dyn.NewValue(42, []diag.Location{leftLocation}), - }, - []diag.Location{leftLocation}, - ), - // location hasn't changed because value hasn't changed - }, - { - name: "sequence (not updated)", - state: visitorState{}, - left: dyn.NewValue( - []dyn.Value{ - dyn.NewValue(42, []diag.Location{leftLocation}), - }, - []diag.Location{leftLocation}, - ), - right: dyn.NewValue( - []dyn.Value{ - dyn.NewValue(42, []diag.Location{rightLocation}), - }, - []diag.Location{rightLocation}, - ), - expected: dyn.NewValue( - []dyn.Value{ - dyn.NewValue(42, []diag.Location{leftLocation}), - }, - []diag.Location{leftLocation}, - ), - }, - { - name: "nil (not updated)", - state: visitorState{}, - left: dyn.NilValue.WithLocations([]diag.Location{leftLocation}), - right: dyn.NilValue.WithLocations([]diag.Location{rightLocation}), - expected: dyn.NilValue.WithLocations([]diag.Location{leftLocation}), - }, - { - name: "nil (updated)", - state: visitorState{updated: []string{"root"}}, - left: dyn.NilValue, - right: dyn.NewValue(42, []diag.Location{rightLocation}), - expected: dyn.NewValue(42, []diag.Location{rightLocation}), - }, - { - name: "change kind (updated)", - state: visitorState{updated: []string{"root"}}, - left: dyn.NewValue(42.0, []diag.Location{leftLocation}), - right: dyn.NewValue(42, []diag.Location{rightLocation}), - expected: dyn.NewValue(42, []diag.Location{rightLocation}), - }, - } - - for _, tc := range modifiedTestCases { - t.Run(tc.name, func(t *testing.T) { - s, visitor := createVisitor(visitorOpts{}) - out, err := override(dyn.NewPath(dyn.Key("root")), tc.left, tc.right, visitor) - - assert.NoError(t, err) - dynassert.Equal(t, tc.state, *s) - dynassert.Equal(t, tc.expected, out) - }) - - modified := len(tc.state.removed)+len(tc.state.added)+len(tc.state.updated) > 0 - - // visitor is not used unless there is a change - - if modified { - t.Run(tc.name+" - visitor has error", func(t *testing.T) { - _, visitor := createVisitor(visitorOpts{error: errors.New("unexpected change in test")}) - _, err := override(dyn.EmptyPath, tc.left, tc.right, visitor) - - assert.EqualError(t, err, "unexpected change in test") - }) - - t.Run(tc.name+" - visitor overrides value", func(t *testing.T) { - expected := dyn.V("return value") - s, visitor := createVisitor(visitorOpts{returnValue: &expected}) - out, err := override(dyn.EmptyPath, tc.left, tc.right, visitor) - - assert.NoError(t, err) - - for _, added := range s.added { - actual, err := dyn.GetByPath(out, dyn.MustPathFromString(added)) - - assert.NoError(t, err) - dynassert.Equal(t, expected, actual) - } - - for _, updated := range s.updated { - actual, err := dyn.GetByPath(out, dyn.MustPathFromString(updated)) - - assert.NoError(t, err) - dynassert.Equal(t, expected, actual) - } - }) - - if len(tc.state.removed) > 0 { - t.Run(tc.name+" - visitor can undo delete", func(t *testing.T) { - s, visitor := createVisitor(visitorOpts{deleteError: ErrOverrideUndoDelete}) - out, err := override(dyn.EmptyPath, tc.left, tc.right, visitor) - require.NoError(t, err) - - for _, removed := range s.removed { - expected, err := dyn.GetByPath(tc.left, dyn.MustPathFromString(removed)) - require.NoError(t, err) - - actual, err := dyn.GetByPath(out, dyn.MustPathFromString(removed)) - - assert.NoError(t, err) - dynassert.Equal(t, expected, actual) - } - }) - } - } - } -} - -func TestOverride_PreserveMappingKeys(t *testing.T) { - leftLocation := diag.Location{File: "left.yml", Line: 1, Column: 1} - leftKeyLocation := diag.Location{File: "left.yml", Line: 2, Column: 1} - leftValueLocation := diag.Location{File: "left.yml", Line: 3, Column: 1} - - rightLocation := diag.Location{File: "right.yml", Line: 1, Column: 1} - rightKeyLocation := diag.Location{File: "right.yml", Line: 2, Column: 1} - rightValueLocation := diag.Location{File: "right.yml", Line: 3, Column: 1} - - left := dyn.NewMapping() - left.SetLoc("a", []diag.Location{leftKeyLocation}, dyn.NewValue(42, []diag.Location{leftValueLocation})) - - right := dyn.NewMapping() - right.SetLoc("a", []diag.Location{rightKeyLocation}, dyn.NewValue(7, []diag.Location{rightValueLocation})) - - state, visitor := createVisitor(visitorOpts{}) - - out, err := override( - dyn.EmptyPath, - dyn.NewValue(left, []diag.Location{leftLocation}), - dyn.NewValue(right, []diag.Location{rightLocation}), - visitor, - ) - - assert.NoError(t, err) - - if err != nil { - outPairs := out.MustMap().Pairs() - - assert.Equal(t, visitorState{updated: []string{"a"}}, state) - assert.Len(t, outPairs, 1) - - // mapping was first defined in left, so it should keep its location - assert.Equal(t, leftLocation, out.Location()) - - // if there is a validation error for key value, it should point - // to where it was initially defined - assert.Equal(t, leftKeyLocation, outPairs[0].Key.Location()) - - // the value should have updated location, because it has changed - assert.Equal(t, rightValueLocation, outPairs[0].Value.Location()) - } -} - -type visitorState struct { - added []string - removed []string - updated []string -} - -type visitorOpts struct { - error error - deleteError error - returnValue *dyn.Value -} - -func createVisitor(opts visitorOpts) (*visitorState, OverrideVisitor) { - s := visitorState{} - - return &s, OverrideVisitor{ - VisitUpdate: func(valuePath dyn.Path, left, right dyn.Value) (dyn.Value, error) { - s.updated = append(s.updated, valuePath.String()) - - if opts.error != nil { - return dyn.NilValue, opts.error - } else if opts.returnValue != nil { - return *opts.returnValue, nil - } else { - return right, nil - } - }, - VisitDelete: func(valuePath dyn.Path, left dyn.Value) error { - s.removed = append(s.removed, valuePath.String()) - - if opts.error != nil { - return opts.error - } else if opts.deleteError != nil { - return opts.deleteError - } else { - return nil - } - }, - VisitInsert: func(valuePath dyn.Path, right dyn.Value) (dyn.Value, error) { - s.added = append(s.added, valuePath.String()) - - if opts.error != nil { - return dyn.NilValue, opts.error - } else if opts.returnValue != nil { - return *opts.returnValue, nil - } else { - return right, nil - } - }, - } -} diff --git a/libs/dyn/merge/select.go b/libs/dyn/merge/select.go deleted file mode 100644 index 9b09bfa2162..00000000000 --- a/libs/dyn/merge/select.go +++ /dev/null @@ -1,53 +0,0 @@ -package merge - -import ( - "fmt" - - "github.com/databricks/cli/libs/dyn" -) - -// Select returns a new map that contains only the keys specified in the included list. -func Select(value dyn.Value, included []string) (dyn.Value, error) { - mapping, ok := value.AsMap() - if !ok { - return dyn.InvalidValue, fmt.Errorf("expected a map, but found %s", value.Kind()) - } - - newMapping := dyn.NewMapping() - for _, key := range included { - pair, ok := mapping.GetPairByString(key) - - if ok { - newMapping.SetLoc(key, pair.Key.Locations(), pair.Value) - } - } - - return dyn.NewValue(newMapping, value.Locations()), nil -} - -// AntiSelect returns a new map with all keys from the input map except for the ones in the excluded list. -func AntiSelect(value dyn.Value, excluded []string) (dyn.Value, error) { - mapping, ok := value.AsMap() - if !ok { - return dyn.InvalidValue, fmt.Errorf("expected a map, but found %s", value.Kind()) - } - - excludedSet := make(map[string]struct{}) - for _, key := range excluded { - excludedSet[key] = struct{}{} - } - - included := make([]string, 0, len(mapping.Pairs())) - for _, pair := range mapping.Pairs() { - key, ok := pair.Key.AsString() - if !ok { - return dyn.InvalidValue, fmt.Errorf("expected a string key, but found %s", pair.Key.Kind()) - } - - if _, ok := excludedSet[key]; !ok { - included = append(included, pair.Key.MustString()) - } - } - - return Select(value, included) -} diff --git a/libs/dyn/merge/select_test.go b/libs/dyn/merge/select_test.go deleted file mode 100644 index c621f226849..00000000000 --- a/libs/dyn/merge/select_test.go +++ /dev/null @@ -1,55 +0,0 @@ -package merge - -import ( - "testing" - - "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/stretchr/testify/assert" -) - -func TestSelect(t *testing.T) { - locations := []diag.Location{{File: "foo.yml", Line: 1, Column: 1}} - included := []string{"foo"} - input := dyn.NewValue( - map[string]dyn.Value{ - "foo": dyn.V("bar"), - "baz": dyn.V("qux"), - }, - locations, - ) - expected := dyn.NewValue( - map[string]dyn.Value{ - "foo": dyn.V("bar"), - }, - locations, - ) - - actual, err := Select(input, included) - - assert.NoError(t, err) - assert.Equal(t, expected, actual) -} - -func TestAntiSelect(t *testing.T) { - locations := []diag.Location{{File: "foo.yml", Line: 1, Column: 1}} - excluded := []string{"foo"} - input := dyn.NewValue( - map[string]dyn.Value{ - "foo": dyn.V("bar"), - "baz": dyn.V("qux"), - }, - locations, - ) - expected := dyn.NewValue( - map[string]dyn.Value{ - "baz": dyn.V("qux"), - }, - locations, - ) - - actual, err := AntiSelect(input, excluded) - - assert.NoError(t, err) - assert.Equal(t, expected, actual) -} diff --git a/libs/dyn/path.go b/libs/dyn/path.go deleted file mode 100644 index 60579103ca8..00000000000 --- a/libs/dyn/path.go +++ /dev/null @@ -1,146 +0,0 @@ -package dyn - -import ( - "bytes" - "fmt" -) - -type pathComponent struct { - key string - index int -} - -func (c pathComponent) Key() string { - return c.key -} - -func (c pathComponent) Index() int { - return c.index -} - -func (c pathComponent) isKey() bool { - return c.key != "" -} - -func (c pathComponent) isIndex() bool { - return c.key == "" -} - -// Path represents a path to a value in a [Value] configuration tree. -type Path []pathComponent - -// EmptyPath is the empty path. -// It is defined for convenience and clarity. -var EmptyPath = Path{} - -// Key returns a path component for a key. -func Key(k string) pathComponent { - return pathComponent{key: k} -} - -// Index returns a path component for an index. -func Index(i int) pathComponent { - return pathComponent{index: i} -} - -// NewPath returns a new path from the given components. -// The individual components may be created with [Key] or [Index]. -func NewPath(cs ...pathComponent) Path { - return cs -} - -// Append appends the given components to the path. -// Mutations to the returned path do not affect the original path. -func (p Path) Append(cs ...pathComponent) Path { - out := make(Path, len(p)+len(cs)) - copy(out, p) - copy(out[len(p):], cs) - return out -} - -// Equal returns true if the paths are equal. -func (p Path) Equal(q Path) bool { - pl := len(p) - ql := len(q) - if pl != ql { - return false - } - for i := range pl { - if p[i] != q[i] { - return false - } - } - return true -} - -// HasPrefix returns true if the path has the specified prefix. -// The empty path is a prefix of all paths. -func (p Path) HasPrefix(q Path) bool { - pl := len(p) - ql := len(q) - if pl < ql { - return false - } - for i := range ql { - if p[i] != q[i] { - return false - } - } - return true -} - -// HasSuffix returns true if the path has the specified suffix. -// The empty path is a suffix of all paths. -func (p Path) HasSuffix(q Path) bool { - pl := len(p) - ql := len(q) - if pl < ql { - return false - } - for i := range ql { - if p[pl-ql+i] != q[i] { - return false - } - } - return true -} - -// CutPrefix returns the path with the specified prefix removed. -// If the path does not have the specified prefix, the original path is returned. -// The second return value is true if the prefix was removed. -// Logically equivalent to [strings.CutPrefix]. -func (p Path) CutPrefix(q Path) (Path, bool) { - if !p.HasPrefix(q) { - return p, false - } - return p[len(q):], true -} - -// CutSuffix returns the path with the specified suffix removed. -// If the path does not have the specified suffix, the original path is returned. -// The second return value is true if the suffix was removed. -// Logically equivalent to [strings.CutSuffix]. -func (p Path) CutSuffix(q Path) (Path, bool) { - if !p.HasSuffix(q) { - return p, false - } - return p[:len(p)-len(q)], true -} - -// String returns a string representation of the path. -func (p Path) String() string { - var buf bytes.Buffer - - for i, c := range p { - if i > 0 && c.key != "" { - buf.WriteRune('.') - } - if c.key != "" { - buf.WriteString(c.key) - } else { - fmt.Fprintf(&buf, "[%d]", c.index) - } - } - - return buf.String() -} diff --git a/libs/dyn/path_string.go b/libs/dyn/path_string.go deleted file mode 100644 index 2f17f9279c2..00000000000 --- a/libs/dyn/path_string.go +++ /dev/null @@ -1,89 +0,0 @@ -package dyn - -import ( - "fmt" - "strconv" - "strings" -) - -// MustPathFromString is like NewPathFromString but panics on error. -func MustPathFromString(input string) Path { - p, err := NewPathFromString(input) - if err != nil { - panic(err) - } - return p -} - -// NewPathFromString parses a path from a string. -// -// The string must be a sequence of keys and indices separated by dots. -// Indices must be enclosed in square brackets. -// The string may include a leading dot. -// -// Examples: -// - foo.bar -// - foo[1].bar -// - foo.bar[1] -// - foo.bar[1][2] -// - . -func NewPathFromString(input string) (Path, error) { - var path Path - - p := input - - // Trim leading dot. - if p != "" && p[0] == '.' { - p = p[1:] - } - - for p != "" { - // Every component may have a leading dot. - if p[0] == '.' { - p = p[1:] - } - - if p == "" { - return nil, fmt.Errorf("invalid path: %s", input) - } - - if p[0] == '[' { - // Find next ] - i := strings.Index(p, "]") - if i < 0 { - return nil, fmt.Errorf("invalid path: %s", input) - } - - // Parse index - j, err := strconv.Atoi(p[1:i]) - if err != nil { - return nil, fmt.Errorf("invalid path: %s", input) - } - - // Append index - path = append(path, Index(j)) - p = p[i+1:] - - // The next character must be a . or [ - if p != "" && strings.IndexAny(p, ".[") != 0 { - return nil, fmt.Errorf("invalid path: %s", input) - } - } else { - // Find next . or [ - i := strings.IndexAny(p, ".[") - if i < 0 { - i = len(p) - } - - if i == 0 { - return nil, fmt.Errorf("invalid path: %s", input) - } - - // Append key - path = append(path, Key(p[:i])) - p = p[i:] - } - } - - return path, nil -} diff --git a/libs/dyn/path_string_test.go b/libs/dyn/path_string_test.go deleted file mode 100644 index d3bfbd7f493..00000000000 --- a/libs/dyn/path_string_test.go +++ /dev/null @@ -1,115 +0,0 @@ -package dyn_test - -import ( - "errors" - "testing" - - . "github.com/databricks/cli/libs/dyn" - "github.com/stretchr/testify/assert" -) - -func TestNewPathFromString(t *testing.T) { - for _, tc := range []struct { - input string - output Path - err error - }{ - { - input: "", - output: NewPath(), - }, - { - input: ".", - output: NewPath(), - }, - { - input: "foo.bar", - output: NewPath(Key("foo"), Key("bar")), - }, - { - input: "[1]", - output: NewPath(Index(1)), - }, - { - input: "foo[1].bar", - output: NewPath(Key("foo"), Index(1), Key("bar")), - }, - { - input: "foo.bar[1]", - output: NewPath(Key("foo"), Key("bar"), Index(1)), - }, - { - input: "foo.bar[1][2]", - output: NewPath(Key("foo"), Key("bar"), Index(1), Index(2)), - }, - { - input: "foo.bar[1][2][3]", - output: NewPath(Key("foo"), Key("bar"), Index(1), Index(2), Index(3)), - }, - { - input: "foo[1234]", - output: NewPath(Key("foo"), Index(1234)), - }, - { - input: "foo[123", - err: errors.New("invalid path: foo[123"), - }, - { - input: "foo[123]]", - err: errors.New("invalid path: foo[123]]"), - }, - { - input: "foo[[123]", - err: errors.New("invalid path: foo[[123]"), - }, - { - input: "foo[[123]]", - err: errors.New("invalid path: foo[[123]]"), - }, - { - input: "foo[foo]", - err: errors.New("invalid path: foo[foo]"), - }, - { - input: "foo..bar", - err: errors.New("invalid path: foo..bar"), - }, - { - input: "foo.bar.", - err: errors.New("invalid path: foo.bar."), - }, - { - // Every component may have a leading dot. - input: ".foo.[1].bar", - output: NewPath(Key("foo"), Index(1), Key("bar")), - }, - { - // But after an index there must be a dot. - input: "foo[1]bar", - err: errors.New("invalid path: foo[1]bar"), - }, - { - // * is parsed as regular string in NewPathFromString - input: "foo.*", - output: NewPath(Key("foo"), Key("*")), - }, - { - // * is parsed as regular string in NewPathFromString - input: "foo.*.bar", - output: NewPath(Key("foo"), Key("*"), Key("bar")), - }, - { - // This is an invalid path (but would be valid for patterns) - input: "foo[*].bar", - err: errors.New("invalid path: foo[*].bar"), - }, - } { - p, err := NewPathFromString(tc.input) - if tc.err != nil { - assert.EqualError(t, err, tc.err.Error(), tc.input) - } else { - assert.NoError(t, err) - assert.Equal(t, tc.output, p) - } - } -} diff --git a/libs/dyn/path_test.go b/libs/dyn/path_test.go deleted file mode 100644 index 49114587175..00000000000 --- a/libs/dyn/path_test.go +++ /dev/null @@ -1,157 +0,0 @@ -package dyn_test - -import ( - "testing" - - "github.com/databricks/cli/libs/dyn" - "github.com/stretchr/testify/assert" -) - -func TestPathAppend(t *testing.T) { - p := dyn.NewPath(dyn.Key("foo")) - - // Single arg. - p1 := p.Append(dyn.Key("bar")) - assert.True(t, p1.Equal(dyn.NewPath(dyn.Key("foo"), dyn.Key("bar")))) - - // Multiple args. - p2 := p.Append(dyn.Key("bar"), dyn.Index(1)) - assert.True(t, p2.Equal(dyn.NewPath(dyn.Key("foo"), dyn.Key("bar"), dyn.Index(1)))) -} - -func TestPathAppendAlwaysNew(t *testing.T) { - p := make(dyn.Path, 0, 2) - p = append(p, dyn.Key("foo")) - - // There is room for a second element in the slice. - p1 := p.Append(dyn.Index(1)) - p2 := p.Append(dyn.Index(2)) - assert.NotEqual(t, p1, p2) -} - -func TestPathEqualEmpty(t *testing.T) { - assert.True(t, dyn.EmptyPath.Equal(dyn.EmptyPath)) -} - -func TestPathEqual(t *testing.T) { - p1 := dyn.NewPath(dyn.Key("foo"), dyn.Index(1)) - p2 := dyn.NewPath(dyn.Key("bar"), dyn.Index(2)) - assert.False(t, p1.Equal(p2), "expected %q to not equal %q", p1, p2) - - p3 := dyn.NewPath(dyn.Key("foo"), dyn.Index(1)) - assert.True(t, p1.Equal(p3), "expected %q to equal %q", p1, p3) - - p4 := dyn.NewPath(dyn.Key("foo"), dyn.Index(1), dyn.Key("bar"), dyn.Index(2)) - assert.False(t, p1.Equal(p4), "expected %q to not equal %q", p1, p4) -} - -func TestPathHasPrefixEmpty(t *testing.T) { - empty := dyn.EmptyPath - nonEmpty := dyn.NewPath(dyn.Key("foo")) - assert.True(t, empty.HasPrefix(empty)) - assert.True(t, nonEmpty.HasPrefix(empty)) - assert.False(t, empty.HasPrefix(nonEmpty)) -} - -func TestPathHasPrefix(t *testing.T) { - p1 := dyn.NewPath(dyn.Key("foo"), dyn.Index(1)) - p2 := dyn.NewPath(dyn.Key("bar"), dyn.Index(2)) - assert.False(t, p1.HasPrefix(p2), "expected %q to not have prefix %q", p1, p2) - - p3 := dyn.NewPath(dyn.Key("foo"), dyn.Index(1), dyn.Key("bar")) - assert.False(t, p1.HasPrefix(p3), "expected %q to not have prefix %q", p1, p3) - - p4 := dyn.NewPath(dyn.Key("foo"), dyn.Index(1)) - assert.True(t, p1.HasPrefix(p4), "expected %q to have prefix %q", p1, p4) - - p5 := dyn.NewPath(dyn.Key("foo")) - assert.True(t, p1.HasPrefix(p5), "expected %q to have prefix %q", p1, p5) -} - -func TestPathHasSuffixEmpty(t *testing.T) { - empty := dyn.EmptyPath - nonEmpty := dyn.NewPath(dyn.Key("foo")) - assert.True(t, empty.HasSuffix(empty)) - assert.True(t, nonEmpty.HasSuffix(empty)) - assert.False(t, empty.HasSuffix(nonEmpty)) -} - -func TestPathHasSuffix(t *testing.T) { - p1 := dyn.NewPath(dyn.Key("foo"), dyn.Index(1)) - p2 := dyn.NewPath(dyn.Key("bar"), dyn.Index(2)) - assert.False(t, p1.HasSuffix(p2), "expected %q to not have suffix %q", p1, p2) - - p3 := dyn.NewPath(dyn.Index(1)) - assert.True(t, p1.HasSuffix(p3), "expected %q to have suffix %q", p1, p3) - - p4 := dyn.NewPath(dyn.Key("foo"), dyn.Index(1)) - assert.True(t, p1.HasSuffix(p4), "expected %q to have suffix %q", p1, p4) - - p5 := dyn.NewPath(dyn.Key("bar"), dyn.Index(2), dyn.Key("baz")) - assert.False(t, p1.HasSuffix(p5), "expected %q to not have suffix %q", p1, p5) -} - -func TestPathCutPrefix(t *testing.T) { - p1 := dyn.NewPath(dyn.Key("foo"), dyn.Index(1), dyn.Key("bar")) - prefix := dyn.NewPath(dyn.Key("foo"), dyn.Index(1)) - - // Cut a valid prefix. - rest, ok := p1.CutPrefix(prefix) - assert.True(t, ok, "expected %q to have prefix %q", p1, prefix) - assert.True(t, rest.Equal(dyn.NewPath(dyn.Key("bar"))), "expected rest to be %q, got %q", dyn.NewPath(dyn.Key("bar")), rest) - - // Try to cut an invalid prefix. - invalidPrefix := dyn.NewPath(dyn.Key("bar")) - rest, ok = p1.CutPrefix(invalidPrefix) - assert.False(t, ok, "expected %q to not have prefix %q", p1, invalidPrefix) - assert.True(t, rest.Equal(p1), "expected rest to be %q, got %q", p1, rest) - - // Cut an empty prefix. - emptyPrefix := dyn.EmptyPath - rest, ok = p1.CutPrefix(emptyPrefix) - assert.True(t, ok, "expected %q to have prefix %q", p1, emptyPrefix) - assert.True(t, rest.Equal(p1), "expected rest to be %q, got %q", p1, rest) - - // Cut a prefix equal to the path. - rest, ok = p1.CutPrefix(p1) - assert.True(t, ok, "expected %q to have prefix %q", p1, p1) - assert.True(t, rest.Equal(dyn.EmptyPath), "expected rest to be %q, got %q", dyn.EmptyPath, rest) -} - -func TestPathCutSuffix(t *testing.T) { - p1 := dyn.NewPath(dyn.Key("foo"), dyn.Index(1), dyn.Key("bar")) - suffix := dyn.NewPath(dyn.Index(1), dyn.Key("bar")) - - // Cut a valid suffix. - rest, ok := p1.CutSuffix(suffix) - assert.True(t, ok, "expected %q to have suffix %q", p1, suffix) - assert.True(t, rest.Equal(dyn.NewPath(dyn.Key("foo"))), "expected rest to be %q, got %q", dyn.NewPath(dyn.Key("foo")), rest) - - // Try to cut an invalid suffix. - invalidSuffix := dyn.NewPath(dyn.Key("foo")) - rest, ok = p1.CutSuffix(invalidSuffix) - assert.False(t, ok, "expected %q to not have suffix %q", p1, invalidSuffix) - assert.True(t, rest.Equal(p1), "expected rest to be %q, got %q", p1, rest) - - // Cut an empty suffix. - emptySuffix := dyn.EmptyPath - rest, ok = p1.CutSuffix(emptySuffix) - assert.True(t, ok, "expected %q to have suffix %q", p1, emptySuffix) - assert.True(t, rest.Equal(p1), "expected rest to be %q, got %q", p1, rest) - - // Cut a suffix equal to the path. - rest, ok = p1.CutSuffix(p1) - assert.True(t, ok, "expected %q to have suffix %q", p1, p1) - assert.True(t, rest.Equal(dyn.EmptyPath), "expected rest to be %q, got %q", dyn.EmptyPath, rest) -} - -func TestPathString(t *testing.T) { - p1 := dyn.NewPath(dyn.Key("foo"), dyn.Index(1)) - assert.Equal(t, "foo[1]", p1.String()) - - p2 := dyn.NewPath(dyn.Key("bar"), dyn.Index(2), dyn.Key("baz")) - assert.Equal(t, "bar[2].baz", p2.String()) - - p3 := dyn.NewPath(dyn.Key("foo"), dyn.Index(1), dyn.Key("bar"), dyn.Index(2), dyn.Key("baz")) - assert.Equal(t, "foo[1].bar[2].baz", p3.String()) -} diff --git a/libs/dyn/pattern.go b/libs/dyn/pattern.go deleted file mode 100644 index cc53a2b0480..00000000000 --- a/libs/dyn/pattern.go +++ /dev/null @@ -1,191 +0,0 @@ -package dyn - -import ( - "errors" - "fmt" - "slices" - "strings" -) - -// Pattern represents a matcher for paths in a [Value] configuration tree. -// It is used by [MapByPattern] to apply a function to the values whose paths match the pattern. -// Every [Path] is a valid [Pattern] that matches a single unique path. -// The reverse is not true; not every [Pattern] is a valid [Path], as patterns may contain wildcards. -type Pattern []patternComponent - -func (p Pattern) String() string { - buf := strings.Builder{} - first := true - - for _, c := range p { - switch c := c.(type) { - case anyKeyComponent: - if !first { - buf.WriteString(".") - } - buf.WriteString("*") - case anyIndexComponent: - buf.WriteString("[*]") - case pathComponent: - if c.isKey() { - if !first { - buf.WriteString(".") - } - buf.WriteString(c.Key()) - } else { - fmt.Fprintf(&buf, "[%d]", c.Index()) - } - default: - buf.WriteString("???") - } - - first = false - } - return buf.String() -} - -// A pattern component can visit a [Value] and recursively call into [visit] for matching elements. -// Fixed components can match a single key or index, while wildcards can match any key or index. -type patternComponent interface { - visit(v Value, prefix Path, suffix Pattern, opts visitOptions) (Value, error) -} - -// NewPattern returns a new pattern from the given components. -// The individual components may be created with [Key], [Index], or [Any]. -func NewPattern(cs ...patternComponent) Pattern { - return cs -} - -// NewPatternFromPath returns a new pattern from the given path. -func NewPatternFromPath(p Path) Pattern { - cs := make(Pattern, len(p)) - for i, c := range p { - cs[i] = c - } - return cs -} - -// Split . into and -// The last component must be dyn.Key() and there must be at least two components. -func (p Pattern) SplitKey() (Pattern, string) { - if len(p) <= 1 { - return nil, "" - } - parent := p[:len(p)-1] - leaf := p[len(p)-1] - pc, ok := leaf.(pathComponent) - if !ok { - return nil, "" - } - key := pc.Key() - if key == "" { - return nil, "" - } - return parent, key -} - -// Append appends the given components to the pattern. -func (p Pattern) Append(cs ...patternComponent) Pattern { - out := make(Pattern, len(p)+len(cs)) - copy(out, p) - copy(out[len(p):], cs) - return out -} - -type anyKeyComponent struct{} - -// AnyKey returns a pattern component that matches any key. -func AnyKey() patternComponent { - return anyKeyComponent{} -} - -type expectedMapError struct { - p Path - v Value -} - -func (e expectedMapError) Error() string { - return fmt.Sprintf("expected a map at %q, found %s", e.p, e.v.Kind()) -} - -type expectedSequenceError struct { - p Path - v Value -} - -func (e expectedSequenceError) Error() string { - return fmt.Sprintf("expected a sequence at %q, found %s", e.p, e.v.Kind()) -} - -// isNoMatchError reports whether err means the pattern suffix didn't match the -// visited value. Wildcard components skip such elements (e.g. a job with an -// empty "tasks:" block) instead of failing the visit for all valid siblings. -func isNoMatchError(err error) bool { - if IsNoSuchKeyError(err) || IsIndexOutOfBoundsError(err) || IsCannotTraverseNilError(err) { - return true - } - if _, ok := errors.AsType[expectedMapError](err); ok { - return true - } - _, ok := errors.AsType[expectedSequenceError](err) - return ok -} - -// This function implements the patternComponent interface. -func (c anyKeyComponent) visit(v Value, prefix Path, suffix Pattern, opts visitOptions) (Value, error) { - m, ok := v.AsMap() - if !ok { - return InvalidValue, expectedMapError{p: prefix, v: v} - } - - m = m.Clone() - for _, pair := range m.Pairs() { - pk := pair.Key - pv := pair.Value - - var err error - nv, err := visit(pv, append(prefix, Key(pk.MustString())), suffix, opts) - if err != nil { - // Leave the value intact if the suffix pattern didn't match any value. - if isNoMatchError(err) { - continue - } - return InvalidValue, err - } - - m.SetLoc(pk.MustString(), pk.Locations(), nv) - } - - return NewValue(m, v.Locations()), nil -} - -type anyIndexComponent struct{} - -// AnyIndex returns a pattern component that matches any index. -func AnyIndex() patternComponent { - return anyIndexComponent{} -} - -// This function implements the patternComponent interface. -func (c anyIndexComponent) visit(v Value, prefix Path, suffix Pattern, opts visitOptions) (Value, error) { - s, ok := v.AsSequence() - if !ok { - return InvalidValue, expectedSequenceError{p: prefix, v: v} - } - - s = slices.Clone(s) - for i, value := range s { - var err error - nv, err := visit(value, append(prefix, Index(i)), suffix, opts) - if err != nil { - // Leave the value intact if the suffix pattern didn't match any value. - if isNoMatchError(err) { - continue - } - return InvalidValue, err - } - s[i] = nv - } - - return NewValue(s, v.Locations()), nil -} diff --git a/libs/dyn/pattern_string.go b/libs/dyn/pattern_string.go deleted file mode 100644 index 6d770faeb92..00000000000 --- a/libs/dyn/pattern_string.go +++ /dev/null @@ -1,102 +0,0 @@ -package dyn - -import ( - "fmt" - "strconv" - "strings" -) - -// MustPatternFromString is like NewPatternFromString but panics on error. -func MustPatternFromString(input string) Pattern { - p, err := NewPatternFromString(input) - if err != nil { - panic(err) - } - return p -} - -// NewPatternFromString parses a pattern from a string. -// -// The string must be a sequence of keys and indices separated by dots. -// Indices must be enclosed in square brackets. -// The string may include a leading dot. -// The wildcard character '*' can be used to match any key or index. -// -// Examples: -// - foo.bar -// - foo[1].bar -// - foo.*.bar -// - foo[*].bar -// - . -func NewPatternFromString(input string) (Pattern, error) { - var pattern Pattern - - p := input - - // Trim leading dot. - if p != "" && p[0] == '.' { - p = p[1:] - } - - for p != "" { - // Every component may have a leading dot. - if p[0] == '.' { - p = p[1:] - } - - if p == "" { - return nil, fmt.Errorf("invalid pattern: %s", input) - } - - if p[0] == '[' { - // Find next ] - i := strings.Index(p, "]") - if i < 0 { - return nil, fmt.Errorf("invalid pattern: %s", input) - } - - // Check for wildcard - if p[1:i] == "*" { - pattern = append(pattern, AnyIndex()) - } else { - // Parse index - j, err := strconv.Atoi(p[1:i]) - if err != nil { - return nil, fmt.Errorf("invalid pattern: %s", input) - } - - // Append index - pattern = append(pattern, Index(j)) - } - - p = p[i+1:] - - // The next character must be a . or [ - if p != "" && strings.IndexAny(p, ".[") != 0 { - return nil, fmt.Errorf("invalid pattern: %s", input) - } - } else { - // Find next . or [ - i := strings.IndexAny(p, ".[") - if i < 0 { - i = len(p) - } - - if i == 0 { - return nil, fmt.Errorf("invalid pattern: %s", input) - } - - // Check for wildcard - if p[:i] == "*" { - pattern = append(pattern, AnyKey()) - } else { - // Append key - pattern = append(pattern, Key(p[:i])) - } - - p = p[i:] - } - } - - return pattern, nil -} diff --git a/libs/dyn/pattern_string_test.go b/libs/dyn/pattern_string_test.go deleted file mode 100644 index 85127c12ed1..00000000000 --- a/libs/dyn/pattern_string_test.go +++ /dev/null @@ -1,147 +0,0 @@ -package dyn_test - -import ( - "errors" - "fmt" - "testing" - - . "github.com/databricks/cli/libs/dyn" - "github.com/stretchr/testify/assert" -) - -func TestNewPatternFromString(t *testing.T) { - for ind, tc := range []struct { - input string - output Pattern - err error - }{ - { - input: "", - output: NewPattern(), - }, - { - input: ".", - output: NewPattern(), - }, - { - input: "foo.bar", - output: NewPattern(Key("foo"), Key("bar")), - }, - { - input: "[1]", - output: NewPattern(Index(1)), - }, - { - input: "foo[1].bar", - output: NewPattern(Key("foo"), Index(1), Key("bar")), - }, - { - input: "foo.bar[1]", - output: NewPattern(Key("foo"), Key("bar"), Index(1)), - }, - { - input: "foo.bar[1][2]", - output: NewPattern(Key("foo"), Key("bar"), Index(1), Index(2)), - }, - { - input: "foo.bar[1][2][3]", - output: NewPattern(Key("foo"), Key("bar"), Index(1), Index(2), Index(3)), - }, - { - input: "foo[1234]", - output: NewPattern(Key("foo"), Index(1234)), - }, - { - input: "foo[123", - err: errors.New("invalid pattern: foo[123"), - }, - { - input: "foo[123]]", - err: errors.New("invalid pattern: foo[123]]"), - }, - { - input: "foo[[123]", - err: errors.New("invalid pattern: foo[[123]"), - }, - { - input: "foo[[123]]", - err: errors.New("invalid pattern: foo[[123]]"), - }, - { - input: "foo[foo]", - err: errors.New("invalid pattern: foo[foo]"), - }, - { - input: "foo..bar", - err: errors.New("invalid pattern: foo..bar"), - }, - { - input: "foo.bar.", - err: errors.New("invalid pattern: foo.bar."), - }, - { - // Every component may have a leading dot. - input: ".foo.[1].bar", - output: NewPattern(Key("foo"), Index(1), Key("bar")), - }, - { - // But after an index there must be a dot. - input: "foo[1]bar", - err: errors.New("invalid pattern: foo[1]bar"), - }, - // Wildcard tests - { - input: "foo.*", - output: NewPattern(Key("foo"), AnyKey()), - }, - { - input: "foo.*.bar", - output: NewPattern(Key("foo"), AnyKey(), Key("bar")), - }, - { - input: "foo[*]", - output: NewPattern(Key("foo"), AnyIndex()), - }, - { - input: "foo[*].bar", - output: NewPattern(Key("foo"), AnyIndex(), Key("bar")), - }, - { - input: "*[1]", - output: NewPattern(AnyKey(), Index(1)), - }, - { - input: "*.*", - output: NewPattern(AnyKey(), AnyKey()), - }, - { - input: "*[*]", - output: NewPattern(AnyKey(), AnyIndex()), - }, - } { - t.Run(fmt.Sprintf("%d %s", ind, tc.input), func(t *testing.T) { - p, err := NewPatternFromString(tc.input) - if tc.err != nil { - assert.EqualError(t, err, tc.err.Error(), tc.input) - } else { - assert.NoError(t, err) - assert.Equal(t, tc.output, p) - } - }) - } -} - -func TestMustPatternFromString(t *testing.T) { - // Test valid pattern - p := MustPatternFromString("foo.bar") - assert.Equal(t, NewPattern(Key("foo"), Key("bar")), p) - - // Test with wildcards - p = MustPatternFromString("foo.*.bar[*]") - assert.Equal(t, NewPattern(Key("foo"), AnyKey(), Key("bar"), AnyIndex()), p) - - // Test that invalid pattern panics - assert.Panics(t, func() { - MustPatternFromString("foo[") - }) -} diff --git a/libs/dyn/pattern_test.go b/libs/dyn/pattern_test.go deleted file mode 100644 index be78b6df61a..00000000000 --- a/libs/dyn/pattern_test.go +++ /dev/null @@ -1,110 +0,0 @@ -package dyn_test - -import ( - "fmt" - "testing" - - "github.com/databricks/cli/libs/dyn" - "github.com/stretchr/testify/assert" -) - -func TestPatternString(t *testing.T) { - patterns := []string{ - "foo.bar", - "foo.bar.baz", - "foo[1]", - "foo[*]", - "*[1].bar", - "foo.*.bar", - "foo[*].bar", - "", - "foo", - "[1]", - "[*]", - "*", - } - - for _, p := range patterns { - pp := dyn.MustPatternFromString(p) - assert.Equal(t, p, pp.String()) - } -} - -func TestNewPattern(t *testing.T) { - pat := dyn.NewPattern( - dyn.Key("foo"), - dyn.Index(1), - ) - - assert.Len(t, pat, 2) -} - -func TestNewPatternFromPath(t *testing.T) { - path := dyn.NewPath( - dyn.Key("foo"), - dyn.Index(1), - ) - - pat1 := dyn.NewPattern(dyn.Key("foo"), dyn.Index(1)) - pat2 := dyn.NewPatternFromPath(path) - assert.Equal(t, pat1, pat2) -} - -func TestPatternAppend(t *testing.T) { - p := dyn.NewPattern(dyn.Key("foo")) - - // Single arg. - p1 := p.Append(dyn.Key("bar")) - assert.Equal(t, dyn.NewPattern(dyn.Key("foo"), dyn.Key("bar")), p1) - - // Multiple args. - p2 := p.Append(dyn.Key("bar"), dyn.Index(1)) - assert.Equal(t, dyn.NewPattern(dyn.Key("foo"), dyn.Key("bar"), dyn.Index(1)), p2) -} - -func TestPatternAppendAlwaysNew(t *testing.T) { - p := make(dyn.Pattern, 0, 2) - p = append(p, dyn.Key("foo")) - - // There is room for a second element in the slice. - p1 := p.Append(dyn.Index(1)) - p2 := p.Append(dyn.Index(2)) - assert.NotEqual(t, p1, p2) -} - -func TestPatternSplitKey(t *testing.T) { - p := dyn.NewPattern( - dyn.Key("foo"), - dyn.Key("bar"), - ) - - pat, key := p.SplitKey() - assert.Equal(t, "bar", key) - assert.Equal(t, dyn.NewPattern(dyn.Key("foo")), pat) -} - -func TestPatternSplitKeyError(t *testing.T) { - patterns := []dyn.Pattern{ - dyn.NewPattern( - dyn.Key("foo"), - dyn.AnyKey(), - ), - dyn.NewPattern( - dyn.Key("foo"), - dyn.AnyIndex(), - ), - dyn.NewPattern( - dyn.Key("foo"), - dyn.Index(1), - ), - dyn.NewPattern(), - } - - for ind, p := range patterns { - t.Run(fmt.Sprintf("%d %#v", ind, p), func(t *testing.T) { - pat, key := p.SplitKey() - assert.Empty(t, key) - assert.Empty(t, pat) - }) - } -} diff --git a/libs/dyn/pattern_trie.go b/libs/dyn/pattern_trie.go deleted file mode 100644 index a4239c3cb53..00000000000 --- a/libs/dyn/pattern_trie.go +++ /dev/null @@ -1,172 +0,0 @@ -package dyn - -import ( - "fmt" -) - -// TrieNode is a trie data structure for storing and querying patterns. -// It supports both exact matches and wildcard matches. You can insert [Pattern]s -// into the trie and then query it to see if a given [Path] matches any of the -// patterns. -// -// TrieNode represents a node in the pattern trie. -// Each node in the array represents one or more of: -// 1. An [AnyKey] component. This is the "*" wildcard which matches any map key. -// 2. An [AnyIndex] component. This is the "[*]" wildcard which matches any array index. -// 3. Multiple [Key] components. These are multiple static path keys for this node would match. -// -// Note: It's valid for both anyKey and pathKey to be set at the same time. -// For example, adding both "foo.*.bar" and "foo.bar" to a trie is valid. -// -// Note: Setting both key (one of pathKey or anyKey) and index (anyIndex) -// is not supported by the [PatternTrie.SearchPath] method. We don't perform validation for this -// case because it's not expected to arise in practice where a field is either a map or an array, -// but not both. -type TrieNode struct { - // If set this indicates the trie node is an AnyKey node. - // Maps to the [AnyKey] component. - AnyKey *TrieNode - - // Indicates the trie node is an AnyIndex node. - // Maps to the [AnyIndex] component. - AnyIndex *TrieNode - - // Set of strings which this trie node matches. - // Maps to the [Key] component. - PathKey map[string]*TrieNode - - // Indicates if this node is the end of a pattern. Encountering a node - // with IsEnd set to true in a trie means the pattern from the root to this - // node is a complete pattern. - IsEnd bool -} - -// Insert adds a pattern to the trie. -func (t *TrieNode) Insert(pattern Pattern) error { - // Empty pattern represents the root. - if len(pattern) == 0 { - t.IsEnd = true - return nil - } - - current := t - for i, component := range pattern { - // Create next node based on component type - var next *TrieNode - switch c := component.(type) { - case anyKeyComponent: - if current.AnyKey == nil { - current.AnyKey = &TrieNode{} - } - next = current.AnyKey - - case anyIndexComponent: - if current.AnyIndex == nil { - current.AnyIndex = &TrieNode{} - } - next = current.AnyIndex - - case pathComponent: - if key := c.Key(); key != "" { - if current.PathKey == nil { - current.PathKey = make(map[string]*TrieNode) - } - if _, exists := current.PathKey[key]; !exists { - current.PathKey[key] = &TrieNode{} - } - next = current.PathKey[key] - } else { - return fmt.Errorf("fixed index patterns are not supported: %#v", pattern) - } - } - - if next == nil { - return fmt.Errorf("invalid component type: %T", component) - } - - // Mark as end of pattern if this is the last component. - if i == len(pattern)-1 { - next.IsEnd = true - } - - // Move to next node - current = next - } - - return nil -} - -// SearchPath checks if the given path matches any pattern in the trie. -// A path matches if it exactly matches a pattern or if it matches a pattern with wildcards. -func (t *TrieNode) SearchPath(path Path) (Pattern, bool) { - // We pre-allocate the prefix array that is used to track the current - // prefix accumulated while walking the prefix tree. Pre-allocating - // ensures that we do not allocate memory on every recursive call. - prefix := make(Pattern, len(path)) - pattern, ok := t.searchPathRecursive(t, path, prefix, 0) - return pattern, ok -} - -// searchPathRecursive is a helper function that recursively checks if a path matches any pattern. -// Arguments: -// - node: the current node in the trie. -// - path: the path to check. -// - prefix: the prefix accumulated while walking the prefix tree. -// - index: the current index in the path / prefix -// -// Note we always expect the path and prefix to be the same length because wildcards like * and [*] -// only match a single path component. -func (t *TrieNode) searchPathRecursive(node *TrieNode, path Path, prefix Pattern, index int) (Pattern, bool) { - if node == nil { - return nil, false - } - - // Zero case, when the query path is the root node. We return nil here to match - // the semantics of [MustPatternFromString] which returns nil for the empty string. - // - // We cannot return a Pattern{} object here because then MustPatternFromString(""), which - // returns nil will not be equal to the Pattern{} object returned by this function. An equality - // is useful because users of this function can use it to check whether the root / empty pattern - // had been inserted into the trie. - if len(path) == 0 { - return nil, node.IsEnd - } - - // If we've reached the end of the path, check if this node is a valid end of a pattern. - isLast := index == len(path) - if isLast { - return prefix, node.IsEnd - } - - currentComponent := path[index] - - // First check if the key wildcard is set for the current index. - if currentComponent.isKey() && node.AnyKey != nil { - prefix[index] = AnyKey() - pattern, ok := t.searchPathRecursive(node.AnyKey, path, prefix, index+1) - if ok { - return pattern, true - } - } - - // If no key wildcard is set, check if the key is an exact match. - if currentComponent.isKey() { - child, exists := node.PathKey[currentComponent.Key()] - if !exists { - return nil, false - } - prefix[index] = currentComponent - return t.searchPathRecursive(child, path, prefix, index+1) - } - - if currentComponent.isIndex() && node.AnyIndex != nil { - prefix[index] = AnyIndex() - pattern, ok := t.searchPathRecursive(node.AnyIndex, path, prefix, index+1) - if ok { - return pattern, true - } - } - - // If we've reached this point, the path does not match any patterns in the trie. - return nil, false -} diff --git a/libs/dyn/pattern_trie_test.go b/libs/dyn/pattern_trie_test.go deleted file mode 100644 index 13bcad457e1..00000000000 --- a/libs/dyn/pattern_trie_test.go +++ /dev/null @@ -1,195 +0,0 @@ -package dyn_test - -import ( - "testing" - - "github.com/databricks/cli/libs/dyn" - "github.com/stretchr/testify/assert" -) - -func TestPatternTrie_SearchPath(t *testing.T) { - tests := []struct { - name string - pattern string - mustMatch []string - mustNotMatch []string - }{ - { - name: "empty pattern", - pattern: "", - mustMatch: []string{""}, - mustNotMatch: []string{"foo"}, - }, - { - name: "simple key pattern", - pattern: "foo", - mustMatch: []string{"foo"}, - mustNotMatch: []string{"foo.bar", "foo[0]", "bar"}, - }, - - { - name: "nested key pattern", - pattern: "foo.bar", - mustMatch: []string{"foo.bar"}, - mustNotMatch: []string{"foo", "foo[0]", "bar.foo", "foo.baz"}, - }, - { - name: "root wildcard", - pattern: "*", - mustMatch: []string{"foo", "bar"}, - mustNotMatch: []string{"", "bar.foo", "foo.baz"}, - }, - { - name: "wildcard * after foo", - pattern: "foo.*", - mustMatch: []string{"foo.bar", "foo.baz"}, - mustNotMatch: []string{"foo", "bar", "foo.bar.baz"}, - }, - { - name: "wildcard [*] after foo", - pattern: "foo[*]", - mustMatch: []string{"foo[0]", "foo[1]", "foo[2025]"}, - mustNotMatch: []string{"foo", "bar", "foo[0].bar"}, - }, - { - name: "key after * wildcard", - pattern: "foo.*.bar", - mustMatch: []string{"foo.abc.bar", "foo.def.bar"}, - mustNotMatch: []string{"foo", "bar", "foo.bar.baz"}, - }, - { - name: "key after [*] wildcard", - pattern: "foo[*].bar", - mustMatch: []string{"foo[0].bar", "foo[1].bar", "foo[2025].bar"}, - mustNotMatch: []string{"foo", "bar", "foo[0].baz"}, - }, - { - name: "multiple * wildcards", - pattern: "*.*.*", - mustMatch: []string{"foo.bar.baz", "foo.bar.qux"}, - mustNotMatch: []string{"foo", "bar", "foo.bar", "foo.bar.baz.qux"}, - }, - { - name: "multiple [*] wildcards", - pattern: "foo[*][*]", - mustMatch: []string{"foo[0][0]", "foo[1][1]", "foo[2025][2025]"}, - mustNotMatch: []string{"foo", "bar", "foo[0][0][0]"}, - }, - { - name: "[*] after * wildcard", - pattern: "*[*]", - mustMatch: []string{"foo[0]", "foo[1]", "foo[2025]"}, - mustNotMatch: []string{"foo", "bar", "foo[0].bar", "[0].foo"}, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - trie := &dyn.TrieNode{} - pattern := dyn.MustPatternFromString(tt.pattern) - - // None of the expected paths should match yet. - for _, path := range tt.mustMatch { - _, ok := trie.SearchPath(dyn.MustPathFromString(path)) - assert.False(t, ok) - } - for _, path := range tt.mustNotMatch { - _, ok := trie.SearchPath(dyn.MustPathFromString(path)) - assert.False(t, ok) - } - - err := trie.Insert(pattern) - assert.NoError(t, err) - - // Now all the expected paths should match. - for _, path := range tt.mustMatch { - pattern, ok := trie.SearchPath(dyn.MustPathFromString(path)) - assert.True(t, ok) - assert.Equal(t, dyn.MustPatternFromString(tt.pattern), pattern) - } - for _, path := range tt.mustNotMatch { - _, ok := trie.SearchPath(dyn.MustPathFromString(path)) - assert.False(t, ok) - } - }) - } -} - -func TestPatternTrie_MultiplePatterns(t *testing.T) { - trie := &dyn.TrieNode{} - - patterns := []string{ - "foo.bar", - "foo.*.baz", - "def[*]", - } - - mustMatch := map[string]string{ - "foo.bar": "foo.bar", - "foo.abc.baz": "foo.*.baz", - "foo.def.baz": "foo.*.baz", - "def[0]": "def[*]", - "def[1]": "def[*]", - } - - mustNotMatch := []string{ - "foo", - "abc[0]", - "abc[1]", - "def[2].x", - "foo.y", - "foo.bar.baz.qux", - } - - for _, pattern := range patterns { - err := trie.Insert(dyn.MustPatternFromString(pattern)) - assert.NoError(t, err) - } - - for path, expectedPattern := range mustMatch { - pattern, ok := trie.SearchPath(dyn.MustPathFromString(path)) - assert.True(t, ok) - assert.Equal(t, dyn.MustPatternFromString(expectedPattern), pattern) - } - - for _, path := range mustNotMatch { - _, ok := trie.SearchPath(dyn.MustPathFromString(path)) - assert.False(t, ok) - } -} - -func TestPatternTrie_OverlappingPatterns(t *testing.T) { - trie := &dyn.TrieNode{} - - // Insert overlapping patterns - patterns := []string{ - "foo.bar", - "foo.*", - "*.bar", - "*.*", - } - - for _, pattern := range patterns { - err := trie.Insert(dyn.MustPatternFromString(pattern)) - assert.NoError(t, err) - } - - for _, path := range []string{ - "foo.bar", - "foo.baz", - "baz.bar", - "baz.qux", - } { - _, ok := trie.SearchPath(dyn.MustPathFromString(path)) - assert.True(t, ok) - } -} - -func TestPatternTrie_FixedIndexPatterns(t *testing.T) { - trie := &dyn.TrieNode{} - - err := trie.Insert(dyn.MustPatternFromString("foo[0]")) - assert.EqualError(t, err, "fixed index patterns are not supported: dyn.Pattern{dyn.pathComponent{key:\"foo\", index:0}, dyn.pathComponent{key:\"\", index:0}}") - - err = trie.Insert(dyn.MustPatternFromString("foo[2]")) - assert.EqualError(t, err, "fixed index patterns are not supported: dyn.Pattern{dyn.pathComponent{key:\"foo\", index:0}, dyn.pathComponent{key:\"\", index:2}}") -} diff --git a/libs/dyn/structpath.go b/libs/dyn/structpath.go deleted file mode 100644 index 1cb740ee0d7..00000000000 --- a/libs/dyn/structpath.go +++ /dev/null @@ -1,41 +0,0 @@ -package dyn - -import "github.com/databricks/cli/libs/structs/structpath" - -// ToStructPath converts a [Path] to a [structpath.PathNode]. -func ToStructPath(p Path) *structpath.PathNode { - var result *structpath.PathNode - for _, c := range p { - if c.isKey() { - result = structpath.NewStringKey(result, c.key) - } else { - result = structpath.NewIndex(result, c.index) - } - } - return result -} - -// ToStructPaths converts each of paths with [ToStructPath], e.g. for diag.Diagnostic.Paths. -func ToStructPaths(paths ...Path) []*structpath.PathNode { - out := make([]*structpath.PathNode, len(paths)) - for i, p := range paths { - out[i] = ToStructPath(p) - } - return out -} - -// FromStructPath converts a [structpath.PathNode] of keys and indices to a [Path]. -// It reports false for nodes a [Path] cannot represent (key-value selectors, wildcards). -func FromStructPath(p *structpath.PathNode) (Path, bool) { - var out Path - for _, n := range p.AsSlice() { - if k, ok := n.StringKey(); ok { - out = append(out, Key(k)) - } else if i, ok := n.Index(); ok { - out = append(out, Index(i)) - } else { - return nil, false - } - } - return out, true -} diff --git a/libs/dyn/structpath_test.go b/libs/dyn/structpath_test.go deleted file mode 100644 index 6eed07a31fa..00000000000 --- a/libs/dyn/structpath_test.go +++ /dev/null @@ -1,39 +0,0 @@ -package dyn_test - -import ( - "testing" - - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/structs/structpath" - "github.com/stretchr/testify/assert" -) - -func TestToStructPath(t *testing.T) { - assert.Nil(t, dyn.ToStructPath(nil)) - - p := dyn.NewPath(dyn.Key("resources"), dyn.Key("jobs"), dyn.Key("a_job"), dyn.Key("tasks"), dyn.Index(1), dyn.Key("task_key")) - assert.Equal(t, "resources.jobs.a_job.tasks[1].task_key", dyn.ToStructPath(p).String()) - - p = dyn.NewPath(dyn.Key("resources"), dyn.Key("jobs"), dyn.Key("${var.env}_job")) - assert.Equal(t, "resources.jobs['${var.env}_job']", dyn.ToStructPath(p).String()) -} - -func TestToStructPaths(t *testing.T) { - paths := dyn.ToStructPaths( - dyn.MustPathFromString("a.b"), - dyn.NewPath(dyn.Key("c"), dyn.Index(1)), - ) - assert.Len(t, paths, 2) - assert.Equal(t, "a.b", paths[0].String()) - assert.Equal(t, "c[1]", paths[1].String()) - assert.Empty(t, dyn.ToStructPaths()) -} - -func TestFromStructPath(t *testing.T) { - p, ok := dyn.FromStructPath(structpath.NewPath(nil, "resources", "jobs", "a.b", "tasks", 2)) - assert.True(t, ok) - assert.Equal(t, dyn.NewPath(dyn.Key("resources"), dyn.Key("jobs"), dyn.Key("a.b"), dyn.Key("tasks"), dyn.Index(2)), p) - - _, ok = dyn.FromStructPath(structpath.MustParsePath("tasks[task_key='x']")) - assert.False(t, ok) -} diff --git a/libs/dyn/suggest_test.go b/libs/dyn/suggest_test.go deleted file mode 100644 index b0aa4a87168..00000000000 --- a/libs/dyn/suggest_test.go +++ /dev/null @@ -1,164 +0,0 @@ -package dyn - -import ( - "errors" - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestLevenshteinDistance(t *testing.T) { - tests := []struct { - a string - b string - want int - }{ - {"", "", 0}, - {"", "abc", 3}, - {"abc", "", 3}, - {"abc", "abc", 0}, - {"abc", "abd", 1}, - {"kitten", "sitting", 3}, - {"host", "hosts", 1}, - } - for _, tt := range tests { - assert.Equal(t, tt.want, levenshteinDistance(tt.a, tt.b), "levenshteinDistance(%q, %q)", tt.a, tt.b) - } -} - -func newSuggestMapping(keys ...string) Mapping { - var m Mapping - for _, k := range keys { - m.SetLoc(k, nil, V(k)) - } - return m -} - -func TestSuggestKeys(t *testing.T) { - tests := []struct { - name string - keys []string - typo string - want []string - }{ - { - // Keys within distance 2 are returned ordered by increasing - // distance; ties keep the map's insertion order. - name: "ordered by distance", - keys: []string{"host", "hosts", "token", "auth_type"}, - typo: "host", - want: []string{"host", "hosts"}, - }, - { - name: "no key close enough", - keys: []string{"host", "hosts", "token", "auth_type"}, - typo: "completely_different", - want: []string{}, - }, - { - // Distance-2 substitutions and insertions are both included. - name: "distance two included", - keys: []string{"profile", "prfile", "prof"}, - typo: "prfil", - want: []string{"prfile", "profile"}, - }, - { - name: "empty map", - keys: nil, - typo: "anything", - want: []string{}, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - assert.Equal(t, tt.want, suggestKeys(newSuggestMapping(tt.keys...), tt.typo)) - }) - } -} - -func TestDidYouMean(t *testing.T) { - tests := []struct { - name string - suggestions []string - want string - }{ - {"nil", nil, ""}, - {"empty", []string{}, ""}, - {"single", []string{"host"}, `, did you mean "host"?`}, - {"multiple", []string{"host", "hosts"}, `, did you mean one of: "host", "hosts"?`}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - assert.Equal(t, tt.want, didYouMean(tt.suggestions)) - }) - } -} - -func TestSuggestedReferences(t *testing.T) { - tests := []struct { - name string - err error - reference string - want []string - }{ - { - name: "single suggestion", - err: noSuchKeyError{p: NewPath(Key("variables"), Key("hst")), suggestions: []string{"host"}}, - reference: "var.hst", - want: []string{"var.host"}, - }, - { - name: "multiple suggestions", - err: noSuchKeyError{p: NewPath(Key("variables"), Key("hst")), suggestions: []string{"host", "hosts"}}, - reference: "var.hst", - want: []string{"var.host", "var.hosts"}, - }, - { - name: "nested outer-key typo keeps suffix", - err: noSuchKeyError{p: NewPath(Key("variables"), Key("clustr")), suggestions: []string{"cluster"}}, - reference: "var.clustr.spark_version", - want: []string{"var.cluster.spark_version"}, - }, - { - name: "deep leaf typo keeps prefix", - err: noSuchKeyError{p: NewPath(Key("variables"), Key("cluster"), Key("value"), Key("spark_versio")), suggestions: []string{"spark_version"}}, - reference: "var.cluster.spark_versio", - want: []string{"var.cluster.spark_version"}, - }, - { - name: "index component preserved", - err: noSuchKeyError{p: NewPath(Key("variables"), Key("librariez")), suggestions: []string{"libraries"}}, - reference: "var.librariez[0].jar", - want: []string{"var.libraries[0].jar"}, - }, - { - name: "non-var prefix", - err: noSuchKeyError{p: NewPath(Key("workspace"), Key("stot_path")), suggestions: []string{"root_path", "state_path"}}, - reference: "workspace.stot_path", - want: []string{"workspace.root_path", "workspace.state_path"}, - }, - { - name: "no suggestions", - err: noSuchKeyError{p: NewPath(Key("xyz"))}, - reference: "var.xyz", - want: nil, - }, - { - name: "other error type", - err: errors.New("some other error"), - reference: "var.xyz", - want: nil, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - assert.Equal(t, tt.want, SuggestedReferences(tt.err, tt.reference)) - }) - } -} - -func TestReplaceKeyFallsBackWhenKeyAbsent(t *testing.T) { - // The failed key is not present in the reference, so only the replacement - // itself is returned rather than a spliced reference. - assert.Equal(t, "host", replaceKey("var.foo", "missing", "host")) -} diff --git a/libs/dyn/time.go b/libs/dyn/time.go deleted file mode 100644 index c7f043f06ca..00000000000 --- a/libs/dyn/time.go +++ /dev/null @@ -1,66 +0,0 @@ -package dyn - -import ( - "fmt" - "time" -) - -// Time represents a time-like primitive value. -// -// It represents a timestamp and includes the original string value -// that was parsed to create the timestamp. This makes it possible -// to coalesce a value that YAML interprets as a timestamp back into -// a string without losing information. -type Time struct { - t time.Time - s string -} - -// NewTime creates a new Time from the given string. -func NewTime(str string) (Time, error) { - // Try a couple of layouts - for _, layout := range []string{ - "2006-1-2T15:4:5.999999999Z07:00", // RCF3339Nano with short date fields. - "2006-1-2t15:4:5.999999999Z07:00", // RFC3339Nano with short date fields and lower-case "t". - "2006-1-2 15:4:5.999999999", // space separated with no time zone - "2006-1-2", // date only - } { - t, terr := time.Parse(layout, str) - if terr == nil { - return Time{t: t, s: str}, nil - } - } - - return Time{}, fmt.Errorf("invalid time value: %q", str) -} - -// MustTime creates a new Time from the given string. -// It panics if the string cannot be parsed. -func MustTime(str string) Time { - t, err := NewTime(str) - if err != nil { - panic(err) - } - return t -} - -// FromTime creates a new Time from the given time.Time. -// It uses the RFC3339Nano format for its string representation. -// This guarantees that it can roundtrip into a string without losing information. -func FromTime(t time.Time) Time { - return Time{t: t, s: t.Format(time.RFC3339Nano)} -} - -// Time returns the time.Time value. -func (t Time) Time() time.Time { - return t.t -} - -// String returns the original string value that was parsed to create the timestamp. -func (t Time) String() string { - return t.s -} - -func (t Time) IsZero() bool { - return t.s == "" && t.t.IsZero() -} diff --git a/libs/dyn/time_test.go b/libs/dyn/time_test.go deleted file mode 100644 index 61cc4fef93d..00000000000 --- a/libs/dyn/time_test.go +++ /dev/null @@ -1,41 +0,0 @@ -package dyn_test - -import ( - "testing" - "time" - - "github.com/databricks/cli/libs/dyn" - "github.com/stretchr/testify/assert" -) - -func TestTimeValid(t *testing.T) { - for _, tc := range []string{ - "2024-08-29", - "2024-01-15T12:34:56.789012345Z", - } { - tm, err := dyn.NewTime(tc) - if assert.NoError(t, err) { - assert.NotEqual(t, time.Time{}, tm.Time()) - assert.Equal(t, tc, tm.String()) - } - } -} - -func TestTimeInvalid(t *testing.T) { - tm, err := dyn.NewTime("invalid") - assert.Error(t, err) - assert.Equal(t, dyn.Time{}, tm) -} - -func TestTimeFromTime(t *testing.T) { - tref := time.Now() - t1 := dyn.FromTime(tref) - - // Verify that the underlying value is the same. - assert.Equal(t, tref, t1.Time()) - - // Verify that the string representation can be used to construct the same. - t2, err := dyn.NewTime(t1.String()) - assert.NoError(t, err) - assert.True(t, t1.Time().Equal(t2.Time())) -} diff --git a/libs/dyn/value.go b/libs/dyn/value.go deleted file mode 100644 index 8cbd9b801ea..00000000000 --- a/libs/dyn/value.go +++ /dev/null @@ -1,245 +0,0 @@ -package dyn - -import ( - "fmt" - "reflect" - "slices" - - "github.com/databricks/cli/libs/diag" -) - -type Value struct { - v any - - k Kind - - // List of locations this value is defined at. The first location in the slice - // is the location returned by the `.Location()` method and is typically used - // for reporting errors and warnings associated with the value. - l []diag.Location - - // Whether or not this value is an anchor. - // If this node doesn't map to a type, we don't need to warn about it. - anchor bool -} - -// InvalidValue is equal to the zero-value of Value. -var InvalidValue = Value{ - k: KindInvalid, -} - -// NilValue is a convenient constant for a nil value. -var NilValue = Value{ - k: KindNil, -} - -// V constructs a new Value with the given value. -func V(v any) Value { - return NewValue(v, nil) -} - -// NewValue constructs a new Value with the given value and location. -func NewValue(v any, loc []diag.Location) Value { - switch vin := v.(type) { - case map[string]Value: - v = newMappingFromGoMap(vin) - } - - return Value{ - v: v, - k: kindOf(v), - - // create a copy of the locations, so that mutations to the original slice - // don't affect new value. - l: slices.Clone(loc), - } -} - -// WithLocations returns a new Value with its location set to the given value. -func (v Value) WithLocations(loc []diag.Location) Value { - return Value{ - v: v.v, - k: v.k, - - // create a copy of the locations, so that mutations to the original slice - // don't affect new value. - l: slices.Clone(loc), - } -} - -func (v Value) AppendLocationsFromValue(w Value) Value { - return Value{ - v: v.v, - k: v.k, - l: append(v.l, w.l...), - } -} - -func (v Value) Kind() Kind { - return v.k -} - -func (v Value) Value() any { - return v.v -} - -func (v Value) Locations() []diag.Location { - return v.l -} - -func (v Value) Location() diag.Location { - if len(v.l) == 0 { - return diag.Location{} - } - - return v.l[0] -} - -func (v Value) IsValid() bool { - return v.k != KindInvalid -} - -func (v Value) AsAny() any { - switch v.k { - case KindInvalid: - panic("invoked AsAny on invalid value") - case KindMap: - m := v.v.(Mapping) - out := make(map[string]any, m.Len()) - for _, pair := range m.pairs { - pk := pair.Key - pv := pair.Value - out[pk.MustString()] = pv.AsAny() - } - return out - case KindSequence: - vv := v.v.([]Value) - a := make([]any, len(vv)) - for i, v := range vv { - a[i] = v.AsAny() - } - return a - case KindNil: - return v.v - case KindString: - return v.v - case KindBool: - return v.v - case KindInt: - return v.v - case KindFloat: - return v.v - case KindTime: - t := v.v.(Time) - return t.Time() - default: - // Panic because we only want to deal with known types. - panic(fmt.Sprintf("invalid kind: %d", v.k)) - } -} - -func (v Value) IsZero() bool { - if v.v == nil { - return true - } - - switch x := v.v.(type) { - case Mapping: - return x.Len() == 0 - case []Value: - return len(x) == 0 - default: - return reflect.ValueOf(x).IsZero() - } -} - -func (v Value) Get(key string) Value { - m, ok := v.AsMap() - if !ok { - return InvalidValue - } - - vv, ok := m.GetByString(key) - if !ok { - return InvalidValue - } - - return vv -} - -func (v Value) Index(i int) Value { - s, ok := v.v.([]Value) - if !ok { - return InvalidValue - } - - if i < 0 || i >= len(s) { - return InvalidValue - } - - return s[i] -} - -func (v Value) MarkAnchor() Value { - return Value{ - v: v.v, - k: v.k, - l: v.l, - - anchor: true, - } -} - -func (v Value) IsAnchor() bool { - return v.anchor -} - -// eq is an internal only method that compares two values. -// It is used to determine if a value has changed during a visit. -// We need a custom implementation because maps and slices -// cannot be compared with the regular == operator. -func (v Value) eq(w Value) bool { - if v.k != w.k { - return false - } - if !slices.Equal(v.l, w.l) { - return false - } - - switch v.k { - case KindMap: - // Comparing &v.v to &w.v is always false (value receivers), so compare the Mapping's backing storage. - vm := v.v.(Mapping) - wm := w.v.(Mapping) - lv := vm.Len() - lw := wm.Len() - if lv == 0 && lw == 0 { - return true - } - if lv != lw { - return false - } - // Compare pointers to the underlying pairs slice. - // This is safe because we don't allow maps to be mutated. - return &vm.pairs[0] == &wm.pairs[0] - case KindSequence: - vs := v.v.([]Value) - ws := w.v.([]Value) - lv := len(vs) - lw := len(ws) - // If both slices are empty, they are equal. - if lv == 0 && lw == 0 { - return true - } - // If they have different lengths, they are not equal. - if lv != lw { - return false - } - // They are both non-empty and have the same length. - // Compare pointers to the underlying slice. - // This is safe because we don't allow slices to be mutated. - return &vs[0] == &ws[0] - default: - return v.v == w.v - } -} diff --git a/libs/dyn/value_eq_test.go b/libs/dyn/value_eq_test.go deleted file mode 100644 index 22fe2df0080..00000000000 --- a/libs/dyn/value_eq_test.go +++ /dev/null @@ -1,77 +0,0 @@ -package dyn - -import ( - "testing" - - "github.com/databricks/cli/libs/diag" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestValueEqMap(t *testing.T) { - loc := []diag.Location{{File: "file", Line: 1, Column: 2}} - v := NewValue(map[string]Value{"key": V("value")}, loc) - - tests := []struct { - name string - a, b Value - want bool - }{ - { - name: "same underlying mapping", - a: v, - b: v, - want: true, - }, - { - name: "cloned mapping with equal contents", - a: v, - b: Value{v: v.v.(Mapping).Clone(), k: KindMap, l: v.l}, - want: false, - }, - { - name: "different lengths", - a: v, - b: NewValue(map[string]Value{"key": V("value"), "other": V("value")}, loc), - want: false, - }, - { - name: "different locations", - a: v, - b: v.WithLocations([]diag.Location{{File: "other", Line: 1, Column: 2}}), - want: false, - }, - { - name: "empty mappings", - a: NewValue(map[string]Value{}, loc), - b: NewValue(map[string]Value{}, loc), - want: true, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - assert.Equal(t, tc.want, tc.a.eq(tc.b)) - }) - } -} - -func TestValueEqMapVisitDoesNotRebuildAncestors(t *testing.T) { - vin := V(map[string]Value{ - "a": V(map[string]Value{ - "b": V("value"), - }), - }) - - vout, err := Map(vin, "a.b", func(_ Path, v Value) (Value, error) { - return v, nil - }) - require.NoError(t, err) - - // The identity transform must return the original value without - // cloning the ancestor maps. - vm := vin.v.(Mapping) - wm := vout.v.(Mapping) - require.Equal(t, vm.Len(), wm.Len()) - assert.Same(t, &vm.pairs[0], &wm.pairs[0]) -} diff --git a/libs/dyn/value_test.go b/libs/dyn/value_test.go deleted file mode 100644 index 667e2cbdd59..00000000000 --- a/libs/dyn/value_test.go +++ /dev/null @@ -1,92 +0,0 @@ -package dyn_test - -import ( - "testing" - - "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/stretchr/testify/assert" -) - -func TestInvalidValue(t *testing.T) { - // Assert that the zero value of [dyn.Value] is the invalid value. - var zero dyn.Value - assert.Equal(t, zero, dyn.InvalidValue) -} - -func TestValueIsAnchor(t *testing.T) { - var zero dyn.Value - assert.False(t, zero.IsAnchor()) - mark := zero.MarkAnchor() - assert.True(t, mark.IsAnchor()) -} - -func TestValueAsMap(t *testing.T) { - var zeroValue dyn.Value - _, ok := zeroValue.AsMap() - assert.False(t, ok) - - intValue := dyn.V(1) - _, ok = intValue.AsMap() - assert.False(t, ok) - - mapValue := dyn.NewValue( - map[string]dyn.Value{ - "key": dyn.NewValue( - "value", - []diag.Location{{File: "file", Line: 1, Column: 2}}), - }, - []diag.Location{{File: "file", Line: 1, Column: 2}}, - ) - - m, ok := mapValue.AsMap() - assert.True(t, ok) - assert.Equal(t, 1, m.Len()) -} - -func TestValueIsValid(t *testing.T) { - var zeroValue dyn.Value - assert.False(t, zeroValue.IsValid()) - intValue := dyn.V(1) - assert.True(t, intValue.IsValid()) -} - -func TestIsZero(t *testing.T) { - assert.True(t, dyn.V(0).IsZero(), "int") - assert.True(t, dyn.V(int(0)).IsZero(), "int") - assert.False(t, dyn.V(int(1)).IsZero(), "int") - assert.True(t, dyn.V(uint(0)).IsZero(), "uint") - - assert.True(t, dyn.V(int8(0)).IsZero(), "int8") - assert.True(t, dyn.V(uint8(0)).IsZero(), "uint8") - - assert.True(t, dyn.V(int16(0)).IsZero(), "int16") - assert.True(t, dyn.V(uint16(0)).IsZero(), "uint16") - - assert.True(t, dyn.V(int32(0)).IsZero(), "int32") - assert.False(t, dyn.V(int32(1)).IsZero(), "int32") - assert.True(t, dyn.V(uint32(0)).IsZero(), "uint32") - - assert.True(t, dyn.V(int64(0)).IsZero(), "int64") - assert.False(t, dyn.V(int64(-1)).IsZero(), "int64") - - assert.True(t, dyn.V(uint64(0)).IsZero(), "uint64") - assert.False(t, dyn.V(uint64(2)).IsZero(), "uint64") - - assert.True(t, dyn.V("").IsZero(), "string") - assert.False(t, dyn.V("x").IsZero(), "string") - - assert.True(t, dyn.V(false).IsZero(), "bool") - assert.False(t, dyn.V(true).IsZero(), "bool") - - assert.True(t, dyn.V(float32(0.0)).IsZero(), "float32") - assert.False(t, dyn.V(float32(0.01)).IsZero(), "float32") - - assert.True(t, dyn.V(float64(0.0)).IsZero(), "float64") - assert.False(t, dyn.V(float64(0.01)).IsZero(), "float64") - - assert.True(t, dyn.V(dyn.Time{}).IsZero(), "time") - assert.True(t, dyn.V(dyn.Mapping{}).IsZero(), "Mapping") - assert.True(t, dyn.V([]dyn.Value{}).IsZero(), "Sequence") - assert.False(t, dyn.V([]dyn.Value{dyn.V(0)}).IsZero(), "Sequence") -} diff --git a/libs/dyn/value_underlying.go b/libs/dyn/value_underlying.go deleted file mode 100644 index a33ecd38ed8..00000000000 --- a/libs/dyn/value_underlying.go +++ /dev/null @@ -1,138 +0,0 @@ -package dyn - -import ( - "fmt" -) - -// AsMap returns the underlying mapping if this value is a map, -// the zero value and false otherwise. -func (v Value) AsMap() (Mapping, bool) { - vv, ok := v.v.(Mapping) - return vv, ok -} - -// MustMap returns the underlying mapping if this value is a map, -// panics otherwise. -func (v Value) MustMap() Mapping { - vv, ok := v.AsMap() - if !ok || v.k != KindMap { - panic(fmt.Sprintf("expected kind %s, got %s", KindMap, v.k)) - } - return vv -} - -// AsSequence returns the underlying sequence if this value is a sequence, -// the zero value and false otherwise. -func (v Value) AsSequence() ([]Value, bool) { - vv, ok := v.v.([]Value) - return vv, ok -} - -// MustSequence returns the underlying sequence if this value is a sequence, -// panics otherwise. -func (v Value) MustSequence() []Value { - vv, ok := v.AsSequence() - if !ok || v.k != KindSequence { - panic(fmt.Sprintf("expected kind %s, got %s", KindSequence, v.k)) - } - return vv -} - -// AsString returns the underlying string if this value is a string, -// the zero value and false otherwise. -func (v Value) AsString() (string, bool) { - vv, ok := v.v.(string) - return vv, ok -} - -// MustString returns the underlying string if this value is a string, -// panics otherwise. -func (v Value) MustString() string { - vv, ok := v.AsString() - if !ok || v.k != KindString { - panic(fmt.Sprintf("expected kind %s, got %s", KindString, v.k)) - } - return vv -} - -// AsBool returns the underlying bool if this value is a bool, -// the zero value and false otherwise. -func (v Value) AsBool() (bool, bool) { - vv, ok := v.v.(bool) - return vv, ok -} - -// MustBool returns the underlying bool if this value is a bool, -// panics otherwise. -func (v Value) MustBool() bool { - vv, ok := v.AsBool() - if !ok || v.k != KindBool { - panic(fmt.Sprintf("expected kind %s, got %s", KindBool, v.k)) - } - return vv -} - -// AsInt returns the underlying int if this value is an int, -// the zero value and false otherwise. -func (v Value) AsInt() (int64, bool) { - switch vv := v.v.(type) { - case int: - return int64(vv), true - case int32: - return int64(vv), true - case int64: - return vv, true - default: - return 0, false - } -} - -// MustInt returns the underlying int if this value is an int, -// panics otherwise. -func (v Value) MustInt() int64 { - vv, ok := v.AsInt() - if !ok || v.k != KindInt { - panic(fmt.Sprintf("expected kind %s, got %s", KindInt, v.k)) - } - return vv -} - -// AsFloat returns the underlying float if this value is a float, -// the zero value and false otherwise. -func (v Value) AsFloat() (float64, bool) { - switch vv := v.v.(type) { - case float32: - return float64(vv), true - case float64: - return float64(vv), true - default: - return 0, false - } -} - -// MustFloat returns the underlying float if this value is a float, -// panics otherwise. -func (v Value) MustFloat() float64 { - vv, ok := v.AsFloat() - if !ok || v.k != KindFloat { - panic(fmt.Sprintf("expected kind %s, got %s", KindFloat, v.k)) - } - return vv -} - -// AsTime returns the underlying time if this value is a time, -// the zero value and false otherwise. -func (v Value) AsTime() (Time, bool) { - vv, ok := v.v.(Time) - return vv, ok -} - -// MustTime returns the underlying time if this value is a time, -// panics otherwise. -func (v Value) MustTime() Time { - vv, ok := v.AsTime() - if !ok || v.k != KindTime { - panic(fmt.Sprintf("expected kind %s, got %s", KindTime, v.k)) - } - return vv -} diff --git a/libs/dyn/value_underlying_test.go b/libs/dyn/value_underlying_test.go deleted file mode 100644 index 189a49a0ea3..00000000000 --- a/libs/dyn/value_underlying_test.go +++ /dev/null @@ -1,162 +0,0 @@ -package dyn_test - -import ( - "testing" - "time" - - "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/stretchr/testify/assert" -) - -func TestValueUnderlyingMap(t *testing.T) { - v := dyn.V( - map[string]dyn.Value{ - "key": dyn.NewValue("value", []diag.Location{{File: "file", Line: 1, Column: 2}}), - }, - ) - - vv1, ok := v.AsMap() - assert.True(t, ok) - - _, ok = dyn.InvalidValue.AsMap() - assert.False(t, ok) - - vv2 := v.MustMap() - assert.Equal(t, vv1, vv2) - - // Test panic. - assert.PanicsWithValue(t, "expected kind map, got invalid", func() { - dyn.InvalidValue.MustMap() - }) -} - -func TestValueUnderlyingSequence(t *testing.T) { - v := dyn.V( - []dyn.Value{ - dyn.NewValue("value", []diag.Location{{File: "file", Line: 1, Column: 2}}), - }, - ) - - vv1, ok := v.AsSequence() - assert.True(t, ok) - - _, ok = dyn.InvalidValue.AsSequence() - assert.False(t, ok) - - vv2 := v.MustSequence() - assert.Equal(t, vv1, vv2) - - // Test panic. - assert.PanicsWithValue(t, "expected kind sequence, got invalid", func() { - dyn.InvalidValue.MustSequence() - }) -} - -func TestValueUnderlyingString(t *testing.T) { - v := dyn.V("value") - - vv1, ok := v.AsString() - assert.True(t, ok) - - _, ok = dyn.InvalidValue.AsString() - assert.False(t, ok) - - vv2 := v.MustString() - assert.Equal(t, vv1, vv2) - - // Test panic. - assert.PanicsWithValue(t, "expected kind string, got invalid", func() { - dyn.InvalidValue.MustString() - }) -} - -func TestValueUnderlyingBool(t *testing.T) { - v := dyn.V(true) - - vv1, ok := v.AsBool() - assert.True(t, ok) - - _, ok = dyn.InvalidValue.AsBool() - assert.False(t, ok) - - vv2 := v.MustBool() - assert.Equal(t, vv1, vv2) - - // Test panic. - assert.PanicsWithValue(t, "expected kind bool, got invalid", func() { - dyn.InvalidValue.MustBool() - }) -} - -func TestValueUnderlyingInt(t *testing.T) { - v := dyn.V(int(1)) - - vv1, ok := v.AsInt() - assert.True(t, ok) - - _, ok = dyn.InvalidValue.AsInt() - assert.False(t, ok) - - vv2 := v.MustInt() - assert.Equal(t, vv1, vv2) - - // Test panic. - assert.PanicsWithValue(t, "expected kind int, got invalid", func() { - dyn.InvalidValue.MustInt() - }) - - // Test int32 type specifically. - v = dyn.V(int32(1)) - vv1, ok = v.AsInt() - assert.True(t, ok) - assert.Equal(t, int64(1), vv1) - - // Test int64 type specifically. - v = dyn.V(int64(1)) - vv1, ok = v.AsInt() - assert.True(t, ok) - assert.Equal(t, int64(1), vv1) -} - -func TestValueUnderlyingFloat(t *testing.T) { - v := dyn.V(float32(1.0)) - - vv1, ok := v.AsFloat() - assert.True(t, ok) - - _, ok = dyn.InvalidValue.AsFloat() - assert.False(t, ok) - - vv2 := v.MustFloat() - assert.Zero(t, vv1-vv2) - - // Test panic. - assert.PanicsWithValue(t, "expected kind float, got invalid", func() { - dyn.InvalidValue.MustFloat() - }) - - // Test float64 type specifically. - v = dyn.V(float64(1.0)) - vv1, ok = v.AsFloat() - assert.True(t, ok) - assert.Zero(t, 1.0-vv1) -} - -func TestValueUnderlyingTime(t *testing.T) { - v := dyn.V(dyn.FromTime(time.Now())) - - vv1, ok := v.AsTime() - assert.True(t, ok) - - _, ok = dyn.InvalidValue.AsTime() - assert.False(t, ok) - - vv2 := v.MustTime() - assert.Equal(t, vv1, vv2) - - // Test panic. - assert.PanicsWithValue(t, "expected kind time, got invalid", func() { - dyn.InvalidValue.MustTime() - }) -} diff --git a/libs/dyn/visit.go b/libs/dyn/visit.go deleted file mode 100644 index 58223e792b9..00000000000 --- a/libs/dyn/visit.go +++ /dev/null @@ -1,210 +0,0 @@ -package dyn - -import ( - "errors" - "fmt" - "slices" -) - -// This error is returned if the path indicates that a map or sequence is expected, but the value is nil. -type cannotTraverseNilError struct { - p Path -} - -func (e cannotTraverseNilError) Error() string { - component := e.p[len(e.p)-1] - switch { - case component.isKey(): - return fmt.Sprintf("expected a map to index %q, found nil", e.p) - case component.isIndex(): - return fmt.Sprintf("expected a sequence to index %q, found nil", e.p) - default: - panic("invalid component") - } -} - -func IsCannotTraverseNilError(err error) bool { - _, ok := errors.AsType[cannotTraverseNilError](err) - return ok -} - -type noSuchKeyError struct { - p Path - suggestions []string -} - -func (e noSuchKeyError) Error() string { - return fmt.Sprintf("key not found at %q%s", e.p, didYouMean(e.suggestions)) -} - -func IsNoSuchKeyError(err error) bool { - _, ok := errors.AsType[noSuchKeyError](err) - return ok -} - -// SuggestedReferences returns drop-in replacement references for a noSuchKeyError -// (nil otherwise), rebuilt by swapping the failed segment of reference for each -// suggestion (e.g. "var.hst" -> ["var.host"]). -func SuggestedReferences(err error, reference string) []string { - e, ok := errors.AsType[noSuchKeyError](err) - if !ok || len(e.suggestions) == 0 { - return nil - } - // Last component of e.p is the failed key (same in original and rewritten space). - failedKey := e.p[len(e.p)-1].Key() - refs := make([]string, len(e.suggestions)) - for i, s := range e.suggestions { - refs[i] = replaceKey(reference, failedKey, s) - } - return refs -} - -type indexOutOfBoundsError struct { - p Path -} - -func (e indexOutOfBoundsError) Error() string { - return fmt.Sprintf("index out of bounds at %q", e.p) -} - -func IsIndexOutOfBoundsError(err error) bool { - _, ok := errors.AsType[indexOutOfBoundsError](err) - return ok -} - -type expectedMapToIndexError struct { - p Path - v Value -} - -func (e expectedMapToIndexError) Error() string { - return fmt.Sprintf("expected a map to index %q, found %s", e.p, e.v.Kind()) -} - -type expectedSequenceToIndexError struct { - p Path - v Value -} - -func (e expectedSequenceToIndexError) Error() string { - return fmt.Sprintf("expected a sequence to index %q, found %s", e.p, e.v.Kind()) -} - -type visitOptions struct { - // The function to apply to the value once found. - // - // If this function returns the same value as it receives as argument, - // the original visit function call returns the original value unmodified. - // - // If this function returns a new value, the original visit function call - // returns a value with all the intermediate values updated. - // - // If this function returns an error, the original visit function call - // returns this error and the value is left unmodified. - fn func(Path, Value) (Value, error) -} - -func visit(v Value, prefix Path, suffix Pattern, opts visitOptions) (Value, error) { - if len(suffix) == 0 { - return opts.fn(slices.Clone(prefix), v) - } - - // Initialize prefix if it is empty. - // It is pre-allocated to its maximum size to avoid additional allocations. - if len(prefix) == 0 { - prefix = make(Path, 0, len(suffix)) - } - - component := suffix[0] - suffix = suffix[1:] - - // Visit the value with the current component. - return component.visit(v, prefix, suffix, opts) -} - -func (c pathComponent) visit(v Value, prefix Path, suffix Pattern, opts visitOptions) (Value, error) { - path := append(prefix, c) - - switch { - case c.isKey(): - // Expect a map to be set if this is a key. - switch v.Kind() { - case KindMap: - // OK - case KindNil: - return InvalidValue, cannotTraverseNilError{path} - default: - return InvalidValue, expectedMapToIndexError{p: path, v: v} - } - - m := v.MustMap() - - // Lookup current value in the map. - ev, ok := m.GetByString(c.key) - if !ok { - return InvalidValue, noSuchKeyError{p: path, suggestions: suggestKeys(m, c.key)} - } - - // Recursively transform the value. - nv, err := visit(ev, path, suffix, opts) - if err != nil { - return InvalidValue, err - } - - // Return the original value if the value hasn't changed. - if nv.eq(ev) { - return v, nil - } - - // Return an updated map value. - m = m.Clone() - m.SetLoc(c.key, nil, nv) - return Value{ - v: m, - k: KindMap, - l: v.l, - }, nil - - case c.isIndex(): - // Expect a sequence to be set if this is an index. - switch v.Kind() { - case KindSequence: - // OK - case KindNil: - return InvalidValue, cannotTraverseNilError{path} - default: - return InvalidValue, expectedSequenceToIndexError{p: path, v: v} - } - - s := v.MustSequence() - - // Lookup current value in the sequence. - if c.index < 0 || c.index >= len(s) { - return InvalidValue, indexOutOfBoundsError{path} - } - - // Recursively transform the value. - ev := s[c.index] - nv, err := visit(ev, path, suffix, opts) - if err != nil { - return InvalidValue, err - } - - // Return the original value if the value hasn't changed. - if nv.eq(ev) { - return v, nil - } - - // Return an updated sequence value. - s = slices.Clone(s) - s[c.index] = nv - return Value{ - v: s, - k: KindSequence, - l: v.l, - }, nil - - default: - panic("invalid component") - } -} diff --git a/libs/dyn/visit_get.go b/libs/dyn/visit_get.go deleted file mode 100644 index d23c1f97427..00000000000 --- a/libs/dyn/visit_get.go +++ /dev/null @@ -1,31 +0,0 @@ -package dyn - -// Get returns the value inside the specified value at the specified path. -// It is identical to [GetByPath], except that it takes a string path instead of a [Path]. -func Get(v Value, path string) (Value, error) { - p, err := NewPathFromString(path) - if err != nil { - return InvalidValue, err - } - return GetByPath(v, p) -} - -// GetValue returns the value without the error (InvalidValue indicates that error did happen) -func GetValue(v Value, path string) Value { - val, _ := Get(v, path) - return val -} - -// GetByPath returns the value inside the specified value at the specified path. -// If the path doesn't exist, it returns InvalidValue and an error. -func GetByPath(v Value, p Path) (Value, error) { - out := InvalidValue - _, err := visit(v, EmptyPath, NewPatternFromPath(p), visitOptions{ - fn: func(_ Path, ev Value) (Value, error) { - // Capture the value argument to return it. - out = ev - return ev, nil - }, - }) - return out, err -} diff --git a/libs/dyn/visit_get_test.go b/libs/dyn/visit_get_test.go deleted file mode 100644 index 524b4ed5cd9..00000000000 --- a/libs/dyn/visit_get_test.go +++ /dev/null @@ -1,83 +0,0 @@ -package dyn_test - -import ( - "testing" - - "github.com/databricks/cli/libs/dyn" - "github.com/stretchr/testify/assert" -) - -func TestGetWithEmptyPath(t *testing.T) { - // An empty path means to return the value itself. - vin := dyn.V(42) - vout, err := dyn.GetByPath(vin, dyn.NewPath()) - assert.NoError(t, err) - assert.Equal(t, vin, vout) -} - -func TestGetOnNilValue(t *testing.T) { - var err error - _, err = dyn.GetByPath(dyn.NilValue, dyn.NewPath(dyn.Key("foo"))) - assert.ErrorContains(t, err, `expected a map to index "foo", found nil`) - _, err = dyn.GetByPath(dyn.NilValue, dyn.NewPath(dyn.Index(42))) - assert.ErrorContains(t, err, `expected a sequence to index "[42]", found nil`) -} - -func TestGetOnMap(t *testing.T) { - vin := dyn.V(map[string]dyn.Value{ - "foo": dyn.V(42), - "bar": dyn.V(43), - }) - - var err error - - _, err = dyn.GetByPath(vin, dyn.NewPath(dyn.Index(42))) - assert.ErrorContains(t, err, `expected a sequence to index "[42]", found map`) - - _, err = dyn.GetByPath(vin, dyn.NewPath(dyn.Key("baz"))) - assert.True(t, dyn.IsNoSuchKeyError(err)) - // "baz" is one edit away from "bar", so the error suggests it. - assert.ErrorContains(t, err, `key not found at "baz", did you mean "bar"?`) - - // A key that is close to no existing key gets no suggestion. - _, err = dyn.GetByPath(vin, dyn.NewPath(dyn.Key("completely_different"))) - assert.True(t, dyn.IsNoSuchKeyError(err)) - assert.ErrorContains(t, err, `key not found at "completely_different"`) - assert.NotContains(t, err.Error(), "did you mean") - - vfoo, err := dyn.GetByPath(vin, dyn.NewPath(dyn.Key("foo"))) - assert.NoError(t, err) - assert.Equal(t, dyn.V(42), vfoo) - - vbar, err := dyn.GetByPath(vin, dyn.NewPath(dyn.Key("bar"))) - assert.NoError(t, err) - assert.Equal(t, dyn.V(43), vbar) -} - -func TestGetOnSequence(t *testing.T) { - vin := dyn.V([]dyn.Value{ - dyn.V(42), - dyn.V(43), - }) - - var err error - - _, err = dyn.GetByPath(vin, dyn.NewPath(dyn.Key("foo"))) - assert.ErrorContains(t, err, `expected a map to index "foo", found sequence`) - - _, err = dyn.GetByPath(vin, dyn.NewPath(dyn.Index(-1))) - assert.True(t, dyn.IsIndexOutOfBoundsError(err)) - assert.ErrorContains(t, err, `index out of bounds at "[-1]"`) - - _, err = dyn.GetByPath(vin, dyn.NewPath(dyn.Index(2))) - assert.True(t, dyn.IsIndexOutOfBoundsError(err)) - assert.ErrorContains(t, err, `index out of bounds at "[2]"`) - - v0, err := dyn.GetByPath(vin, dyn.NewPath(dyn.Index(0))) - assert.NoError(t, err) - assert.Equal(t, dyn.V(42), v0) - - v1, err := dyn.GetByPath(vin, dyn.NewPath(dyn.Index(1))) - assert.NoError(t, err) - assert.Equal(t, dyn.V(43), v1) -} diff --git a/libs/dyn/visit_map.go b/libs/dyn/visit_map.go deleted file mode 100644 index a20655e5c21..00000000000 --- a/libs/dyn/visit_map.go +++ /dev/null @@ -1,89 +0,0 @@ -package dyn - -import ( - "fmt" - "slices" -) - -// MapFunc is a function that maps a value to another value. -type MapFunc func(Path, Value) (Value, error) - -// Foreach returns a [MapFunc] that applies the specified [MapFunc] to each -// value in a map or sequence and returns the new map or sequence. -// If the input is nil, it returns nil. -func Foreach(fn MapFunc) MapFunc { - return func(p Path, v Value) (Value, error) { - switch v.Kind() { - case KindNil: - return v, nil - case KindMap: - m := v.MustMap().Clone() - for _, pair := range m.Pairs() { - pk := pair.Key - pv := pair.Value - nv, err := fn(p.Append(Key(pk.MustString())), pv) - if err != nil { - return InvalidValue, err - } - m.SetLoc(pk.MustString(), pk.Locations(), nv) - } - return NewValue(m, v.Locations()), nil - case KindSequence: - s := slices.Clone(v.MustSequence()) - for i, value := range s { - var err error - s[i], err = fn(p.Append(Index(i)), value) - if err != nil { - return InvalidValue, err - } - } - return NewValue(s, v.Locations()), nil - default: - return InvalidValue, fmt.Errorf("expected a map or sequence, found %s", v.Kind()) - } - } -} - -// Map applies a function to the value at the given path in the given value. -// It is identical to [MapByPath], except that it takes a string path instead of a [Path]. -func Map(v Value, path string, fn MapFunc) (Value, error) { - p, err := NewPathFromString(path) - if err != nil { - return InvalidValue, err - } - return MapByPath(v, p, fn) -} - -// MapByPath applies a function to the value at the given path in the given value. -// It is identical to [MapByPattern], except that it takes a [Path] instead of a [Pattern]. -// This means it only matches a single value, not a pattern of values. -func MapByPath(v Value, p Path, fn MapFunc) (Value, error) { - return MapByPattern(v, NewPatternFromPath(p), fn) -} - -// MapByPattern applies a function to the values whose paths match the given pattern in the given value. -// If successful, it returns the new value with all intermediate values copied and updated. -// -// If the pattern contains a key that doesn't exist, or an index that is out of bounds, -// it returns the original value and no error. -// -// If the pattern is invalid for the given value, it returns InvalidValue and an error. -func MapByPattern(v Value, p Pattern, fn MapFunc) (Value, error) { - nv, err := visit(v, EmptyPath, p, visitOptions{ - fn: fn, - }) - - // Check for success. - if err == nil { - return nv, nil - } - - // Return original value if: - // - any map or sequence is a nil, or - // - a key or index is missing - if IsCannotTraverseNilError(err) || IsNoSuchKeyError(err) || IsIndexOutOfBoundsError(err) { - return v, nil - } - - return nv, err -} diff --git a/libs/dyn/visit_map_test.go b/libs/dyn/visit_map_test.go deleted file mode 100644 index bff4b0bd3f9..00000000000 --- a/libs/dyn/visit_map_test.go +++ /dev/null @@ -1,477 +0,0 @@ -package dyn_test - -import ( - "errors" - "fmt" - "testing" - - "github.com/databricks/cli/libs/dyn" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestMapWithEmptyPath(t *testing.T) { - // An empty path means to return the value itself. - vin := dyn.V(42) - vout, err := dyn.MapByPath(dyn.InvalidValue, dyn.EmptyPath, func(_ dyn.Path, v dyn.Value) (dyn.Value, error) { - return vin, nil - }) - assert.NoError(t, err) - assert.Equal(t, vin, vout) -} - -func TestMapOnNilValue(t *testing.T) { - var nv dyn.Value - var err error - nv, err = dyn.MapByPath(dyn.NilValue, dyn.NewPath(dyn.Key("foo")), nil) - assert.NoError(t, err) - assert.Equal(t, dyn.NilValue, nv) - nv, err = dyn.MapByPath(dyn.NilValue, dyn.NewPath(dyn.Index(42)), nil) - assert.NoError(t, err) - assert.Equal(t, dyn.NilValue, nv) -} - -func TestMapFuncOnMap(t *testing.T) { - vin := dyn.V(map[string]dyn.Value{ - "foo": dyn.V(42), - "bar": dyn.V(43), - }) - - var err error - - _, err = dyn.MapByPath(vin, dyn.NewPath(dyn.Index(42)), nil) - assert.ErrorContains(t, err, `expected a sequence to index "[42]", found map`) - - // A key that does not exist is not an error. - vout, err := dyn.MapByPath(vin, dyn.NewPath(dyn.Key("baz")), nil) - assert.NoError(t, err) - assert.Equal(t, vin, vout) - - // Note: in the test cases below we implicitly test that the original - // value is not modified as we repeatedly set values on it. - vfoo, err := dyn.MapByPath(vin, dyn.NewPath(dyn.Key("foo")), func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - assert.Equal(t, dyn.NewPath(dyn.Key("foo")), p) - assert.Equal(t, dyn.V(42), v) - return dyn.V(44), nil - }) - assert.NoError(t, err) - assert.Equal(t, map[string]any{ - "foo": 44, - "bar": 43, - }, vfoo.AsAny()) - - vbar, err := dyn.MapByPath(vin, dyn.NewPath(dyn.Key("bar")), func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - assert.Equal(t, dyn.NewPath(dyn.Key("bar")), p) - assert.Equal(t, dyn.V(43), v) - return dyn.V(45), nil - }) - assert.NoError(t, err) - assert.Equal(t, map[string]any{ - "foo": 42, - "bar": 45, - }, vbar.AsAny()) - - // Return error from map function. - ref := errors.New("error") - verr, err := dyn.MapByPath(vin, dyn.NewPath(dyn.Key("foo")), func(_ dyn.Path, v dyn.Value) (dyn.Value, error) { - return dyn.InvalidValue, ref - }) - assert.Equal(t, dyn.InvalidValue, verr) - assert.ErrorIs(t, err, ref) -} - -func TestMapFuncOnMapWithEmptySequence(t *testing.T) { - variants := []dyn.Value{ - // empty sequence - dyn.V([]dyn.Value{}), - // non-empty sequence - dyn.V([]dyn.Value{dyn.V(42)}), - } - - for i := range variants { - vin := dyn.V(map[string]dyn.Value{ - "key": variants[i], - }) - - for j := range variants { - vout, err := dyn.MapByPath(vin, dyn.NewPath(dyn.Key("key")), func(_ dyn.Path, v dyn.Value) (dyn.Value, error) { - return variants[j], nil - }) - assert.NoError(t, err) - assert.Equal(t, variants[j], vout.Get("key")) - } - } -} - -func TestMapFuncOnSequence(t *testing.T) { - vin := dyn.V([]dyn.Value{ - dyn.V(42), - dyn.V(43), - }) - - var err error - - _, err = dyn.MapByPath(vin, dyn.NewPath(dyn.Key("foo")), nil) - assert.ErrorContains(t, err, `expected a map to index "foo", found sequence`) - - // An index that does not exist is not an error. - vout, err := dyn.MapByPath(vin, dyn.NewPath(dyn.Index(2)), nil) - assert.NoError(t, err) - assert.Equal(t, vin, vout) - - // Note: in the test cases below we implicitly test that the original - // value is not modified as we repeatedly set values on it. - v0, err := dyn.MapByPath(vin, dyn.NewPath(dyn.Index(0)), func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - assert.Equal(t, dyn.NewPath(dyn.Index(0)), p) - assert.Equal(t, dyn.V(42), v) - return dyn.V(44), nil - }) - assert.NoError(t, err) - assert.Equal(t, []any{44, 43}, v0.AsAny()) - - v1, err := dyn.MapByPath(vin, dyn.NewPath(dyn.Index(1)), func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - assert.Equal(t, dyn.NewPath(dyn.Index(1)), p) - assert.Equal(t, dyn.V(43), v) - return dyn.V(45), nil - }) - assert.NoError(t, err) - assert.Equal(t, []any{42, 45}, v1.AsAny()) - - // Return error from map function. - ref := errors.New("error") - verr, err := dyn.MapByPath(vin, dyn.NewPath(dyn.Index(0)), func(_ dyn.Path, v dyn.Value) (dyn.Value, error) { - return dyn.InvalidValue, ref - }) - assert.Equal(t, dyn.InvalidValue, verr) - assert.ErrorIs(t, err, ref) -} - -func TestMapFuncOnSequenceWithEmptySequence(t *testing.T) { - variants := []dyn.Value{ - // empty sequence - dyn.V([]dyn.Value{}), - // non-empty sequence - dyn.V([]dyn.Value{dyn.V(42)}), - } - - for i := range variants { - vin := dyn.V([]dyn.Value{ - variants[i], - }) - - for j := range variants { - vout, err := dyn.MapByPath(vin, dyn.NewPath(dyn.Index(0)), func(_ dyn.Path, v dyn.Value) (dyn.Value, error) { - return variants[j], nil - }) - assert.NoError(t, err) - assert.Equal(t, variants[j], vout.Index(0)) - } - } -} - -func TestMapForeachOnMap(t *testing.T) { - vin := dyn.V(map[string]dyn.Value{ - "foo": dyn.V(42), - "bar": dyn.V(43), - }) - - var err error - - // Run foreach, adding 1 to each of the elements. - vout, err := dyn.Map(vin, ".", dyn.Foreach(func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - i, ok := v.AsInt() - require.True(t, ok, "expected an integer") - switch p[0].Key() { - case "foo": - assert.EqualValues(t, 42, i) - return dyn.V(43), nil - case "bar": - assert.EqualValues(t, 43, i) - return dyn.V(44), nil - default: - return dyn.InvalidValue, fmt.Errorf("unexpected key %q", p[0].Key()) - } - })) - assert.NoError(t, err) - assert.Equal(t, map[string]any{ - "foo": 43, - "bar": 44, - }, vout.AsAny()) - - // Check that the original has not been modified. - assert.Equal(t, map[string]any{ - "foo": 42, - "bar": 43, - }, vin.AsAny()) -} - -func TestMapForeachOnMapError(t *testing.T) { - vin := dyn.V(map[string]dyn.Value{ - "foo": dyn.V(42), - "bar": dyn.V(43), - }) - - // Check that an error from the map function propagates. - ref := errors.New("error") - _, err := dyn.Map(vin, ".", dyn.Foreach(func(_ dyn.Path, v dyn.Value) (dyn.Value, error) { - return dyn.InvalidValue, ref - })) - assert.ErrorIs(t, err, ref) -} - -func TestMapForeachOnSequence(t *testing.T) { - vin := dyn.V([]dyn.Value{ - dyn.V(42), - dyn.V(43), - }) - - var err error - - // Run foreach, adding 1 to each of the elements. - vout, err := dyn.Map(vin, ".", dyn.Foreach(func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - i, ok := v.AsInt() - require.True(t, ok, "expected an integer") - switch p[0].Index() { - case 0: - assert.EqualValues(t, 42, i) - return dyn.V(43), nil - case 1: - assert.EqualValues(t, 43, i) - return dyn.V(44), nil - default: - return dyn.InvalidValue, fmt.Errorf("unexpected index %d", p[0].Index()) - } - })) - assert.NoError(t, err) - assert.Equal(t, []any{43, 44}, vout.AsAny()) - - // Check that the original has not been modified. - assert.Equal(t, []any{42, 43}, vin.AsAny()) -} - -func TestMapForeachOnSequenceError(t *testing.T) { - vin := dyn.V([]dyn.Value{ - dyn.V(42), - dyn.V(43), - }) - - // Check that an error from the map function propagates. - ref := errors.New("error") - _, err := dyn.Map(vin, ".", dyn.Foreach(func(_ dyn.Path, v dyn.Value) (dyn.Value, error) { - return dyn.InvalidValue, ref - })) - assert.ErrorIs(t, err, ref) -} - -func TestMapForeachOnOtherError(t *testing.T) { - vin := dyn.V(42) - - // Check that if foreach is applied to something other than a map or a sequence, it returns an error. - _, err := dyn.Map(vin, ".", dyn.Foreach(func(_ dyn.Path, v dyn.Value) (dyn.Value, error) { - return dyn.InvalidValue, nil - })) - assert.ErrorContains(t, err, "expected a map or sequence, found int") -} - -func TestMapForeachOnNil(t *testing.T) { - vin := dyn.NilValue - - // Check that if foreach is applied to nil, it returns nil. - vout, err := dyn.Map(vin, ".", dyn.Foreach(func(_ dyn.Path, v dyn.Value) (dyn.Value, error) { - return dyn.InvalidValue, nil - })) - assert.NoError(t, err) - assert.Equal(t, dyn.NilValue, vout) -} - -func TestMapByPatternOnNilValue(t *testing.T) { - var err error - _, err = dyn.MapByPattern(dyn.NilValue, dyn.NewPattern(dyn.AnyKey()), nil) - assert.ErrorContains(t, err, `expected a map at "", found nil`) - _, err = dyn.MapByPattern(dyn.NilValue, dyn.NewPattern(dyn.AnyIndex()), nil) - assert.ErrorContains(t, err, `expected a sequence at "", found nil`) -} - -func TestMapByPatternOnMap(t *testing.T) { - vin := dyn.V(map[string]dyn.Value{ - "a": dyn.V(map[string]dyn.Value{ - "b": dyn.V(42), - }), - "b": dyn.V(map[string]dyn.Value{ - "c": dyn.V(43), - }), - }) - - var err error - - // Expect an error if the pattern structure doesn't match the value structure. - _, err = dyn.MapByPattern(vin, dyn.NewPattern(dyn.AnyKey(), dyn.Index(0)), nil) - assert.ErrorContains(t, err, `expected a sequence to index`) - - // Apply function to pattern "*.b". - vout, err := dyn.MapByPattern(vin, dyn.NewPattern(dyn.AnyKey(), dyn.Key("b")), func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - assert.Equal(t, dyn.NewPath(dyn.Key("a"), dyn.Key("b")), p) - assert.Equal(t, dyn.V(42), v) - return dyn.V(44), nil - }) - assert.NoError(t, err) - assert.Equal(t, map[string]any{ - "a": map[string]any{ - "b": 44, - }, - "b": map[string]any{ - "c": 43, - }, - }, vout.AsAny()) -} - -func TestMapByPatternOnMapWithoutMatch(t *testing.T) { - vin := dyn.V(map[string]dyn.Value{ - "a": dyn.V(map[string]dyn.Value{ - "b": dyn.V(42), - }), - "b": dyn.V(map[string]dyn.Value{ - "c": dyn.V(43), - }), - }) - - // Apply function to pattern "*.zzz". - vout, err := dyn.MapByPattern(vin, dyn.NewPattern(dyn.AnyKey(), dyn.Key("zzz")), nil) - assert.NoError(t, err) - assert.Equal(t, vin, vout) -} - -func TestMapByPatternOnSequence(t *testing.T) { - vin := dyn.V([]dyn.Value{ - dyn.V([]dyn.Value{ - dyn.V(42), - }), - dyn.V([]dyn.Value{ - dyn.V(43), - dyn.V(44), - }), - }) - - var err error - - // Expect an error if the pattern structure doesn't match the value structure. - _, err = dyn.MapByPattern(vin, dyn.NewPattern(dyn.AnyIndex(), dyn.Key("a")), nil) - assert.ErrorContains(t, err, `expected a map to index`) - - // Apply function to pattern "*.c". - vout, err := dyn.MapByPattern(vin, dyn.NewPattern(dyn.AnyIndex(), dyn.Index(1)), func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - assert.Equal(t, dyn.NewPath(dyn.Index(1), dyn.Index(1)), p) - assert.Equal(t, dyn.V(44), v) - return dyn.V(45), nil - }) - assert.NoError(t, err) - assert.Equal(t, []any{ - []any{42}, - []any{43, 45}, - }, vout.AsAny()) -} - -func TestMapByPatternWildcardSkipsNonMatchingSiblings(t *testing.T) { - tests := []struct { - name string - vin dyn.Value - pattern dyn.Pattern - want any - visited []string - }{ - { - name: "nil sibling under any key", - vin: dyn.V(map[string]dyn.Value{ - "a": dyn.NilValue, - "b": dyn.V(map[string]dyn.Value{"x": dyn.V(1)}), - }), - pattern: dyn.NewPattern(dyn.AnyKey(), dyn.Key("x")), - want: map[string]any{ - "a": nil, - "b": map[string]any{"x": 42}, - }, - visited: []string{"b.x"}, - }, - { - name: "nil sequence sibling under any key", - vin: dyn.V(map[string]dyn.Value{ - "a": dyn.V(map[string]dyn.Value{"tasks": dyn.NilValue}), - "b": dyn.V(map[string]dyn.Value{"tasks": dyn.V([]dyn.Value{dyn.V(1)})}), - }), - pattern: dyn.NewPattern(dyn.AnyKey(), dyn.Key("tasks"), dyn.AnyIndex()), - want: map[string]any{ - "a": map[string]any{"tasks": nil}, - "b": map[string]any{"tasks": []any{42}}, - }, - visited: []string{"b.tasks[0]"}, - }, - { - name: "wrong kind sibling under any key", - vin: dyn.V(map[string]dyn.Value{ - "a": dyn.V(map[string]dyn.Value{"tasks": dyn.V("oops")}), - "b": dyn.V(map[string]dyn.Value{"tasks": dyn.V([]dyn.Value{dyn.V(1)})}), - }), - pattern: dyn.NewPattern(dyn.AnyKey(), dyn.Key("tasks"), dyn.AnyIndex()), - want: map[string]any{ - "a": map[string]any{"tasks": "oops"}, - "b": map[string]any{"tasks": []any{42}}, - }, - visited: []string{"b.tasks[0]"}, - }, - { - name: "nil element under any index", - vin: dyn.V([]dyn.Value{ - dyn.NilValue, - dyn.V(map[string]dyn.Value{"x": dyn.V(1)}), - }), - pattern: dyn.NewPattern(dyn.AnyIndex(), dyn.Key("x")), - want: []any{ - nil, - map[string]any{"x": 42}, - }, - visited: []string{"[1].x"}, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - var visited []string - vout, err := dyn.MapByPattern(tc.vin, tc.pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - visited = append(visited, p.String()) - return dyn.V(42), nil - }) - assert.NoError(t, err) - assert.Equal(t, tc.want, vout.AsAny()) - assert.Equal(t, tc.visited, visited) - }) - } -} - -func TestMapByPatternWildcardPropagatesMapFuncError(t *testing.T) { - vin := dyn.V(map[string]dyn.Value{ - "a": dyn.V(map[string]dyn.Value{"x": dyn.V(1)}), - }) - - ref := errors.New("error") - _, err := dyn.MapByPattern(vin, dyn.NewPattern(dyn.AnyKey(), dyn.Key("x")), func(_ dyn.Path, v dyn.Value) (dyn.Value, error) { - return dyn.InvalidValue, ref - }) - assert.ErrorIs(t, err, ref) -} - -func TestMapByPatternOnSequenceWithoutMatch(t *testing.T) { - vin := dyn.V([]dyn.Value{ - dyn.V([]dyn.Value{ - dyn.V(42), - }), - dyn.V([]dyn.Value{ - dyn.V(43), - dyn.V(44), - }), - }) - - // Apply function to pattern "*.zzz". - vout, err := dyn.MapByPattern(vin, dyn.NewPattern(dyn.AnyIndex(), dyn.Index(42)), nil) - assert.NoError(t, err) - assert.Equal(t, vin, vout) -} diff --git a/libs/dyn/visit_set.go b/libs/dyn/visit_set.go deleted file mode 100644 index 1b3c7f1b661..00000000000 --- a/libs/dyn/visit_set.go +++ /dev/null @@ -1,77 +0,0 @@ -package dyn - -import ( - "fmt" - "slices" -) - -// Set assigns a new value at the specified path in the specified value. -// It is identical to [SetByPath], except that it takes a string path instead of a [Path]. -func Set(v Value, path string, nv Value) (Value, error) { - p, err := NewPathFromString(path) - if err != nil { - return InvalidValue, err - } - return SetByPath(v, p, nv) -} - -// SetByPath assigns the given value at the specified path in the specified value. -// If successful, it returns the new value with all intermediate values copied and updated. -// If the path doesn't exist, it returns InvalidValue and an error. -func SetByPath(v Value, p Path, nv Value) (Value, error) { - lp := len(p) - if lp == 0 { - return nv, nil - } - - component := p[lp-1] - p = p[:lp-1] - - return visit(v, EmptyPath, NewPatternFromPath(p), visitOptions{ - fn: func(prefix Path, v Value) (Value, error) { - path := append(prefix, component) - - switch { - case component.isKey(): - // Expect a map to be set if this is a key. - m, ok := v.AsMap() - if !ok { - return InvalidValue, fmt.Errorf("expected a map to index %q, found %s", path, v.Kind()) - } - - // Return an updated map value. - m = m.Clone() - m.SetLoc(component.key, nil, nv) - return Value{ - v: m, - k: KindMap, - l: v.l, - }, nil - - case component.isIndex(): - // Expect a sequence to be set if this is an index. - s, ok := v.AsSequence() - if !ok { - return InvalidValue, fmt.Errorf("expected a sequence to index %q, found %s", path, v.Kind()) - } - - // Lookup current value in the sequence. - if component.index < 0 || component.index >= len(s) { - return InvalidValue, indexOutOfBoundsError{prefix} - } - - // Return an updated sequence value. - s = slices.Clone(s) - s[component.index] = nv - return Value{ - v: s, - k: KindSequence, - l: v.l, - }, nil - - default: - panic("invalid component") - } - }, - }) -} diff --git a/libs/dyn/visit_set_test.go b/libs/dyn/visit_set_test.go deleted file mode 100644 index b3847158757..00000000000 --- a/libs/dyn/visit_set_test.go +++ /dev/null @@ -1,90 +0,0 @@ -package dyn_test - -import ( - "testing" - - "github.com/databricks/cli/libs/dyn" - "github.com/stretchr/testify/assert" -) - -func TestSetWithEmptyPath(t *testing.T) { - // An empty path means to return the value itself. - vin := dyn.V(42) - vout, err := dyn.SetByPath(dyn.InvalidValue, dyn.EmptyPath, vin) - assert.NoError(t, err) - assert.Equal(t, vin, vout) -} - -func TestSetOnNilValue(t *testing.T) { - var err error - _, err = dyn.SetByPath(dyn.NilValue, dyn.NewPath(dyn.Key("foo")), dyn.V(42)) - assert.ErrorContains(t, err, `expected a map to index "foo", found nil`) - _, err = dyn.SetByPath(dyn.NilValue, dyn.NewPath(dyn.Index(42)), dyn.V(42)) - assert.ErrorContains(t, err, `expected a sequence to index "[42]", found nil`) -} - -func TestSetOnMap(t *testing.T) { - vin := dyn.V(map[string]dyn.Value{ - "foo": dyn.V(42), - "bar": dyn.V(43), - }) - - var err error - - _, err = dyn.SetByPath(vin, dyn.NewPath(dyn.Index(42)), dyn.V(42)) - assert.ErrorContains(t, err, `expected a sequence to index "[42]", found map`) - - // Note: in the test cases below we implicitly test that the original - // value is not modified as we repeatedly set values on it. - - vfoo, err := dyn.SetByPath(vin, dyn.NewPath(dyn.Key("foo")), dyn.V(44)) - assert.NoError(t, err) - assert.Equal(t, map[string]any{ - "foo": 44, - "bar": 43, - }, vfoo.AsAny()) - - vbar, err := dyn.SetByPath(vin, dyn.NewPath(dyn.Key("bar")), dyn.V(45)) - assert.NoError(t, err) - assert.Equal(t, map[string]any{ - "foo": 42, - "bar": 45, - }, vbar.AsAny()) - - vbaz, err := dyn.SetByPath(vin, dyn.NewPath(dyn.Key("baz")), dyn.V(46)) - assert.NoError(t, err) - assert.Equal(t, map[string]any{ - "foo": 42, - "bar": 43, - "baz": 46, - }, vbaz.AsAny()) -} - -func TestSetOnSequence(t *testing.T) { - vin := dyn.V([]dyn.Value{ - dyn.V(42), - dyn.V(43), - }) - - var err error - - _, err = dyn.SetByPath(vin, dyn.NewPath(dyn.Key("foo")), dyn.V(42)) - assert.ErrorContains(t, err, `expected a map to index "foo", found sequence`) - - // It is not allowed to set a value at an index that is out of bounds. - _, err = dyn.SetByPath(vin, dyn.NewPath(dyn.Index(-1)), dyn.V(42)) - assert.True(t, dyn.IsIndexOutOfBoundsError(err)) - _, err = dyn.SetByPath(vin, dyn.NewPath(dyn.Index(2)), dyn.V(42)) - assert.True(t, dyn.IsIndexOutOfBoundsError(err)) - - // Note: in the test cases below we implicitly test that the original - // value is not modified as we repeatedly set values on it. - - v0, err := dyn.SetByPath(vin, dyn.NewPath(dyn.Index(0)), dyn.V(44)) - assert.NoError(t, err) - assert.Equal(t, []any{44, 43}, v0.AsAny()) - - v1, err := dyn.SetByPath(vin, dyn.NewPath(dyn.Index(1)), dyn.V(45)) - assert.NoError(t, err) - assert.Equal(t, []any{42, 45}, v1.AsAny()) -} diff --git a/libs/dyn/visit_test.go b/libs/dyn/visit_test.go deleted file mode 100644 index b54e6ad37c7..00000000000 --- a/libs/dyn/visit_test.go +++ /dev/null @@ -1,36 +0,0 @@ -package dyn_test - -import ( - "testing" - - "github.com/databricks/cli/libs/dyn" - "github.com/stretchr/testify/assert" -) - -func TestVisitCallbackPathCopy(t *testing.T) { - vin := dyn.V(map[string]dyn.Value{ - "foo": dyn.V(42), - "bar": dyn.V(43), - }) - - var paths []dyn.Path - - // The callback should receive a copy of the path. - // If the same underlying value is used, all collected paths will be the same. - // This test uses `MapByPattern` to collect all paths in the map. - // Visit itself doesn't have public functions and we exclusively use black-box testing for this package. - _, _ = dyn.MapByPattern(vin, dyn.NewPattern(dyn.AnyKey()), func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - paths = append(paths, p) - return v, nil - }) - - // Verify that the paths retained their original values. - var strings []string - for _, p := range paths { - strings = append(strings, p.String()) - } - assert.ElementsMatch(t, strings, []string{ - "foo", - "bar", - }) -} diff --git a/libs/dyn/walk.go b/libs/dyn/walk.go deleted file mode 100644 index af03ef27743..00000000000 --- a/libs/dyn/walk.go +++ /dev/null @@ -1,95 +0,0 @@ -package dyn - -import "errors" - -// WalkValueFunc is the type of the function called by Walk to traverse the configuration tree. -type WalkValueFunc func(p Path, v Value) (Value, error) - -// ErrDrop may be returned by WalkValueFunc to remove a value from the subtree. -var ErrDrop = errors.New("drop value from subtree") - -// ErrSkip may be returned by WalkValueFunc to skip traversal of a subtree. -var ErrSkip = errors.New("skip traversal of subtree") - -// Walk walks the configuration tree and calls the given function on each node. -// The callback may return ErrDrop to remove a value from the subtree. -// The callback may return ErrSkip to skip traversal of a subtree. -// If the callback returns another error, the walk is aborted, and the error is returned. -func Walk(v Value, fn func(p Path, v Value) (Value, error)) (Value, error) { - return walk(v, EmptyPath, fn) -} - -// Unexported counterpart to Walk. -// It carries the path leading up to the current node, -// such that it can be passed to the WalkValueFunc. -func walk(v Value, p Path, fn func(p Path, v Value) (Value, error)) (Value, error) { - v, err := fn(p, v) - if err != nil { - if err == ErrSkip { - return v, nil - } - return InvalidValue, err - } - - switch v.Kind() { - case KindMap: - m := v.MustMap() - out := newMappingWithSize(m.Len()) - for _, pair := range m.Pairs() { - pk := pair.Key - pv := pair.Value - nv, err := walk(pv, append(p, Key(pk.MustString())), fn) - if err == ErrDrop { - continue - } - if err != nil { - return InvalidValue, err - } - out.SetLoc(pk.MustString(), pk.Locations(), nv) - } - v.v = out - case KindSequence: - s := v.MustSequence() - out := make([]Value, 0, len(s)) - for i := range s { - nv, err := walk(s[i], append(p, Index(i)), fn) - if err == ErrDrop { - continue - } - if err != nil { - return InvalidValue, err - } - out = append(out, nv) - } - v.v = out - default: - // Other kinds are not walkable. - // This was a leaf node. - } - - return v, nil -} - -// CollectLeafPaths traverses the value and returns all paths (as dot notation strings) to leaf nodes (non-map, non-sequence). -// The return value is not ordered. -func CollectLeafPaths(v Value) []string { - var paths []string - - Walk(v, func(p Path, v Value) (Value, error) { //nolint:errcheck - if len(p) == 0 { - return v, nil - } - - switch v.Kind() { - case KindMap, KindSequence: - // Ignore internal nodes. - default: - // This was a leaf node. - paths = append(paths, p.String()) - } - - return v, nil - }) - - return paths -} diff --git a/libs/dyn/walk_read_only.go b/libs/dyn/walk_read_only.go deleted file mode 100644 index caead36289e..00000000000 --- a/libs/dyn/walk_read_only.go +++ /dev/null @@ -1,44 +0,0 @@ -package dyn - -// WalkReadOnly walks the configuration tree in readonly mode and calls the given function on each node. -// The callback may return ErrSkip to skip traversal of a subtree. -// If the callback returns another error, the walk is aborted, and the error is returned. -func WalkReadOnly(v Value, fn func(p Path, v Value) error) error { - return walkReadOnly(v, EmptyPath, fn) -} - -// Unexported counterpart to WalkReadOnly. -// It carries the path leading up to the current node, -// such that it can be passed to the callback function. -func walkReadOnly(v Value, p Path, fn func(p Path, v Value) error) error { - if err := fn(p, v); err != nil { - if err == ErrSkip { - return nil - } - return err - } - - switch v.Kind() { - case KindMap: - m := v.MustMap() - for _, pair := range m.Pairs() { - pk := pair.Key - pv := pair.Value - if err := walkReadOnly(pv, append(p, Key(pk.MustString())), fn); err != nil { - return err - } - } - case KindSequence: - s := v.MustSequence() - for i := range s { - if err := walkReadOnly(s[i], append(p, Index(i)), fn); err != nil { - return err - } - } - default: - // Other kinds are not walkable. - // This was a leaf node. - } - - return nil -} diff --git a/libs/dyn/walk_read_only_test.go b/libs/dyn/walk_read_only_test.go deleted file mode 100644 index f6df5344d8c..00000000000 --- a/libs/dyn/walk_read_only_test.go +++ /dev/null @@ -1,245 +0,0 @@ -package dyn_test - -import ( - "errors" - "testing" - - "github.com/databricks/cli/libs/dyn" - "github.com/stretchr/testify/assert" -) - -func TestWalkReadOnly(t *testing.T) { - tests := []struct { - name string - input dyn.Value - expectedPaths []dyn.Path - expectedValues []dyn.Value - }{ - { - name: "simple map", - input: dyn.V(dyn.NewMappingFromPairs( - []dyn.Pair{ - {Key: dyn.V("a"), Value: dyn.V("1")}, - {Key: dyn.V("b"), Value: dyn.V("2")}, - }, - )), - expectedPaths: []dyn.Path{ - dyn.EmptyPath, - {dyn.Key("a")}, - {dyn.Key("b")}, - }, - expectedValues: []dyn.Value{ - dyn.V(dyn.NewMappingFromPairs( - []dyn.Pair{ - {Key: dyn.V("a"), Value: dyn.V("1")}, - {Key: dyn.V("b"), Value: dyn.V("2")}, - }, - )), - dyn.V("1"), - dyn.V("2"), - }, - }, - { - name: "nested map", - input: dyn.V(map[string]dyn.Value{ - "a": dyn.V(map[string]dyn.Value{ - "b": dyn.V("1"), - }), - }), - expectedPaths: []dyn.Path{ - dyn.EmptyPath, - {dyn.Key("a")}, - {dyn.Key("a"), dyn.Key("b")}, - }, - expectedValues: []dyn.Value{ - dyn.V(map[string]dyn.Value{ - "a": dyn.V(map[string]dyn.Value{ - "b": dyn.V("1"), - }), - }), - dyn.V(map[string]dyn.Value{ - "b": dyn.V("1"), - }), - dyn.V("1"), - }, - }, - { - name: "sequence", - input: dyn.V([]dyn.Value{ - dyn.V("1"), - dyn.V("2"), - }), - expectedPaths: []dyn.Path{ - dyn.EmptyPath, - {dyn.Index(0)}, - {dyn.Index(1)}, - }, - expectedValues: []dyn.Value{ - dyn.V([]dyn.Value{ - dyn.V("1"), - dyn.V("2"), - }), - dyn.V("1"), - dyn.V("2"), - }, - }, - { - name: "nested sequence", - input: dyn.V([]dyn.Value{ - dyn.V([]dyn.Value{ - dyn.V("1"), - }), - }), - expectedPaths: []dyn.Path{ - dyn.EmptyPath, - {dyn.Index(0)}, - {dyn.Index(0), dyn.Index(0)}, - }, - expectedValues: []dyn.Value{ - dyn.V([]dyn.Value{ - dyn.V([]dyn.Value{ - dyn.V("1"), - }), - }), - dyn.V([]dyn.Value{ - dyn.V("1"), - }), - dyn.V("1"), - }, - }, - { - name: "complex structure", - input: dyn.V(map[string]dyn.Value{ - "a": dyn.V([]dyn.Value{ - dyn.V(map[string]dyn.Value{ - "b": dyn.V("1"), - }), - }), - }), - expectedPaths: []dyn.Path{ - dyn.EmptyPath, - {dyn.Key("a")}, - {dyn.Key("a"), dyn.Index(0)}, - {dyn.Key("a"), dyn.Index(0), dyn.Key("b")}, - }, - expectedValues: []dyn.Value{ - dyn.V(map[string]dyn.Value{ - "a": dyn.V([]dyn.Value{ - dyn.V(map[string]dyn.Value{ - "b": dyn.V("1"), - }), - }), - }), - dyn.V([]dyn.Value{ - dyn.V(map[string]dyn.Value{ - "b": dyn.V("1"), - }), - }), - dyn.V(map[string]dyn.Value{ - "b": dyn.V("1"), - }), - dyn.V("1"), - }, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - visitedPaths := make([]dyn.Path, 0, len(tt.expectedPaths)) - visitedValues := make([]dyn.Value, 0, len(tt.expectedValues)) - - err := dyn.WalkReadOnly(tt.input, func(p dyn.Path, v dyn.Value) error { - visitedPaths = append(visitedPaths, p) - visitedValues = append(visitedValues, v) - return nil - }) - assert.NoError(t, err) - assert.Equal(t, tt.expectedPaths, visitedPaths) - assert.Equal(t, tt.expectedValues, visitedValues) - }) - } -} - -func TestWalkReadOnly_Error(t *testing.T) { - testErr := errors.New("test error") - input := dyn.V(map[string]dyn.Value{ - "a": dyn.V("1"), - }) - - err := dyn.WalkReadOnly(input, func(p dyn.Path, v dyn.Value) error { - if p.Equal(dyn.Path{dyn.Key("a")}) { - return testErr - } - return nil - }) - - assert.Equal(t, err, testErr) -} - -func TestWalkReadOnly_SkipPaths(t *testing.T) { - va := dyn.V(dyn.NewMappingFromPairs( - []dyn.Pair{ - {Key: dyn.V("b"), Value: dyn.V("1")}, - {Key: dyn.V("c"), Value: dyn.V("2")}, - }, - )) - - vd := dyn.V(dyn.NewMappingFromPairs( - []dyn.Pair{ - {Key: dyn.V("e"), Value: dyn.V("3")}, - }, - )) - - input := dyn.V(dyn.NewMappingFromPairs( - []dyn.Pair{ - { - Key: dyn.V("a"), - Value: va, - }, - { - Key: dyn.V("d"), - Value: vd, - }, - { - Key: dyn.V("f"), - Value: dyn.V("4"), - }, - }, - )) - - skipPaths := map[string]bool{ - "a.b": true, - "d": true, - } - - var visitedPaths []dyn.Path - var visitedValues []dyn.Value - - err := dyn.WalkReadOnly(input, func(p dyn.Path, v dyn.Value) error { - _, ok := skipPaths[p.String()] - if ok { - return dyn.ErrSkip - } - - visitedPaths = append(visitedPaths, p) - visitedValues = append(visitedValues, v) - return nil - }) - assert.NoError(t, err) - - expectedPaths := []dyn.Path{ - dyn.EmptyPath, - {dyn.Key("a")}, - {dyn.Key("a"), dyn.Key("c")}, - {dyn.Key("f")}, - } - expectedValues := []dyn.Value{ - input, - va, - dyn.V("2"), - dyn.V("4"), - } - - assert.Equal(t, expectedPaths, visitedPaths) - assert.Equal(t, expectedValues, visitedValues) -} diff --git a/libs/dyn/walk_test.go b/libs/dyn/walk_test.go deleted file mode 100644 index f59683f0381..00000000000 --- a/libs/dyn/walk_test.go +++ /dev/null @@ -1,269 +0,0 @@ -package dyn_test - -import ( - "errors" - "testing" - - . "github.com/databricks/cli/libs/dyn" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// Return values for specific paths. -type walkReturn struct { - path Path - - // Return values. - fn func(Value) Value - err error -} - -// Track the calls to the callback. -type walkCall struct { - path Path - value Value -} - -// Track the calls to the callback. -type walkCallTracker struct { - returns []walkReturn - calls []walkCall -} - -func (w *walkCallTracker) on(path string, fn func(Value) Value, err error) { - w.returns = append(w.returns, walkReturn{MustPathFromString(path), fn, err}) -} - -func (w *walkCallTracker) returnSkip(path string) { - w.on(path, func(v Value) Value { return v }, ErrSkip) -} - -func (w *walkCallTracker) returnDrop(path string) { - w.on(path, func(v Value) Value { return InvalidValue }, ErrDrop) -} - -func (w *walkCallTracker) track(p Path, v Value) (Value, error) { - w.calls = append(w.calls, walkCall{p, v}) - - // Look for matching return. - for _, r := range w.returns { - if p.Equal(r.path) { - return r.fn(v), r.err - } - } - - return v, nil -} - -func TestWalkEmpty(t *testing.T) { - var tracker walkCallTracker - - value := V(nil) - out, err := Walk(value, tracker.track) - require.NoError(t, err) - assert.Equal(t, value, out) - - // The callback should have been called once. - assert.Len(t, tracker.calls, 1) - - // The call should have been made with the empty path. - assert.Equal(t, EmptyPath, tracker.calls[0].path) - - // The value should be the same as the input. - assert.Equal(t, value, tracker.calls[0].value) -} - -func TestWalkMapSkip(t *testing.T) { - var tracker walkCallTracker - - // Skip traversal of the root value. - tracker.returnSkip(".") - - value := V(map[string]Value{ - "key": V("value"), - }) - out, err := Walk(value, tracker.track) - require.NoError(t, err) - assert.Equal( - t, - V(map[string]Value{ - "key": V("value"), - }), - out, - ) - - // The callback should have been called once. - assert.Len(t, tracker.calls, 1) - - // The call should have been made with the empty path. - assert.Equal(t, EmptyPath, tracker.calls[0].path) - - // The value should be the same as the input. - assert.Equal(t, value, tracker.calls[0].value) -} - -func TestWalkMapDrop(t *testing.T) { - var tracker walkCallTracker - - // Drop the value at key "foo". - tracker.returnDrop(".foo") - - value := V(map[string]Value{ - "foo": V("bar"), - "bar": V("baz"), - }) - out, err := Walk(value, tracker.track) - require.NoError(t, err) - assert.Equal( - t, - V(map[string]Value{ - "bar": V("baz"), - }), - out, - ) - - // The callback should have been called for the root and every key in the map. - assert.Len(t, tracker.calls, 3) - - // Calls 2 and 3 have been made for the keys in the map. - assert.ElementsMatch(t, - []Path{ - tracker.calls[1].path, - tracker.calls[2].path, - }, []Path{ - MustPathFromString(".foo"), - MustPathFromString(".bar"), - }) -} - -func TestWalkMapError(t *testing.T) { - var tracker walkCallTracker - - // Return an error from the callback for key "foo". - cerr := errors.New("error!") - tracker.on(".foo", func(v Value) Value { return v }, cerr) - - value := V(map[string]Value{ - "foo": V("bar"), - }) - out, err := Walk(value, tracker.track) - assert.Equal(t, cerr, err) - assert.Equal(t, InvalidValue, out) - - // The callback should have been called twice. - assert.Len(t, tracker.calls, 2) - - // The second call was for the value at key "foo". - assert.Equal(t, MustPathFromString(".foo"), tracker.calls[1].path) -} - -func TestWalkSequenceSkip(t *testing.T) { - var tracker walkCallTracker - - // Skip traversal of the root value. - tracker.returnSkip(".") - - value := V([]Value{ - V("foo"), - V("bar"), - }) - out, err := Walk(value, tracker.track) - require.NoError(t, err) - assert.Equal( - t, - V([]Value{ - V("foo"), - V("bar"), - }), - out, - ) - - // The callback should have been called once. - assert.Len(t, tracker.calls, 1) - - // The call should have been made with the empty path. - assert.Equal(t, EmptyPath, tracker.calls[0].path) - - // The value should be the same as the input. - assert.Equal(t, value, tracker.calls[0].value) -} - -func TestWalkSequenceDrop(t *testing.T) { - var tracker walkCallTracker - - // Drop the value at index 1. - tracker.returnDrop(".[1]") - - value := V([]Value{ - V("foo"), - V("bar"), - V("baz"), - }) - out, err := Walk(value, tracker.track) - require.NoError(t, err) - assert.Equal( - t, - V([]Value{ - V("foo"), - V("baz"), - }), - out, - ) - - // The callback should have been called for the root and every value in the sequence. - assert.Len(t, tracker.calls, 4) - - // The second call was for the value at index 0. - assert.Equal(t, MustPathFromString(".[0]"), tracker.calls[1].path) - assert.Equal(t, V("foo"), tracker.calls[1].value) - - // The third call was for the value at index 1. - assert.Equal(t, MustPathFromString(".[1]"), tracker.calls[2].path) - assert.Equal(t, V("bar"), tracker.calls[2].value) - - // The fourth call was for the value at index 2. - assert.Equal(t, MustPathFromString(".[2]"), tracker.calls[3].path) - assert.Equal(t, V("baz"), tracker.calls[3].value) -} - -func TestWalkSequenceError(t *testing.T) { - var tracker walkCallTracker - - // Return an error from the callback for index 1. - cerr := errors.New("error!") - tracker.on(".[1]", func(v Value) Value { return v }, cerr) - - value := V([]Value{ - V("foo"), - V("bar"), - }) - out, err := Walk(value, tracker.track) - assert.Equal(t, cerr, err) - assert.Equal(t, InvalidValue, out) - - // The callback should have been called three times. - assert.Len(t, tracker.calls, 3) - - // The second call was for the value at index 0. - assert.Equal(t, MustPathFromString(".[0]"), tracker.calls[1].path) - assert.Equal(t, V("foo"), tracker.calls[1].value) - - // The third call was for the value at index 1. - assert.Equal(t, MustPathFromString(".[1]"), tracker.calls[2].path) - assert.Equal(t, V("bar"), tracker.calls[2].value) -} - -func TestCollectLeafPaths(t *testing.T) { - v := V(map[string]Value{ - "a": V(1), - "b": V(map[string]Value{ - "c": V(2), - "d": V(map[string]Value{ - "e": V(3), - }), - }), - "f": V([]Value{V(4), V(5)}), - }) - paths := CollectLeafPaths(v) - assert.ElementsMatch(t, []string{"a", "b.c", "b.d.e", "f[0]", "f[1]"}, paths) -} diff --git a/libs/dyn/yamlloader/loader.go b/libs/dyn/yamlloader/loader.go deleted file mode 100644 index aebbf0883e9..00000000000 --- a/libs/dyn/yamlloader/loader.go +++ /dev/null @@ -1,291 +0,0 @@ -package yamlloader - -import ( - "fmt" - "math" - "strconv" - "strings" - - "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "go.yaml.in/yaml/v3" -) - -// LocationError is an error with a YAML source location that can be displayed -// to the user with a file path, line, and column number. -type LocationError struct { - Loc diag.Location - Summary string -} - -func (e *LocationError) Error() string { - return fmt.Sprintf("yaml (%s): %s", e.Loc, e.Summary) -} - -type loader struct { - path string - - // Aliases currently being expanded. A self-referential anchor (e.g. - // "a: &x {b: *x}") makes the yaml.v3 node graph cyclic; without this - // bookkeeping, expansion would recurse until stack overflow. - activeAliases map[*yaml.Node]bool -} - -func errorf(loc diag.Location, format string, args ...any) error { - return fmt.Errorf("yaml (%s): %s", loc, fmt.Sprintf(format, args...)) -} - -func newLoader(path string) *loader { - return &loader{ - path: path, - activeAliases: make(map[*yaml.Node]bool), - } -} - -func (d *loader) location(node *yaml.Node) diag.Location { - return diag.Location{ - File: d.path, - Line: node.Line, - Column: node.Column, - } -} - -func (d *loader) load(node *yaml.Node) (dyn.Value, error) { - loc := diag.Location{ - File: d.path, - Line: node.Line, - Column: node.Column, - } - - var value dyn.Value - var err error - - switch node.Kind { - case yaml.DocumentNode: - value, err = d.loadDocument(node, loc) - case yaml.SequenceNode: - value, err = d.loadSequence(node, loc) - case yaml.MappingNode: - value, err = d.loadMapping(node, loc) - case yaml.ScalarNode: - value, err = d.loadScalar(node, loc) - case yaml.AliasNode: - value, err = d.loadAlias(node, loc) - default: - return dyn.InvalidValue, errorf(loc, "unknown node kind: %v", node.Kind) - } - - if err != nil { - return value, err - } - - // Mark value as anchor if needed. - // If this node doesn't map to a type, we don't need to warn about it. - if node.Anchor != "" { - value = value.MarkAnchor() - } - - return value, nil -} - -func (d *loader) loadDocument(node *yaml.Node, loc diag.Location) (dyn.Value, error) { - return d.load(node.Content[0]) -} - -func (d *loader) loadSequence(node *yaml.Node, loc diag.Location) (dyn.Value, error) { - acc := make([]dyn.Value, len(node.Content)) - for i, n := range node.Content { - v, err := d.load(n) - if err != nil { - return dyn.InvalidValue, err - } - - acc[i] = v - } - - return dyn.NewValue(acc, []diag.Location{loc}), nil -} - -func (d *loader) loadMapping(node *yaml.Node, loc diag.Location) (dyn.Value, error) { - var merge *yaml.Node - - acc := dyn.NewMapping() - for i := 0; i < len(node.Content); i += 2 { - key := node.Content[i] - val := node.Content[i+1] - - // Assert that keys are strings - if key.Kind != yaml.ScalarNode { - return dyn.InvalidValue, errorf(loc, "key is not a scalar") - } - - st := key.ShortTag() - switch st { - case "!!str": - // OK - case "!!null": - // A literal unquoted "null" is treated as a null value by the YAML parser. - // However, when used as a key, it is treated as the string "null". - case "!!merge": - if merge != nil { - // The YAML merge key spec allows a single '<<' key per mapping. - // To merge multiple maps, use a sequence: '<<: [*anchor1, *anchor2]'. - return dyn.InvalidValue, &LocationError{ - Loc: d.location(key), - Summary: "duplicate YAML merge key ('<<') is not allowed; to merge multiple maps, use a sequence: '<<: [*anchor1, *anchor2]'", - } - } - merge = val - continue - default: - return dyn.InvalidValue, errorf(loc, "invalid key tag: %v", st) - } - - loc := []diag.Location{{ - File: d.path, - Line: key.Line, - Column: key.Column, - }} - - v, err := d.load(val) - if err != nil { - return dyn.InvalidValue, err - } - - acc.SetLoc(key.Value, loc, v) - } - - if merge == nil { - return dyn.NewValue(acc, []diag.Location{loc}), nil - } - - // Build location for the merge node. - mloc := d.location(merge) - merr := errorf(mloc, "map merge requires map or sequence of maps as the value") - - // Flatten the merge node into a slice of nodes. - // It can be either a single node or a sequence of nodes. - var mnodes []*yaml.Node - switch merge.Kind { - case yaml.SequenceNode: - mnodes = merge.Content - case yaml.AliasNode: - mnodes = []*yaml.Node{merge} - default: - return dyn.NilValue, merr - } - - // Build a sequence of values to merge. - // The entries that we already accumulated have precedence. - var seq []dyn.Mapping - for _, n := range mnodes { - v, err := d.load(n) - if err != nil { - return dyn.InvalidValue, err - } - m, ok := v.AsMap() - if !ok { - return dyn.NilValue, merr - } - seq = append(seq, m) - } - - // Append the accumulated entries to the sequence. - seq = append(seq, acc) - out := dyn.NewMapping() - for _, m := range seq { - out.Merge(m) - } - - return dyn.NewValue(out, []diag.Location{loc}), nil -} - -func newIntValue(i64 int64, loc diag.Location) dyn.Value { - // Use regular int type instead of int64 if possible. - if i64 >= math.MinInt32 && i64 <= math.MaxInt32 { - return dyn.NewValue(int(i64), []diag.Location{loc}) - } - return dyn.NewValue(i64, []diag.Location{loc}) -} - -func (d *loader) loadScalar(node *yaml.Node, loc diag.Location) (dyn.Value, error) { - st := node.ShortTag() - switch st { - case "!!str": - return dyn.NewValue(node.Value, []diag.Location{loc}), nil - case "!!bool": - switch strings.ToLower(node.Value) { - case "true": - return dyn.NewValue(true, []diag.Location{loc}), nil - case "false": - return dyn.NewValue(false, []diag.Location{loc}), nil - default: - return dyn.InvalidValue, errorf(loc, "invalid bool value: %v", node.Value) - } - case "!!int": - // Try to parse the an integer value in base 10. - // We trim leading zeros to avoid octal parsing of the "0" prefix. - // See "testdata/spec_example_2.19.yml" for background. - i64, err := strconv.ParseInt(strings.TrimLeft(node.Value, "0"), 10, 64) - if err == nil { - return newIntValue(i64, loc), nil - } - // Let the [ParseInt] function figure out the base. - i64, err = strconv.ParseInt(node.Value, 0, 64) - if err == nil { - return newIntValue(i64, loc), nil - } - return dyn.InvalidValue, errorf(loc, "invalid int value: %v", node.Value) - case "!!float": - f64, err := strconv.ParseFloat(node.Value, 64) - if err != nil { - // Deal with infinity prefixes. - v := strings.ToLower(node.Value) - switch { - case strings.HasPrefix(v, "+"): - v = strings.TrimPrefix(v, "+") - f64 = math.Inf(1) - case strings.HasPrefix(v, "-"): - v = strings.TrimPrefix(v, "-") - f64 = math.Inf(-1) - default: - // No prefix. - f64 = math.Inf(1) - } - - // Deal with infinity and NaN values. - switch v { - case ".inf": - return dyn.NewValue(f64, []diag.Location{loc}), nil - case ".nan": - return dyn.NewValue(math.NaN(), []diag.Location{loc}), nil - } - - return dyn.InvalidValue, errorf(loc, "invalid float value: %v", node.Value) - } - return dyn.NewValue(f64, []diag.Location{loc}), nil - case "!!null": - return dyn.NewValue(nil, []diag.Location{loc}), nil - case "!!timestamp": - t, err := dyn.NewTime(node.Value) - if err == nil { - return dyn.NewValue(t, []diag.Location{loc}), nil - } - return dyn.InvalidValue, errorf(loc, "invalid timestamp value: %v", node.Value) - default: - return dyn.InvalidValue, errorf(loc, "unknown tag: %v", st) - } -} - -func (d *loader) loadAlias(node *yaml.Node, loc diag.Location) (dyn.Value, error) { - if d.activeAliases[node] { - return dyn.InvalidValue, errorf(loc, "cyclic reference to anchor %q", node.Value) - } - - // The same alias node may be reached again through another alias to an - // enclosing anchor, which is not a cycle. - d.activeAliases[node] = true - defer delete(d.activeAliases, node) - - return d.load(node.Alias) -} diff --git a/libs/dyn/yamlloader/testdata/anchor_01.yml b/libs/dyn/yamlloader/testdata/anchor_01.yml deleted file mode 100644 index b677d60b808..00000000000 --- a/libs/dyn/yamlloader/testdata/anchor_01.yml +++ /dev/null @@ -1,12 +0,0 @@ -# 1. Basic Anchor and Alias -defaults: &DEFAULTS - color: red - size: large - -shirt1: - <<: *DEFAULTS - pattern: striped - -shirt2: - <<: *DEFAULTS - pattern: plain diff --git a/libs/dyn/yamlloader/testdata/anchor_02.yml b/libs/dyn/yamlloader/testdata/anchor_02.yml deleted file mode 100644 index 86650987f3d..00000000000 --- a/libs/dyn/yamlloader/testdata/anchor_02.yml +++ /dev/null @@ -1,13 +0,0 @@ -# 2. Merging Anchors -# Here, multiple anchors can be merged into a single item. -defaults: &DEFAULTS - color: red - size: large - -materials: &MATERIALS - primary: cotton - secondary: polyester - -shirt: - <<: [*DEFAULTS, *MATERIALS] - pattern: striped diff --git a/libs/dyn/yamlloader/testdata/anchor_03.yml b/libs/dyn/yamlloader/testdata/anchor_03.yml deleted file mode 100644 index f06b7999ef0..00000000000 --- a/libs/dyn/yamlloader/testdata/anchor_03.yml +++ /dev/null @@ -1,10 +0,0 @@ -# 3. Overriding Merged Anchor Values -# You can override values when merging. -defaults: &DEFAULTS - color: red - size: large - pattern: plain - -shirt: - <<: *DEFAULTS - color: blue diff --git a/libs/dyn/yamlloader/testdata/anchor_04.yml b/libs/dyn/yamlloader/testdata/anchor_04.yml deleted file mode 100644 index bbc4bc02707..00000000000 --- a/libs/dyn/yamlloader/testdata/anchor_04.yml +++ /dev/null @@ -1,16 +0,0 @@ -# 4. Nested Anchors -# This demonstrates the reuse of nested content. -address: &ADDRESS - city: San Francisco - state: CA - country: USA - -person1: - name: Alice - address: *ADDRESS - -person2: - name: Bob - address: - <<: *ADDRESS - city: Los Angeles diff --git a/libs/dyn/yamlloader/testdata/anchor_05.yml b/libs/dyn/yamlloader/testdata/anchor_05.yml deleted file mode 100644 index 9a4c961b7c8..00000000000 --- a/libs/dyn/yamlloader/testdata/anchor_05.yml +++ /dev/null @@ -1,15 +0,0 @@ -# 5. Using Anchors for List Items -# You can use anchors for list items too. -features: &FEATURES - - wifi - - bluetooth - -phone1: - name: PhoneA - features: *FEATURES - -phone2: - name: PhoneB - features: - - camera - - *FEATURES diff --git a/libs/dyn/yamlloader/testdata/anchor_06.yml b/libs/dyn/yamlloader/testdata/anchor_06.yml deleted file mode 100644 index f27650fad47..00000000000 --- a/libs/dyn/yamlloader/testdata/anchor_06.yml +++ /dev/null @@ -1,5 +0,0 @@ -# 6. String Anchors -commonMessage: &msg "Hello, World!" - -greeting1: *msg -greeting2: *msg diff --git a/libs/dyn/yamlloader/testdata/anchor_07.yml b/libs/dyn/yamlloader/testdata/anchor_07.yml deleted file mode 100644 index b8f32214568..00000000000 --- a/libs/dyn/yamlloader/testdata/anchor_07.yml +++ /dev/null @@ -1,10 +0,0 @@ -# 7. Number Anchors -defaultAge: &age 25 - -person1: - name: Alice - age: *age - -person2: - name: Bob - age: *age diff --git a/libs/dyn/yamlloader/testdata/anchor_08.yml b/libs/dyn/yamlloader/testdata/anchor_08.yml deleted file mode 100644 index 8b01e21f14b..00000000000 --- a/libs/dyn/yamlloader/testdata/anchor_08.yml +++ /dev/null @@ -1,10 +0,0 @@ -# 8. Boolean Anchors -isActive: &active true - -user1: - username: user1 - active: *active - -user2: - username: user2 - active: *active diff --git a/libs/dyn/yamlloader/testdata/anchor_09.yml b/libs/dyn/yamlloader/testdata/anchor_09.yml deleted file mode 100644 index 8005bf445ef..00000000000 --- a/libs/dyn/yamlloader/testdata/anchor_09.yml +++ /dev/null @@ -1,2 +0,0 @@ -anchortest: &x - child: *x diff --git a/libs/dyn/yamlloader/testdata/anchor_10.yml b/libs/dyn/yamlloader/testdata/anchor_10.yml deleted file mode 100644 index 013832ab3ac..00000000000 --- a/libs/dyn/yamlloader/testdata/anchor_10.yml +++ /dev/null @@ -1,5 +0,0 @@ -leaf: &leaf 1 -shared: &shared - value: *leaf -use1: *shared -use2: *shared diff --git a/libs/dyn/yamlloader/testdata/empty.yml b/libs/dyn/yamlloader/testdata/empty.yml deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/libs/dyn/yamlloader/testdata/error_01.yml b/libs/dyn/yamlloader/testdata/error_01.yml deleted file mode 100644 index 11b96486e32..00000000000 --- a/libs/dyn/yamlloader/testdata/error_01.yml +++ /dev/null @@ -1,6 +0,0 @@ -# Use string anchor to extend a mapping. -str: &str "Hello world!" - -map: - <<: *str - key: value diff --git a/libs/dyn/yamlloader/testdata/error_02.yml b/libs/dyn/yamlloader/testdata/error_02.yml deleted file mode 100644 index 5e9c6276d09..00000000000 --- a/libs/dyn/yamlloader/testdata/error_02.yml +++ /dev/null @@ -1,6 +0,0 @@ -# Use string anchor inside sequence to extend a mapping. -str: &str "Hello world!" - -map: - <<: [*str] - key: value diff --git a/libs/dyn/yamlloader/testdata/error_03.yml b/libs/dyn/yamlloader/testdata/error_03.yml deleted file mode 100644 index c4d3c397dd9..00000000000 --- a/libs/dyn/yamlloader/testdata/error_03.yml +++ /dev/null @@ -1,4 +0,0 @@ -# Extend a mapping with a literal string. -map: - <<: value - key: value diff --git a/libs/dyn/yamlloader/testdata/mix_01.yml b/libs/dyn/yamlloader/testdata/mix_01.yml deleted file mode 100644 index 98706bf6f01..00000000000 --- a/libs/dyn/yamlloader/testdata/mix_01.yml +++ /dev/null @@ -1,90 +0,0 @@ -# Scalars - -# Strings can be unquoted -name: John Doe - -# Strings can be single quoted -single_quoted_string: 'This is a single quoted string' - -# Strings can be double quoted (allows for escape sequences) -double_quoted_string: "This is a double quoted string with an escaped newline \n and tab \t." - -# Multiline string with folded style (newlines become spaces) -folded_style: > - This is a very - long string that - spans several lines. - -# Multiline string with literal block style (newlines are preserved) -literal_style: | - Line 1 - Line 2 - Line 3 - -# Integers -age: 30 - -# Floating point -price: 19.99 - -# Boolean values -is_student: true -is_employed: false - -# Null value -middle_name: ~ # null can also be represented with 'null' - -# Timestamp -timestamp: 2023-01-01T12:00:00Z - -# Sequences (lists) - -# Inline list -fruits: [apple, banana, cherry] - -# Block style list -books: - - 'Moby Dick' - - '1984' - - 'Pride and Prejudice' - -# Mappings (dictionaries) - -# Inline dictionary -address: {street: '123 Main St', city: 'Anytown', zip: '12345'} - -# Block style dictionary -employee: - first_name: Jane - last_name: Smith - age: 40 - -# Edge cases and advanced features - -# Strings that look like other types must be quoted -looks_like_number: '12345' -looks_like_boolean: 'yes' -looks_like_null: 'null' -looks_like_timestamp: '2023-01-01T12:00:00Z' - -# Using anchors and aliases to reuse properties -base_address: &base - street: '456 Elm St' - city: 'Sometown' - -office_address: - <<: *base # Merge the base address into this mapping - suite: 500 - -# Nested structures -users: - - name: Alice - age: 28 - interests: - - reading - - cycling - - name: Bob - age: 35 - interests: - - cooking - - hiking diff --git a/libs/dyn/yamlloader/testdata/mix_02.yml b/libs/dyn/yamlloader/testdata/mix_02.yml deleted file mode 100644 index 757d626dbfe..00000000000 --- a/libs/dyn/yamlloader/testdata/mix_02.yml +++ /dev/null @@ -1,49 +0,0 @@ -# Scalars with special characters - -# Commas and square brackets in strings should be enclosed in quotes -special_chars: "[This, string] has, special chars." - -# Strings starting with reserved indicators must be quoted -reserved_string: "@not_a_directive" -colon_string: "this: looks like a mapping, but it's not" - -# Explicit data type declaration -explicit_string: !!str 12345 - -# Sequences with nested mappings and lists - -teams: - - - name: Team A - members: - - Alice - - Bob - tasks: - - task1: - status: incomplete - due_date: 2023-01-15 - - task2: - status: complete - -# Complex mapping keys -? | - Multi-line key - which is unusual but valid -: multi-line key's value - -"complex key: with colon": complex key's value - -# Set (a mapping with null values) -set_example: - item1: ~ - item2: ~ - -# Merge multiple mappings (with override) -base_colors: &colors - red: "#FF0000" - blue: "#0000FF" - green: "#00FF00" - -theme: - <<: *colors - blue: "#001122" # Overriding the blue color from base_colors diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.1.yml b/libs/dyn/yamlloader/testdata/spec_example_2.1.yml deleted file mode 100644 index c9e26274c09..00000000000 --- a/libs/dyn/yamlloader/testdata/spec_example_2.1.yml +++ /dev/null @@ -1,5 +0,0 @@ -# Example 2.1 Sequence of Scalars (ball players) - -- Mark McGwire -- Sammy Sosa -- Ken Griffey diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.10.yml b/libs/dyn/yamlloader/testdata/spec_example_2.10.yml deleted file mode 100644 index a3459dedfc8..00000000000 --- a/libs/dyn/yamlloader/testdata/spec_example_2.10.yml +++ /dev/null @@ -1,10 +0,0 @@ -# Example 2.10 Node for “Sammy Sosa” appears twice in this document - ---- -hr: -- Mark McGwire -# Following node labeled SS -- &SS Sammy Sosa -rbi: -- *SS # Subsequent occurrence -- Ken Griffey diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.11.yml b/libs/dyn/yamlloader/testdata/spec_example_2.11.yml deleted file mode 100644 index e3c5c115e02..00000000000 --- a/libs/dyn/yamlloader/testdata/spec_example_2.11.yml +++ /dev/null @@ -1,10 +0,0 @@ -# Example 2.11 Mapping between Sequences - -? - Detroit Tigers - - Chicago cubs -: - 2001-07-23 - -? [ New York Yankees, - Atlanta Braves ] -: [ 2001-07-02, 2001-08-12, - 2001-08-14 ] diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.12.yml b/libs/dyn/yamlloader/testdata/spec_example_2.12.yml deleted file mode 100644 index eb4a526f65b..00000000000 --- a/libs/dyn/yamlloader/testdata/spec_example_2.12.yml +++ /dev/null @@ -1,10 +0,0 @@ -# Example 2.12 Compact Nested Mapping - ---- -# Products purchased -- item : Super Hoop - quantity: 1 -- item : Basketball - quantity: 4 -- item : Big Shoes - quantity: 1 diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.13.yml b/libs/dyn/yamlloader/testdata/spec_example_2.13.yml deleted file mode 100644 index e55abff107e..00000000000 --- a/libs/dyn/yamlloader/testdata/spec_example_2.13.yml +++ /dev/null @@ -1,6 +0,0 @@ -# Example 2.13 In literals, newlines are preserved - -# ASCII Art ---- | - \//||\/|| - // || ||__ diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.14.yml b/libs/dyn/yamlloader/testdata/spec_example_2.14.yml deleted file mode 100644 index 439fca30cb1..00000000000 --- a/libs/dyn/yamlloader/testdata/spec_example_2.14.yml +++ /dev/null @@ -1,6 +0,0 @@ -# Example 2.14 In the folded scalars, newlines become spaces - ---- > - Mark McGwire's - year was crippled - by a knee injury. diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.15.yml b/libs/dyn/yamlloader/testdata/spec_example_2.15.yml deleted file mode 100644 index 266e7ce4711..00000000000 --- a/libs/dyn/yamlloader/testdata/spec_example_2.15.yml +++ /dev/null @@ -1,10 +0,0 @@ -# Example 2.15 Folded newlines are preserved for “more indented” and blank lines - ---- > - Sammy Sosa completed another - fine season with great stats. - - 63 Home Runs - 0.288 Batting Average - - What a year! diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.16.yml b/libs/dyn/yamlloader/testdata/spec_example_2.16.yml deleted file mode 100644 index 6db6b0878f8..00000000000 --- a/libs/dyn/yamlloader/testdata/spec_example_2.16.yml +++ /dev/null @@ -1,9 +0,0 @@ -# Example 2.16 Indentation determines scope - -name: Mark McGwire -accomplishment: > - Mark set a major league - home run record in 1998. -stats: | - 65 Home Runs - 0.278 Batting Average diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.17.yml b/libs/dyn/yamlloader/testdata/spec_example_2.17.yml deleted file mode 100644 index af0777abab5..00000000000 --- a/libs/dyn/yamlloader/testdata/spec_example_2.17.yml +++ /dev/null @@ -1,9 +0,0 @@ -# Example 2.17 Quoted Scalars - -unicode: "Sosa did fine.\u263A" -control: "\b1998\t1999\t2000\n" -hex esc: "\x0d\x0a is \r\n" - -single: '"Howdy!" he cried.' -quoted: ' # Not a ''comment''.' -tie-fighter: '|\-*-/|' diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.18.yml b/libs/dyn/yamlloader/testdata/spec_example_2.18.yml deleted file mode 100644 index 741bcd8c0c0..00000000000 --- a/libs/dyn/yamlloader/testdata/spec_example_2.18.yml +++ /dev/null @@ -1,8 +0,0 @@ -# Example 2.18 Multi-line Flow Scalars - -plain: - This unquoted scalar - spans many lines. - -quoted: "So does this - quoted scalar.\n" diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.19.yml b/libs/dyn/yamlloader/testdata/spec_example_2.19.yml deleted file mode 100644 index 6ed95e092aa..00000000000 --- a/libs/dyn/yamlloader/testdata/spec_example_2.19.yml +++ /dev/null @@ -1,15 +0,0 @@ -# Example 2.19 Integers - -canonical: 12345 -decimal: +12345 -octal: 0o14 -hexadecimal: 0xC - -# Note: this example is not part of the spec but added for completeness. -# -# Octal numbers: -# - YAML 1.1: prefix is "0" -# - YAML 1.2: prefix is "0o" -# The "gopkg.in/yaml.v3" package accepts both for backwards compat. -# We accept only the YAML 1.2 prefix "0o". -octal11: 012345 diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.2.yml b/libs/dyn/yamlloader/testdata/spec_example_2.2.yml deleted file mode 100644 index 29c16105e3b..00000000000 --- a/libs/dyn/yamlloader/testdata/spec_example_2.2.yml +++ /dev/null @@ -1,5 +0,0 @@ -# Example 2.2 Mapping Scalars to Scalars (player statistics) - -hr: 65 # Home runs -avg: 0.278 # Batting average -rbi: 147 # Runs Batted In diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.20.yml b/libs/dyn/yamlloader/testdata/spec_example_2.20.yml deleted file mode 100644 index 77a79a0c3e0..00000000000 --- a/libs/dyn/yamlloader/testdata/spec_example_2.20.yml +++ /dev/null @@ -1,7 +0,0 @@ -# Example 2.20 Floating Point - -canonical: 1.23015e+3 -exponential: 12.3015e+02 -fixed: 1230.15 -negative infinity: -.inf -not a number: .nan diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.21.yml b/libs/dyn/yamlloader/testdata/spec_example_2.21.yml deleted file mode 100644 index cdb423c566e..00000000000 --- a/libs/dyn/yamlloader/testdata/spec_example_2.21.yml +++ /dev/null @@ -1,5 +0,0 @@ -# Example 2.21 Miscellaneous - -null: -booleans: [ true, false ] -string: '012345' diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.22.yml b/libs/dyn/yamlloader/testdata/spec_example_2.22.yml deleted file mode 100644 index bef2addfa07..00000000000 --- a/libs/dyn/yamlloader/testdata/spec_example_2.22.yml +++ /dev/null @@ -1,6 +0,0 @@ -# Example 2.22 Timestamps - -canonical: 2001-12-15T02:59:43.1Z -iso8601: 2001-12-14t21:59:43.10-05:00 -spaced: 2001-12-14 21:59:43.10 -5 -date: 2002-12-14 diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.23.yml b/libs/dyn/yamlloader/testdata/spec_example_2.23.yml deleted file mode 100644 index 56e9898e21b..00000000000 --- a/libs/dyn/yamlloader/testdata/spec_example_2.23.yml +++ /dev/null @@ -1,15 +0,0 @@ -# Example 2.23 Various Explicit Tags - ---- -not-date: !!str 2002-04-28 - -picture: !!binary | - R0lGODlhDAAMAIQAAP//9/X - 17unp5WZmZgAAAOfn515eXv - Pz7Y6OjuDg4J+fn5OTk6enp - 56enmleECcgggoBADs= - -application specific tag: !something | - The semantics of the tag - above may be different for - different documents. diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.24.yml b/libs/dyn/yamlloader/testdata/spec_example_2.24.yml deleted file mode 100644 index f7c11d0fb11..00000000000 --- a/libs/dyn/yamlloader/testdata/spec_example_2.24.yml +++ /dev/null @@ -1,16 +0,0 @@ -# Example 2.24 Global Tags - -%TAG ! tag:clarkevans.com,2002: ---- !shape - # Use the ! handle for presenting - # tag:clarkevans.com,2002:circle -- !circle - center: &ORIGIN {x: 73, y: 129} - radius: 7 -- !line - start: *ORIGIN - finish: { x: 89, y: 102 } -- !label - start: *ORIGIN - color: 0xFFEEBB - text: Pretty vector drawing. diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.25.yml b/libs/dyn/yamlloader/testdata/spec_example_2.25.yml deleted file mode 100644 index 73bac862755..00000000000 --- a/libs/dyn/yamlloader/testdata/spec_example_2.25.yml +++ /dev/null @@ -1,9 +0,0 @@ -# Example 2.25 Unordered Sets - -# Sets are represented as a -# Mapping where each key is -# associated with a null value ---- !!set -? Mark McGwire -? Sammy Sosa -? Ken Griffey diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.26.yml b/libs/dyn/yamlloader/testdata/spec_example_2.26.yml deleted file mode 100644 index 00863a6b854..00000000000 --- a/libs/dyn/yamlloader/testdata/spec_example_2.26.yml +++ /dev/null @@ -1,9 +0,0 @@ -# Example 2.26 Ordered Mappings - -# Ordered maps are represented as -# A sequence of mappings, with -# each mapping having one key ---- !!omap -- Mark McGwire: 65 -- Sammy Sosa: 63 -- Ken Griffey: 58 diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.27.yml b/libs/dyn/yamlloader/testdata/spec_example_2.27.yml deleted file mode 100644 index fc9b460c299..00000000000 --- a/libs/dyn/yamlloader/testdata/spec_example_2.27.yml +++ /dev/null @@ -1,31 +0,0 @@ -# Example 2.27 Invoice - ---- ! -invoice: 34843 -date : 2001-01-23 -bill-to: &id001 - given : Chris - family : Dumars - address: - lines: | - 458 Walkman Dr. - Suite #292 - city : Royal Oak - state : MI - postal : 48046 -ship-to: *id001 -product: -- sku : BL394D - quantity : 4 - description : Basketball - price : 450.00 -- sku : BL4438H - quantity : 1 - description : Super Hoop - price : 2392.00 -tax : 251.42 -total: 4443.52 -comments: - Late afternoon is best. - Backup contact is Nancy - Billsmer @ 338-4338. diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.28.yml b/libs/dyn/yamlloader/testdata/spec_example_2.28.yml deleted file mode 100644 index 35369472967..00000000000 --- a/libs/dyn/yamlloader/testdata/spec_example_2.28.yml +++ /dev/null @@ -1,28 +0,0 @@ -# Example 2.28 Log File - ---- -Time: 2001-11-23 15:01:42 -5 -User: ed -Warning: - This is an error message - for the log file ---- -Time: 2001-11-23 15:02:31 -5 -User: ed -Warning: - A slightly different error - message. ---- -Date: 2001-11-23 15:03:17 -5 -User: ed -Fatal: - Unknown variable "bar" -Stack: -- file: TopClass.py - line: 23 - code: | - x = MoreObject("345\n") -- file: MoreClass.py - line: 58 - code: |- - foo = bar diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.3.yml b/libs/dyn/yamlloader/testdata/spec_example_2.3.yml deleted file mode 100644 index 70cbe07d391..00000000000 --- a/libs/dyn/yamlloader/testdata/spec_example_2.3.yml +++ /dev/null @@ -1,10 +0,0 @@ -# Example 2.3 Mapping Scalars to Sequences (ball clubs in each league) - -american: -- Boston Red Sox -- Detroit Tigers -- New York Yankees -national: -- New York Mets -- Chicago Cubs -- Atlanta Braves diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.4.yml b/libs/dyn/yamlloader/testdata/spec_example_2.4.yml deleted file mode 100644 index cce28625f56..00000000000 --- a/libs/dyn/yamlloader/testdata/spec_example_2.4.yml +++ /dev/null @@ -1,10 +0,0 @@ -# Example 2.4 Sequence of Mappings (players’ statistics) - -- - name: Mark McGwire - hr: 65 - avg: 0.278 -- - name: Sammy Sosa - hr: 63 - avg: 0.288 diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.5.yml b/libs/dyn/yamlloader/testdata/spec_example_2.5.yml deleted file mode 100644 index a585faee790..00000000000 --- a/libs/dyn/yamlloader/testdata/spec_example_2.5.yml +++ /dev/null @@ -1,5 +0,0 @@ -# Example 2.5 Sequence of Sequences - -- [name , hr, avg ] -- [Mark McGwire, 65, 0.278] -- [Sammy Sosa , 63, 0.288] diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.6.yml b/libs/dyn/yamlloader/testdata/spec_example_2.6.yml deleted file mode 100644 index cc137e5d56d..00000000000 --- a/libs/dyn/yamlloader/testdata/spec_example_2.6.yml +++ /dev/null @@ -1,7 +0,0 @@ -# Example 2.6 Mapping of Mappings - -Mark McGwire: {hr: 65, avg: 0.278} -Sammy Sosa: { - hr: 63, - avg: 0.288, - } diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.7.yml b/libs/dyn/yamlloader/testdata/spec_example_2.7.yml deleted file mode 100644 index 35c2541d6cc..00000000000 --- a/libs/dyn/yamlloader/testdata/spec_example_2.7.yml +++ /dev/null @@ -1,12 +0,0 @@ -# Example 2.7 Two Documents in a Stream (each with a leading comment) - -# Ranking of 1998 home runs ---- -- Mark McGwire -- Sammy Sosa -- Ken Griffey - -# Team ranking ---- -- Chicago Cubs -- St Louis Cardinals diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.8.yml b/libs/dyn/yamlloader/testdata/spec_example_2.8.yml deleted file mode 100644 index ae6e8bf279f..00000000000 --- a/libs/dyn/yamlloader/testdata/spec_example_2.8.yml +++ /dev/null @@ -1,12 +0,0 @@ -# Example 2.8 Play by Play Feed from a Game - ---- -time: 20:03:20 -player: Sammy Sosa -action: strike (miss) -... ---- -time: 20:03:47 -player: Sammy Sosa -action: grand slam -... diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.9.yml b/libs/dyn/yamlloader/testdata/spec_example_2.9.yml deleted file mode 100644 index 75217b25ec9..00000000000 --- a/libs/dyn/yamlloader/testdata/spec_example_2.9.yml +++ /dev/null @@ -1,10 +0,0 @@ -# Example 2.9 Single Document with Two Comments - ---- -hr: # 1998 hr ranking -- Mark McGwire -- Sammy Sosa -# 1998 rbi ranking -rbi: -- Sammy Sosa -- Ken Griffey diff --git a/libs/dyn/yamlloader/yaml.go b/libs/dyn/yamlloader/yaml.go deleted file mode 100644 index 1225fcc7e2f..00000000000 --- a/libs/dyn/yamlloader/yaml.go +++ /dev/null @@ -1,22 +0,0 @@ -package yamlloader - -import ( - "io" - - "github.com/databricks/cli/libs/dyn" - "go.yaml.in/yaml/v3" -) - -func LoadYAML(path string, r io.Reader) (dyn.Value, error) { - var node yaml.Node - dec := yaml.NewDecoder(r) - err := dec.Decode(&node) - if err != nil { - if err == io.EOF { - return dyn.NilValue, nil - } - return dyn.InvalidValue, err - } - - return newLoader(path).load(&node) -} diff --git a/libs/dyn/yamlloader/yaml_anchor_test.go b/libs/dyn/yamlloader/yaml_anchor_test.go deleted file mode 100644 index 00f4576a3e2..00000000000 --- a/libs/dyn/yamlloader/yaml_anchor_test.go +++ /dev/null @@ -1,140 +0,0 @@ -package yamlloader_test - -import ( - "bytes" - "os" - "testing" - - "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/yamlloader" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestYAMLAnchor01(t *testing.T) { - file := "testdata/anchor_01.yml" - self := loadYAML(t, file) - assert.NotEqual(t, dyn.NilValue, self) - - assert.True(t, self.Get("defaults").IsAnchor()) - assert.False(t, self.Get("shirt1").IsAnchor()) - assert.False(t, self.Get("shirt2").IsAnchor()) - - pattern := self.Get("shirt1").Get("pattern") - assert.Equal(t, "striped", pattern.AsAny()) - assert.Equal(t, diag.Location{File: file, Line: 8, Column: 12}, pattern.Location()) -} - -func TestYAMLAnchor02(t *testing.T) { - file := "testdata/anchor_02.yml" - self := loadYAML(t, file) - assert.NotEqual(t, dyn.NilValue, self) - - color := self.Get("shirt").Get("color") - assert.Equal(t, "red", color.AsAny()) - assert.Equal(t, diag.Location{File: file, Line: 4, Column: 10}, color.Location()) - - primary := self.Get("shirt").Get("primary") - assert.Equal(t, "cotton", primary.AsAny()) - assert.Equal(t, diag.Location{File: file, Line: 8, Column: 12}, primary.Location()) - - pattern := self.Get("shirt").Get("pattern") - assert.Equal(t, "striped", pattern.AsAny()) - assert.Equal(t, diag.Location{File: file, Line: 13, Column: 12}, pattern.Location()) -} - -func TestYAMLAnchor03(t *testing.T) { - file := "testdata/anchor_03.yml" - self := loadYAML(t, file) - assert.NotEqual(t, dyn.NilValue, self) - - // Assert the override took place. - blue := self.Get("shirt").Get("color") - assert.Equal(t, "blue", blue.AsAny()) - assert.Equal(t, file, blue.Location().File) - assert.Equal(t, 10, blue.Location().Line) - assert.Equal(t, 10, blue.Location().Column) -} - -func TestYAMLAnchor04(t *testing.T) { - file := "testdata/anchor_04.yml" - self := loadYAML(t, file) - assert.NotEqual(t, dyn.NilValue, self) - - p1 := self.Get("person1").Get("address").Get("city") - assert.Equal(t, "San Francisco", p1.AsAny()) - assert.Equal(t, diag.Location{File: file, Line: 4, Column: 9}, p1.Location()) - - p2 := self.Get("person2").Get("address").Get("city") - assert.Equal(t, "Los Angeles", p2.AsAny()) - assert.Equal(t, diag.Location{File: file, Line: 16, Column: 11}, p2.Location()) -} - -func TestYAMLAnchor05(t *testing.T) { - file := "testdata/anchor_05.yml" - self := loadYAML(t, file) - assert.NotEqual(t, dyn.NilValue, self) - - features := self.Get("phone1").Get("features") - assert.Equal(t, "wifi", features.Index(0).AsAny()) - assert.Equal(t, diag.Location{File: file, Line: 4, Column: 5}, features.Index(0).Location()) - assert.Equal(t, "bluetooth", features.Index(1).AsAny()) - assert.Equal(t, diag.Location{File: file, Line: 5, Column: 5}, features.Index(1).Location()) -} - -func TestYAMLAnchor06(t *testing.T) { - file := "testdata/anchor_06.yml" - self := loadYAML(t, file) - assert.NotEqual(t, dyn.NilValue, self) - - greeting := self.Get("greeting1") - assert.Equal(t, "Hello, World!", greeting.AsAny()) - assert.Equal(t, diag.Location{File: file, Line: 2, Column: 16}, greeting.Location()) -} - -func TestYAMLAnchor07(t *testing.T) { - file := "testdata/anchor_07.yml" - self := loadYAML(t, file) - assert.NotEqual(t, dyn.NilValue, self) - - name := self.Get("person1").Get("name") - assert.Equal(t, "Alice", name.AsAny()) - assert.Equal(t, diag.Location{File: file, Line: 5, Column: 9}, name.Location()) - - age := self.Get("person1").Get("age") - assert.Equal(t, 25, age.AsAny()) - assert.Equal(t, diag.Location{File: file, Line: 2, Column: 13}, age.Location()) -} - -func TestYAMLAnchor08(t *testing.T) { - file := "testdata/anchor_08.yml" - self := loadYAML(t, file) - assert.NotEqual(t, dyn.NilValue, self) - - username := self.Get("user1").Get("username") - assert.Equal(t, "user1", username.AsAny()) - assert.Equal(t, diag.Location{File: file, Line: 5, Column: 13}, username.Location()) - - active := self.Get("user1").Get("active") - assert.Equal(t, true, active.AsAny()) - assert.Equal(t, diag.Location{File: file, Line: 2, Column: 11}, active.Location()) -} - -func TestYAMLAnchor09(t *testing.T) { - file := "testdata/anchor_09.yml" - input, err := os.ReadFile(file) - require.NoError(t, err) - - _, err = yamlloader.LoadYAML(file, bytes.NewBuffer(input)) - assert.ErrorContains(t, err, `cyclic reference to anchor "x"`) -} - -func TestYAMLAnchor10(t *testing.T) { - file := "testdata/anchor_10.yml" - self := loadYAML(t, file) - assert.NotEqual(t, dyn.NilValue, self) - - assert.Equal(t, 1, self.Get("use1").Get("value").AsAny()) - assert.Equal(t, 1, self.Get("use2").Get("value").AsAny()) -} diff --git a/libs/dyn/yamlloader/yaml_error_test.go b/libs/dyn/yamlloader/yaml_error_test.go deleted file mode 100644 index a18f9599b05..00000000000 --- a/libs/dyn/yamlloader/yaml_error_test.go +++ /dev/null @@ -1,36 +0,0 @@ -package yamlloader_test - -import ( - "bytes" - "os" - "testing" - - "github.com/databricks/cli/libs/dyn/yamlloader" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "go.yaml.in/yaml/v3" -) - -func TestYAMLErrorMapMerge(t *testing.T) { - for _, file := range []string{ - "testdata/error_01.yml", - "testdata/error_02.yml", - "testdata/error_03.yml", - } { - input, err := os.ReadFile(file) - require.NoError(t, err) - - t.Run(file, func(t *testing.T) { - t.Run("reference", func(t *testing.T) { - var ref any - err = yaml.Unmarshal(input, &ref) - assert.ErrorContains(t, err, "map merge requires map or sequence of maps as the value") - }) - - t.Run("self", func(t *testing.T) { - _, err := yamlloader.LoadYAML(file, bytes.NewBuffer(input)) - assert.ErrorContains(t, err, "map merge requires map or sequence of maps as the value") - }) - }) - } -} diff --git a/libs/dyn/yamlloader/yaml_mix_test.go b/libs/dyn/yamlloader/yaml_mix_test.go deleted file mode 100644 index 307b93dbf3a..00000000000 --- a/libs/dyn/yamlloader/yaml_mix_test.go +++ /dev/null @@ -1,26 +0,0 @@ -package yamlloader_test - -import ( - "testing" - - "github.com/databricks/cli/libs/dyn" - "github.com/stretchr/testify/assert" -) - -func TestYAMLMix01(t *testing.T) { - file := "testdata/mix_01.yml" - self := loadYAML(t, file) - assert.NotEqual(t, dyn.NilValue, self) - - assert.True(t, self.Get("base_address").IsAnchor()) - assert.False(t, self.Get("office_address").IsAnchor()) -} - -func TestYAMLMix02(t *testing.T) { - file := "testdata/mix_02.yml" - self := loadYAML(t, file) - assert.NotEqual(t, dyn.NilValue, self) - - assert.True(t, self.Get("base_colors").IsAnchor()) - assert.False(t, self.Get("theme").IsAnchor()) -} diff --git a/libs/dyn/yamlloader/yaml_spec_test.go b/libs/dyn/yamlloader/yaml_spec_test.go deleted file mode 100644 index 28ad2e7480c..00000000000 --- a/libs/dyn/yamlloader/yaml_spec_test.go +++ /dev/null @@ -1,824 +0,0 @@ -package yamlloader_test - -import ( - "bytes" - "math" - "os" - "testing" - - "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/dynassert" - "github.com/databricks/cli/libs/dyn/yamlloader" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -const NL = "\n" - -func loadExample(t *testing.T, file string) dyn.Value { - input, err := os.ReadFile(file) - require.NoError(t, err) - self, err := yamlloader.LoadYAML(file, bytes.NewBuffer(input)) - require.NoError(t, err) - return self -} - -func TestYAMLSpecExample_2_1(t *testing.T) { - file := "testdata/spec_example_2.1.yml" - self := loadExample(t, file) - - assert.Equal(t, dyn.NewValue( - []dyn.Value{ - dyn.NewValue("Mark McGwire", []diag.Location{{File: file, Line: 3, Column: 3}}), - dyn.NewValue("Sammy Sosa", []diag.Location{{File: file, Line: 4, Column: 3}}), - dyn.NewValue("Ken Griffey", []diag.Location{{File: file, Line: 5, Column: 3}}), - }, - []diag.Location{{File: file, Line: 3, Column: 1}}, - ), self) -} - -func TestYAMLSpecExample_2_2(t *testing.T) { - file := "testdata/spec_example_2.2.yml" - self := loadExample(t, file) - - dynassert.Equal(t, dyn.NewValue( - map[string]dyn.Value{ - "hr": dyn.NewValue(65, []diag.Location{{File: file, Line: 3, Column: 6}}), - "avg": dyn.NewValue(0.278, []diag.Location{{File: file, Line: 4, Column: 6}}), - "rbi": dyn.NewValue(147, []diag.Location{{File: file, Line: 5, Column: 6}}), - }, - []diag.Location{{File: file, Line: 3, Column: 1}}, - ), self) -} - -func TestYAMLSpecExample_2_3(t *testing.T) { - file := "testdata/spec_example_2.3.yml" - self := loadExample(t, file) - - dynassert.Equal(t, dyn.NewValue( - map[string]dyn.Value{ - "american": dyn.NewValue( - []dyn.Value{ - dyn.NewValue("Boston Red Sox", []diag.Location{{File: file, Line: 4, Column: 3}}), - dyn.NewValue("Detroit Tigers", []diag.Location{{File: file, Line: 5, Column: 3}}), - dyn.NewValue("New York Yankees", []diag.Location{{File: file, Line: 6, Column: 3}}), - }, - []diag.Location{{File: file, Line: 4, Column: 1}}, - ), - "national": dyn.NewValue( - []dyn.Value{ - dyn.NewValue("New York Mets", []diag.Location{{File: file, Line: 8, Column: 3}}), - dyn.NewValue("Chicago Cubs", []diag.Location{{File: file, Line: 9, Column: 3}}), - dyn.NewValue("Atlanta Braves", []diag.Location{{File: file, Line: 10, Column: 3}}), - }, - []diag.Location{{File: file, Line: 8, Column: 1}}, - ), - }, - []diag.Location{{File: file, Line: 3, Column: 1}}, - ), self) -} - -func TestYAMLSpecExample_2_4(t *testing.T) { - file := "testdata/spec_example_2.4.yml" - self := loadExample(t, file) - - dynassert.Equal(t, dyn.NewValue( - []dyn.Value{ - dyn.NewValue( - map[string]dyn.Value{ - "name": dyn.NewValue("Mark McGwire", []diag.Location{{File: file, Line: 4, Column: 9}}), - "hr": dyn.NewValue(65, []diag.Location{{File: file, Line: 5, Column: 9}}), - "avg": dyn.NewValue(0.278, []diag.Location{{File: file, Line: 6, Column: 9}}), - }, - []diag.Location{{File: file, Line: 4, Column: 3}}, - ), - dyn.NewValue( - map[string]dyn.Value{ - "name": dyn.NewValue("Sammy Sosa", []diag.Location{{File: file, Line: 8, Column: 9}}), - "hr": dyn.NewValue(63, []diag.Location{{File: file, Line: 9, Column: 9}}), - "avg": dyn.NewValue(0.288, []diag.Location{{File: file, Line: 10, Column: 9}}), - }, - []diag.Location{{File: file, Line: 8, Column: 3}}, - ), - }, - []diag.Location{{File: file, Line: 3, Column: 1}}, - ), self) -} - -func TestYAMLSpecExample_2_5(t *testing.T) { - file := "testdata/spec_example_2.5.yml" - self := loadExample(t, file) - - dynassert.Equal(t, dyn.NewValue( - []dyn.Value{ - dyn.NewValue( - []dyn.Value{ - dyn.NewValue("name", []diag.Location{{File: file, Line: 3, Column: 4}}), - dyn.NewValue("hr", []diag.Location{{File: file, Line: 3, Column: 18}}), - dyn.NewValue("avg", []diag.Location{{File: file, Line: 3, Column: 22}}), - }, - []diag.Location{{File: file, Line: 3, Column: 3}}, - ), - dyn.NewValue( - []dyn.Value{ - dyn.NewValue("Mark McGwire", []diag.Location{{File: file, Line: 4, Column: 4}}), - dyn.NewValue(65, []diag.Location{{File: file, Line: 4, Column: 18}}), - dyn.NewValue(0.278, []diag.Location{{File: file, Line: 4, Column: 22}}), - }, - []diag.Location{{File: file, Line: 4, Column: 3}}, - ), - dyn.NewValue( - []dyn.Value{ - dyn.NewValue("Sammy Sosa", []diag.Location{{File: file, Line: 5, Column: 4}}), - dyn.NewValue(63, []diag.Location{{File: file, Line: 5, Column: 18}}), - dyn.NewValue(0.288, []diag.Location{{File: file, Line: 5, Column: 22}}), - }, - []diag.Location{{File: file, Line: 5, Column: 3}}, - ), - }, - []diag.Location{{File: file, Line: 3, Column: 1}}, - ), self) -} - -func TestYAMLSpecExample_2_6(t *testing.T) { - file := "testdata/spec_example_2.6.yml" - self := loadExample(t, file) - - dynassert.Equal(t, dyn.NewValue( - map[string]dyn.Value{ - "Mark McGwire": dyn.NewValue( - map[string]dyn.Value{ - "hr": dyn.NewValue(65, []diag.Location{{File: file, Line: 3, Column: 20}}), - "avg": dyn.NewValue(0.278, []diag.Location{{File: file, Line: 3, Column: 29}}), - }, - []diag.Location{{File: file, Line: 3, Column: 15}}, - ), - "Sammy Sosa": dyn.NewValue( - map[string]dyn.Value{ - "hr": dyn.NewValue(63, []diag.Location{{File: file, Line: 5, Column: 9}}), - "avg": dyn.NewValue(0.288, []diag.Location{{File: file, Line: 6, Column: 10}}), - }, - []diag.Location{{File: file, Line: 4, Column: 13}}, - ), - }, - []diag.Location{{File: file, Line: 3, Column: 1}}, - ), self) -} - -func TestYAMLSpecExample_2_7(t *testing.T) { - file := "testdata/spec_example_2.7.yml" - self := loadExample(t, file) - - // Note: we do not support multiple documents in a single YAML file. - - dynassert.Equal(t, dyn.NewValue( - []dyn.Value{ - dyn.NewValue( - "Mark McGwire", - []diag.Location{{File: file, Line: 5, Column: 3}}, - ), - dyn.NewValue( - "Sammy Sosa", - []diag.Location{{File: file, Line: 6, Column: 3}}, - ), - dyn.NewValue( - "Ken Griffey", - []diag.Location{{File: file, Line: 7, Column: 3}}, - ), - }, - []diag.Location{{File: file, Line: 5, Column: 1}}, - ), self) -} - -func TestYAMLSpecExample_2_8(t *testing.T) { - file := "testdata/spec_example_2.8.yml" - self := loadExample(t, file) - - // Note: we do not support multiple documents in a single YAML file. - - dynassert.Equal(t, dyn.NewValue( - map[string]dyn.Value{ - "time": dyn.NewValue("20:03:20", []diag.Location{{File: file, Line: 4, Column: 7}}), - "player": dyn.NewValue("Sammy Sosa", []diag.Location{{File: file, Line: 5, Column: 9}}), - "action": dyn.NewValue("strike (miss)", []diag.Location{{File: file, Line: 6, Column: 9}}), - }, - []diag.Location{{File: file, Line: 4, Column: 1}}, - ), self) -} - -func TestYAMLSpecExample_2_9(t *testing.T) { - file := "testdata/spec_example_2.9.yml" - self := loadExample(t, file) - - // Note: we do not support multiple documents in a single YAML file. - - dynassert.Equal(t, dyn.NewValue( - map[string]dyn.Value{ - "hr": dyn.NewValue( - []dyn.Value{ - dyn.NewValue("Mark McGwire", []diag.Location{{File: file, Line: 5, Column: 3}}), - dyn.NewValue("Sammy Sosa", []diag.Location{{File: file, Line: 6, Column: 3}}), - }, - []diag.Location{{File: file, Line: 5, Column: 1}}, - ), - "rbi": dyn.NewValue( - []dyn.Value{ - dyn.NewValue("Sammy Sosa", []diag.Location{{File: file, Line: 9, Column: 3}}), - dyn.NewValue("Ken Griffey", []diag.Location{{File: file, Line: 10, Column: 3}}), - }, - []diag.Location{{File: file, Line: 9, Column: 1}}, - ), - }, - []diag.Location{{File: file, Line: 4, Column: 1}}, - ), self) -} - -func TestYAMLSpecExample_2_10(t *testing.T) { - file := "testdata/spec_example_2.10.yml" - self := loadExample(t, file) - - dynassert.Equal(t, dyn.NewValue( - map[string]dyn.Value{ - "hr": dyn.NewValue( - []dyn.Value{ - dyn.NewValue("Mark McGwire", []diag.Location{{File: file, Line: 5, Column: 3}}), - dyn.NewValue("Sammy Sosa", []diag.Location{{File: file, Line: 7, Column: 3}}), - }, - []diag.Location{{File: file, Line: 5, Column: 1}}, - ), - "rbi": dyn.NewValue( - []dyn.Value{ - // The location for an anchored value refers to the anchor, not the reference. - // This is the same location as the anchor that appears in the "hr" mapping. - dyn.NewValue("Sammy Sosa", []diag.Location{{File: file, Line: 7, Column: 3}}), - dyn.NewValue("Ken Griffey", []diag.Location{{File: file, Line: 10, Column: 3}}), - }, - []diag.Location{{File: file, Line: 9, Column: 1}}, - ), - }, - []diag.Location{{File: file, Line: 4, Column: 1}}, - ), self) -} - -func TestYAMLSpecExample_2_11(t *testing.T) { - file := "testdata/spec_example_2.11.yml" - input, err := os.ReadFile(file) - require.NoError(t, err) - - // Note: non-string mapping keys are not supported by "go.yaml.in/yaml/v3". - _, err = yamlloader.LoadYAML(file, bytes.NewBuffer(input)) - assert.ErrorContains(t, err, `: key is not a scalar`) -} - -func TestYAMLSpecExample_2_12(t *testing.T) { - file := "testdata/spec_example_2.12.yml" - self := loadExample(t, file) - - dynassert.Equal(t, dyn.NewValue( - []dyn.Value{ - dyn.NewValue( - map[string]dyn.Value{ - "item": dyn.NewValue("Super Hoop", []diag.Location{{File: file, Line: 5, Column: 13}}), - "quantity": dyn.NewValue(1, []diag.Location{{File: file, Line: 6, Column: 13}}), - }, - []diag.Location{{File: file, Line: 5, Column: 3}}, - ), - dyn.NewValue( - map[string]dyn.Value{ - "item": dyn.NewValue("Basketball", []diag.Location{{File: file, Line: 7, Column: 13}}), - "quantity": dyn.NewValue(4, []diag.Location{{File: file, Line: 8, Column: 13}}), - }, - []diag.Location{{File: file, Line: 7, Column: 3}}, - ), - dyn.NewValue( - map[string]dyn.Value{ - "item": dyn.NewValue("Big Shoes", []diag.Location{{File: file, Line: 9, Column: 13}}), - "quantity": dyn.NewValue(1, []diag.Location{{File: file, Line: 10, Column: 13}}), - }, - []diag.Location{{File: file, Line: 9, Column: 3}}, - ), - }, - []diag.Location{{File: file, Line: 5, Column: 1}}, - ), self) -} - -func TestYAMLSpecExample_2_13(t *testing.T) { - file := "testdata/spec_example_2.13.yml" - self := loadExample(t, file) - - dynassert.Equal(t, dyn.NewValue( - ``+ - `\//||\/||`+NL+ - "// || ||__"+NL, - []diag.Location{{File: file, Line: 4, Column: 5}}, - ), self) -} - -func TestYAMLSpecExample_2_14(t *testing.T) { - file := "testdata/spec_example_2.14.yml" - self := loadExample(t, file) - - dynassert.Equal(t, dyn.NewValue( - `Mark McGwire's year was crippled by a knee injury.`+NL, - []diag.Location{{File: file, Line: 3, Column: 5}}, - ), self) -} - -func TestYAMLSpecExample_2_15(t *testing.T) { - file := "testdata/spec_example_2.15.yml" - self := loadExample(t, file) - - dynassert.Equal(t, dyn.NewValue( - ``+ - `Sammy Sosa completed another fine season with great stats.`+NL+ - NL+ - ` 63 Home Runs`+NL+ - ` 0.288 Batting Average`+NL+ - NL+ - `What a year!`+NL, - []diag.Location{{File: file, Line: 3, Column: 5}}, - ), self) -} - -func TestYAMLSpecExample_2_16(t *testing.T) { - file := "testdata/spec_example_2.16.yml" - self := loadExample(t, file) - - dynassert.Equal(t, dyn.NewValue( - map[string]dyn.Value{ - "name": dyn.NewValue( - "Mark McGwire", - []diag.Location{{File: file, Line: 3, Column: 7}}, - ), - "accomplishment": dyn.NewValue( - `Mark set a major league home run record in 1998.`+NL, - []diag.Location{{File: file, Line: 4, Column: 17}}, - ), - "stats": dyn.NewValue( - ``+ - `65 Home Runs`+NL+ - `0.278 Batting Average`+NL, - []diag.Location{{File: file, Line: 7, Column: 8}}, - ), - }, - []diag.Location{{File: file, Line: 3, Column: 1}}, - ), self) -} - -func TestYAMLSpecExample_2_17(t *testing.T) { - file := "testdata/spec_example_2.17.yml" - self := loadExample(t, file) - - dynassert.Equal(t, dyn.NewValue( - map[string]dyn.Value{ - "unicode": dyn.NewValue( - `Sosa did fine.`+"\u263A", - []diag.Location{{File: file, Line: 3, Column: 10}}, - ), - "control": dyn.NewValue( - "\b1998\t1999\t2000\n", - []diag.Location{{File: file, Line: 4, Column: 10}}, - ), - "hex esc": dyn.NewValue( - "\x0d\x0a is \r\n", - []diag.Location{{File: file, Line: 5, Column: 10}}, - ), - "single": dyn.NewValue( - `"Howdy!" he cried.`, - []diag.Location{{File: file, Line: 7, Column: 9}}, - ), - "quoted": dyn.NewValue( - ` # Not a 'comment'.`, - []diag.Location{{File: file, Line: 8, Column: 9}}, - ), - "tie-fighter": dyn.NewValue( - `|\-*-/|`, - []diag.Location{{File: file, Line: 9, Column: 14}}, - ), - }, - []diag.Location{{File: file, Line: 3, Column: 1}}, - ), self) -} - -func TestYAMLSpecExample_2_18(t *testing.T) { - file := "testdata/spec_example_2.18.yml" - self := loadExample(t, file) - - dynassert.Equal(t, dyn.NewValue( - map[string]dyn.Value{ - "plain": dyn.NewValue( - `This unquoted scalar spans many lines.`, - []diag.Location{{File: file, Line: 4, Column: 3}}, - ), - "quoted": dyn.NewValue( - `So does this quoted scalar.`+NL, - []diag.Location{{File: file, Line: 7, Column: 9}}, - ), - }, - []diag.Location{{File: file, Line: 3, Column: 1}}, - ), self) -} - -func TestYAMLSpecExample_2_19(t *testing.T) { - file := "testdata/spec_example_2.19.yml" - self := loadExample(t, file) - - dynassert.Equal(t, dyn.NewValue( - map[string]dyn.Value{ - "canonical": dyn.NewValue( - 12345, - []diag.Location{{File: file, Line: 3, Column: 12}}, - ), - "decimal": dyn.NewValue( - 12345, - []diag.Location{{File: file, Line: 4, Column: 10}}, - ), - "octal": dyn.NewValue( - 12, - []diag.Location{{File: file, Line: 5, Column: 8}}, - ), - "hexadecimal": dyn.NewValue( - 12, - []diag.Location{{File: file, Line: 6, Column: 14}}, - ), - "octal11": dyn.NewValue( - 12345, - []diag.Location{{File: file, Line: 15, Column: 10}}, - ), - }, - []diag.Location{{File: file, Line: 3, Column: 1}}, - ), self) -} - -func TestYAMLSpecExample_2_20(t *testing.T) { - file := "testdata/spec_example_2.20.yml" - self := loadExample(t, file) - - // Equality assertion doesn't work with NaNs. - // See https://github.com/stretchr/testify/issues/624. - // - // Remove the NaN entry. - self, _ = dyn.Walk(self, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - if f, ok := v.AsFloat(); ok && math.IsNaN(f) { - return dyn.InvalidValue, dyn.ErrDrop - } - return v, nil - }) - - dynassert.Equal(t, dyn.NewValue( - map[string]dyn.Value{ - "canonical": dyn.NewValue( - 1230.15, - []diag.Location{{File: file, Line: 3, Column: 12}}, - ), - "exponential": dyn.NewValue( - 1230.15, - []diag.Location{{File: file, Line: 4, Column: 14}}, - ), - "fixed": dyn.NewValue( - 1230.15, - []diag.Location{{File: file, Line: 5, Column: 8}}, - ), - "negative infinity": dyn.NewValue( - math.Inf(-1), - []diag.Location{{File: file, Line: 6, Column: 20}}, - ), - }, - []diag.Location{{File: file, Line: 3, Column: 1}}, - ), self) -} - -func TestYAMLSpecExample_2_21(t *testing.T) { - file := "testdata/spec_example_2.21.yml" - self := loadExample(t, file) - - dynassert.Equal(t, dyn.NewValue( - map[string]dyn.Value{ - "null": dyn.NewValue( - nil, - []diag.Location{{File: file, Line: 3, Column: 6}}, - ), - "booleans": dyn.NewValue( - []dyn.Value{ - dyn.NewValue(true, []diag.Location{{File: file, Line: 4, Column: 13}}), - dyn.NewValue(false, []diag.Location{{File: file, Line: 4, Column: 19}}), - }, - []diag.Location{{File: file, Line: 4, Column: 11}}, - ), - "string": dyn.NewValue( - "012345", - []diag.Location{{File: file, Line: 5, Column: 9}}, - ), - }, - []diag.Location{{File: file, Line: 3, Column: 1}}, - ), self) -} - -func TestYAMLSpecExample_2_22(t *testing.T) { - file := "testdata/spec_example_2.22.yml" - self := loadExample(t, file) - - dynassert.Equal(t, dyn.NewValue( - map[string]dyn.Value{ - "canonical": dyn.NewValue( - dyn.MustTime("2001-12-15T02:59:43.1Z"), - []diag.Location{{File: file, Line: 3, Column: 12}}, - ), - "iso8601": dyn.NewValue( - dyn.MustTime("2001-12-14t21:59:43.10-05:00"), - []diag.Location{{File: file, Line: 4, Column: 10}}, - ), - "spaced": dyn.NewValue( - // This is parsed as a string, not a timestamp, - // both by "go.yaml.in/yaml/v3" and by our implementation. - "2001-12-14 21:59:43.10 -5", - []diag.Location{{File: file, Line: 5, Column: 9}}, - ), - "date": dyn.NewValue( - dyn.MustTime("2002-12-14"), - []diag.Location{{File: file, Line: 6, Column: 7}}, - ), - }, - []diag.Location{{File: file, Line: 3, Column: 1}}, - ), self) -} - -func TestYAMLSpecExample_2_23(t *testing.T) { - file := "testdata/spec_example_2.23.yml" - input, err := os.ReadFile(file) - require.NoError(t, err) - - // Note: the !!binary tag is not supported by us. - - _, err = yamlloader.LoadYAML(file, bytes.NewBuffer(input)) - assert.ErrorContains(t, err, `: unknown tag: !!binary`) -} - -func TestYAMLSpecExample_2_24(t *testing.T) { - file := "testdata/spec_example_2.24.yml" - self := loadExample(t, file) - - dynassert.Equal(t, dyn.NewValue( - []dyn.Value{ - dyn.NewValue( - map[string]dyn.Value{ - "center": dyn.NewValue( - map[string]dyn.Value{ - "x": dyn.NewValue(73, []diag.Location{{File: file, Line: 8, Column: 23}}), - "y": dyn.NewValue(129, []diag.Location{{File: file, Line: 8, Column: 30}}), - }, - []diag.Location{{File: file, Line: 8, Column: 11}}, - ), - "radius": dyn.NewValue(7, []diag.Location{{File: file, Line: 9, Column: 11}}), - }, - []diag.Location{{File: file, Line: 7, Column: 3}}, - ), - dyn.NewValue( - map[string]dyn.Value{ - "start": dyn.NewValue( - map[string]dyn.Value{ - "x": dyn.NewValue(73, []diag.Location{{File: file, Line: 8, Column: 23}}), - "y": dyn.NewValue(129, []diag.Location{{File: file, Line: 8, Column: 30}}), - }, - []diag.Location{{File: file, Line: 8, Column: 11}}, - ), - "finish": dyn.NewValue( - map[string]dyn.Value{ - "x": dyn.NewValue(89, []diag.Location{{File: file, Line: 12, Column: 16}}), - "y": dyn.NewValue(102, []diag.Location{{File: file, Line: 12, Column: 23}}), - }, - []diag.Location{{File: file, Line: 12, Column: 11}}, - ), - }, - []diag.Location{{File: file, Line: 10, Column: 3}}, - ), - dyn.NewValue( - map[string]dyn.Value{ - "start": dyn.NewValue( - map[string]dyn.Value{ - "x": dyn.NewValue(73, []diag.Location{{File: file, Line: 8, Column: 23}}), - "y": dyn.NewValue(129, []diag.Location{{File: file, Line: 8, Column: 30}}), - }, - []diag.Location{{File: file, Line: 8, Column: 11}}, - ), - "color": dyn.NewValue(16772795, []diag.Location{{File: file, Line: 15, Column: 10}}), - "text": dyn.NewValue("Pretty vector drawing.", []diag.Location{{File: file, Line: 16, Column: 9}}), - }, - []diag.Location{{File: file, Line: 13, Column: 3}}, - ), - }, - []diag.Location{{File: file, Line: 4, Column: 5}}, - ), self) -} - -func TestYAMLSpecExample_2_25(t *testing.T) { - file := "testdata/spec_example_2.25.yml" - self := loadExample(t, file) - - dynassert.Equal(t, dyn.NewValue( - map[string]dyn.Value{ - "Mark McGwire": dyn.NewValue(nil, []diag.Location{{File: file, Line: 8, Column: 1}}), - "Sammy Sosa": dyn.NewValue(nil, []diag.Location{{File: file, Line: 9, Column: 1}}), - "Ken Griffey": dyn.NewValue(nil, []diag.Location{{File: file, Line: 10, Column: 1}}), - }, - []diag.Location{{File: file, Line: 6, Column: 5}}, - ), self) -} - -func TestYAMLSpecExample_2_26(t *testing.T) { - file := "testdata/spec_example_2.26.yml" - self := loadExample(t, file) - - dynassert.Equal(t, dyn.NewValue( - []dyn.Value{ - dyn.NewValue( - map[string]dyn.Value{ - "Mark McGwire": dyn.NewValue(65, []diag.Location{{File: file, Line: 7, Column: 17}}), - }, - []diag.Location{{File: file, Line: 7, Column: 3}}, - ), - dyn.NewValue( - map[string]dyn.Value{ - "Sammy Sosa": dyn.NewValue(63, []diag.Location{{File: file, Line: 8, Column: 15}}), - }, - []diag.Location{{File: file, Line: 8, Column: 3}}, - ), - dyn.NewValue( - map[string]dyn.Value{ - "Ken Griffey": dyn.NewValue(58, []diag.Location{{File: file, Line: 9, Column: 16}}), - }, - []diag.Location{{File: file, Line: 9, Column: 3}}, - ), - }, - []diag.Location{{File: file, Line: 6, Column: 5}}, - ), self) -} - -func TestYAMLSpecExample_2_27(t *testing.T) { - file := "testdata/spec_example_2.27.yml" - self := loadExample(t, file) - - dynassert.Equal(t, dyn.NewValue( - map[string]dyn.Value{ - "invoice": dyn.NewValue( - 34843, - []diag.Location{{File: file, Line: 4, Column: 10}}, - ), - "date": dyn.NewValue( - dyn.MustTime("2001-01-23"), - []diag.Location{{File: file, Line: 5, Column: 10}}, - ), - "bill-to": dyn.NewValue( - map[string]dyn.Value{ - "given": dyn.NewValue( - "Chris", - []diag.Location{{File: file, Line: 7, Column: 12}}, - ), - "family": dyn.NewValue( - "Dumars", - []diag.Location{{File: file, Line: 8, Column: 12}}, - ), - "address": dyn.NewValue( - map[string]dyn.Value{ - "lines": dyn.NewValue( - "458 Walkman Dr.\nSuite #292\n", - []diag.Location{{File: file, Line: 10, Column: 12}}, - ), - "city": dyn.NewValue( - "Royal Oak", - []diag.Location{{File: file, Line: 13, Column: 15}}, - ), - "state": dyn.NewValue( - "MI", - []diag.Location{{File: file, Line: 14, Column: 15}}, - ), - "postal": dyn.NewValue( - 48046, - []diag.Location{{File: file, Line: 15, Column: 15}}, - ), - }, - []diag.Location{{File: file, Line: 10, Column: 5}}, - ), - }, - []diag.Location{{File: file, Line: 6, Column: 10}}, - ), - "ship-to": dyn.NewValue( - map[string]dyn.Value{ - "given": dyn.NewValue( - "Chris", - []diag.Location{{File: file, Line: 7, Column: 12}}, - ), - "family": dyn.NewValue( - "Dumars", - []diag.Location{{File: file, Line: 8, Column: 12}}, - ), - "address": dyn.NewValue( - map[string]dyn.Value{ - "lines": dyn.NewValue( - "458 Walkman Dr.\nSuite #292\n", - []diag.Location{{File: file, Line: 10, Column: 12}}, - ), - "city": dyn.NewValue( - "Royal Oak", - []diag.Location{{File: file, Line: 13, Column: 15}}, - ), - "state": dyn.NewValue( - "MI", - []diag.Location{{File: file, Line: 14, Column: 15}}, - ), - "postal": dyn.NewValue( - 48046, - []diag.Location{{File: file, Line: 15, Column: 15}}, - ), - }, - []diag.Location{{File: file, Line: 10, Column: 5}}, - ), - }, - []diag.Location{{File: file, Line: 6, Column: 10}}, - ), - "product": dyn.NewValue( - []dyn.Value{ - dyn.NewValue( - map[string]dyn.Value{ - "sku": dyn.NewValue( - "BL394D", - []diag.Location{{File: file, Line: 18, Column: 17}}, - ), - "quantity": dyn.NewValue( - 4, - []diag.Location{{File: file, Line: 19, Column: 17}}, - ), - "description": dyn.NewValue( - "Basketball", - []diag.Location{{File: file, Line: 20, Column: 17}}, - ), - "price": dyn.NewValue( - 450.0, - []diag.Location{{File: file, Line: 21, Column: 17}}, - ), - }, - []diag.Location{{File: file, Line: 18, Column: 3}}, - ), dyn.NewValue( - map[string]dyn.Value{ - "sku": dyn.NewValue( - "BL4438H", - []diag.Location{{File: file, Line: 22, Column: 17}}, - ), - "quantity": dyn.NewValue( - 1, - []diag.Location{{File: file, Line: 23, Column: 17}}, - ), - "description": dyn.NewValue( - "Super Hoop", - []diag.Location{{File: file, Line: 24, Column: 17}}, - ), - "price": dyn.NewValue( - 2392.0, - []diag.Location{{File: file, Line: 25, Column: 17}}, - ), - }, - []diag.Location{{File: file, Line: 22, Column: 3}}, - ), - }, - []diag.Location{{File: file, Line: 18, Column: 1}}, - ), - "tax": dyn.NewValue( - 251.42, - []diag.Location{{File: file, Line: 26, Column: 8}}, - ), - "total": dyn.NewValue( - 4443.52, - []diag.Location{{File: file, Line: 27, Column: 8}}, - ), - "comments": dyn.NewValue( - "Late afternoon is best. Backup contact is Nancy Billsmer @ 338-4338.", - []diag.Location{{File: file, Line: 29, Column: 3}}, - ), - }, - []diag.Location{{File: file, Line: 3, Column: 5}}, - ), self) -} - -func TestYAMLSpecExample_2_28(t *testing.T) { - file := "testdata/spec_example_2.28.yml" - self := loadExample(t, file) - - dynassert.Equal(t, dyn.NewValue( - map[string]dyn.Value{ - "Time": dyn.NewValue( - "2001-11-23 15:01:42 -5", - []diag.Location{{File: file, Line: 4, Column: 7}}, - ), - "User": dyn.NewValue( - "ed", - []diag.Location{{File: file, Line: 5, Column: 7}}, - ), - "Warning": dyn.NewValue( - "This is an error message for the log file", - []diag.Location{{File: file, Line: 7, Column: 3}}, - ), - }, - []diag.Location{{File: file, Line: 4, Column: 1}}, - ), self) -} diff --git a/libs/dyn/yamlloader/yaml_test.go b/libs/dyn/yamlloader/yaml_test.go deleted file mode 100644 index e21065e92c0..00000000000 --- a/libs/dyn/yamlloader/yaml_test.go +++ /dev/null @@ -1,35 +0,0 @@ -package yamlloader_test - -import ( - "bytes" - "os" - "testing" - - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/yamlloader" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "go.yaml.in/yaml/v3" -) - -func loadYAML(t *testing.T, path string) dyn.Value { - input, err := os.ReadFile(path) - require.NoError(t, err) - - var ref any - err = yaml.Unmarshal(input, &ref) - require.NoError(t, err) - - self, err := yamlloader.LoadYAML(path, bytes.NewBuffer(input)) - require.NoError(t, err) - assert.NotNil(t, self) - - // Deep-equal the two values to ensure that the loader is producing - assert.EqualValues(t, ref, self.AsAny()) - return self -} - -func TestYAMLEmpty(t *testing.T) { - self := loadYAML(t, "testdata/empty.yml") - assert.Equal(t, dyn.NilValue, self) -} diff --git a/libs/dyn/yamlsaver/order.go b/libs/dyn/yamlsaver/order.go deleted file mode 100644 index 439be8474a6..00000000000 --- a/libs/dyn/yamlsaver/order.go +++ /dev/null @@ -1,33 +0,0 @@ -package yamlsaver - -import "slices" - -// This struct is used to generate indexes for ordering of map keys. -// The ordering defined based on any predefined Order in `Order` field -// or running Order based on `index` -type Order struct { - index int - order []string -} - -func NewOrder(o []string) *Order { - return &Order{index: 0, order: o} -} - -// Returns an integer which represents the order of map key in resulting -// The lower the index, the earlier in the list the key is. -// If the order is not predefined, it uses running order and any subsequential call to -// order.Get returns an increasing index. -func (o *Order) Get(key string) int { - index := slices.Index(o.order, key) - // If the key is found in predefined order list - // We return a negative index which put the value at the top of the order compared to other - // not predefined keys. The earlier value in predefined list, the lower negative index value - if index != -1 { - return index - len(o.order) - } - - // Otherwise we just increase the order index - o.index += 1 - return o.index -} diff --git a/libs/dyn/yamlsaver/order_test.go b/libs/dyn/yamlsaver/order_test.go deleted file mode 100644 index ed2877f6c8c..00000000000 --- a/libs/dyn/yamlsaver/order_test.go +++ /dev/null @@ -1,24 +0,0 @@ -package yamlsaver - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestOrderReturnsIncreasingIndex(t *testing.T) { - o := NewOrder([]string{}) - assert.Equal(t, 1, o.Get("a")) - assert.Equal(t, 2, o.Get("b")) - assert.Equal(t, 3, o.Get("c")) -} - -func TestOrderReturnsNegativeIndexForPredefinedKeys(t *testing.T) { - o := NewOrder([]string{"a", "b", "c"}) - assert.Equal(t, -3, o.Get("a")) - assert.Equal(t, -2, o.Get("b")) - assert.Equal(t, -1, o.Get("c")) - assert.Equal(t, 1, o.Get("d")) - assert.Equal(t, 2, o.Get("e")) - assert.Equal(t, 3, o.Get("f")) -} diff --git a/libs/dyn/yamlsaver/saver.go b/libs/dyn/yamlsaver/saver.go deleted file mode 100644 index 4c302e26f0d..00000000000 --- a/libs/dyn/yamlsaver/saver.go +++ /dev/null @@ -1,172 +0,0 @@ -package yamlsaver - -import ( - "cmp" - "fmt" - "io" - "os" - "path/filepath" - "slices" - "strconv" - - "github.com/databricks/cli/libs/dyn" - "go.yaml.in/yaml/v3" -) - -type saver struct { - nodesWithStyle map[string]yaml.Style -} - -func NewSaver() *saver { - return &saver{} -} - -func NewSaverWithStyle(nodesWithStyle map[string]yaml.Style) *saver { - return &saver{ - nodesWithStyle: nodesWithStyle, - } -} - -func (s *saver) SaveAsYAML(data any, filename string, force bool) error { - err := os.MkdirAll(filepath.Dir(filename), 0o755) - if err != nil { - return err - } - - // check that file exists - info, err := os.Stat(filename) - if err == nil { - if info.IsDir() { - return fmt.Errorf("%s is a directory", filename) - } - if !force { - return fmt.Errorf("%s already exists. Use --force to overwrite", filename) - } - } - - file, err := os.Create(filename) - if err != nil { - return err - } - defer file.Close() - - err = s.encode(data, file) - if err != nil { - return err - } - return nil -} - -func (s *saver) encode(data any, w io.Writer) error { - yamlNode, err := s.toYamlNode(dyn.V(data)) - if err != nil { - return err - } - enc := yaml.NewEncoder(w) - enc.SetIndent(2) - return enc.Encode(yamlNode) -} - -func (s *saver) toYamlNode(v dyn.Value) (*yaml.Node, error) { - return s.toYamlNodeWithStyle(v, yaml.Style(0)) -} - -func (s *saver) toYamlNodeWithStyle(v dyn.Value, style yaml.Style) (*yaml.Node, error) { - switch v.Kind() { - case dyn.KindMap: - m, _ := v.AsMap() - - // We're using location lines to define the order of keys in YAML. - // The location is set when we convert API response struct to config.Value representation - // See convert.convertMap for details - pairs := m.Pairs() - slices.SortStableFunc(pairs, func(a, b dyn.Pair) int { - return cmp.Compare(a.Value.Location().Line, b.Value.Location().Line) - }) - - var content []*yaml.Node - for _, pair := range pairs { - pk := pair.Key - pv := pair.Value - node := yaml.Node{Kind: yaml.ScalarNode, Value: pk.MustString(), Style: style} - var nestedNodeStyle yaml.Style - if customStyle, ok := s.hasStyle(pk.MustString()); ok { - nestedNodeStyle = customStyle - } else { - nestedNodeStyle = style - } - c, err := s.toYamlNodeWithStyle(pv, nestedNodeStyle) - if err != nil { - return nil, err - } - content = append(content, &node) - content = append(content, c) - } - - return &yaml.Node{Kind: yaml.MappingNode, Content: content, Style: style}, nil - case dyn.KindSequence: - seq, _ := v.AsSequence() - var content []*yaml.Node - for _, item := range seq { - node, err := s.toYamlNodeWithStyle(item, style) - if err != nil { - return nil, err - } - content = append(content, node) - } - return &yaml.Node{Kind: yaml.SequenceNode, Content: content, Style: style}, nil - case dyn.KindNil: - return &yaml.Node{Kind: yaml.ScalarNode, Value: "null", Style: style}, nil - case dyn.KindString: - // If the string is a scalar value (bool, int, float and etc.), we want to quote it. - if isScalarValueInString(v) { - return &yaml.Node{Kind: yaml.ScalarNode, Value: v.MustString(), Style: yaml.DoubleQuotedStyle}, nil - } - return &yaml.Node{Kind: yaml.ScalarNode, Value: v.MustString(), Style: style}, nil - case dyn.KindBool: - return &yaml.Node{Kind: yaml.ScalarNode, Value: strconv.FormatBool(v.MustBool()), Style: style}, nil - case dyn.KindInt: - return &yaml.Node{Kind: yaml.ScalarNode, Value: strconv.FormatInt(v.MustInt(), 10), Style: style}, nil - case dyn.KindFloat: - return &yaml.Node{Kind: yaml.ScalarNode, Value: fmt.Sprint(v.MustFloat()), Style: style}, nil - case dyn.KindTime: - return &yaml.Node{Kind: yaml.ScalarNode, Value: v.MustTime().String(), Style: style}, nil - default: - // Panic because we only want to deal with known types. - panic(fmt.Sprintf("invalid kind: %d", v.Kind())) - } -} - -func (s *saver) hasStyle(key string) (yaml.Style, bool) { - style, ok := s.nodesWithStyle[key] - return style, ok -} - -func isScalarValueInString(v dyn.Value) bool { - if v.Kind() != dyn.KindString { - return false - } - - // Parse value of the string and check if it's a scalar value. - // If it's a scalar value, we want to quote it. - switch v.MustString() { - case "true", "false": - return true - case "": - return true - default: - _, err := parseNumber(v.MustString()) - return err == nil - } -} - -func parseNumber(s string) (any, error) { - if i, err := strconv.ParseInt(s, 0, 64); err == nil { - return i, nil - } - - if f, err := strconv.ParseFloat(s, 64); err == nil { - return f, nil - } - return nil, fmt.Errorf("invalid number: %s", s) -} diff --git a/libs/dyn/yamlsaver/saver_test.go b/libs/dyn/yamlsaver/saver_test.go deleted file mode 100644 index a4f50e0b8fb..00000000000 --- a/libs/dyn/yamlsaver/saver_test.go +++ /dev/null @@ -1,275 +0,0 @@ -package yamlsaver - -import ( - "testing" - - "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "go.yaml.in/yaml/v3" -) - -func TestMarshalNilValue(t *testing.T) { - s := NewSaver() - nilValue := dyn.NilValue - v, err := s.toYamlNode(nilValue) - assert.NoError(t, err) - assert.Equal(t, "null", v.Value) -} - -func TestMarshalIntValue(t *testing.T) { - s := NewSaver() - intValue := dyn.V(1) - v, err := s.toYamlNode(intValue) - assert.NoError(t, err) - assert.Equal(t, "1", v.Value) - assert.Equal(t, yaml.ScalarNode, v.Kind) -} - -func TestMarshalFloatValue(t *testing.T) { - s := NewSaver() - floatValue := dyn.V(1.0) - v, err := s.toYamlNode(floatValue) - assert.NoError(t, err) - assert.Equal(t, "1", v.Value) - assert.Equal(t, yaml.ScalarNode, v.Kind) -} - -func TestMarshalBoolValue(t *testing.T) { - s := NewSaver() - boolValue := dyn.V(true) - v, err := s.toYamlNode(boolValue) - assert.NoError(t, err) - assert.Equal(t, "true", v.Value) - assert.Equal(t, yaml.ScalarNode, v.Kind) -} - -func TestMarshalTimeValue(t *testing.T) { - tm, err := dyn.NewTime("1970-01-01") - require.NoError(t, err) - - s := NewSaver() - timeValue := dyn.V(tm) - v, err := s.toYamlNode(timeValue) - assert.NoError(t, err) - assert.Equal(t, "1970-01-01", v.Value) - assert.Equal(t, yaml.ScalarNode, v.Kind) -} - -func TestMarshalSequenceValue(t *testing.T) { - s := NewSaver() - sequenceValue := dyn.NewValue( - []dyn.Value{ - dyn.NewValue("value1", []diag.Location{{File: "file", Line: 1, Column: 2}}), - dyn.NewValue("value2", []diag.Location{{File: "file", Line: 2, Column: 2}}), - }, - []diag.Location{{File: "file", Line: 1, Column: 2}}, - ) - v, err := s.toYamlNode(sequenceValue) - assert.NoError(t, err) - assert.Equal(t, yaml.SequenceNode, v.Kind) - assert.Equal(t, "value1", v.Content[0].Value) - assert.Equal(t, "value2", v.Content[1].Value) -} - -func TestMarshalStringValue(t *testing.T) { - s := NewSaver() - stringValue := dyn.V("value") - v, err := s.toYamlNode(stringValue) - assert.NoError(t, err) - assert.Equal(t, "value", v.Value) - assert.Equal(t, yaml.ScalarNode, v.Kind) -} - -func TestMarshalMapValue(t *testing.T) { - s := NewSaver() - mapValue := dyn.NewValue( - map[string]dyn.Value{ - "key3": dyn.NewValue("value3", []diag.Location{{File: "file", Line: 3, Column: 2}}), - "key2": dyn.NewValue("value2", []diag.Location{{File: "file", Line: 2, Column: 2}}), - "key1": dyn.NewValue("value1", []diag.Location{{File: "file", Line: 1, Column: 2}}), - }, - []diag.Location{{File: "file", Line: 1, Column: 2}}, - ) - - v, err := s.toYamlNode(mapValue) - assert.NoError(t, err) - assert.Equal(t, yaml.MappingNode, v.Kind) - assert.Equal(t, "key1", v.Content[0].Value) - assert.Equal(t, "value1", v.Content[1].Value) - - assert.Equal(t, "key2", v.Content[2].Value) - assert.Equal(t, "value2", v.Content[3].Value) - - assert.Equal(t, "key3", v.Content[4].Value) - assert.Equal(t, "value3", v.Content[5].Value) -} - -func TestMarshalNestedValues(t *testing.T) { - s := NewSaver() - mapValue := dyn.NewValue( - map[string]dyn.Value{ - "key1": dyn.NewValue( - map[string]dyn.Value{ - "key2": dyn.NewValue("value", []diag.Location{{File: "file", Line: 1, Column: 2}}), - }, - []diag.Location{{File: "file", Line: 1, Column: 2}}, - ), - }, - []diag.Location{{File: "file", Line: 1, Column: 2}}, - ) - v, err := s.toYamlNode(mapValue) - assert.NoError(t, err) - assert.Equal(t, yaml.MappingNode, v.Kind) - assert.Equal(t, "key1", v.Content[0].Value) - assert.Equal(t, yaml.MappingNode, v.Content[1].Kind) - assert.Equal(t, "key2", v.Content[1].Content[0].Value) - assert.Equal(t, "value", v.Content[1].Content[1].Value) -} - -func TestMarshalHexadecimalValueIsQuoted(t *testing.T) { - s := NewSaver() - hexValue := dyn.V(0x123) - v, err := s.toYamlNode(hexValue) - assert.NoError(t, err) - assert.Equal(t, "291", v.Value) - assert.Equal(t, yaml.Style(0), v.Style) - assert.Equal(t, yaml.ScalarNode, v.Kind) - - stringValue := dyn.V("0x123") - v, err = s.toYamlNode(stringValue) - assert.NoError(t, err) - assert.Equal(t, "0x123", v.Value) - assert.Equal(t, yaml.DoubleQuotedStyle, v.Style) - assert.Equal(t, yaml.ScalarNode, v.Kind) -} - -func TestMarshalBinaryValueIsQuoted(t *testing.T) { - s := NewSaver() - binaryValue := dyn.V(0b101) - v, err := s.toYamlNode(binaryValue) - assert.NoError(t, err) - assert.Equal(t, "5", v.Value) - assert.Equal(t, yaml.Style(0), v.Style) - assert.Equal(t, yaml.ScalarNode, v.Kind) - - stringValue := dyn.V("0b101") - v, err = s.toYamlNode(stringValue) - assert.NoError(t, err) - assert.Equal(t, "0b101", v.Value) - assert.Equal(t, yaml.DoubleQuotedStyle, v.Style) - assert.Equal(t, yaml.ScalarNode, v.Kind) -} - -func TestMarshalOctalValueIsQuoted(t *testing.T) { - s := NewSaver() - octalValue := dyn.V(0o123) - v, err := s.toYamlNode(octalValue) - assert.NoError(t, err) - assert.Equal(t, "83", v.Value) - assert.Equal(t, yaml.Style(0), v.Style) - assert.Equal(t, yaml.ScalarNode, v.Kind) - - stringValue := dyn.V("0123") - v, err = s.toYamlNode(stringValue) - assert.NoError(t, err) - assert.Equal(t, "0123", v.Value) - assert.Equal(t, yaml.DoubleQuotedStyle, v.Style) - assert.Equal(t, yaml.ScalarNode, v.Kind) -} - -func TestMarshalFloatValueIsQuoted(t *testing.T) { - s := NewSaver() - floatValue := dyn.V(1.0) - v, err := s.toYamlNode(floatValue) - assert.NoError(t, err) - assert.Equal(t, "1", v.Value) - assert.Equal(t, yaml.Style(0), v.Style) - assert.Equal(t, yaml.ScalarNode, v.Kind) - - stringValue := dyn.V("1.0") - v, err = s.toYamlNode(stringValue) - assert.NoError(t, err) - assert.Equal(t, "1.0", v.Value) - assert.Equal(t, yaml.DoubleQuotedStyle, v.Style) - assert.Equal(t, yaml.ScalarNode, v.Kind) -} - -func TestMarshalBoolValueIsQuoted(t *testing.T) { - s := NewSaver() - boolValue := dyn.V(true) - v, err := s.toYamlNode(boolValue) - assert.NoError(t, err) - assert.Equal(t, "true", v.Value) - assert.Equal(t, yaml.Style(0), v.Style) - assert.Equal(t, yaml.ScalarNode, v.Kind) - - stringValue := dyn.V("true") - v, err = s.toYamlNode(stringValue) - assert.NoError(t, err) - assert.Equal(t, "true", v.Value) - assert.Equal(t, yaml.DoubleQuotedStyle, v.Style) - assert.Equal(t, yaml.ScalarNode, v.Kind) -} - -func TestCustomStylingWithNestedMap(t *testing.T) { - s := NewSaverWithStyle(map[string]yaml.Style{ - "styled": yaml.DoubleQuotedStyle, - }) - - styledMap := dyn.NewValue( - map[string]dyn.Value{ - "key1": dyn.NewValue("value1", []diag.Location{{File: "file", Line: 1, Column: 2}}), - "key2": dyn.NewValue("value2", []diag.Location{{File: "file", Line: 2, Column: 2}}), - }, - []diag.Location{{File: "file", Line: -2, Column: 2}}, - ) - - unstyledMap := dyn.NewValue( - map[string]dyn.Value{ - "key3": dyn.NewValue("value3", []diag.Location{{File: "file", Line: 1, Column: 2}}), - "key4": dyn.NewValue("value4", []diag.Location{{File: "file", Line: 2, Column: 2}}), - }, - []diag.Location{{File: "file", Line: -1, Column: 2}}, - ) - - val := dyn.NewValue( - map[string]dyn.Value{ - "styled": styledMap, - "unstyled": unstyledMap, - }, - []diag.Location{{File: "file", Line: 1, Column: 2}}, - ) - - mv, err := s.toYamlNode(val) - assert.NoError(t, err) - - // Check that the styled map is quoted - v := mv.Content[1] - - assert.Equal(t, yaml.MappingNode, v.Kind) - assert.Equal(t, "key1", v.Content[0].Value) - assert.Equal(t, "value1", v.Content[1].Value) - assert.Equal(t, yaml.DoubleQuotedStyle, v.Content[0].Style) - assert.Equal(t, yaml.DoubleQuotedStyle, v.Content[1].Style) - - assert.Equal(t, "key2", v.Content[2].Value) - assert.Equal(t, "value2", v.Content[3].Value) - assert.Equal(t, yaml.DoubleQuotedStyle, v.Content[2].Style) - assert.Equal(t, yaml.DoubleQuotedStyle, v.Content[3].Style) - - // Check that the unstyled map is not quoted - v = mv.Content[3] - - assert.Equal(t, yaml.MappingNode, v.Kind) - assert.Equal(t, "key3", v.Content[0].Value) - assert.Equal(t, "value3", v.Content[1].Value) - assert.Equal(t, yaml.Style(0), v.Content[0].Style) - assert.Equal(t, yaml.Style(0), v.Content[1].Style) - - assert.Equal(t, "key4", v.Content[2].Value) - assert.Equal(t, "value4", v.Content[3].Value) - assert.Equal(t, yaml.Style(0), v.Content[2].Style) - assert.Equal(t, yaml.Style(0), v.Content[3].Style) -} diff --git a/libs/dyn/yamlsaver/utils.go b/libs/dyn/yamlsaver/utils.go deleted file mode 100644 index 61f41790029..00000000000 --- a/libs/dyn/yamlsaver/utils.go +++ /dev/null @@ -1,94 +0,0 @@ -package yamlsaver - -import ( - "fmt" - "slices" - - "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/convert" -) - -// Converts a struct to map. Skips any nil fields. -// It uses `skipFields` to skip unnecessary fields. -// Uses `order` to define the order of keys in resulting outout -func ConvertToMapValue(strct any, order *Order, skipFields []string, dst map[string]dyn.Value) (dyn.Value, error) { - ref := dyn.NilValue - mv, err := convert.FromTyped(strct, ref) - if err != nil { - return dyn.InvalidValue, err - } - - if mv.Kind() != dyn.KindMap { - return dyn.InvalidValue, fmt.Errorf("expected map, got %s", mv.Kind()) - } - - mv, err = sortMapAlphabetically(mv) - if err != nil { - return dyn.InvalidValue, err - } - - return skipAndOrder(mv, order, skipFields, dst) -} - -// Sort the map alphabetically by keys. This is used to produce stable output for generated YAML files. -func sortMapAlphabetically(mv dyn.Value) (dyn.Value, error) { - sortedMap := dyn.NewMapping() - mapV := mv.MustMap() - keys := mapV.Keys() - slices.SortStableFunc(keys, func(i, j dyn.Value) int { - iKey := i.MustString() - jKey := j.MustString() - if iKey < jKey { - return -1 - } - - if iKey > jKey { - return 1 - } - return 0 - }) - - for _, key := range keys { - value, _ := mapV.Get(key) - var err error - if value.Kind() == dyn.KindMap { - value, err = sortMapAlphabetically(value) - if err != nil { - return dyn.InvalidValue, err - } - } - sortedMap.SetLoc(key.MustString(), key.Locations(), value) - } - - return dyn.V(sortedMap), nil -} - -func skipAndOrder(mv dyn.Value, order *Order, skipFields []string, dst map[string]dyn.Value) (dyn.Value, error) { - for _, pair := range mv.MustMap().Pairs() { - k := pair.Key.MustString() - v := pair.Value - if v.Kind() == dyn.KindNil { - continue - } - - if slices.Contains(skipFields, k) { - continue - } - - // If the value is already defined in destination, it means it was - // manually set due to custom ordering or other customisation required - // So we're skipping processing it again - if _, ok := dst[k]; ok { - continue - } - - if order == nil { - dst[k] = v - } else { - dst[k] = dyn.NewValue(v.Value(), []diag.Location{{Line: order.Get(k)}}) - } - } - - return dyn.V(dst), nil -} diff --git a/libs/dyn/yamlsaver/utils_test.go b/libs/dyn/yamlsaver/utils_test.go deleted file mode 100644 index 8b056e0076f..00000000000 --- a/libs/dyn/yamlsaver/utils_test.go +++ /dev/null @@ -1,104 +0,0 @@ -package yamlsaver - -import ( - "testing" - - "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/dynassert" - "github.com/stretchr/testify/assert" -) - -func TestConvertToMap(t *testing.T) { - type test struct { - Name string `json:"name"` - Map map[string]string `json:"map"` - List []string `json:"list"` - LongNameField string `json:"long_name_field"` - ForceSendFields []string `json:"-"` - Format string `json:"format"` - } - - v := &test{ - Name: "test", - Map: map[string]string{ - "key2": "value2", - "key1": "value1", - }, - List: []string{"a", "b", "c"}, - ForceSendFields: []string{ - "Name", - }, - LongNameField: "long name goes here", - } - result, err := ConvertToMapValue(v, nil, []string{"format"}, map[string]dyn.Value{}) - assert.NoError(t, err) - dynassert.Equal(t, dyn.V(map[string]dyn.Value{ - "list": dyn.V( - []dyn.Value{ - dyn.V("a"), - dyn.V("b"), - dyn.V("c"), - }, - ), - "long_name_field": dyn.V("long name goes here"), - "map": dyn.V( - map[string]dyn.Value{ - "key1": dyn.V("value1"), - "key2": dyn.V("value2"), - }, - ), - "name": dyn.V( - "test", - ), - }), result) -} - -func TestConvertToMapValueWithOrder(t *testing.T) { - type test struct { - Name string `json:"name"` - Map map[string]string `json:"map"` - List []string `json:"list"` - LongNameField string `json:"long_name_field"` - ForceSendFields []string `json:"-"` - Format string `json:"format"` - } - - v := &test{ - Name: "test", - Map: map[string]string{ - "key1": "value1", - "key2": "value2", - }, - List: []string{"a", "b", "c"}, - ForceSendFields: []string{ - "Name", - }, - LongNameField: "long name goes here", - } - result, err := ConvertToMapValue(v, NewOrder([]string{"list", "name", "map"}), []string{"format"}, map[string]dyn.Value{}) - assert.NoError(t, err) - - dynassert.Equal(t, dyn.V(map[string]dyn.Value{ - "list": dyn.NewValue( - []dyn.Value{ - dyn.V("a"), - dyn.V("b"), - dyn.V("c"), - }, - []diag.Location{{Line: -3}}, - ), - "name": dyn.NewValue( - "test", - []diag.Location{{Line: -2}}, - ), - "map": dyn.NewValue( - map[string]dyn.Value{ - "key1": dyn.V("value1"), - "key2": dyn.V("value2"), - }, - []diag.Location{{Line: -1}}, - ), - "long_name_field": dyn.NewValue("long name goes here", []diag.Location{{Line: 1}}), - }), result) -} diff --git a/libs/flags/json_flag.go b/libs/flags/json_flag.go index 3ea4834a5f2..775ea346c70 100644 --- a/libs/flags/json_flag.go +++ b/libs/flags/json_flag.go @@ -8,9 +8,9 @@ import ( "reflect" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn/convert" - "github.com/databricks/cli/libs/dyn/jsonloader" + "github.com/databricks/cli/libs/structs/structvar" "github.com/databricks/databricks-sdk-go/marshal" + "go.yaml.in/yaml/v3" ) type JsonFlag struct { @@ -48,27 +48,18 @@ func (j *JsonFlag) Unmarshal(v any) diag.Diagnostics { return nil } - dv, err := jsonloader.LoadJSON(j.raw, j.source) + node, err := structvar.ParseJSON(j.source, j.raw) if err != nil { return diag.FromErr(err) } - // First normalize the input data. - // It will convert all the values to the correct types. + // Convert the input to the types of the target, in a fresh value: fields set by + // other flags are already in v and must be kept. // For example string literals for booleans and integers will be converted to the correct types. - nv, diags := convert.Normalize(v, dv) - if diags.HasError() { - return diags - } - - if !nv.IsValid() { - kind := reflect.TypeOf(v).Kind() - if kind == reflect.Pointer { - kind = reflect.TypeOf(v).Elem().Kind() - } - + sv, diags, err := structvar.DecodeYAMLNode(j.source, node, reflect.New(reflect.TypeOf(v).Elem()).Interface(), nil) + if err != nil { var expectedJsonType string - switch kind { + switch reflect.TypeOf(v).Elem().Kind() { case reflect.Struct, reflect.Map: expectedJsonType = "object" case reflect.Slice: @@ -80,37 +71,28 @@ func (j *JsonFlag) Unmarshal(v any) diag.Diagnostics { return diags.Append(diag.Diagnostic{ Severity: diag.Error, Summary: "Invalid command input", - Detail: fmt.Sprintf("expected JSON %s, received %s", expectedJsonType, dv.Kind()), + Detail: fmt.Sprintf("expected JSON %s, received %s", expectedJsonType, nodeKind(node)), }) } // Then marshal the normalized data to the output. // It will serialize all set data with the correct types. - data, err := json.Marshal(nv.AsAny()) + data, err := json.Marshal(sv.View().AsAny()) if err != nil { return diags.Extend(diag.FromErr(err)) } - kind := reflect.ValueOf(v).Kind() - if kind == reflect.Pointer { - kind = reflect.ValueOf(v).Elem().Kind() - } - - if kind == reflect.Struct { + if reflect.TypeOf(v).Elem().Kind() == reflect.Struct { // Finally unmarshal the normalized data to the output. // It will fill in the ForceSendFields field if the struct contains it. err = marshal.Unmarshal(data, v) - if err != nil { - return diags.Extend(diag.FromErr(err)) - } } else { // If the output is not a struct, just unmarshal the data to the output. err = json.Unmarshal(data, v) - if err != nil { - return diags.Extend(diag.FromErr(err)) - } } - + if err != nil { + return diags.Extend(diag.FromErr(err)) + } return diags } @@ -133,7 +115,7 @@ func (j *JsonFlag) Validate() error { if j.raw == nil { return nil } - _, err := jsonloader.LoadJSON(j.raw, j.source) + _, err := structvar.ParseJSON(j.source, j.raw) return err } @@ -170,3 +152,15 @@ func (j *JsonFlag) RejectWrappedJSON(outerKey, example string) error { } return fmt.Errorf("%s", msg) } + +// nodeKind returns the kind of the JSON value in node, as a user sees it. +func nodeKind(node *yaml.Node) string { + switch node.Kind { + case yaml.MappingNode: + return "map" + case yaml.SequenceNode: + return "sequence" + default: + return map[string]string{"!!str": "string", "!!bool": "bool", "!!int": "int", "!!float": "float", "!!null": "nil"}[node.Tag] + } +} diff --git a/libs/flags/json_flag_test.go b/libs/flags/json_flag_test.go index 675e1014922..f5142e48e6a 100644 --- a/libs/flags/json_flag_test.go +++ b/libs/flags/json_flag_test.go @@ -6,7 +6,6 @@ import ( "testing" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/stretchr/testify/assert" @@ -202,7 +201,7 @@ func TestJsonUnmarshalRequestMismatch(t *testing.T) { Column: 6, }, }, - Paths: dyn.ToStructPaths(dyn.EmptyPath), + Paths: []*structpath.PathNode{nil}, }) } @@ -260,7 +259,7 @@ func TestJsonUnmarshalWrongTypeReportsCorrectLocation(t *testing.T) { Column: 15, }, }, - Paths: structpath.NewPathSlice("job_id"), + Paths: structpath.MustParsePaths("job_id"), }) } @@ -285,7 +284,7 @@ func TestJsonUnmarshalArrayInsteadOfIntReportsCorrectLocation(t *testing.T) { Column: 40, }, }, - Paths: structpath.NewPathSlice("new_settings", "timeout_seconds"), + Paths: structpath.MustParsePaths("new_settings.timeout_seconds"), }) } diff --git a/libs/structs/structaccess/bundle_test.go b/libs/structs/structaccess/bundle_test.go index 4de21ede392..12c06830dbc 100644 --- a/libs/structs/structaccess/bundle_test.go +++ b/libs/structs/structaccess/bundle_test.go @@ -1,4 +1,4 @@ -package structaccess +package structaccess_test import ( "reflect" @@ -6,6 +6,7 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/structs/structaccess" "github.com/databricks/databricks-sdk-go/service/apps" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/stretchr/testify/require" @@ -45,37 +46,37 @@ func TestGet_ConfigRoot_JobTagsAccess(t *testing.T) { } // Access a value inside the tags map - v, err := GetByString(root, "resources.jobs.my_job.tags.env") + v, err := structaccess.GetByString(root, "resources.jobs.my_job.tags.env") require.NoError(t, err) require.Equal(t, "dev", v) - require.NoError(t, ValidateByString(reflect.TypeFor[config.Root](), "resources.jobs.my_job.tags.env")) - require.NoError(t, ValidateByString(reflect.TypeFor[config.Root](), "resources.jobs.my_job.tags.anything")) - require.Error(t, ValidateByString(reflect.TypeFor[config.Root](), "resources.jobs.my_job.tags.env.inner")) - require.Error(t, ValidateByString(reflect.TypeFor[config.Root](), "resources.jobs.my_job.tags1")) + require.NoError(t, structaccess.ValidateByString(reflect.TypeFor[config.Root](), "resources.jobs.my_job.tags.env")) + require.NoError(t, structaccess.ValidateByString(reflect.TypeFor[config.Root](), "resources.jobs.my_job.tags.anything")) + require.Error(t, structaccess.ValidateByString(reflect.TypeFor[config.Root](), "resources.jobs.my_job.tags.env.inner")) + require.Error(t, structaccess.ValidateByString(reflect.TypeFor[config.Root](), "resources.jobs.my_job.tags1")) // Array indexing test (1) - v, err = GetByString(root, "resources.jobs.my_job.tasks[0].task_key") + v, err = structaccess.GetByString(root, "resources.jobs.my_job.tasks[0].task_key") require.NoError(t, err) require.Equal(t, "t1", v) - require.NoError(t, ValidateByString(reflect.TypeFor[config.Root](), "resources.jobs.my_job.tasks[0].task_key")) - require.Error(t, ValidateByString(reflect.TypeFor[config.Root](), "resources.jobs.my_job.tasks[0].task_key.inner")) - require.Error(t, ValidateByString(reflect.TypeFor[config.Root](), "resources.jobs.my_job.tasks[0].task_key1")) + require.NoError(t, structaccess.ValidateByString(reflect.TypeFor[config.Root](), "resources.jobs.my_job.tasks[0].task_key")) + require.Error(t, structaccess.ValidateByString(reflect.TypeFor[config.Root](), "resources.jobs.my_job.tasks[0].task_key.inner")) + require.Error(t, structaccess.ValidateByString(reflect.TypeFor[config.Root](), "resources.jobs.my_job.tasks[0].task_key1")) // Array indexing test (2) - v, err = GetByString(root, "resources.jobs.my_job.tasks[0].notebook_task.notebook_path") + v, err = structaccess.GetByString(root, "resources.jobs.my_job.tasks[0].notebook_task.notebook_path") require.NoError(t, err) require.Equal(t, "/Workspace/Users/user@example.com/nb", v) - require.NoError(t, ValidateByString(reflect.TypeFor[config.Root](), "resources.jobs.my_job.tasks[0].notebook_task.notebook_path")) - require.Error(t, ValidateByString(reflect.TypeFor[config.Root](), "resources.jobs.my_job.tasks[0].notebook_task.notebook_path.inner")) - require.Error(t, ValidateByString(reflect.TypeFor[config.Root](), "resources.jobs.my_job.tasks[0].notebook_task.notebook_path1")) + require.NoError(t, structaccess.ValidateByString(reflect.TypeFor[config.Root](), "resources.jobs.my_job.tasks[0].notebook_task.notebook_path")) + require.Error(t, structaccess.ValidateByString(reflect.TypeFor[config.Root](), "resources.jobs.my_job.tasks[0].notebook_task.notebook_path.inner")) + require.Error(t, structaccess.ValidateByString(reflect.TypeFor[config.Root](), "resources.jobs.my_job.tasks[0].notebook_task.notebook_path1")) // Test ambiguous field access: outer is ignored because it has bundle tag - v, err = GetByString(root, "resources.apps.my_app.url") + v, err = structaccess.GetByString(root, "resources.apps.my_app.url") require.NoError(t, err) require.Equal(t, "app_inner_url", v) - require.NoError(t, ValidateByString(reflect.TypeFor[config.Root](), "resources.apps.my_app.url")) - require.Error(t, ValidateByString(reflect.TypeFor[config.Root](), "resources.apps.my_app.url.inner")) - require.Error(t, ValidateByString(reflect.TypeFor[config.Root](), "resources.apps.my_app.url1")) + require.NoError(t, structaccess.ValidateByString(reflect.TypeFor[config.Root](), "resources.apps.my_app.url")) + require.Error(t, structaccess.ValidateByString(reflect.TypeFor[config.Root](), "resources.apps.my_app.url.inner")) + require.Error(t, structaccess.ValidateByString(reflect.TypeFor[config.Root](), "resources.apps.my_app.url1")) } // A bundle resource embeds a config struct that embeds the SDK request struct, so its @@ -86,29 +87,29 @@ func TestGetSet_DoublyEmbeddedField(t *testing.T) { project := &resources.PostgresProject{} //exhaustruct:ignore project.ProjectId = "p" - require.NoError(t, ValidateByString(reflect.TypeOf(project), "budget_policy_id")) + require.NoError(t, structaccess.ValidateByString(reflect.TypeOf(project), "budget_policy_id")) - require.NoError(t, SetByString(project, "budget_policy_id", "abc")) + require.NoError(t, structaccess.SetByString(project, "budget_policy_id", "abc")) require.Equal(t, "abc", project.BudgetPolicyId) - value, err := GetByString(project, "budget_policy_id") + value, err := structaccess.GetByString(project, "budget_policy_id") require.NoError(t, err) require.Equal(t, "abc", value) // An explicit empty value is recorded on ProjectSpec, which declares the field. - require.NoError(t, SetByString(project, "budget_policy_id", "")) + require.NoError(t, structaccess.SetByString(project, "budget_policy_id", "")) require.Contains(t, project.ProjectSpec.ForceSendFields, "BudgetPolicyId") require.NotContains(t, project.ForceSendFields, "BudgetPolicyId") - value, err = GetByString(project, "budget_policy_id") + value, err = structaccess.GetByString(project, "budget_policy_id") require.NoError(t, err) // The empty string, not nil: that is what separates an explicit "" from an absent field. require.Equal(t, any(""), value) // And dropping it again leaves the field absent. - require.NoError(t, SetByString(project, "budget_policy_id", nil)) + require.NoError(t, structaccess.SetByString(project, "budget_policy_id", nil)) require.NotContains(t, project.ProjectSpec.ForceSendFields, "BudgetPolicyId") - value, err = GetByString(project, "budget_policy_id") + value, err = structaccess.GetByString(project, "budget_policy_id") require.NoError(t, err) require.Nil(t, value) } diff --git a/libs/structs/structpath/path.go b/libs/structs/structpath/path.go index 8e608253e33..4aa4f4aa9be 100644 --- a/libs/structs/structpath/path.go +++ b/libs/structs/structpath/path.go @@ -191,6 +191,15 @@ func NewPathSlice(parts ...any) []*PathNode { return []*PathNode{NewPath(nil, parts...)} } +// Join appends the components of nodes (each node's own key, index or key-value, not +// its parents) to prefix. +func Join(prefix *PathNode, nodes ...*PathNode) *PathNode { + for _, n := range nodes { + prefix = &PathNode{prev: prefix, key: n.key, index: n.index, value: n.value} + } + return prefix +} + func NewKeyValue(prev *PathNode, key, value string) *PathNode { return &PathNode{ prev: prev, @@ -696,6 +705,15 @@ func MustParsePath(s string) *PathNode { return path } +// MustParsePattern parses a pattern string and panics on error. Wildcards are allowed. +func MustParsePattern(s string) *PatternNode { + pattern, err := ParsePattern(s) + if err != nil { + panic(err) + } + return pattern +} + // MustParsePaths parses each of paths like [MustParsePath], e.g. for diag.Diagnostic.Paths. func MustParsePaths(paths ...string) []*PathNode { out := make([]*PathNode, len(paths)) @@ -1017,3 +1035,13 @@ func (p *PatternNode) UnmarshalYAML(unmarshal func(any) error) error { *p = *parsed return nil } + +// KeyAt returns the string key of the i-th component of the path, or "" if it is not a key. +func (p *PathNode) KeyAt(i int) string { + nodes := p.AsSlice() + if i < 0 || i >= len(nodes) { + return "" + } + k, _ := nodes[i].StringKey() + return k +} diff --git a/libs/structs/structpath/path_test.go b/libs/structs/structpath/path_test.go index 3b7b3d3c896..c0a735cfd42 100644 --- a/libs/structs/structpath/path_test.go +++ b/libs/structs/structpath/path_test.go @@ -1275,6 +1275,19 @@ paths: assert.Contains(t, err.Error(), "unexpected end of input") } +func TestNewStringKeys(t *testing.T) { + assert.Equal(t, "resources.jobs['${var.env}_job']", NewStringKeys(nil, "resources", "jobs", "${var.env}_job").String()) + assert.Equal(t, "a.b.c", NewStringKeys(MustParsePath("a"), "b", "c").String()) + assert.Equal(t, "a", NewStringKeys(MustParsePath("a")).String()) +} + +func TestJoin(t *testing.T) { + p := MustParsePath("tasks[task_key='x'].libraries[0]['a.b']") + assert.Equal(t, "resources.jobs.j.tasks[task_key='x'].libraries[0]['a.b']", Join(MustParsePath("resources.jobs.j"), p.AsSlice()...).String()) + assert.Equal(t, p.String(), Join(nil, p.AsSlice()...).String()) + assert.Equal(t, "a", Join(MustParsePath("a")).String()) +} + func TestMustParsePaths(t *testing.T) { paths := MustParsePaths("bundle.terraform", "resources.jobs.foo.tasks[0]") require.Len(t, paths, 2) @@ -1283,19 +1296,10 @@ func TestMustParsePaths(t *testing.T) { assert.Empty(t, MustParsePaths()) } -func TestNewPath(t *testing.T) { - assert.Equal(t, "a.b['c d']", NewPath(NewStringKey(nil, "a"), "b", "c d").String()) - assert.Equal(t, "resources.jobs.j.tasks[2].job_cluster_key", NewPath(nil, "resources", "jobs", "j", "tasks", 2, "job_cluster_key").String()) - assert.Nil(t, NewPath(nil)) - assert.Equal(t, "a", NewPath(NewStringKey(nil, "a")).String()) - assert.Panics(t, func() { NewPath(nil, int64(1)) }) -} - -func TestNewPathSlice(t *testing.T) { - paths := NewPathSlice("resources", "jobs", "${var.env}_job") +func TestNewStringKeysSlice(t *testing.T) { + paths := NewStringKeysSlice("resources", "jobs", "${var.env}_job") require.Len(t, paths, 1) assert.Equal(t, "resources.jobs['${var.env}_job']", paths[0].String()) - assert.Equal(t, "sync.paths[3]", NewPathSlice("sync", "paths", 3)[0].String()) } func TestNewPattern(t *testing.T) { diff --git a/libs/structs/structvar/decode.go b/libs/structs/structvar/decode.go new file mode 100644 index 00000000000..7c74ddcdee9 --- /dev/null +++ b/libs/structs/structvar/decode.go @@ -0,0 +1,776 @@ +package structvar + +import ( + "encoding/json" + "fmt" + "io" + "maps" + "reflect" + "strconv" + + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/structs/structpath" + "go.yaml.in/yaml/v3" +) + +// source is a configuration tree to decode from (a parsed YAML document or a view). +type source interface { + kind() Kind + locations() []diag.Location + // anchor reports whether the value is a YAML anchor (or an alias of one). + anchor() bool + // scalar returns the value of a string, bool, int (int or int64), float or time (string). + scalar() any + pairs() ([]sourcePair, error) + elems() ([]source, error) +} + +type sourcePair struct { + key string + keyLocs []diag.Location + value source +} + +// DecodeYAML decodes the YAML document in r (read from file) into dst, a pointer to a +// typed configuration value, and returns its references and locations. +// +// Values are converted to the field types the way configuration is normalized: scalars +// are converted where possible, pure variable references are recorded in the returned +// references for fields that cannot hold a string, and values that cannot be converted +// or fields that do not exist are dropped with a warning. An error is returned for +// invalid YAML. +func DecodeYAML(file string, r io.Reader, dst any) (*StructVar, diag.Diagnostics, error) { + node, err := ParseYAML(r) + if err != nil { + return nil, nil, err + } + return DecodeYAMLNode(file, node, dst, nil) +} + +// LocationMapper returns the locations to record for the value at path, given the +// locations it has in the YAML document. +type LocationMapper func(path *structpath.PathNode, locs []diag.Location) []diag.Location + +// DecodeYAMLNode is [DecodeYAML] for a parsed YAML document (nil if it is empty). +// If mapLocations is not nil, it is applied to the location of every value. +func DecodeYAMLNode(file string, node *yaml.Node, dst any, mapLocations LocationMapper) (*StructVar, diag.Diagnostics, error) { + sv := &StructVar{Value: dst} + v := reflect.ValueOf(dst).Elem() + if node == nil { + // An empty document is null. + v.SetZero() + return sv, nil, nil + } + src, err := newYAMLSource(file, node, nil) + if err != nil { + return nil, nil, err + } + // Report YAML errors anywhere in the document, also in values that are dropped. + if err := validate(src); err != nil { + return nil, nil, err + } + var d decoder + var s source = src + if mapLocations != nil { + s = mappedSource{source: src, fn: mapLocations} + } + n, ok, err := d.decode(v, s, nil) + if err == nil && !ok { + // The value could not be converted to dst at all (e.g. a list for a struct). + err = fmt.Errorf("expected a %s, found a %s", kindOf(v.Type()), src.kind()) + } + sv.Refs = n.refs(nil, nil) + sv.Locations = n.locations() + return sv, d.diags, err +} + +// Assign sets the value described by src at path, replacing what was there, with the +// references and locations of src. The value is converted to the type at path like +// [DecodeYAML] converts values; the diagnostics explain values that were dropped. +func (sv *StructVar) Assign(path *structpath.PathNode, src View) (diag.Diagnostics, error) { + var d decoder + var n *dnode + err := update(reflect.ValueOf(sv.Value), components(path), func(dst reflect.Value) (bool, error) { + var ok bool + var err error + n, ok, err = d.decode(dst, viewSource{src}, path) + if !ok { + dst.SetZero() + return false, err + } + return n.isZeroValue(dst), err + }) + if err != nil { + return d.diags, err + } + sv.Refs = n.refs(path, withoutRefs(sv.Refs, path)) + sv.Locations = sv.Locations.With(path, n.locations()) + return d.diags, nil +} + +func validate(s source) error { + switch s.kind() { + case KindMap: + pairs, err := s.pairs() + if err != nil { + return err + } + for _, p := range pairs { + if err := validate(p.value); err != nil { + return err + } + } + case KindSequence: + elems, err := s.elems() + if err != nil { + return err + } + for _, e := range elems { + if err := validate(e); err != nil { + return err + } + } + default: + } + return nil +} + +type decoder struct { + diags diag.Diagnostics +} + +// dnode is the decoded shape of a value: what the typed value does not hold. +type dnode struct { + kind Kind + locs []diag.Location + ref string + keys map[string]*dnode + elems []*dnode +} + +// locations returns the locations of the decoded value. +func (n *dnode) locations() *Locations { + if n == nil { + return nil + } + out := &Locations{locs: n.locs} + for k, c := range n.keys { + if out.keys == nil { + out.keys = make(map[string]*Locations, len(n.keys)) + } + out.keys[k] = c.locations() + } + for _, c := range n.elems { + out.elems = append(out.elems, c.locations()) + } + return out +} + +// refs returns refs (copied) plus the references of the decoded value at path. +func (n *dnode) refs(path *structpath.PathNode, refs map[string]string) map[string]string { + var out map[string]string + n.collectRefs(path, func(p *structpath.PathNode, ref string) { + if out == nil { + out = maps.Clone(refs) + if out == nil { + out = map[string]string{} + } + } + out[p.String()] = ref + }) + if out == nil { + return refs + } + return out +} + +func (n *dnode) collectRefs(path *structpath.PathNode, fn func(*structpath.PathNode, string)) { + if n == nil { + return + } + if n.ref != "" { + fn(path, n.ref) + } + for k, c := range n.keys { + c.collectRefs(structpath.NewStringKey(path, k), fn) + } + for i, c := range n.elems { + c.collectRefs(structpath.NewIndex(path, i), fn) + } +} + +func (d *decoder) warn(summary string, src source, path *structpath.PathNode) { + d.diags = d.diags.Append(diag.Diagnostic{ + Severity: diag.Warning, + Summary: summary, + Locations: []diag.Location{firstLocation(src)}, + Paths: []*structpath.PathNode{path}, + }) +} + +func firstLocation(src source) diag.Location { + locs := src.locations() + if len(locs) == 0 { + return diag.Location{} + } + return locs[0] +} + +func (d *decoder) typeMismatch(expected Kind, src source, path *structpath.PathNode) { + d.warn(fmt.Sprintf("expected %s, found %s", expected, src.kind()), src, path) +} + +func (d *decoder) nullWarning(expected Kind, src source, path *structpath.PathNode) { + d.warn(fmt.Sprintf("expected a %s value, found null", expected), src, path) +} + +func pureRefOf(src source) string { + if src.kind() != KindString { + return "" + } + s, _ := src.scalar().(string) + if IsPureVariableReference(s) { + return s + } + return "" +} + +// decode decodes src into dst (settable) and returns its meta. It reports false if the +// value cannot be decoded and is dropped (the diagnostics say why). +func (d *decoder) decode(dst reflect.Value, src source, path *structpath.PathNode) (*dnode, bool, error) { + typ := dst.Type() + if typ.Kind() == reflect.Pointer { + if src.kind() == KindNil { + n, ok, err := d.decode(reflect.New(typ.Elem()).Elem(), src, path) + if ok { + dst.SetZero() + } + return n, ok, err + } + p := reflect.New(typ.Elem()) + n, ok, err := d.decode(p.Elem(), src, path) + if ok { + dst.Set(p) + } + return n, ok, err + } + + n := &dnode{kind: src.kind(), locs: src.locations()} + ref := pureRefOf(src) + + switch typ.Kind() { + case reflect.Struct: + if IsSDKNativeType(typ) { + s, ok := d.decodeString(src, path) + if !ok { + return nil, false, nil + } + n.kind = KindString + if ref != "" { + n.ref = ref + dst.SetZero() + return n, true, nil + } + buf, err := json.Marshal(s) + if err != nil { + return nil, false, err + } + return n, true, json.Unmarshal(buf, dst.Addr().Interface()) + } + return d.decodeStruct(dst, src, path, n, ref) + case reflect.Map: + switch src.kind() { + case KindMap: + pairs, err := src.pairs() + if err != nil { + return nil, false, err + } + out := reflect.MakeMapWithSize(typ, len(pairs)) + n.keys = make(map[string]*dnode, len(pairs)) + for _, p := range pairs { + e := reflect.New(typ.Elem()).Elem() + en, ok, err := d.decode(e, p.value, structpath.NewStringKey(path, p.key)) + if err != nil { + return nil, false, err + } + if !ok { + continue + } + out.SetMapIndex(reflect.ValueOf(p.key).Convert(typ.Key()), e) + n.keys[p.key] = en + } + dst.Set(out) + return n, true, nil + case KindNil: + dst.SetZero() + return n, true, nil + default: + } + if ref != "" { + n.ref = ref + dst.SetZero() + return n, true, nil + } + d.typeMismatch(KindMap, src, path) + return nil, false, nil + case reflect.Slice: + switch src.kind() { + case KindSequence: + elems, err := src.elems() + if err != nil { + return nil, false, err + } + out := reflect.MakeSlice(typ, 0, len(elems)) + for _, ev := range elems { + e := reflect.New(typ.Elem()).Elem() + en, ok, err := d.decode(e, ev, structpath.NewIndex(path, out.Len())) + if err != nil { + return nil, false, err + } + if !ok { + continue + } + out = reflect.Append(out, e) + n.elems = append(n.elems, en) + } + dst.Set(out) + return n, true, nil + case KindNil: + dst.SetZero() + return n, true, nil + default: + } + if ref != "" { + n.ref = ref + dst.SetZero() + return n, true, nil + } + d.typeMismatch(KindSequence, src, path) + return nil, false, nil + case reflect.String: + s, ok := d.decodeString(src, path) + if !ok { + return nil, false, nil + } + n.kind = KindString + dst.SetString(s) + return n, true, nil + case reflect.Bool: + return d.decodeBool(dst, src, path, n, ref) + case reflect.Int, reflect.Int32, reflect.Int64: + return d.decodeInt(dst, src, path, n, ref) + case reflect.Float32, reflect.Float64: + return d.decodeFloat(dst, src, path, n, ref) + case reflect.Interface: + v, err := d.decodeAny(src, n) + if err != nil { + return nil, false, err + } + if v == nil { + dst.SetZero() + } else { + dst.Set(reflect.ValueOf(v)) + } + return n, true, nil + default: + return nil, false, fmt.Errorf("unsupported type: %s", typ.Kind()) + } +} + +// isAnchorContainer reports whether v is a YAML anchor or a non-empty sequence or map +// composed entirely of anchor containers. Anchors define reusable blocks and must not +// trigger "unknown field" warnings, including when nested inside a container. +func isAnchorContainer(v source) bool { + if v.anchor() { + return true + } + var elements []source + switch v.kind() { + case KindSequence: + elements, _ = v.elems() + case KindMap: + pairs, _ := v.pairs() + for _, p := range pairs { + elements = append(elements, p.value) + } + default: + return false + } + if len(elements) == 0 { + return false + } + for _, e := range elements { + if !isAnchorContainer(e) { + return false + } + } + return true +} + +func (d *decoder) decodeStruct(dst reflect.Value, src source, path *structpath.PathNode, n *dnode, ref string) (*dnode, bool, error) { + switch src.kind() { + case KindMap: + case KindNil: + dst.SetZero() + return n, true, nil + default: + if ref != "" { + n.ref = ref + dst.SetZero() + return n, true, nil + } + d.typeMismatch(KindMap, src, path) + return nil, false, nil + } + + pairs, err := src.pairs() + if err != nil { + return nil, false, err + } + + dst.SetZero() + info := GetStructInfo(dst.Type()) + n.keys = make(map[string]*dnode, len(pairs)) + for _, p := range pairs { + index, ok := info.Fields[p.key] + if !ok { + if isAnchorContainer(p.value) { + continue + } + // Special case: provide a more helpful message for "valueFrom" vs "value_from". + if _, hasValueFrom := info.Fields["value_from"]; p.key == "valueFrom" && hasValueFrom { + d.diags = d.diags.Append(diag.Diagnostic{ + Severity: diag.Warning, + Summary: "Use 'value_from' instead of 'valueFrom'", + Detail: "The field 'valueFrom' should be 'value_from' (snake_case). The 'valueFrom' field will be ignored.", + Locations: p.keyLocs, + Paths: []*structpath.PathNode{path}, + }) + continue + } + d.diags = d.diags.Append(diag.Diagnostic{ + Severity: diag.Warning, + Summary: "unknown field: " + p.key, + Locations: p.keyLocs, + Paths: []*structpath.PathNode{path}, + }) + continue + } + + // Decode into a fresh value so a dropped value leaves the field (and any + // embedded struct pointers on the way to it) untouched. + ft := dst.Type().FieldByIndex(index).Type + fv := reflect.New(ft).Elem() + fn, ok, err := d.decode(fv, p.value, structpath.NewStringKey(path, p.key)) + if err != nil { + return nil, false, err + } + if !ok { + continue + } + GetOrNewFieldByIndex(dst, index).Set(fv) + n.keys[p.key] = fn + + // An explicitly set zero value must still serialize: add it to ForceSendFields. + if fn.isZeroValue(fv) { + addForceSend(dst, &info, p.key) + } + } + return n, true, nil +} + +// isZeroValue mirrors dyn.Value.IsZero for the decoded value: nil, an empty map or +// sequence, or a zero scalar (a pure reference is not zero). +func (n *dnode) isZeroValue(v reflect.Value) bool { + switch n.kind { + case KindNil: + return true + case KindMap: + return len(n.keys) == 0 + case KindSequence: + return len(n.elems) == 0 + default: + } + if n.ref != "" { + return false + } + for v.Kind() == reflect.Pointer || v.Kind() == reflect.Interface { + if v.IsNil() { + return true + } + v = v.Elem() + } + if v.Kind() == reflect.Struct { + // SDK native types are strings in the configuration tree. + s, ok := sdkNativeString(v, includeZero) + return !ok || s == "" + } + return v.IsZero() +} + +func (d *decoder) decodeString(src source, path *structpath.PathNode) (string, bool) { + switch src.kind() { + case KindString, KindTime: + return src.scalar().(string), true + case KindBool: + return strconv.FormatBool(src.scalar().(bool)), true + case KindInt: + return strconv.FormatInt(toInt64(src.scalar()), 10), true + case KindFloat: + return strconv.FormatFloat(src.scalar().(float64), 'f', -1, 64), true + case KindNil: + d.nullWarning(KindString, src, path) + return "", false + default: + d.typeMismatch(KindString, src, path) + return "", false + } +} + +func toInt64(v any) int64 { + switch v := v.(type) { + case int: + return int64(v) + case int64: + return v + default: + panic(fmt.Sprintf("unexpected int type %T", v)) + } +} + +func (d *decoder) decodeBool(dst reflect.Value, src source, path *structpath.PathNode, n *dnode, ref string) (*dnode, bool, error) { + switch src.kind() { + case KindBool: + dst.SetBool(src.scalar().(bool)) + return n, true, nil + case KindString: + // See https://yaml.org/type/bool.html. + switch src.scalar().(string) { + case "true", "True", "TRUE", "y", "Y", "yes", "Yes", "YES", "on", "On", "ON": + dst.SetBool(true) + case "false", "False", "FALSE", "n", "N", "no", "No", "NO", "off", "Off", "OFF": + dst.SetBool(false) + default: + if ref != "" { + n.ref = ref + dst.SetZero() + return n, true, nil + } + d.typeMismatch(KindBool, src, path) + return nil, false, nil + } + n.kind = KindBool + return n, true, nil + case KindNil: + d.nullWarning(KindBool, src, path) + return nil, false, nil + default: + d.typeMismatch(KindBool, src, path) + return nil, false, nil + } +} + +func (d *decoder) decodeInt(dst reflect.Value, src source, path *structpath.PathNode, n *dnode, ref string) (*dnode, bool, error) { + switch src.kind() { + case KindInt: + dst.SetInt(toInt64(src.scalar())) + return n, true, nil + case KindFloat: + f := src.scalar().(float64) + out := int64(f) + if f != float64(out) { + d.warn(fmt.Sprintf(`cannot accurately represent "%g" as integer due to precision loss`, f), src, path) + return nil, false, nil + } + dst.SetInt(out) + n.kind = KindInt + return n, true, nil + case KindString: + s := src.scalar().(string) + out, err := strconv.ParseInt(s, 10, 64) + if err != nil { + if ref != "" { + n.ref = ref + dst.SetZero() + return n, true, nil + } + d.warn(fmt.Sprintf("cannot parse %q as an integer", s), src, path) + return nil, false, nil + } + dst.SetInt(out) + n.kind = KindInt + return n, true, nil + case KindNil: + d.nullWarning(KindInt, src, path) + return nil, false, nil + default: + d.typeMismatch(KindInt, src, path) + return nil, false, nil + } +} + +func (d *decoder) decodeFloat(dst reflect.Value, src source, path *structpath.PathNode, n *dnode, ref string) (*dnode, bool, error) { + switch src.kind() { + case KindFloat: + dst.SetFloat(src.scalar().(float64)) + return n, true, nil + case KindInt: + i := toInt64(src.scalar()) + out := float64(i) + if i != int64(out) { + d.warn(fmt.Sprintf(`cannot accurately represent "%d" as floating point number due to precision loss`, i), src, path) + return nil, false, nil + } + dst.SetFloat(out) + n.kind = KindFloat + return n, true, nil + case KindString: + s := src.scalar().(string) + out, err := strconv.ParseFloat(s, 64) + if err != nil { + if ref != "" { + n.ref = ref + dst.SetZero() + return n, true, nil + } + d.warn(fmt.Sprintf("cannot parse %q as a floating point number", s), src, path) + return nil, false, nil + } + dst.SetFloat(out) + n.kind = KindFloat + return n, true, nil + case KindNil: + d.nullWarning(KindFloat, src, path) + return nil, false, nil + default: + d.typeMismatch(KindFloat, src, path) + return nil, false, nil + } +} + +// decodeAny returns the generic Go value for src (maps, slices, scalars) and fills in +// the meta of everything below n. Timestamps become the string they were written as. +func (d *decoder) decodeAny(src source, n *dnode) (any, error) { + switch src.kind() { + case KindMap: + pairs, err := src.pairs() + if err != nil { + return nil, err + } + out := make(map[string]any, len(pairs)) + n.keys = make(map[string]*dnode, len(pairs)) + for _, p := range pairs { + cn := &dnode{kind: p.value.kind(), locs: p.value.locations()} + v, err := d.decodeAny(p.value, cn) + if err != nil { + return nil, err + } + out[p.key] = v + n.keys[p.key] = cn + } + return out, nil + case KindSequence: + elems, err := src.elems() + if err != nil { + return nil, err + } + out := make([]any, len(elems)) + for i, e := range elems { + cn := &dnode{kind: e.kind(), locs: e.locations()} + v, err := d.decodeAny(e, cn) + if err != nil { + return nil, err + } + out[i] = v + n.elems = append(n.elems, cn) + } + return out, nil + case KindTime: + n.kind = KindString + return src.scalar(), nil + case KindNil: + return nil, nil + default: + return src.scalar(), nil + } +} + +// viewSource is a [source] backed by a view. +type viewSource struct { + v View +} + +func (s viewSource) kind() Kind { return s.v.Kind() } +func (s viewSource) locations() []diag.Location { return s.v.Locations() } +func (s viewSource) anchor() bool { return false } + +func (s viewSource) scalar() any { + if str, ok := s.v.AsString(); ok { + return str + } + return s.v.AsAny() +} + +func (s viewSource) pairs() ([]sourcePair, error) { + var out []sourcePair + for k, c := range s.v.MapItems() { + out = append(out, sourcePair{key: k, value: viewSource{c}}) + } + return out, nil +} + +func (s viewSource) elems() ([]source, error) { + var out []source + for _, c := range s.v.Sequence() { + out = append(out, viewSource{c}) + } + return out, nil +} + +// mappedSource is a [source] whose locations are mapped by fn. +type mappedSource struct { + source + path *structpath.PathNode + fn LocationMapper +} + +func (s mappedSource) locations() []diag.Location { + return s.fn(s.path, s.source.locations()) +} + +func (s mappedSource) pairs() ([]sourcePair, error) { + pairs, err := s.source.pairs() + for i := range pairs { + pairs[i].value = mappedSource{source: pairs[i].value, path: structpath.NewStringKey(s.path, pairs[i].key), fn: s.fn} + } + return pairs, err +} + +func (s mappedSource) elems() ([]source, error) { + elems, err := s.source.elems() + for i := range elems { + elems[i] = mappedSource{source: elems[i], path: structpath.NewIndex(s.path, i), fn: s.fn} + } + return elems, err +} + +// kindOf returns the kind of configuration tree values of type t. +func kindOf(t reflect.Type) Kind { + for t.Kind() == reflect.Pointer { + t = t.Elem() + } + switch t.Kind() { + case reflect.Struct, reflect.Map: + return KindMap + case reflect.Slice: + return KindSequence + case reflect.Bool: + return KindBool + case reflect.Int, reflect.Int32, reflect.Int64: + return KindInt + case reflect.Float32, reflect.Float64: + return KindFloat + default: + return KindString + } +} diff --git a/libs/structs/structvar/decode_json.go b/libs/structs/structvar/decode_json.go new file mode 100644 index 00000000000..c0b91b3699b --- /dev/null +++ b/libs/structs/structvar/decode_json.go @@ -0,0 +1,126 @@ +package structvar + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "slices" + "strconv" + + "github.com/databricks/cli/libs/diag" + "go.yaml.in/yaml/v3" +) + +// ParseJSON parses data into a YAML node tree that carries the positions of the values +// in data: the start of an object or array, the first character of a key and the end of +// a scalar (the decoder does not report where a scalar starts). +func ParseJSON(source string, data []byte) (*yaml.Node, error) { + decoder := json.NewDecoder(bytes.NewReader(data)) + // Use json.Number to avoid losing precision on int64 values above 2^53 (e.g. job and run IDs). + decoder.UseNumber() + + lines := newLineIndex(data) + node, err := decodeJSONValue(decoder, lines) + if err == nil && decoder.More() { + err = errors.New("unexpected additional content") + } + if err != nil { + if errors.Is(err, io.EOF) { + err = errors.New("unexpected end of JSON input") + } + line, column := lines.position(decoder.InputOffset()) + return nil, fmt.Errorf("error decoding JSON at %s: %v", diag.Location{File: source, Line: line, Column: column}, err) + } + return node, nil +} + +// lineIndex holds the offsets at which the lines of the input start. +type lineIndex []int64 + +func newLineIndex(data []byte) lineIndex { + lines := lineIndex{0} + for i, b := range data { + if b == '\n' { + lines = append(lines, int64(i+1)) + } + } + return lines +} + +// position returns the 1-based line and column of offset. +func (l lineIndex) position(offset int64) (line, column int) { + // The line is the last one starting at or before offset. + i, found := slices.BinarySearch(l, offset) + if !found { + i-- + } + return i + 1, int(offset-l[i]) + 1 +} + +func newNode(lines lineIndex, offset int64, kind yaml.Kind, tag, value string) *yaml.Node { + line, column := lines.position(offset) + return &yaml.Node{Kind: kind, Tag: tag, Value: value, Line: line, Column: column} +} + +func decodeJSONValue(decoder *json.Decoder, lines lineIndex) (*yaml.Node, error) { + token, err := decoder.Token() + if err != nil { + return nil, err + } + offset := decoder.InputOffset() + + switch tok := token.(type) { + case json.Delim: + if tok == '{' { + node := newNode(lines, offset-1, yaml.MappingNode, "!!map", "") + for decoder.More() { + keyToken, err := decoder.Token() + if err != nil { + return nil, err + } + key, ok := keyToken.(string) + if !ok { + return nil, errors.New("expected string for object key") + } + // The decoder reports the end of the key: step back over the key and the closing quote. + keyNode := newNode(lines, decoder.InputOffset()-int64(len(key)+1), yaml.ScalarNode, "!!str", key) + value, err := decodeJSONValue(decoder, lines) + if err != nil { + return nil, err + } + node.Content = append(node.Content, keyNode, value) + } + // Consume the closing '}'. + _, err := decoder.Token() + return node, err + } + node := newNode(lines, offset-1, yaml.SequenceNode, "!!seq", "") + for decoder.More() { + value, err := decodeJSONValue(decoder, lines) + if err != nil { + return nil, err + } + node.Content = append(node.Content, value) + } + // Consume the closing ']'. + _, err := decoder.Token() + return node, err + case json.Number: + if _, err := tok.Int64(); err == nil { + return newNode(lines, offset, yaml.ScalarNode, "!!int", tok.String()), nil + } + // Integers that overflow int64 fall back to float64. + if _, err := tok.Float64(); err != nil { + return nil, fmt.Errorf("invalid number %q: %w", tok.String(), err) + } + return newNode(lines, offset, yaml.ScalarNode, "!!float", tok.String()), nil + case string: + return newNode(lines, offset, yaml.ScalarNode, "!!str", tok), nil + case bool: + return newNode(lines, offset, yaml.ScalarNode, "!!bool", strconv.FormatBool(tok)), nil + default: + return newNode(lines, offset, yaml.ScalarNode, "!!null", "null"), nil + } +} diff --git a/libs/structs/structvar/decode_yaml.go b/libs/structs/structvar/decode_yaml.go new file mode 100644 index 00000000000..ceb0634918c --- /dev/null +++ b/libs/structs/structvar/decode_yaml.go @@ -0,0 +1,327 @@ +package structvar + +import ( + "fmt" + "io" + "math" + "strconv" + "strings" + "time" + + "github.com/databricks/cli/libs/diag" + "go.yaml.in/yaml/v3" +) + +// LocationError is a YAML syntax error with the location it occurred at. +type LocationError struct { + Loc diag.Location + Summary string +} + +func (e *LocationError) Error() string { + return fmt.Sprintf("yaml (%s): %s", e.Loc, e.Summary) +} + +// yamlSource is a [source] backed by a YAML node. Aliases are followed, merge keys +// are applied, and scalars are resolved by their tag. +type yamlSource struct { + file string + node *yaml.Node + + // The node the value comes from, after following aliases. + target *yaml.Node + + // Aliases followed on the way from the document root to this value, to detect + // cyclic anchors. + aliases *aliasChain + + // Resolved scalar. + k Kind + v any +} + +// ParseYAML parses the YAML document in r. It returns nil for an empty document. +func ParseYAML(r io.Reader) (*yaml.Node, error) { + var doc yaml.Node + err := yaml.NewDecoder(r).Decode(&doc) + if err == io.EOF { + return nil, nil + } + if err != nil { + return nil, err + } + return doc.Content[0], nil +} + +// aliasChain is an immutable list of alias nodes. +type aliasChain struct { + node *yaml.Node + prev *aliasChain +} + +func (c *aliasChain) contains(n *yaml.Node) bool { + for ; c != nil; c = c.prev { + if c.node == n { + return true + } + } + return false +} + +func newYAMLSource(file string, node *yaml.Node, aliases *aliasChain) (*yamlSource, error) { + s := &yamlSource{file: file, node: node, target: node, aliases: aliases} + loc := s.loc(node) + for s.target.Kind == yaml.AliasNode { + // The same alias may be reached again through another path, which is not a + // cycle; only an alias inside its own expansion is. + if s.aliases.contains(s.target) { + return nil, yamlErrorf(loc, "cyclic reference to anchor %q", s.target.Value) + } + s.aliases = &aliasChain{node: s.target, prev: s.aliases} + s.target = s.target.Alias + } + + switch s.target.Kind { + case yaml.MappingNode: + s.k = KindMap + case yaml.SequenceNode: + s.k = KindSequence + case yaml.ScalarNode: + if err := s.resolveScalar(loc); err != nil { + return nil, err + } + default: + return nil, yamlErrorf(loc, "unknown node kind: %v", s.target.Kind) + } + return s, nil +} + +func yamlErrorf(loc diag.Location, format string, args ...any) error { + return fmt.Errorf("yaml (%s): %s", loc, fmt.Sprintf(format, args...)) +} + +func (s *yamlSource) loc(n *yaml.Node) diag.Location { + return diag.Location{File: s.file, Line: n.Line, Column: n.Column} +} + +func intValue(i64 int64) any { + // Use regular int type instead of int64 if possible. + if i64 >= math.MinInt32 && i64 <= math.MaxInt32 { + return int(i64) + } + return i64 +} + +func (s *yamlSource) resolveScalar(loc diag.Location) error { + n := s.target + switch st := n.ShortTag(); st { + case "!!str": + s.k, s.v = KindString, n.Value + case "!!bool": + switch strings.ToLower(n.Value) { + case "true": + s.k, s.v = KindBool, true + case "false": + s.k, s.v = KindBool, false + default: + return yamlErrorf(loc, "invalid bool value: %v", n.Value) + } + case "!!int": + // Try to parse the integer value in base 10. Trim leading zeros to avoid + // octal parsing of the "0" prefix (YAML 1.2 spec example 2.19). + i64, err := strconv.ParseInt(strings.TrimLeft(n.Value, "0"), 10, 64) + if err != nil { + // Let ParseInt figure out the base. + i64, err = strconv.ParseInt(n.Value, 0, 64) + } + if err != nil { + return yamlErrorf(loc, "invalid int value: %v", n.Value) + } + s.k, s.v = KindInt, intValue(i64) + case "!!float": + f64, err := strconv.ParseFloat(n.Value, 64) + if err == nil { + s.k, s.v = KindFloat, f64 + break + } + // Deal with infinity prefixes. + v := strings.ToLower(n.Value) + switch { + case strings.HasPrefix(v, "+"): + v = strings.TrimPrefix(v, "+") + f64 = math.Inf(1) + case strings.HasPrefix(v, "-"): + v = strings.TrimPrefix(v, "-") + f64 = math.Inf(-1) + default: + f64 = math.Inf(1) + } + switch v { + case ".inf": + s.k, s.v = KindFloat, f64 + case ".nan": + s.k, s.v = KindFloat, math.NaN() + default: + return yamlErrorf(loc, "invalid float value: %v", n.Value) + } + case "!!null": + s.k = KindNil + case "!!timestamp": + if !isTimestamp(n.Value) { + return yamlErrorf(loc, "invalid timestamp value: %v", n.Value) + } + // Timestamps are kept as the string they were written as. + s.k, s.v = KindTime, n.Value + default: + return yamlErrorf(loc, "unknown tag: %v", st) + } + return nil +} + +func isTimestamp(s string) bool { + for _, layout := range []string{ + "2006-1-2T15:4:5.999999999Z07:00", // RCF3339Nano with short date fields. + "2006-1-2t15:4:5.999999999Z07:00", // RFC3339Nano with short date fields and lower-case "t". + "2006-1-2 15:4:5.999999999", // space separated with no time zone + "2006-1-2", // date only + } { + if _, err := time.Parse(layout, s); err == nil { + return true + } + } + return false +} + +func (s *yamlSource) kind() Kind { + return s.k +} + +func (s *yamlSource) locations() []diag.Location { + // An alias has the location of the anchored value. + return []diag.Location{s.loc(s.target)} +} + +func (s *yamlSource) anchor() bool { + return s.node.Anchor != "" || s.target.Anchor != "" +} + +func (s *yamlSource) scalar() any { + return s.v +} + +func (s *yamlSource) child(n *yaml.Node) (source, error) { + return newYAMLSource(s.file, n, s.aliases) +} + +func (s *yamlSource) pairs() ([]sourcePair, error) { + node := s.target + loc := s.loc(node) + var merge *yaml.Node + var acc pairSet + for i := 0; i < len(node.Content); i += 2 { + key := node.Content[i] + val := node.Content[i+1] + + if key.Kind != yaml.ScalarNode { + return nil, yamlErrorf(loc, "key is not a scalar") + } + + switch st := key.ShortTag(); st { + case "!!str": + case "!!null": + // A literal unquoted "null" is treated as a null value by the YAML parser. + // However, when used as a key, it is treated as the string "null". + case "!!merge": + if merge != nil { + // The YAML merge key spec allows a single '<<' key per mapping. + return nil, &LocationError{ + Loc: s.loc(key), + Summary: "duplicate YAML merge key ('<<') is not allowed; to merge multiple maps, use a sequence: '<<: [*anchor1, *anchor2]'", + } + } + merge = val + continue + default: + return nil, yamlErrorf(loc, "invalid key tag: %v", st) + } + + v, err := s.child(val) + if err != nil { + return nil, err + } + acc.set(sourcePair{key: key.Value, keyLocs: []diag.Location{s.loc(key)}, value: v}) + } + + if merge == nil { + return acc.pairs, nil + } + + mloc := s.loc(merge) + merr := yamlErrorf(mloc, "map merge requires map or sequence of maps as the value") + + var mnodes []*yaml.Node + switch merge.Kind { + case yaml.SequenceNode: + mnodes = merge.Content + case yaml.AliasNode: + mnodes = []*yaml.Node{merge} + default: + return nil, merr + } + + // Merged maps come first; the entries of this mapping take precedence. + var out pairSet + for _, n := range mnodes { + ms, err := s.child(n) + if err != nil { + return nil, err + } + if ms.kind() != KindMap { + return nil, merr + } + mps, err := ms.pairs() + if err != nil { + return nil, err + } + for _, p := range mps { + out.set(p) + } + } + for _, p := range acc.pairs { + out.set(p) + } + return out.pairs, nil +} + +// setPair sets the pair in pairs: an existing key keeps its position and key location. +// pairSet is a list of pairs with unique keys, in the order the keys were first set. +type pairSet struct { + pairs []sourcePair + index map[string]int +} + +// set adds p, or replaces the value of the pair with the same key (keeping its key +// location and position). +func (s *pairSet) set(p sourcePair) { + if i, ok := s.index[p.key]; ok { + s.pairs[i].value = p.value + return + } + if s.index == nil { + s.index = map[string]int{} + } + s.index[p.key] = len(s.pairs) + s.pairs = append(s.pairs, p) +} + +func (s *yamlSource) elems() ([]source, error) { + var out []source + for _, n := range s.target.Content { + v, err := s.child(n) + if err != nil { + return nil, err + } + out = append(out, v) + } + return out, nil +} diff --git a/libs/dyn/kind.go b/libs/structs/structvar/kind.go similarity index 54% rename from libs/dyn/kind.go rename to libs/structs/structvar/kind.go index 1890e9e2ccd..a0775f98afe 100644 --- a/libs/dyn/kind.go +++ b/libs/structs/structvar/kind.go @@ -1,9 +1,8 @@ -package dyn +package structvar -import ( - "fmt" -) +import "fmt" +// Kind is the kind of a value in the configuration tree. type Kind int const ( @@ -19,31 +18,6 @@ const ( KindNil ) -func kindOf(v any) Kind { - switch v.(type) { - case Mapping: - return KindMap - case []Value: - return KindSequence - case string: - return KindString - case bool: - return KindBool - case int, int8, int16, int32, int64: - return KindInt - case uint, uint8, uint16, uint32, uint64: - return KindInt - case float32, float64: - return KindFloat - case Time: - return KindTime - case nil: - return KindNil - default: - panic(fmt.Sprintf("not handled: %T", v)) - } -} - func (k Kind) String() string { switch k { case KindInvalid: diff --git a/libs/structs/structvar/locations.go b/libs/structs/structvar/locations.go new file mode 100644 index 00000000000..733c9b34df6 --- /dev/null +++ b/libs/structs/structvar/locations.go @@ -0,0 +1,262 @@ +package structvar + +import ( + "maps" + "slices" + + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/structs/structpath" +) + +// Locations records the source locations of the values of a typed value, as a tree +// that follows the value: one node per map key or sequence index. +// +// Locations is immutable: operations that change it return a new tree that shares +// unchanged subtrees, so copies of a struct holding one can share it safely. +// A nil *Locations is an empty tree. +type Locations struct { + locs []diag.Location + keys map[string]*Locations + elems []*Locations +} + +// pathComponent is a key or an index of a path. +type pathComponent struct { + key string + index int + isKey bool +} + +// components returns the keys and indices of p. +func components(p *structpath.PathNode) []pathComponent { + var out []pathComponent + for _, n := range p.AsSlice() { + if k, ok := n.StringKey(); ok { + out = append(out, pathComponent{key: k, isKey: true}) + } else if i, ok := n.Index(); ok { + out = append(out, pathComponent{index: i}) + } + } + return out +} + +func (l *Locations) child(c pathComponent) *Locations { + if l == nil { + return nil + } + if c.isKey { + return l.keys[c.key] + } + if c.index < 0 || c.index >= len(l.elems) { + return nil + } + return l.elems[c.index] +} + +// Key returns the locations of map key or struct field k. +func (l *Locations) Key(k string) *Locations { + return l.child(pathComponent{key: k, isKey: true}) +} + +// Index returns the locations of sequence element i. +func (l *Locations) Index(i int) *Locations { + return l.child(pathComponent{index: i}) +} + +// Get returns the locations recorded for the value itself. +func (l *Locations) Get() []diag.Location { + if l == nil { + return nil + } + return slices.Clone(l.locs) +} + +// At returns the locations of the value at path. Values without recorded locations +// (e.g. set by code) have none. +func (l *Locations) At(path *structpath.PathNode) []diag.Location { + return l.lookup(components(path)).Get() +} + +// Nearest returns the locations of the value at path or, if it has none, of the +// closest ancestor that has some. Use it to point diagnostics at the most specific +// place there is, not to decide where a value was defined. +func (l *Locations) Nearest(path *structpath.PathNode) []diag.Location { + var nearest []diag.Location + n := l + for _, c := range components(path) { + if n == nil { + break + } + if len(n.locs) > 0 { + nearest = n.locs + } + n = n.child(c) + } + if n != nil && len(n.locs) > 0 { + nearest = n.locs + } + return slices.Clone(nearest) +} + +func (l *Locations) lookup(path []pathComponent) *Locations { + n := l + for _, c := range path { + n = n.child(c) + if n == nil { + return nil + } + } + return n +} + +// Sub returns the locations of the value at path and below. +func (l *Locations) Sub(path *structpath.PathNode) *Locations { + return l.lookup(components(path)) +} + +// With returns a copy of l where the locations of the value at path and below are sub. +func (l *Locations) With(path *structpath.PathNode, sub *Locations) *Locations { + return l.with(components(path), sub) +} + +func (l *Locations) with(path []pathComponent, sub *Locations) *Locations { + if len(path) == 0 { + return sub + } + child := l.child(path[0]).with(path[1:], sub) + return l.withChild(path[0], child) +} + +func (l *Locations) clone() *Locations { + if l == nil { + return &Locations{} + } + c := *l + c.keys = maps.Clone(l.keys) + c.elems = slices.Clone(l.elems) + return &c +} + +func (l *Locations) withChild(c pathComponent, child *Locations) *Locations { + out := l.clone() + out.setChild(c, child) + return out +} + +// setChild sets the child c of l in place. Only use it on a node the caller owns, +// e.g. one just returned by [Locations.WithLocations], which builds a fresh node. +func (l *Locations) setChild(c pathComponent, child *Locations) { + if c.isKey { + if child == nil { + delete(l.keys, c.key) + return + } + if l.keys == nil { + l.keys = map[string]*Locations{} + } + l.keys[c.key] = child + return + } + if c.index < 0 { + return + } + for len(l.elems) <= c.index { + l.elems = append(l.elems, nil) + } + l.elems[c.index] = child +} + +// WithLocations returns a copy of l where the value itself has the locations locs. +func (l *Locations) WithLocations(locs []diag.Location) *Locations { + c := l.clone() + c.locs = slices.Clone(locs) + return c +} + +// WithSequence returns a copy of l where the elements of the sequence at path are +// rebuilt from the old ones: new element i has the locations of old elements sources[i] +// merged (see [MergeLocations]). Use it after reordering, merging, splitting or +// dropping sequence elements. +func (l *Locations) WithSequence(path *structpath.PathNode, sources [][]int) *Locations { + p := components(path) + n := l.lookup(p) + if n == nil { + return l + } + c := n.clone() + c.elems = make([]*Locations, len(sources)) + for i, src := range sources { + var merged *Locations + for j, k := range src { + if j == 0 { + merged = n.Index(k) + continue + } + merged = MergeLocations(merged, n.Index(k)) + } + c.elems[i] = merged + } + return l.with(p, c) +} + +// MergeLocations returns the locations of the value obtained by merging the value +// described by b into the one described by a, following merge.Merge: maps and +// sequences keep a's locations and accumulate b's, other values take b's and +// accumulate a's. +func MergeLocations(a, b *Locations) *Locations { + switch { + case a == nil: + return b + case b == nil: + return a + } + isComposite := len(a.keys) > 0 || len(a.elems) > 0 || len(b.keys) > 0 || len(b.elems) > 0 + if !isComposite { + return b.WithLocations(concatLocations(b.locs, a.locs)) + } + c := a.clone() + c.locs = concatLocations(a.locs, b.locs) + for k, bv := range b.keys { + if c.keys == nil { + c.keys = map[string]*Locations{} + } + c.keys[k] = MergeLocations(a.keys[k], bv) + } + c.elems = append(slices.Clone(a.elems), b.elems...) + return c +} + +func concatLocations(a, b []diag.Location) []diag.Location { + if len(b) == 0 { + return a + } + return append(slices.Clone(a), b...) +} + +// Map returns a copy of l where the locations of every value are replaced by fn(path, locations). +func (l *Locations) Map(fn func(path *structpath.PathNode, locs []diag.Location) []diag.Location) *Locations { + return l.mapLocations(nil, fn) +} + +func (l *Locations) mapLocations(path *structpath.PathNode, fn func(*structpath.PathNode, []diag.Location) []diag.Location) *Locations { + if l == nil { + return nil + } + c := l.clone() + c.locs = fn(path, slices.Clone(l.locs)) + for k, v := range c.keys { + c.keys[k] = v.mapLocations(structpath.NewStringKey(path, k), fn) + } + for i, v := range c.elems { + c.elems[i] = v.mapLocations(structpath.NewIndex(path, i), fn) + } + return c +} + +// child returns the path of this component below prefix. +func (c pathComponent) child(prefix *structpath.PathNode) *structpath.PathNode { + if c.isKey { + return structpath.NewStringKey(prefix, c.key) + } + return structpath.NewIndex(prefix, c.index) +} diff --git a/libs/structs/structvar/lookup.go b/libs/structs/structvar/lookup.go new file mode 100644 index 00000000000..b4f3e3f1465 --- /dev/null +++ b/libs/structs/structvar/lookup.go @@ -0,0 +1,157 @@ +package structvar + +import ( + "fmt" + "reflect" + "slices" + + "github.com/databricks/cli/libs/structs/structpath" +) + +// LookupError is returned by [View.LookupWithDefaults] when the path cannot be followed. +type LookupError struct { + msg string +} + +func (e *LookupError) Error() string { + return e.msg +} + +// KeyNotFoundError is returned by [View.LookupWithDefaults] for a missing key. +type KeyNotFoundError struct { + Path *structpath.PathNode + Suggestions []string +} + +func (e *KeyNotFoundError) Error() string { + return fmt.Sprintf("key not found at %q%s", e.Path, DidYouMean(e.Suggestions)) +} + +func indexedKind(c pathComponent) string { + if c.isKey { + return "map" + } + return "sequence" +} + +// LookupWithDefaults returns the value at path. Fields that are declared in the type +// but not set resolve to their zero value: e.g. ${bundle.git.origin_url} resolves to an +// empty string if a bundle isn't located in a Git repository (yet). +func (x View) LookupWithDefaults(path *structpath.PathNode) (View, error) { + var p *structpath.PathNode + for _, c := range components(path) { + p = c.child(p) + kind := x.Kind() + if c.isKey { + switch kind { + case KindMap: + case KindNil: + return View{}, &LookupError{fmt.Sprintf("expected a %s to index %q, found nil", indexedKind(c), p)} + default: + return View{}, &LookupError{fmt.Sprintf("expected a map to index %q, found %s", p, kind)} + } + child := x.Get(c.key) + if !child.IsValid() { + zero, ok := zeroField(x, c.key) + if !ok { + return View{}, &KeyNotFoundError{Path: p, Suggestions: SuggestKeys(lookupKeys(x), c.key)} + } + child = zero + } + x = child + continue + } + + switch kind { + case KindSequence: + case KindNil: + return View{}, &LookupError{fmt.Sprintf("expected a %s to index %q, found nil", indexedKind(c), p)} + default: + return View{}, &LookupError{fmt.Sprintf("expected a sequence to index %q, found %s", p, kind)} + } + child := x.Index(c.index) + if !child.IsValid() { + return View{}, &LookupError{fmt.Sprintf("index out of bounds at %q", p)} + } + x = child + } + return x, nil +} + +// structType returns the struct type of the value, if it is a struct. +func structType(x View) (reflect.Type, bool) { + t := x.Reflect().Type() + for t.Kind() == reflect.Pointer || t.Kind() == reflect.Interface { + if t.Kind() == reflect.Interface { + v := x.Reflect() + for v.Kind() == reflect.Pointer || v.Kind() == reflect.Interface { + if v.IsNil() { + return nil, false + } + v = v.Elem() + } + t = v.Type() + continue + } + t = t.Elem() + } + return t, t.Kind() == reflect.Struct && !IsSDKNativeType(t) +} + +// fillableFieldType returns the type of the struct field with JSON name k, if a field of +// that type is given a zero value when it is not set (see convert.IncludeMissingFields). +func fillableFieldType(t reflect.Type, k string) (reflect.Type, bool) { + info := GetStructInfo(t) + index, ok := info.Fields[k] + if !ok { + return nil, false + } + ft := t.FieldByIndex(index).Type + for ft.Kind() == reflect.Pointer { + ft = ft.Elem() + } + switch ft.Kind() { + case reflect.Struct, reflect.Map, reflect.Slice, reflect.String, reflect.Bool, + reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Float32, reflect.Float64: + return ft, true + default: + return nil, false + } +} + +// zeroField returns the zero value of the struct field k of x that is not set. +func zeroField(x View, k string) (View, bool) { + t, ok := structType(x) + if !ok { + return View{}, false + } + ft, ok := fillableFieldType(t, k) + if !ok { + return View{}, false + } + zero := reflect.New(ft) + switch ft.Kind() { + case reflect.Map: + zero.Elem().Set(reflect.MakeMap(ft)) + case reflect.Slice: + zero.Elem().Set(reflect.MakeSlice(ft, 0, 0)) + default: + } + return NewView(zero.Interface(), nil, nil), true +} + +// lookupKeys returns the keys a lookup in x can find, for "did you mean" suggestions. +func lookupKeys(x View) []string { + var keys []string + for k := range x.MapItems() { + keys = append(keys, k) + } + if t, ok := structType(x); ok { + for _, k := range GetStructInfo(t).FieldNames { + if _, ok := fillableFieldType(t, k); ok && !slices.Contains(keys, k) { + keys = append(keys, k) + } + } + } + return keys +} diff --git a/libs/structs/structvar/merge.go b/libs/structs/structvar/merge.go new file mode 100644 index 00000000000..8aab3370149 --- /dev/null +++ b/libs/structs/structvar/merge.go @@ -0,0 +1,213 @@ +package structvar + +import ( + "fmt" + "maps" + "reflect" + "slices" + + "github.com/databricks/cli/libs/structs/structpath" +) + +// Merge merges the value described by src into the value at path, with the semantics +// of merge.Merge: maps are merged recursively, sequences are concatenated, primitive +// values are replaced, and a nil value on either side yields the other one. +// Locations accumulate like merge.Merge accumulates them. A failed merge changes nothing. +func (sv *StructVar) Merge(path *structpath.PathNode, src View) error { + dstView := sv.View().Lookup(path) + if err := CheckMerge(dstView, src); err != nil { + return err + } + + m := merger{refs: maps.Clone(sv.Refs)} + var locs *Locations + err := update(reflect.ValueOf(sv.Value), components(path), func(dst reflect.Value) (bool, error) { + dstView.v = dst + var err error + locs, err = m.merge(dst, dstView, src, path) + return isZeroScalar(src), err + }) + if err != nil { + return err + } + sv.Refs = m.refs + sv.Locations = sv.Locations.With(path, locs) + return nil +} + +// isZeroScalar reports whether src is a present zero scalar, which must be kept as an +// explicitly set value. +func isZeroScalar(src View) bool { + k := src.Kind() + return k != KindNil && k != KindMap && k != KindSequence && src.isZero() +} + +// CheckMerge returns the error [StructVar.Merge] would return for merging src into dst. +func CheckMerge(dst, src View) error { + ak, bk := dst.Kind(), src.Kind() + if ak == KindInvalid || ak == KindNil || bk == KindInvalid || bk == KindNil { + return nil + } + switch ak { + case KindMap: + if bk != KindMap { + return fmt.Errorf("cannot merge map with %s", bk) + } + for k, sc := range src.MapItems() { + if err := CheckMerge(dst.Get(k), sc); err != nil { + return err + } + } + return nil + case KindSequence: + if bk != KindSequence { + return fmt.Errorf("cannot merge sequence with %s", bk) + } + return nil + default: + if ak != bk { + return fmt.Errorf("cannot merge %s with %s", ak, bk) + } + return nil + } +} + +type merger struct { + refs map[string]string +} + +// assign sets dst to src (see [StructVar.Assign]) and returns its locations. +func (m *merger) assign(dst reflect.Value, src View, path *structpath.PathNode) (*Locations, error) { + var d decoder + n, ok, err := d.decode(dst, viewSource{src}, path) + if err != nil { + return nil, err + } + if !ok { + dst.SetZero() + } + m.refs = n.refs(path, withoutRefs(m.refs, path)) + return n.locations(), nil +} + +// merge merges src into dst (settable; dstView is its view, invalid if absent). +func (m *merger) merge(dst reflect.Value, dstView, src View, path *structpath.PathNode) (*Locations, error) { + ak, bk := dstView.Kind(), src.Kind() + switch { + case bk == KindInvalid: + return dstView.loc, nil + case ak == KindInvalid: + return m.assign(dst, src, path) + case ak == KindNil: + locs, err := m.assign(dst, src, path) + return locs.WithLocations(concatLocations(src.loc.Get(), dstView.loc.Get())), err + case bk == KindNil: + return dstView.loc.WithLocations(concatLocations(dstView.loc.Get(), src.loc.Get())), nil + } + + switch ak { + case KindMap: + out := dstView.loc.WithLocations(concatLocations(dstView.loc.Get(), src.loc.Get())) + v := derefAlloc(dst) + for k, sc := range src.MapItems() { + c := pathComponent{key: k, isKey: true} + cp := structpath.NewStringKey(path, k) + var locs *Locations + err := updateChild(v, c, func(child reflect.Value) (bool, error) { + childView := dstView.Get(k) + childView.v = child + var err error + locs, err = m.merge(child, childView, sc, cp) + return isZeroScalar(sc), err + }) + if err != nil { + return nil, err + } + out.setChild(c, locs) + } + return out, nil + case KindSequence: + out := dstView.loc.WithLocations(concatLocations(dstView.loc.Get(), src.loc.Get())) + v := derefAlloc(dst) + holder := v + if v.Kind() == reflect.Interface { + // A sequence held in an interface (e.g. a complex variable default) is + // appended to as a copy and stored back. + v = reflect.New(holder.Elem().Type()).Elem() + v.Set(holder.Elem()) + } + for _, sc := range src.Sequence() { + i := v.Len() + e := reflect.New(v.Type().Elem()).Elem() + locs, err := m.assign(e, sc, structpath.NewIndex(path, i)) + if err != nil { + return nil, err + } + v.Set(reflect.Append(v, e)) + out.setChild(pathComponent{index: i}, locs) + } + if holder.Kind() == reflect.Interface { + holder.Set(v) + } + return out, nil + default: + locs, err := m.assign(dst, src, path) + return locs.WithLocations(concatLocations(src.loc.Get(), dstView.loc.Get())), err + } +} + +// MergeElementsByKey merges the elements of the sequence at path that have the same +// key, like merge.ElementsByKey: elements are merged into the first one with the same +// key (in order), the key field of every element is set to its key, and the keys are +// sorted if sortKeys is set. +func (sv *StructVar) MergeElementsByKey(path *structpath.PathNode, keyField string, keyFn func(View) string, sortKeys bool) error { + seq := sv.View().Lookup(path) + if seq.Kind() != KindSequence { + return nil + } + + var keys []string + groups := map[string][]int{} + for i, e := range seq.Sequence() { + k := keyFn(e.Get(keyField)) + if _, ok := groups[k]; !ok { + keys = append(keys, k) + } + groups[k] = append(groups[k], i) + } + if len(keys) == 0 { + return nil + } + if sortKeys { + slices.Sort(keys) + } + + // Build the merged sequence separately: the first element of each group is + // assigned, the others are merged into it. + sliceType := seq.Reflect().Type() + for sliceType.Kind() == reflect.Pointer { + sliceType = sliceType.Elem() + } + tmp := &StructVar{Value: reflect.New(sliceType).Interface()} + for i, k := range keys { + elem := structpath.NewIndex(nil, i) + for j, idx := range groups[k] { + var err error + if j == 0 { + _, err = tmp.Assign(elem, seq.Index(idx)) + } else { + err = tmp.Merge(elem, seq.Index(idx)) + } + if err != nil { + return err + } + } + // The key field holds the key (e.g. a normalized one), keeping its location. + if err := tmp.Set(structpath.NewStringKey(elem, keyField), k); err != nil { + return err + } + } + tmp.Locations = tmp.Locations.WithLocations(seq.loc.Get()) + _, err := sv.Assign(path, tmp.View()) + return err +} diff --git a/libs/structs/structvar/override.go b/libs/structs/structvar/override.go new file mode 100644 index 00000000000..f44b5e08f5c --- /dev/null +++ b/libs/structs/structvar/override.go @@ -0,0 +1,163 @@ +package structvar + +import ( + "errors" + "fmt" + "reflect" + + "github.com/databricks/cli/libs/structs/structpath" +) + +// ErrOverrideUndoDelete can be returned by [OverrideVisitor.VisitDelete] to keep the +// deleted value. +var ErrOverrideUndoDelete = errors.New("undo delete operation") + +// OverrideVisitor is notified of the differences found by [PlanOverride]. +// Any error aborts the override. +type OverrideVisitor struct { + VisitDelete func(path *structpath.PathNode, left View) error + VisitInsert func(path *structpath.PathNode, right View) error + VisitUpdate func(path *structpath.PathNode, left, right View) error +} + +// OverridePlan is the result of [PlanOverride]; apply it with [StructVar.Override]. +type OverridePlan struct { + src View + locs *Locations + undos []undo +} + +type undo struct { + path *structpath.PathNode + value *StructVar +} + +// PlanOverride compares dst with src like merge.Override and reports the differences +// to the visitor. Applying the plan replaces dst with src, except that values unchanged +// in src keep their locations in dst, and values whose deletion the visitor undid are kept. +func PlanOverride(dst, src View, visitor OverrideVisitor) (*OverridePlan, error) { + plan := &OverridePlan{src: src} + locs, err := plan.override(nil, dst, src, visitor) + if err != nil { + return nil, err + } + plan.locs = locs + return plan, nil +} + +func (p *OverridePlan) override(path *structpath.PathNode, left, right View, visitor OverrideVisitor) (*Locations, error) { + lk, rk := left.Kind(), right.Kind() + if lk != rk { + return right.loc, visitor.VisitUpdate(path, left, right) + } + + switch lk { + case KindMap: + out := (*Locations)(nil).WithLocations(left.loc.Get()) + for k, lv := range left.MapItems() { + if right.Get(k).IsValid() { + continue + } + kp := structpath.NewStringKey(path, k) + err := visitor.VisitDelete(kp, lv) + if errors.Is(err, ErrOverrideUndoDelete) { + p.keep(kp, lv) + out.setChild(pathComponent{key: k, isKey: true}, lv.loc) + } else if err != nil { + return nil, err + } + } + for k, rv := range right.MapItems() { + kp := structpath.NewStringKey(path, k) + locs := rv.loc + if lv := left.Get(k); lv.IsValid() { + var err error + locs, err = p.override(kp, lv, rv, visitor) + if err != nil { + return nil, err + } + } else if err := visitor.VisitInsert(kp, rv); err != nil { + return nil, err + } + out.setChild(pathComponent{key: k, isKey: true}, locs) + } + return out, nil + case KindSequence: + out := (*Locations)(nil).WithLocations(left.loc.Get()) + var ls, rs []View + for _, v := range left.Sequence() { + ls = append(ls, v) + } + for _, v := range right.Sequence() { + rs = append(rs, v) + } + n := 0 + for i := range min(len(ls), len(rs)) { + locs, err := p.override(structpath.NewIndex(path, i), ls[i], rs[i], visitor) + if err != nil { + return nil, err + } + out.setChild(pathComponent{index: n}, locs) + n++ + } + for i := len(ls); i < len(rs); i++ { + if err := visitor.VisitInsert(structpath.NewIndex(path, i), rs[i]); err != nil { + return nil, err + } + out.setChild(pathComponent{index: n}, rs[i].loc) + n++ + } + for i := len(rs); i < len(ls); i++ { + err := visitor.VisitDelete(structpath.NewIndex(path, i), ls[i]) + if errors.Is(err, ErrOverrideUndoDelete) { + p.keep(structpath.NewIndex(path, n), ls[i]) + out.setChild(pathComponent{index: n}, ls[i].loc) + n++ + } else if err != nil { + return nil, err + } + } + return out, nil + case KindNil: + return left.loc, nil + case KindString, KindBool, KindInt, KindFloat: + if left.AsAny() == right.AsAny() || sameNumber(left, right) { + return left.loc, nil + } + return right.loc, visitor.VisitUpdate(path, left, right) + default: + return nil, fmt.Errorf("unexpected kind %s at %s", lk, path) + } +} + +func sameNumber(a, b View) bool { + if ai, ok := a.AsInt(); ok { + bi, ok := b.AsInt() + return ok && ai == bi + } + return false +} + +// keep records that the value at path (in the result) is the kept left value. It is +// copied now, because applying the plan overwrites the left side. +func (p *OverridePlan) keep(path *structpath.PathNode, v View) { + cp := &StructVar{Value: reflect.New(v.v.Type()).Interface()} + _, _ = cp.Assign(nil, v) + p.undos = append(p.undos, undo{path: path, value: cp}) +} + +// Override replaces sv.Value with the result of plan (see [PlanOverride]). +func (sv *StructVar) Override(plan *OverridePlan) error { + sv.Refs = nil + sv.Locations = nil + if _, err := sv.Assign(nil, plan.src); err != nil { + return err + } + for _, u := range plan.undos { + if _, err := sv.Assign(u.path, u.value.View()); err != nil { + return err + } + } + sv.Locations = plan.locs + return nil +} diff --git a/libs/structs/structvar/parity_walk_test.go b/libs/structs/structvar/parity_walk_test.go new file mode 100644 index 00000000000..f3111c64342 --- /dev/null +++ b/libs/structs/structvar/parity_walk_test.go @@ -0,0 +1,71 @@ +package structvar_test + +import ( + "testing" + + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestWalkVisitsParentsBeforeChildren(t *testing.T) { + root := config.Root{ + Bundle: config.Bundle{Name: "x"}, + Sync: config.Sync{Include: []string{"a", "b"}}, + } + + var visited []string + err := structvar.Walk(root.View(), func(p *structpath.PathNode, v structvar.View) error { + if s, ok := v.AsString(); ok { + visited = append(visited, p.String()+"="+s) + } + return nil + }) + require.NoError(t, err) + assert.Equal(t, []string{"bundle.name=x", "sync.include[0]=a", "sync.include[1]=b"}, visited) +} + +func TestWalkVisitsAllValues(t *testing.T) { + root, diags := config.LoadFromBytes("test.yml", []byte(` +resources: + jobs: + j: + name: n + max_concurrent_runs: ${var.x} + tags: + a: b +`)) + require.NoError(t, diags.Error()) + + var visited []string + err := structvar.Walk(root.View().Lookup(structpath.MustParsePath("resources.jobs.j")), func(p *structpath.PathNode, _ structvar.View) error { + visited = append(visited, p.String()) + return nil + }) + require.NoError(t, err) + assert.Equal(t, []string{"", "max_concurrent_runs", "name", "tags", "tags.a"}, visited) +} + +func TestSetReference(t *testing.T) { + root, diags := config.LoadFromBytes("test.yml", []byte(` +resources: + jobs: + j: + name: n + max_concurrent_runs: ${var.x} +`)) + require.NoError(t, diags.Error()) + + path := structpath.MustParsePath("resources.jobs.j.max_concurrent_runs") + require.NoError(t, root.SetReference(path, "${var.y}")) + s, ok := root.View().Lookup(path).AsString() + require.True(t, ok) + assert.Equal(t, "${var.y}", s) + + // A string field holds the reference itself. + name := structpath.MustParsePath("resources.jobs.j.name") + require.NoError(t, root.SetReference(name, "${var.y}")) + assert.Equal(t, "${var.y}", root.Resources.Jobs["j"].Name) +} diff --git a/libs/structs/structvar/pattern.go b/libs/structs/structvar/pattern.go new file mode 100644 index 00000000000..63cbdd0658e --- /dev/null +++ b/libs/structs/structvar/pattern.go @@ -0,0 +1,110 @@ +package structvar + +import ( + "errors" + "fmt" + + "github.com/databricks/cli/libs/structs/structpath" +) + +// ForEach calls fn for every value in x that matches pattern, like dyn.MapByPattern +// visits them: map keys in order, wildcards skip values the rest of the pattern does +// not match, and a missing key or index outside a wildcard visits nothing. +// +// fn may change the configuration at or below the visited path (e.g. with [Set]), +// but must not add or remove the keys and elements being iterated. +func ForEach(x View, pattern *structpath.PatternNode, fn func(*structpath.PathNode, View) error) error { + err := forEach(x, nil, pattern.AsSlice(), fn) + if errors.Is(err, errNoMatch) { + return nil + } + return err +} + +// errNoMatch means the pattern did not match: a key or index is missing or a value is nil. +var errNoMatch = errors.New("no match") + +// errUnexpectedKind means a wildcard was applied to a value of the wrong kind. +type errUnexpectedKind struct { + msg string +} + +func (e errUnexpectedKind) Error() string { + return e.msg +} + +func forEach(x View, prefix *structpath.PathNode, pattern []*structpath.PatternNode, fn func(*structpath.PathNode, View) error) error { + if len(pattern) == 0 { + return fn(prefix, x) + } + + c := pattern[0] + rest := pattern[1:] + kind := x.Kind() + + if c.DotStar() { + if kind != KindMap { + return errUnexpectedKind{fmt.Sprintf("expected a map at %q, found %s", prefix, kind)} + } + for k, child := range x.MapItems() { + if err := forEach(child, structpath.NewStringKey(prefix, k), rest, fn); err != nil && !isNoMatch(err) { + return err + } + } + return nil + } + + if c.BracketStar() { + if kind != KindSequence { + return errUnexpectedKind{fmt.Sprintf("expected a sequence at %q, found %s", prefix, kind)} + } + for i, child := range x.Sequence() { + if err := forEach(child, structpath.NewIndex(prefix, i), rest, fn); err != nil && !isNoMatch(err) { + return err + } + } + return nil + } + + if key, ok := c.StringKey(); ok { + path := structpath.NewStringKey(prefix, key) + switch kind { + case KindMap: + case KindNil: + return errNoMatch + default: + return fmt.Errorf("expected a map to index %q, found %s", path, kind) + } + child := x.Get(key) + if !child.IsValid() { + return errNoMatch + } + return forEach(child, path, rest, fn) + } + + index, ok := c.Index() + if !ok { + return fmt.Errorf("unsupported pattern component %q", c) + } + path := structpath.NewIndex(prefix, index) + switch kind { + case KindSequence: + case KindNil: + return errNoMatch + default: + return fmt.Errorf("expected a sequence to index %q, found %s", path, kind) + } + child := x.Index(index) + if !child.IsValid() { + return errNoMatch + } + return forEach(child, path, rest, fn) +} + +func isNoMatch(err error) bool { + if errors.Is(err, errNoMatch) { + return true + } + _, ok := errors.AsType[errUnexpectedKind](err) + return ok +} diff --git a/libs/dyn/dynvar/ref.go b/libs/structs/structvar/ref.go similarity index 88% rename from libs/dyn/dynvar/ref.go rename to libs/structs/structvar/ref.go index de25f44a212..d19421c8015 100644 --- a/libs/dyn/dynvar/ref.go +++ b/libs/structs/structvar/ref.go @@ -1,11 +1,9 @@ -package dynvar +package structvar import ( "fmt" "regexp" "strings" - - "github.com/databricks/cli/libs/dyn" ) // BaseVarDef matches a single dot-separated path segment in ${...} references. @@ -17,14 +15,9 @@ const BaseVarDef = `_*\p{L}+([-_]*[\p{L}\p{N}]+)*` var re = regexp.MustCompile(fmt.Sprintf(`\$\{(%s(\.%s(\[[0-9]+\])*)*(\[[0-9]+\])*)\}`, BaseVarDef, BaseVarDef)) -// Ref represents a variable reference. -// It is a string [dyn.Value] contained in a larger [dyn.Value]. -// Its path within the containing [dyn.Value] is also stored. +// Ref represents the variable references in a string. type Ref struct { - // Original value. - Value dyn.Value - - // String value in the original [dyn.Value]. + // The string. Str string // Matches of the variable reference in the string. @@ -36,9 +29,7 @@ type Ref struct { Spans [][2]int } -// NewRef returns a new Ref if the given [dyn.Value] contains a string -// with one or more variable references. It returns false if the given -// [dyn.Value] does not contain variable references. +// NewRef returns a new Ref if s contains one or more variable references. // // A reference preceded by "$" is treated as an escaped literal and is not // included in the returned Ref. For example, "$${foo}" contains no references @@ -49,12 +40,7 @@ type Ref struct { // - "${a.b.c}" // - "${a.b[0].c}" // - "${a} ${b} ${c}" -func NewRef(v dyn.Value) (Ref, bool) { - s, ok := v.AsString() - if !ok { - return Ref{}, false - } - +func NewRef(s string) (Ref, bool) { // Find all matches with their byte positions so we can detect escaped references. indices := re.FindAllStringSubmatchIndex(s, -1) if len(indices) == 0 { @@ -84,7 +70,6 @@ func NewRef(v dyn.Value) (Ref, bool) { } return Ref{ - Value: v, Str: s, Matches: m, Spans: spans, diff --git a/libs/dyn/dynvar/ref_test.go b/libs/structs/structvar/ref_test.go similarity index 56% rename from libs/dyn/dynvar/ref_test.go rename to libs/structs/structvar/ref_test.go index 7b93ac633b9..107c612f0b3 100644 --- a/libs/dyn/dynvar/ref_test.go +++ b/libs/structs/structvar/ref_test.go @@ -1,22 +1,16 @@ -package dynvar +package structvar import ( "testing" - "github.com/databricks/cli/libs/dyn" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -func TestNewRefNoString(t *testing.T) { - _, ok := NewRef(dyn.V(1)) - require.False(t, ok, "should not match non-string") -} - func TestNewRefSpans(t *testing.T) { // Spans must point at the unescaped occurrence, not the escaped one that // appears earlier with identical text. - ref, ok := NewRef(dyn.V("$${a.b} ${a.b}")) + ref, ok := NewRef("$${a.b} ${a.b}") require.True(t, ok) require.Len(t, ref.Matches, 1) require.Len(t, ref.Spans, 1) @@ -63,3 +57,52 @@ func TestReplaceRef(t *testing.T) { assert.Equal(t, tt.want, ReplaceRef(tt.in, "${a.b}", "V"), "input %q", tt.in) } } + +func TestPureReferenceToPath(t *testing.T) { + tests := []struct { + name string + input string + expected string + ok bool + }{ + { + name: "simple reference", + input: "${resources.jobs.foo.id}", + expected: "resources.jobs.foo.id", + ok: true, + }, + { + name: "simple reference", + input: "${resources.jobs.foo.tasks[1].env.key}", + expected: "resources.jobs.foo.tasks[1].env.key", + ok: true, + }, + { + name: "complex nested reference", + input: "${var.resources.jobs['my_job'].tasks[0]}", + // we use regex from dyn module which only support integers inside brackets: + // expected: "resources.jobs['my_job'].tasks[0]", + }, + { + name: "not a pure reference", + input: "prefix_${var.field}", + }, + { + name: "not a variable reference", + input: "plain_string", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + pathNode, ok := PureReferenceToPath(tt.input) + assert.Equal(t, tt.ok, ok) + if tt.ok { + assert.NotNil(t, pathNode) + assert.Equal(t, tt.expected, pathNode.String()) + } else { + assert.Nil(t, pathNode) + } + }) + } +} diff --git a/libs/dyn/dynvar/ref_vectors_test.go b/libs/structs/structvar/ref_vectors_test.go similarity index 91% rename from libs/dyn/dynvar/ref_vectors_test.go rename to libs/structs/structvar/ref_vectors_test.go index d0084a6ae5c..ea58cc03fba 100644 --- a/libs/dyn/dynvar/ref_vectors_test.go +++ b/libs/structs/structvar/ref_vectors_test.go @@ -1,4 +1,4 @@ -package dynvar +package structvar import ( "encoding/json" @@ -6,7 +6,6 @@ import ( "path/filepath" "testing" - "github.com/databricks/cli/libs/dyn" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -41,7 +40,7 @@ func loadReferenceVectors(t *testing.T) []referenceVector { func TestReferenceVectors(t *testing.T) { for _, v := range loadReferenceVectors(t) { t.Run(v.ID, func(t *testing.T) { - ref, ok := NewRef(dyn.V(v.Input)) + ref, ok := NewRef(v.Input) assert.Equal(t, v.Match, ok, "NewRef match") if v.Match { @@ -59,7 +58,7 @@ func TestReferenceVectors(t *testing.T) { case v.Path != nil: path, ok := PureReferenceToPath(v.Input) require.True(t, ok, "PureReferenceToPath") - assert.Equal(t, dyn.MustPathFromString(*v.Path), path) + assert.Equal(t, *v.Path, path.String()) case v.PathOK != nil: _, ok := PureReferenceToPath(v.Input) assert.Equal(t, *v.PathOK, ok, "PureReferenceToPath") diff --git a/libs/structs/structvar/resolve.go b/libs/structs/structvar/resolve.go new file mode 100644 index 00000000000..3cb8ec44683 --- /dev/null +++ b/libs/structs/structvar/resolve.go @@ -0,0 +1,200 @@ +package structvar + +import ( + "errors" + "fmt" + "maps" + "slices" + "strings" + + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/structs/structpath" +) + +// ErrSkipResolution is returned by a [Lookup] to leave a reference in place. +var ErrSkipResolution = errors.New("skip resolution") + +// ReferenceError is returned for a reference to a path that does not exist. +// Suggestions are carried as data so callers (which can import libs/diag) format them. +type ReferenceError struct { + Reference string // original reference text, e.g. "var.hst" + Suggestions []string // corrected references, e.g. ["var.host", "var.hosts"] +} + +func (e *ReferenceError) Error() string { + return fmt.Sprintf("reference does not exist: ${%s}", e.Reference) +} + +// Template is a string with variable references and the locations it was written at. +type Template struct { + Value string + Locations []diag.Location +} + +// Lookup returns the value a reference path refers to. It returns +// ErrSkipResolution to leave the reference in place. +type Lookup func(path *structpath.PathNode) (View, error) + +// Resolve resolves the references in templates (keyed by path; the keys appear in +// cycle errors) using lookup, with the semantics of dynvar.Resolve: a referenced value +// that is a reference itself is resolved first, cycles are an error, a pure reference +// is replaced by the referenced value (with the location of the reference), and other +// references are interpolated into the string. It returns the views of the resolved +// values; templates that resolve to themselves are omitted. +func Resolve(templates map[string]Template, lookup Lookup) (map[string]View, error) { + r := resolver{lookup: lookup, lookups: map[string]lookupResult{}} + out := map[string]View{} + + // Resolve in key order, so the cycle detected is deterministic. + for _, key := range slices.Sorted(maps.Keys(templates)) { + t := templates[key] + ref, ok := NewRef(t.Value) + if !ok { + continue + } + v, err := r.resolveRef(ref, t.Locations, []string{key}) + if err != nil { + return nil, err + } + if s, ok := v.AsString(); ok && s == t.Value { + continue + } + out[key] = v + } + return out, nil +} + +type lookupResult struct { + v View + err error +} + +type resolver struct { + lookup Lookup + lookups map[string]lookupResult +} + +// stringView returns a view of the string s with locations locs. +func stringView(s string, locs []diag.Location) View { + return NewView(&s, nil, (*Locations)(nil).WithLocations(locs)) +} + +func (r *resolver) resolveRef(ref Ref, locs []diag.Location, seen []string) (View, error) { + deps := ref.References() + + // Resolve each of the dependencies, then interpolate them in the ref. + resolved := make([]View, len(deps)) + complete := true + + for j, dep := range deps { + // Cycle detection. + if slices.Contains(seen, dep) { + return View{}, fmt.Errorf( + "cycle detected in field resolution: %s", + strings.Join(append(seen, dep), " -> "), + ) + } + + v, err := r.resolveKey(dep, append(seen, dep)) + + // If we should skip resolution of this key, index j holds an invalid view. + if errors.Is(err, ErrSkipResolution) { + complete = false + continue + } else if err != nil { + return View{}, err + } + + resolved[j] = v + } + + // A pure reference is replaced by the value, which keeps its type. It takes the + // location of the reference, so relative paths resolve relative to where a + // variable is used, not where it is defined. + if ref.IsPure() && complete { + return resolved[0].WithLocations(locs), nil + } + + // Not pure; perform string interpolation. Substitute by byte offset: the same + // reference may also appear escaped ("$${foo} ${foo}"). + var sb strings.Builder + consumed := 0 + for j := range ref.Matches { + // Leave references that were skipped in place. + if !resolved[j].IsValid() { + continue + } + + s, ok := resolved[j].AsString() + if !ok { + // Only allow primitive types to be converted to string. + switch kind := resolved[j].Kind(); kind { + case KindBool, KindInt, KindFloat, KindNil: + s = fmt.Sprint(resolved[j].AsAny()) + default: + return View{}, fmt.Errorf("cannot interpolate non-primitive value of type %s into string", kind) + } + } + + start, end := ref.Spans[j][0], ref.Spans[j][1] + sb.WriteString(ref.Str[consumed:start]) + sb.WriteString(s) + consumed = end + } + sb.WriteString(ref.Str[consumed:]) + + return stringView(sb.String(), locs), nil +} + +func (r *resolver) resolveKey(key string, seen []string) (View, error) { + if v, ok := r.lookups[key]; ok { + return v.v, v.err + } + + p, err := structpath.ParsePath(key) + if err != nil { + return View{}, err + } + + v, err := r.lookup(p) + if err != nil { + if knf, ok := errors.AsType[*KeyNotFoundError](err); ok { + // Carry suggestions as data; the caller formats them. + err = &ReferenceError{Reference: key, Suggestions: suggestedReferences(knf, key)} + } + r.lookups[key] = lookupResult{err: err} + return View{}, err + } + + // If the value is a reference itself, resolve it. + if s, ok := v.AsString(); ok { + if ref, ok := NewRef(s); ok { + v, err = r.resolveRef(ref, v.Locations(), seen) + } + } + + r.lookups[key] = lookupResult{v: v, err: err} + return v, err +} + +// WithLocations returns the view with its own locations replaced by locs; the values +// below it keep theirs. +func (x View) WithLocations(locs []diag.Location) View { + x.loc = x.loc.WithLocations(locs) + return x +} + +// suggestedReferences returns drop-in replacement references for reference, rebuilt +// by swapping the key that was not found for each suggestion. +func suggestedReferences(err *KeyNotFoundError, reference string) []string { + if len(err.Suggestions) == 0 { + return nil + } + nodes := err.Path.AsSlice() + failedKey, _ := nodes[len(nodes)-1].StringKey() + refs := make([]string, len(err.Suggestions)) + for i, s := range err.Suggestions { + refs[i] = ReplaceKey(reference, failedKey, s) + } + return refs +} diff --git a/libs/dyn/convert/struct_info.go b/libs/structs/structvar/struct_info.go similarity index 65% rename from libs/dyn/convert/struct_info.go rename to libs/structs/structvar/struct_info.go index 69e8f868ed5..46ca6cde995 100644 --- a/libs/dyn/convert/struct_info.go +++ b/libs/structs/structvar/struct_info.go @@ -1,27 +1,25 @@ -package convert +package structvar import ( "reflect" "slices" "sync" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/structs/structtag" + sdkduration "github.com/databricks/databricks-sdk-go/common/types/duration" + sdkfieldmask "github.com/databricks/databricks-sdk-go/common/types/fieldmask" + sdktime "github.com/databricks/databricks-sdk-go/common/types/time" ) -// structInfo holds the type information we need to efficiently -// convert data from a [dyn.Value] to a Go struct. -type structInfo struct { +// StructInfo holds the type information of a struct for converting between +// the configuration tree and the typed value: JSON names, omitempty, ForceSendFields. +type StructInfo struct { // FieldNames is ordered list of fields FieldNames []string // Fields maps the JSON-name of the field to the field's index for use with [FieldByIndex]. Fields map[string][]int - // ValueField maps to the field with a [dyn.Value]. - // The underlying type is expected to only have one of these. - ValueField []int - // Tracks which fields do not have omitempty annotation ForceEmpty map[string]bool @@ -39,31 +37,31 @@ type structInfo struct { ForceSendFieldsIndex map[string][]int } -// structInfoCache caches type information. -var structInfoCache = make(map[reflect.Type]structInfo) +// StructInfoCache caches type information. +var StructInfoCache = make(map[reflect.Type]StructInfo) -// structInfoCacheLock guards concurrent access to structInfoCache. -var structInfoCacheLock sync.Mutex +// StructInfoCacheLock guards concurrent access to StructInfoCache. +var StructInfoCacheLock sync.Mutex -// getStructInfo returns the [structInfo] for the given type. +// getStructInfo returns the [StructInfo] for the given type. // It lazily populates a cache, so the first call for a given // type is slower than subsequent calls for that same type. -func getStructInfo(typ reflect.Type) structInfo { - structInfoCacheLock.Lock() - defer structInfoCacheLock.Unlock() +func GetStructInfo(typ reflect.Type) StructInfo { + StructInfoCacheLock.Lock() + defer StructInfoCacheLock.Unlock() - si, ok := structInfoCache[typ] + si, ok := StructInfoCache[typ] if !ok { si = buildStructInfo(typ) - structInfoCache[typ] = si + StructInfoCache[typ] = si } return si } -// buildStructInfo populates a new [structInfo] for the given type. -func buildStructInfo(typ reflect.Type) structInfo { - out := structInfo{ +// buildStructInfo populates a new [StructInfo] for the given type. +func buildStructInfo(typ reflect.Type) StructInfo { + out := StructInfo{ Fields: make(map[string][]int), ForceEmpty: make(map[string]bool), GolangNames: make(map[string]string), @@ -106,15 +104,6 @@ func buildStructInfo(typ reflect.Type) structInfo { continue } - // If this field has type [dyn.Value], we populate it with the source [dyn.Value] from [ToTyped]. - if sf.IsExported() && sf.Type == configValueType { - if out.ValueField != nil { - panic("multiple dyn.Value fields") - } - out.ValueField = append(prefix, sf.Index...) - continue - } - jtag := structtag.JSONTag(sf.Tag.Get("json")) name := jtag.Name() if name == "" || name == "-" { @@ -151,18 +140,18 @@ type FieldValue struct { IsForced bool } -func (s *structInfo) FieldValues(v reflect.Value) []FieldValue { +func (s *StructInfo) FieldValues(v reflect.Value) []FieldValue { out := make([]FieldValue, 0, len(s.Fields)) for _, k := range s.FieldNames { - fv := fieldByIndex(v, s.Fields[k]) + fv := FieldByIndex(v, s.Fields[k]) if fv.IsValid() { isForced := true // TODO: we should use isEmptyForOmitEmpty instead of IsZero() if fv.IsZero() { - isForced = s.isForceSend(v, k) + isForced = s.IsForceSend(v, k) } out = append(out, FieldValue{ @@ -177,13 +166,13 @@ func (s *structInfo) FieldValues(v reflect.Value) []FieldValue { } // isForceSend reports whether the field named k is listed in the ForceSendFields -// that governs it (see structInfo.ForceSendFieldsIndex). -func (s *structInfo) isForceSend(v reflect.Value, k string) bool { +// that governs it (see StructInfo.ForceSendFieldsIndex). +func (s *StructInfo) IsForceSend(v reflect.Value, k string) bool { index, ok := s.ForceSendFieldsIndex[k] if !ok { return false } - fsf := fieldByIndex(v, index) + fsf := FieldByIndex(v, index) if !fsf.IsValid() { return false } @@ -192,7 +181,7 @@ func (s *structInfo) isForceSend(v reflect.Value, k string) bool { // fieldByIndex resolves the value at the given index path, dereferencing embedded // pointer structs on the way. It returns an invalid value if a nil pointer is met. -func fieldByIndex(v reflect.Value, index []int) reflect.Value { +func FieldByIndex(v reflect.Value, index []int) reflect.Value { for i, x := range index { if i > 0 { if v.Kind() == reflect.Pointer && v.Type().Elem().Kind() == reflect.Struct { @@ -207,5 +196,32 @@ func fieldByIndex(v reflect.Value, index []int) reflect.Value { return v } -// Type of [dyn.Value]. -var configValueType = reflect.TypeFor[dyn.Value]() +// GetOrNewFieldByIndex resolves the value at the given index path within an addressable +// struct, allocating intermediate structs embedded as pointer types along the way. +func GetOrNewFieldByIndex(v reflect.Value, index []int) reflect.Value { + for i, x := range index { + if i > 0 { + if v.Kind() == reflect.Pointer { + if v.IsNil() { + v.Set(reflect.New(v.Type().Elem())) + } + v = v.Elem() + } + } + v = v.Field(x) + } + return v +} + +// sdkNativeTypes are SDK types with custom JSON marshaling to and from a string. +// The configuration represents them as strings. +var sdkNativeTypes = []reflect.Type{ + reflect.TypeFor[sdkduration.Duration](), // Protobuf duration format (e.g., "300s") + reflect.TypeFor[sdktime.Time](), // RFC3339 timestamp format (e.g., "2023-12-25T10:30:00Z") + reflect.TypeFor[sdkfieldmask.FieldMask](), // Comma-separated paths (e.g., "name,age,email") +} + +// IsSDKNativeType reports whether typ is an SDK type represented as a string. +func IsSDKNativeType(typ reflect.Type) bool { + return slices.Contains(sdkNativeTypes, typ) +} diff --git a/libs/structs/structvar/structvar.go b/libs/structs/structvar/structvar.go index cc894aced7f..2f5bde6642e 100644 --- a/libs/structs/structvar/structvar.go +++ b/libs/structs/structvar/structvar.go @@ -8,7 +8,6 @@ import ( "strings" "sync" - "github.com/databricks/cli/libs/dyn/dynvar" "github.com/databricks/cli/libs/structs/structaccess" "github.com/databricks/cli/libs/structs/structpath" ) @@ -20,6 +19,9 @@ type StructVar struct { // Refs holds unresolved references. Key is serialized PathNode pointing inside a struct (e.g. "name") // and value is either pure or multiple references string: "${resources.foo.jobs.id}" or "${a} ${b}" Refs map[string]string `json:"vars,omitempty"` + + // Locations holds the source locations of the values, if known (configuration loaded from files). + Locations *Locations `json:"-"` } // StructVarJSON is the serialized form of StructVar for persisting in plan files. @@ -103,18 +105,18 @@ func (sv *StructVar) ResolveRef(reference string, value any) error { return fmt.Errorf("cannot set %s to %T (%#v): %w", pathNode.String(), value, value, err) } - newValue := dynvar.ReplaceRef(refValue, reference, valueStr) + newValue := ReplaceRef(refValue, reference, valueStr) // The struct gets the unescaped form ("$${x}" -> "${x}"), since that is what // is sent to the API. sv.Refs keeps the escaped form so the still-pending // check below can tell a literal apart from a real reference. - err = structaccess.Set(sv.Value, pathNode, dynvar.Unescape(newValue)) + err = structaccess.Set(sv.Value, pathNode, Unescape(newValue)) if err != nil { return fmt.Errorf("cannot update %s to string: %w", pathNode.String(), err) } // Check if fully resolved (no unescaped ${} patterns left) - if !dynvar.ContainsVariableReference(newValue) { + if !ContainsVariableReference(newValue) { delete(sv.Refs, pathKey) } else { sv.Refs[pathKey] = newValue diff --git a/libs/dyn/suggest.go b/libs/structs/structvar/suggest.go similarity index 60% rename from libs/dyn/suggest.go rename to libs/structs/structvar/suggest.go index ef8008aae6f..2d2f4bccef7 100644 --- a/libs/dyn/suggest.go +++ b/libs/structs/structvar/suggest.go @@ -1,15 +1,17 @@ -package dyn +package structvar import ( "fmt" "slices" "strings" + + "github.com/databricks/cli/libs/structs/structpath" ) const maxSuggestionDistance = 2 -// levenshteinDistance computes the edit distance between two strings. -func levenshteinDistance(a, b string) int { +// LevenshteinDistance computes the edit distance between two strings. +func LevenshteinDistance(a, b string) int { if len(a) == 0 { return len(b) } @@ -43,19 +45,18 @@ func levenshteinDistance(a, b string) int { return prev[len(b)] } -// suggestKeys returns the keys in m whose edit distance from name is at most +// SuggestKeys returns the keys whose edit distance from name is at most // maxSuggestionDistance, ordered by increasing distance. It is used to build -// "did you mean" hints for a key that was not found in the map. -func suggestKeys(m Mapping, name string) []string { +// "did you mean" hints for a key that was not found in a map. +func SuggestKeys(keys []string, name string) []string { type candidate struct { key string dist int } var candidates []candidate - for _, kv := range m.Keys() { - key := kv.MustString() - d := levenshteinDistance(name, key) + for _, key := range keys { + d := LevenshteinDistance(name, key) if d <= maxSuggestionDistance { candidates = append(candidates, candidate{key, d}) } @@ -72,10 +73,10 @@ func suggestKeys(m Mapping, name string) []string { return suggestions } -// didYouMean formats a suggestion clause like `, did you mean "x"?` (or, for +// DidYouMean formats a suggestion clause like `, did you mean "x"?` (or, for // multiple candidates, `, did you mean one of: "x", "y"?`). It returns an empty // string when there are no suggestions. -func didYouMean(suggestions []string) string { +func DidYouMean(suggestions []string) string { switch len(suggestions) { case 0: return "" @@ -90,19 +91,28 @@ func didYouMean(suggestions []string) string { } } -// replaceKey returns reference with the component matching failedKey swapped for +// ReplaceKey returns reference with the component matching failedKey swapped for // replacement, or just replacement if reference can't be parsed or has no match. -func replaceKey(reference, failedKey, replacement string) string { - p, err := NewPathFromString(reference) +func ReplaceKey(reference, failedKey, replacement string) string { + p, err := structpath.ParsePath(reference) if err != nil { return replacement } - for i, c := range p { - if c.Key() == failedKey { - out := p.Append() - out[i] = Key(replacement) - return out.String() + var out *structpath.PathNode + replaced := false + for _, n := range p.AsSlice() { + if k, ok := n.StringKey(); ok { + if k == failedKey && !replaced { + k = replacement + replaced = true + } + out = structpath.NewStringKey(out, k) + } else if i, ok := n.Index(); ok { + out = structpath.NewIndex(out, i) } } - return replacement + if !replaced { + return replacement + } + return out.String() } diff --git a/libs/dyn/dynvar/testdata/reference_vectors.json b/libs/structs/structvar/testdata/reference_vectors.json similarity index 99% rename from libs/dyn/dynvar/testdata/reference_vectors.json rename to libs/structs/structvar/testdata/reference_vectors.json index 2c47b135450..c9c380ad7d0 100644 --- a/libs/dyn/dynvar/testdata/reference_vectors.json +++ b/libs/structs/structvar/testdata/reference_vectors.json @@ -1,5 +1,5 @@ { - "comment": "Shared contract for ${...} reference parsing. Consumed by libs/dyn/dynvar and python/databricks/bundles/core/_transform.py tests.", + "comment": "Shared contract for ${...} reference parsing. Consumed by libs/structs/structvar and python/databricks/bundles/core/_transform.py tests.", "vectors": [ { "id": "ascii-simple", diff --git a/libs/structs/structvar/view.go b/libs/structs/structvar/view.go new file mode 100644 index 00000000000..ecdc0c73825 --- /dev/null +++ b/libs/structs/structvar/view.go @@ -0,0 +1,504 @@ +package structvar + +import ( + "encoding/json" + "iter" + "reflect" + "slices" + + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/structs/structpath" +) + +type option uint8 + +const ( + // includeZero keeps zero values: the value is behind a pointer, a map value, + // a slice element, or held in an interface. + includeZero option = 1 << iota + + // fromInterface is set for values held in an interface (e.g. variable defaults). + fromInterface +) + +// View is a read-only view of a typed value with its references and locations, as the +// configuration tree they describe. It is what [convert.FromTyped] would return for the typed value given +// the source tree as reference, without building that tree: it decides presence the +// same way (zero values are absent unless forced, present in the source, behind a +// pointer, or a map value or slice element) and keeps pure references. +// +// The zero View is invalid and represents an absent value. +type View struct { + v reflect.Value + + // Locations of the value. + loc *Locations + + // Path of the value and the references of the whole value the view was created for. + // The path is only tracked if there are references. + path *structpath.PathNode + refs map[string]string + + opts option + valid bool +} + +// NewView returns the view of the typed value v (typically a pointer to a struct) +// with references refs (keyed by path string) and locations. +func NewView(v any, refs map[string]string, locs *Locations) View { + return View{v: reflect.ValueOf(v), loc: locs, refs: refs, valid: true} +} + +// View returns the view of sv. +func (sv *StructVar) View() View { + return NewView(sv.Value, sv.Refs, sv.Locations) +} + +// ref returns the pure reference held by the value, if its type cannot hold a string. +func (x View) ref() string { + if len(x.refs) == 0 { + return "" + } + return x.refs[x.path.String()] +} + +// child returns a view of the child value v at key k (or index i). +func (x View) childKey(v reflect.Value, k string, opts option) View { + c := View{v: v, loc: x.loc.Key(k), refs: x.refs, opts: opts, valid: true} + if len(x.refs) > 0 { + c.path = structpath.NewStringKey(x.path, k) + } + return c +} + +func (x View) childIndex(v reflect.Value, i int, opts option) View { + c := View{v: v, loc: x.loc.Index(i), refs: x.refs, opts: opts, valid: true} + if len(x.refs) > 0 { + c.path = structpath.NewIndex(x.path, i) + } + return c +} + +// IsValid reports whether the value is present. +func (x View) IsValid() bool { + return x.valid +} + +// Locations returns the source locations of the value. +func (x View) Locations() []diag.Location { + if !x.valid { + return nil + } + return x.loc.Get() +} + +// Location returns the first source location of the value. +func (x View) Location() diag.Location { + locs := x.loc.Get() + if !x.valid || len(locs) == 0 { + return diag.Location{} + } + return locs[0] +} + +// Reflect returns the typed value the view is based on (it may be a pointer or interface). +// It is settable if it was reached through pointers or is the root pointer's element. +func (x View) Reflect() reflect.Value { + return x.v +} + +// Locs returns the locations of this value and the values below it. +func (x View) Locs() *Locations { + return x.loc +} + +// elemOptions returns the options for a map value or a slice element: they keep +// zero values. +func elemOptions(v reflect.Value) option { + if v.Kind() == reflect.Interface { + return includeZero | fromInterface + } + return includeZero +} + +// deref follows pointers and interfaces; it returns an invalid value for nil. +func (x View) deref() (reflect.Value, option) { + v, opts := x.v, x.opts + for { + switch v.Kind() { + case reflect.Pointer: + if v.IsNil() { + return reflect.Value{}, opts + } + v = v.Elem() + // A pointer to a zero value was intentionally set. + opts |= includeZero + case reflect.Interface: + if v.IsNil() { + return reflect.Value{}, opts + } + v = v.Elem() + default: + return v, opts + } + } +} + +// Kind returns the kind of the value in the configuration tree. +func (x View) Kind() Kind { + if !x.valid { + return KindInvalid + } + v, opts := x.deref() + if !v.IsValid() { + return KindNil + } + ref := x.ref() + switch v.Kind() { + case reflect.Struct: + if ref != "" { + return KindString + } + if IsSDKNativeType(v.Type()) { + if _, ok := sdkNativeString(v, opts); ok { + return KindString + } + return KindNil + } + // A struct is present if it was present in the source, even if empty. + if opts&includeZero != 0 || x.loc != nil || x.hasFields(v) { + return KindMap + } + return KindNil + case reflect.Map: + if ref != "" { + return KindString + } + if v.IsNil() { + return KindNil + } + return KindMap + case reflect.Slice: + if ref != "" { + return KindString + } + if v.IsNil() { + return KindNil + } + return KindSequence + case reflect.String, reflect.Bool, reflect.Int, reflect.Int32, reflect.Int64, reflect.Float32, reflect.Float64: + if ref != "" { + return KindString + } + // A zero value is absent unless it was present in the source or is kept. + if v.IsZero() && opts&includeZero == 0 && x.loc == nil { + return KindNil + } + switch v.Kind() { + case reflect.String: + return KindString + case reflect.Bool: + return KindBool + case reflect.Float32, reflect.Float64: + return KindFloat + default: + return KindInt + } + default: + return KindInvalid + } +} + +func sdkNativeString(v reflect.Value, opts option) (string, bool) { + if v.IsZero() && opts&includeZero == 0 { + return "", false + } + // SDK native types implement json.Marshaler and marshal to a JSON string. + buf, err := json.Marshal(v.Interface()) + if err != nil { + return "", false + } + var s string + if json.Unmarshal(buf, &s) != nil { + return "", false + } + if s == "" && opts&includeZero == 0 { + return "", false + } + return s, true +} + +// isZero mirrors [dyn.Value.IsZero] for the value. +func (x View) isZero() bool { + switch x.Kind() { + case KindInvalid, KindNil: + return true + case KindMap: + for range x.MapItems() { + return false + } + return true + case KindSequence: + v, _ := x.deref() + return v.Len() == 0 + default: + if x.ref() != "" { + return false + } + s, ok := x.AsString() + if ok { + return s == "" + } + v, _ := x.deref() + return v.IsZero() + } +} + +// field returns the view of the struct field k, and whether it is represented. +func (x View) field(info *StructInfo, sv reflect.Value, k string, fv reflect.Value, isForced bool) (View, bool) { + var opts option + if fv.Kind() == reflect.Interface { + opts = includeZero | fromInterface + } + child := x.childKey(fv, k, opts) + kind := child.Kind() + + // The field was present in the source (it has a node in the locations tree), it is + // not zero-valued, or it's forced. + if child.loc == nil && kind == KindNil && !isForced { + return View{}, false + } + + // A zero field is absent unless forced (it may still be a reference: check the view too). + if fv.Kind() != reflect.Struct && fv.IsZero() && child.isZero() && !info.ForceEmpty[k] && !isForced { + return View{}, false + } + + // A forced field with a nil value is represented by its zero value. + if isForced && kind == KindNil { + child.opts |= includeZero + } + return child, true +} + +func (x View) hasFields(v reflect.Value) bool { + for range x.structFields(v) { + return true + } + return false +} + +func (x View) structFields(v reflect.Value) iter.Seq2[string, View] { + return func(yield func(string, View) bool) { + info := GetStructInfo(v.Type()) + for _, fv := range info.FieldValues(v) { + child, ok := x.field(&info, v, fv.Key, fv.Value, fv.IsForced) + if ok && !yield(fv.Key, child) { + return + } + } + } +} + +// MapItems returns the entries of a map value: struct fields in declaration order, +// map entries in key order. +func (x View) MapItems() iter.Seq2[string, View] { + return func(yield func(string, View) bool) { + if x.ref() != "" { + return + } + v, _ := x.deref() + switch { + case !v.IsValid(): + case v.Kind() == reflect.Struct && !IsSDKNativeType(v.Type()): + for k, c := range x.structFields(v) { + if !yield(k, c) { + return + } + } + case v.Kind() == reflect.Map: + keys := v.MapKeys() + slices.SortFunc(keys, func(a, b reflect.Value) int { + switch { + case a.String() < b.String(): + return -1 + case a.String() > b.String(): + return 1 + } + return 0 + }) + for _, key := range keys { + ev := v.MapIndex(key) + child := x.childKey(ev, key.String(), elemOptions(ev)) + if !yield(key.String(), child) { + return + } + } + } + } +} + +// Sequence returns the elements of a sequence value. +func (x View) Sequence() iter.Seq2[int, View] { + return func(yield func(int, View) bool) { + if x.Kind() != KindSequence { + return + } + v, _ := x.deref() + for i := range v.Len() { + ev := v.Index(i) + child := x.childIndex(ev, i, elemOptions(ev)) + if !yield(i, child) { + return + } + } + } +} + +// Get returns the value of key k of a map value, or an invalid value. +func (x View) Get(k string) View { + if x.Kind() != KindMap { + return View{} + } + v, _ := x.deref() + switch v.Kind() { + case reflect.Struct: + info := GetStructInfo(v.Type()) + index, ok := info.Fields[k] + if !ok { + return View{} + } + fv := FieldByIndex(v, index) + if !fv.IsValid() { + return View{} + } + isForced := !fv.IsZero() || info.IsForceSend(v, k) + child, ok := x.field(&info, v, k, fv, isForced) + if !ok { + return View{} + } + return child + case reflect.Map: + ev := v.MapIndex(reflect.ValueOf(k).Convert(v.Type().Key())) + if !ev.IsValid() { + return View{} + } + child := x.childKey(ev, k, elemOptions(ev)) + return child + default: + return View{} + } +} + +// Index returns element i of a sequence value, or an invalid value. +func (x View) Index(i int) View { + if x.Kind() != KindSequence { + return View{} + } + v, _ := x.deref() + if i < 0 || i >= v.Len() { + return View{} + } + ev := v.Index(i) + child := x.childIndex(ev, i, elemOptions(ev)) + return child +} + +// Lookup returns the value at path, or an invalid value if there is none. +func (x View) Lookup(path *structpath.PathNode) View { + return x.lookupComponents(components(path)) +} + +func (x View) lookupComponents(path []pathComponent) View { + for _, c := range path { + if c.isKey { + x = x.Get(c.key) + } else { + x = x.Index(c.index) + } + if !x.valid { + return x + } + } + return x +} + +// AsString returns the string value (a string, a pure reference, or an SDK native value). +func (x View) AsString() (string, bool) { + if x.Kind() != KindString { + return "", false + } + if ref := x.ref(); ref != "" { + return ref, true + } + v, opts := x.deref() + if v.Kind() == reflect.Struct { + return sdkNativeString(v, opts) + } + return v.String(), true +} + +// AsBool returns the bool value. +func (x View) AsBool() (bool, bool) { + if x.Kind() != KindBool { + return false, false + } + v, _ := x.deref() + return v.Bool(), true +} + +// AsInt returns the int value. +func (x View) AsInt() (int64, bool) { + if x.Kind() != KindInt { + return 0, false + } + v, _ := x.deref() + return v.Int(), true +} + +// AsFloat returns the float value. +func (x View) AsFloat() (float64, bool) { + if x.Kind() != KindFloat { + return 0, false + } + v, _ := x.deref() + return v.Float(), true +} + +// AsAny returns the value as a generic Go value (map[string]any, []any, string, +// bool, int64 or int, float64, or nil), like [dyn.Value.AsAny]. +func (x View) AsAny() any { + switch x.Kind() { + case KindMap: + out := map[string]any{} + for k, c := range x.MapItems() { + out[k] = c.AsAny() + } + return out + case KindSequence: + out := []any{} //nolint:gocritic // an empty sequence renders as [], not null + for _, c := range x.Sequence() { + out = append(out, c.AsAny()) + } + return out + case KindString: + s, _ := x.AsString() + return s + case KindBool: + b, _ := x.AsBool() + return b + case KindInt: + v, opts := x.deref() + // Values held in an interface keep the type the YAML loader produced (int), + // typed fields hold the normalized int64. + if opts&fromInterface != 0 && v.Type() == reflect.TypeFor[int]() { + return int(v.Int()) + } + return v.Int() + case KindFloat: + f, _ := x.AsFloat() + return f + default: + return nil + } +} diff --git a/libs/structs/structvar/walk.go b/libs/structs/structvar/walk.go new file mode 100644 index 00000000000..b7a11a91499 --- /dev/null +++ b/libs/structs/structvar/walk.go @@ -0,0 +1,47 @@ +package structvar + +import ( + "errors" + + "github.com/databricks/cli/libs/structs/structpath" +) + +// Walk calls fn for x and every value below it, parents before children, like +// [dyn.WalkReadOnly] visits a tree. Maps are visited in [View.MapItems] order. +// fn may return [ErrSkip] to skip the subtree of the visited value. +// +// fn may change the configuration at or below the visited path (e.g. with [Set]), +// but must not add or remove the keys and elements being iterated. +func Walk(x View, fn func(*structpath.PathNode, View) error) error { + return walk(x, nil, fn) +} + +func walk(x View, p *structpath.PathNode, fn func(*structpath.PathNode, View) error) error { + if err := fn(p, x); err != nil { + if errors.Is(err, ErrSkip) { + return nil + } + return err + } + + switch x.Kind() { + case KindMap: + for k, c := range x.MapItems() { + if err := walk(c, structpath.NewStringKey(p, k), fn); err != nil { + return err + } + } + case KindSequence: + for i, c := range x.Sequence() { + if err := walk(c, structpath.NewIndex(p, i), fn); err != nil { + return err + } + } + default: + } + return nil +} + +// ErrSkip can be returned by the function passed to [Walk] to skip the subtree of the +// visited value. +var ErrSkip = errors.New("skip traversal of subtree") diff --git a/libs/structs/structvar/write.go b/libs/structs/structvar/write.go new file mode 100644 index 00000000000..42c3c6f464d --- /dev/null +++ b/libs/structs/structvar/write.go @@ -0,0 +1,352 @@ +package structvar + +import ( + "errors" + "fmt" + "reflect" + "slices" + "strings" + + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/structs/structpath" +) + +// The methods in this file change sv.Value (a pointer) and keep sv.Refs and +// sv.Locations in sync. Values along the path are allocated as needed (nil pointers, +// nil maps), and map values are written back, so maps of structs can be changed too. +// Refs is replaced rather than modified, so copies of a StructVar can share it. + +// Set sets value at path, like setting it in the configuration tree: a zero value +// is kept as explicitly set (it is added to the governing ForceSendFields). The +// locations recorded for path are kept; references at and below path are dropped. +func (sv *StructVar) Set(path *structpath.PathNode, value any) error { + err := update(reflect.ValueOf(sv.Value), components(path), func(dst reflect.Value) (bool, error) { + rv := reflect.ValueOf(value) + if !rv.IsValid() { + dst.SetZero() + return false, nil + } + if err := setGoValue(dst, rv); err != nil { + return false, fmt.Errorf("cannot set %s: %w", path, err) + } + return rv.IsZero(), nil + }) + if err != nil { + return err + } + sv.Refs = withoutRefs(sv.Refs, path) + // The node marks the value as present even if it is zero (e.g. an empty struct). + var locs *Locations + if value != nil { + locs = (*Locations)(nil).WithLocations(sv.Locations.At(path)) + } + sv.Locations = sv.Locations.With(path, locs) + return nil +} + +func setGoValue(dst, src reflect.Value) error { + for src.Kind() == reflect.Interface || (src.Kind() == reflect.Pointer && dst.Kind() != reflect.Pointer) { + if src.IsNil() { + dst.SetZero() + return nil + } + src = src.Elem() + } + switch { + case src.Type().AssignableTo(dst.Type()): + dst.Set(src) + case dst.Kind() == reflect.Pointer && src.Type().AssignableTo(dst.Type().Elem()): + p := reflect.New(dst.Type().Elem()) + p.Elem().Set(src) + dst.Set(p) + case isScalarKind(src.Kind()) && isScalarKind(dst.Kind()) && src.Type().ConvertibleTo(dst.Type()) && (src.Kind() == reflect.String) == (dst.Kind() == reflect.String): + dst.Set(src.Convert(dst.Type())) + default: + return fmt.Errorf("cannot assign %s to %s", src.Type(), dst.Type()) + } + return nil +} + +func isScalarKind(k reflect.Kind) bool { + switch k { + case reflect.String, reflect.Bool, reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, + reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Float32, reflect.Float64: + return true + default: + return false + } +} + +// Delete removes the value at path: a map entry is deleted, a struct field is zeroed. +func (sv *StructVar) Delete(path *structpath.PathNode) error { + p := components(path) + if len(p) == 0 { + return errors.New("cannot delete the root") + } + parent, last := p[:len(p)-1], p[len(p)-1] + err := walkTo(reflect.ValueOf(sv.Value), parent, func(v reflect.Value) error { + v = derefAlloc(v) + switch { + case v.Kind() == reflect.Map && last.isKey: + if !v.IsNil() { + v.SetMapIndex(reflect.ValueOf(last.key).Convert(v.Type().Key()), reflect.Value{}) + } + case v.Kind() == reflect.Struct && last.isKey: + info := GetStructInfo(v.Type()) + index, ok := info.Fields[last.key] + if !ok { + return fmt.Errorf("no field %q at %s", last.key, path.Parent()) + } + if f := FieldByIndex(v, index); f.IsValid() { + f.SetZero() + } + removeForceSend(v, &info, last.key) + default: + return fmt.Errorf("cannot delete %s", path) + } + return nil + }) + if err != nil { + return err + } + sv.Refs = withoutRefs(sv.Refs, path) + sv.Locations = sv.Locations.With(path, nil) + return nil +} + +// SetLocations sets the locations of the value at path and of every value below it, +// including values that have no locations yet (e.g. set in Go code). +func (sv *StructVar) SetLocations(path *structpath.PathNode, locs []diag.Location) { + x := sv.View().Lookup(path) + if !x.IsValid() { + return + } + sv.Locations = sv.Locations.With(path, locationsOf(x, locs)) +} + +func locationsOf(x View, locs []diag.Location) *Locations { + out := (*Locations)(nil).WithLocations(locs) + switch x.Kind() { + case KindMap: + for k, c := range x.MapItems() { + out.setChild(pathComponent{key: k, isKey: true}, locationsOf(c, locs)) + } + case KindSequence: + for i, c := range x.Sequence() { + out.setChild(pathComponent{index: i}, locationsOf(c, locs)) + } + default: + } + return out +} + +// UpdateSequence records that the elements of the sequence at path were rebuilt from +// the old ones: new element i comes from old elements sources[i] (see +// [Locations.WithSequence]). References of the old elements move along. +func (sv *StructVar) UpdateSequence(path *structpath.PathNode, sources [][]int) { + sv.Locations = sv.Locations.WithSequence(path, sources) + if len(sv.Refs) == 0 { + return + } + elems := path.String() + "[" + refs := make(map[string]string, len(sv.Refs)) + for k, v := range sv.Refs { + if !strings.HasPrefix(k, elems) { + refs[k] = v + } + } + for i, src := range sources { + to := structpath.NewIndex(path, i).String() + for _, old := range src { + from := structpath.NewIndex(path, old).String() + for k, v := range sv.Refs { + if isUnder(k, from) { + refs[to+strings.TrimPrefix(k, from)] = v + } + } + } + } + sv.Refs = refs +} + +// isUnder reports whether the path string key is prefix or a path below it. +func isUnder(key, prefix string) bool { + if prefix == "" || key == prefix { + return true + } + if !strings.HasPrefix(key, prefix) { + return false + } + c := key[len(prefix)] + return c == '.' || c == '[' +} + +// withoutRefs returns a copy of refs without the references at and below path. +func withoutRefs(refs map[string]string, path *structpath.PathNode) map[string]string { + if len(refs) == 0 { + return refs + } + prefix := path.String() + out := make(map[string]string, len(refs)) + for k, v := range refs { + if !isUnder(k, prefix) { + out[k] = v + } + } + return out +} + +// SetReference sets the value at path to the reference ref. A string (or interface) +// field holds the reference itself; for other fields the typed value is set to zero, +// like decoding does, and the reference is recorded in Refs. +func (sv *StructVar) SetReference(path *structpath.PathNode, ref string) error { + isText := false + err := walkTo(reflect.ValueOf(sv.Value), components(path), func(dst reflect.Value) error { + switch dst.Kind() { + case reflect.String, reflect.Interface: + isText = true + case reflect.Pointer: + // The reference is recorded on an allocated zero value, so it is present. + dst.Set(reflect.New(dst.Type().Elem())) + default: + dst.SetZero() + } + return nil + }) + if err != nil { + return err + } + if isText { + return sv.Set(path, ref) + } + locs := sv.Locations.At(path) + // withoutRefs returns a fresh map unless there is nothing to filter. + refs := withoutRefs(sv.Refs, path) + if len(refs) == 0 { + refs = map[string]string{} + } + refs[path.String()] = ref + sv.Refs = refs + sv.Locations = sv.Locations.With(path, (*Locations)(nil).WithLocations(locs)) + return nil +} + +func addForceSend(v reflect.Value, info *StructInfo, k string) { + index, ok := info.ForceSendFieldsIndex[k] + if !ok { + return + } + fsf := GetOrNewFieldByIndex(v, index) + name := info.GolangNames[k] + if slices.Contains(fsf.Interface().([]string), name) { + return + } + fsf.Set(reflect.Append(fsf, reflect.ValueOf(name))) +} + +func removeForceSend(v reflect.Value, info *StructInfo, k string) { + index, ok := info.ForceSendFieldsIndex[k] + if !ok { + return + } + fsf := FieldByIndex(v, index) + if !fsf.IsValid() { + return + } + name := info.GolangNames[k] + names := fsf.Interface().([]string) + if i := slices.Index(names, name); i >= 0 { + fsf.Set(reflect.ValueOf(slices.Delete(slices.Clone(names), i, i+1))) + } +} + +// derefAlloc follows pointers, allocating nil ones, and returns the value they point to. +func derefAlloc(v reflect.Value) reflect.Value { + for v.Kind() == reflect.Pointer { + if v.IsNil() { + v.Set(reflect.New(v.Type().Elem())) + } + v = v.Elem() + } + return v +} + +// walkTo calls fn with the settable value at path. +func walkTo(v reflect.Value, path []pathComponent, fn func(reflect.Value) error) error { + if len(path) == 0 { + return fn(v) + } + return updateChild(derefAlloc(v), path[0], func(child reflect.Value) (bool, error) { + return false, walkTo(child, path[1:], fn) + }) +} + +// update calls fn with the settable value at path. If fn reports that it set a zero +// value into a struct field, the field is added to the governing ForceSendFields. +func update(v reflect.Value, path []pathComponent, fn func(reflect.Value) (bool, error)) error { + if len(path) == 0 { + if v.Kind() == reflect.Pointer && !v.CanSet() { + v = v.Elem() + } + _, err := fn(v) + return err + } + return walkTo(v, path[:len(path)-1], func(parent reflect.Value) error { + return updateChild(derefAlloc(parent), path[len(path)-1], fn) + }) +} + +// updateChild calls fn with the settable child c of v. Interfaces and map values are +// copied and written back; an index one past the end of a slice appends an element. +func updateChild(v reflect.Value, c pathComponent, fn func(reflect.Value) (bool, error)) error { + if v.Kind() == reflect.Interface { + if v.IsNil() { + return errors.New("cannot index nil value") + } + cp := reflect.New(v.Elem().Type()).Elem() + cp.Set(v.Elem()) + err := updateChild(derefAlloc(cp), c, fn) + v.Set(cp) + return err + } + + switch { + case c.isKey && v.Kind() == reflect.Struct: + info := GetStructInfo(v.Type()) + index, ok := info.Fields[c.key] + if !ok { + return fmt.Errorf("no field %q", c.key) + } + zero, err := fn(GetOrNewFieldByIndex(v, index)) + if err != nil { + return err + } + if zero { + addForceSend(v, &info, c.key) + } + return nil + case c.isKey && v.Kind() == reflect.Map: + if v.IsNil() { + v.Set(reflect.MakeMap(v.Type())) + } + key := reflect.ValueOf(c.key).Convert(v.Type().Key()) + e := reflect.New(v.Type().Elem()).Elem() + if old := v.MapIndex(key); old.IsValid() { + e.Set(old) + } + _, err := fn(e) + v.SetMapIndex(key, e) + return err + case !c.isKey && v.Kind() == reflect.Slice: + if c.index == v.Len() { + v.Set(reflect.Append(v, reflect.New(v.Type().Elem()).Elem())) + } + if c.index < 0 || c.index >= v.Len() { + return fmt.Errorf("index %d out of range", c.index) + } + _, err := fn(v.Index(c.index)) + return err + default: + return fmt.Errorf("cannot index %s", v.Kind()) + } +} diff --git a/libs/structs/structyaml/structyaml.go b/libs/structs/structyaml/structyaml.go new file mode 100644 index 00000000000..9c88f6fb21c --- /dev/null +++ b/libs/structs/structyaml/structyaml.go @@ -0,0 +1,187 @@ +// Package structyaml writes typed values as YAML with a controlled key order. +package structyaml + +import ( + "cmp" + "fmt" + "os" + "path/filepath" + "slices" + "strconv" + + "github.com/databricks/cli/libs/structs/structvar" + "go.yaml.in/yaml/v3" +) + +// Pair is an entry of a [Map]. +type Pair struct { + Key string + Value any +} + +// Map is a YAML mapping that keeps the order of its keys. Values are maps (Map or +// map[string]any, keys sorted), sequences ([]any or []string), strings, bools, ints, floats or nil. +type Map []Pair + +// M returns a Map of the given alternating keys (strings) and values. +func M(kv ...any) Map { + m := make(Map, 0, len(kv)/2) + for i := 0; i < len(kv); i += 2 { + m = append(m, Pair{Key: kv[i].(string), Value: kv[i+1]}) + } + return m +} + +// Add appends an entry. +func (m *Map) Add(key string, value any) { + *m = append(*m, Pair{Key: key, Value: value}) +} + +// Order moves the given keys to the front, in the given order. The other keys keep +// their relative order. +func (m *Map) Order(keys ...string) Map { + index := func(p Pair) int { + if i := slices.Index(keys, p.Key); i >= 0 { + return i + } + return len(keys) + } + slices.SortStableFunc(*m, func(a, b Pair) int { return cmp.Compare(index(a), index(b)) }) + return *m +} + +// Struct returns the fields of the struct (or map) v that are set, except skip, as a +// Map with the keys sorted alphabetically, and so are the keys of maps and structs +// nested in maps. Sequence elements keep the field order of their type. +func Struct(v any, skip ...string) (Map, error) { + x := structvar.NewView(v, nil, nil) + if x.Kind() != structvar.KindMap { + return nil, fmt.Errorf("expected map, got %s", x.Kind()) + } + m := fromView(x, true).(Map) + return slices.DeleteFunc(m, func(p Pair) bool { return slices.Contains(skip, p.Key) }), nil +} + +// Value returns the typed value v as a value for a Map, in field order. Unset +// fields are dropped. It returns nil if v is unset. +func Value(v any) any { + return fromView(structvar.NewView(v, nil, nil), false) +} + +func fromView(x structvar.View, sorted bool) any { + switch x.Kind() { + case structvar.KindMap: + var m Map + for k, c := range x.MapItems() { + m.Add(k, fromView(c, sorted)) + } + if sorted { + slices.SortStableFunc(m, func(a, b Pair) int { return cmp.Compare(a.Key, b.Key) }) + } + return m + case structvar.KindSequence: + s := []any{} //nolint:gocritic // an empty sequence is written as [], not null + for _, c := range x.Sequence() { + s = append(s, fromView(c, false)) + } + return s + default: + return x.AsAny() + } +} + +// Node returns v as a YAML node. Values of keys in styles (and everything below +// them) get that style. +func Node(v any, styles map[string]yaml.Style) *yaml.Node { + return toNode(v, 0, styles) +} + +func toNode(v any, style yaml.Style, styles map[string]yaml.Style) *yaml.Node { + switch v := v.(type) { + case Map: + n := &yaml.Node{Kind: yaml.MappingNode, Style: style} + for _, p := range v { + s, ok := styles[p.Key] + if !ok { + s = style + } + n.Content = append(n.Content, &yaml.Node{Kind: yaml.ScalarNode, Value: p.Key, Style: style}, toNode(p.Value, s, styles)) + } + return n + case map[string]any: + m := make(Map, 0, len(v)) + for k, e := range v { + m.Add(k, e) + } + slices.SortFunc(m, func(a, b Pair) int { return cmp.Compare(a.Key, b.Key) }) + return toNode(m, style, styles) + case []any: + n := &yaml.Node{Kind: yaml.SequenceNode, Style: style} + for _, e := range v { + n.Content = append(n.Content, toNode(e, style, styles)) + } + return n + case []string: + n := &yaml.Node{Kind: yaml.SequenceNode, Style: style} + for _, e := range v { + n.Content = append(n.Content, toNode(e, style, styles)) + } + return n + case nil: + return &yaml.Node{Kind: yaml.ScalarNode, Value: "null", Style: style} + case string: + // A string that reads as another scalar (bool, number or empty) is quoted. + if v == "" || v == "true" || v == "false" || isNumber(v) { + style = yaml.DoubleQuotedStyle + } + return &yaml.Node{Kind: yaml.ScalarNode, Value: v, Style: style} + case bool: + return &yaml.Node{Kind: yaml.ScalarNode, Value: strconv.FormatBool(v), Style: style} + case int: + return &yaml.Node{Kind: yaml.ScalarNode, Value: strconv.Itoa(v), Style: style} + case int64: + return &yaml.Node{Kind: yaml.ScalarNode, Value: strconv.FormatInt(v, 10), Style: style} + case float64: + return &yaml.Node{Kind: yaml.ScalarNode, Value: fmt.Sprint(v), Style: style} + default: + // Panic because we only want to deal with known types. + panic(fmt.Sprintf("invalid type: %T", v)) + } +} + +func isNumber(s string) bool { + if _, err := strconv.ParseInt(s, 0, 64); err == nil { + return true + } + _, err := strconv.ParseFloat(s, 64) + return err == nil +} + +// Save writes v as YAML to path, creating parent directories. It fails if path exists +// and force is not set. See [Node] for styles. +func Save(path string, v any, force bool, styles map[string]yaml.Style) error { + err := os.MkdirAll(filepath.Dir(path), 0o755) + if err != nil { + return err + } + + info, err := os.Stat(path) + if err == nil { + if info.IsDir() { + return fmt.Errorf("%s is a directory", path) + } + if !force { + return fmt.Errorf("%s already exists. Use --force to overwrite", path) + } + } + + file, err := os.Create(path) + if err != nil { + return err + } + defer file.Close() + + enc := yaml.NewEncoder(file) + enc.SetIndent(2) + return enc.Encode(Node(v, styles)) +} diff --git a/libs/structs/structyaml/structyaml_test.go b/libs/structs/structyaml/structyaml_test.go new file mode 100644 index 00000000000..6c62fcfcac2 --- /dev/null +++ b/libs/structs/structyaml/structyaml_test.go @@ -0,0 +1,163 @@ +package structyaml + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.yaml.in/yaml/v3" +) + +type testStruct struct { + Name string `json:"name"` + Map map[string]string `json:"map"` + List []string `json:"list"` + LongNameField string `json:"long_name_field"` + ForceSendFields []string `json:"-"` + Format string `json:"format"` +} + +func newTestStruct() *testStruct { + return &testStruct{ + Name: "test", + Map: map[string]string{"key2": "value2", "key1": "value1"}, + List: []string{"a", "b", "c"}, + ForceSendFields: []string{"Name"}, + LongNameField: "long name goes here", + } +} + +func TestStruct(t *testing.T) { + result, err := Struct(newTestStruct(), "format") + require.NoError(t, err) + assert.Equal(t, M( + "list", []any{"a", "b", "c"}, + "long_name_field", "long name goes here", + "map", M("key1", "value1", "key2", "value2"), + "name", "test", + ), result) +} + +func TestStructWithOrder(t *testing.T) { + result, err := Struct(newTestStruct(), "format") + require.NoError(t, err) + assert.Equal(t, M( + "list", []any{"a", "b", "c"}, + "name", "test", + "map", M("key1", "value1", "key2", "value2"), + "long_name_field", "long name goes here", + ), result.Order("list", "name", "map")) +} + +func TestStructNotAMap(t *testing.T) { + _, err := Struct([]string{"a"}) + assert.ErrorContains(t, err, "expected map") +} + +func TestOrderUnknownKeysKeepOrder(t *testing.T) { + m := M("d", 1, "c", 2, "b", 3, "a", 4) + assert.Equal(t, M("a", 4, "b", 3, "d", 1, "c", 2), m.Order("a", "x", "b")) +} + +func TestValueKeepsFieldOrder(t *testing.T) { + type inner struct { + Zeta string `json:"zeta"` + Alpha string `json:"alpha"` + } + assert.Equal(t, []any{M("zeta", "z", "alpha", "a")}, Value([]inner{{Zeta: "z", Alpha: "a"}})) + assert.Nil(t, Value([]inner(nil))) +} + +func TestNodeScalars(t *testing.T) { + tests := []struct { + value any + want string + style yaml.Style + }{ + {nil, "null", 0}, + {1, "1", 0}, + {int64(291), "291", 0}, + {1.0, "1", 0}, + {true, "true", 0}, + {"value", "value", 0}, + {"0x123", "0x123", yaml.DoubleQuotedStyle}, + {"0b101", "0b101", yaml.DoubleQuotedStyle}, + {"0123", "0123", yaml.DoubleQuotedStyle}, + {"1.0", "1.0", yaml.DoubleQuotedStyle}, + {"true", "true", yaml.DoubleQuotedStyle}, + {"", "", yaml.DoubleQuotedStyle}, + } + for _, tt := range tests { + n := Node(tt.value, nil) + assert.Equal(t, yaml.ScalarNode, n.Kind) + assert.Equal(t, tt.want, n.Value) + assert.Equal(t, tt.style, n.Style, "%v", tt.value) + } +} + +func TestNodeSequence(t *testing.T) { + for _, v := range []any{[]any{"value1", "value2"}, []string{"value1", "value2"}} { + n := Node(v, nil) + assert.Equal(t, yaml.SequenceNode, n.Kind) + assert.Equal(t, "value1", n.Content[0].Value) + assert.Equal(t, "value2", n.Content[1].Value) + } +} + +func TestNodeMapKeepsOrder(t *testing.T) { + n := Node(M("key3", "value3", "key1", "value1"), nil) + assert.Equal(t, yaml.MappingNode, n.Kind) + assert.Equal(t, "key3", n.Content[0].Value) + assert.Equal(t, "value3", n.Content[1].Value) + assert.Equal(t, "key1", n.Content[2].Value) + assert.Equal(t, "value1", n.Content[3].Value) +} + +func TestNodeGoMapSortsKeys(t *testing.T) { + n := Node(map[string]any{"b": 1, "a": M("y", 1, "x", 2)}, nil) + assert.Equal(t, "a", n.Content[0].Value) + assert.Equal(t, "y", n.Content[1].Content[0].Value) + assert.Equal(t, "b", n.Content[2].Value) +} + +func TestNodeStyle(t *testing.T) { + styles := map[string]yaml.Style{"styled": yaml.DoubleQuotedStyle} + n := Node(M( + "styled", M("key1", "value1", "key2", "value2"), + "unstyled", M("key3", "value3"), + ), styles) + + styled := n.Content[1] + assert.Equal(t, yaml.MappingNode, styled.Kind) + for _, c := range styled.Content { + assert.Equal(t, yaml.DoubleQuotedStyle, c.Style) + } + + unstyled := n.Content[3] + assert.Equal(t, yaml.MappingNode, unstyled.Kind) + for _, c := range unstyled.Content { + assert.Equal(t, yaml.Style(0), c.Style) + } +} + +func TestSave(t *testing.T) { + path := filepath.Join(t.TempDir(), "dir", "out.yml") + require.NoError(t, Save(path, M("b", "1", "a", M("c", []any{1, true})), false, nil)) + + data, err := os.ReadFile(path) + require.NoError(t, err) + assert.Equal(t, "b: \"1\"\na:\n c:\n - 1\n - true\n", string(data)) + + err = Save(path, M("a", 1), false, nil) + assert.ErrorContains(t, err, "already exists. Use --force to overwrite") + + require.NoError(t, Save(path, M("a", 1), true, nil)) + data, err = os.ReadFile(path) + require.NoError(t, err) + assert.Equal(t, "a: 1\n", string(data)) + + err = Save(filepath.Dir(path), M("a", 1), true, nil) + assert.ErrorContains(t, err, "is a directory") +} diff --git a/python/Taskfile.yml b/python/Taskfile.yml index 137b567050a..68c452e228f 100644 --- a/python/Taskfile.yml +++ b/python/Taskfile.yml @@ -16,7 +16,7 @@ tasks: desc: Run pydabs tests sources: - "**/*.py" - - ../libs/dyn/dynvar/testdata/reference_vectors.json + - ../libs/structs/structvar/testdata/reference_vectors.json - pyproject.toml - uv.lock - README.md diff --git a/python/codegen/codegen/packages.py b/python/codegen/codegen/packages.py index 46674518dca..69d4944fa9a 100644 --- a/python/codegen/codegen/packages.py +++ b/python/codegen/codegen/packages.py @@ -63,7 +63,7 @@ def _load_resource_namespace() -> dict[str, str]: # pass through untyped. The Go config mutators validate the value. "interface": "Any", # time.Time is a scalar serialized as an RFC3339 string; the Go side models - # it as a string too (see libs/dyn/convert/sdk_native_types.go). + # it as a string too (see libs/structs/structvar/struct_info.go). "time.Time": "str", # duration.Duration is a scalar serialized as a seconds string (e.g. "3600s"). "duration.Duration": "str", diff --git a/python/databricks/bundles/core/_transform.py b/python/databricks/bundles/core/_transform.py index 8e3b4a6ec12..c39b3555869 100644 --- a/python/databricks/bundles/core/_transform.py +++ b/python/databricks/bundles/core/_transform.py @@ -19,7 +19,7 @@ _T = TypeVar("_T") -# Same strings as normalizeBool in libs/dyn/convert/normalize.go. +# Same strings as the bool decoding in libs/structs/structvar/decode.go. # See https://yaml.org/type/bool.html. _TRUE_STRINGS = [ "true", @@ -311,8 +311,8 @@ def _unwrap_variable(tpe: type) -> Optional[type]: # Regex for string corresponding to variables. # -# The source of truth is regex in libs/dyn/dynvar/ref.go. -# Behavioral parity is enforced by libs/dyn/dynvar/testdata/reference_vectors.json. +# The source of truth is regex in libs/structs/structvar/ref.go. +# Behavioral parity is enforced by libs/structs/structvar/testdata/reference_vectors.json. # # The (? Date: Wed, 7 Oct 2026 16:24:55 +0200 Subject: [PATCH 03/22] structvar: map YAML locations in the YAML source; drop mappedSource Co-authored-by: Isaac --- libs/structs/structvar/decode.go | 34 ++------------------------- libs/structs/structvar/decode_yaml.go | 34 +++++++++++++++++++++------ 2 files changed, 29 insertions(+), 39 deletions(-) diff --git a/libs/structs/structvar/decode.go b/libs/structs/structvar/decode.go index 7c74ddcdee9..2ea9d008e64 100644 --- a/libs/structs/structvar/decode.go +++ b/libs/structs/structvar/decode.go @@ -69,12 +69,9 @@ func DecodeYAMLNode(file string, node *yaml.Node, dst any, mapLocations Location if err := validate(src); err != nil { return nil, nil, err } + src.mapLocations = mapLocations var d decoder - var s source = src - if mapLocations != nil { - s = mappedSource{source: src, fn: mapLocations} - } - n, ok, err := d.decode(v, s, nil) + n, ok, err := d.decode(v, src, nil) if err == nil && !ok { // The value could not be converted to dst at all (e.g. a list for a struct). err = fmt.Errorf("expected a %s, found a %s", kindOf(v.Type()), src.kind()) @@ -727,33 +724,6 @@ func (s viewSource) elems() ([]source, error) { return out, nil } -// mappedSource is a [source] whose locations are mapped by fn. -type mappedSource struct { - source - path *structpath.PathNode - fn LocationMapper -} - -func (s mappedSource) locations() []diag.Location { - return s.fn(s.path, s.source.locations()) -} - -func (s mappedSource) pairs() ([]sourcePair, error) { - pairs, err := s.source.pairs() - for i := range pairs { - pairs[i].value = mappedSource{source: pairs[i].value, path: structpath.NewStringKey(s.path, pairs[i].key), fn: s.fn} - } - return pairs, err -} - -func (s mappedSource) elems() ([]source, error) { - elems, err := s.source.elems() - for i := range elems { - elems[i] = mappedSource{source: elems[i], path: structpath.NewIndex(s.path, i), fn: s.fn} - } - return elems, err -} - // kindOf returns the kind of configuration tree values of type t. func kindOf(t reflect.Type) Kind { for t.Kind() == reflect.Pointer { diff --git a/libs/structs/structvar/decode_yaml.go b/libs/structs/structvar/decode_yaml.go index ceb0634918c..f4b540ce44a 100644 --- a/libs/structs/structvar/decode_yaml.go +++ b/libs/structs/structvar/decode_yaml.go @@ -9,6 +9,7 @@ import ( "time" "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/structs/structpath" "go.yaml.in/yaml/v3" ) @@ -35,6 +36,10 @@ type yamlSource struct { // cyclic anchors. aliases *aliasChain + // If set, mapLocations is applied to the locations of the value at path. + mapLocations LocationMapper + path *structpath.PathNode + // Resolved scalar. k Kind v any @@ -198,7 +203,11 @@ func (s *yamlSource) kind() Kind { func (s *yamlSource) locations() []diag.Location { // An alias has the location of the anchored value. - return []diag.Location{s.loc(s.target)} + locs := []diag.Location{s.loc(s.target)} + if s.mapLocations != nil { + return s.mapLocations(s.path, locs) + } + return locs } func (s *yamlSource) anchor() bool { @@ -209,8 +218,18 @@ func (s *yamlSource) scalar() any { return s.v } -func (s *yamlSource) child(n *yaml.Node) (source, error) { - return newYAMLSource(s.file, n, s.aliases) +// child returns the source of n, the value at path below this one (computed by path, +// only when locations are mapped). +func (s *yamlSource) child(n *yaml.Node, path func() *structpath.PathNode) (*yamlSource, error) { + c, err := newYAMLSource(s.file, n, s.aliases) + if err != nil { + return nil, err + } + if s.mapLocations != nil { + c.mapLocations = s.mapLocations + c.path = path() + } + return c, nil } func (s *yamlSource) pairs() ([]sourcePair, error) { @@ -245,7 +264,7 @@ func (s *yamlSource) pairs() ([]sourcePair, error) { return nil, yamlErrorf(loc, "invalid key tag: %v", st) } - v, err := s.child(val) + v, err := s.child(val, func() *structpath.PathNode { return structpath.NewStringKey(s.path, key.Value) }) if err != nil { return nil, err } @@ -272,7 +291,8 @@ func (s *yamlSource) pairs() ([]sourcePair, error) { // Merged maps come first; the entries of this mapping take precedence. var out pairSet for _, n := range mnodes { - ms, err := s.child(n) + // The merged mapping's values are at the paths of this mapping's. + ms, err := s.child(n, func() *structpath.PathNode { return s.path }) if err != nil { return nil, err } @@ -316,8 +336,8 @@ func (s *pairSet) set(p sourcePair) { func (s *yamlSource) elems() ([]source, error) { var out []source - for _, n := range s.target.Content { - v, err := s.child(n) + for i, n := range s.target.Content { + v, err := s.child(n, func() *structpath.PathNode { return structpath.NewIndex(s.path, i) }) if err != nil { return nil, err } From dc8c9b18dafec514a334f4940ab6fd8ecf10f437 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 7 Oct 2026 16:33:30 +0200 Subject: [PATCH 04/22] Use NewStringKeysSlice for single key paths Co-authored-by: Isaac --- bundle/apps/validate.go | 2 +- bundle/config/validate/single_node_cluster_test.go | 2 +- bundle/config/validate/validate_dashboard_etags.go | 2 +- bundle/config/validate/validate_genie_space_etags.go | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bundle/apps/validate.go b/bundle/apps/validate.go index e00a134bcf5..50caddfde23 100644 --- a/bundle/apps/validate.go +++ b/bundle/apps/validate.go @@ -167,7 +167,7 @@ func warnForAppResourcePermissions(b *bundle.Bundle, appKey string, app *resourc ref.permission, appKey, ), - Paths: []*structpath.PathNode{structpath.NewStringKeys(nil, "resources", "apps", appKey)}, + Paths: structpath.NewStringKeysSlice("resources", "apps", appKey), Locations: b.Config.GetLocations(appPath), }) } diff --git a/bundle/config/validate/single_node_cluster_test.go b/bundle/config/validate/single_node_cluster_test.go index fc32796b9a1..4cb9b52f352 100644 --- a/bundle/config/validate/single_node_cluster_test.go +++ b/bundle/config/validate/single_node_cluster_test.go @@ -121,7 +121,7 @@ func TestValidateSingleNodeClusterFailForInteractiveClusters(t *testing.T) { Summary: singleNodeWarningSummary, Detail: singleNodeWarningDetail, Locations: []diag.Location{{File: "a.yml", Line: 1, Column: 1}}, - Paths: []*structpath.PathNode{structpath.NewStringKeys(nil, "resources", "clusters", "foo")}, + Paths: structpath.NewStringKeysSlice("resources", "clusters", "foo"), }, }, diags) }) diff --git a/bundle/config/validate/validate_dashboard_etags.go b/bundle/config/validate/validate_dashboard_etags.go index 2510f52e78f..b0b61cf4d1b 100644 --- a/bundle/config/validate/validate_dashboard_etags.go +++ b/bundle/config/validate/validate_dashboard_etags.go @@ -27,7 +27,7 @@ func (v *validateDashboardEtags) Apply(ctx context.Context, b *bundle.Bundle) di { Severity: diag.Error, Summary: fmt.Sprintf("dashboard %q has an etag set. Etags must not be set in bundle configuration", dashboard.DisplayName), - Paths: []*structpath.PathNode{structpath.NewStringKeys(nil, "resources", "dashboards", k)}, + Paths: structpath.NewStringKeysSlice("resources", "dashboards", k), Locations: b.Config.GetLocationsOf(structpath.NewPath(nil, "resources", "dashboards", k)), }, } diff --git a/bundle/config/validate/validate_genie_space_etags.go b/bundle/config/validate/validate_genie_space_etags.go index 53252c720a3..f783da030e3 100644 --- a/bundle/config/validate/validate_genie_space_etags.go +++ b/bundle/config/validate/validate_genie_space_etags.go @@ -29,7 +29,7 @@ func (v *validateGenieSpaceEtags) Apply(ctx context.Context, b *bundle.Bundle) d { Severity: diag.Error, Summary: fmt.Sprintf("genie space %q has an etag set. Etags must not be set in bundle configuration", genieSpace.Title), - Paths: []*structpath.PathNode{structpath.NewStringKeys(nil, "resources", "genie_spaces", k)}, + Paths: structpath.NewStringKeysSlice("resources", "genie_spaces", k), Locations: b.Config.GetLocationsOf(structpath.NewPath(nil, "resources", "genie_spaces", k)), }, } From a703bfccec657543d74988e70cfd5588202d9af5 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 7 Oct 2026 18:05:38 +0200 Subject: [PATCH 05/22] Build literal diagnostic paths with NewStringKeysSlice Co-authored-by: Isaac --- bundle/config/mutator/paths/app_paths_visitor_test.go | 2 +- bundle/config/mutator/paths/artifact_paths_visitor_test.go | 2 +- bundle/config/mutator/paths/dashboard_paths_visitor_test.go | 2 +- bundle/config/validate/validate_artifact_path.go | 2 +- libs/flags/json_flag_test.go | 4 ++-- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/bundle/config/mutator/paths/app_paths_visitor_test.go b/bundle/config/mutator/paths/app_paths_visitor_test.go index 79dc3df22ef..42e17f67180 100644 --- a/bundle/config/mutator/paths/app_paths_visitor_test.go +++ b/bundle/config/mutator/paths/app_paths_visitor_test.go @@ -24,7 +24,7 @@ func TestAppPathsVisitor(t *testing.T) { } actual := collectVisitedPaths(t, root, VisitAppPaths) - expected := structpath.MustParsePaths("resources.apps.app0.source_code_path") + expected := structpath.NewStringKeysSlice("resources", "apps", "app0", "source_code_path") assert.ElementsMatch(t, expected, actual) } diff --git a/bundle/config/mutator/paths/artifact_paths_visitor_test.go b/bundle/config/mutator/paths/artifact_paths_visitor_test.go index 9e9cb2b9ce7..e7f85a15579 100644 --- a/bundle/config/mutator/paths/artifact_paths_visitor_test.go +++ b/bundle/config/mutator/paths/artifact_paths_visitor_test.go @@ -18,7 +18,7 @@ func TestArtifactPathsVisitor(t *testing.T) { } actual := collectVisitedPaths(t, root, VisitArtifactPaths) - expected := structpath.MustParsePaths("artifacts.artifact0.path") + expected := structpath.NewStringKeysSlice("artifacts", "artifact0", "path") assert.ElementsMatch(t, expected, actual) } diff --git a/bundle/config/mutator/paths/dashboard_paths_visitor_test.go b/bundle/config/mutator/paths/dashboard_paths_visitor_test.go index 88a0ef4679e..2bd720d5daf 100644 --- a/bundle/config/mutator/paths/dashboard_paths_visitor_test.go +++ b/bundle/config/mutator/paths/dashboard_paths_visitor_test.go @@ -21,7 +21,7 @@ func TestVisitDashboardPaths(t *testing.T) { } actual := collectVisitedPaths(t, root, VisitDashboardPaths) - expected := structpath.MustParsePaths("resources.dashboards.dashboard0.file_path") + expected := structpath.NewStringKeysSlice("resources", "dashboards", "dashboard0", "file_path") assert.ElementsMatch(t, expected, actual) } diff --git a/bundle/config/validate/validate_artifact_path.go b/bundle/config/validate/validate_artifact_path.go index 76d7fdc52f4..658cf6e1b8c 100644 --- a/bundle/config/validate/validate_artifact_path.go +++ b/bundle/config/validate/validate_artifact_path.go @@ -86,7 +86,7 @@ func (v *validateArtifactPath) Apply(ctx context.Context, b *bundle.Bundle) diag Summary: s, Severity: diag.Error, Locations: b.Config.GetLocations("workspace.artifact_path"), - Paths: structpath.MustParsePaths("workspace.artifact_path"), + Paths: structpath.NewStringKeysSlice("workspace", "artifact_path"), }, } } diff --git a/libs/flags/json_flag_test.go b/libs/flags/json_flag_test.go index f5142e48e6a..cb8208b60a7 100644 --- a/libs/flags/json_flag_test.go +++ b/libs/flags/json_flag_test.go @@ -259,7 +259,7 @@ func TestJsonUnmarshalWrongTypeReportsCorrectLocation(t *testing.T) { Column: 15, }, }, - Paths: structpath.MustParsePaths("job_id"), + Paths: structpath.NewStringKeysSlice("job_id"), }) } @@ -284,7 +284,7 @@ func TestJsonUnmarshalArrayInsteadOfIntReportsCorrectLocation(t *testing.T) { Column: 40, }, }, - Paths: structpath.MustParsePaths("new_settings.timeout_seconds"), + Paths: structpath.NewStringKeysSlice("new_settings", "timeout_seconds"), }) } From 6b66d26aae0125f4b5b1ea59f763425b48bc8be5 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 7 Oct 2026 18:11:38 +0200 Subject: [PATCH 06/22] Restack fixes: structvar grammar for structpath's PureReferenceToPath test Co-authored-by: Isaac --- .../mutator/python/python_mutator_test.go | 2 ++ cmd/apps/import.go | 5 ++--- libs/structs/structpath/path.go | 3 +-- libs/structs/structpath/reference_test.go | 6 +++--- libs/structs/structvar/ref.go | 17 +++++++---------- 5 files changed, 15 insertions(+), 18 deletions(-) diff --git a/bundle/config/mutator/python/python_mutator_test.go b/bundle/config/mutator/python/python_mutator_test.go index bf5ea03eaab..bbba487332a 100644 --- a/bundle/config/mutator/python/python_mutator_test.go +++ b/bundle/config/mutator/python/python_mutator_test.go @@ -12,7 +12,9 @@ import ( "testing" "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/bundle/env" + "github.com/stretchr/testify/require" "github.com/databricks/cli/bundle" diff --git a/cmd/apps/import.go b/cmd/apps/import.go index dc679fdce58..4aefe33f057 100644 --- a/cmd/apps/import.go +++ b/cmd/apps/import.go @@ -12,8 +12,6 @@ import ( "slices" "strings" - "go.yaml.in/yaml/v3" - "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/generate" "github.com/databricks/cli/bundle/phases" @@ -27,13 +25,14 @@ import ( "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/env" "github.com/databricks/cli/libs/logdiag" + "github.com/databricks/cli/libs/structs/structyaml" "github.com/databricks/cli/libs/textutil" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/service/apps" "github.com/databricks/databricks-sdk-go/service/iam" "github.com/databricks/databricks-sdk-go/service/workspace" "github.com/spf13/cobra" - "github.com/databricks/cli/libs/structs/structyaml" + "go.yaml.in/yaml/v3" ) func newImportCommand() *cobra.Command { diff --git a/libs/structs/structpath/path.go b/libs/structs/structpath/path.go index 4aa4f4aa9be..2d02187b58d 100644 --- a/libs/structs/structpath/path.go +++ b/libs/structs/structpath/path.go @@ -9,7 +9,7 @@ import ( ) // baseVarDef matches a path segment of a ${...} reference. It is the grammar of -// dynvar.BaseVarDef, which structpath cannot import; TestPureReferenceMatchesDynvar +// structvar.BaseVarDef, which structpath cannot import; TestPureReferenceMatchesStructvar // keeps them in agreement. const baseVarDef = `_*\p{L}+([-_]*[\p{L}\p{N}]+)*` @@ -761,7 +761,6 @@ func isValidField(s string) bool { } // PureReferenceToPath returns a PathNode if s is a pure variable reference, otherwise false. -// This function is similar to dynvar.PureReferenceToPath but returns a *PathNode instead of dyn.Path. func PureReferenceToPath(s string) (*PathNode, bool) { m := pureReference.FindStringSubmatch(s) if m == nil { diff --git a/libs/structs/structpath/reference_test.go b/libs/structs/structpath/reference_test.go index 485b756c7fb..45360c24b4b 100644 --- a/libs/structs/structpath/reference_test.go +++ b/libs/structs/structpath/reference_test.go @@ -3,12 +3,12 @@ package structpath_test import ( "testing" - "github.com/databricks/cli/libs/dyn/dynvar" "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" "github.com/stretchr/testify/assert" ) -func TestPureReferenceMatchesDynvar(t *testing.T) { +func TestPureReferenceMatchesStructvar(t *testing.T) { for _, s := range []string{ "${var.foo}", "${resources.jobs.foo.tasks[1].env.key}", @@ -25,6 +25,6 @@ func TestPureReferenceMatchesDynvar(t *testing.T) { "plain_string", } { _, ok := structpath.PureReferenceToPath(s) - assert.Equal(t, dynvar.IsPureVariableReference(s), ok, s) + assert.Equal(t, structvar.IsPureVariableReference(s), ok, s) } } diff --git a/libs/structs/structvar/ref.go b/libs/structs/structvar/ref.go index d19421c8015..b619ae096d2 100644 --- a/libs/structs/structvar/ref.go +++ b/libs/structs/structvar/ref.go @@ -4,6 +4,8 @@ import ( "fmt" "regexp" "strings" + + "github.com/databricks/cli/libs/structs/structpath" ) // BaseVarDef matches a single dot-separated path segment in ${...} references. @@ -147,19 +149,14 @@ func ContainsVariableReference(s string) bool { return false } -// If s is a pure variable reference, this function returns the corresponding -// dyn.Path. Otherwise, it returns false. -func PureReferenceToPath(s string) (dyn.Path, bool) { - ref, ok := NewRef(dyn.V(s)) - if !ok { - return nil, false - } - - if !ref.IsPure() { +// PureReferenceToPath returns the path s refers to if s is a pure variable reference. +func PureReferenceToPath(s string) (*structpath.PathNode, bool) { + ref, ok := NewRef(s) + if !ok || !ref.IsPure() { return nil, false } - p, err := dyn.NewPathFromString(ref.References()[0]) + p, err := structpath.ParsePath(ref.References()[0]) if err != nil { return nil, false } From 9f08470b10c04e7167e87ad5db7ece5231bd2333 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 7 Oct 2026 18:17:28 +0200 Subject: [PATCH 07/22] Restore import block after restack Co-authored-by: Isaac --- .../mutator/resourcemutator/expand_pipeline_glob_paths_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/bundle/config/mutator/resourcemutator/expand_pipeline_glob_paths_test.go b/bundle/config/mutator/resourcemutator/expand_pipeline_glob_paths_test.go index 515d82f81f8..1a8e331caf4 100644 --- a/bundle/config/mutator/resourcemutator/expand_pipeline_glob_paths_test.go +++ b/bundle/config/mutator/resourcemutator/expand_pipeline_glob_paths_test.go @@ -7,7 +7,6 @@ import ( "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/bundle/config/mutator/resourcemutator" - "github.com/stretchr/testify/assert" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" From 18b64b844059c03415da833f296391273c4e6845 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 7 Oct 2026 23:11:22 +0200 Subject: [PATCH 08/22] Use structpath.NewPath for key/index paths Co-authored-by: Isaac --- bundle/apps/validate.go | 2 +- bundle/artifacts/expand_globs.go | 4 ++-- .../apply_source_linked_deployment_preset.go | 4 ++-- bundle/config/mutator/drop_empty_strings.go | 2 +- bundle/config/mutator/load_dbalert_files.go | 2 +- bundle/config/mutator/normalize_paths.go | 2 +- .../mutator/paths/app_paths_visitor_test.go | 2 +- .../mutator/paths/artifact_paths_visitor_test.go | 2 +- .../mutator/paths/dashboard_paths_visitor_test.go | 2 +- .../config/mutator/resolve_variable_references.go | 2 +- .../config/mutator/resourcemutator/keyed_merge.go | 2 +- .../config/mutator/resourcemutator/merge_apps.go | 4 ++-- .../mutator/resourcemutator/merge_grants.go | 4 ++-- .../mutator/resourcemutator/resource_mutator.go | 2 +- .../resourcemutator/secret_scope_fixups.go | 2 +- bundle/config/mutator/set_variables.go | 2 +- .../mutator/validate_secret_value_is_variable.go | 4 ++-- bundle/config/root.go | 4 ++-- .../config/validate/all_resources_have_values.go | 2 +- .../validate/interpolation_in_auth_config.go | 2 +- bundle/config/validate/job_cluster_key_defined.go | 6 +++--- bundle/config/validate/job_task_cluster_spec.go | 8 ++++---- .../no_variable_reference_in_resource_key.go | 6 +++--- .../validate/pipeline_development_deprecated.go | 2 +- bundle/config/validate/required.go | 10 +++++----- bundle/config/validate/scripts.go | 4 ++-- .../config/validate/single_node_cluster_test.go | 2 +- bundle/config/validate/unique_resource_keys.go | 4 ++-- bundle/config/validate/validate_artifact_path.go | 2 +- .../config/validate/validate_dashboard_etags.go | 2 +- .../config/validate/validate_deployment_fields.go | 10 +++++----- .../config/validate/validate_genie_space_etags.go | 2 +- .../validate_job_run_idempotency_token.go | 2 +- bundle/configsync/sequences.go | 2 +- bundle/configsync/variables.go | 2 +- .../deploy/check_dashboards_modified_remotely.go | 2 +- bundle/libraries/expand_glob_references.go | 2 +- bundle/phases/plan.go | 2 +- bundle/statemgmt/state_load.go | 2 +- cmd/bundle/run.go | 2 +- libs/flags/json_flag_test.go | 4 ++-- libs/structs/structpath/path_test.go | 15 +++++++++------ 42 files changed, 74 insertions(+), 71 deletions(-) diff --git a/bundle/apps/validate.go b/bundle/apps/validate.go index 50caddfde23..d2b4355c635 100644 --- a/bundle/apps/validate.go +++ b/bundle/apps/validate.go @@ -167,7 +167,7 @@ func warnForAppResourcePermissions(b *bundle.Bundle, appKey string, app *resourc ref.permission, appKey, ), - Paths: structpath.NewStringKeysSlice("resources", "apps", appKey), + Paths: structpath.NewPathSlice("resources", "apps", appKey), Locations: b.Config.GetLocations(appPath), }) } diff --git a/bundle/artifacts/expand_globs.go b/bundle/artifacts/expand_globs.go index 0f5a7585aa2..6b1c46a8f43 100644 --- a/bundle/artifacts/expand_globs.go +++ b/bundle/artifacts/expand_globs.go @@ -43,7 +43,7 @@ func (e expandGlobs) Name() string { func (e expandGlobs) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { // Base path for this mutator. // This path is set with the list of expanded globs when done. - base := structpath.NewStringKeys(nil, "artifacts", e.name, "files") + base := structpath.NewPath(nil, "artifacts", e.name, "files") // Pattern to match the source key in the files sequence. pattern := structpath.NewPatternStringKey(structpath.NewPatternBracketStar(structpath.NewPatternStringKey(structpath.NewPatternStringKey(structpath.NewPatternStringKey(nil, "artifacts"), e.name), "files")), "source") @@ -113,7 +113,7 @@ func (e expandGlobs) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnosti b.Config.UpdateSequence(base, sources) for i, locs := range locations { if locs != nil { - b.Config.SetLocations(structpath.NewStringKey(structpath.NewIndex(base, i), "source"), locs) + b.Config.SetLocations(structpath.NewPath(base, i, "source"), locs) } } diff --git a/bundle/config/mutator/apply_source_linked_deployment_preset.go b/bundle/config/mutator/apply_source_linked_deployment_preset.go index 99e06d56d24..64f4a47bc91 100644 --- a/bundle/config/mutator/apply_source_linked_deployment_preset.go +++ b/bundle/config/mutator/apply_source_linked_deployment_preset.go @@ -14,8 +14,8 @@ import ( type applySourceLinkedDeploymentPreset struct{} func targetPresetPath(target string) *structpath.PathNode { - p := structpath.NewStringKeys(nil, "targets", target) - return structpath.NewStringKeys(p, "presets", "source_linked_deployment") + p := structpath.NewPath(nil, "targets", target) + return structpath.NewPath(p, "presets", "source_linked_deployment") } // Apply source-linked deployment preset diff --git a/bundle/config/mutator/drop_empty_strings.go b/bundle/config/mutator/drop_empty_strings.go index f80a7d1f583..243a2fb9f58 100644 --- a/bundle/config/mutator/drop_empty_strings.go +++ b/bundle/config/mutator/drop_empty_strings.go @@ -50,7 +50,7 @@ func (m *dropEmptyStrings) Apply(ctx context.Context, b *bundle.Bundle) diag.Dia } } for _, name := range apps { - err := b.Config.Set(structpath.NewStringKeys(appsPath, name, "description"), "") + err := b.Config.Set(structpath.NewPath(appsPath, name, "description"), "") if err != nil { return diag.FromErr(err) } diff --git a/bundle/config/mutator/load_dbalert_files.go b/bundle/config/mutator/load_dbalert_files.go index 00f30b920b9..b6914230731 100644 --- a/bundle/config/mutator/load_dbalert_files.go +++ b/bundle/config/mutator/load_dbalert_files.go @@ -56,7 +56,7 @@ func (m *loadDBAlertFiles) Apply(ctx context.Context, b *bundle.Bundle) diag.Dia continue } - alertPath := structpath.NewStringKeys(nil, "resources", "alerts", alertKey) + alertPath := structpath.NewPath(nil, "resources", "alerts", alertKey) alertV := b.Config.View().Lookup(alertPath) // No other fields other than allowedInYAML should be set in the bundle YAML. diff --git a/bundle/config/mutator/normalize_paths.go b/bundle/config/mutator/normalize_paths.go index bbf322b3429..9c44894f48c 100644 --- a/bundle/config/mutator/normalize_paths.go +++ b/bundle/config/mutator/normalize_paths.go @@ -76,7 +76,7 @@ func collectGitSourcePaths(b *bundle.Bundle) []*structpath.PathNode { continue } if job.GitSource != nil { - jobs = append(jobs, structpath.NewStringKeys(nil, "resources", "jobs", name)) + jobs = append(jobs, structpath.NewPath(nil, "resources", "jobs", name)) } } diff --git a/bundle/config/mutator/paths/app_paths_visitor_test.go b/bundle/config/mutator/paths/app_paths_visitor_test.go index 42e17f67180..bf08e3d94d2 100644 --- a/bundle/config/mutator/paths/app_paths_visitor_test.go +++ b/bundle/config/mutator/paths/app_paths_visitor_test.go @@ -24,7 +24,7 @@ func TestAppPathsVisitor(t *testing.T) { } actual := collectVisitedPaths(t, root, VisitAppPaths) - expected := structpath.NewStringKeysSlice("resources", "apps", "app0", "source_code_path") + expected := structpath.NewPathSlice("resources", "apps", "app0", "source_code_path") assert.ElementsMatch(t, expected, actual) } diff --git a/bundle/config/mutator/paths/artifact_paths_visitor_test.go b/bundle/config/mutator/paths/artifact_paths_visitor_test.go index e7f85a15579..980b284cb11 100644 --- a/bundle/config/mutator/paths/artifact_paths_visitor_test.go +++ b/bundle/config/mutator/paths/artifact_paths_visitor_test.go @@ -18,7 +18,7 @@ func TestArtifactPathsVisitor(t *testing.T) { } actual := collectVisitedPaths(t, root, VisitArtifactPaths) - expected := structpath.NewStringKeysSlice("artifacts", "artifact0", "path") + expected := structpath.NewPathSlice("artifacts", "artifact0", "path") assert.ElementsMatch(t, expected, actual) } diff --git a/bundle/config/mutator/paths/dashboard_paths_visitor_test.go b/bundle/config/mutator/paths/dashboard_paths_visitor_test.go index 2bd720d5daf..da360776db6 100644 --- a/bundle/config/mutator/paths/dashboard_paths_visitor_test.go +++ b/bundle/config/mutator/paths/dashboard_paths_visitor_test.go @@ -21,7 +21,7 @@ func TestVisitDashboardPaths(t *testing.T) { } actual := collectVisitedPaths(t, root, VisitDashboardPaths) - expected := structpath.NewStringKeysSlice("resources", "dashboards", "dashboard0", "file_path") + expected := structpath.NewPathSlice("resources", "dashboards", "dashboard0", "file_path") assert.ElementsMatch(t, expected, actual) } diff --git a/bundle/config/mutator/resolve_variable_references.go b/bundle/config/mutator/resolve_variable_references.go index bab21234c41..f1e779ad832 100644 --- a/bundle/config/mutator/resolve_variable_references.go +++ b/bundle/config/mutator/resolve_variable_references.go @@ -285,7 +285,7 @@ func (m *resolveVariableReferences) resolveOnce(b *bundle.Bundle, prefixes []*st path := sp // Rewrite the shorthand path ${var.foo} into ${variables.foo.value}. if path.HasPrefix(varPath) { - path = structpath.Join(structpath.NewStringKeys(nil, "variables", path.KeyAt(1), "value"), path.SkipPrefix(2).AsSlice()...) + path = structpath.Join(structpath.NewPath(nil, "variables", path.KeyAt(1), "value"), path.SkipPrefix(2).AsSlice()...) } // If the path starts with "artifacts", we need to add a metric to track if this reference is used. diff --git a/bundle/config/mutator/resourcemutator/keyed_merge.go b/bundle/config/mutator/resourcemutator/keyed_merge.go index baf184f01b5..a263f3acbf2 100644 --- a/bundle/config/mutator/resourcemutator/keyed_merge.go +++ b/bundle/config/mutator/resourcemutator/keyed_merge.go @@ -35,7 +35,7 @@ func mergeByKey[R, E any]( if keyedMergeIsNoop(b, "resources."+resourceType+"."+name+"."+field, elems(r), noopKey, sortKeys) { continue } - path := structpath.NewStringKeys(nil, "resources", resourceType, name, field) + path := structpath.NewPath(nil, "resources", resourceType, name, field) if err := b.Config.MergeElementsByKey(path, keyField, keyFn, sortKeys); err != nil { return err } diff --git a/bundle/config/mutator/resourcemutator/merge_apps.go b/bundle/config/mutator/resourcemutator/merge_apps.go index 07396ce7537..d0646c2db5c 100644 --- a/bundle/config/mutator/resourcemutator/merge_apps.go +++ b/bundle/config/mutator/resourcemutator/merge_apps.go @@ -32,7 +32,7 @@ func (m *mergeApps) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostic if app == nil { continue } - path := structpath.NewStringKeys(nil, "resources", "apps", name, "resources") + path := structpath.NewPath(nil, "resources", "apps", name, "resources") if keyedMergeIsNoop(b, "resources.apps."+name+".resources", app.Resources, func(r apps.AppResource) string { return r.Name }, false) { continue } @@ -62,7 +62,7 @@ func (m *mergeApps) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostic if r.Name != "" { continue } - if err := b.Config.Set(structpath.NewStringKey(structpath.NewIndex(path, i), "name"), ""); err != nil { + if err := b.Config.Set(structpath.NewPath(path, i, "name"), ""); err != nil { return diag.FromErr(err) } } diff --git a/bundle/config/mutator/resourcemutator/merge_grants.go b/bundle/config/mutator/resourcemutator/merge_grants.go index 0702396a9df..ed3c7f7fe4b 100644 --- a/bundle/config/mutator/resourcemutator/merge_grants.go +++ b/bundle/config/mutator/resourcemutator/merge_grants.go @@ -66,7 +66,7 @@ func mergeGrantsOf[R any](b *bundle.Bundle, resourceType string, rs map[string]* // Merge grant entries by principal. This concatenates privileges // for entries with the same principal via the standard merge semantics. - path := structpath.NewStringKeys(nil, "resources", resourceType, name, "grants") + path := structpath.NewPath(nil, "resources", resourceType, name, "grants") err := b.Config.MergeElementsByKey(path, "principal", func(v structvar.View) string { s, _ := v.AsString() return s @@ -83,7 +83,7 @@ func mergeGrantsOf[R any](b *bundle.Bundle, resourceType string, rs map[string]* continue } gs[i].Privileges = privileges - b.Config.UpdateSequence(structpath.NewStringKey(structpath.NewIndex(path, i), "privileges"), sources) + b.Config.UpdateSequence(structpath.NewPath(path, i, "privileges"), sources) } } return nil diff --git a/bundle/config/mutator/resourcemutator/resource_mutator.go b/bundle/config/mutator/resourcemutator/resource_mutator.go index 768f1e875f4..5f424eb82fd 100644 --- a/bundle/config/mutator/resourcemutator/resource_mutator.go +++ b/bundle/config/mutator/resourcemutator/resource_mutator.go @@ -326,7 +326,7 @@ func selectResources(b *bundle.Bundle, resourceKeys ResourceKeySet) func() error continue } - path := structpath.NewStringKeys(nil, "resources", resourceType, name) + path := structpath.NewPath(nil, "resources", resourceType, name) err := b.Config.Assign(path, updatedView.Lookup(path)) if err != nil { return err diff --git a/bundle/config/mutator/resourcemutator/secret_scope_fixups.go b/bundle/config/mutator/resourcemutator/secret_scope_fixups.go index 3c7b4a033c3..468d324905d 100644 --- a/bundle/config/mutator/resourcemutator/secret_scope_fixups.go +++ b/bundle/config/mutator/resourcemutator/secret_scope_fixups.go @@ -131,7 +131,7 @@ func (m *secretScopeFixups) Apply(ctx context.Context, b *bundle.Bundle) diag.Di currentUser := b.Config.Workspace.CurrentUser.User addManageForCurrentUser(scope, currentUser) - path := structpath.NewStringKeys(nil, "resources", "secret_scopes", key) + path := structpath.NewPath(nil, "resources", "secret_scopes", key) err := collapsePermissions(scope) if err != nil { return diag.Diagnostics{ diff --git a/bundle/config/mutator/set_variables.go b/bundle/config/mutator/set_variables.go index 6060d1d0bc1..c12df9f6d3c 100644 --- a/bundle/config/mutator/set_variables.go +++ b/bundle/config/mutator/set_variables.go @@ -37,7 +37,7 @@ func setVariable(ctx context.Context, cfg *config.Root, variable *variable.Varia return nil } - variablePath := structpath.NewStringKeys(nil, "variables", name) + variablePath := structpath.NewPath(nil, "variables", name) valuePath := structpath.NewStringKey(variablePath, "value") // case: read and set variable value from process environment diff --git a/bundle/config/mutator/validate_secret_value_is_variable.go b/bundle/config/mutator/validate_secret_value_is_variable.go index 70770022221..467f8a6d2ae 100644 --- a/bundle/config/mutator/validate_secret_value_is_variable.go +++ b/bundle/config/mutator/validate_secret_value_is_variable.go @@ -25,7 +25,7 @@ func (v *validateSecretValueIsVariable) Apply(ctx context.Context, b *bundle.Bun // Iterate over all secrets in the bundle for key := range b.Config.Resources.Secrets { - p := structpath.NewStringKeys(nil, "resources", "secrets", key, "value") + p := structpath.NewPath(nil, "resources", "secrets", key, "value") val := b.Config.View().Lookup(p) if !val.IsValid() { diags = append(diags, diag.Diagnostic{ @@ -89,7 +89,7 @@ func (v *validateSecretValueIsVariable) checkVariableDefault(b *bundle.Bundle, s } // The default path in the dynamic config for the variable's default field. - defaultPath := structpath.NewStringKeys(nil, "variables", varName, "default") + defaultPath := structpath.NewPath(nil, "variables", varName, "default") locations := append(b.Config.LocationsAt(defaultPath), val.Locations()...) return diag.Diagnostics{{ diff --git a/bundle/config/root.go b/bundle/config/root.go index 34c2c9dee63..2a2870b2d6d 100644 --- a/bundle/config/root.go +++ b/bundle/config/root.go @@ -206,7 +206,7 @@ func (r *Root) Merge(others ...*Root) error { var bundleGitPath = structpath.MustParsePath("bundle.git") func (r *Root) MergeTargetOverrides(name string) error { - targetPath := structpath.NewStringKeys(nil, "targets", name) + targetPath := structpath.NewPath(nil, "targets", name) target := r.View().Lookup(targetPath) if !target.IsValid() { return fmt.Errorf("target %s not found", name) @@ -245,7 +245,7 @@ func (r *Root) MergeTargetOverrides(name string) error { // Merge `variables`. This field must be overwritten if set, not merged. for varName, variable := range target.Get("variables").MapItems() { - varPath := structpath.NewStringKeys(nil, "variables", varName) + varPath := structpath.NewPath(nil, "variables", varName) if vDefault := variable.Get("default"); vDefault.IsValid() { if err := r.Assign(structpath.NewStringKey(varPath, "default"), vDefault); err != nil { diff --git a/bundle/config/validate/all_resources_have_values.go b/bundle/config/validate/all_resources_have_values.go index 166115d5557..9e3aecb9743 100644 --- a/bundle/config/validate/all_resources_have_values.go +++ b/bundle/config/validate/all_resources_have_values.go @@ -37,7 +37,7 @@ func (m *allResourcesHaveValues) Apply(ctx context.Context, b *bundle.Bundle) di // singular. rType := strings.TrimSuffix(group.Description.PluralName, "s") - p := structpath.NewStringKeys(nil, "resources", group.Description.PluralName, rName) + p := structpath.NewPath(nil, "resources", group.Description.PluralName, rName) diags = append(diags, diag.Diagnostic{ Severity: diag.Error, Summary: fmt.Sprintf("%s %s is not defined", rType, rName), diff --git a/bundle/config/validate/interpolation_in_auth_config.go b/bundle/config/validate/interpolation_in_auth_config.go index e1b9d6fba18..a282b72790e 100644 --- a/bundle/config/validate/interpolation_in_auth_config.go +++ b/bundle/config/validate/interpolation_in_auth_config.go @@ -51,7 +51,7 @@ func (f *noInterpolationInAuthConfig) Apply(ctx context.Context, b *bundle.Bundl diags := diag.Diagnostics{} for _, fieldName := range authFields { - p := structpath.NewStringKeys(nil, "workspace", fieldName) + p := structpath.NewPath(nil, "workspace", fieldName) v := b.Config.View().Lookup(p) if !v.IsValid() { continue diff --git a/bundle/config/validate/job_cluster_key_defined.go b/bundle/config/validate/job_cluster_key_defined.go index f3641a658ae..23f7f4161c0 100644 --- a/bundle/config/validate/job_cluster_key_defined.go +++ b/bundle/config/validate/job_cluster_key_defined.go @@ -31,14 +31,14 @@ func (v *jobClusterKeyDefined) Apply(ctx context.Context, b *bundle.Bundle) diag } for index, task := range job.Tasks { - taskPath := structpath.NewIndex(structpath.NewStringKeys(nil, "resources", "jobs", k, "tasks"), index) + taskPath := structpath.NewPath(nil, "resources", "jobs", k, "tasks", index) diags = diags.Extend(checkJobClusterKey(b, jobClusterKeys, task.JobClusterKey, - structpath.NewStringKeys(taskPath, "job_cluster_key"))) + structpath.NewPath(taskPath, "job_cluster_key"))) // The Jobs API rejects nested for_each_task, so one level is sufficient. if task.ForEachTask != nil { diags = diags.Extend(checkJobClusterKey(b, jobClusterKeys, task.ForEachTask.Task.JobClusterKey, - structpath.NewStringKeys(taskPath, "for_each_task", "task", "job_cluster_key"))) + structpath.NewPath(taskPath, "for_each_task", "task", "job_cluster_key"))) } } } diff --git a/bundle/config/validate/job_task_cluster_spec.go b/bundle/config/validate/job_task_cluster_spec.go index 66f7bf8fc3d..56191d114b7 100644 --- a/bundle/config/validate/job_task_cluster_spec.go +++ b/bundle/config/validate/job_task_cluster_spec.go @@ -26,13 +26,13 @@ func (v *jobTaskClusterSpec) Name() string { func (v *jobTaskClusterSpec) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { diags := diag.Diagnostics{} - jobsPath := structpath.NewStringKeys(nil, "resources", "jobs") + jobsPath := structpath.NewPath(nil, "resources", "jobs") for resourceName, job := range b.Config.Resources.Jobs { - resourcePath := structpath.NewStringKeys(jobsPath, resourceName) + resourcePath := structpath.NewPath(jobsPath, resourceName) for taskIndex, task := range job.Tasks { - taskPath := structpath.NewIndex(structpath.NewStringKeys(resourcePath, "tasks"), taskIndex) + taskPath := structpath.NewPath(resourcePath, "tasks", taskIndex) diags = diags.Extend(validateJobTask(b, task, taskPath)) } @@ -72,7 +72,7 @@ func validateJobTask(b *bundle.Bundle, task jobs.Task, taskPath *structpath.Path } if task.ForEachTask != nil { - forEachTaskPath := structpath.NewStringKeys(taskPath, "for_each_task", "task") + forEachTaskPath := structpath.NewPath(taskPath, "for_each_task", "task") diags = diags.Extend(validateJobTask(b, task.ForEachTask.Task, forEachTaskPath)) } diff --git a/bundle/config/validate/no_variable_reference_in_resource_key.go b/bundle/config/validate/no_variable_reference_in_resource_key.go index 743d47bd7fd..a2db01ffa14 100644 --- a/bundle/config/validate/no_variable_reference_in_resource_key.go +++ b/bundle/config/validate/no_variable_reference_in_resource_key.go @@ -34,7 +34,7 @@ func (m *noVariableReferenceInResourceKey) Apply(_ context.Context, b *bundle.Bu if !structvar.ContainsVariableReference(key) { continue } - p := structpath.NewStringKeys(prefix, group.Description.PluralName, key) + p := structpath.NewPath(prefix, group.Description.PluralName, key) diags = append(diags, diag.Diagnostic{ Severity: diag.Error, Summary: fmt.Sprintf("resource key %q must not contain variable references", key), @@ -45,10 +45,10 @@ func (m *noVariableReferenceInResourceKey) Apply(_ context.Context, b *bundle.Bu } } - check(structpath.NewStringKeys(nil, "resources"), &b.Config.Resources) + check(structpath.NewPath(nil, "resources"), &b.Config.Resources) for _, name := range slices.Sorted(maps.Keys(b.Config.Targets)) { if t := b.Config.Targets[name]; t != nil && t.Resources != nil { - check(structpath.NewStringKeys(nil, "targets", name, "resources"), t.Resources) + check(structpath.NewPath(nil, "targets", name, "resources"), t.Resources) } } diff --git a/bundle/config/validate/pipeline_development_deprecated.go b/bundle/config/validate/pipeline_development_deprecated.go index d837daa5908..f6672a4646f 100644 --- a/bundle/config/validate/pipeline_development_deprecated.go +++ b/bundle/config/validate/pipeline_development_deprecated.go @@ -42,7 +42,7 @@ func (v *pipelineDevelopmentDeprecated) Apply(_ context.Context, b *bundle.Bundl continue } - p := structpath.NewStringKeys(nil, "resources", "pipelines", key, "development") + p := structpath.NewPath(nil, "resources", "pipelines", key, "development") // Only user-written values have a location; the value set by "mode: development" does not. locs := b.Config.LocationsAt(p) diff --git a/bundle/config/validate/required.go b/bundle/config/validate/required.go index b8bb148b8e4..3de90d4c335 100644 --- a/bundle/config/validate/required.go +++ b/bundle/config/validate/required.go @@ -97,11 +97,11 @@ func errorForMissingFields(ctx context.Context, b *bundle.Bundle) diag.Diagnosti for key, dashboard := range b.Config.Resources.Dashboards { if dashboard.DisplayName == "" { nameLocations = append(nameLocations, b.Config.GetLocations("resources.dashboards."+key)...) - namePaths = append(namePaths, structpath.NewStringKeys(nil, "resources", "dashboards", key)) + namePaths = append(namePaths, structpath.NewPath(nil, "resources", "dashboards", key)) } if dashboard.WarehouseId == "" { warehouseIdLocations = append(warehouseIdLocations, b.Config.GetLocations("resources.dashboards."+key)...) - warehouseIdPaths = append(warehouseIdPaths, structpath.NewStringKeys(nil, "resources", "dashboards", key)) + warehouseIdPaths = append(warehouseIdPaths, structpath.NewPath(nil, "resources", "dashboards", key)) } } @@ -126,7 +126,7 @@ func errorForMissingFields(ctx context.Context, b *bundle.Bundle) diag.Diagnosti // by the backend, which rejects whitespace-only names (name.trim.nonEmpty). for key, warehouse := range b.Config.Resources.SqlWarehouses { if strings.TrimSpace(warehouse.Name) == "" { - path := structpath.NewStringKeys(nil, "resources", "sql_warehouses", key) + path := structpath.NewPath(nil, "resources", "sql_warehouses", key) diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, Summary: "sql_warehouse name is required", @@ -191,8 +191,8 @@ func errorForInvalidSecretScopePermissions(ctx context.Context, b *bundle.Bundle if perm.UserName != "" || perm.GroupName != "" || perm.ServicePrincipalName != "" { continue } - scopePath := structpath.NewStringKeys(nil, "resources", "secret_scopes", key) - path := structpath.NewIndex(structpath.NewStringKeys(scopePath, "permissions"), i) + scopePath := structpath.NewPath(nil, "resources", "secret_scopes", key) + path := structpath.NewPath(scopePath, "permissions", i) // ApplyBundlePermissions rebuilds permissions via convert.FromTyped and drops // per-entry locations, so point at the scope. diags = diags.Append(diag.Diagnostic{ diff --git a/bundle/config/validate/scripts.go b/bundle/config/validate/scripts.go index f7e5586d73e..328ba2404b6 100644 --- a/bundle/config/validate/scripts.go +++ b/bundle/config/validate/scripts.go @@ -38,7 +38,7 @@ func (f *validateScripts) Apply(ctx context.Context, b *bundle.Bundle) diag.Diag for _, k := range scriptKeys { script := b.Config.Scripts[k] - contentPath := structpath.NewStringKeys(nil, "scripts", k, "content") + contentPath := structpath.NewPath(nil, "scripts", k, "content") if script.Content == "" { diags = append(diags, diag.Diagnostic{ @@ -92,7 +92,7 @@ func validateScriptEnv(b *bundle.Bundle, key string, env map[string]string) diag continue } - envValuePath := structpath.NewStringKeys(nil, "scripts", key, "env", name) + envValuePath := structpath.NewPath(nil, "scripts", key, "env", name) for _, refPath := range ref.References() { prefix, _, _ := strings.Cut(refPath, ".") diff --git a/bundle/config/validate/single_node_cluster_test.go b/bundle/config/validate/single_node_cluster_test.go index 4cb9b52f352..1c8d79af4f2 100644 --- a/bundle/config/validate/single_node_cluster_test.go +++ b/bundle/config/validate/single_node_cluster_test.go @@ -121,7 +121,7 @@ func TestValidateSingleNodeClusterFailForInteractiveClusters(t *testing.T) { Summary: singleNodeWarningSummary, Detail: singleNodeWarningDetail, Locations: []diag.Location{{File: "a.yml", Line: 1, Column: 1}}, - Paths: structpath.NewStringKeysSlice("resources", "clusters", "foo"), + Paths: structpath.NewPathSlice("resources", "clusters", "foo"), }, }, diags) }) diff --git a/bundle/config/validate/unique_resource_keys.go b/bundle/config/validate/unique_resource_keys.go index 488edd4e8a6..bced70c6ffc 100644 --- a/bundle/config/validate/unique_resource_keys.go +++ b/bundle/config/validate/unique_resource_keys.go @@ -59,13 +59,13 @@ func (m *uniqueResourceKeys) Apply(ctx context.Context, b *bundle.Bundle) diag.D // Gather the paths and locations of all resources for _, group := range b.Config.Resources.AllResources() { for k := range group.Resources { - addLocationToMetadata(k, structpath.NewStringKeys(nil, "resources", group.Description.PluralName, k)) + addLocationToMetadata(k, structpath.NewPath(nil, "resources", group.Description.PluralName, k)) } } // track locations for all scripts. for k := range b.Config.Scripts { - addLocationToMetadata(k, structpath.NewStringKeys(nil, "scripts", k)) + addLocationToMetadata(k, structpath.NewPath(nil, "scripts", k)) } // If duplicate keys are found, report an error. diff --git a/bundle/config/validate/validate_artifact_path.go b/bundle/config/validate/validate_artifact_path.go index 658cf6e1b8c..1eaa41518cc 100644 --- a/bundle/config/validate/validate_artifact_path.go +++ b/bundle/config/validate/validate_artifact_path.go @@ -86,7 +86,7 @@ func (v *validateArtifactPath) Apply(ctx context.Context, b *bundle.Bundle) diag Summary: s, Severity: diag.Error, Locations: b.Config.GetLocations("workspace.artifact_path"), - Paths: structpath.NewStringKeysSlice("workspace", "artifact_path"), + Paths: structpath.NewPathSlice("workspace", "artifact_path"), }, } } diff --git a/bundle/config/validate/validate_dashboard_etags.go b/bundle/config/validate/validate_dashboard_etags.go index b0b61cf4d1b..1715f22cb84 100644 --- a/bundle/config/validate/validate_dashboard_etags.go +++ b/bundle/config/validate/validate_dashboard_etags.go @@ -27,7 +27,7 @@ func (v *validateDashboardEtags) Apply(ctx context.Context, b *bundle.Bundle) di { Severity: diag.Error, Summary: fmt.Sprintf("dashboard %q has an etag set. Etags must not be set in bundle configuration", dashboard.DisplayName), - Paths: structpath.NewStringKeysSlice("resources", "dashboards", k), + Paths: structpath.NewPathSlice("resources", "dashboards", k), Locations: b.Config.GetLocationsOf(structpath.NewPath(nil, "resources", "dashboards", k)), }, } diff --git a/bundle/config/validate/validate_deployment_fields.go b/bundle/config/validate/validate_deployment_fields.go index f824329adbf..10d95644d13 100644 --- a/bundle/config/validate/validate_deployment_fields.go +++ b/bundle/config/validate/validate_deployment_fields.go @@ -30,7 +30,7 @@ func (v *validateDeploymentFields) Apply(_ context.Context, b *bundle.Bundle) di if value == "" { return } - path := structpath.NewStringKeys(resource, "deployment", field) + path := structpath.NewPath(resource, "deployment", field) diags = append(diags, diag.Diagnostic{ Severity: diag.Error, Summary: field + " must not be set in bundle configuration; it is managed by Declarative Automation Bundles", @@ -41,14 +41,14 @@ func (v *validateDeploymentFields) Apply(_ context.Context, b *bundle.Bundle) di for name, job := range b.Config.Resources.Jobs { if d := job.Deployment; d != nil { - reject(structpath.NewStringKeys(nil, "resources", "jobs", name), "deployment_id", d.DeploymentId) - reject(structpath.NewStringKeys(nil, "resources", "jobs", name), "version_id", d.VersionId) + reject(structpath.NewPath(nil, "resources", "jobs", name), "deployment_id", d.DeploymentId) + reject(structpath.NewPath(nil, "resources", "jobs", name), "version_id", d.VersionId) } } for name, pipeline := range b.Config.Resources.Pipelines { if d := pipeline.Deployment; d != nil { - reject(structpath.NewStringKeys(nil, "resources", "pipelines", name), "deployment_id", d.DeploymentId) - reject(structpath.NewStringKeys(nil, "resources", "pipelines", name), "version_id", d.VersionId) + reject(structpath.NewPath(nil, "resources", "pipelines", name), "deployment_id", d.DeploymentId) + reject(structpath.NewPath(nil, "resources", "pipelines", name), "version_id", d.VersionId) } } diff --git a/bundle/config/validate/validate_genie_space_etags.go b/bundle/config/validate/validate_genie_space_etags.go index f783da030e3..963e572346f 100644 --- a/bundle/config/validate/validate_genie_space_etags.go +++ b/bundle/config/validate/validate_genie_space_etags.go @@ -29,7 +29,7 @@ func (v *validateGenieSpaceEtags) Apply(ctx context.Context, b *bundle.Bundle) d { Severity: diag.Error, Summary: fmt.Sprintf("genie space %q has an etag set. Etags must not be set in bundle configuration", genieSpace.Title), - Paths: structpath.NewStringKeysSlice("resources", "genie_spaces", k), + Paths: structpath.NewPathSlice("resources", "genie_spaces", k), Locations: b.Config.GetLocationsOf(structpath.NewPath(nil, "resources", "genie_spaces", k)), }, } diff --git a/bundle/config/validate/validate_job_run_idempotency_token.go b/bundle/config/validate/validate_job_run_idempotency_token.go index c211e5a6364..4ed6cc75694 100644 --- a/bundle/config/validate/validate_job_run_idempotency_token.go +++ b/bundle/config/validate/validate_job_run_idempotency_token.go @@ -32,7 +32,7 @@ func (v *validateJobRunIdempotencyToken) Apply(_ context.Context, b *bundle.Bund } // The CLI mints the token; a configured one would also remain reserved after // the run is deleted and break the next deploy. - path := structpath.NewStringKeys(nil, "resources", "job_runs", name, "idempotency_token") + path := structpath.NewPath(nil, "resources", "job_runs", name, "idempotency_token") diags = append(diags, diag.Diagnostic{ Severity: diag.Error, Summary: "idempotency_token must not be set in bundle configuration; the CLI sets it on each run-now request", diff --git a/bundle/configsync/sequences.go b/bundle/configsync/sequences.go index 1be8c0d484b..3a9bd84dd1e 100644 --- a/bundle/configsync/sequences.go +++ b/bundle/configsync/sequences.go @@ -155,7 +155,7 @@ func (r *blockResolver) regionPath(block sourceBlock, path *structpath.PathNode) if !block.override { return path } - return structpath.Join(structpath.NewStringKeys(nil, "targets", r.target), path.AsSlice()...) + return structpath.Join(structpath.NewPath(nil, "targets", r.target), path.AsSlice()...) } // candidatePath renders a resolved path the way the patch layer addresses it, diff --git a/bundle/configsync/variables.go b/bundle/configsync/variables.go index 5382ea1da1f..1030a912719 100644 --- a/bundle/configsync/variables.go +++ b/bundle/configsync/variables.go @@ -217,7 +217,7 @@ func resolveReferencePath(refStr string) (*structpath.PathNode, bool) { } if p.HasPrefix(varPrefix) && p.Len() >= 2 { - newPath := structpath.NewStringKeys(nil, "variables", p.KeyAt(1), "value") + newPath := structpath.NewPath(nil, "variables", p.KeyAt(1), "value") return structpath.Join(newPath, p.AsSlice()[2:]...), true } diff --git a/bundle/deploy/check_dashboards_modified_remotely.go b/bundle/deploy/check_dashboards_modified_remotely.go index 97288de568a..06e3e29f3a1 100644 --- a/bundle/deploy/check_dashboards_modified_remotely.go +++ b/bundle/deploy/check_dashboards_modified_remotely.go @@ -72,7 +72,7 @@ func (l *checkDashboardsModifiedRemotely) Apply(ctx context.Context, b *bundle.B continue } - path := structpath.NewStringKeys(nil, "resources", "dashboards", dashboard.Name) + path := structpath.NewPath(nil, "resources", "dashboards", dashboard.Name) loc := b.Config.GetLocation(path.String()) actual, err := b.WorkspaceClient(ctx).Lakeview.GetByDashboardId(ctx, dashboard.ID) if err != nil { diff --git a/bundle/libraries/expand_glob_references.go b/bundle/libraries/expand_glob_references.go index 632bf1be60e..9277c349100 100644 --- a/bundle/libraries/expand_glob_references.go +++ b/bundle/libraries/expand_glob_references.go @@ -106,7 +106,7 @@ func expandSequence[T any](ctx context.Context, b *bundle.Bundle, p *structpath. b.Config.UpdateSequence(p, sources) for i, relocate := range relocates { if relocate != "" { - b.Config.SetLocations(structpath.NewStringKey(structpath.NewIndex(p, i), relocate), locations[i]) + b.Config.SetLocations(structpath.NewPath(p, i, relocate), locations[i]) } } diff --git a/bundle/phases/plan.go b/bundle/phases/plan.go index 25516d2777d..0bd7fa5e432 100644 --- a/bundle/phases/plan.go +++ b/bundle/phases/plan.go @@ -64,7 +64,7 @@ func checkForPreventDestroy(b *bundle.Bundle, actions []deployplan.Action) error return fmt.Errorf("failed to parse %q", action.ResourceKey) } - path = structpath.NewStringKeys(path, "lifecycle", "prevent_destroy") + path = structpath.NewPath(path, "lifecycle", "prevent_destroy") preventDestroyV := root.Lookup(path) if !preventDestroyV.IsValid() { diff --git a/bundle/statemgmt/state_load.go b/bundle/statemgmt/state_load.go index a3327cf6688..2e3849c8bf9 100644 --- a/bundle/statemgmt/state_load.go +++ b/bundle/statemgmt/state_load.go @@ -120,7 +120,7 @@ func StateToBundle(ctx context.Context, state ExportedResourcesMap, cfg *config. continue } - path := structpath.NewStringKeys(resourcesPath, groupName, resourceName) + path := structpath.NewPath(resourcesPath, groupName, resourceName) if !cfg.View().Lookup(path).IsValid() { if err := cfg.Set(structpath.NewStringKey(path, "modified_status"), resources.ModifiedStatusDeleted); err != nil { return err diff --git a/cmd/bundle/run.go b/cmd/bundle/run.go index b6ce23c18f3..4ca32fd92a1 100644 --- a/cmd/bundle/run.go +++ b/cmd/bundle/run.go @@ -189,7 +189,7 @@ Example usage: // runner sees concrete values (e.g. an app's env vars referencing another // resource, or its source_code_path pointing at the immutable snapshot's // full_path, which lives only in the deployed state). - target := structpath.NewStringKeys(nil, "resources", ref.Description.PluralName, ref.Key) + target := structpath.NewPath(nil, "resources", ref.Description.PluralName, ref.Key) if err := b.DeploymentBundle.ResolveConfigAgainstState(&b.Config, target); err != nil { return err } diff --git a/libs/flags/json_flag_test.go b/libs/flags/json_flag_test.go index cb8208b60a7..2dc73942719 100644 --- a/libs/flags/json_flag_test.go +++ b/libs/flags/json_flag_test.go @@ -259,7 +259,7 @@ func TestJsonUnmarshalWrongTypeReportsCorrectLocation(t *testing.T) { Column: 15, }, }, - Paths: structpath.NewStringKeysSlice("job_id"), + Paths: structpath.NewPathSlice("job_id"), }) } @@ -284,7 +284,7 @@ func TestJsonUnmarshalArrayInsteadOfIntReportsCorrectLocation(t *testing.T) { Column: 40, }, }, - Paths: structpath.NewStringKeysSlice("new_settings", "timeout_seconds"), + Paths: structpath.NewPathSlice("new_settings", "timeout_seconds"), }) } diff --git a/libs/structs/structpath/path_test.go b/libs/structs/structpath/path_test.go index c0a735cfd42..58cf6d050cb 100644 --- a/libs/structs/structpath/path_test.go +++ b/libs/structs/structpath/path_test.go @@ -1275,10 +1275,12 @@ paths: assert.Contains(t, err.Error(), "unexpected end of input") } -func TestNewStringKeys(t *testing.T) { - assert.Equal(t, "resources.jobs['${var.env}_job']", NewStringKeys(nil, "resources", "jobs", "${var.env}_job").String()) - assert.Equal(t, "a.b.c", NewStringKeys(MustParsePath("a"), "b", "c").String()) - assert.Equal(t, "a", NewStringKeys(MustParsePath("a")).String()) +func TestNewPath(t *testing.T) { + assert.Equal(t, "a.b['c d']", NewPath(NewStringKey(nil, "a"), "b", "c d").String()) + assert.Equal(t, "resources.jobs.j.tasks[2].job_cluster_key", NewPath(nil, "resources", "jobs", "j", "tasks", 2, "job_cluster_key").String()) + assert.Nil(t, NewPath(nil)) + assert.Equal(t, "a", NewPath(NewStringKey(nil, "a")).String()) + assert.Panics(t, func() { NewPath(nil, int64(1)) }) } func TestJoin(t *testing.T) { @@ -1296,10 +1298,11 @@ func TestMustParsePaths(t *testing.T) { assert.Empty(t, MustParsePaths()) } -func TestNewStringKeysSlice(t *testing.T) { - paths := NewStringKeysSlice("resources", "jobs", "${var.env}_job") +func TestNewPathSlice(t *testing.T) { + paths := NewPathSlice("resources", "jobs", "${var.env}_job") require.Len(t, paths, 1) assert.Equal(t, "resources.jobs['${var.env}_job']", paths[0].String()) + assert.Equal(t, "sync.paths[3]", NewPathSlice("sync", "paths", 3)[0].String()) } func TestNewPattern(t *testing.T) { From 0671d48248a807437d759bedbd83d1423f397f0c Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 8 Oct 2026 13:34:01 +0200 Subject: [PATCH 09/22] Build the target preset path in one NewPath call Co-authored-by: Isaac --- bundle/config/mutator/apply_source_linked_deployment_preset.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bundle/config/mutator/apply_source_linked_deployment_preset.go b/bundle/config/mutator/apply_source_linked_deployment_preset.go index 64f4a47bc91..dd175aabd7d 100644 --- a/bundle/config/mutator/apply_source_linked_deployment_preset.go +++ b/bundle/config/mutator/apply_source_linked_deployment_preset.go @@ -14,8 +14,7 @@ import ( type applySourceLinkedDeploymentPreset struct{} func targetPresetPath(target string) *structpath.PathNode { - p := structpath.NewPath(nil, "targets", target) - return structpath.NewPath(p, "presets", "source_linked_deployment") + return structpath.NewPath(nil, "targets", target, "presets", "source_linked_deployment") } // Apply source-linked deployment preset From abb002ff49eb1adf90f11f5700372c68a02c6bc3 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 8 Oct 2026 18:09:03 +0200 Subject: [PATCH 10/22] GetLocationsOf on the location tree Co-authored-by: Isaac --- bundle/config/root.go | 29 ++++++++--------------------- 1 file changed, 8 insertions(+), 21 deletions(-) diff --git a/bundle/config/root.go b/bundle/config/root.go index 2a2870b2d6d..07c2933eb6b 100644 --- a/bundle/config/root.go +++ b/bundle/config/root.go @@ -565,34 +565,21 @@ func (r Root) GetLocations(path string) []diag.Location { if err != nil { return nil } - return r.locations.Nearest(p) -} - -// GetLocationOf is [Root.GetLocation] for a path node. -func (r Root) GetLocationOf(path *structpath.PathNode) diag.Location { - v, ok := r.valueOf(path) - if !ok { - return diag.Location{} - } - return v.Location() + return r.GetLocationsOf(p) } // GetLocationsOf is [Root.GetLocations] for a path node. func (r Root) GetLocationsOf(path *structpath.PathNode) []diag.Location { - v, ok := r.valueOf(path) - if !ok { - return nil - } - return v.Locations() + return r.locations.Nearest(path) } -func (r Root) valueOf(path *structpath.PathNode) (dyn.Value, bool) { - p, ok := dyn.FromStructPath(path) - if !ok { - return dyn.InvalidValue, false +// GetLocationOf is [Root.GetLocation] for a path node. +func (r Root) GetLocationOf(path *structpath.PathNode) diag.Location { + locs := r.GetLocationsOf(path) + if len(locs) == 0 { + return diag.Location{} } - v, err := dyn.GetByPath(r.value, p) - return v, err == nil + return locs[0] } // DefinitionLocation returns the primary location the value at path is defined at, From fbe25ac0dfdd93469669907d926b0b7e843bed67 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 8 Oct 2026 18:27:30 +0200 Subject: [PATCH 11/22] Minimal import edits; drop unused diag import Co-authored-by: Isaac --- .../mutator/python/apply_python_output_test.go | 2 +- bundle/config/mutator/python/python_mutator_test.go | 12 ++---------- .../resourcemutator/apply_bundle_permissions.go | 3 ++- .../expand_pipeline_glob_paths_test.go | 2 ++ .../mutator/resourcemutator/resource_mutator.go | 6 ++++-- .../mutator/resourcemutator/validate_target_mode.go | 3 ++- bundle/config/mutator/translate_paths.go | 5 +++-- bundle/config/resources_types_test.go | 5 +++-- bundle/libraries/upload.go | 1 + cmd/apps/import.go | 3 ++- .../assumptions/dashboard_assumptions_test.go | 1 + 11 files changed, 23 insertions(+), 20 deletions(-) diff --git a/bundle/config/mutator/python/apply_python_output_test.go b/bundle/config/mutator/python/apply_python_output_test.go index fcba01a24ae..7c807f0e2fa 100644 --- a/bundle/config/mutator/python/apply_python_output_test.go +++ b/bundle/config/mutator/python/apply_python_output_test.go @@ -7,9 +7,9 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/mutator/resourcemutator" - "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/cli/libs/structs/structvar" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/bundle/config/mutator/python/python_mutator_test.go b/bundle/config/mutator/python/python_mutator_test.go index bbba487332a..1b4adaed884 100644 --- a/bundle/config/mutator/python/python_mutator_test.go +++ b/bundle/config/mutator/python/python_mutator_test.go @@ -9,29 +9,21 @@ import ( "path/filepath" "runtime" "slices" + "strings" "testing" "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/cli/bundle/env" - "github.com/stretchr/testify/require" "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/config" - "github.com/databricks/cli/internal/testutil" - "github.com/databricks/cli/libs/cmdio" - "github.com/databricks/cli/libs/process" - "github.com/stretchr/testify/assert" - - "strings" - - "github.com/databricks/cli/libs/structs/structpath" ) func TestPythonMutator_Name_loadResources(t *testing.T) { diff --git a/bundle/config/mutator/resourcemutator/apply_bundle_permissions.go b/bundle/config/mutator/resourcemutator/apply_bundle_permissions.go index 33a09135222..bfc9e373616 100644 --- a/bundle/config/mutator/resourcemutator/apply_bundle_permissions.go +++ b/bundle/config/mutator/resourcemutator/apply_bundle_permissions.go @@ -7,9 +7,10 @@ import ( "slices" "strings" + "github.com/databricks/cli/bundle/permissions" + "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config/resources" - "github.com/databricks/cli/bundle/permissions" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/logdiag" "github.com/databricks/cli/libs/structs/structpath" diff --git a/bundle/config/mutator/resourcemutator/expand_pipeline_glob_paths_test.go b/bundle/config/mutator/resourcemutator/expand_pipeline_glob_paths_test.go index 1a8e331caf4..ead2084d835 100644 --- a/bundle/config/mutator/resourcemutator/expand_pipeline_glob_paths_test.go +++ b/bundle/config/mutator/resourcemutator/expand_pipeline_glob_paths_test.go @@ -7,7 +7,9 @@ import ( "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/bundle/config/mutator/resourcemutator" + "github.com/stretchr/testify/assert" + "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/resources" diff --git a/bundle/config/mutator/resourcemutator/resource_mutator.go b/bundle/config/mutator/resourcemutator/resource_mutator.go index 5f424eb82fd..4704d854032 100644 --- a/bundle/config/mutator/resourcemutator/resource_mutator.go +++ b/bundle/config/mutator/resourcemutator/resource_mutator.go @@ -6,12 +6,14 @@ import ( "reflect" "strings" - "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/bundle/config/validate" + "github.com/databricks/databricks-sdk-go/service/jobs" + "github.com/databricks/cli/libs/logdiag" "github.com/databricks/cli/libs/structs/structpath" - "github.com/databricks/databricks-sdk-go/service/jobs" + + "github.com/databricks/cli/bundle" ) // When a new resource is added to configuration, we apply bundle diff --git a/bundle/config/mutator/resourcemutator/validate_target_mode.go b/bundle/config/mutator/resourcemutator/validate_target_mode.go index c3832ceb090..bf6386264ad 100644 --- a/bundle/config/mutator/resourcemutator/validate_target_mode.go +++ b/bundle/config/mutator/resourcemutator/validate_target_mode.go @@ -5,11 +5,12 @@ import ( "fmt" "strings" + "github.com/databricks/cli/libs/iamutil" + "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/iamutil" ) type validateTargetMode struct{} diff --git a/bundle/config/mutator/translate_paths.go b/bundle/config/mutator/translate_paths.go index 9ad6c53dbaf..21c8f212cf6 100644 --- a/bundle/config/mutator/translate_paths.go +++ b/bundle/config/mutator/translate_paths.go @@ -11,11 +11,12 @@ import ( "path/filepath" "strings" + "github.com/databricks/cli/bundle/config/mutator/paths" + "github.com/databricks/cli/bundle/libraries" + "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" - "github.com/databricks/cli/bundle/config/mutator/paths" "github.com/databricks/cli/bundle/config/resources" - "github.com/databricks/cli/bundle/libraries" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/notebook" "github.com/databricks/cli/libs/structs/structpath" diff --git a/bundle/config/resources_types_test.go b/bundle/config/resources_types_test.go index af53fe0ef04..d2aadd7f530 100644 --- a/bundle/config/resources_types_test.go +++ b/bundle/config/resources_types_test.go @@ -9,13 +9,14 @@ import ( "strings" "testing" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/cli/libs/structs/structtag" "github.com/databricks/cli/libs/structs/structvar" "github.com/databricks/cli/libs/structs/structwalk" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" ) func TestResourcesTypesMap(t *testing.T) { diff --git a/bundle/libraries/upload.go b/bundle/libraries/upload.go index 52e34de1915..7c7087aca36 100644 --- a/bundle/libraries/upload.go +++ b/bundle/libraries/upload.go @@ -16,6 +16,7 @@ import ( "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/retry" "github.com/databricks/cli/libs/structs/structpath" + "golang.org/x/sync/errgroup" ) diff --git a/cmd/apps/import.go b/cmd/apps/import.go index 4aefe33f057..d7138979ec1 100644 --- a/cmd/apps/import.go +++ b/cmd/apps/import.go @@ -12,6 +12,8 @@ import ( "slices" "strings" + "go.yaml.in/yaml/v3" + "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/generate" "github.com/databricks/cli/bundle/phases" @@ -32,7 +34,6 @@ import ( "github.com/databricks/databricks-sdk-go/service/iam" "github.com/databricks/databricks-sdk-go/service/workspace" "github.com/spf13/cobra" - "go.yaml.in/yaml/v3" ) func newImportCommand() *cobra.Command { diff --git a/integration/assumptions/dashboard_assumptions_test.go b/integration/assumptions/dashboard_assumptions_test.go index be991dd081d..e7d6bd44a2e 100644 --- a/integration/assumptions/dashboard_assumptions_test.go +++ b/integration/assumptions/dashboard_assumptions_test.go @@ -11,6 +11,7 @@ import ( "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/service/dashboards" "github.com/databricks/databricks-sdk-go/service/workspace" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) From 6476693084cde746c52075a6acf31ebab7e78050 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 9 Oct 2026 10:22:45 +0200 Subject: [PATCH 12/22] structvar: test the YAML decoder with the yamlloader fixtures Co-authored-by: Isaac --- .../structvar/testdata/yaml/anchor_01.yml | 12 + .../structvar/testdata/yaml/anchor_02.yml | 13 + .../structvar/testdata/yaml/anchor_03.yml | 10 + .../structvar/testdata/yaml/anchor_04.yml | 16 + .../structvar/testdata/yaml/anchor_05.yml | 15 + .../structvar/testdata/yaml/anchor_06.yml | 5 + .../structvar/testdata/yaml/anchor_07.yml | 10 + .../structvar/testdata/yaml/anchor_08.yml | 10 + .../structvar/testdata/yaml/anchor_09.yml | 2 + .../structvar/testdata/yaml/anchor_10.yml | 5 + .../structs/structvar/testdata/yaml/empty.yml | 0 .../structvar/testdata/yaml/error_01.yml | 6 + .../structvar/testdata/yaml/error_02.yml | 6 + .../structvar/testdata/yaml/error_03.yml | 4 + .../structvar/testdata/yaml/mix_01.yml | 90 ++ .../structvar/testdata/yaml/mix_02.yml | 49 ++ .../testdata/yaml/spec_example_2.1.yml | 5 + .../testdata/yaml/spec_example_2.10.yml | 10 + .../testdata/yaml/spec_example_2.11.yml | 10 + .../testdata/yaml/spec_example_2.12.yml | 10 + .../testdata/yaml/spec_example_2.13.yml | 6 + .../testdata/yaml/spec_example_2.14.yml | 6 + .../testdata/yaml/spec_example_2.15.yml | 10 + .../testdata/yaml/spec_example_2.16.yml | 9 + .../testdata/yaml/spec_example_2.17.yml | 9 + .../testdata/yaml/spec_example_2.18.yml | 8 + .../testdata/yaml/spec_example_2.19.yml | 15 + .../testdata/yaml/spec_example_2.2.yml | 5 + .../testdata/yaml/spec_example_2.20.yml | 7 + .../testdata/yaml/spec_example_2.21.yml | 5 + .../testdata/yaml/spec_example_2.22.yml | 6 + .../testdata/yaml/spec_example_2.23.yml | 15 + .../testdata/yaml/spec_example_2.24.yml | 16 + .../testdata/yaml/spec_example_2.25.yml | 9 + .../testdata/yaml/spec_example_2.26.yml | 9 + .../testdata/yaml/spec_example_2.27.yml | 31 + .../testdata/yaml/spec_example_2.28.yml | 28 + .../testdata/yaml/spec_example_2.3.yml | 10 + .../testdata/yaml/spec_example_2.4.yml | 10 + .../testdata/yaml/spec_example_2.5.yml | 5 + .../testdata/yaml/spec_example_2.6.yml | 7 + .../testdata/yaml/spec_example_2.7.yml | 12 + .../testdata/yaml/spec_example_2.8.yml | 12 + .../testdata/yaml/spec_example_2.9.yml | 10 + libs/structs/structvar/yaml_anchor_test.go | 115 +++ libs/structs/structvar/yaml_error_test.go | 34 + libs/structs/structvar/yaml_mix_test.go | 48 ++ libs/structs/structvar/yaml_spec_test.go | 803 ++++++++++++++++++ yamlfmt.yml | 1 + 49 files changed, 1549 insertions(+) create mode 100644 libs/structs/structvar/testdata/yaml/anchor_01.yml create mode 100644 libs/structs/structvar/testdata/yaml/anchor_02.yml create mode 100644 libs/structs/structvar/testdata/yaml/anchor_03.yml create mode 100644 libs/structs/structvar/testdata/yaml/anchor_04.yml create mode 100644 libs/structs/structvar/testdata/yaml/anchor_05.yml create mode 100644 libs/structs/structvar/testdata/yaml/anchor_06.yml create mode 100644 libs/structs/structvar/testdata/yaml/anchor_07.yml create mode 100644 libs/structs/structvar/testdata/yaml/anchor_08.yml create mode 100644 libs/structs/structvar/testdata/yaml/anchor_09.yml create mode 100644 libs/structs/structvar/testdata/yaml/anchor_10.yml create mode 100644 libs/structs/structvar/testdata/yaml/empty.yml create mode 100644 libs/structs/structvar/testdata/yaml/error_01.yml create mode 100644 libs/structs/structvar/testdata/yaml/error_02.yml create mode 100644 libs/structs/structvar/testdata/yaml/error_03.yml create mode 100644 libs/structs/structvar/testdata/yaml/mix_01.yml create mode 100644 libs/structs/structvar/testdata/yaml/mix_02.yml create mode 100644 libs/structs/structvar/testdata/yaml/spec_example_2.1.yml create mode 100644 libs/structs/structvar/testdata/yaml/spec_example_2.10.yml create mode 100644 libs/structs/structvar/testdata/yaml/spec_example_2.11.yml create mode 100644 libs/structs/structvar/testdata/yaml/spec_example_2.12.yml create mode 100644 libs/structs/structvar/testdata/yaml/spec_example_2.13.yml create mode 100644 libs/structs/structvar/testdata/yaml/spec_example_2.14.yml create mode 100644 libs/structs/structvar/testdata/yaml/spec_example_2.15.yml create mode 100644 libs/structs/structvar/testdata/yaml/spec_example_2.16.yml create mode 100644 libs/structs/structvar/testdata/yaml/spec_example_2.17.yml create mode 100644 libs/structs/structvar/testdata/yaml/spec_example_2.18.yml create mode 100644 libs/structs/structvar/testdata/yaml/spec_example_2.19.yml create mode 100644 libs/structs/structvar/testdata/yaml/spec_example_2.2.yml create mode 100644 libs/structs/structvar/testdata/yaml/spec_example_2.20.yml create mode 100644 libs/structs/structvar/testdata/yaml/spec_example_2.21.yml create mode 100644 libs/structs/structvar/testdata/yaml/spec_example_2.22.yml create mode 100644 libs/structs/structvar/testdata/yaml/spec_example_2.23.yml create mode 100644 libs/structs/structvar/testdata/yaml/spec_example_2.24.yml create mode 100644 libs/structs/structvar/testdata/yaml/spec_example_2.25.yml create mode 100644 libs/structs/structvar/testdata/yaml/spec_example_2.26.yml create mode 100644 libs/structs/structvar/testdata/yaml/spec_example_2.27.yml create mode 100644 libs/structs/structvar/testdata/yaml/spec_example_2.28.yml create mode 100644 libs/structs/structvar/testdata/yaml/spec_example_2.3.yml create mode 100644 libs/structs/structvar/testdata/yaml/spec_example_2.4.yml create mode 100644 libs/structs/structvar/testdata/yaml/spec_example_2.5.yml create mode 100644 libs/structs/structvar/testdata/yaml/spec_example_2.6.yml create mode 100644 libs/structs/structvar/testdata/yaml/spec_example_2.7.yml create mode 100644 libs/structs/structvar/testdata/yaml/spec_example_2.8.yml create mode 100644 libs/structs/structvar/testdata/yaml/spec_example_2.9.yml create mode 100644 libs/structs/structvar/yaml_anchor_test.go create mode 100644 libs/structs/structvar/yaml_error_test.go create mode 100644 libs/structs/structvar/yaml_mix_test.go create mode 100644 libs/structs/structvar/yaml_spec_test.go diff --git a/libs/structs/structvar/testdata/yaml/anchor_01.yml b/libs/structs/structvar/testdata/yaml/anchor_01.yml new file mode 100644 index 00000000000..b677d60b808 --- /dev/null +++ b/libs/structs/structvar/testdata/yaml/anchor_01.yml @@ -0,0 +1,12 @@ +# 1. Basic Anchor and Alias +defaults: &DEFAULTS + color: red + size: large + +shirt1: + <<: *DEFAULTS + pattern: striped + +shirt2: + <<: *DEFAULTS + pattern: plain diff --git a/libs/structs/structvar/testdata/yaml/anchor_02.yml b/libs/structs/structvar/testdata/yaml/anchor_02.yml new file mode 100644 index 00000000000..86650987f3d --- /dev/null +++ b/libs/structs/structvar/testdata/yaml/anchor_02.yml @@ -0,0 +1,13 @@ +# 2. Merging Anchors +# Here, multiple anchors can be merged into a single item. +defaults: &DEFAULTS + color: red + size: large + +materials: &MATERIALS + primary: cotton + secondary: polyester + +shirt: + <<: [*DEFAULTS, *MATERIALS] + pattern: striped diff --git a/libs/structs/structvar/testdata/yaml/anchor_03.yml b/libs/structs/structvar/testdata/yaml/anchor_03.yml new file mode 100644 index 00000000000..f06b7999ef0 --- /dev/null +++ b/libs/structs/structvar/testdata/yaml/anchor_03.yml @@ -0,0 +1,10 @@ +# 3. Overriding Merged Anchor Values +# You can override values when merging. +defaults: &DEFAULTS + color: red + size: large + pattern: plain + +shirt: + <<: *DEFAULTS + color: blue diff --git a/libs/structs/structvar/testdata/yaml/anchor_04.yml b/libs/structs/structvar/testdata/yaml/anchor_04.yml new file mode 100644 index 00000000000..bbc4bc02707 --- /dev/null +++ b/libs/structs/structvar/testdata/yaml/anchor_04.yml @@ -0,0 +1,16 @@ +# 4. Nested Anchors +# This demonstrates the reuse of nested content. +address: &ADDRESS + city: San Francisco + state: CA + country: USA + +person1: + name: Alice + address: *ADDRESS + +person2: + name: Bob + address: + <<: *ADDRESS + city: Los Angeles diff --git a/libs/structs/structvar/testdata/yaml/anchor_05.yml b/libs/structs/structvar/testdata/yaml/anchor_05.yml new file mode 100644 index 00000000000..9a4c961b7c8 --- /dev/null +++ b/libs/structs/structvar/testdata/yaml/anchor_05.yml @@ -0,0 +1,15 @@ +# 5. Using Anchors for List Items +# You can use anchors for list items too. +features: &FEATURES + - wifi + - bluetooth + +phone1: + name: PhoneA + features: *FEATURES + +phone2: + name: PhoneB + features: + - camera + - *FEATURES diff --git a/libs/structs/structvar/testdata/yaml/anchor_06.yml b/libs/structs/structvar/testdata/yaml/anchor_06.yml new file mode 100644 index 00000000000..f27650fad47 --- /dev/null +++ b/libs/structs/structvar/testdata/yaml/anchor_06.yml @@ -0,0 +1,5 @@ +# 6. String Anchors +commonMessage: &msg "Hello, World!" + +greeting1: *msg +greeting2: *msg diff --git a/libs/structs/structvar/testdata/yaml/anchor_07.yml b/libs/structs/structvar/testdata/yaml/anchor_07.yml new file mode 100644 index 00000000000..b8f32214568 --- /dev/null +++ b/libs/structs/structvar/testdata/yaml/anchor_07.yml @@ -0,0 +1,10 @@ +# 7. Number Anchors +defaultAge: &age 25 + +person1: + name: Alice + age: *age + +person2: + name: Bob + age: *age diff --git a/libs/structs/structvar/testdata/yaml/anchor_08.yml b/libs/structs/structvar/testdata/yaml/anchor_08.yml new file mode 100644 index 00000000000..8b01e21f14b --- /dev/null +++ b/libs/structs/structvar/testdata/yaml/anchor_08.yml @@ -0,0 +1,10 @@ +# 8. Boolean Anchors +isActive: &active true + +user1: + username: user1 + active: *active + +user2: + username: user2 + active: *active diff --git a/libs/structs/structvar/testdata/yaml/anchor_09.yml b/libs/structs/structvar/testdata/yaml/anchor_09.yml new file mode 100644 index 00000000000..8005bf445ef --- /dev/null +++ b/libs/structs/structvar/testdata/yaml/anchor_09.yml @@ -0,0 +1,2 @@ +anchortest: &x + child: *x diff --git a/libs/structs/structvar/testdata/yaml/anchor_10.yml b/libs/structs/structvar/testdata/yaml/anchor_10.yml new file mode 100644 index 00000000000..013832ab3ac --- /dev/null +++ b/libs/structs/structvar/testdata/yaml/anchor_10.yml @@ -0,0 +1,5 @@ +leaf: &leaf 1 +shared: &shared + value: *leaf +use1: *shared +use2: *shared diff --git a/libs/structs/structvar/testdata/yaml/empty.yml b/libs/structs/structvar/testdata/yaml/empty.yml new file mode 100644 index 00000000000..e69de29bb2d diff --git a/libs/structs/structvar/testdata/yaml/error_01.yml b/libs/structs/structvar/testdata/yaml/error_01.yml new file mode 100644 index 00000000000..11b96486e32 --- /dev/null +++ b/libs/structs/structvar/testdata/yaml/error_01.yml @@ -0,0 +1,6 @@ +# Use string anchor to extend a mapping. +str: &str "Hello world!" + +map: + <<: *str + key: value diff --git a/libs/structs/structvar/testdata/yaml/error_02.yml b/libs/structs/structvar/testdata/yaml/error_02.yml new file mode 100644 index 00000000000..5e9c6276d09 --- /dev/null +++ b/libs/structs/structvar/testdata/yaml/error_02.yml @@ -0,0 +1,6 @@ +# Use string anchor inside sequence to extend a mapping. +str: &str "Hello world!" + +map: + <<: [*str] + key: value diff --git a/libs/structs/structvar/testdata/yaml/error_03.yml b/libs/structs/structvar/testdata/yaml/error_03.yml new file mode 100644 index 00000000000..c4d3c397dd9 --- /dev/null +++ b/libs/structs/structvar/testdata/yaml/error_03.yml @@ -0,0 +1,4 @@ +# Extend a mapping with a literal string. +map: + <<: value + key: value diff --git a/libs/structs/structvar/testdata/yaml/mix_01.yml b/libs/structs/structvar/testdata/yaml/mix_01.yml new file mode 100644 index 00000000000..98706bf6f01 --- /dev/null +++ b/libs/structs/structvar/testdata/yaml/mix_01.yml @@ -0,0 +1,90 @@ +# Scalars + +# Strings can be unquoted +name: John Doe + +# Strings can be single quoted +single_quoted_string: 'This is a single quoted string' + +# Strings can be double quoted (allows for escape sequences) +double_quoted_string: "This is a double quoted string with an escaped newline \n and tab \t." + +# Multiline string with folded style (newlines become spaces) +folded_style: > + This is a very + long string that + spans several lines. + +# Multiline string with literal block style (newlines are preserved) +literal_style: | + Line 1 + Line 2 + Line 3 + +# Integers +age: 30 + +# Floating point +price: 19.99 + +# Boolean values +is_student: true +is_employed: false + +# Null value +middle_name: ~ # null can also be represented with 'null' + +# Timestamp +timestamp: 2023-01-01T12:00:00Z + +# Sequences (lists) + +# Inline list +fruits: [apple, banana, cherry] + +# Block style list +books: + - 'Moby Dick' + - '1984' + - 'Pride and Prejudice' + +# Mappings (dictionaries) + +# Inline dictionary +address: {street: '123 Main St', city: 'Anytown', zip: '12345'} + +# Block style dictionary +employee: + first_name: Jane + last_name: Smith + age: 40 + +# Edge cases and advanced features + +# Strings that look like other types must be quoted +looks_like_number: '12345' +looks_like_boolean: 'yes' +looks_like_null: 'null' +looks_like_timestamp: '2023-01-01T12:00:00Z' + +# Using anchors and aliases to reuse properties +base_address: &base + street: '456 Elm St' + city: 'Sometown' + +office_address: + <<: *base # Merge the base address into this mapping + suite: 500 + +# Nested structures +users: + - name: Alice + age: 28 + interests: + - reading + - cycling + - name: Bob + age: 35 + interests: + - cooking + - hiking diff --git a/libs/structs/structvar/testdata/yaml/mix_02.yml b/libs/structs/structvar/testdata/yaml/mix_02.yml new file mode 100644 index 00000000000..757d626dbfe --- /dev/null +++ b/libs/structs/structvar/testdata/yaml/mix_02.yml @@ -0,0 +1,49 @@ +# Scalars with special characters + +# Commas and square brackets in strings should be enclosed in quotes +special_chars: "[This, string] has, special chars." + +# Strings starting with reserved indicators must be quoted +reserved_string: "@not_a_directive" +colon_string: "this: looks like a mapping, but it's not" + +# Explicit data type declaration +explicit_string: !!str 12345 + +# Sequences with nested mappings and lists + +teams: + - + name: Team A + members: + - Alice + - Bob + tasks: + - task1: + status: incomplete + due_date: 2023-01-15 + - task2: + status: complete + +# Complex mapping keys +? | + Multi-line key + which is unusual but valid +: multi-line key's value + +"complex key: with colon": complex key's value + +# Set (a mapping with null values) +set_example: + item1: ~ + item2: ~ + +# Merge multiple mappings (with override) +base_colors: &colors + red: "#FF0000" + blue: "#0000FF" + green: "#00FF00" + +theme: + <<: *colors + blue: "#001122" # Overriding the blue color from base_colors diff --git a/libs/structs/structvar/testdata/yaml/spec_example_2.1.yml b/libs/structs/structvar/testdata/yaml/spec_example_2.1.yml new file mode 100644 index 00000000000..c9e26274c09 --- /dev/null +++ b/libs/structs/structvar/testdata/yaml/spec_example_2.1.yml @@ -0,0 +1,5 @@ +# Example 2.1 Sequence of Scalars (ball players) + +- Mark McGwire +- Sammy Sosa +- Ken Griffey diff --git a/libs/structs/structvar/testdata/yaml/spec_example_2.10.yml b/libs/structs/structvar/testdata/yaml/spec_example_2.10.yml new file mode 100644 index 00000000000..a3459dedfc8 --- /dev/null +++ b/libs/structs/structvar/testdata/yaml/spec_example_2.10.yml @@ -0,0 +1,10 @@ +# Example 2.10 Node for “Sammy Sosa” appears twice in this document + +--- +hr: +- Mark McGwire +# Following node labeled SS +- &SS Sammy Sosa +rbi: +- *SS # Subsequent occurrence +- Ken Griffey diff --git a/libs/structs/structvar/testdata/yaml/spec_example_2.11.yml b/libs/structs/structvar/testdata/yaml/spec_example_2.11.yml new file mode 100644 index 00000000000..e3c5c115e02 --- /dev/null +++ b/libs/structs/structvar/testdata/yaml/spec_example_2.11.yml @@ -0,0 +1,10 @@ +# Example 2.11 Mapping between Sequences + +? - Detroit Tigers + - Chicago cubs +: - 2001-07-23 + +? [ New York Yankees, + Atlanta Braves ] +: [ 2001-07-02, 2001-08-12, + 2001-08-14 ] diff --git a/libs/structs/structvar/testdata/yaml/spec_example_2.12.yml b/libs/structs/structvar/testdata/yaml/spec_example_2.12.yml new file mode 100644 index 00000000000..eb4a526f65b --- /dev/null +++ b/libs/structs/structvar/testdata/yaml/spec_example_2.12.yml @@ -0,0 +1,10 @@ +# Example 2.12 Compact Nested Mapping + +--- +# Products purchased +- item : Super Hoop + quantity: 1 +- item : Basketball + quantity: 4 +- item : Big Shoes + quantity: 1 diff --git a/libs/structs/structvar/testdata/yaml/spec_example_2.13.yml b/libs/structs/structvar/testdata/yaml/spec_example_2.13.yml new file mode 100644 index 00000000000..e55abff107e --- /dev/null +++ b/libs/structs/structvar/testdata/yaml/spec_example_2.13.yml @@ -0,0 +1,6 @@ +# Example 2.13 In literals, newlines are preserved + +# ASCII Art +--- | + \//||\/|| + // || ||__ diff --git a/libs/structs/structvar/testdata/yaml/spec_example_2.14.yml b/libs/structs/structvar/testdata/yaml/spec_example_2.14.yml new file mode 100644 index 00000000000..439fca30cb1 --- /dev/null +++ b/libs/structs/structvar/testdata/yaml/spec_example_2.14.yml @@ -0,0 +1,6 @@ +# Example 2.14 In the folded scalars, newlines become spaces + +--- > + Mark McGwire's + year was crippled + by a knee injury. diff --git a/libs/structs/structvar/testdata/yaml/spec_example_2.15.yml b/libs/structs/structvar/testdata/yaml/spec_example_2.15.yml new file mode 100644 index 00000000000..266e7ce4711 --- /dev/null +++ b/libs/structs/structvar/testdata/yaml/spec_example_2.15.yml @@ -0,0 +1,10 @@ +# Example 2.15 Folded newlines are preserved for “more indented” and blank lines + +--- > + Sammy Sosa completed another + fine season with great stats. + + 63 Home Runs + 0.288 Batting Average + + What a year! diff --git a/libs/structs/structvar/testdata/yaml/spec_example_2.16.yml b/libs/structs/structvar/testdata/yaml/spec_example_2.16.yml new file mode 100644 index 00000000000..6db6b0878f8 --- /dev/null +++ b/libs/structs/structvar/testdata/yaml/spec_example_2.16.yml @@ -0,0 +1,9 @@ +# Example 2.16 Indentation determines scope + +name: Mark McGwire +accomplishment: > + Mark set a major league + home run record in 1998. +stats: | + 65 Home Runs + 0.278 Batting Average diff --git a/libs/structs/structvar/testdata/yaml/spec_example_2.17.yml b/libs/structs/structvar/testdata/yaml/spec_example_2.17.yml new file mode 100644 index 00000000000..af0777abab5 --- /dev/null +++ b/libs/structs/structvar/testdata/yaml/spec_example_2.17.yml @@ -0,0 +1,9 @@ +# Example 2.17 Quoted Scalars + +unicode: "Sosa did fine.\u263A" +control: "\b1998\t1999\t2000\n" +hex esc: "\x0d\x0a is \r\n" + +single: '"Howdy!" he cried.' +quoted: ' # Not a ''comment''.' +tie-fighter: '|\-*-/|' diff --git a/libs/structs/structvar/testdata/yaml/spec_example_2.18.yml b/libs/structs/structvar/testdata/yaml/spec_example_2.18.yml new file mode 100644 index 00000000000..741bcd8c0c0 --- /dev/null +++ b/libs/structs/structvar/testdata/yaml/spec_example_2.18.yml @@ -0,0 +1,8 @@ +# Example 2.18 Multi-line Flow Scalars + +plain: + This unquoted scalar + spans many lines. + +quoted: "So does this + quoted scalar.\n" diff --git a/libs/structs/structvar/testdata/yaml/spec_example_2.19.yml b/libs/structs/structvar/testdata/yaml/spec_example_2.19.yml new file mode 100644 index 00000000000..6ed95e092aa --- /dev/null +++ b/libs/structs/structvar/testdata/yaml/spec_example_2.19.yml @@ -0,0 +1,15 @@ +# Example 2.19 Integers + +canonical: 12345 +decimal: +12345 +octal: 0o14 +hexadecimal: 0xC + +# Note: this example is not part of the spec but added for completeness. +# +# Octal numbers: +# - YAML 1.1: prefix is "0" +# - YAML 1.2: prefix is "0o" +# The "gopkg.in/yaml.v3" package accepts both for backwards compat. +# We accept only the YAML 1.2 prefix "0o". +octal11: 012345 diff --git a/libs/structs/structvar/testdata/yaml/spec_example_2.2.yml b/libs/structs/structvar/testdata/yaml/spec_example_2.2.yml new file mode 100644 index 00000000000..29c16105e3b --- /dev/null +++ b/libs/structs/structvar/testdata/yaml/spec_example_2.2.yml @@ -0,0 +1,5 @@ +# Example 2.2 Mapping Scalars to Scalars (player statistics) + +hr: 65 # Home runs +avg: 0.278 # Batting average +rbi: 147 # Runs Batted In diff --git a/libs/structs/structvar/testdata/yaml/spec_example_2.20.yml b/libs/structs/structvar/testdata/yaml/spec_example_2.20.yml new file mode 100644 index 00000000000..77a79a0c3e0 --- /dev/null +++ b/libs/structs/structvar/testdata/yaml/spec_example_2.20.yml @@ -0,0 +1,7 @@ +# Example 2.20 Floating Point + +canonical: 1.23015e+3 +exponential: 12.3015e+02 +fixed: 1230.15 +negative infinity: -.inf +not a number: .nan diff --git a/libs/structs/structvar/testdata/yaml/spec_example_2.21.yml b/libs/structs/structvar/testdata/yaml/spec_example_2.21.yml new file mode 100644 index 00000000000..cdb423c566e --- /dev/null +++ b/libs/structs/structvar/testdata/yaml/spec_example_2.21.yml @@ -0,0 +1,5 @@ +# Example 2.21 Miscellaneous + +null: +booleans: [ true, false ] +string: '012345' diff --git a/libs/structs/structvar/testdata/yaml/spec_example_2.22.yml b/libs/structs/structvar/testdata/yaml/spec_example_2.22.yml new file mode 100644 index 00000000000..bef2addfa07 --- /dev/null +++ b/libs/structs/structvar/testdata/yaml/spec_example_2.22.yml @@ -0,0 +1,6 @@ +# Example 2.22 Timestamps + +canonical: 2001-12-15T02:59:43.1Z +iso8601: 2001-12-14t21:59:43.10-05:00 +spaced: 2001-12-14 21:59:43.10 -5 +date: 2002-12-14 diff --git a/libs/structs/structvar/testdata/yaml/spec_example_2.23.yml b/libs/structs/structvar/testdata/yaml/spec_example_2.23.yml new file mode 100644 index 00000000000..56e9898e21b --- /dev/null +++ b/libs/structs/structvar/testdata/yaml/spec_example_2.23.yml @@ -0,0 +1,15 @@ +# Example 2.23 Various Explicit Tags + +--- +not-date: !!str 2002-04-28 + +picture: !!binary | + R0lGODlhDAAMAIQAAP//9/X + 17unp5WZmZgAAAOfn515eXv + Pz7Y6OjuDg4J+fn5OTk6enp + 56enmleECcgggoBADs= + +application specific tag: !something | + The semantics of the tag + above may be different for + different documents. diff --git a/libs/structs/structvar/testdata/yaml/spec_example_2.24.yml b/libs/structs/structvar/testdata/yaml/spec_example_2.24.yml new file mode 100644 index 00000000000..f7c11d0fb11 --- /dev/null +++ b/libs/structs/structvar/testdata/yaml/spec_example_2.24.yml @@ -0,0 +1,16 @@ +# Example 2.24 Global Tags + +%TAG ! tag:clarkevans.com,2002: +--- !shape + # Use the ! handle for presenting + # tag:clarkevans.com,2002:circle +- !circle + center: &ORIGIN {x: 73, y: 129} + radius: 7 +- !line + start: *ORIGIN + finish: { x: 89, y: 102 } +- !label + start: *ORIGIN + color: 0xFFEEBB + text: Pretty vector drawing. diff --git a/libs/structs/structvar/testdata/yaml/spec_example_2.25.yml b/libs/structs/structvar/testdata/yaml/spec_example_2.25.yml new file mode 100644 index 00000000000..73bac862755 --- /dev/null +++ b/libs/structs/structvar/testdata/yaml/spec_example_2.25.yml @@ -0,0 +1,9 @@ +# Example 2.25 Unordered Sets + +# Sets are represented as a +# Mapping where each key is +# associated with a null value +--- !!set +? Mark McGwire +? Sammy Sosa +? Ken Griffey diff --git a/libs/structs/structvar/testdata/yaml/spec_example_2.26.yml b/libs/structs/structvar/testdata/yaml/spec_example_2.26.yml new file mode 100644 index 00000000000..00863a6b854 --- /dev/null +++ b/libs/structs/structvar/testdata/yaml/spec_example_2.26.yml @@ -0,0 +1,9 @@ +# Example 2.26 Ordered Mappings + +# Ordered maps are represented as +# A sequence of mappings, with +# each mapping having one key +--- !!omap +- Mark McGwire: 65 +- Sammy Sosa: 63 +- Ken Griffey: 58 diff --git a/libs/structs/structvar/testdata/yaml/spec_example_2.27.yml b/libs/structs/structvar/testdata/yaml/spec_example_2.27.yml new file mode 100644 index 00000000000..fc9b460c299 --- /dev/null +++ b/libs/structs/structvar/testdata/yaml/spec_example_2.27.yml @@ -0,0 +1,31 @@ +# Example 2.27 Invoice + +--- ! +invoice: 34843 +date : 2001-01-23 +bill-to: &id001 + given : Chris + family : Dumars + address: + lines: | + 458 Walkman Dr. + Suite #292 + city : Royal Oak + state : MI + postal : 48046 +ship-to: *id001 +product: +- sku : BL394D + quantity : 4 + description : Basketball + price : 450.00 +- sku : BL4438H + quantity : 1 + description : Super Hoop + price : 2392.00 +tax : 251.42 +total: 4443.52 +comments: + Late afternoon is best. + Backup contact is Nancy + Billsmer @ 338-4338. diff --git a/libs/structs/structvar/testdata/yaml/spec_example_2.28.yml b/libs/structs/structvar/testdata/yaml/spec_example_2.28.yml new file mode 100644 index 00000000000..35369472967 --- /dev/null +++ b/libs/structs/structvar/testdata/yaml/spec_example_2.28.yml @@ -0,0 +1,28 @@ +# Example 2.28 Log File + +--- +Time: 2001-11-23 15:01:42 -5 +User: ed +Warning: + This is an error message + for the log file +--- +Time: 2001-11-23 15:02:31 -5 +User: ed +Warning: + A slightly different error + message. +--- +Date: 2001-11-23 15:03:17 -5 +User: ed +Fatal: + Unknown variable "bar" +Stack: +- file: TopClass.py + line: 23 + code: | + x = MoreObject("345\n") +- file: MoreClass.py + line: 58 + code: |- + foo = bar diff --git a/libs/structs/structvar/testdata/yaml/spec_example_2.3.yml b/libs/structs/structvar/testdata/yaml/spec_example_2.3.yml new file mode 100644 index 00000000000..70cbe07d391 --- /dev/null +++ b/libs/structs/structvar/testdata/yaml/spec_example_2.3.yml @@ -0,0 +1,10 @@ +# Example 2.3 Mapping Scalars to Sequences (ball clubs in each league) + +american: +- Boston Red Sox +- Detroit Tigers +- New York Yankees +national: +- New York Mets +- Chicago Cubs +- Atlanta Braves diff --git a/libs/structs/structvar/testdata/yaml/spec_example_2.4.yml b/libs/structs/structvar/testdata/yaml/spec_example_2.4.yml new file mode 100644 index 00000000000..cce28625f56 --- /dev/null +++ b/libs/structs/structvar/testdata/yaml/spec_example_2.4.yml @@ -0,0 +1,10 @@ +# Example 2.4 Sequence of Mappings (players’ statistics) + +- + name: Mark McGwire + hr: 65 + avg: 0.278 +- + name: Sammy Sosa + hr: 63 + avg: 0.288 diff --git a/libs/structs/structvar/testdata/yaml/spec_example_2.5.yml b/libs/structs/structvar/testdata/yaml/spec_example_2.5.yml new file mode 100644 index 00000000000..a585faee790 --- /dev/null +++ b/libs/structs/structvar/testdata/yaml/spec_example_2.5.yml @@ -0,0 +1,5 @@ +# Example 2.5 Sequence of Sequences + +- [name , hr, avg ] +- [Mark McGwire, 65, 0.278] +- [Sammy Sosa , 63, 0.288] diff --git a/libs/structs/structvar/testdata/yaml/spec_example_2.6.yml b/libs/structs/structvar/testdata/yaml/spec_example_2.6.yml new file mode 100644 index 00000000000..cc137e5d56d --- /dev/null +++ b/libs/structs/structvar/testdata/yaml/spec_example_2.6.yml @@ -0,0 +1,7 @@ +# Example 2.6 Mapping of Mappings + +Mark McGwire: {hr: 65, avg: 0.278} +Sammy Sosa: { + hr: 63, + avg: 0.288, + } diff --git a/libs/structs/structvar/testdata/yaml/spec_example_2.7.yml b/libs/structs/structvar/testdata/yaml/spec_example_2.7.yml new file mode 100644 index 00000000000..35c2541d6cc --- /dev/null +++ b/libs/structs/structvar/testdata/yaml/spec_example_2.7.yml @@ -0,0 +1,12 @@ +# Example 2.7 Two Documents in a Stream (each with a leading comment) + +# Ranking of 1998 home runs +--- +- Mark McGwire +- Sammy Sosa +- Ken Griffey + +# Team ranking +--- +- Chicago Cubs +- St Louis Cardinals diff --git a/libs/structs/structvar/testdata/yaml/spec_example_2.8.yml b/libs/structs/structvar/testdata/yaml/spec_example_2.8.yml new file mode 100644 index 00000000000..ae6e8bf279f --- /dev/null +++ b/libs/structs/structvar/testdata/yaml/spec_example_2.8.yml @@ -0,0 +1,12 @@ +# Example 2.8 Play by Play Feed from a Game + +--- +time: 20:03:20 +player: Sammy Sosa +action: strike (miss) +... +--- +time: 20:03:47 +player: Sammy Sosa +action: grand slam +... diff --git a/libs/structs/structvar/testdata/yaml/spec_example_2.9.yml b/libs/structs/structvar/testdata/yaml/spec_example_2.9.yml new file mode 100644 index 00000000000..75217b25ec9 --- /dev/null +++ b/libs/structs/structvar/testdata/yaml/spec_example_2.9.yml @@ -0,0 +1,10 @@ +# Example 2.9 Single Document with Two Comments + +--- +hr: # 1998 hr ranking +- Mark McGwire +- Sammy Sosa +# 1998 rbi ranking +rbi: +- Sammy Sosa +- Ken Griffey diff --git a/libs/structs/structvar/yaml_anchor_test.go b/libs/structs/structvar/yaml_anchor_test.go new file mode 100644 index 00000000000..f4fe8e9dfd1 --- /dev/null +++ b/libs/structs/structvar/yaml_anchor_test.go @@ -0,0 +1,115 @@ +package structvar_test + +import ( + "testing" + + "github.com/databricks/cli/libs/diag" + "github.com/stretchr/testify/assert" +) + +func loc(file string, line, column int) []diag.Location { + return []diag.Location{{File: file, Line: line, Column: column}} +} + +func TestYAMLAnchor01(t *testing.T) { + file := "testdata/yaml/anchor_01.yml" + v, sv := loadYAML(t, file) + + // The anchor is not reported as an unknown field, the other keys are. + type known struct{} + warnings := decodeFields[known](t, file) + assert.NotContains(t, warnings, "unknown field: defaults") + assert.Contains(t, warnings, "unknown field: shirt1") + assert.Contains(t, warnings, "unknown field: shirt2") + + assert.Equal(t, "striped", at(t, v, "shirt1", "pattern")) + assert.Equal(t, loc(file, 8, 12), locsAt(sv, "shirt1", "pattern")) +} + +func TestYAMLAnchor02(t *testing.T) { + file := "testdata/yaml/anchor_02.yml" + v, sv := loadYAML(t, file) + + assert.Equal(t, "red", at(t, v, "shirt", "color")) + assert.Equal(t, loc(file, 4, 10), locsAt(sv, "shirt", "color")) + + assert.Equal(t, "cotton", at(t, v, "shirt", "primary")) + assert.Equal(t, loc(file, 8, 12), locsAt(sv, "shirt", "primary")) + + assert.Equal(t, "striped", at(t, v, "shirt", "pattern")) + assert.Equal(t, loc(file, 13, 12), locsAt(sv, "shirt", "pattern")) +} + +func TestYAMLAnchor03(t *testing.T) { + file := "testdata/yaml/anchor_03.yml" + v, sv := loadYAML(t, file) + + // Assert the override took place. + assert.Equal(t, "blue", at(t, v, "shirt", "color")) + assert.Equal(t, loc(file, 10, 10), locsAt(sv, "shirt", "color")) +} + +func TestYAMLAnchor04(t *testing.T) { + file := "testdata/yaml/anchor_04.yml" + v, sv := loadYAML(t, file) + + assert.Equal(t, "San Francisco", at(t, v, "person1", "address", "city")) + assert.Equal(t, loc(file, 4, 9), locsAt(sv, "person1", "address", "city")) + + assert.Equal(t, "Los Angeles", at(t, v, "person2", "address", "city")) + assert.Equal(t, loc(file, 16, 11), locsAt(sv, "person2", "address", "city")) +} + +func TestYAMLAnchor05(t *testing.T) { + file := "testdata/yaml/anchor_05.yml" + v, sv := loadYAML(t, file) + + assert.Equal(t, "wifi", at(t, v, "phone1", "features", 0)) + assert.Equal(t, loc(file, 4, 5), locsAt(sv, "phone1", "features", 0)) + assert.Equal(t, "bluetooth", at(t, v, "phone1", "features", 1)) + assert.Equal(t, loc(file, 5, 5), locsAt(sv, "phone1", "features", 1)) +} + +func TestYAMLAnchor06(t *testing.T) { + file := "testdata/yaml/anchor_06.yml" + v, sv := loadYAML(t, file) + + assert.Equal(t, "Hello, World!", at(t, v, "greeting1")) + assert.Equal(t, loc(file, 2, 16), locsAt(sv, "greeting1")) +} + +func TestYAMLAnchor07(t *testing.T) { + file := "testdata/yaml/anchor_07.yml" + v, sv := loadYAML(t, file) + + assert.Equal(t, "Alice", at(t, v, "person1", "name")) + assert.Equal(t, loc(file, 5, 9), locsAt(sv, "person1", "name")) + + assert.Equal(t, 25, at(t, v, "person1", "age")) + assert.Equal(t, loc(file, 2, 13), locsAt(sv, "person1", "age")) +} + +func TestYAMLAnchor08(t *testing.T) { + file := "testdata/yaml/anchor_08.yml" + v, sv := loadYAML(t, file) + + assert.Equal(t, "user1", at(t, v, "user1", "username")) + assert.Equal(t, loc(file, 5, 13), locsAt(sv, "user1", "username")) + + assert.Equal(t, true, at(t, v, "user1", "active")) + assert.Equal(t, loc(file, 2, 11), locsAt(sv, "user1", "active")) +} + +func TestYAMLAnchor09(t *testing.T) { + file := "testdata/yaml/anchor_09.yml" + _, _, err := decodeFile(t, file) + assert.ErrorContains(t, err, `cyclic reference to anchor "x"`) +} + +func TestYAMLAnchor10(t *testing.T) { + file := "testdata/yaml/anchor_10.yml" + v, _ := loadYAML(t, file) + + assert.Equal(t, 1, at(t, v, "use1", "value")) + assert.Equal(t, 1, at(t, v, "use2", "value")) +} diff --git a/libs/structs/structvar/yaml_error_test.go b/libs/structs/structvar/yaml_error_test.go new file mode 100644 index 00000000000..1e1954e8c0e --- /dev/null +++ b/libs/structs/structvar/yaml_error_test.go @@ -0,0 +1,34 @@ +package structvar_test + +import ( + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.yaml.in/yaml/v3" +) + +func TestYAMLErrorMapMerge(t *testing.T) { + for _, file := range []string{ + "testdata/yaml/error_01.yml", + "testdata/yaml/error_02.yml", + "testdata/yaml/error_03.yml", + } { + input, err := os.ReadFile(file) + require.NoError(t, err) + + t.Run(file, func(t *testing.T) { + t.Run("reference", func(t *testing.T) { + var ref any + err = yaml.Unmarshal(input, &ref) + assert.ErrorContains(t, err, "map merge requires map or sequence of maps as the value") + }) + + t.Run("self", func(t *testing.T) { + _, _, err := decodeFile(t, file) + assert.ErrorContains(t, err, "map merge requires map or sequence of maps as the value") + }) + }) + } +} diff --git a/libs/structs/structvar/yaml_mix_test.go b/libs/structs/structvar/yaml_mix_test.go new file mode 100644 index 00000000000..b9529424d20 --- /dev/null +++ b/libs/structs/structvar/yaml_mix_test.go @@ -0,0 +1,48 @@ +package structvar_test + +import ( + "bytes" + "os" + "testing" + + "github.com/databricks/cli/libs/structs/structvar" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// decodeFields decodes the file into a struct that has only the given fields and +// returns the names of the unknown fields reported. Anchors must not be reported. +func decodeFields[T any](t *testing.T, file string) []string { + input, err := os.ReadFile(file) + require.NoError(t, err) + var dst T + _, diags, err := structvar.DecodeYAML(file, bytes.NewBuffer(input), &dst) + require.NoError(t, err) + var out []string + for _, d := range diags { + out = append(out, d.Summary) + } + return out +} + +// The old tests checked dyn.Value.IsAnchor. The decoder exposes anchors only through +// the "unknown field" warning, which is not reported for anchors. +func TestYAMLMix01(t *testing.T) { + file := "testdata/yaml/mix_01.yml" + loadYAML(t, file) + + type known struct{} + warnings := decodeFields[known](t, file) + assert.Contains(t, warnings, "unknown field: office_address") + assert.NotContains(t, warnings, "unknown field: base_address") +} + +func TestYAMLMix02(t *testing.T) { + file := "testdata/yaml/mix_02.yml" + loadYAML(t, file) + + type known struct{} + warnings := decodeFields[known](t, file) + assert.Contains(t, warnings, "unknown field: theme") + assert.NotContains(t, warnings, "unknown field: base_colors") +} diff --git a/libs/structs/structvar/yaml_spec_test.go b/libs/structs/structvar/yaml_spec_test.go new file mode 100644 index 00000000000..16eeb7e22d4 --- /dev/null +++ b/libs/structs/structvar/yaml_spec_test.go @@ -0,0 +1,803 @@ +package structvar_test + +import ( + "math" + "testing" + + "github.com/databricks/cli/libs/diag" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const NL = "\n" + +func TestYAMLSpecExample_2_1(t *testing.T) { + file := "testdata/yaml/spec_example_2.1.yml" + v, sv := loadExample(t, file) + + assertDecoded(t, nv( + []ev{ + nv("Mark McGwire", []diag.Location{{File: file, Line: 3, Column: 3}}), + nv("Sammy Sosa", []diag.Location{{File: file, Line: 4, Column: 3}}), + nv("Ken Griffey", []diag.Location{{File: file, Line: 5, Column: 3}}), + }, + []diag.Location{{File: file, Line: 3, Column: 1}}, + ), v, sv) +} + +func TestYAMLSpecExample_2_2(t *testing.T) { + file := "testdata/yaml/spec_example_2.2.yml" + v, sv := loadExample(t, file) + + assertDecoded(t, nv( + map[string]ev{ + "hr": nv(65, []diag.Location{{File: file, Line: 3, Column: 6}}), + "avg": nv(0.278, []diag.Location{{File: file, Line: 4, Column: 6}}), + "rbi": nv(147, []diag.Location{{File: file, Line: 5, Column: 6}}), + }, + []diag.Location{{File: file, Line: 3, Column: 1}}, + ), v, sv) +} + +func TestYAMLSpecExample_2_3(t *testing.T) { + file := "testdata/yaml/spec_example_2.3.yml" + v, sv := loadExample(t, file) + + assertDecoded(t, nv( + map[string]ev{ + "american": nv( + []ev{ + nv("Boston Red Sox", []diag.Location{{File: file, Line: 4, Column: 3}}), + nv("Detroit Tigers", []diag.Location{{File: file, Line: 5, Column: 3}}), + nv("New York Yankees", []diag.Location{{File: file, Line: 6, Column: 3}}), + }, + []diag.Location{{File: file, Line: 4, Column: 1}}, + ), + "national": nv( + []ev{ + nv("New York Mets", []diag.Location{{File: file, Line: 8, Column: 3}}), + nv("Chicago Cubs", []diag.Location{{File: file, Line: 9, Column: 3}}), + nv("Atlanta Braves", []diag.Location{{File: file, Line: 10, Column: 3}}), + }, + []diag.Location{{File: file, Line: 8, Column: 1}}, + ), + }, + []diag.Location{{File: file, Line: 3, Column: 1}}, + ), v, sv) +} + +func TestYAMLSpecExample_2_4(t *testing.T) { + file := "testdata/yaml/spec_example_2.4.yml" + v, sv := loadExample(t, file) + + assertDecoded(t, nv( + []ev{ + nv( + map[string]ev{ + "name": nv("Mark McGwire", []diag.Location{{File: file, Line: 4, Column: 9}}), + "hr": nv(65, []diag.Location{{File: file, Line: 5, Column: 9}}), + "avg": nv(0.278, []diag.Location{{File: file, Line: 6, Column: 9}}), + }, + []diag.Location{{File: file, Line: 4, Column: 3}}, + ), + nv( + map[string]ev{ + "name": nv("Sammy Sosa", []diag.Location{{File: file, Line: 8, Column: 9}}), + "hr": nv(63, []diag.Location{{File: file, Line: 9, Column: 9}}), + "avg": nv(0.288, []diag.Location{{File: file, Line: 10, Column: 9}}), + }, + []diag.Location{{File: file, Line: 8, Column: 3}}, + ), + }, + []diag.Location{{File: file, Line: 3, Column: 1}}, + ), v, sv) +} + +func TestYAMLSpecExample_2_5(t *testing.T) { + file := "testdata/yaml/spec_example_2.5.yml" + v, sv := loadExample(t, file) + + assertDecoded(t, nv( + []ev{ + nv( + []ev{ + nv("name", []diag.Location{{File: file, Line: 3, Column: 4}}), + nv("hr", []diag.Location{{File: file, Line: 3, Column: 18}}), + nv("avg", []diag.Location{{File: file, Line: 3, Column: 22}}), + }, + []diag.Location{{File: file, Line: 3, Column: 3}}, + ), + nv( + []ev{ + nv("Mark McGwire", []diag.Location{{File: file, Line: 4, Column: 4}}), + nv(65, []diag.Location{{File: file, Line: 4, Column: 18}}), + nv(0.278, []diag.Location{{File: file, Line: 4, Column: 22}}), + }, + []diag.Location{{File: file, Line: 4, Column: 3}}, + ), + nv( + []ev{ + nv("Sammy Sosa", []diag.Location{{File: file, Line: 5, Column: 4}}), + nv(63, []diag.Location{{File: file, Line: 5, Column: 18}}), + nv(0.288, []diag.Location{{File: file, Line: 5, Column: 22}}), + }, + []diag.Location{{File: file, Line: 5, Column: 3}}, + ), + }, + []diag.Location{{File: file, Line: 3, Column: 1}}, + ), v, sv) +} + +func TestYAMLSpecExample_2_6(t *testing.T) { + file := "testdata/yaml/spec_example_2.6.yml" + v, sv := loadExample(t, file) + + assertDecoded(t, nv( + map[string]ev{ + "Mark McGwire": nv( + map[string]ev{ + "hr": nv(65, []diag.Location{{File: file, Line: 3, Column: 20}}), + "avg": nv(0.278, []diag.Location{{File: file, Line: 3, Column: 29}}), + }, + []diag.Location{{File: file, Line: 3, Column: 15}}, + ), + "Sammy Sosa": nv( + map[string]ev{ + "hr": nv(63, []diag.Location{{File: file, Line: 5, Column: 9}}), + "avg": nv(0.288, []diag.Location{{File: file, Line: 6, Column: 10}}), + }, + []diag.Location{{File: file, Line: 4, Column: 13}}, + ), + }, + []diag.Location{{File: file, Line: 3, Column: 1}}, + ), v, sv) +} + +func TestYAMLSpecExample_2_7(t *testing.T) { + file := "testdata/yaml/spec_example_2.7.yml" + v, sv := loadExample(t, file) + + // Note: we do not support multiple documents in a single YAML file. + + assertDecoded(t, nv( + []ev{ + nv( + "Mark McGwire", + []diag.Location{{File: file, Line: 5, Column: 3}}, + ), + nv( + "Sammy Sosa", + []diag.Location{{File: file, Line: 6, Column: 3}}, + ), + nv( + "Ken Griffey", + []diag.Location{{File: file, Line: 7, Column: 3}}, + ), + }, + []diag.Location{{File: file, Line: 5, Column: 1}}, + ), v, sv) +} + +func TestYAMLSpecExample_2_8(t *testing.T) { + file := "testdata/yaml/spec_example_2.8.yml" + v, sv := loadExample(t, file) + + // Note: we do not support multiple documents in a single YAML file. + + assertDecoded(t, nv( + map[string]ev{ + "time": nv("20:03:20", []diag.Location{{File: file, Line: 4, Column: 7}}), + "player": nv("Sammy Sosa", []diag.Location{{File: file, Line: 5, Column: 9}}), + "action": nv("strike (miss)", []diag.Location{{File: file, Line: 6, Column: 9}}), + }, + []diag.Location{{File: file, Line: 4, Column: 1}}, + ), v, sv) +} + +func TestYAMLSpecExample_2_9(t *testing.T) { + file := "testdata/yaml/spec_example_2.9.yml" + v, sv := loadExample(t, file) + + // Note: we do not support multiple documents in a single YAML file. + + assertDecoded(t, nv( + map[string]ev{ + "hr": nv( + []ev{ + nv("Mark McGwire", []diag.Location{{File: file, Line: 5, Column: 3}}), + nv("Sammy Sosa", []diag.Location{{File: file, Line: 6, Column: 3}}), + }, + []diag.Location{{File: file, Line: 5, Column: 1}}, + ), + "rbi": nv( + []ev{ + nv("Sammy Sosa", []diag.Location{{File: file, Line: 9, Column: 3}}), + nv("Ken Griffey", []diag.Location{{File: file, Line: 10, Column: 3}}), + }, + []diag.Location{{File: file, Line: 9, Column: 1}}, + ), + }, + []diag.Location{{File: file, Line: 4, Column: 1}}, + ), v, sv) +} + +func TestYAMLSpecExample_2_10(t *testing.T) { + file := "testdata/yaml/spec_example_2.10.yml" + v, sv := loadExample(t, file) + + assertDecoded(t, nv( + map[string]ev{ + "hr": nv( + []ev{ + nv("Mark McGwire", []diag.Location{{File: file, Line: 5, Column: 3}}), + nv("Sammy Sosa", []diag.Location{{File: file, Line: 7, Column: 3}}), + }, + []diag.Location{{File: file, Line: 5, Column: 1}}, + ), + "rbi": nv( + []ev{ + // The location for an anchored value refers to the anchor, not the reference. + // This is the same location as the anchor that appears in the "hr" mapping. + nv("Sammy Sosa", []diag.Location{{File: file, Line: 7, Column: 3}}), + nv("Ken Griffey", []diag.Location{{File: file, Line: 10, Column: 3}}), + }, + []diag.Location{{File: file, Line: 9, Column: 1}}, + ), + }, + []diag.Location{{File: file, Line: 4, Column: 1}}, + ), v, sv) +} + +func TestYAMLSpecExample_2_11(t *testing.T) { + file := "testdata/yaml/spec_example_2.11.yml" + // Note: non-string mapping keys are not supported by "go.yaml.in/yaml/v3". + _, _, err := decodeFile(t, file) + assert.ErrorContains(t, err, `: key is not a scalar`) +} + +func TestYAMLSpecExample_2_12(t *testing.T) { + file := "testdata/yaml/spec_example_2.12.yml" + v, sv := loadExample(t, file) + + assertDecoded(t, nv( + []ev{ + nv( + map[string]ev{ + "item": nv("Super Hoop", []diag.Location{{File: file, Line: 5, Column: 13}}), + "quantity": nv(1, []diag.Location{{File: file, Line: 6, Column: 13}}), + }, + []diag.Location{{File: file, Line: 5, Column: 3}}, + ), + nv( + map[string]ev{ + "item": nv("Basketball", []diag.Location{{File: file, Line: 7, Column: 13}}), + "quantity": nv(4, []diag.Location{{File: file, Line: 8, Column: 13}}), + }, + []diag.Location{{File: file, Line: 7, Column: 3}}, + ), + nv( + map[string]ev{ + "item": nv("Big Shoes", []diag.Location{{File: file, Line: 9, Column: 13}}), + "quantity": nv(1, []diag.Location{{File: file, Line: 10, Column: 13}}), + }, + []diag.Location{{File: file, Line: 9, Column: 3}}, + ), + }, + []diag.Location{{File: file, Line: 5, Column: 1}}, + ), v, sv) +} + +func TestYAMLSpecExample_2_13(t *testing.T) { + file := "testdata/yaml/spec_example_2.13.yml" + v, sv := loadExample(t, file) + + assertDecoded(t, nv( + ``+ + `\//||\/||`+NL+ + "// || ||__"+NL, + []diag.Location{{File: file, Line: 4, Column: 5}}, + ), v, sv) +} + +func TestYAMLSpecExample_2_14(t *testing.T) { + file := "testdata/yaml/spec_example_2.14.yml" + v, sv := loadExample(t, file) + + assertDecoded(t, nv( + `Mark McGwire's year was crippled by a knee injury.`+NL, + []diag.Location{{File: file, Line: 3, Column: 5}}, + ), v, sv) +} + +func TestYAMLSpecExample_2_15(t *testing.T) { + file := "testdata/yaml/spec_example_2.15.yml" + v, sv := loadExample(t, file) + + assertDecoded(t, nv( + ``+ + `Sammy Sosa completed another fine season with great stats.`+NL+ + NL+ + ` 63 Home Runs`+NL+ + ` 0.288 Batting Average`+NL+ + NL+ + `What a year!`+NL, + []diag.Location{{File: file, Line: 3, Column: 5}}, + ), v, sv) +} + +func TestYAMLSpecExample_2_16(t *testing.T) { + file := "testdata/yaml/spec_example_2.16.yml" + v, sv := loadExample(t, file) + + assertDecoded(t, nv( + map[string]ev{ + "name": nv( + "Mark McGwire", + []diag.Location{{File: file, Line: 3, Column: 7}}, + ), + "accomplishment": nv( + `Mark set a major league home run record in 1998.`+NL, + []diag.Location{{File: file, Line: 4, Column: 17}}, + ), + "stats": nv( + ``+ + `65 Home Runs`+NL+ + `0.278 Batting Average`+NL, + []diag.Location{{File: file, Line: 7, Column: 8}}, + ), + }, + []diag.Location{{File: file, Line: 3, Column: 1}}, + ), v, sv) +} + +func TestYAMLSpecExample_2_17(t *testing.T) { + file := "testdata/yaml/spec_example_2.17.yml" + v, sv := loadExample(t, file) + + assertDecoded(t, nv( + map[string]ev{ + "unicode": nv( + `Sosa did fine.`+"\u263A", + []diag.Location{{File: file, Line: 3, Column: 10}}, + ), + "control": nv( + "\b1998\t1999\t2000\n", + []diag.Location{{File: file, Line: 4, Column: 10}}, + ), + "hex esc": nv( + "\x0d\x0a is \r\n", + []diag.Location{{File: file, Line: 5, Column: 10}}, + ), + "single": nv( + `"Howdy!" he cried.`, + []diag.Location{{File: file, Line: 7, Column: 9}}, + ), + "quoted": nv( + ` # Not a 'comment'.`, + []diag.Location{{File: file, Line: 8, Column: 9}}, + ), + "tie-fighter": nv( + `|\-*-/|`, + []diag.Location{{File: file, Line: 9, Column: 14}}, + ), + }, + []diag.Location{{File: file, Line: 3, Column: 1}}, + ), v, sv) +} + +func TestYAMLSpecExample_2_18(t *testing.T) { + file := "testdata/yaml/spec_example_2.18.yml" + v, sv := loadExample(t, file) + + assertDecoded(t, nv( + map[string]ev{ + "plain": nv( + `This unquoted scalar spans many lines.`, + []diag.Location{{File: file, Line: 4, Column: 3}}, + ), + "quoted": nv( + `So does this quoted scalar.`+NL, + []diag.Location{{File: file, Line: 7, Column: 9}}, + ), + }, + []diag.Location{{File: file, Line: 3, Column: 1}}, + ), v, sv) +} + +func TestYAMLSpecExample_2_19(t *testing.T) { + file := "testdata/yaml/spec_example_2.19.yml" + v, sv := loadExample(t, file) + + assertDecoded(t, nv( + map[string]ev{ + "canonical": nv( + 12345, + []diag.Location{{File: file, Line: 3, Column: 12}}, + ), + "decimal": nv( + 12345, + []diag.Location{{File: file, Line: 4, Column: 10}}, + ), + "octal": nv( + 12, + []diag.Location{{File: file, Line: 5, Column: 8}}, + ), + "hexadecimal": nv( + 12, + []diag.Location{{File: file, Line: 6, Column: 14}}, + ), + "octal11": nv( + 12345, + []diag.Location{{File: file, Line: 15, Column: 10}}, + ), + }, + []diag.Location{{File: file, Line: 3, Column: 1}}, + ), v, sv) +} + +func TestYAMLSpecExample_2_20(t *testing.T) { + file := "testdata/yaml/spec_example_2.20.yml" + v, sv := loadExample(t, file) + + // Equality assertion doesn't work with NaNs. + // See https://github.com/stretchr/testify/issues/624. + // + // Remove the NaN entry. + m := v.(map[string]any) + nan, ok := m["not a number"].(float64) + require.True(t, ok) + assert.True(t, math.IsNaN(nan)) + delete(m, "not a number") + + assertDecoded(t, nv( + map[string]ev{ + "canonical": nv( + 1230.15, + []diag.Location{{File: file, Line: 3, Column: 12}}, + ), + "exponential": nv( + 1230.15, + []diag.Location{{File: file, Line: 4, Column: 14}}, + ), + "fixed": nv( + 1230.15, + []diag.Location{{File: file, Line: 5, Column: 8}}, + ), + "negative infinity": nv( + math.Inf(-1), + []diag.Location{{File: file, Line: 6, Column: 20}}, + ), + }, + []diag.Location{{File: file, Line: 3, Column: 1}}, + ), v, sv) +} + +func TestYAMLSpecExample_2_21(t *testing.T) { + file := "testdata/yaml/spec_example_2.21.yml" + v, sv := loadExample(t, file) + + assertDecoded(t, nv( + map[string]ev{ + "null": nv( + nil, + []diag.Location{{File: file, Line: 3, Column: 6}}, + ), + "booleans": nv( + []ev{ + nv(true, []diag.Location{{File: file, Line: 4, Column: 13}}), + nv(false, []diag.Location{{File: file, Line: 4, Column: 19}}), + }, + []diag.Location{{File: file, Line: 4, Column: 11}}, + ), + "string": nv( + "012345", + []diag.Location{{File: file, Line: 5, Column: 9}}, + ), + }, + []diag.Location{{File: file, Line: 3, Column: 1}}, + ), v, sv) +} + +func TestYAMLSpecExample_2_22(t *testing.T) { + file := "testdata/yaml/spec_example_2.22.yml" + v, sv := loadExample(t, file) + + assertDecoded(t, nv( + map[string]ev{ + "canonical": nv( + "2001-12-15T02:59:43.1Z", + []diag.Location{{File: file, Line: 3, Column: 12}}, + ), + "iso8601": nv( + "2001-12-14t21:59:43.10-05:00", + []diag.Location{{File: file, Line: 4, Column: 10}}, + ), + "spaced": nv( + // This is parsed as a string, not a timestamp, + // both by "go.yaml.in/yaml/v3" and by our implementation. + "2001-12-14 21:59:43.10 -5", + []diag.Location{{File: file, Line: 5, Column: 9}}, + ), + "date": nv( + "2002-12-14", + []diag.Location{{File: file, Line: 6, Column: 7}}, + ), + }, + []diag.Location{{File: file, Line: 3, Column: 1}}, + ), v, sv) +} + +func TestYAMLSpecExample_2_23(t *testing.T) { + file := "testdata/yaml/spec_example_2.23.yml" + // Note: the !!binary tag is not supported by us. + _, _, err := decodeFile(t, file) + assert.ErrorContains(t, err, `: unknown tag: !!binary`) +} + +func TestYAMLSpecExample_2_24(t *testing.T) { + file := "testdata/yaml/spec_example_2.24.yml" + v, sv := loadExample(t, file) + + assertDecoded(t, nv( + []ev{ + nv( + map[string]ev{ + "center": nv( + map[string]ev{ + "x": nv(73, []diag.Location{{File: file, Line: 8, Column: 23}}), + "y": nv(129, []diag.Location{{File: file, Line: 8, Column: 30}}), + }, + []diag.Location{{File: file, Line: 8, Column: 11}}, + ), + "radius": nv(7, []diag.Location{{File: file, Line: 9, Column: 11}}), + }, + []diag.Location{{File: file, Line: 7, Column: 3}}, + ), + nv( + map[string]ev{ + "start": nv( + map[string]ev{ + "x": nv(73, []diag.Location{{File: file, Line: 8, Column: 23}}), + "y": nv(129, []diag.Location{{File: file, Line: 8, Column: 30}}), + }, + []diag.Location{{File: file, Line: 8, Column: 11}}, + ), + "finish": nv( + map[string]ev{ + "x": nv(89, []diag.Location{{File: file, Line: 12, Column: 16}}), + "y": nv(102, []diag.Location{{File: file, Line: 12, Column: 23}}), + }, + []diag.Location{{File: file, Line: 12, Column: 11}}, + ), + }, + []diag.Location{{File: file, Line: 10, Column: 3}}, + ), + nv( + map[string]ev{ + "start": nv( + map[string]ev{ + "x": nv(73, []diag.Location{{File: file, Line: 8, Column: 23}}), + "y": nv(129, []diag.Location{{File: file, Line: 8, Column: 30}}), + }, + []diag.Location{{File: file, Line: 8, Column: 11}}, + ), + "color": nv(16772795, []diag.Location{{File: file, Line: 15, Column: 10}}), + "text": nv("Pretty vector drawing.", []diag.Location{{File: file, Line: 16, Column: 9}}), + }, + []diag.Location{{File: file, Line: 13, Column: 3}}, + ), + }, + []diag.Location{{File: file, Line: 4, Column: 5}}, + ), v, sv) +} + +func TestYAMLSpecExample_2_25(t *testing.T) { + file := "testdata/yaml/spec_example_2.25.yml" + v, sv := loadExample(t, file) + + assertDecoded(t, nv( + map[string]ev{ + "Mark McGwire": nv(nil, []diag.Location{{File: file, Line: 8, Column: 1}}), + "Sammy Sosa": nv(nil, []diag.Location{{File: file, Line: 9, Column: 1}}), + "Ken Griffey": nv(nil, []diag.Location{{File: file, Line: 10, Column: 1}}), + }, + []diag.Location{{File: file, Line: 6, Column: 5}}, + ), v, sv) +} + +func TestYAMLSpecExample_2_26(t *testing.T) { + file := "testdata/yaml/spec_example_2.26.yml" + v, sv := loadExample(t, file) + + assertDecoded(t, nv( + []ev{ + nv( + map[string]ev{ + "Mark McGwire": nv(65, []diag.Location{{File: file, Line: 7, Column: 17}}), + }, + []diag.Location{{File: file, Line: 7, Column: 3}}, + ), + nv( + map[string]ev{ + "Sammy Sosa": nv(63, []diag.Location{{File: file, Line: 8, Column: 15}}), + }, + []diag.Location{{File: file, Line: 8, Column: 3}}, + ), + nv( + map[string]ev{ + "Ken Griffey": nv(58, []diag.Location{{File: file, Line: 9, Column: 16}}), + }, + []diag.Location{{File: file, Line: 9, Column: 3}}, + ), + }, + []diag.Location{{File: file, Line: 6, Column: 5}}, + ), v, sv) +} + +func TestYAMLSpecExample_2_27(t *testing.T) { + file := "testdata/yaml/spec_example_2.27.yml" + v, sv := loadExample(t, file) + + assertDecoded(t, nv( + map[string]ev{ + "invoice": nv( + 34843, + []diag.Location{{File: file, Line: 4, Column: 10}}, + ), + "date": nv( + "2001-01-23", + []diag.Location{{File: file, Line: 5, Column: 10}}, + ), + "bill-to": nv( + map[string]ev{ + "given": nv( + "Chris", + []diag.Location{{File: file, Line: 7, Column: 12}}, + ), + "family": nv( + "Dumars", + []diag.Location{{File: file, Line: 8, Column: 12}}, + ), + "address": nv( + map[string]ev{ + "lines": nv( + "458 Walkman Dr.\nSuite #292\n", + []diag.Location{{File: file, Line: 10, Column: 12}}, + ), + "city": nv( + "Royal Oak", + []diag.Location{{File: file, Line: 13, Column: 15}}, + ), + "state": nv( + "MI", + []diag.Location{{File: file, Line: 14, Column: 15}}, + ), + "postal": nv( + 48046, + []diag.Location{{File: file, Line: 15, Column: 15}}, + ), + }, + []diag.Location{{File: file, Line: 10, Column: 5}}, + ), + }, + []diag.Location{{File: file, Line: 6, Column: 10}}, + ), + "ship-to": nv( + map[string]ev{ + "given": nv( + "Chris", + []diag.Location{{File: file, Line: 7, Column: 12}}, + ), + "family": nv( + "Dumars", + []diag.Location{{File: file, Line: 8, Column: 12}}, + ), + "address": nv( + map[string]ev{ + "lines": nv( + "458 Walkman Dr.\nSuite #292\n", + []diag.Location{{File: file, Line: 10, Column: 12}}, + ), + "city": nv( + "Royal Oak", + []diag.Location{{File: file, Line: 13, Column: 15}}, + ), + "state": nv( + "MI", + []diag.Location{{File: file, Line: 14, Column: 15}}, + ), + "postal": nv( + 48046, + []diag.Location{{File: file, Line: 15, Column: 15}}, + ), + }, + []diag.Location{{File: file, Line: 10, Column: 5}}, + ), + }, + []diag.Location{{File: file, Line: 6, Column: 10}}, + ), + "product": nv( + []ev{ + nv( + map[string]ev{ + "sku": nv( + "BL394D", + []diag.Location{{File: file, Line: 18, Column: 17}}, + ), + "quantity": nv( + 4, + []diag.Location{{File: file, Line: 19, Column: 17}}, + ), + "description": nv( + "Basketball", + []diag.Location{{File: file, Line: 20, Column: 17}}, + ), + "price": nv( + 450.0, + []diag.Location{{File: file, Line: 21, Column: 17}}, + ), + }, + []diag.Location{{File: file, Line: 18, Column: 3}}, + ), nv( + map[string]ev{ + "sku": nv( + "BL4438H", + []diag.Location{{File: file, Line: 22, Column: 17}}, + ), + "quantity": nv( + 1, + []diag.Location{{File: file, Line: 23, Column: 17}}, + ), + "description": nv( + "Super Hoop", + []diag.Location{{File: file, Line: 24, Column: 17}}, + ), + "price": nv( + 2392.0, + []diag.Location{{File: file, Line: 25, Column: 17}}, + ), + }, + []diag.Location{{File: file, Line: 22, Column: 3}}, + ), + }, + []diag.Location{{File: file, Line: 18, Column: 1}}, + ), + "tax": nv( + 251.42, + []diag.Location{{File: file, Line: 26, Column: 8}}, + ), + "total": nv( + 4443.52, + []diag.Location{{File: file, Line: 27, Column: 8}}, + ), + "comments": nv( + "Late afternoon is best. Backup contact is Nancy Billsmer @ 338-4338.", + []diag.Location{{File: file, Line: 29, Column: 3}}, + ), + }, + []diag.Location{{File: file, Line: 3, Column: 5}}, + ), v, sv) +} + +func TestYAMLSpecExample_2_28(t *testing.T) { + file := "testdata/yaml/spec_example_2.28.yml" + v, sv := loadExample(t, file) + + assertDecoded(t, nv( + map[string]ev{ + "Time": nv( + "2001-11-23 15:01:42 -5", + []diag.Location{{File: file, Line: 4, Column: 7}}, + ), + "User": nv( + "ed", + []diag.Location{{File: file, Line: 5, Column: 7}}, + ), + "Warning": nv( + "This is an error message for the log file", + []diag.Location{{File: file, Line: 7, Column: 3}}, + ), + }, + []diag.Location{{File: file, Line: 4, Column: 1}}, + ), v, sv) +} diff --git a/yamlfmt.yml b/yamlfmt.yml index 2bc2e891d20..f76d5d66271 100644 --- a/yamlfmt.yml +++ b/yamlfmt.yml @@ -1,6 +1,7 @@ exclude: - acceptance/selftest/bundleconfig # https://github.com/google/yamlfmt/issues/254 - acceptance/bundle/artifacts/nil_artifacts/databricks.yml # https://github.com/google/yamlfmt/issues/253 + - libs/structs/structvar/testdata/yaml - bundle/direct/dresources/apitypes.yml # remove comments - acceptance/bundle/resources/permissions/jobs/update/databricks.yml # comment indentation is significant: it is uncommented into a top-level permissions: [] key formatter: From b189c7777567a3c1d076982e2d9c7891bf37c2b6 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 9 Oct 2026 10:22:51 +0200 Subject: [PATCH 13/22] structvar: add the YAML test helpers Co-authored-by: Isaac --- libs/structs/structvar/yaml_test.go | 159 ++++++++++++++++++++++++++++ 1 file changed, 159 insertions(+) create mode 100644 libs/structs/structvar/yaml_test.go diff --git a/libs/structs/structvar/yaml_test.go b/libs/structs/structvar/yaml_test.go new file mode 100644 index 00000000000..d75410f6fc0 --- /dev/null +++ b/libs/structs/structvar/yaml_test.go @@ -0,0 +1,159 @@ +package structvar_test + +import ( + "bytes" + "fmt" + "os" + "testing" + "time" + + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/structs/structvar" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.yaml.in/yaml/v3" +) + +// ev is an expected value with the locations it is expected to have. +// val is a scalar, a []ev or a map[string]ev. +type ev struct { + val any + locs []diag.Location +} + +func nv(val any, locs []diag.Location) ev { + return ev{val: val, locs: locs} +} + +// plain converts the expected value to the plain Go value the decoder produces. +func (e ev) plain() any { + switch v := e.val.(type) { + case []ev: + out := make([]any, len(v)) + for i, x := range v { + out[i] = x.plain() + } + return out + case map[string]ev: + out := make(map[string]any, len(v)) + for k, x := range v { + out[k] = x.plain() + } + return out + default: + return v + } +} + +// assertLocations checks the locations of every value in the expected tree. +func assertLocations(t *testing.T, l *structvar.Locations, e ev, where string) { + assert.Equal(t, e.locs, l.Get(), "locations of %s", where) + switch v := e.val.(type) { + case []ev: + for i, x := range v { + assertLocations(t, l.Index(i), x, fmt.Sprintf("%s[%d]", where, i)) + } + case map[string]ev: + for k, x := range v { + assertLocations(t, l.Key(k), x, where+"."+k) + } + } +} + +func decodeFile(t *testing.T, file string) (any, *structvar.StructVar, error) { + input, err := os.ReadFile(file) + require.NoError(t, err) + var v any + sv, _, err := structvar.DecodeYAML(file, bytes.NewBuffer(input), &v) + return v, sv, err +} + +func loadExample(t *testing.T, file string) (any, *structvar.StructVar) { + v, sv, err := decodeFile(t, file) + require.NoError(t, err) + return v, sv +} + +// assertDecoded checks the decoded value and the locations of all its parts. +func assertDecoded(t *testing.T, expected ev, v any, sv *structvar.StructVar) { + assert.Equal(t, expected.plain(), v) + assertLocations(t, sv.Locations, expected, "root") +} + +// equalToReference compares the decoded value to the one decoded by yaml.v3. +// Timestamps are kept as the string they were written as, so only their presence is compared. +func equalToReference(t *testing.T, ref, v any) { + switch r := ref.(type) { + case time.Time: + assert.IsType(t, "", v) + case map[string]any: + got, ok := v.(map[string]any) + require.True(t, ok, "expected a map, got %T", v) + require.Len(t, got, len(r)) + for k, x := range r { + equalToReference(t, x, got[k]) + } + case []any: + got, ok := v.([]any) + require.True(t, ok, "expected a sequence, got %T", v) + require.Len(t, got, len(r)) + for i, x := range r { + equalToReference(t, x, got[i]) + } + default: + assert.EqualValues(t, ref, v) + } +} + +// loadYAML decodes the file and checks that the result matches yaml.v3. +func loadYAML(t *testing.T, path string) (any, *structvar.StructVar) { + input, err := os.ReadFile(path) + require.NoError(t, err) + + var ref any + err = yaml.Unmarshal(input, &ref) + require.NoError(t, err) + + v, sv := loadExample(t, path) + require.NotNil(t, sv) + + equalToReference(t, ref, v) + return v, sv +} + +// at returns the value at the path of keys (strings) and indexes (ints). +func at(t *testing.T, v any, path ...any) any { + for _, p := range path { + switch p := p.(type) { + case string: + m, ok := v.(map[string]any) + require.True(t, ok, "expected a map at %q, got %T", p, v) + v = m[p] + case int: + s, ok := v.([]any) + require.True(t, ok, "expected a sequence at %d, got %T", p, v) + v = s[p] + } + } + return v +} + +// locsAt returns the locations of the value at the path of keys (strings) and indexes (ints). +func locsAt(sv *structvar.StructVar, path ...any) []diag.Location { + l := sv.Locations + for _, p := range path { + switch p := p.(type) { + case string: + l = l.Key(p) + case int: + l = l.Index(p) + } + } + return l.Get() +} + +func TestYAMLEmpty(t *testing.T) { + v, sv := loadYAML(t, "testdata/yaml/empty.yml") + assert.Nil(t, v) + assert.Empty(t, sv.Locations.Get()) +} From 1ed2dd9b858e9640405283d9da9963e843df4887 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 9 Oct 2026 10:35:33 +0200 Subject: [PATCH 14/22] Port the job environment_variables files pattern (#6985) to structpath Co-authored-by: Isaac --- bundle/config/mutator/paths/job_paths_visitor.go | 11 +---------- bundle/config/mutator/paths/job_paths_visitor_test.go | 8 ++++---- 2 files changed, 5 insertions(+), 14 deletions(-) diff --git a/bundle/config/mutator/paths/job_paths_visitor.go b/bundle/config/mutator/paths/job_paths_visitor.go index 1e6244b21ed..36f28c5e559 100644 --- a/bundle/config/mutator/paths/job_paths_visitor.go +++ b/bundle/config/mutator/paths/job_paths_visitor.go @@ -67,16 +67,7 @@ func jobRewritePatterns() []jobRewritePattern { patterns := append(taskPatterns, forEachPatterns...) return append(patterns, jobRewritePattern{ - dyn.NewPattern( - dyn.Key("resources"), - dyn.Key("jobs"), - dyn.AnyKey(), - dyn.Key("environment_variables"), - dyn.AnyIndex(), - dyn.Key("spec"), - dyn.Key("files"), - dyn.AnyIndex(), - ), + structpath.MustParsePattern("resources.jobs.*.environment_variables[*].spec.files[*]"), TranslateModeFile, noSkipRewrite, }, diff --git a/bundle/config/mutator/paths/job_paths_visitor_test.go b/bundle/config/mutator/paths/job_paths_visitor_test.go index 23d8b3fa652..7999533f96b 100644 --- a/bundle/config/mutator/paths/job_paths_visitor_test.go +++ b/bundle/config/mutator/paths/job_paths_visitor_test.go @@ -156,10 +156,10 @@ func TestVisitJobPaths_environmentVariables(t *testing.T) { } actual := collectVisitedPaths(t, root, VisitJobPaths) - expected := []dyn.Path{ - dyn.MustPathFromString("resources.jobs.job0.environment_variables[0].spec.files[0]"), - dyn.MustPathFromString("resources.jobs.job0.environment_variables[0].spec.files[1]"), - } + expected := structpath.MustParsePaths( + "resources.jobs.job0.environment_variables[0].spec.files[0]", + "resources.jobs.job0.environment_variables[0].spec.files[1]", + ) assert.ElementsMatch(t, expected, actual) } From 96e6767686dccd67b9c1c8c2007d6a8f92695eb2 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 9 Oct 2026 13:15:47 +0200 Subject: [PATCH 15/22] Fix codex review findings - configsync: compare int and int64 values as equal when restoring variable references (variables decode as int, remote fields are int64). - schema annotations: merge object/sequence defaults as documented; accept an empty annotations file. - python mutator: use ResourceKeySet.AddPattern instead of a duplicate helper. - structvar.Set: don't rewrite an unchanged leaf location node (avoids cloning every ancestor per write). - ResolveConfigAgainstState: resolve all references in one call so lookups are shared. Co-authored-by: Isaac --- .../mutator/python/apply_python_output.go | 20 +++------- bundle/configsync/variables.go | 29 +++++++++++++-- bundle/configsync/variables_test.go | 9 +++++ bundle/direct/resolve_config.go | 37 +++++++++++-------- bundle/internal/schema/annotations.go | 26 +++++++++++++ bundle/internal/schema/annotations_file.go | 4 ++ .../internal/schema/annotations_file_test.go | 12 ++++++ bundle/internal/schema/annotations_test.go | 13 +++++++ libs/structs/structvar/write.go | 15 ++++++-- 9 files changed, 128 insertions(+), 37 deletions(-) diff --git a/bundle/config/mutator/python/apply_python_output.go b/bundle/config/mutator/python/apply_python_output.go index 9ed4d7dd56b..a3b165a1384 100644 --- a/bundle/config/mutator/python/apply_python_output.go +++ b/bundle/config/mutator/python/apply_python_output.go @@ -33,14 +33,6 @@ func applyPythonOutput(root, output structvar.View) (*structvar.OverridePlan, ap return plan, result, nil } -// addResourceKeys adds the keys of the resources in root that match pattern. -func addResourceKeys(set resourcemutator.ResourceKeySet, pattern *structpath.PatternNode, root structvar.View) error { - return structvar.ForEach(root, pattern, func(np *structpath.PathNode, _ structvar.View) error { - set.AddResourceKey(resourcemutator.ResourceKey{Type: np.KeyAt(1), Name: np.KeyAt(2)}) - return nil - }) -} - func createOverrideVisitor(leftRoot, rightRoot structvar.View) (applyPythonOutputResult, structvar.OverrideVisitor) { resourcesPath := structpath.MustParsePath("resources") deleted := resourcemutator.NewResourceKeySet() @@ -66,7 +58,7 @@ func createOverrideVisitor(leftRoot, rightRoot structvar.View) (applyPythonOutpu // leftRoot: {"bundle": ..., "resources": ...}, // rightRoot: {"bundle": ...} - return addResourceKeys(deleted, + return deleted.AddPattern( structpath.MustParsePattern("resources.*.*"), leftRoot, ) @@ -77,7 +69,7 @@ func createOverrideVisitor(leftRoot, rightRoot structvar.View) (applyPythonOutpu // leftRoot: {"resources": { "jobs": ..., "pipeline": ...}}}, // rightRoot: {"resources": { "jobs": ...}}}, - return addResourceKeys(deleted, + return deleted.AddPattern( structpath.NewPatternDotStar(structpath.NewPatternStringKey(structpath.MustParsePattern("resources"), np.KeyAt(1))), leftRoot, ) @@ -109,7 +101,7 @@ func createOverrideVisitor(leftRoot, rightRoot structvar.View) (applyPythonOutpu // leftRoot: {"bundle": ..., } // rightRoot: {"bundle": ..., "resources": {...} } - return addResourceKeys(added, + return added.AddPattern( structpath.MustParsePattern("resources.*.*"), rightRoot, ) @@ -120,7 +112,7 @@ func createOverrideVisitor(leftRoot, rightRoot structvar.View) (applyPythonOutpu // leftRoot: {"resources": { }} // rightRoot: {"resources": { "jobs": {...} }} - return addResourceKeys(added, + return added.AddPattern( structpath.NewPatternDotStar(structpath.NewPatternStringKey(structpath.MustParsePattern("resources"), np.KeyAt(1))), rightRoot, ) @@ -156,7 +148,7 @@ func createOverrideVisitor(leftRoot, rightRoot structvar.View) (applyPythonOutpu // leftRoot: {"bundle": ..., "resources": null } // rightRoot: {"bundle": ..., "resources": {...} } - return addResourceKeys(added, + return added.AddPattern( structpath.MustParsePattern("resources.*.*"), rightRoot, ) @@ -167,7 +159,7 @@ func createOverrideVisitor(leftRoot, rightRoot structvar.View) (applyPythonOutpu // leftRoot: {"resources": { "jobs": null }} // rightRoot: {"resources": { "jobs": {...} }} - return addResourceKeys(added, + return added.AddPattern( structpath.NewPatternDotStar(structpath.NewPatternStringKey(structpath.MustParsePattern("resources"), np.KeyAt(1))), rightRoot, ) diff --git a/bundle/configsync/variables.go b/bundle/configsync/variables.go index 1030a912719..c6e98e1bb73 100644 --- a/bundle/configsync/variables.go +++ b/bundle/configsync/variables.go @@ -3,6 +3,7 @@ package configsync import ( "context" "errors" + "reflect" "strings" "github.com/databricks/cli/bundle" @@ -297,7 +298,7 @@ func restoreFromSiblingsAt(value any, siblings []structvar.View, resolved resolv if !found { continue } - if rv == value { + if equalValues(rv, value) { refs[s] = struct{}{} } } else if isStr && structvar.ContainsVariableReference(s) { @@ -360,7 +361,7 @@ func matchAnyVariable(remoteValue any, resolved resolvedConfig) (string, bool) { v := variable.Get("value") switch v.Kind() { case structvar.KindString, structvar.KindInt, structvar.KindBool: - if v.AsAny() == remoteValue { + if equalValues(v.AsAny(), remoteValue) { match = pathToRef(structpath.NewStringKey(varPrefix, name)) count++ } @@ -400,7 +401,7 @@ func matchOriginalRef(remoteValue any, preResolved structvar.View, resolved reso return "", false } - if resolvedV == remoteValue { + if equalValues(resolvedV, remoteValue) { return s, true } return "", false @@ -580,3 +581,25 @@ func sequenceSiblings(preResolved structvar.View, fieldPath string) ([]structvar } return seq, true } + +// equalValues reports whether two configuration values are equal, treating integers of +// different Go types as equal: variables decoded from YAML hold int, while typed +// fields of the remote resource hold int64. +func equalValues(a, b any) bool { + ai, aok := asInt64(a) + bi, bok := asInt64(b) + if aok || bok { + return aok && bok && ai == bi + } + return a == b +} + +func asInt64(v any) (int64, bool) { + rv := reflect.ValueOf(v) + switch rv.Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return rv.Int(), true + default: + return 0, false + } +} diff --git a/bundle/configsync/variables_test.go b/bundle/configsync/variables_test.go index e25917c2f9e..b82664aa9c4 100644 --- a/bundle/configsync/variables_test.go +++ b/bundle/configsync/variables_test.go @@ -85,6 +85,15 @@ func TestRestoreFromSiblings_ValueMatchesVariableButDifferentPath(t *testing.T) // TestRestoreFromSiblings_AmbiguousAcrossSiblings fences the multi-variable // same-value rule: when two siblings use different variables at the same // relative path that both resolve to the same value, restoration is skipped. +func TestRestoreFromSiblings_IntVariableMatchesInt64Field(t *testing.T) { + // The variable decodes from YAML as int; the remote field is a typed int64. + siblings := tasksView(t, `task_key: main, max_retries: "${var.retry_count}"`) + resolved := variablesConfig(t, map[string]any{"retry_count": 5}) + value := map[string]any{"task_key": "other", "max_retries": int64(5)} + result := restoreFromSiblings(value, siblings, resolved, &RestoreStats{}).(map[string]any) + assert.Equal(t, "${var.retry_count}", result["max_retries"]) +} + func TestRestoreFromSiblings_AmbiguousAcrossSiblings(t *testing.T) { siblings := tasksView(t, `task_key: "${var.landing_schema}"`, `task_key: "${var.curated_schema}"`) resolved := variablesConfig(t, map[string]any{"landing_schema": "raw_data", "curated_schema": "raw_data"}) diff --git a/bundle/direct/resolve_config.go b/bundle/direct/resolve_config.go index 45ab83ef1fb..f35ceaef5b3 100644 --- a/bundle/direct/resolve_config.go +++ b/bundle/direct/resolve_config.go @@ -2,6 +2,8 @@ package direct import ( "fmt" + "maps" + "slices" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/libs/structs/structaccess" @@ -32,13 +34,10 @@ func (b *DeploymentBundle) ResolveConfigAgainstState(cfg *config.Root, target *s return fmt.Errorf("resource %s not found in configuration", target) } - // Each string is resolved on its own (references only point at other paths), then - // written back; collect first because writing changes the view. - type update struct { - path *structpath.PathNode - value any - } - var updates []update + // Collect every reference string first: resolving them in one call shares the + // lookups, and writing back changes the view. + templates := map[string]structvar.Template{} + paths := map[string]*structpath.PathNode{} lookup := func(path *structpath.PathNode) (structvar.View, error) { if !path.HasPrefix(resourcesPrefix) { @@ -67,22 +66,28 @@ func (b *DeploymentBundle) ResolveConfigAgainstState(cfg *config.Root, target *s return nil } key := p.String() - out, err := structvar.Resolve(map[string]structvar.Template{key: {Value: s}}, lookup) - if err != nil { - return err - } - if resolved, ok := out[key]; ok { - updates = append(updates, update{structpath.Join(target, p.AsSlice()...), resolved.AsAny()}) - } + templates[key] = structvar.Template{Value: s} + paths[key] = structpath.Join(target, p.AsSlice()...) return nil }) if err != nil { return err } - for _, u := range updates { + resolved, err := structvar.Resolve(templates, lookup) + if err != nil { + return err + } + + // Snapshot all values before writing: a resolved view may point into the configuration. + values := map[string]any{} + for key, v := range resolved { + values[key] = v.AsAny() + } + for _, key := range slices.Sorted(maps.Keys(values)) { + value := values[key] // Decode converts like loading did, e.g. a string id into an int job_id. - diags, err := cfg.Decode(u.path, structvar.NewView(&u.value, nil, nil)) + diags, err := cfg.Decode(paths[key], structvar.NewView(&value, nil, nil)) if err != nil { return err } diff --git a/bundle/internal/schema/annotations.go b/bundle/internal/schema/annotations.go index 7cc5776ff4c..894930f5491 100644 --- a/bundle/internal/schema/annotations.go +++ b/bundle/internal/schema/annotations.go @@ -2,8 +2,10 @@ package main import ( "fmt" + "maps" "reflect" "regexp" + "slices" "strings" "github.com/databricks/cli/bundle/internal/annotation" @@ -95,6 +97,8 @@ func mergeDescriptor(a, b annotation.Descriptor) annotation.Descriptor { case g.IsZero(): case f.Kind() == reflect.Slice: f.Set(reflect.AppendSlice(reflect.AppendSlice(reflect.Zero(f.Type()), f), g)) + case f.Kind() == reflect.Interface: + f.Set(reflect.ValueOf(mergeAny(f.Interface(), g.Interface()))) case f.Kind() == reflect.Map: m := reflect.MakeMap(f.Type()) for _, v := range []reflect.Value{f, g} { @@ -110,6 +114,28 @@ func mergeDescriptor(a, b annotation.Descriptor) annotation.Descriptor { return a } +// mergeAny merges b over a like mergeDescriptor merges fields: maps merge by key +// (recursively), sequences concatenate and other values are replaced. +func mergeAny(a, b any) any { + switch bv := b.(type) { + case map[string]any: + av, ok := a.(map[string]any) + if !ok { + return b + } + out := maps.Clone(av) + for k, v := range bv { + out[k] = mergeAny(av[k], v) + } + return out + case []any: + if av, ok := a.([]any); ok { + return slices.Concat(av, bv) + } + } + return b +} + func (d *annotationHandler) addAnnotations(typ reflect.Type, s jsonschema.Schema) jsonschema.Schema { refPath := getPath(typ) shouldHandle := strings.HasPrefix(refPath, "github.com") diff --git a/bundle/internal/schema/annotations_file.go b/bundle/internal/schema/annotations_file.go index 5d9f96b55bc..df759fe29c4 100644 --- a/bundle/internal/schema/annotations_file.go +++ b/bundle/internal/schema/annotations_file.go @@ -153,6 +153,10 @@ func loadAnnotationsFile(path string, g *typeGraph) (annotation.File, []string, if err != nil { return nil, nil, err } + if v == nil { + // An empty document has no annotations. + return annotation.File{}, nil, nil + } l := &fileLoader{graph: g, data: annotation.File{}} err = l.block(v, g.root, "") diff --git a/bundle/internal/schema/annotations_file_test.go b/bundle/internal/schema/annotations_file_test.go index 42899b0c184..d50dcf6ff1b 100644 --- a/bundle/internal/schema/annotations_file_test.go +++ b/bundle/internal/schema/annotations_file_test.go @@ -167,3 +167,15 @@ func TestAnnotationsFileDetachedEntries(t *testing.T) { "github.com/databricks/cli/no/such.Type: field", }, detached) } + +func TestAnnotationsFileEmpty(t *testing.T) { + g, err := newTypeGraph(reflect.TypeFor[tgRoot]()) + require.NoError(t, err) + path := filepath.Join(t.TempDir(), "annotations.yml") + require.NoError(t, os.WriteFile(path, []byte("# no annotations yet\n"), 0o644)) + + data, unknown, err := loadAnnotationsFile(path, g) + require.NoError(t, err) + assert.Empty(t, data) + assert.Empty(t, unknown) +} diff --git a/bundle/internal/schema/annotations_test.go b/bundle/internal/schema/annotations_test.go index 3ad9bae6fd6..e43acc97c20 100644 --- a/bundle/internal/schema/annotations_test.go +++ b/bundle/internal/schema/annotations_test.go @@ -270,3 +270,16 @@ func TestBuildEnumDescriptions(t *testing.T) { assert.Equal(t, []string{"[Public Preview]", "", "[Beta]"}, got) }) } + +func TestMergeAnnotationFilesMergesDefaults(t *testing.T) { + a := annotation.File{} + a.SetField("t", "f", annotation.Descriptor{Default: map[string]any{"a": 1, "nested": map[string]any{"x": 1}}}) + a.SetField("t", "g", annotation.Descriptor{Default: []any{1}}) + b := annotation.File{} + b.SetField("t", "f", annotation.Descriptor{Default: map[string]any{"b": 2, "nested": map[string]any{"y": 2}}}) + b.SetField("t", "g", annotation.Descriptor{Default: []any{2}}) + + merged := mergeAnnotationFiles(a, b) + assert.Equal(t, map[string]any{"a": 1, "b": 2, "nested": map[string]any{"x": 1, "y": 2}}, merged["t"].Fields["f"].Default) + assert.Equal(t, []any{1, 2}, merged["t"].Fields["g"].Default) +} diff --git a/libs/structs/structvar/write.go b/libs/structs/structvar/write.go index 42c3c6f464d..e0f605fa4c0 100644 --- a/libs/structs/structvar/write.go +++ b/libs/structs/structvar/write.go @@ -36,11 +36,18 @@ func (sv *StructVar) Set(path *structpath.PathNode, value any) error { } sv.Refs = withoutRefs(sv.Refs, path) // The node marks the value as present even if it is zero (e.g. an empty struct). - var locs *Locations - if value != nil { - locs = (*Locations)(nil).WithLocations(sv.Locations.At(path)) + // A leaf node is already what we would write; skip rewriting the tree, which clones + // every ancestor (e.g. the map of all jobs). + cur := sv.Locations.Sub(path) + if value == nil { + if cur != nil { + sv.Locations = sv.Locations.With(path, nil) + } + return nil + } + if cur == nil || len(cur.keys) > 0 || len(cur.elems) > 0 { + sv.Locations = sv.Locations.With(path, (*Locations)(nil).WithLocations(cur.Get())) } - sv.Locations = sv.Locations.With(path, locs) return nil } From 3ca961608c322aea46f1f3c74e576e8de7eccba6 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 12 Oct 2026 12:59:19 +0200 Subject: [PATCH 16/22] Port the job environments project_environment pattern (#6983) to structpath Co-authored-by: Isaac --- bundle/config/mutator/paths/job_paths_visitor.go | 10 +--------- bundle/config/mutator/paths/job_paths_visitor_test.go | 8 ++++---- 2 files changed, 5 insertions(+), 13 deletions(-) diff --git a/bundle/config/mutator/paths/job_paths_visitor.go b/bundle/config/mutator/paths/job_paths_visitor.go index 36f28c5e559..9b8c3ea5a94 100644 --- a/bundle/config/mutator/paths/job_paths_visitor.go +++ b/bundle/config/mutator/paths/job_paths_visitor.go @@ -73,15 +73,7 @@ func jobRewritePatterns() []jobRewritePattern { }, // Environment-level pattern for all jobs. jobRewritePattern{ - dyn.NewPattern( - dyn.Key("resources"), - dyn.Key("jobs"), - dyn.AnyKey(), - dyn.Key("environments"), - dyn.AnyIndex(), - dyn.Key("spec"), - dyn.Key("project_environment"), - ), + structpath.MustParsePattern("resources.jobs.*.environments[*].spec.project_environment"), TranslateModeFile, noSkipRewrite, }, diff --git a/bundle/config/mutator/paths/job_paths_visitor_test.go b/bundle/config/mutator/paths/job_paths_visitor_test.go index 7999533f96b..2d5e0616312 100644 --- a/bundle/config/mutator/paths/job_paths_visitor_test.go +++ b/bundle/config/mutator/paths/job_paths_visitor_test.go @@ -124,10 +124,10 @@ func TestVisitJobPaths_environments(t *testing.T) { } actual := collectVisitedPaths(t, root, VisitJobPaths) - expected := []*structpath.PathNode{ - dyn.MustPathFromString("resources.jobs.job0.environments[1].spec.project_environment"), - dyn.MustPathFromString("resources.jobs.job0.environments[2].spec.project_environment"), - } + expected := structpath.MustParsePaths( + "resources.jobs.job0.environments[1].spec.project_environment", + "resources.jobs.job0.environments[2].spec.project_environment", + ) assert.ElementsMatch(t, expected, actual) } From 651eecce65398eec9565d6b28cbc13b0e2e51201 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 14 Oct 2026 23:01:22 +0200 Subject: [PATCH 17/22] Fix regressions recorded by the config edge-case tests - apply_default_task_source: an explicit source: "" is user-set; detect it by its definition location (not every task type has ForceSendFields). - pipeline_development_deprecated: warn on any user-written development value, including a reference resolving to an unset field. - rewriteShorthands: a later merged map takes precedence over an earlier one, and merge keys are expanded before detecting a full variable definition. - required: look dashboards up by a path that round-trips keys with path syntax. Co-authored-by: Isaac --- .../apply_default_task_source.go | 36 ++++++++++++------- bundle/config/root.go | 26 ++++++++++---- .../pipeline_development_deprecated.go | 3 +- bundle/config/validate/required.go | 9 ++--- 4 files changed, 48 insertions(+), 26 deletions(-) diff --git a/bundle/config/mutator/resourcemutator/apply_default_task_source.go b/bundle/config/mutator/resourcemutator/apply_default_task_source.go index a95130597f3..df0ca1fec68 100644 --- a/bundle/config/mutator/resourcemutator/apply_default_task_source.go +++ b/bundle/config/mutator/resourcemutator/apply_default_task_source.go @@ -2,6 +2,7 @@ package resourcemutator import ( "context" + "fmt" "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" @@ -49,10 +50,11 @@ func (a *applyDefaultTaskSource) Apply(ctx context.Context, b *bundle.Bundle) di for i := range job.Tasks { task := &job.Tasks[i] + taskPath := fmt.Sprintf("resources.jobs.%s.tasks[%d]", name, i) if task.ForEachTask != nil { - setGitTaskSource(&task.ForEachTask.Task) + setGitTaskSource(b, taskPath+".for_each_task.task", &task.ForEachTask.Task) } - setGitTaskSource(task) + setGitTaskSource(b, taskPath, task) } } return nil @@ -62,27 +64,35 @@ func (a *applyDefaultTaskSource) Apply(ctx context.Context, b *bundle.Bundle) di // supports the field, unless the user already set it. // The task types are the ones that support the `source` field: // https://docs.databricks.com/api/workspace/jobs/create -func setGitTaskSource(task *jobs.Task) { - var sources []*jobs.Source +func setGitTaskSource(b *bundle.Bundle, taskPath string, task *jobs.Task) { + type taskSource struct { + key string + source *jobs.Source + } + var sources []taskSource if task.DbtTask != nil { - sources = append(sources, &task.DbtTask.Source) + sources = append(sources, taskSource{"dbt_task", &task.DbtTask.Source}) } if task.GenAiComputeTask != nil { - sources = append(sources, &task.GenAiComputeTask.Source) + sources = append(sources, taskSource{"gen_ai_compute_task", &task.GenAiComputeTask.Source}) } if task.NotebookTask != nil { - sources = append(sources, &task.NotebookTask.Source) + sources = append(sources, taskSource{"notebook_task", &task.NotebookTask.Source}) } if task.SparkPythonTask != nil { - sources = append(sources, &task.SparkPythonTask.Source) + sources = append(sources, taskSource{"spark_python_task", &task.SparkPythonTask.Source}) } if task.SqlTask != nil && task.SqlTask.File != nil { - sources = append(sources, &task.SqlTask.File.Source) + sources = append(sources, taskSource{"sql_task.file", &task.SqlTask.File.Source}) } - for _, source := range sources { - if *source == "" { - *source = jobs.SourceGit - return + for _, s := range sources { + // An explicit `source: ""` is set by the user, although the typed value is + // empty; not every task type has ForceSendFields to record it, but it is + // defined in the configuration. + if *s.source != "" || b.Config.DefinitionLocation(taskPath+"."+s.key+".source").File != "" { + continue } + *s.source = jobs.SourceGit + return } } diff --git a/bundle/config/root.go b/bundle/config/root.go index 07c2933eb6b..0ed17499e08 100644 --- a/bundle/config/root.go +++ b/bundle/config/root.go @@ -361,8 +361,7 @@ func mappingValue(node *yaml.Node, key string) *yaml.Node { } // expandMergeKeys returns a copy of the mapping node with "<<" merge keys replaced -// by the pairs they merge in (keys set explicitly take precedence), so the pairs -// can be rewritten individually. +// by the pairs they merge in, so the pairs can be rewritten individually. func expandMergeKeys(node *yaml.Node) *yaml.Node { var merged, explicit []*yaml.Node for i := 0; i+1 < len(node.Content); i += 2 { @@ -383,16 +382,29 @@ func expandMergeKeys(node *yaml.Node) *yaml.Node { if merged == nil { return node } + // Like the YAML decoder, a later merged map takes precedence over an earlier one and + // keys set explicitly take precedence over all merged maps. out := *node - out.Content = explicit - for i := 0; i+1 < len(merged); i += 2 { - if mappingValue(&out, merged[i].Value) == nil { - out.Content = append(out.Content, merged[i], merged[i+1]) + out.Content = nil + for _, kvs := range [][]*yaml.Node{merged, explicit} { + for i := 0; i+1 < len(kvs); i += 2 { + setMappingValue(&out, kvs[i], kvs[i+1]) } } return &out } +// setMappingValue replaces the value of key in the mapping node, or appends the pair. +func setMappingValue(node, key, value *yaml.Node) { + for i := 0; i+1 < len(node.Content); i += 2 { + if node.Content[i].Value == key.Value { + node.Content[i+1] = value + return + } + } + node.Content = append(node.Content, key, value) +} + func mappingKeys(node *yaml.Node) []string { var keys []string for i := 0; i+1 < len(node.Content); i += 2 { @@ -438,7 +450,7 @@ func rewriteShorthands(root *yaml.Node) { case variable.Kind == yaml.MappingNode || variable.Kind == yaml.SequenceNode: // If it's a full variable definition, leave it as is. - if variable.Kind == yaml.MappingNode && isFullVariableOverrideDef(mappingKeys(variable)) { + if variable.Kind == yaml.MappingNode && isFullVariableOverrideDef(mappingKeys(expandMergeKeys(variable))) { continue } diff --git a/bundle/config/validate/pipeline_development_deprecated.go b/bundle/config/validate/pipeline_development_deprecated.go index f6672a4646f..62c838beae3 100644 --- a/bundle/config/validate/pipeline_development_deprecated.go +++ b/bundle/config/validate/pipeline_development_deprecated.go @@ -37,8 +37,7 @@ func (v *pipelineDevelopmentDeprecated) Apply(_ context.Context, b *bundle.Bundl pipelines := b.Config.Resources.Pipelines for _, key := range slices.Sorted(maps.Keys(pipelines)) { pipeline := pipelines[key] - //nolint:staticcheck // SA1019: pipeline development is deprecated in the SDK but remains a supported bundle config field - if pipeline == nil || (!pipeline.Development && !slices.Contains(pipeline.ForceSendFields, "Development")) { + if pipeline == nil { continue } diff --git a/bundle/config/validate/required.go b/bundle/config/validate/required.go index 3de90d4c335..a61986954cc 100644 --- a/bundle/config/validate/required.go +++ b/bundle/config/validate/required.go @@ -95,13 +95,14 @@ func errorForMissingFields(ctx context.Context, b *bundle.Bundle) diag.Diagnosti diags := diag.Diagnostics{} for key, dashboard := range b.Config.Resources.Dashboards { + path := structpath.NewPath(nil, "resources", "dashboards", key) if dashboard.DisplayName == "" { - nameLocations = append(nameLocations, b.Config.GetLocations("resources.dashboards."+key)...) - namePaths = append(namePaths, structpath.NewPath(nil, "resources", "dashboards", key)) + nameLocations = append(nameLocations, b.Config.GetLocations(path.String())...) + namePaths = append(namePaths, path) } if dashboard.WarehouseId == "" { - warehouseIdLocations = append(warehouseIdLocations, b.Config.GetLocations("resources.dashboards."+key)...) - warehouseIdPaths = append(warehouseIdPaths, structpath.NewPath(nil, "resources", "dashboards", key)) + warehouseIdLocations = append(warehouseIdLocations, b.Config.GetLocations(path.String())...) + warehouseIdPaths = append(warehouseIdPaths, path) } } From 3ecf7d60e5e021037b5d7333ef9a38aa3649542d Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 15 Oct 2026 11:32:52 +0200 Subject: [PATCH 18/22] Drop the embedded JobSettings literal flagged by embedlit Co-authored-by: Isaac --- bundle/config/mutator/resourcemutator/resource_mutator_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bundle/config/mutator/resourcemutator/resource_mutator_test.go b/bundle/config/mutator/resourcemutator/resource_mutator_test.go index 572b688946b..286362d4871 100644 --- a/bundle/config/mutator/resourcemutator/resource_mutator_test.go +++ b/bundle/config/mutator/resourcemutator/resource_mutator_test.go @@ -7,13 +7,12 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/structs/structpath" - "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func newJob(name string) *resources.Job { - return &resources.Job{JobSettings: jobs.JobSettings{Name: name}} + return &resources.Job{Name: name} } func TestSelectResources(t *testing.T) { From 86b1c45bf301cf9dc231bfdf18ec1d782a4f4acf Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 15 Oct 2026 16:14:34 +0200 Subject: [PATCH 19/22] Keep main's path-node location lookups; build app permission paths from parts Earlier rebases resolved conflicts with #6980 in favor of string paths, reverting call sites that main already builds with structpath.NewPath. Restore main's versions where this PR has no reason to change them, and make the app permission checks take a path node instead of parsing a string. Co-authored-by: Isaac --- bundle/apps/validate.go | 40 ++++++------------- .../apply_source_linked_deployment_preset.go | 16 +++----- .../resourcemutator/secret_scope_fixups.go | 5 +-- bundle/config/mutator/sync_infer_root.go | 5 ++- bundle/config/validate/files_to_sync.go | 6 +-- .../validate/job_cluster_key_defined.go | 7 ++-- .../config/validate/job_task_cluster_spec.go | 2 +- bundle/config/validate/required.go | 8 ++-- .../config/validate/validate_artifact_path.go | 4 +- .../validate/validate_deployment_fields.go | 14 +++---- .../validate_job_run_idempotency_token.go | 2 +- .../check_dashboards_modified_remotely.go | 2 +- 12 files changed, 45 insertions(+), 66 deletions(-) diff --git a/bundle/apps/validate.go b/bundle/apps/validate.go index d2b4355c635..339764396b3 100644 --- a/bundle/apps/validate.go +++ b/bundle/apps/validate.go @@ -9,7 +9,6 @@ import ( "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/structs/structpath" - "github.com/databricks/cli/libs/structs/structvar" "github.com/databricks/databricks-sdk-go/service/apps" ) @@ -28,7 +27,7 @@ func (v *validate) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics Severity: diag.Error, Summary: "Missing app source code path or git source", Detail: fmt.Sprintf("app resource '%s' should have either source_code_path or git_source field", key), - Locations: b.Config.GetLocations("resources.apps." + key), + Locations: b.Config.GetLocationsOf(structpath.NewPath(nil, "resources", "apps", key)), }) continue } @@ -38,7 +37,7 @@ func (v *validate) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics Severity: diag.Error, Summary: "Both source_code_path and git_source fields are set", Detail: fmt.Sprintf("app resource '%s' should have either source_code_path or git_source field, not both", key), - Locations: b.Config.GetLocations("resources.apps." + key), + Locations: b.Config.GetLocationsOf(structpath.NewPath(nil, "resources", "apps", key)), }) continue } @@ -48,7 +47,7 @@ func (v *validate) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics Severity: diag.Error, Summary: "Duplicate app source code path", Detail: fmt.Sprintf("app resource '%s' has the same source code path as app resource '%s', this will lead to the app configuration being overridden by each other", key, usedSourceCodePaths[app.SourceCodePath]), - Locations: b.Config.GetLocations(fmt.Sprintf("resources.apps.%s.source_code_path", key)), + Locations: b.Config.GetLocationsOf(structpath.NewPath(nil, "resources", "apps", key, "source_code_path")), }) } usedSourceCodePaths[app.SourceCodePath] = key @@ -92,12 +91,9 @@ type appResourceReference struct { permission string } -// hasPermissions checks if a bundle resource at the given dyn path has a non-empty permissions list. -func hasPermissions(b *bundle.Bundle, resourcePath string) bool { - pv, err := valueAt(b, resourcePath+".permissions") - if err != nil { - return false - } +// hasPermissions checks if a bundle resource at the given path has a non-empty permissions list. +func hasPermissions(b *bundle.Bundle, resourcePath *structpath.PathNode) bool { + pv := b.Config.View().Lookup(structpath.NewPath(resourcePath, "permissions")) for range pv.Sequence() { return true } @@ -106,12 +102,9 @@ func hasPermissions(b *bundle.Bundle, resourcePath string) bool { // hasAppSPInPermissions checks if any permission entry for the given resource // references the app's service principal via variable interpolation. -func hasAppSPInPermissions(b *bundle.Bundle, resourcePath, appKey string) bool { +func hasAppSPInPermissions(b *bundle.Bundle, resourcePath *structpath.PathNode, appKey string) bool { appSPRef := fmt.Sprintf("${resources.apps.%s.service_principal_client_id}", appKey) - pv, err := valueAt(b, resourcePath+".permissions") - if err != nil { - return false - } + pv := b.Config.View().Lookup(structpath.NewPath(resourcePath, "permissions")) for _, entry := range pv.Sequence() { if str, ok := entry.Get("service_principal_name").AsString(); ok && str == appSPRef { return true @@ -140,7 +133,7 @@ func warnForAppResourcePermissions(b *bundle.Bundle, appKey string, app *resourc } refType, resourceKey := matches[1], matches[2] - resourcePath := fmt.Sprintf("resources.%s.%s", refType, resourceKey) + resourcePath := structpath.NewPath(nil, "resources", refType, resourceKey) if !hasPermissions(b, resourcePath) { continue } @@ -149,7 +142,7 @@ func warnForAppResourcePermissions(b *bundle.Bundle, appKey string, app *resourc continue } - appPath := "resources.apps." + appKey + appPath := structpath.NewPath(nil, "resources", "apps", appKey) diags = append(diags, diag.Diagnostic{ Severity: diag.Warning, Summary: fmt.Sprintf("app %q references %s %q which has permissions set. To prevent permission override after deploying the app, please add the app service principal to the %s permissions", appKey, refType, resourceKey, refType), @@ -167,8 +160,8 @@ func warnForAppResourcePermissions(b *bundle.Bundle, appKey string, app *resourc ref.permission, appKey, ), - Paths: structpath.NewPathSlice("resources", "apps", appKey), - Locations: b.Config.GetLocations(appPath), + Paths: []*structpath.PathNode{appPath}, + Locations: b.Config.GetLocationsOf(appPath), }) } @@ -182,12 +175,3 @@ func (v *validate) Name() string { func Validate() bundle.Mutator { return &validate{} } - -// valueAt returns the view of the configuration value at path. -func valueAt(b *bundle.Bundle, path string) (structvar.View, error) { - p, err := structpath.ParsePath(path) - if err != nil { - return structvar.View{}, err - } - return b.Config.View().Lookup(p), nil -} diff --git a/bundle/config/mutator/apply_source_linked_deployment_preset.go b/bundle/config/mutator/apply_source_linked_deployment_preset.go index dd175aabd7d..695edabc68a 100644 --- a/bundle/config/mutator/apply_source_linked_deployment_preset.go +++ b/bundle/config/mutator/apply_source_linked_deployment_preset.go @@ -13,10 +13,6 @@ import ( type applySourceLinkedDeploymentPreset struct{} -func targetPresetPath(target string) *structpath.PathNode { - return structpath.NewPath(nil, "targets", target, "presets", "source_linked_deployment") -} - // Apply source-linked deployment preset func ApplySourceLinkedDeploymentPreset() *applySourceLinkedDeploymentPreset { return &applySourceLinkedDeploymentPreset{} @@ -37,13 +33,13 @@ func (m *applySourceLinkedDeploymentPreset) Apply(ctx context.Context, b *bundle if config.IsExplicitlyEnabled(b.Config.Presets.SourceLinkedDeployment) { if !isDatabricksWorkspace { - path := targetPresetPath(target) + path := structpath.NewPath(nil, "targets", target, "presets", "source_linked_deployment") diags = diags.Append( diag.Diagnostic{ Severity: diag.Warning, Summary: "source-linked deployment is available only in the Databricks Workspace", Paths: []*structpath.PathNode{path}, - Locations: b.Config.GetLocations("presets.source_linked_deployment"), + Locations: b.Config.GetLocationsOf(path.SkipPrefix(2)), }, ) @@ -55,13 +51,13 @@ func (m *applySourceLinkedDeploymentPreset) Apply(ctx context.Context, b *bundle b.Metrics.AddBoolValue("source_linked_set_for_non_development", b.Config.Bundle.Mode != config.Development) if b.Config.Bundle.Mode != config.Development { - path := targetPresetPath(target) + path := structpath.NewPath(nil, "targets", target, "presets", "source_linked_deployment") diags = diags.Append( diag.Diagnostic{ Severity: diag.Warning, Summary: "source-linked deployment in non-development mode is deprecated and will not be supported in a future release", Paths: []*structpath.PathNode{path}, - Locations: b.Config.GetLocations("presets.source_linked_deployment"), + Locations: b.Config.GetLocationsOf(path.SkipPrefix(2)), }, ) } @@ -74,14 +70,14 @@ func (m *applySourceLinkedDeploymentPreset) Apply(ctx context.Context, b *bundle // This mutator runs before workspace paths are defaulted so it's safe to check for the user-defined value if b.Config.Workspace.FilePath != "" && config.IsExplicitlyEnabled(b.Config.Presets.SourceLinkedDeployment) { - path := structpath.MustParsePath("workspace.file_path") + path := structpath.NewPath(nil, "workspace", "file_path") diags = diags.Append( diag.Diagnostic{ Severity: diag.Warning, Summary: "workspace.file_path setting will be ignored in source-linked deployment mode", Detail: "In source-linked deployment files are not copied to the destination and resources use source files instead", Paths: []*structpath.PathNode{path}, - Locations: b.Config.GetLocations(path.String()), + Locations: b.Config.GetLocationsOf(path), }, ) } diff --git a/bundle/config/mutator/resourcemutator/secret_scope_fixups.go b/bundle/config/mutator/resourcemutator/secret_scope_fixups.go index 468d324905d..8404d2444eb 100644 --- a/bundle/config/mutator/resourcemutator/secret_scope_fixups.go +++ b/bundle/config/mutator/resourcemutator/secret_scope_fixups.go @@ -131,7 +131,6 @@ func (m *secretScopeFixups) Apply(ctx context.Context, b *bundle.Bundle) diag.Di currentUser := b.Config.Workspace.CurrentUser.User addManageForCurrentUser(scope, currentUser) - path := structpath.NewPath(nil, "resources", "secret_scopes", key) err := collapsePermissions(scope) if err != nil { return diag.Diagnostics{ @@ -139,8 +138,8 @@ func (m *secretScopeFixups) Apply(ctx context.Context, b *bundle.Bundle) diag.Di Severity: diag.Error, Summary: "Failed to collapse permissions for secret scope", Detail: err.Error(), - Paths: []*structpath.PathNode{path}, - Locations: []diag.Location{b.Config.GetLocation(path.String())}, + Paths: structpath.NewPathSlice("resources", "secret_scopes", key), + Locations: []diag.Location{b.Config.GetLocationOf(structpath.NewPath(nil, "resources", "secret_scopes", key))}, }, } } diff --git a/bundle/config/mutator/sync_infer_root.go b/bundle/config/mutator/sync_infer_root.go index 75223780081..bd1a529380a 100644 --- a/bundle/config/mutator/sync_infer_root.go +++ b/bundle/config/mutator/sync_infer_root.go @@ -90,11 +90,12 @@ func (m *syncInferRoot) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagno continue } + syncPath := structpath.NewPath(nil, "sync", "paths", i) diags = append(diags, diag.Diagnostic{ Severity: diag.Error, Summary: fmt.Sprintf("invalid sync path %q", path), - Locations: b.Config.GetLocations(fmt.Sprintf("sync.paths[%d]", i)), - Paths: []*structpath.PathNode{structpath.NewIndex(structpath.MustParsePath("sync.paths"), i)}, + Locations: b.Config.GetLocationsOf(syncPath), + Paths: []*structpath.PathNode{syncPath}, }) } diff --git a/bundle/config/validate/files_to_sync.go b/bundle/config/validate/files_to_sync.go index d1b811e17a7..14b032e8da1 100644 --- a/bundle/config/validate/files_to_sync.go +++ b/bundle/config/validate/files_to_sync.go @@ -56,14 +56,14 @@ func (v *filesToSync) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnost Summary: "There are no files to sync, please check your .gitignore", }) } else { - path := "sync.exclude" + path := structpath.NewPath(nil, "sync", "exclude") diags = diags.Append(diag.Diagnostic{ Severity: diag.Warning, Summary: "There are no files to sync, please check your .gitignore and sync.exclude configuration", // Show all locations where sync.exclude is defined, since merging // sync.exclude is additive. - Locations: b.Config.GetLocations(path), - Paths: structpath.MustParsePaths(path), + Locations: b.Config.GetLocationsOf(path), + Paths: []*structpath.PathNode{path}, }) } diff --git a/bundle/config/validate/job_cluster_key_defined.go b/bundle/config/validate/job_cluster_key_defined.go index 23f7f4161c0..eb9bea4df70 100644 --- a/bundle/config/validate/job_cluster_key_defined.go +++ b/bundle/config/validate/job_cluster_key_defined.go @@ -31,14 +31,13 @@ func (v *jobClusterKeyDefined) Apply(ctx context.Context, b *bundle.Bundle) diag } for index, task := range job.Tasks { - taskPath := structpath.NewPath(nil, "resources", "jobs", k, "tasks", index) diags = diags.Extend(checkJobClusterKey(b, jobClusterKeys, task.JobClusterKey, - structpath.NewPath(taskPath, "job_cluster_key"))) + structpath.NewPath(nil, "resources", "jobs", k, "tasks", index, "job_cluster_key"))) // The Jobs API rejects nested for_each_task, so one level is sufficient. if task.ForEachTask != nil { diags = diags.Extend(checkJobClusterKey(b, jobClusterKeys, task.ForEachTask.Task.JobClusterKey, - structpath.NewPath(taskPath, "for_each_task", "task", "job_cluster_key"))) + structpath.NewPath(nil, "resources", "jobs", k, "tasks", index, "for_each_task", "task", "job_cluster_key"))) } } } @@ -61,7 +60,7 @@ func checkJobClusterKey(b *bundle.Bundle, jobClusterKeys map[string]bool, jobClu // Show only the location where the job_cluster_key is defined. // Other associated locations are not relevant since they are // overridden during merging. - Locations: b.Config.GetLocations(path.String()), + Locations: b.Config.GetLocationsOf(path), Paths: []*structpath.PathNode{path}, }} } diff --git a/bundle/config/validate/job_task_cluster_spec.go b/bundle/config/validate/job_task_cluster_spec.go index 56191d114b7..71bce7173d5 100644 --- a/bundle/config/validate/job_task_cluster_spec.go +++ b/bundle/config/validate/job_task_cluster_spec.go @@ -92,7 +92,7 @@ func validateJobTask(b *bundle.Bundle, task jobs.Task, taskPath *structpath.Path Severity: diag.Error, Summary: "Missing required cluster or environment settings", Detail: detail, - Locations: b.Config.GetLocations(taskPath.String()), + Locations: b.Config.GetLocationsOf(taskPath), Paths: []*structpath.PathNode{taskPath}, }) } diff --git a/bundle/config/validate/required.go b/bundle/config/validate/required.go index a61986954cc..21e546738c6 100644 --- a/bundle/config/validate/required.go +++ b/bundle/config/validate/required.go @@ -97,11 +97,11 @@ func errorForMissingFields(ctx context.Context, b *bundle.Bundle) diag.Diagnosti for key, dashboard := range b.Config.Resources.Dashboards { path := structpath.NewPath(nil, "resources", "dashboards", key) if dashboard.DisplayName == "" { - nameLocations = append(nameLocations, b.Config.GetLocations(path.String())...) + nameLocations = append(nameLocations, b.Config.GetLocationsOf(path)...) namePaths = append(namePaths, path) } if dashboard.WarehouseId == "" { - warehouseIdLocations = append(warehouseIdLocations, b.Config.GetLocations(path.String())...) + warehouseIdLocations = append(warehouseIdLocations, b.Config.GetLocationsOf(path)...) warehouseIdPaths = append(warehouseIdPaths, path) } } @@ -131,7 +131,7 @@ func errorForMissingFields(ctx context.Context, b *bundle.Bundle) diag.Diagnosti diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, Summary: "sql_warehouse name is required", - Locations: b.Config.GetLocations(path.String()), + Locations: b.Config.GetLocationsOf(path), Paths: []*structpath.PathNode{path}, }) } @@ -200,7 +200,7 @@ func errorForInvalidSecretScopePermissions(ctx context.Context, b *bundle.Bundle Severity: diag.Error, Summary: "secret scope permission principal is required", Detail: "Set one of user_name, group_name or service_principal_name", - Locations: b.Config.GetLocations(scopePath.String()), + Locations: b.Config.GetLocationsOf(scopePath), Paths: []*structpath.PathNode{path}, }) } diff --git a/bundle/config/validate/validate_artifact_path.go b/bundle/config/validate/validate_artifact_path.go index 1eaa41518cc..572c3a0de5a 100644 --- a/bundle/config/validate/validate_artifact_path.go +++ b/bundle/config/validate/validate_artifact_path.go @@ -68,8 +68,8 @@ func findVolumeInBundle(r config.Root, catalogName, schemaName, volumeName strin if v.SchemaName != schemaName && !isSchemaDefinedInBundle { continue } - pathString := "resources.volumes." + k - return structpath.MustParsePath(pathString), r.GetLocations(pathString), true + volumePath := structpath.NewPath(nil, "resources", "volumes", k) + return volumePath, r.GetLocationsOf(volumePath), true } return nil, nil, false } diff --git a/bundle/config/validate/validate_deployment_fields.go b/bundle/config/validate/validate_deployment_fields.go index 10d95644d13..d1ca337f4ed 100644 --- a/bundle/config/validate/validate_deployment_fields.go +++ b/bundle/config/validate/validate_deployment_fields.go @@ -26,29 +26,29 @@ func (v *validateDeploymentFields) Apply(_ context.Context, b *bundle.Bundle) di // deployment_id and version_id identify the bundle deployment and its version // in the Deployment Metadata Service. The CLI sets them on every deploy, so a // value provided by hand would be overwritten; reject it up front. - reject := func(resource *structpath.PathNode, field, value string) { + reject := func(resourceType, name, field, value string) { if value == "" { return } - path := structpath.NewPath(resource, "deployment", field) + path := structpath.NewPath(nil, "resources", resourceType, name, "deployment", field) diags = append(diags, diag.Diagnostic{ Severity: diag.Error, Summary: field + " must not be set in bundle configuration; it is managed by Declarative Automation Bundles", Paths: []*structpath.PathNode{path}, - Locations: b.Config.GetLocations(path.String()), + Locations: b.Config.GetLocationsOf(path), }) } for name, job := range b.Config.Resources.Jobs { if d := job.Deployment; d != nil { - reject(structpath.NewPath(nil, "resources", "jobs", name), "deployment_id", d.DeploymentId) - reject(structpath.NewPath(nil, "resources", "jobs", name), "version_id", d.VersionId) + reject("jobs", name, "deployment_id", d.DeploymentId) + reject("jobs", name, "version_id", d.VersionId) } } for name, pipeline := range b.Config.Resources.Pipelines { if d := pipeline.Deployment; d != nil { - reject(structpath.NewPath(nil, "resources", "pipelines", name), "deployment_id", d.DeploymentId) - reject(structpath.NewPath(nil, "resources", "pipelines", name), "version_id", d.VersionId) + reject("pipelines", name, "deployment_id", d.DeploymentId) + reject("pipelines", name, "version_id", d.VersionId) } } diff --git a/bundle/config/validate/validate_job_run_idempotency_token.go b/bundle/config/validate/validate_job_run_idempotency_token.go index 4ed6cc75694..61e07e302ca 100644 --- a/bundle/config/validate/validate_job_run_idempotency_token.go +++ b/bundle/config/validate/validate_job_run_idempotency_token.go @@ -37,7 +37,7 @@ func (v *validateJobRunIdempotencyToken) Apply(_ context.Context, b *bundle.Bund Severity: diag.Error, Summary: "idempotency_token must not be set in bundle configuration; the CLI sets it on each run-now request", Paths: []*structpath.PathNode{path}, - Locations: b.Config.GetLocations(path.String()), + Locations: b.Config.GetLocationsOf(path), }) } diff --git a/bundle/deploy/check_dashboards_modified_remotely.go b/bundle/deploy/check_dashboards_modified_remotely.go index 06e3e29f3a1..d685f7ebeb6 100644 --- a/bundle/deploy/check_dashboards_modified_remotely.go +++ b/bundle/deploy/check_dashboards_modified_remotely.go @@ -73,7 +73,7 @@ func (l *checkDashboardsModifiedRemotely) Apply(ctx context.Context, b *bundle.B } path := structpath.NewPath(nil, "resources", "dashboards", dashboard.Name) - loc := b.Config.GetLocation(path.String()) + loc := b.Config.GetLocationOf(path) actual, err := b.WorkspaceClient(ctx).Lakeview.GetByDashboardId(ctx, dashboard.ID) if err != nil { diags = diags.Append(diag.Diagnostic{ From 1bcb66da2a03dc91da31883e6294da1214c471f2 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 15 Oct 2026 17:46:09 +0200 Subject: [PATCH 20/22] config: Nearest/Exact location lookups by path node; build paths from parts Root's location lookups take a *structpath.PathNode and say which value they return: NearestLocations/NearestLocation fall back to the closest ancestor with a location (for diagnostics), ExactLocations/ExactLocation return where the value is written (empty if a mutator set it). The string variants are gone, so callers build paths from parts instead of formatting and parsing them. Literal MustParsePattern/MustParsePath calls in production code are replaced with NewPattern/NewPath, and the app permission checks read the resource view directly. Co-authored-by: Isaac --- bundle/apps/validate.go | 27 ++++---- bundle/artifacts/expand_globs.go | 2 +- bundle/artifacts/prepare.go | 5 +- bundle/config/loader/process_include.go | 6 +- bundle/config/loader/process_root_includes.go | 3 +- .../apply_source_linked_deployment_preset.go | 6 +- bundle/config/mutator/compute_id_compat.go | 2 +- .../config/mutator/initialize_volume_paths.go | 2 +- bundle/config/mutator/normalize_paths_test.go | 4 +- .../mutator/paths/alert_paths_visitor.go | 2 +- .../config/mutator/paths/app_paths_visitor.go | 2 +- .../mutator/paths/artifact_paths_visitor.go | 4 +- .../mutator/paths/dashboard_paths_visitor.go | 2 +- .../paths/genie_space_paths_visitor.go | 2 +- .../paths/job_libraries_paths_visitor.go | 14 ++-- .../config/mutator/paths/job_paths_visitor.go | 27 ++++---- .../mutator/paths/job_run_paths_visitor.go | 2 +- .../mutator/paths/pipeline_paths_visitor.go | 14 ++-- bundle/config/mutator/populate_locations.go | 10 +-- .../mutator/prepend_workspace_prefix.go | 10 +-- .../mutator/prepend_workspace_prefix_test.go | 3 +- .../mutator/python/apply_python_output.go | 14 ++-- .../mutator/python/python_locations_test.go | 2 +- .../config/mutator/python/python_mutator.go | 2 +- .../mutator/reject_internal_resources.go | 2 +- .../mutator/resolve_job_run_file_triggers.go | 21 +++--- .../mutator/resolve_variable_references.go | 11 ++-- .../apply_bundle_permissions.go | 2 +- .../apply_default_task_source.go | 22 +++---- .../mutator/resourcemutator/apply_presets.go | 3 +- .../configure_cluster_policy_definition.go | 2 +- ...nfigure_dashboards_serialized_dashboard.go | 2 +- .../configure_genie_space_serialized_space.go | 2 +- .../expand_pipeline_glob_paths.go | 2 +- .../resourcemutator/fix_permissions.go | 2 +- .../model_serving_endpoint_fixups.go | 5 +- .../config/mutator/resourcemutator/run_as.go | 12 ++-- .../mutator/resourcemutator/run_as_test.go | 3 +- .../resourcemutator/secret_scope_fixups.go | 2 +- .../resourcemutator/validate_target_mode.go | 5 +- bundle/config/mutator/rewrite_sync_paths.go | 2 +- bundle/config/mutator/sync_infer_root.go | 2 +- bundle/config/mutator/translate_paths.go | 2 +- .../mutator/validate_job_run_triggers.go | 15 +++-- .../validate_secret_value_is_variable.go | 2 +- bundle/config/root.go | 66 ++++++------------- .../validate/all_resources_have_values.go | 2 +- bundle/config/validate/files_to_sync.go | 2 +- .../validate/job_cluster_key_defined.go | 2 +- .../config/validate/job_task_cluster_spec.go | 2 +- .../no_interpolation_in_bundle_name.go | 2 +- .../no_variable_reference_in_resource_key.go | 2 +- .../pipeline_development_deprecated.go | 2 +- bundle/config/validate/required.go | 10 +-- bundle/config/validate/scripts.go | 4 +- bundle/config/validate/single_node_cluster.go | 10 +-- .../config/validate/unique_resource_keys.go | 2 +- .../config/validate/validate_artifact_path.go | 10 +-- .../validate/validate_dashboard_etags.go | 2 +- .../validate/validate_deployment_fields.go | 2 +- bundle/config/validate/validate_engine.go | 4 +- .../validate/validate_genie_space_etags.go | 2 +- .../validate_job_run_idempotency_token.go | 2 +- .../config/validate/validate_sync_patterns.go | 2 +- .../config/validate/validate_volume_path.go | 15 ++--- bundle/configsync/resolve.go | 7 +- .../check_dashboards_modified_remotely.go | 2 +- bundle/deploy/metadata/compute.go | 7 +- bundle/direct/bundle_plan.go | 6 +- bundle/direct/dresources/cluster.go | 2 +- .../dresources/model_serving_endpoint.go | 10 +-- .../dresources/vector_search_endpoint.go | 4 +- bundle/direct/resolve_config.go | 2 +- bundle/libraries/expand_glob_references.go | 20 +++--- .../libraries/expand_glob_references_test.go | 8 +-- bundle/libraries/remote_path.go | 18 ++--- bundle/libraries/same_name_libraries.go | 16 ++--- bundle/migrate/build_state.go | 6 +- bundle/permissions/permission_diagnostics.go | 3 +- bundle/phases/telemetry.go | 2 +- bundle/statemgmt/reverse_interpolate_test.go | 2 +- bundle/statemgmt/state_load.go | 2 +- bundle/tests/include_test.go | 7 +- cmd/bundle/utils/process.go | 3 +- libs/structs/structvar/locations.go | 4 +- libs/structs/structvar/write.go | 2 +- 86 files changed, 279 insertions(+), 287 deletions(-) diff --git a/bundle/apps/validate.go b/bundle/apps/validate.go index 339764396b3..568b4db02fc 100644 --- a/bundle/apps/validate.go +++ b/bundle/apps/validate.go @@ -9,6 +9,7 @@ import ( "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" "github.com/databricks/databricks-sdk-go/service/apps" ) @@ -27,7 +28,7 @@ func (v *validate) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics Severity: diag.Error, Summary: "Missing app source code path or git source", Detail: fmt.Sprintf("app resource '%s' should have either source_code_path or git_source field", key), - Locations: b.Config.GetLocationsOf(structpath.NewPath(nil, "resources", "apps", key)), + Locations: b.Config.NearestLocations(structpath.NewPath(nil, "resources", "apps", key)), }) continue } @@ -37,7 +38,7 @@ func (v *validate) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics Severity: diag.Error, Summary: "Both source_code_path and git_source fields are set", Detail: fmt.Sprintf("app resource '%s' should have either source_code_path or git_source field, not both", key), - Locations: b.Config.GetLocationsOf(structpath.NewPath(nil, "resources", "apps", key)), + Locations: b.Config.NearestLocations(structpath.NewPath(nil, "resources", "apps", key)), }) continue } @@ -47,7 +48,7 @@ func (v *validate) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics Severity: diag.Error, Summary: "Duplicate app source code path", Detail: fmt.Sprintf("app resource '%s' has the same source code path as app resource '%s', this will lead to the app configuration being overridden by each other", key, usedSourceCodePaths[app.SourceCodePath]), - Locations: b.Config.GetLocationsOf(structpath.NewPath(nil, "resources", "apps", key, "source_code_path")), + Locations: b.Config.NearestLocations(structpath.NewPath(nil, "resources", "apps", key, "source_code_path")), }) } usedSourceCodePaths[app.SourceCodePath] = key @@ -91,10 +92,9 @@ type appResourceReference struct { permission string } -// hasPermissions checks if a bundle resource at the given path has a non-empty permissions list. -func hasPermissions(b *bundle.Bundle, resourcePath *structpath.PathNode) bool { - pv := b.Config.View().Lookup(structpath.NewPath(resourcePath, "permissions")) - for range pv.Sequence() { +// hasPermissions checks if a bundle resource has a non-empty permissions list. +func hasPermissions(resource structvar.View) bool { + for range resource.Get("permissions").Sequence() { return true } return false @@ -102,10 +102,9 @@ func hasPermissions(b *bundle.Bundle, resourcePath *structpath.PathNode) bool { // hasAppSPInPermissions checks if any permission entry for the given resource // references the app's service principal via variable interpolation. -func hasAppSPInPermissions(b *bundle.Bundle, resourcePath *structpath.PathNode, appKey string) bool { +func hasAppSPInPermissions(resource structvar.View, appKey string) bool { appSPRef := fmt.Sprintf("${resources.apps.%s.service_principal_client_id}", appKey) - pv := b.Config.View().Lookup(structpath.NewPath(resourcePath, "permissions")) - for _, entry := range pv.Sequence() { + for _, entry := range resource.Get("permissions").Sequence() { if str, ok := entry.Get("service_principal_name").AsString(); ok && str == appSPRef { return true } @@ -133,12 +132,12 @@ func warnForAppResourcePermissions(b *bundle.Bundle, appKey string, app *resourc } refType, resourceKey := matches[1], matches[2] - resourcePath := structpath.NewPath(nil, "resources", refType, resourceKey) - if !hasPermissions(b, resourcePath) { + resource := b.Config.View().Get("resources").Get(refType).Get(resourceKey) + if !hasPermissions(resource) { continue } - if hasAppSPInPermissions(b, resourcePath, appKey) { + if hasAppSPInPermissions(resource, appKey) { continue } @@ -161,7 +160,7 @@ func warnForAppResourcePermissions(b *bundle.Bundle, appKey string, app *resourc appKey, ), Paths: []*structpath.PathNode{appPath}, - Locations: b.Config.GetLocationsOf(appPath), + Locations: b.Config.NearestLocations(appPath), }) } diff --git a/bundle/artifacts/expand_globs.go b/bundle/artifacts/expand_globs.go index 6b1c46a8f43..9b08566eb33 100644 --- a/bundle/artifacts/expand_globs.go +++ b/bundle/artifacts/expand_globs.go @@ -46,7 +46,7 @@ func (e expandGlobs) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnosti base := structpath.NewPath(nil, "artifacts", e.name, "files") // Pattern to match the source key in the files sequence. - pattern := structpath.NewPatternStringKey(structpath.NewPatternBracketStar(structpath.NewPatternStringKey(structpath.NewPatternStringKey(structpath.NewPatternStringKey(nil, "artifacts"), e.name), "files")), "source") + pattern := structpath.NewPattern(nil, "artifacts", e.name, "files", structpath.AnyIndex, "source") artifact := b.Config.Artifacts[e.name] if artifact == nil { diff --git a/bundle/artifacts/prepare.go b/bundle/artifacts/prepare.go index d5330208eeb..740700815ff 100644 --- a/bundle/artifacts/prepare.go +++ b/bundle/artifacts/prepare.go @@ -17,6 +17,7 @@ import ( "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/logdiag" "github.com/databricks/cli/libs/python" + "github.com/databricks/cli/libs/structs/structpath" ) func Prepare() bundle.Mutator { @@ -38,7 +39,7 @@ func (m *prepare) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics for _, artifactName := range slices.Sorted(maps.Keys(b.Config.Artifacts)) { artifact := b.Config.Artifacts[artifactName] if artifact == nil { - l := b.Config.GetLocation("artifacts." + artifactName) + l := b.Config.NearestLocation(structpath.NewPath(nil, "artifacts", artifactName)) logdiag.LogDiag(ctx, diag.Diagnostic{ Severity: diag.Error, Summary: "Artifact not properly configured", @@ -61,7 +62,7 @@ func (m *prepare) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics logdiag.LogError(ctx, fmt.Errorf("artifact %q: a tgz artifact needs a `files` entry naming the output path", artifactName)) } - l := b.Config.DefinitionLocation("artifacts." + artifactName) + l := b.Config.ExactLocation(structpath.NewPath(nil, "artifacts", artifactName)) dirPath := filepath.Dir(l.File) // Check if source paths are absolute, if not, make them absolute diff --git a/bundle/config/loader/process_include.go b/bundle/config/loader/process_include.go index 9fc6e13dfc5..c2dfa79b500 100644 --- a/bundle/config/loader/process_include.go +++ b/bundle/config/loader/process_include.go @@ -43,7 +43,7 @@ func validateSingleResourceDefined(configRoot structvar.View, ext, typ string) d // Gather all resources defined in the resources block. err := structvar.ForEach( configRoot, - structpath.MustParsePattern("resources.*.*"), + structpath.NewPattern(nil, "resources", structpath.AnyKey, structpath.AnyKey), func(np *structpath.PathNode, v structvar.View) error { // The key for the resource, e.g. "my_job" for jobs.my_job. k := np.KeyAt(2) @@ -60,7 +60,7 @@ func validateSingleResourceDefined(configRoot structvar.View, ext, typ string) d // Gather all resources defined in a target block. err = structvar.ForEach( configRoot, - structpath.MustParsePattern("targets.*.resources.*.*"), + structpath.NewPattern(nil, "targets", structpath.AnyKey, "resources", structpath.AnyKey, structpath.AnyKey), func(np *structpath.PathNode, v structvar.View) error { // The key for the resource, e.g. "my_job" for jobs.my_job. k := np.KeyAt(4) @@ -182,7 +182,7 @@ func (m *processInclude) load() (*config.Root, diag.Diagnostics) { Summary: "Include section is defined outside root file", Detail: `An include section is defined in a file that is not databricks.yml. Only includes defined in databricks.yml are applied.`, - Locations: this.GetLocations("include"), + Locations: this.NearestLocations(structpath.NewPath(nil, "include")), Paths: structpath.NewPathSlice("include"), }) } diff --git a/bundle/config/loader/process_root_includes.go b/bundle/config/loader/process_root_includes.go index 36166fa8512..f83c127449c 100644 --- a/bundle/config/loader/process_root_includes.go +++ b/bundle/config/loader/process_root_includes.go @@ -11,6 +11,7 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/logdiag" + "github.com/databricks/cli/libs/structs/structpath" ) type processRootIncludes struct{} @@ -112,7 +113,7 @@ func (m *processRootIncludes) Apply(ctx context.Context, b *bundle.Bundle) diag. Summary: "Files in the 'include' configuration section must be YAML or JSON files.", Detail: fmt.Sprintf("The file %s in the 'include' configuration section is not a YAML or JSON file, and only such files are supported. To include files to sync, specify them in the 'sync.include' configuration section instead.", rel), // The match's index within the glob is unrelated to the entry's position in the include list. - Locations: b.Config.GetLocations(fmt.Sprintf("include[%d]", entryIndex)), + Locations: b.Config.NearestLocations(structpath.NewPath(nil, "include", entryIndex)), }) continue } diff --git a/bundle/config/mutator/apply_source_linked_deployment_preset.go b/bundle/config/mutator/apply_source_linked_deployment_preset.go index 695edabc68a..698696b6e5d 100644 --- a/bundle/config/mutator/apply_source_linked_deployment_preset.go +++ b/bundle/config/mutator/apply_source_linked_deployment_preset.go @@ -39,7 +39,7 @@ func (m *applySourceLinkedDeploymentPreset) Apply(ctx context.Context, b *bundle Severity: diag.Warning, Summary: "source-linked deployment is available only in the Databricks Workspace", Paths: []*structpath.PathNode{path}, - Locations: b.Config.GetLocationsOf(path.SkipPrefix(2)), + Locations: b.Config.NearestLocations(path.SkipPrefix(2)), }, ) @@ -57,7 +57,7 @@ func (m *applySourceLinkedDeploymentPreset) Apply(ctx context.Context, b *bundle Severity: diag.Warning, Summary: "source-linked deployment in non-development mode is deprecated and will not be supported in a future release", Paths: []*structpath.PathNode{path}, - Locations: b.Config.GetLocationsOf(path.SkipPrefix(2)), + Locations: b.Config.NearestLocations(path.SkipPrefix(2)), }, ) } @@ -77,7 +77,7 @@ func (m *applySourceLinkedDeploymentPreset) Apply(ctx context.Context, b *bundle Summary: "workspace.file_path setting will be ignored in source-linked deployment mode", Detail: "In source-linked deployment files are not copied to the destination and resources use source files instead", Paths: []*structpath.PathNode{path}, - Locations: b.Config.GetLocationsOf(path), + Locations: b.Config.NearestLocations(path), }, ) } diff --git a/bundle/config/mutator/compute_id_compat.go b/bundle/config/mutator/compute_id_compat.go index 605d4db5a30..297769503b4 100644 --- a/bundle/config/mutator/compute_id_compat.go +++ b/bundle/config/mutator/compute_id_compat.go @@ -38,7 +38,7 @@ func (m *computeIdToClusterId) Apply(ctx context.Context, b *bundle.Bundle) diag diags = diags.Extend(rewriteComputeIdToClusterId(b, bundlePath, view.Get("bundle"))) // Check if the "compute_id" key is set in any target overrides. - err := structvar.ForEach(view, structpath.MustParsePattern("targets.*"), func(p *structpath.PathNode, v structvar.View) error { + err := structvar.ForEach(view, structpath.NewPattern(nil, "targets", structpath.AnyKey), func(p *structpath.PathNode, v structvar.View) error { diags = diags.Extend(rewriteComputeIdToClusterId(b, p, v)) return nil }) diff --git a/bundle/config/mutator/initialize_volume_paths.go b/bundle/config/mutator/initialize_volume_paths.go index fa971a92298..96733c09740 100644 --- a/bundle/config/mutator/initialize_volume_paths.go +++ b/bundle/config/mutator/initialize_volume_paths.go @@ -36,7 +36,7 @@ func (m *initializeVolumePaths) Apply(_ context.Context, b *bundle.Bundle) diag. } view := b.Config.View() - pattern := structpath.MustParsePattern("resources.volumes.*") + pattern := structpath.NewPattern(nil, "resources", "volumes", structpath.AnyKey) err := structvar.ForEach(view, pattern, func(p *structpath.PathNode, v structvar.View) error { // volume_path is computed and read-only; reject a user-provided value instead of overwriting it. if existing, ok := v.Get("volume_path").AsString(); ok && existing != "" { diff --git a/bundle/config/mutator/normalize_paths_test.go b/bundle/config/mutator/normalize_paths_test.go index faefb4b2a63..a3d1c95b3e0 100644 --- a/bundle/config/mutator/normalize_paths_test.go +++ b/bundle/config/mutator/normalize_paths_test.go @@ -45,7 +45,7 @@ func TestNormalizePaths(t *testing.T) { require.NoError(t, diags.Error()) require.Equal(t, "src/notebook.py", b.Config.Resources.Jobs["job1"].Tasks[0].NotebookTask.NotebookPath) - require.Equal(t, []diag.Location{location}, b.Config.LocationsAt(path)) + require.Equal(t, []diag.Location{location}, b.Config.ExactLocations(path)) } func TestNormalizePaths_jobRunOnFileChange(t *testing.T) { @@ -78,7 +78,7 @@ func TestNormalizePaths_jobRunOnFileChange(t *testing.T) { require.NoError(t, diags.Error()) require.Equal(t, "data/*.txt", *b.Config.Resources.JobRuns["run1"].Lifecycle.Triggers[0].OnFileChange) - require.Equal(t, []diag.Location{location}, b.Config.LocationsAt(path)) + require.Equal(t, []diag.Location{location}, b.Config.ExactLocations(path)) } func TestNormalizePath_absolutePath(t *testing.T) { diff --git a/bundle/config/mutator/paths/alert_paths_visitor.go b/bundle/config/mutator/paths/alert_paths_visitor.go index 91265736264..7c3d62b950e 100644 --- a/bundle/config/mutator/paths/alert_paths_visitor.go +++ b/bundle/config/mutator/paths/alert_paths_visitor.go @@ -6,7 +6,7 @@ import ( ) func VisitAlertPaths(root structvar.View, fn VisitFunc) error { - pattern := structpath.MustParsePattern("resources.alerts.*.file_path") + pattern := structpath.NewPattern(nil, "resources", "alerts", structpath.AnyKey, "file_path") return structvar.ForEach(root, pattern, func(path *structpath.PathNode, value structvar.View) error { return fn(path, TranslateModeLocalRelative, value) diff --git a/bundle/config/mutator/paths/app_paths_visitor.go b/bundle/config/mutator/paths/app_paths_visitor.go index 25040263f67..a54b5db71ca 100644 --- a/bundle/config/mutator/paths/app_paths_visitor.go +++ b/bundle/config/mutator/paths/app_paths_visitor.go @@ -6,7 +6,7 @@ import ( ) func VisitAppPaths(root structvar.View, fn VisitFunc) error { - pattern := structpath.MustParsePattern("resources.apps.*.source_code_path") + pattern := structpath.NewPattern(nil, "resources", "apps", structpath.AnyKey, "source_code_path") return structvar.ForEach(root, pattern, func(path *structpath.PathNode, value structvar.View) error { return fn(path, TranslateModeDirectory, value) diff --git a/bundle/config/mutator/paths/artifact_paths_visitor.go b/bundle/config/mutator/paths/artifact_paths_visitor.go index 4f934e42ba8..017d8d1835a 100644 --- a/bundle/config/mutator/paths/artifact_paths_visitor.go +++ b/bundle/config/mutator/paths/artifact_paths_visitor.go @@ -12,12 +12,12 @@ type artifactRewritePattern struct { func artifactRewritePatterns() []artifactRewritePattern { // Base pattern to match all artifacts. - base := "artifacts.*" + base := structpath.NewPattern(nil, "artifacts", structpath.AnyKey) // Compile list of configuration paths to rewrite. return []artifactRewritePattern{ { - pattern: structpath.MustParsePattern(base + ".path"), + pattern: structpath.NewPattern(base, "path"), mode: TranslateModeLocalAbsoluteDirectory, }, } diff --git a/bundle/config/mutator/paths/dashboard_paths_visitor.go b/bundle/config/mutator/paths/dashboard_paths_visitor.go index 40c9de427bd..741ac658cf8 100644 --- a/bundle/config/mutator/paths/dashboard_paths_visitor.go +++ b/bundle/config/mutator/paths/dashboard_paths_visitor.go @@ -6,7 +6,7 @@ import ( ) func VisitDashboardPaths(root structvar.View, fn VisitFunc) error { - pattern := structpath.MustParsePattern("resources.dashboards.*.file_path") + pattern := structpath.NewPattern(nil, "resources", "dashboards", structpath.AnyKey, "file_path") return structvar.ForEach(root, pattern, func(path *structpath.PathNode, value structvar.View) error { return fn(path, TranslateModeLocalRelative, value) diff --git a/bundle/config/mutator/paths/genie_space_paths_visitor.go b/bundle/config/mutator/paths/genie_space_paths_visitor.go index 8cc61bc09c4..b95c7eb95a4 100644 --- a/bundle/config/mutator/paths/genie_space_paths_visitor.go +++ b/bundle/config/mutator/paths/genie_space_paths_visitor.go @@ -6,7 +6,7 @@ import ( ) func VisitGenieSpacePaths(root structvar.View, fn VisitFunc) error { - pattern := structpath.MustParsePattern("resources.genie_spaces.*.file_path") + pattern := structpath.NewPattern(nil, "resources", "genie_spaces", structpath.AnyKey, "file_path") return structvar.ForEach(root, pattern, func(path *structpath.PathNode, value structvar.View) error { return fn(path, TranslateModeLocalRelative, value) diff --git a/bundle/config/mutator/paths/job_libraries_paths_visitor.go b/bundle/config/mutator/paths/job_libraries_paths_visitor.go index e96e136f814..c624797685e 100644 --- a/bundle/config/mutator/paths/job_libraries_paths_visitor.go +++ b/bundle/config/mutator/paths/job_libraries_paths_visitor.go @@ -6,15 +6,15 @@ import ( "github.com/databricks/cli/libs/structs/structvar" ) -func jobTaskLibrariesRewritePatterns(base string) []jobRewritePattern { +func jobTaskLibrariesRewritePatterns(base *structpath.PatternNode) []jobRewritePattern { return []jobRewritePattern{ { - structpath.MustParsePattern(base + ".libraries[*].whl"), + structpath.NewPattern(base, "libraries", structpath.AnyIndex, "whl"), TranslateModeLocalRelative, noSkipRewrite, }, { - structpath.MustParsePattern(base + ".libraries[*].jar"), + structpath.NewPattern(base, "libraries", structpath.AnyIndex, "jar"), TranslateModeLocalRelative, noSkipRewrite, }, @@ -23,12 +23,12 @@ func jobTaskLibrariesRewritePatterns(base string) []jobRewritePattern { func jobLibrariesRewritePatterns() []jobRewritePattern { // Base pattern to match all tasks in all jobs. - base := "resources.jobs.*.tasks[*]" + base := jobTasksPattern // Compile list of patterns and their respective rewrite functions. jobEnvironmentsPatterns := []jobRewritePattern{ { - structpath.MustParsePattern("resources.jobs.*.environments[*].spec.dependencies[*]"), + structpath.NewPattern(nil, "resources", "jobs", structpath.AnyKey, "environments", structpath.AnyIndex, "spec", "dependencies", structpath.AnyIndex), TranslateModeLocalRelativeWithPrefix, func(s string) bool { return !libraries.IsLibraryLocal(s) @@ -38,7 +38,7 @@ func jobLibrariesRewritePatterns() []jobRewritePattern { jobEnvironmentsWithRequirementsPatterns := []jobRewritePattern{ { - structpath.MustParsePattern("resources.jobs.*.environments[*].spec.dependencies[*]"), + structpath.NewPattern(nil, "resources", "jobs", structpath.AnyKey, "environments", structpath.AnyIndex, "spec", "dependencies", structpath.AnyIndex), TranslateModeEnvironmentPipFlag, func(s string) bool { _, _, ok := libraries.IsLocalPathInPipFlag(s) @@ -48,7 +48,7 @@ func jobLibrariesRewritePatterns() []jobRewritePattern { } taskPatterns := jobTaskLibrariesRewritePatterns(base) - forEachPatterns := jobTaskLibrariesRewritePatterns(base + ".for_each_task.task") + forEachPatterns := jobTaskLibrariesRewritePatterns(structpath.NewPattern(base, "for_each_task", "task")) allPatterns := append(taskPatterns, jobEnvironmentsPatterns...) allPatterns = append(allPatterns, jobEnvironmentsWithRequirementsPatterns...) allPatterns = append(allPatterns, forEachPatterns...) diff --git a/bundle/config/mutator/paths/job_paths_visitor.go b/bundle/config/mutator/paths/job_paths_visitor.go index 9b8c3ea5a94..0f26483c3b7 100644 --- a/bundle/config/mutator/paths/job_paths_visitor.go +++ b/bundle/config/mutator/paths/job_paths_visitor.go @@ -11,39 +11,42 @@ type jobRewritePattern struct { skipRewrite func(string) bool } +// jobTasksPattern matches all tasks in all jobs. +var jobTasksPattern = structpath.NewPattern(nil, "resources", "jobs", structpath.AnyKey, "tasks", structpath.AnyIndex) + func noSkipRewrite(string) bool { return false } -func jobTaskRewritePatterns(base string) []jobRewritePattern { +func jobTaskRewritePatterns(base *structpath.PatternNode) []jobRewritePattern { return []jobRewritePattern{ { - structpath.MustParsePattern(base + ".notebook_task.notebook_path"), + structpath.NewPattern(base, "notebook_task", "notebook_path"), TranslateModeNotebook, noSkipRewrite, }, { - structpath.MustParsePattern(base + ".spark_python_task.python_file"), + structpath.NewPattern(base, "spark_python_task", "python_file"), TranslateModeFile, noSkipRewrite, }, { - structpath.MustParsePattern(base + ".dbt_task.project_directory"), + structpath.NewPattern(base, "dbt_task", "project_directory"), TranslateModeDirectory, noSkipRewrite, }, { - structpath.MustParsePattern(base + ".sql_task.file.path"), + structpath.NewPattern(base, "sql_task", "file", "path"), TranslateModeFile, noSkipRewrite, }, { - structpath.MustParsePattern(base + ".alert_task.workspace_path"), + structpath.NewPattern(base, "alert_task", "workspace_path"), TranslateModeFile, noSkipRewrite, }, { - structpath.MustParsePattern(base + ".libraries[*].requirements"), + structpath.NewPattern(base, "libraries", structpath.AnyIndex, "requirements"), TranslateModeFile, noSkipRewrite, }, @@ -51,7 +54,7 @@ func jobTaskRewritePatterns(base string) []jobRewritePattern { // The AI Runtime task runs this bash script on each node; the backend // reads it as a workspace file, so translate the local path to its // remote (or immutable-snapshot) location like any other file. - structpath.MustParsePattern(base + ".ai_runtime_task.deployments[*].command_path"), + structpath.NewPattern(base, "ai_runtime_task", "deployments", structpath.AnyIndex, "command_path"), TranslateModeFile, noSkipRewrite, }, @@ -60,20 +63,20 @@ func jobTaskRewritePatterns(base string) []jobRewritePattern { func jobRewritePatterns() []jobRewritePattern { // Task-level base pattern for all jobs. - base := "resources.jobs.*.tasks[*]" + base := jobTasksPattern taskPatterns := jobTaskRewritePatterns(base) - forEachPatterns := jobTaskRewritePatterns(base + ".for_each_task.task") + forEachPatterns := jobTaskRewritePatterns(structpath.NewPattern(base, "for_each_task", "task")) patterns := append(taskPatterns, forEachPatterns...) return append(patterns, jobRewritePattern{ - structpath.MustParsePattern("resources.jobs.*.environment_variables[*].spec.files[*]"), + structpath.NewPattern(nil, "resources", "jobs", structpath.AnyKey, "environment_variables", structpath.AnyIndex, "spec", "files", structpath.AnyIndex), TranslateModeFile, noSkipRewrite, }, // Environment-level pattern for all jobs. jobRewritePattern{ - structpath.MustParsePattern("resources.jobs.*.environments[*].spec.project_environment"), + structpath.NewPattern(nil, "resources", "jobs", structpath.AnyKey, "environments", structpath.AnyIndex, "spec", "project_environment"), TranslateModeFile, noSkipRewrite, }, diff --git a/bundle/config/mutator/paths/job_run_paths_visitor.go b/bundle/config/mutator/paths/job_run_paths_visitor.go index 507a5343b44..0a0f6c86bb6 100644 --- a/bundle/config/mutator/paths/job_run_paths_visitor.go +++ b/bundle/config/mutator/paths/job_run_paths_visitor.go @@ -9,7 +9,7 @@ import ( // them relative to the bundle root. Not used by TranslatePaths: hashing still // needs a local glob, not a workspace path. func VisitJobRunPaths(root structvar.View, fn VisitFunc) error { - pattern := structpath.MustParsePattern("resources.job_runs.*.lifecycle.triggers[*].on_file_change") + pattern := structpath.NewPattern(nil, "resources", "job_runs", structpath.AnyKey, "lifecycle", "triggers", structpath.AnyIndex, "on_file_change") return structvar.ForEach(root, pattern, func(path *structpath.PathNode, value structvar.View) error { return fn(path, TranslateModeLocalRelative, value) diff --git a/bundle/config/mutator/paths/pipeline_paths_visitor.go b/bundle/config/mutator/paths/pipeline_paths_visitor.go index ff50c637db8..9570ba4adfb 100644 --- a/bundle/config/mutator/paths/pipeline_paths_visitor.go +++ b/bundle/config/mutator/paths/pipeline_paths_visitor.go @@ -16,28 +16,28 @@ type pipelineRewritePattern struct { } // Base pattern to match all libraries in all pipelines. -var base = "resources.pipelines.*" +var base = structpath.NewPattern(nil, "resources", "pipelines", structpath.AnyKey) func pipelineRewritePatterns() []pipelineRewritePattern { // Compile list of configuration paths to rewrite. allPatterns := []pipelineRewritePattern{ { - pattern: structpath.MustParsePattern(base + ".libraries[*].notebook.path"), + pattern: structpath.NewPattern(base, "libraries", structpath.AnyIndex, "notebook", "path"), mode: TranslateModeNotebook, skipRewrite: noSkipRewrite, }, { - pattern: structpath.MustParsePattern(base + ".libraries[*].file.path"), + pattern: structpath.NewPattern(base, "libraries", structpath.AnyIndex, "file", "path"), mode: TranslateModeFile, skipRewrite: noSkipRewrite, }, { - pattern: structpath.MustParsePattern(base + ".libraries[*].glob.include"), + pattern: structpath.NewPattern(base, "libraries", structpath.AnyIndex, "glob", "include"), mode: TranslateModeGlob, skipRewrite: noSkipRewrite, }, { - pattern: structpath.MustParsePattern(base + ".root_path"), + pattern: structpath.NewPattern(base, "root_path"), mode: TranslateModeDirectory, skipRewrite: noSkipRewrite, }, @@ -49,7 +49,7 @@ func pipelineRewritePatterns() []pipelineRewritePattern { func pipelineLibrariesRewritePatterns() []pipelineRewritePattern { pipelineEnvironmentsPatterns := []pipelineRewritePattern{ { - pattern: structpath.MustParsePattern("resources.pipelines.*.environment.dependencies[*]"), + pattern: structpath.NewPattern(nil, "resources", "pipelines", structpath.AnyKey, "environment", "dependencies", structpath.AnyIndex), mode: TranslateModeLocalRelativeWithPrefix, skipRewrite: func(s string) bool { return !libraries.IsLibraryLocal(s) @@ -59,7 +59,7 @@ func pipelineLibrariesRewritePatterns() []pipelineRewritePattern { pipelineEnvironmentsPatternsWithPipFlags := []pipelineRewritePattern{ { - structpath.MustParsePattern("resources.pipelines.*.environment.dependencies[*]"), + structpath.NewPattern(nil, "resources", "pipelines", structpath.AnyKey, "environment", "dependencies", structpath.AnyIndex), TranslateModeEnvironmentPipFlag, func(s string) bool { _, _, ok := libraries.IsLocalPathInPipFlag(s) diff --git a/bundle/config/mutator/populate_locations.go b/bundle/config/mutator/populate_locations.go index a2a217e0c68..315fc4e8b72 100644 --- a/bundle/config/mutator/populate_locations.go +++ b/bundle/config/mutator/populate_locations.go @@ -26,11 +26,11 @@ func (m *populateLocations) Name() string { // locationPatterns are the paths for which locations are recorded. var locationPatterns = []*structpath.PatternNode{ - structpath.MustParsePattern("*"), // Top level fields - structpath.MustParsePattern("resources.*"), // Resource groups ("resources.jobs") - structpath.MustParsePattern("resources.*.*"), // Resources for all types ("resources.jobs.my_job") - structpath.MustParsePattern("resources.jobs.*.tasks"), // Job tasks ("resources.jobs.my_job.tasks") - structpath.MustParsePattern("resources.jobs.*.tasks[*]"), // Job task items ("resources.jobs.my_job.tasks[2]") + structpath.NewPattern(nil, structpath.AnyKey), // Top level fields + structpath.NewPattern(nil, "resources", structpath.AnyKey), // Resource groups ("resources.jobs") + structpath.NewPattern(nil, "resources", structpath.AnyKey, structpath.AnyKey), // Resources for all types ("resources.jobs.my_job") + structpath.NewPattern(nil, "resources", "jobs", structpath.AnyKey, "tasks"), // Job tasks ("resources.jobs.my_job.tasks") + structpath.NewPattern(nil, "resources", "jobs", structpath.AnyKey, "tasks", structpath.AnyIndex), // Job task items ("resources.jobs.my_job.tasks[2]") } // pathString returns the path with keys separated by dots, whatever characters they contain. diff --git a/bundle/config/mutator/prepend_workspace_prefix.go b/bundle/config/mutator/prepend_workspace_prefix.go index 05892d50f86..0c923439dd2 100644 --- a/bundle/config/mutator/prepend_workspace_prefix.go +++ b/bundle/config/mutator/prepend_workspace_prefix.go @@ -29,11 +29,11 @@ var skipPrefixes = []string{ func (m *prependWorkspacePrefix) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { patterns := []*structpath.PatternNode{ - structpath.MustParsePattern("workspace.root_path"), - structpath.MustParsePattern("workspace.file_path"), - structpath.MustParsePattern("workspace.artifact_path"), - structpath.MustParsePattern("workspace.state_path"), - structpath.MustParsePattern("workspace.resource_path"), + structpath.NewPattern(nil, "workspace", "root_path"), + structpath.NewPattern(nil, "workspace", "file_path"), + structpath.NewPattern(nil, "workspace", "artifact_path"), + structpath.NewPattern(nil, "workspace", "state_path"), + structpath.NewPattern(nil, "workspace", "resource_path"), } for _, pattern := range patterns { diff --git a/bundle/config/mutator/prepend_workspace_prefix_test.go b/bundle/config/mutator/prepend_workspace_prefix_test.go index 6c439247051..e77047d8330 100644 --- a/bundle/config/mutator/prepend_workspace_prefix_test.go +++ b/bundle/config/mutator/prepend_workspace_prefix_test.go @@ -7,6 +7,7 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/internal/bundletest" "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/databricks-sdk-go/service/iam" "github.com/stretchr/testify/require" ) @@ -79,7 +80,7 @@ func TestPrependWorkspacePrefixPreservesLocations(t *testing.T) { diags := bundle.Apply(t.Context(), b, PrependWorkspacePrefix()) require.Empty(t, diags) require.Equal(t, "/Workspace/Users/test", b.Config.Workspace.RootPath) - require.Equal(t, locations, b.Config.GetLocations("workspace.root_path")) + require.Equal(t, locations, b.Config.NearestLocations(structpath.NewPath(nil, "workspace", "root_path"))) } func TestPrependWorkspaceForDefaultConfig(t *testing.T) { diff --git a/bundle/config/mutator/python/apply_python_output.go b/bundle/config/mutator/python/apply_python_output.go index a3b165a1384..7701aa789ba 100644 --- a/bundle/config/mutator/python/apply_python_output.go +++ b/bundle/config/mutator/python/apply_python_output.go @@ -34,7 +34,7 @@ func applyPythonOutput(root, output structvar.View) (*structvar.OverridePlan, ap } func createOverrideVisitor(leftRoot, rightRoot structvar.View) (applyPythonOutputResult, structvar.OverrideVisitor) { - resourcesPath := structpath.MustParsePath("resources") + resourcesPath := structpath.NewPath(nil, "resources") deleted := resourcemutator.NewResourceKeySet() updated := resourcemutator.NewResourceKeySet() added := resourcemutator.NewResourceKeySet() @@ -59,7 +59,7 @@ func createOverrideVisitor(leftRoot, rightRoot structvar.View) (applyPythonOutpu // rightRoot: {"bundle": ...} return deleted.AddPattern( - structpath.MustParsePattern("resources.*.*"), + structpath.NewPattern(nil, "resources", structpath.AnyKey, structpath.AnyKey), leftRoot, ) } else if np.Len() == 2 { @@ -70,7 +70,7 @@ func createOverrideVisitor(leftRoot, rightRoot structvar.View) (applyPythonOutpu // rightRoot: {"resources": { "jobs": ...}}}, return deleted.AddPattern( - structpath.NewPatternDotStar(structpath.NewPatternStringKey(structpath.MustParsePattern("resources"), np.KeyAt(1))), + structpath.NewPattern(nil, "resources", np.KeyAt(1), structpath.AnyKey), leftRoot, ) } else if np.Len() == 3 { @@ -102,7 +102,7 @@ func createOverrideVisitor(leftRoot, rightRoot structvar.View) (applyPythonOutpu // rightRoot: {"bundle": ..., "resources": {...} } return added.AddPattern( - structpath.MustParsePattern("resources.*.*"), + structpath.NewPattern(nil, "resources", structpath.AnyKey, structpath.AnyKey), rightRoot, ) } else if np.Len() == 2 { @@ -113,7 +113,7 @@ func createOverrideVisitor(leftRoot, rightRoot structvar.View) (applyPythonOutpu // rightRoot: {"resources": { "jobs": {...} }} return added.AddPattern( - structpath.NewPatternDotStar(structpath.NewPatternStringKey(structpath.MustParsePattern("resources"), np.KeyAt(1))), + structpath.NewPattern(nil, "resources", np.KeyAt(1), structpath.AnyKey), rightRoot, ) } else if np.Len() == 3 { @@ -149,7 +149,7 @@ func createOverrideVisitor(leftRoot, rightRoot structvar.View) (applyPythonOutpu // rightRoot: {"bundle": ..., "resources": {...} } return added.AddPattern( - structpath.MustParsePattern("resources.*.*"), + structpath.NewPattern(nil, "resources", structpath.AnyKey, structpath.AnyKey), rightRoot, ) } else if np.Len() == 2 { @@ -160,7 +160,7 @@ func createOverrideVisitor(leftRoot, rightRoot structvar.View) (applyPythonOutpu // rightRoot: {"resources": { "jobs": {...} }} return added.AddPattern( - structpath.NewPatternDotStar(structpath.NewPatternStringKey(structpath.MustParsePattern("resources"), np.KeyAt(1))), + structpath.NewPattern(nil, "resources", np.KeyAt(1), structpath.AnyKey), rightRoot, ) } else if np.Len() == 3 { diff --git a/bundle/config/mutator/python/python_locations_test.go b/bundle/config/mutator/python/python_locations_test.go index f8326c18578..bcc08d72852 100644 --- a/bundle/config/mutator/python/python_locations_test.go +++ b/bundle/config/mutator/python/python_locations_test.go @@ -130,7 +130,7 @@ func TestLoadOutput(t *testing.T) { ) assert.Equal(t, diag.Diagnostics{}, diags) - require.Equal(t, []diag.Location{location}, root.LocationsAt(structpath.MustParsePath("resources.jobs.my_job.name"))) + require.Equal(t, []diag.Location{location}, root.ExactLocations(structpath.MustParsePath("resources.jobs.my_job.name"))) } func TestParsePythonLocations_absolutePath(t *testing.T) { diff --git a/bundle/config/mutator/python/python_mutator.go b/bundle/config/mutator/python/python_mutator.go index f05fee37d7b..35ecd150026 100644 --- a/bundle/config/mutator/python/python_mutator.go +++ b/bundle/config/mutator/python/python_mutator.go @@ -168,7 +168,7 @@ func applyBackwardsCompatibilityFixes(b *bundle.Bundle) error { } // if 'experimental.python' section exists, nothing to do - experimentalPythonPath := structpath.MustParsePath("experimental.python") + experimentalPythonPath := structpath.NewPath(nil, "experimental", "python") if root.Lookup(experimentalPythonPath).IsValid() { return nil } diff --git a/bundle/config/mutator/reject_internal_resources.go b/bundle/config/mutator/reject_internal_resources.go index 0e4e28e53b5..de44c09298c 100644 --- a/bundle/config/mutator/reject_internal_resources.go +++ b/bundle/config/mutator/reject_internal_resources.go @@ -27,7 +27,7 @@ func (m *rejectInternalResources) Apply(ctx context.Context, b *bundle.Bundle) d Severity: diag.Error, Summary: "Internal resources cannot be set in bundle configuration", Paths: structpath.NewPathSlice("resources"), - Locations: b.Config.GetLocations("resources"), + Locations: b.Config.NearestLocations(structpath.NewPath(nil, "resources")), }) } diff --git a/bundle/config/mutator/resolve_job_run_file_triggers.go b/bundle/config/mutator/resolve_job_run_file_triggers.go index 35cffd0dcbe..8b952bae235 100644 --- a/bundle/config/mutator/resolve_job_run_file_triggers.go +++ b/bundle/config/mutator/resolve_job_run_file_triggers.go @@ -15,6 +15,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/structs/structpath" libsync "github.com/databricks/cli/libs/sync" ) @@ -65,7 +66,7 @@ func (*resolveJobRunFileTriggers) Apply(ctx context.Context, b *bundle.Bundle) d if t.OnFileChange == nil { continue } - path := fmt.Sprintf("resources.job_runs.%s.lifecycle.triggers[%d].on_file_change", name, i) + path := structpath.NewPath(nil, "resources", "job_runs", name, "lifecycle", "triggers", i, "on_file_change") pattern, fingerprint, d := resolveFileTrigger(b, path, *t.OnFileChange, syncable) diags = diags.Extend(d) if !d.HasError() { @@ -105,7 +106,7 @@ func listSyncableRelPaths(ctx context.Context, b *bundle.Bundle) ([]string, erro return out, nil } -func resolveFileTrigger(b *bundle.Bundle, loc, pattern string, syncable []string) (string, string, diag.Diagnostics) { +func resolveFileTrigger(b *bundle.Bundle, loc *structpath.PathNode, pattern string, syncable []string) (string, string, diag.Diagnostics) { relPattern, diags := validateFileTriggerPattern(b, loc, pattern) if diags.HasError() { return "", "", diags @@ -119,7 +120,7 @@ func resolveFileTrigger(b *bundle.Bundle, loc, pattern string, syncable []string diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, Summary: fileTriggerPrefix + fmt.Sprintf("invalid pattern %q: %s", pattern, err), - Locations: b.Config.GetLocations(loc), + Locations: b.Config.NearestLocations(loc), }) continue } @@ -131,7 +132,7 @@ func resolveFileTrigger(b *bundle.Bundle, loc, pattern string, syncable []string diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, Summary: fileTriggerPrefix + fmt.Sprintf("hash %q: %s", rel, err), - Locations: b.Config.GetLocations(loc), + Locations: b.Config.NearestLocations(loc), }) continue } @@ -145,20 +146,20 @@ func resolveFileTrigger(b *bundle.Bundle, loc, pattern string, syncable []string diags = diags.Append(diag.Diagnostic{ Severity: diag.Warning, Summary: fileTriggerPrefix + fmt.Sprintf("no synced files match %q", pattern), - Locations: b.Config.GetLocations(loc), + Locations: b.Config.NearestLocations(loc), }) } return relPattern, hex.EncodeToString(h.Sum(nil)), diags } -func validateFileTriggerPattern(b *bundle.Bundle, loc, pattern string) (string, diag.Diagnostics) { +func validateFileTriggerPattern(b *bundle.Bundle, loc *structpath.PathNode, pattern string) (string, diag.Diagnostics) { var diags diag.Diagnostics // A double star looks recursive but path.Match treats it as two ordinary stars. if strings.Contains(pattern, "**") { return "", diags.Append(diag.Diagnostic{ Severity: diag.Error, Summary: fileTriggerPrefix + fmt.Sprintf("** in %q is not supported; use * for a single directory level", pattern), - Locations: b.Config.GetLocations(loc), + Locations: b.Config.NearestLocations(loc), }) } // Reject a genuinely absolute path; Join would otherwise silently reinterpret it @@ -170,7 +171,7 @@ func validateFileTriggerPattern(b *bundle.Bundle, loc, pattern string) (string, return "", diags.Append(diag.Diagnostic{ Severity: diag.Error, Summary: fileTriggerPrefix + fmt.Sprintf("pattern %q must be relative to the defining YAML file", pattern), - Locations: b.Config.GetLocations(loc), + Locations: b.Config.NearestLocations(loc), }) } // NormalizePaths has already rewritten YAML-relative globs to be bundle-root @@ -182,7 +183,7 @@ func validateFileTriggerPattern(b *bundle.Bundle, loc, pattern string) (string, return "", diags.Append(diag.Diagnostic{ Severity: diag.Error, Summary: fileTriggerPrefix + fmt.Sprintf("pattern %q is not under the sync root", pattern), - Locations: b.Config.GetLocations(loc), + Locations: b.Config.NearestLocations(loc), }) } relPattern = filepath.ToSlash(relPattern) @@ -191,7 +192,7 @@ func validateFileTriggerPattern(b *bundle.Bundle, loc, pattern string) (string, return "", diags.Append(diag.Diagnostic{ Severity: diag.Error, Summary: fileTriggerPrefix + fmt.Sprintf("invalid pattern %q: %s", pattern, err), - Locations: b.Config.GetLocations(loc), + Locations: b.Config.NearestLocations(loc), }) } return relPattern, diags diff --git a/bundle/config/mutator/resolve_variable_references.go b/bundle/config/mutator/resolve_variable_references.go index f1e779ad832..123cc2a7b67 100644 --- a/bundle/config/mutator/resolve_variable_references.go +++ b/bundle/config/mutator/resolve_variable_references.go @@ -44,9 +44,10 @@ var defaultPrefixes = []string{ "variables", } -var artifactPath = structpath.MustParsePath("artifacts") +var artifactPath = structpath.NewPath(nil, "artifacts") type resolveVariableReferences struct { + // prefixes are top-level config keys. prefixes []string pattern *structpath.PatternNode lookupFn func(structvar.View, *structpath.PathNode, *bundle.Bundle) (structvar.View, error) @@ -75,7 +76,7 @@ func ResolveVariableReferencesOnlyResources(prefixes ...string) bundle.Mutator { prefixes: prefixes, lookupFn: lookup, extraRounds: maxResolutionRounds - 1, - pattern: structpath.MustParsePattern("resources"), + pattern: structpath.NewPattern(nil, "resources"), includeResources: true, } } @@ -94,7 +95,7 @@ func ResolveVariableReferencesWithoutResources(prefixes ...string) bundle.Mutato func ResolveVariableReferencesInLookup() bundle.Mutator { return &resolveVariableReferences{ prefixes: defaultPrefixes, - pattern: structpath.MustParsePattern("variables.*.lookup"), + pattern: structpath.NewPattern(nil, "variables", structpath.AnyKey, "lookup"), lookupFn: lookupForVariables, extraRounds: maxResolutionRounds - 1, } @@ -169,12 +170,12 @@ func (m *resolveVariableReferences) Validate(ctx context.Context, b *bundle.Bund func (m *resolveVariableReferences) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { prefixes := make([]*structpath.PathNode, len(m.prefixes)) for i, prefix := range m.prefixes { - prefixes[i] = structpath.MustParsePath(prefix) + prefixes[i] = structpath.NewPath(nil, prefix) } // The path ${var.foo} is a shorthand for ${variables.foo.value}. // We rewrite it here to make the resolution logic simpler. - varPath := structpath.MustParsePath("var") + varPath := structpath.NewPath(nil, "var") // Resolution converts the whole configuration on every round; skip it if no // reference would be resolved. diff --git a/bundle/config/mutator/resourcemutator/apply_bundle_permissions.go b/bundle/config/mutator/resourcemutator/apply_bundle_permissions.go index bfc9e373616..96a819c57f1 100644 --- a/bundle/config/mutator/resourcemutator/apply_bundle_permissions.go +++ b/bundle/config/mutator/resourcemutator/apply_bundle_permissions.go @@ -116,7 +116,7 @@ func (m *bundlePermissions) Apply(ctx context.Context, b *bundle.Bundle) diag.Di slices.Sort(keys) for _, key := range keys { - pattern := structpath.MustParsePattern("resources." + key + ".*") + pattern := structpath.NewPattern(nil, "resources", key, structpath.AnyKey) err = structvar.ForEach(b.Config.View(), pattern, func(p *structpath.PathNode, v structvar.View) error { had := v.Get("permissions").IsValid() diff --git a/bundle/config/mutator/resourcemutator/apply_default_task_source.go b/bundle/config/mutator/resourcemutator/apply_default_task_source.go index df0ca1fec68..ee190fd13d7 100644 --- a/bundle/config/mutator/resourcemutator/apply_default_task_source.go +++ b/bundle/config/mutator/resourcemutator/apply_default_task_source.go @@ -2,10 +2,10 @@ package resourcemutator import ( "context" - "fmt" "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/databricks-sdk-go/service/jobs" ) @@ -50,9 +50,9 @@ func (a *applyDefaultTaskSource) Apply(ctx context.Context, b *bundle.Bundle) di for i := range job.Tasks { task := &job.Tasks[i] - taskPath := fmt.Sprintf("resources.jobs.%s.tasks[%d]", name, i) + taskPath := structpath.NewPath(nil, "resources", "jobs", name, "tasks", i) if task.ForEachTask != nil { - setGitTaskSource(b, taskPath+".for_each_task.task", &task.ForEachTask.Task) + setGitTaskSource(b, structpath.NewPath(taskPath, "for_each_task", "task"), &task.ForEachTask.Task) } setGitTaskSource(b, taskPath, task) } @@ -64,32 +64,32 @@ func (a *applyDefaultTaskSource) Apply(ctx context.Context, b *bundle.Bundle) di // supports the field, unless the user already set it. // The task types are the ones that support the `source` field: // https://docs.databricks.com/api/workspace/jobs/create -func setGitTaskSource(b *bundle.Bundle, taskPath string, task *jobs.Task) { +func setGitTaskSource(b *bundle.Bundle, taskPath *structpath.PathNode, task *jobs.Task) { type taskSource struct { - key string + path []any source *jobs.Source } var sources []taskSource if task.DbtTask != nil { - sources = append(sources, taskSource{"dbt_task", &task.DbtTask.Source}) + sources = append(sources, taskSource{[]any{"dbt_task", "source"}, &task.DbtTask.Source}) } if task.GenAiComputeTask != nil { - sources = append(sources, taskSource{"gen_ai_compute_task", &task.GenAiComputeTask.Source}) + sources = append(sources, taskSource{[]any{"gen_ai_compute_task", "source"}, &task.GenAiComputeTask.Source}) } if task.NotebookTask != nil { - sources = append(sources, taskSource{"notebook_task", &task.NotebookTask.Source}) + sources = append(sources, taskSource{[]any{"notebook_task", "source"}, &task.NotebookTask.Source}) } if task.SparkPythonTask != nil { - sources = append(sources, taskSource{"spark_python_task", &task.SparkPythonTask.Source}) + sources = append(sources, taskSource{[]any{"spark_python_task", "source"}, &task.SparkPythonTask.Source}) } if task.SqlTask != nil && task.SqlTask.File != nil { - sources = append(sources, taskSource{"sql_task.file", &task.SqlTask.File.Source}) + sources = append(sources, taskSource{[]any{"sql_task", "file", "source"}, &task.SqlTask.File.Source}) } for _, s := range sources { // An explicit `source: ""` is set by the user, although the typed value is // empty; not every task type has ForceSendFields to record it, but it is // defined in the configuration. - if *s.source != "" || b.Config.DefinitionLocation(taskPath+"."+s.key+".source").File != "" { + if *s.source != "" || b.Config.ExactLocation(structpath.NewPath(taskPath, s.path...)).File != "" { continue } *s.source = jobs.SourceGit diff --git a/bundle/config/mutator/resourcemutator/apply_presets.go b/bundle/config/mutator/resourcemutator/apply_presets.go index bb84240d7cc..9bac80ee5ed 100644 --- a/bundle/config/mutator/resourcemutator/apply_presets.go +++ b/bundle/config/mutator/resourcemutator/apply_presets.go @@ -11,6 +11,7 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/metrics" "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/cli/libs/textutil" "github.com/databricks/databricks-sdk-go/service/catalog" "github.com/databricks/databricks-sdk-go/service/jobs" @@ -352,7 +353,7 @@ func validatePauseStatus(b *bundle.Bundle) diag.Diagnostics { return diag.Diagnostics{{ Summary: "Invalid value for trigger_pause_status, should be PAUSED or UNPAUSED", Severity: diag.Error, - Locations: []diag.Location{b.Config.GetLocation("presets.trigger_pause_status")}, + Locations: []diag.Location{b.Config.NearestLocation(structpath.NewPath(nil, "presets", "trigger_pause_status"))}, }} } diff --git a/bundle/config/mutator/resourcemutator/configure_cluster_policy_definition.go b/bundle/config/mutator/resourcemutator/configure_cluster_policy_definition.go index 84dddb367ba..08252135556 100644 --- a/bundle/config/mutator/resourcemutator/configure_cluster_policy_definition.go +++ b/bundle/config/mutator/resourcemutator/configure_cluster_policy_definition.go @@ -32,7 +32,7 @@ func (c configureClusterPolicyDefinition) Apply(_ context.Context, b *bundle.Bun return nil } - pattern := structpath.MustParsePattern("resources.cluster_policies.*") + pattern := structpath.NewPattern(nil, "resources", "cluster_policies", structpath.AnyKey) err := structvar.ForEach(b.Config.View(), pattern, func(p *structpath.PathNode, v structvar.View) error { for _, field := range jsonPolicyFields { diff --git a/bundle/config/mutator/resourcemutator/configure_dashboards_serialized_dashboard.go b/bundle/config/mutator/resourcemutator/configure_dashboards_serialized_dashboard.go index 5251c92716d..a48f93c7715 100644 --- a/bundle/config/mutator/resourcemutator/configure_dashboards_serialized_dashboard.go +++ b/bundle/config/mutator/resourcemutator/configure_dashboards_serialized_dashboard.go @@ -34,7 +34,7 @@ func (c configureDashboardSerializedDashboard) Apply(_ context.Context, b *bundl return nil } - pattern := structpath.MustParsePattern("resources.dashboards.*") + pattern := structpath.NewPattern(nil, "resources", "dashboards", structpath.AnyKey) // Configure serialized_dashboard field for all dashboards. err := structvar.ForEach(b.Config.View(), pattern, func(p *structpath.PathNode, v structvar.View) error { diff --git a/bundle/config/mutator/resourcemutator/configure_genie_space_serialized_space.go b/bundle/config/mutator/resourcemutator/configure_genie_space_serialized_space.go index 0eecb3b2bb9..91078058e53 100644 --- a/bundle/config/mutator/resourcemutator/configure_genie_space_serialized_space.go +++ b/bundle/config/mutator/resourcemutator/configure_genie_space_serialized_space.go @@ -31,7 +31,7 @@ func (c configureGenieSpaceSerializedSpace) Apply(_ context.Context, b *bundle.B return nil } - pattern := structpath.MustParsePattern("resources.genie_spaces.*") + pattern := structpath.NewPattern(nil, "resources", "genie_spaces", structpath.AnyKey) err := structvar.ForEach(b.Config.View(), pattern, func(p *structpath.PathNode, v structvar.View) error { filePath, hasFilePath := v.Get(filePathFieldName).AsString() diff --git a/bundle/config/mutator/resourcemutator/expand_pipeline_glob_paths.go b/bundle/config/mutator/resourcemutator/expand_pipeline_glob_paths.go index 1f9f2427851..77aaa702610 100644 --- a/bundle/config/mutator/resourcemutator/expand_pipeline_glob_paths.go +++ b/bundle/config/mutator/resourcemutator/expand_pipeline_glob_paths.go @@ -76,7 +76,7 @@ func (m *expandPipelineGlobPaths) expandLibrary(ctx context.Context, dir string, } func (m *expandPipelineGlobPaths) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - p := structpath.MustParsePattern("resources.pipelines.*.libraries") + p := structpath.NewPattern(nil, "resources", "pipelines", structpath.AnyKey, "libraries") // Visit each pipeline's "libraries" field and expand any glob patterns. err := structvar.ForEach(b.Config.View(), p, func(path *structpath.PathNode, value structvar.View) error { diff --git a/bundle/config/mutator/resourcemutator/fix_permissions.go b/bundle/config/mutator/resourcemutator/fix_permissions.go index 6f631fe6dd9..fcc49f72f64 100644 --- a/bundle/config/mutator/resourcemutator/fix_permissions.go +++ b/bundle/config/mutator/resourcemutator/fix_permissions.go @@ -192,7 +192,7 @@ func (m *fixPermissions) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagn } currentUser := b.Config.Workspace.CurrentUser.UserName - err := structvar.ForEach(b.Config.View(), structpath.MustParsePattern("resources.*.*.permissions"), func(p *structpath.PathNode, v structvar.View) error { + err := structvar.ForEach(b.Config.View(), structpath.NewPattern(nil, "resources", structpath.AnyKey, structpath.AnyKey, "permissions"), func(p *structpath.PathNode, v structvar.View) error { // Extract resource type from path: resources...permissions resourceType := p.KeyAt(1) if ignoredResources[resourceType] || v.Kind() != structvar.KindSequence { diff --git a/bundle/config/mutator/resourcemutator/model_serving_endpoint_fixups.go b/bundle/config/mutator/resourcemutator/model_serving_endpoint_fixups.go index 6c25146e6bd..17e9bdc2ea0 100644 --- a/bundle/config/mutator/resourcemutator/model_serving_endpoint_fixups.go +++ b/bundle/config/mutator/resourcemutator/model_serving_endpoint_fixups.go @@ -5,6 +5,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/cli/libs/utils" "github.com/databricks/databricks-sdk-go/service/serving" ) @@ -56,7 +57,7 @@ func (m *modelServingEndpointFixups) Apply(ctx context.Context, b *bundle.Bundle Summary: "Cannot use both served_models and served_entities", Detail: "Model serving endpoint cannot specify both served_models and served_entities at the same time.", Locations: []diag.Location{ - b.Config.GetLocation("resources.model_serving_endpoints." + key), + b.Config.NearestLocation(structpath.NewPath(nil, "resources", "model_serving_endpoints", key)), }, }) continue @@ -71,7 +72,7 @@ func (m *modelServingEndpointFixups) Apply(ctx context.Context, b *bundle.Bundle Summary: "Using served_models is deprecated", Detail: "The served_models field is deprecated. Please use served_entities instead.", Locations: []diag.Location{ - b.Config.GetLocation("resources.model_serving_endpoints." + key + ".config.served_models"), + b.Config.NearestLocation(structpath.NewPath(nil, "resources", "model_serving_endpoints", key, "config", "served_models")), }, }) diff --git a/bundle/config/mutator/resourcemutator/run_as.go b/bundle/config/mutator/resourcemutator/run_as.go index 21751e32813..9a3f4b4544c 100644 --- a/bundle/config/mutator/resourcemutator/run_as.go +++ b/bundle/config/mutator/resourcemutator/run_as.go @@ -56,7 +56,7 @@ func validateRunAs(b *bundle.Bundle) diag.Diagnostics { if identityCount != 1 { return diag.Diagnostics{{ Summary: "run_as section must specify exactly one non-empty identity: user_name, service_principal_name, or group_name", - Locations: []diag.Location{b.Config.GetLocation("run_as")}, + Locations: []diag.Location{b.Config.NearestLocation(structpath.NewPath(nil, "run_as"))}, Severity: diag.Error, }} } @@ -84,7 +84,7 @@ func validateRunAs(b *bundle.Bundle) diag.Diagnostics { if len(b.Config.Resources.ModelServingEndpoints) > 0 { diags = diags.Extend(reportRunAsNotSupported( "model_serving_endpoints", - b.Config.GetLocation("resources.model_serving_endpoints"), + b.Config.NearestLocation(structpath.NewPath(nil, "resources", "model_serving_endpoints")), b.Config.Workspace.CurrentUser.UserName, identity, )) @@ -94,7 +94,7 @@ func validateRunAs(b *bundle.Bundle) diag.Diagnostics { if len(b.Config.Resources.QualityMonitors) > 0 { diags = diags.Extend(reportRunAsNotSupported( "quality_monitors", - b.Config.GetLocation("resources.quality_monitors"), + b.Config.NearestLocation(structpath.NewPath(nil, "resources", "quality_monitors")), b.Config.Workspace.CurrentUser.UserName, identity, )) @@ -108,7 +108,7 @@ func validateRunAs(b *bundle.Bundle) diag.Diagnostics { } diags = diags.Extend(reportRunAsNotSupported( "dashboards with embed_credentials set to true", - b.Config.GetLocationOf(structpath.NewPath(nil, "resources", "dashboards", key)), + b.Config.NearestLocation(structpath.NewPath(nil, "resources", "dashboards", key)), b.Config.Workspace.CurrentUser.UserName, identity, )) @@ -119,7 +119,7 @@ func validateRunAs(b *bundle.Bundle) diag.Diagnostics { if len(b.Config.Resources.Apps) > 0 { diags = diags.Extend(reportRunAsNotSupported( "apps", - b.Config.GetLocation("resources.apps"), + b.Config.NearestLocation(structpath.NewPath(nil, "resources", "apps")), b.Config.Workspace.CurrentUser.UserName, identity, )) @@ -239,7 +239,7 @@ func (m *setRunAs) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { Severity: diag.Warning, Summary: "You are using the legacy mode of run_as. The support for this mode is experimental and might be removed in a future release of the CLI. In order to run the pipelines in your DABs project as the run_as user this mode changes the owners of the pipelines to the run_as identity, which requires the user deploying the bundle to be a workspace admin, and also a Metastore admin if the pipeline target is in UC.", Paths: structpath.NewPathSlice("experimental", "use_legacy_run_as"), - Locations: b.Config.GetLocations("experimental.use_legacy_run_as"), + Locations: b.Config.NearestLocations(structpath.NewPath(nil, "experimental", "use_legacy_run_as")), }, } } diff --git a/bundle/config/mutator/resourcemutator/run_as_test.go b/bundle/config/mutator/resourcemutator/run_as_test.go index 1be8d37957c..cfb98cbb84c 100644 --- a/bundle/config/mutator/resourcemutator/run_as_test.go +++ b/bundle/config/mutator/resourcemutator/run_as_test.go @@ -11,6 +11,7 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/databricks-sdk-go/service/iam" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/databricks/databricks-sdk-go/service/pipelines" @@ -383,7 +384,7 @@ func TestRunAsIdentities(t *testing.T) { diags = bundle.Apply(t.Context(), b, SetRunAs()) if tc.wantError { require.ErrorContains(t, diags.Error(), "run_as section must specify exactly one non-empty identity: user_name, service_principal_name, or group_name") - assert.Equal(t, []diag.Location{r.GetLocation("run_as")}, diags[0].Locations) + assert.Equal(t, []diag.Location{r.NearestLocation(structpath.NewPath(nil, "run_as"))}, diags[0].Locations) } else { require.NoError(t, diags.Error()) } diff --git a/bundle/config/mutator/resourcemutator/secret_scope_fixups.go b/bundle/config/mutator/resourcemutator/secret_scope_fixups.go index 8404d2444eb..0bec7b469fc 100644 --- a/bundle/config/mutator/resourcemutator/secret_scope_fixups.go +++ b/bundle/config/mutator/resourcemutator/secret_scope_fixups.go @@ -139,7 +139,7 @@ func (m *secretScopeFixups) Apply(ctx context.Context, b *bundle.Bundle) diag.Di Summary: "Failed to collapse permissions for secret scope", Detail: err.Error(), Paths: structpath.NewPathSlice("resources", "secret_scopes", key), - Locations: []diag.Location{b.Config.GetLocationOf(structpath.NewPath(nil, "resources", "secret_scopes", key))}, + Locations: []diag.Location{b.Config.NearestLocation(structpath.NewPath(nil, "resources", "secret_scopes", key))}, }, } } diff --git a/bundle/config/mutator/resourcemutator/validate_target_mode.go b/bundle/config/mutator/resourcemutator/validate_target_mode.go index bf6386264ad..812d4df66fd 100644 --- a/bundle/config/mutator/resourcemutator/validate_target_mode.go +++ b/bundle/config/mutator/resourcemutator/validate_target_mode.go @@ -6,6 +6,7 @@ import ( "strings" "github.com/databricks/cli/libs/iamutil" + "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" @@ -55,7 +56,7 @@ func validateDevelopmentMode(b *bundle.Bundle) diag.Diagnostics { diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, Summary: "target with 'mode: development' cannot set trigger pause status to UNPAUSED by default", - Locations: []diag.Location{b.Config.GetLocation("presets.trigger_pause_status")}, + Locations: []diag.Location{b.Config.NearestLocation(structpath.NewPath(nil, "presets", "trigger_pause_status"))}, }) } @@ -77,7 +78,7 @@ func validateDevelopmentMode(b *bundle.Bundle) diag.Diagnostics { diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, Summary: "prefix should contain the current username or ${workspace.current_user.short_name} to ensure uniqueness when using 'mode: development'", - Locations: []diag.Location{b.Config.GetLocation("presets.name_prefix")}, + Locations: []diag.Location{b.Config.NearestLocation(structpath.NewPath(nil, "presets", "name_prefix"))}, }) } return diags diff --git a/bundle/config/mutator/rewrite_sync_paths.go b/bundle/config/mutator/rewrite_sync_paths.go index c4eefebae2e..91a3451e4b6 100644 --- a/bundle/config/mutator/rewrite_sync_paths.go +++ b/bundle/config/mutator/rewrite_sync_paths.go @@ -48,7 +48,7 @@ func (m *rewriteSyncPaths) makeRelativeTo(root string, v structvar.View) (string func (m *rewriteSyncPaths) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { rewrite := func(field string, toSlash bool) error { - pattern := structpath.MustParsePattern("sync." + field + "[*]") + pattern := structpath.NewPattern(nil, "sync", field, structpath.AnyIndex) return structvar.ForEach(b.Config.View(), pattern, func(p *structpath.PathNode, v structvar.View) error { path, err := m.makeRelativeTo(b.BundleRootPath, v) if err != nil { diff --git a/bundle/config/mutator/sync_infer_root.go b/bundle/config/mutator/sync_infer_root.go index bd1a529380a..d8d4a3f6266 100644 --- a/bundle/config/mutator/sync_infer_root.go +++ b/bundle/config/mutator/sync_infer_root.go @@ -94,7 +94,7 @@ func (m *syncInferRoot) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagno diags = append(diags, diag.Diagnostic{ Severity: diag.Error, Summary: fmt.Sprintf("invalid sync path %q", path), - Locations: b.Config.GetLocationsOf(syncPath), + Locations: b.Config.NearestLocations(syncPath), Paths: []*structpath.PathNode{syncPath}, }) } diff --git a/bundle/config/mutator/translate_paths.go b/bundle/config/mutator/translate_paths.go index 21c8f212cf6..e6359fbc925 100644 --- a/bundle/config/mutator/translate_paths.go +++ b/bundle/config/mutator/translate_paths.go @@ -341,7 +341,7 @@ func applyTranslations(ctx context.Context, b *bundle.Bundle, t *translateContex // Reject an explicit workspace.file_path: immutable bundles set it // automatically to the content-addressed snapshot location. A user-supplied // value would be silently discarded during path translation, so we error early. - if loc := b.Config.DefinitionLocation("workspace.file_path"); loc.File != "" { + if loc := b.Config.ExactLocation(structpath.NewPath(nil, "workspace", "file_path")); loc.File != "" { return diag.Diagnostics{{ Severity: diag.Error, Summary: "workspace.file_path cannot be configured when experimental.immutable_folder is enabled", diff --git a/bundle/config/mutator/validate_job_run_triggers.go b/bundle/config/mutator/validate_job_run_triggers.go index d33a7263bde..7d09a50fc6d 100644 --- a/bundle/config/mutator/validate_job_run_triggers.go +++ b/bundle/config/mutator/validate_job_run_triggers.go @@ -2,11 +2,11 @@ package mutator import ( "context" - "fmt" "strings" "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/structs/structpath" ) type validateJobRunTriggers struct{} @@ -27,12 +27,12 @@ func (*validateJobRunTriggers) Apply(_ context.Context, b *bundle.Bundle) diag.D continue } for i, t := range jr.Lifecycle.Triggers { - path := fmt.Sprintf("resources.job_runs.%s.lifecycle.triggers[%d]", name, i) + path := structpath.NewPath(nil, "resources", "job_runs", name, "lifecycle", "triggers", i) if t.OnBundleDeploy == nil && t.OnFileChange == nil { diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, Summary: "lifecycle.triggers entry must set on_bundle_deploy or on_file_change", - Locations: b.Config.GetLocations(path), + Locations: b.Config.NearestLocations(path), }) continue } @@ -40,7 +40,7 @@ func (*validateJobRunTriggers) Apply(_ context.Context, b *bundle.Bundle) diag.D diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, Summary: "lifecycle.triggers entry must set only one of on_bundle_deploy or on_file_change", - Locations: b.Config.GetLocations(path), + Locations: b.Config.NearestLocations(path), }) continue } @@ -48,20 +48,21 @@ func (*validateJobRunTriggers) Apply(_ context.Context, b *bundle.Bundle) diag.D diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, Summary: "lifecycle.triggers.on_bundle_deploy must be true when set", - Locations: b.Config.GetLocations(path + ".on_bundle_deploy"), + Locations: b.Config.NearestLocations(structpath.NewPath(path, "on_bundle_deploy")), }) } if t.OnFileChange != nil { + onFileChange := structpath.NewPath(path, "on_file_change") if strings.TrimSpace(*t.OnFileChange) == "" { diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, Summary: "lifecycle.triggers.on_file_change must be non-empty when set", - Locations: b.Config.GetLocations(path + ".on_file_change"), + Locations: b.Config.NearestLocations(onFileChange), }) continue } // Report bad patterns at validate time; hashing only runs on deploy. - _, patternDiags := validateFileTriggerPattern(b, path+".on_file_change", *t.OnFileChange) + _, patternDiags := validateFileTriggerPattern(b, onFileChange, *t.OnFileChange) diags = diags.Extend(patternDiags) } } diff --git a/bundle/config/mutator/validate_secret_value_is_variable.go b/bundle/config/mutator/validate_secret_value_is_variable.go index 467f8a6d2ae..0b14e37f4c4 100644 --- a/bundle/config/mutator/validate_secret_value_is_variable.go +++ b/bundle/config/mutator/validate_secret_value_is_variable.go @@ -90,7 +90,7 @@ func (v *validateSecretValueIsVariable) checkVariableDefault(b *bundle.Bundle, s // The default path in the dynamic config for the variable's default field. defaultPath := structpath.NewPath(nil, "variables", varName, "default") - locations := append(b.Config.LocationsAt(defaultPath), val.Locations()...) + locations := append(b.Config.ExactLocations(defaultPath), val.Locations()...) return diag.Diagnostics{{ Severity: diag.Error, diff --git a/bundle/config/root.go b/bundle/config/root.go index 0ed17499e08..c2c557dfe83 100644 --- a/bundle/config/root.go +++ b/bundle/config/root.go @@ -203,7 +203,7 @@ func (r *Root) Merge(others ...*Root) error { return nil } -var bundleGitPath = structpath.MustParsePath("bundle.git") +var bundleGitPath = structpath.NewPath(nil, "bundle", "git") func (r *Root) MergeTargetOverrides(name string) error { targetPath := structpath.NewPath(nil, "targets", name) @@ -281,14 +281,14 @@ func (r *Root) MergeTargetOverrides(name string) error { // Merge `mode`. This field must be overwritten if set, not merged. if v := target.Get("mode"); v.IsValid() { - if err := r.Assign(structpath.MustParsePath("bundle.mode"), v); err != nil { + if err := r.Assign(structpath.NewPath(nil, "bundle", "mode"), v); err != nil { return err } } // Merge `cluster_id`. This field must be overwritten if set, not merged. if v := target.Get("cluster_id"); v.IsValid() { - if err := r.Assign(structpath.MustParsePath("bundle.cluster_id"), v); err != nil { + if err := r.Assign(structpath.NewPath(nil, "bundle", "cluster_id"), v); err != nil { return err } } @@ -551,57 +551,31 @@ func (r *Root) UpdateSequence(path *structpath.PathNode, sources [][]int) { r.store(sv) } -// LocationsAt returns all locations of the configuration value at the specified path. -func (r Root) LocationsAt(path *structpath.PathNode) []diag.Location { - return r.locations.At(path) -} - -// Best effort to get the location of configuration value at the specified path. -// This function is useful to annotate error messages with the location, because -// we don't want to fail with a different error message if we cannot retrieve the location. -func (r Root) GetLocation(path string) diag.Location { - locs := r.GetLocations(path) - if len(locs) == 0 { - return diag.Location{} - } - return locs[0] +// NearestLocations returns the locations of the configuration value at path or, if it +// has none (e.g. it was set by a mutator), those of its closest ancestor that has some. +// Use it to point diagnostics at the most specific place there is; use +// [Root.ExactLocations] to find where a value is written. +func (r Root) NearestLocations(path *structpath.PathNode) []diag.Location { + return r.locations.Nearest(path) } -// Get all locations of the configuration value at the specified path. We need both -// this function and it's singular version (GetLocation) because some diagnostics just need -// the primary location and some need all locations associated with a configuration value. -// A value without locations (e.g. set by a mutator) gets those of its closest ancestor -// that has some; use [Root.DefinitionLocation] to find where a value is defined. -func (r Root) GetLocations(path string) []diag.Location { - p, err := structpath.ParsePath(path) - if err != nil { - return nil - } - return r.GetLocationsOf(p) +// NearestLocation returns the primary location of [Root.NearestLocations]. +func (r Root) NearestLocation(path *structpath.PathNode) diag.Location { + return firstLocation(r.NearestLocations(path)) } -// GetLocationsOf is [Root.GetLocations] for a path node. -func (r Root) GetLocationsOf(path *structpath.PathNode) []diag.Location { - return r.locations.Nearest(path) +// ExactLocations returns the locations the configuration value at path is written at, +// or none if it was set by a mutator. +func (r Root) ExactLocations(path *structpath.PathNode) []diag.Location { + return r.locations.Exact(path) } -// GetLocationOf is [Root.GetLocation] for a path node. -func (r Root) GetLocationOf(path *structpath.PathNode) diag.Location { - locs := r.GetLocationsOf(path) - if len(locs) == 0 { - return diag.Location{} - } - return locs[0] +// ExactLocation returns the primary location of [Root.ExactLocations]. +func (r Root) ExactLocation(path *structpath.PathNode) diag.Location { + return firstLocation(r.ExactLocations(path)) } -// DefinitionLocation returns the primary location the value at path is defined at, -// or an empty location if it has none (e.g. it was set by a mutator). -func (r Root) DefinitionLocation(path string) diag.Location { - p, err := structpath.ParsePath(path) - if err != nil { - return diag.Location{} - } - locs := r.locations.At(p) +func firstLocation(locs []diag.Location) diag.Location { if len(locs) == 0 { return diag.Location{} } diff --git a/bundle/config/validate/all_resources_have_values.go b/bundle/config/validate/all_resources_have_values.go index 9e3aecb9743..434fdba513a 100644 --- a/bundle/config/validate/all_resources_have_values.go +++ b/bundle/config/validate/all_resources_have_values.go @@ -41,7 +41,7 @@ func (m *allResourcesHaveValues) Apply(ctx context.Context, b *bundle.Bundle) di diags = append(diags, diag.Diagnostic{ Severity: diag.Error, Summary: fmt.Sprintf("%s %s is not defined", rType, rName), - Locations: b.Config.LocationsAt(p), + Locations: b.Config.ExactLocations(p), Paths: []*structpath.PathNode{p}, }) } diff --git a/bundle/config/validate/files_to_sync.go b/bundle/config/validate/files_to_sync.go index 14b032e8da1..b719d46282c 100644 --- a/bundle/config/validate/files_to_sync.go +++ b/bundle/config/validate/files_to_sync.go @@ -62,7 +62,7 @@ func (v *filesToSync) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnost Summary: "There are no files to sync, please check your .gitignore and sync.exclude configuration", // Show all locations where sync.exclude is defined, since merging // sync.exclude is additive. - Locations: b.Config.GetLocationsOf(path), + Locations: b.Config.NearestLocations(path), Paths: []*structpath.PathNode{path}, }) } diff --git a/bundle/config/validate/job_cluster_key_defined.go b/bundle/config/validate/job_cluster_key_defined.go index eb9bea4df70..dbed1bb9a43 100644 --- a/bundle/config/validate/job_cluster_key_defined.go +++ b/bundle/config/validate/job_cluster_key_defined.go @@ -60,7 +60,7 @@ func checkJobClusterKey(b *bundle.Bundle, jobClusterKeys map[string]bool, jobClu // Show only the location where the job_cluster_key is defined. // Other associated locations are not relevant since they are // overridden during merging. - Locations: b.Config.GetLocationsOf(path), + Locations: b.Config.NearestLocations(path), Paths: []*structpath.PathNode{path}, }} } diff --git a/bundle/config/validate/job_task_cluster_spec.go b/bundle/config/validate/job_task_cluster_spec.go index 71bce7173d5..b4ed9fecb53 100644 --- a/bundle/config/validate/job_task_cluster_spec.go +++ b/bundle/config/validate/job_task_cluster_spec.go @@ -92,7 +92,7 @@ func validateJobTask(b *bundle.Bundle, task jobs.Task, taskPath *structpath.Path Severity: diag.Error, Summary: "Missing required cluster or environment settings", Detail: detail, - Locations: b.Config.GetLocationsOf(taskPath), + Locations: b.Config.NearestLocations(taskPath), Paths: []*structpath.PathNode{taskPath}, }) } diff --git a/bundle/config/validate/no_interpolation_in_bundle_name.go b/bundle/config/validate/no_interpolation_in_bundle_name.go index 0850f27ac15..c4e2aa3f8f9 100644 --- a/bundle/config/validate/no_interpolation_in_bundle_name.go +++ b/bundle/config/validate/no_interpolation_in_bundle_name.go @@ -28,7 +28,7 @@ func (m *noInterpolationInBundleName) Apply(ctx context.Context, b *bundle.Bundl is a part of the path at which your bundle state is stored at by default. Parameterizing it at runtime can have unexpected consequences like duplicate deployments or resources not being cleaned up during bundle destroy.`, - Locations: b.Config.GetLocations("bundle.name"), + Locations: b.Config.NearestLocations(structpath.NewPath(nil, "bundle", "name")), Paths: structpath.NewPathSlice("bundle", "name"), }) } diff --git a/bundle/config/validate/no_variable_reference_in_resource_key.go b/bundle/config/validate/no_variable_reference_in_resource_key.go index a2db01ffa14..9f568f2262d 100644 --- a/bundle/config/validate/no_variable_reference_in_resource_key.go +++ b/bundle/config/validate/no_variable_reference_in_resource_key.go @@ -38,7 +38,7 @@ func (m *noVariableReferenceInResourceKey) Apply(_ context.Context, b *bundle.Bu diags = append(diags, diag.Diagnostic{ Severity: diag.Error, Summary: fmt.Sprintf("resource key %q must not contain variable references", key), - Locations: b.Config.LocationsAt(p), + Locations: b.Config.ExactLocations(p), Paths: []*structpath.PathNode{p}, }) } diff --git a/bundle/config/validate/pipeline_development_deprecated.go b/bundle/config/validate/pipeline_development_deprecated.go index 62c838beae3..cf89e41556c 100644 --- a/bundle/config/validate/pipeline_development_deprecated.go +++ b/bundle/config/validate/pipeline_development_deprecated.go @@ -44,7 +44,7 @@ func (v *pipelineDevelopmentDeprecated) Apply(_ context.Context, b *bundle.Bundl p := structpath.NewPath(nil, "resources", "pipelines", key, "development") // Only user-written values have a location; the value set by "mode: development" does not. - locs := b.Config.LocationsAt(p) + locs := b.Config.ExactLocations(p) if len(locs) == 0 { continue } diff --git a/bundle/config/validate/required.go b/bundle/config/validate/required.go index 21e546738c6..d39eee5bce0 100644 --- a/bundle/config/validate/required.go +++ b/bundle/config/validate/required.go @@ -97,11 +97,11 @@ func errorForMissingFields(ctx context.Context, b *bundle.Bundle) diag.Diagnosti for key, dashboard := range b.Config.Resources.Dashboards { path := structpath.NewPath(nil, "resources", "dashboards", key) if dashboard.DisplayName == "" { - nameLocations = append(nameLocations, b.Config.GetLocationsOf(path)...) + nameLocations = append(nameLocations, b.Config.NearestLocations(path)...) namePaths = append(namePaths, path) } if dashboard.WarehouseId == "" { - warehouseIdLocations = append(warehouseIdLocations, b.Config.GetLocationsOf(path)...) + warehouseIdLocations = append(warehouseIdLocations, b.Config.NearestLocations(path)...) warehouseIdPaths = append(warehouseIdPaths, path) } } @@ -131,7 +131,7 @@ func errorForMissingFields(ctx context.Context, b *bundle.Bundle) diag.Diagnosti diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, Summary: "sql_warehouse name is required", - Locations: b.Config.GetLocationsOf(path), + Locations: b.Config.NearestLocations(path), Paths: []*structpath.PathNode{path}, }) } @@ -152,7 +152,7 @@ func errorForInvalidGrants(ctx context.Context, b *bundle.Bundle) diag.Diagnosti err := structvar.ForEach( b.Config.View(), - structpath.MustParsePattern("resources.*.*.grants[*]"), + structpath.NewPattern(nil, "resources", structpath.AnyKey, structpath.AnyKey, "grants", structpath.AnyIndex), func(np *structpath.PathNode, v structvar.View) error { if isMissingOrEmptyString(v.Get("principal")) { diags = diags.Append(diag.Diagnostic{ @@ -200,7 +200,7 @@ func errorForInvalidSecretScopePermissions(ctx context.Context, b *bundle.Bundle Severity: diag.Error, Summary: "secret scope permission principal is required", Detail: "Set one of user_name, group_name or service_principal_name", - Locations: b.Config.GetLocationsOf(scopePath), + Locations: b.Config.NearestLocations(scopePath), Paths: []*structpath.PathNode{path}, }) } diff --git a/bundle/config/validate/scripts.go b/bundle/config/validate/scripts.go index 328ba2404b6..0b71a27edc7 100644 --- a/bundle/config/validate/scripts.go +++ b/bundle/config/validate/scripts.go @@ -78,7 +78,7 @@ from "content" with $NAME: env: MY_VAR: ${var.foo} content: echo "$MY_VAR"`, - Locations: b.Config.LocationsAt(p), + Locations: b.Config.ExactLocations(p), Paths: []*structpath.PathNode{p}, }} } @@ -102,7 +102,7 @@ func validateScriptEnv(b *bundle.Bundle, key string, env map[string]string) diag diags = append(diags, diag.Diagnostic{ Severity: diag.Error, Summary: fmt.Sprintf("${%s} cannot be used in scripts.%s.env.%s; only ${bundle.*}, ${workspace.*}, and ${var.*} are resolved before scripts execute", refPath, key, name), - Locations: b.Config.LocationsAt(envValuePath), + Locations: b.Config.ExactLocations(envValuePath), Paths: []*structpath.PathNode{envValuePath}, }) } diff --git a/bundle/config/validate/single_node_cluster.go b/bundle/config/validate/single_node_cluster.go index b5a1355030b..65c5c647867 100644 --- a/bundle/config/validate/single_node_cluster.go +++ b/bundle/config/validate/single_node_cluster.go @@ -128,15 +128,15 @@ func (m *singleNodeCluster) Apply(ctx context.Context, b *bundle.Bundle) diag.Di patterns := []*structpath.PatternNode{ // Interactive clusters - structpath.MustParsePattern("resources.clusters.*"), + structpath.NewPattern(nil, "resources", "clusters", structpath.AnyKey), // Job clusters - structpath.MustParsePattern("resources.jobs.*.job_clusters[*].new_cluster"), + structpath.NewPattern(nil, "resources", "jobs", structpath.AnyKey, "job_clusters", structpath.AnyIndex, "new_cluster"), // Job task clusters - structpath.MustParsePattern("resources.jobs.*.tasks[*].new_cluster"), + structpath.NewPattern(nil, "resources", "jobs", structpath.AnyKey, "tasks", structpath.AnyIndex, "new_cluster"), // Job for each task clusters - structpath.MustParsePattern("resources.jobs.*.tasks[*].for_each_task.task.new_cluster"), + structpath.NewPattern(nil, "resources", "jobs", structpath.AnyKey, "tasks", structpath.AnyIndex, "for_each_task", "task", "new_cluster"), // Pipeline clusters - structpath.MustParsePattern("resources.pipelines.*.clusters[*]"), + structpath.NewPattern(nil, "resources", "pipelines", structpath.AnyKey, "clusters", structpath.AnyIndex), } root := b.Config.View() diff --git a/bundle/config/validate/unique_resource_keys.go b/bundle/config/validate/unique_resource_keys.go index bced70c6ffc..dcbfeacb3cb 100644 --- a/bundle/config/validate/unique_resource_keys.go +++ b/bundle/config/validate/unique_resource_keys.go @@ -51,7 +51,7 @@ func (m *uniqueResourceKeys) Apply(ctx context.Context, b *bundle.Bundle) diag.D } mv.paths = append(mv.paths, fullPath) - mv.locations = append(mv.locations, b.Config.LocationsAt(fullPath)...) + mv.locations = append(mv.locations, b.Config.ExactLocations(fullPath)...) resourceAndScriptMetadata[k] = mv } diff --git a/bundle/config/validate/validate_artifact_path.go b/bundle/config/validate/validate_artifact_path.go index 572c3a0de5a..6d6a83e86e9 100644 --- a/bundle/config/validate/validate_artifact_path.go +++ b/bundle/config/validate/validate_artifact_path.go @@ -64,12 +64,12 @@ func findVolumeInBundle(r config.Root, catalogName, schemaName, volumeName strin // We only have to check for ${resources.schemas...} references because any // other valid reference (like ${var.foo}) would have been interpolated by this point. p, ok := structvar.PureReferenceToPath(v.SchemaName) - isSchemaDefinedInBundle := ok && p.HasPrefix(structpath.MustParsePath("resources.schemas")) + isSchemaDefinedInBundle := ok && p.HasPrefix(structpath.NewPath(nil, "resources", "schemas")) if v.SchemaName != schemaName && !isSchemaDefinedInBundle { continue } volumePath := structpath.NewPath(nil, "resources", "volumes", k) - return volumePath, r.GetLocationsOf(volumePath), true + return volumePath, r.NearestLocations(volumePath), true } return nil, nil, false } @@ -85,7 +85,7 @@ func (v *validateArtifactPath) Apply(ctx context.Context, b *bundle.Bundle) diag { Summary: s, Severity: diag.Error, - Locations: b.Config.GetLocations("workspace.artifact_path"), + Locations: b.Config.NearestLocations(structpath.NewPath(nil, "workspace", "artifact_path")), Paths: structpath.NewPathSlice("workspace", "artifact_path"), }, } @@ -117,8 +117,8 @@ func (v *validateArtifactPath) Apply(ctx context.Context, b *bundle.Bundle) diag this bundle but which has not been deployed yet. Please first deploy the volume using 'bundle deploy' and then switch over to using it in the artifact_path.`, - Locations: slices.Concat(b.Config.GetLocations("workspace.artifact_path"), locations), - Paths: []*structpath.PathNode{structpath.MustParsePath("workspace.artifact_path"), path}, + Locations: slices.Concat(b.Config.NearestLocations(structpath.NewPath(nil, "workspace", "artifact_path")), locations), + Paths: []*structpath.PathNode{structpath.NewPath(nil, "workspace", "artifact_path"), path}, }} } diff --git a/bundle/config/validate/validate_dashboard_etags.go b/bundle/config/validate/validate_dashboard_etags.go index 1715f22cb84..265683c9f65 100644 --- a/bundle/config/validate/validate_dashboard_etags.go +++ b/bundle/config/validate/validate_dashboard_etags.go @@ -28,7 +28,7 @@ func (v *validateDashboardEtags) Apply(ctx context.Context, b *bundle.Bundle) di Severity: diag.Error, Summary: fmt.Sprintf("dashboard %q has an etag set. Etags must not be set in bundle configuration", dashboard.DisplayName), Paths: structpath.NewPathSlice("resources", "dashboards", k), - Locations: b.Config.GetLocationsOf(structpath.NewPath(nil, "resources", "dashboards", k)), + Locations: b.Config.NearestLocations(structpath.NewPath(nil, "resources", "dashboards", k)), }, } } diff --git a/bundle/config/validate/validate_deployment_fields.go b/bundle/config/validate/validate_deployment_fields.go index d1ca337f4ed..4d2fbd6fb0a 100644 --- a/bundle/config/validate/validate_deployment_fields.go +++ b/bundle/config/validate/validate_deployment_fields.go @@ -35,7 +35,7 @@ func (v *validateDeploymentFields) Apply(_ context.Context, b *bundle.Bundle) di Severity: diag.Error, Summary: field + " must not be set in bundle configuration; it is managed by Declarative Automation Bundles", Paths: []*structpath.PathNode{path}, - Locations: b.Config.GetLocationsOf(path), + Locations: b.Config.NearestLocations(path), }) } diff --git a/bundle/config/validate/validate_engine.go b/bundle/config/validate/validate_engine.go index 11f7b106b87..f04409924d8 100644 --- a/bundle/config/validate/validate_engine.go +++ b/bundle/config/validate/validate_engine.go @@ -25,7 +25,7 @@ func (v *validateEngine) Name() string { func (v *validateEngine) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { var diags diag.Diagnostics - tf := b.Config.View().Lookup(structpath.MustParsePath("bundle.terraform")) + tf := b.Config.View().Lookup(structpath.NewPath(nil, "bundle", "terraform")) if tf.IsValid() { diags = diags.Append(diag.Diagnostic{ Severity: diag.Warning, @@ -40,7 +40,7 @@ func (v *validateEngine) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnos return diags } - loc := b.Config.GetLocation("bundle.engine") + loc := b.Config.NearestLocation(structpath.NewPath(nil, "bundle", "engine")) parsed, ok := engine.Parse(string(configEngine)) if !ok { diff --git a/bundle/config/validate/validate_genie_space_etags.go b/bundle/config/validate/validate_genie_space_etags.go index 963e572346f..559d929d8dc 100644 --- a/bundle/config/validate/validate_genie_space_etags.go +++ b/bundle/config/validate/validate_genie_space_etags.go @@ -30,7 +30,7 @@ func (v *validateGenieSpaceEtags) Apply(ctx context.Context, b *bundle.Bundle) d Severity: diag.Error, Summary: fmt.Sprintf("genie space %q has an etag set. Etags must not be set in bundle configuration", genieSpace.Title), Paths: structpath.NewPathSlice("resources", "genie_spaces", k), - Locations: b.Config.GetLocationsOf(structpath.NewPath(nil, "resources", "genie_spaces", k)), + Locations: b.Config.NearestLocations(structpath.NewPath(nil, "resources", "genie_spaces", k)), }, } } diff --git a/bundle/config/validate/validate_job_run_idempotency_token.go b/bundle/config/validate/validate_job_run_idempotency_token.go index 61e07e302ca..9a6a161941a 100644 --- a/bundle/config/validate/validate_job_run_idempotency_token.go +++ b/bundle/config/validate/validate_job_run_idempotency_token.go @@ -37,7 +37,7 @@ func (v *validateJobRunIdempotencyToken) Apply(_ context.Context, b *bundle.Bund Severity: diag.Error, Summary: "idempotency_token must not be set in bundle configuration; the CLI sets it on each run-now request", Paths: []*structpath.PathNode{path}, - Locations: b.Config.GetLocationsOf(path), + Locations: b.Config.NearestLocations(path), }) } diff --git a/bundle/config/validate/validate_sync_patterns.go b/bundle/config/validate/validate_sync_patterns.go index c888a05a7dd..e441af87a3f 100644 --- a/bundle/config/validate/validate_sync_patterns.go +++ b/bundle/config/validate/validate_sync_patterns.go @@ -61,7 +61,7 @@ func checkPatterns(ctx context.Context, patterns []string, path *structpath.Path logdiag.LogDiag(ctx, diag.Diagnostic{ Severity: diag.Warning, Summary: fmt.Sprintf("Pattern %s does not match any files", pattern), - Locations: b.Config.GetLocationsOf(path), + Locations: b.Config.NearestLocations(path), Paths: []*structpath.PathNode{path}, }) } diff --git a/bundle/config/validate/validate_volume_path.go b/bundle/config/validate/validate_volume_path.go index c440dfbf06f..fe058732684 100644 --- a/bundle/config/validate/validate_volume_path.go +++ b/bundle/config/validate/validate_volume_path.go @@ -23,13 +23,12 @@ func (m *validateVolumePath) Apply(ctx context.Context, b *bundle.Bundle) diag.D // Define paths to check and their corresponding config field names pathChecks := []struct { path string - configName string configPath *structpath.PathNode }{ - {b.Config.Workspace.RootPath, "workspace.root_path", structpath.NewPath(nil, "workspace", "root_path")}, - {b.Config.Workspace.FilePath, "workspace.file_path", structpath.NewPath(nil, "workspace", "file_path")}, - {b.Config.Workspace.StatePath, "workspace.state_path", structpath.NewPath(nil, "workspace", "state_path")}, - {b.Config.Workspace.ResourcePath, "workspace.resource_path", structpath.NewPath(nil, "workspace", "resource_path")}, + {b.Config.Workspace.RootPath, structpath.NewPath(nil, "workspace", "root_path")}, + {b.Config.Workspace.FilePath, structpath.NewPath(nil, "workspace", "file_path")}, + {b.Config.Workspace.StatePath, structpath.NewPath(nil, "workspace", "state_path")}, + {b.Config.Workspace.ResourcePath, structpath.NewPath(nil, "workspace", "resource_path")}, } // Check each path @@ -37,14 +36,14 @@ func (m *validateVolumePath) Apply(ctx context.Context, b *bundle.Bundle) diag.D if check.path != "" && strings.HasPrefix(check.path, "/Volumes/") { diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, - Summary: fmt.Sprintf("%s %s starts with /Volumes. /Volumes can only be used with workspace.artifact_path.", check.configName, check.path), + Summary: fmt.Sprintf("%s %s starts with /Volumes. /Volumes can only be used with workspace.artifact_path.", check.configPath, check.path), Detail: "For more information, see https://docs.databricks.com/aws/en/dev-tools/bundles/settings#workspace", - Locations: b.Config.GetLocationsOf(check.configPath), + Locations: b.Config.NearestLocations(check.configPath), Paths: []*structpath.PathNode{check.configPath}, }) // Return early for root path validation - if check.configName == "workspace.root_path" { + if check.configPath.String() == "workspace.root_path" { return diags } } diff --git a/bundle/configsync/resolve.go b/bundle/configsync/resolve.go index abd74295216..baf951ec67c 100644 --- a/bundle/configsync/resolve.go +++ b/bundle/configsync/resolve.go @@ -351,8 +351,11 @@ func ResolveChanges(ctx context.Context, b *bundle.Bundle, configChanges Changes // on it belongs in that same block. filePath = block.file } else { - resourceLocation := b.Config.DefinitionLocation(resourceKey) - filePath = resourceLocation.File + resourcePath, err := structpath.ParsePath(resourceKey) + if err != nil { + return nil, 0, fmt.Errorf("failed to parse resource key %s: %w", resourceKey, err) + } + filePath = b.Config.ExactLocation(resourcePath).File } if filePath == "" { return nil, 0, fmt.Errorf("failed to find location for resource %s for a field %s", resourceKey, fieldPath) diff --git a/bundle/deploy/check_dashboards_modified_remotely.go b/bundle/deploy/check_dashboards_modified_remotely.go index d685f7ebeb6..d3308a9ded7 100644 --- a/bundle/deploy/check_dashboards_modified_remotely.go +++ b/bundle/deploy/check_dashboards_modified_remotely.go @@ -73,7 +73,7 @@ func (l *checkDashboardsModifiedRemotely) Apply(ctx context.Context, b *bundle.B } path := structpath.NewPath(nil, "resources", "dashboards", dashboard.Name) - loc := b.Config.GetLocationOf(path) + loc := b.Config.NearestLocation(path) actual, err := b.WorkspaceClient(ctx).Lakeview.GetByDashboardId(ctx, dashboard.ID) if err != nil { diags = diags.Append(diag.Diagnostic{ diff --git a/bundle/deploy/metadata/compute.go b/bundle/deploy/metadata/compute.go index c8775748151..d33f32baba7 100644 --- a/bundle/deploy/metadata/compute.go +++ b/bundle/deploy/metadata/compute.go @@ -10,6 +10,7 @@ import ( "github.com/databricks/cli/bundle/metadata" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/structs/structpath" ) type compute struct{} @@ -46,7 +47,7 @@ func (m *compute) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics for name, job := range b.Config.Resources.Jobs { // Compute config file path the job is defined in, relative to the bundle // root - l := b.Config.DefinitionLocation("resources.jobs." + name) + l := b.Config.ExactLocation(structpath.NewPath(nil, "resources", "jobs", name)) if l.File == "" { // Skip resources that exist only in the deployment state: statemgmt.Load, // which runs before this mutator, injects them into the config without a @@ -73,7 +74,7 @@ func (m *compute) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics for name, pipeline := range b.Config.Resources.Pipelines { // Compute config file path the pipeline is defined in, relative to the bundle // root - l := b.Config.DefinitionLocation("resources.pipelines." + name) + l := b.Config.ExactLocation(structpath.NewPath(nil, "resources", "pipelines", name)) if l.File == "" { // Skip resources that exist only in the deployment state: statemgmt.Load, // which runs before this mutator, injects them into the config without a @@ -98,7 +99,7 @@ func (m *compute) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics for name, dashboard := range b.Config.Resources.Dashboards { // Compute config file path the dashboard is defined in, relative to the bundle // root - l := b.Config.DefinitionLocation("resources.dashboards." + name) + l := b.Config.ExactLocation(structpath.NewPath(nil, "resources", "dashboards", name)) if l.File == "" { // Skip resources that exist only in the deployment state: statemgmt.Load, // which runs before this mutator, injects them into the config without a diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 6503ba402b8..8cf8e414019 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -1159,9 +1159,9 @@ func (b *DeploymentBundle) makePlan(ctx context.Context, configRoot *config.Root existingKeys := maps.Clone(db.State) patterns := []*structpath.PatternNode{ - structpath.MustParsePattern("resources.*.*"), - structpath.MustParsePattern("resources.*.*.permissions"), - structpath.MustParsePattern("resources.*.*.grants"), + structpath.NewPattern(nil, "resources", structpath.AnyKey, structpath.AnyKey), + structpath.NewPattern(nil, "resources", structpath.AnyKey, structpath.AnyKey, "permissions"), + structpath.NewPattern(nil, "resources", structpath.AnyKey, structpath.AnyKey, "grants"), } var rootView structvar.View diff --git a/bundle/direct/dresources/cluster.go b/bundle/direct/dresources/cluster.go index 82167080c4d..bc3467786d2 100644 --- a/bundle/direct/dresources/cluster.go +++ b/bundle/direct/dresources/cluster.go @@ -528,7 +528,7 @@ func makeEditCluster(id string, config *compute.ClusterSpec) compute.EditCluster } // librariesPath is the ClusterState path of the libraries slice, used to detect library changes. -var librariesPath = structpath.MustParsePath("libraries") +var librariesPath = structpath.NewPath(nil, "libraries") // KeyedSlices compares libraries by identity rather than index so reordering (or the // read-order the status API returns) does not produce phantom diffs. diff --git a/bundle/direct/dresources/model_serving_endpoint.go b/bundle/direct/dresources/model_serving_endpoint.go index f9c0a2314ac..b763a08fd05 100644 --- a/bundle/direct/dresources/model_serving_endpoint.go +++ b/bundle/direct/dresources/model_serving_endpoint.go @@ -15,11 +15,11 @@ import ( // Precalculated paths for HasChange checks var ( - pathTags = structpath.MustParsePath("tags") - pathAiGateway = structpath.MustParsePath("ai_gateway") - pathConfig = structpath.MustParsePath("config") - pathEmailNotifications = structpath.MustParsePath("email_notifications") - pathTelemetryConfig = structpath.MustParsePath("telemetry_config") + pathTags = structpath.NewPath(nil, "tags") + pathAiGateway = structpath.NewPath(nil, "ai_gateway") + pathConfig = structpath.NewPath(nil, "config") + pathEmailNotifications = structpath.NewPath(nil, "email_notifications") + pathTelemetryConfig = structpath.NewPath(nil, "telemetry_config") ) type ResourceModelServingEndpoint struct { diff --git a/bundle/direct/dresources/vector_search_endpoint.go b/bundle/direct/dresources/vector_search_endpoint.go index 12470872b62..3775f793b60 100644 --- a/bundle/direct/dresources/vector_search_endpoint.go +++ b/bundle/direct/dresources/vector_search_endpoint.go @@ -13,8 +13,8 @@ import ( ) var ( - pathBudgetPolicyId = structpath.MustParsePath("budget_policy_id") - pathTargetQps = structpath.MustParsePath("target_qps") + pathBudgetPolicyId = structpath.NewPath(nil, "budget_policy_id") + pathTargetQps = structpath.NewPath(nil, "target_qps") ) // VectorSearchEndpointRemote is remote state for a vector search endpoint. It embeds API response diff --git a/bundle/direct/resolve_config.go b/bundle/direct/resolve_config.go index f35ceaef5b3..de316f34030 100644 --- a/bundle/direct/resolve_config.go +++ b/bundle/direct/resolve_config.go @@ -11,7 +11,7 @@ import ( "github.com/databricks/cli/libs/structs/structvar" ) -var resourcesPrefix = structpath.MustParsePath("resources") +var resourcesPrefix = structpath.NewPath(nil, "resources") // ResolveConfigAgainstState resolves ${resources.*} references within the resource at // target so its runner ("bundle run") sees concrete values rather than references. For a diff --git a/bundle/libraries/expand_glob_references.go b/bundle/libraries/expand_glob_references.go index 9277c349100..9c32c496e8a 100644 --- a/bundle/libraries/expand_glob_references.go +++ b/bundle/libraries/expand_glob_references.go @@ -93,7 +93,7 @@ func expandSequence[T any](ctx context.Context, b *bundle.Bundle, p *structpath. ip := structpath.NewIndex(p, i) expanded, relocate, d := fn(ctx, b, ip, item) diags = diags.Extend(d) - locs := b.Config.LocationsAt(ip) + locs := b.Config.ExactLocations(ip) for _, e := range expanded { output = append(output, e) sources = append(sources, []int{i}) @@ -139,7 +139,7 @@ func expandLibraries(ctx context.Context, lv structvar.View, p *structpath.PathN matches, err := findMatches(ctx, b, path) if err != nil { - return nil, "", diag.Diagnostics{matchError(structpath.NewStringKey(ip, libType), b.Config.LocationsAt(ip), err.Error())} + return nil, "", diag.Diagnostics{matchError(structpath.NewStringKey(ip, libType), b.Config.ExactLocations(ip), err.Error())} } var output []compute.Library @@ -165,7 +165,7 @@ func expandEnvironmentDeps(ctx context.Context, lv structvar.View, p *structpath matches, err := findMatches(ctx, b, path) if err != nil { - return nil, "", diag.Diagnostics{matchError(ip, b.Config.LocationsAt(ip), err.Error())} + return nil, "", diag.Diagnostics{matchError(ip, b.Config.ExactLocations(ip), err.Error())} } var output []string @@ -182,13 +182,13 @@ type expandPattern struct { } var ( - taskLibrariesPattern = structpath.MustParsePattern("resources.jobs.*.tasks[*].libraries") - forEachTaskLibrariesPattern = structpath.MustParsePattern("resources.jobs.*.tasks[*].for_each_task.task.libraries") - aiRuntimeCodeSourcePattern = structpath.MustParsePattern("resources.jobs.*.tasks[*].ai_runtime_task.code_source_path") - forEachAiRuntimeCodeSourcePattern = structpath.MustParsePattern("resources.jobs.*.tasks[*].for_each_task.task.ai_runtime_task.code_source_path") - envDepsPattern = structpath.MustParsePattern("resources.jobs.*.environments[*].spec.dependencies") - pipelineEnvDepsPattern = structpath.MustParsePattern("resources.pipelines.*.environment.dependencies") - clusterLibrariesPattern = structpath.MustParsePattern("resources.clusters.*.libraries") + taskLibrariesPattern = structpath.NewPattern(nil, "resources", "jobs", structpath.AnyKey, "tasks", structpath.AnyIndex, "libraries") + forEachTaskLibrariesPattern = structpath.NewPattern(nil, "resources", "jobs", structpath.AnyKey, "tasks", structpath.AnyIndex, "for_each_task", "task", "libraries") + aiRuntimeCodeSourcePattern = structpath.NewPattern(nil, "resources", "jobs", structpath.AnyKey, "tasks", structpath.AnyIndex, "ai_runtime_task", "code_source_path") + forEachAiRuntimeCodeSourcePattern = structpath.NewPattern(nil, "resources", "jobs", structpath.AnyKey, "tasks", structpath.AnyIndex, "for_each_task", "task", "ai_runtime_task", "code_source_path") + envDepsPattern = structpath.NewPattern(nil, "resources", "jobs", structpath.AnyKey, "environments", structpath.AnyIndex, "spec", "dependencies") + pipelineEnvDepsPattern = structpath.NewPattern(nil, "resources", "pipelines", structpath.AnyKey, "environment", "dependencies") + clusterLibrariesPattern = structpath.NewPattern(nil, "resources", "clusters", structpath.AnyKey, "libraries") ) func (e *expand) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { diff --git a/bundle/libraries/expand_glob_references_test.go b/bundle/libraries/expand_glob_references_test.go index 5bb4a49fff8..4032201b76b 100644 --- a/bundle/libraries/expand_glob_references_test.go +++ b/bundle/libraries/expand_glob_references_test.go @@ -283,19 +283,19 @@ func TestExpandGlobReferencesPreservesLocations(t *testing.T) { diags := bundle.Apply(t.Context(), b, ExpandGlobReferences()) require.Empty(t, diags) - libs := b.Config.LocationsAt(structpath.MustParsePath("resources.jobs.job.tasks[0].libraries")) + libs := b.Config.ExactLocations(structpath.MustParsePath("resources.jobs.job.tasks[0].libraries")) require.NotEmpty(t, libs) assert.Equal(t, loc.File, libs[0].File) - lib := b.Config.LocationsAt(structpath.MustParsePath("resources.jobs.job.tasks[0].libraries[0].whl")) + lib := b.Config.ExactLocations(structpath.MustParsePath("resources.jobs.job.tasks[0].libraries[0].whl")) require.NotEmpty(t, lib) assert.Equal(t, loc.File, lib[0].File) - deps := b.Config.LocationsAt(structpath.MustParsePath("resources.pipelines.pipeline.environment.dependencies")) + deps := b.Config.ExactLocations(structpath.MustParsePath("resources.pipelines.pipeline.environment.dependencies")) require.NotEmpty(t, deps) assert.Equal(t, loc.File, deps[0].File) - dep := b.Config.LocationsAt(structpath.MustParsePath("resources.pipelines.pipeline.environment.dependencies[0]")) + dep := b.Config.ExactLocations(structpath.MustParsePath("resources.pipelines.pipeline.environment.dependencies[0]")) require.NotEmpty(t, dep) assert.Equal(t, loc.File, dep[0].File) } diff --git a/bundle/libraries/remote_path.go b/bundle/libraries/remote_path.go index d28e21c10c1..d879cef676f 100644 --- a/bundle/libraries/remote_path.go +++ b/bundle/libraries/remote_path.go @@ -131,14 +131,14 @@ func collectLocalLibraries(b *bundle.Bundle) (map[string][]LocationToUpdate, err libs := make(map[string]([]LocationToUpdate)) patterns := []*structpath.PatternNode{ - structpath.MustParsePattern(taskLibrariesPattern.String() + "[*].whl"), - structpath.MustParsePattern(taskLibrariesPattern.String() + "[*].jar"), - structpath.MustParsePattern(forEachTaskLibrariesPattern.String() + "[*].whl"), - structpath.MustParsePattern(forEachTaskLibrariesPattern.String() + "[*].jar"), - structpath.MustParsePattern(clusterLibrariesPattern.String() + "[*].whl"), - structpath.MustParsePattern(clusterLibrariesPattern.String() + "[*].jar"), - structpath.MustParsePattern(envDepsPattern.String() + "[*]"), - structpath.MustParsePattern(pipelineEnvDepsPattern.String() + "[*]"), + structpath.NewPattern(taskLibrariesPattern, structpath.AnyIndex, "whl"), + structpath.NewPattern(taskLibrariesPattern, structpath.AnyIndex, "jar"), + structpath.NewPattern(forEachTaskLibrariesPattern, structpath.AnyIndex, "whl"), + structpath.NewPattern(forEachTaskLibrariesPattern, structpath.AnyIndex, "jar"), + structpath.NewPattern(clusterLibrariesPattern, structpath.AnyIndex, "whl"), + structpath.NewPattern(clusterLibrariesPattern, structpath.AnyIndex, "jar"), + structpath.NewPattern(envDepsPattern, structpath.AnyIndex), + structpath.NewPattern(pipelineEnvDepsPattern, structpath.AnyIndex), // The AI Runtime task's code_source_path is a local archive (typically an // artifact-built .tar.gz) that must be uploaded and referenced by its remote // path, exactly like a wheel or jar library. @@ -176,7 +176,7 @@ func collectLocalLibraries(b *bundle.Bundle) (map[string][]LocationToUpdate, err } } - artifactPattern := structpath.MustParsePattern("artifacts.*.files[*]") + artifactPattern := structpath.NewPattern(nil, "artifacts", structpath.AnyKey, "files", structpath.AnyIndex) err := structvar.ForEach(root, artifactPattern, func(p *structpath.PathNode, v structvar.View) error { if v.Kind() != structvar.KindMap { diff --git a/bundle/libraries/same_name_libraries.go b/bundle/libraries/same_name_libraries.go index a2ba9591a4f..3dcb4ba03b0 100644 --- a/bundle/libraries/same_name_libraries.go +++ b/bundle/libraries/same_name_libraries.go @@ -14,14 +14,14 @@ import ( type checkForSameNameLibraries struct{} var patterns = []*structpath.PatternNode{ - structpath.MustParsePattern(taskLibrariesPattern.String() + "[*].whl"), - structpath.MustParsePattern(taskLibrariesPattern.String() + "[*].jar"), - structpath.MustParsePattern(forEachTaskLibrariesPattern.String() + "[*].whl"), - structpath.MustParsePattern(forEachTaskLibrariesPattern.String() + "[*].jar"), - structpath.MustParsePattern(clusterLibrariesPattern.String() + "[*].whl"), - structpath.MustParsePattern(clusterLibrariesPattern.String() + "[*].jar"), - structpath.MustParsePattern(envDepsPattern.String() + "[*]"), - structpath.MustParsePattern(pipelineEnvDepsPattern.String() + "[*]"), + structpath.NewPattern(taskLibrariesPattern, structpath.AnyIndex, "whl"), + structpath.NewPattern(taskLibrariesPattern, structpath.AnyIndex, "jar"), + structpath.NewPattern(forEachTaskLibrariesPattern, structpath.AnyIndex, "whl"), + structpath.NewPattern(forEachTaskLibrariesPattern, structpath.AnyIndex, "jar"), + structpath.NewPattern(clusterLibrariesPattern, structpath.AnyIndex, "whl"), + structpath.NewPattern(clusterLibrariesPattern, structpath.AnyIndex, "jar"), + structpath.NewPattern(envDepsPattern, structpath.AnyIndex), + structpath.NewPattern(pipelineEnvDepsPattern, structpath.AnyIndex), } type libData struct { diff --git a/bundle/migrate/build_state.go b/bundle/migrate/build_state.go index 4bc28889fc5..d81b25a0452 100644 --- a/bundle/migrate/build_state.go +++ b/bundle/migrate/build_state.go @@ -59,9 +59,9 @@ func BuildStateFromTF( rootView := configRoot.View() patterns := []*structpath.PatternNode{ - structpath.MustParsePattern("resources.*.*"), - structpath.MustParsePattern("resources.*.*.permissions"), - structpath.MustParsePattern("resources.*.*.grants"), + structpath.NewPattern(nil, "resources", structpath.AnyKey, structpath.AnyKey), + structpath.NewPattern(nil, "resources", structpath.AnyKey, structpath.AnyKey, "permissions"), + structpath.NewPattern(nil, "resources", structpath.AnyKey, structpath.AnyKey, "grants"), } for _, pat := range patterns { err := structvar.ForEach( diff --git a/bundle/permissions/permission_diagnostics.go b/bundle/permissions/permission_diagnostics.go index 0e01f72a0f4..8d4aa37a703 100644 --- a/bundle/permissions/permission_diagnostics.go +++ b/bundle/permissions/permission_diagnostics.go @@ -10,6 +10,7 @@ import ( "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/iamutil" "github.com/databricks/cli/libs/set" + "github.com/databricks/cli/libs/structs/structpath" ) type permissionDiagnostics struct{} @@ -58,7 +59,7 @@ func (m *permissionDiagnostics) Apply(ctx context.Context, b *bundle.Bundle) dia identityType, b.Config.Workspace.CurrentUser.UserName, ), - Locations: []diag.Location{b.Config.GetLocation("permissions")}, + Locations: []diag.Location{b.Config.NearestLocation(structpath.NewPath(nil, "permissions"))}, ID: diag.PermissionNotIncluded, }} } diff --git a/bundle/phases/telemetry.go b/bundle/phases/telemetry.go index 159afeb9560..4d25646542f 100644 --- a/bundle/phases/telemetry.go +++ b/bundle/phases/telemetry.go @@ -202,7 +202,7 @@ func LogDeployTelemetry(ctx context.Context, b *bundle.Bundle, errMsg string) { b.Metrics.SetBoolValue(metrics.DeploymentHistoryEnabled, b.DeploymentBundle.StateDB.IsDeploymentMetadataService()) resourcesCount := int64(0) - err := structvar.ForEach(b.Config.View(), structpath.MustParsePattern("resources.*.*"), func(p *structpath.PathNode, v structvar.View) error { + err := structvar.ForEach(b.Config.View(), structpath.NewPattern(nil, "resources", structpath.AnyKey, structpath.AnyKey), func(p *structpath.PathNode, v structvar.View) error { resourcesCount++ return nil }) diff --git a/bundle/statemgmt/reverse_interpolate_test.go b/bundle/statemgmt/reverse_interpolate_test.go index ef6a40ee5da..35ce1ed692e 100644 --- a/bundle/statemgmt/reverse_interpolate_test.go +++ b/bundle/statemgmt/reverse_interpolate_test.go @@ -45,7 +45,7 @@ resources: assert.Equal(t, "${resources.jobs.other.id}", maxRuns) // Locations are kept. - assert.Equal(t, b.Config.GetLocations("resources.jobs.my_job.description"), uninterpolatedConfig.GetLocations("resources.jobs.my_job.description")) + assert.Equal(t, b.Config.NearestLocations(structpath.NewPath(nil, "resources", "jobs", "my_job", "description")), uninterpolatedConfig.NearestLocations(structpath.NewPath(nil, "resources", "jobs", "my_job", "description"))) afterJSON, err := json.Marshal(b.Config.View().AsAny()) require.NoError(t, err) diff --git a/bundle/statemgmt/state_load.go b/bundle/statemgmt/state_load.go index 2e3849c8bf9..bda44241b26 100644 --- a/bundle/statemgmt/state_load.go +++ b/bundle/statemgmt/state_load.go @@ -131,7 +131,7 @@ func StateToBundle(ctx context.Context, state ExportedResourcesMap, cfg *config. } } - return structvar.ForEach(cfg.View(), structpath.MustParsePattern("resources.*.*"), func(p *structpath.PathNode, inner structvar.View) error { + return structvar.ForEach(cfg.View(), structpath.NewPattern(nil, "resources", structpath.AnyKey, structpath.AnyKey), func(p *structpath.PathNode, inner structvar.View) error { group, _ := p.Parent().StringKey() if !hasID(group) || inner.Get("id").IsValid() || inner.Get("modified_status").IsValid() { return nil diff --git a/bundle/tests/include_test.go b/bundle/tests/include_test.go index 50bf177fdfc..a03d1521bc7 100644 --- a/bundle/tests/include_test.go +++ b/bundle/tests/include_test.go @@ -9,6 +9,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/phases" "github.com/databricks/cli/libs/logdiag" + "github.com/databricks/cli/libs/structs/structpath" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -32,7 +33,7 @@ func TestIncludeWithGlob(t *testing.T) { job := b.Config.Resources.Jobs["my_job"] assert.Equal(t, "1", job.ID) - l := b.Config.GetLocation("resources.jobs.my_job") + l := b.Config.NearestLocation(structpath.NewPath(nil, "resources", "jobs", "my_job")) assert.Equal(t, "include_with_glob/job.yml", filepath.ToSlash(l.File)) } @@ -52,11 +53,11 @@ func TestIncludeForMultipleMatches(t *testing.T) { first := b.Config.Resources.Jobs["my_first_job"] assert.Equal(t, "1", first.ID) - fl := b.Config.GetLocation("resources.jobs.my_first_job") + fl := b.Config.NearestLocation(structpath.NewPath(nil, "resources", "jobs", "my_first_job")) assert.Equal(t, "include_multiple/my_first_job/resource.yml", filepath.ToSlash(fl.File)) second := b.Config.Resources.Jobs["my_second_job"] assert.Equal(t, "2", second.ID) - sl := b.Config.GetLocation("resources.jobs.my_second_job") + sl := b.Config.NearestLocation(structpath.NewPath(nil, "resources", "jobs", "my_second_job")) assert.Equal(t, "include_multiple/my_second_job/resource.yml", filepath.ToSlash(sl.File)) } diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index 6332da33089..91d04e9c899 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -30,6 +30,7 @@ import ( "github.com/databricks/cli/libs/dms" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/logdiag" + "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/cli/libs/sync" "github.com/databricks/cli/libs/telemetry" "github.com/databricks/cli/libs/telemetry/protos" @@ -714,7 +715,7 @@ func isNewerVersion(stateVersion, currentVersion string) bool { func rejectDefinitions(ctx context.Context, b *bundle.Bundle) { if b.Config.Definitions != nil { - loc := b.Config.GetLocations("definitions") + loc := b.Config.NearestLocations(structpath.NewPath(nil, "definitions")) filename := "input yaml" if len(loc) > 0 { filename = filepath.ToSlash(loc[0].File) diff --git a/libs/structs/structvar/locations.go b/libs/structs/structvar/locations.go index 733c9b34df6..a40567d40a9 100644 --- a/libs/structs/structvar/locations.go +++ b/libs/structs/structvar/locations.go @@ -71,9 +71,9 @@ func (l *Locations) Get() []diag.Location { return slices.Clone(l.locs) } -// At returns the locations of the value at path. Values without recorded locations +// Exact returns the locations of the value at path. Values without recorded locations // (e.g. set by code) have none. -func (l *Locations) At(path *structpath.PathNode) []diag.Location { +func (l *Locations) Exact(path *structpath.PathNode) []diag.Location { return l.lookup(components(path)).Get() } diff --git a/libs/structs/structvar/write.go b/libs/structs/structvar/write.go index e0f605fa4c0..6eba0b8dda6 100644 --- a/libs/structs/structvar/write.go +++ b/libs/structs/structvar/write.go @@ -226,7 +226,7 @@ func (sv *StructVar) SetReference(path *structpath.PathNode, ref string) error { if isText { return sv.Set(path, ref) } - locs := sv.Locations.At(path) + locs := sv.Locations.Exact(path) // withoutRefs returns a fresh map unless there is nothing to filter. refs := withoutRefs(sv.Refs, path) if len(refs) == 0 { From e4cdd6c9bfd18e1114d7a9f73d929e75c78420bf Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 16 Oct 2026 10:24:48 +0200 Subject: [PATCH 21/22] structvar: don't allocate per struct field when traversing views IsForceSend boxed the ForceSendFields slice on every call and FieldValues built a slice per struct; both now run on every View traversal rather than once per conversion. Iterate with reflect and yield the fields instead. BenchmarkWalkReadOnly (10k jobs): 7.16M -> 4.77M allocs. Co-authored-by: Isaac --- libs/structs/structvar/struct_info.go | 36 ++++++++++++++++----------- libs/structs/structvar/view.go | 2 +- 2 files changed, 23 insertions(+), 15 deletions(-) diff --git a/libs/structs/structvar/struct_info.go b/libs/structs/structvar/struct_info.go index 46ca6cde995..12af491fc61 100644 --- a/libs/structs/structvar/struct_info.go +++ b/libs/structs/structvar/struct_info.go @@ -1,6 +1,7 @@ package structvar import ( + "iter" "reflect" "slices" "sync" @@ -140,13 +141,16 @@ type FieldValue struct { IsForced bool } -func (s *StructInfo) FieldValues(v reflect.Value) []FieldValue { - out := make([]FieldValue, 0, len(s.Fields)) - - for _, k := range s.FieldNames { - fv := FieldByIndex(v, s.Fields[k]) +// FieldValues yields the fields of the struct v. It is an iterator rather than a slice +// because views call it on every struct they traverse. +func (s *StructInfo) FieldValues(v reflect.Value) iter.Seq[FieldValue] { + return func(yield func(FieldValue) bool) { + for _, k := range s.FieldNames { + fv := FieldByIndex(v, s.Fields[k]) + if !fv.IsValid() { + continue + } - if fv.IsValid() { isForced := true // TODO: we should use isEmptyForOmitEmpty instead of IsZero() @@ -154,15 +158,11 @@ func (s *StructInfo) FieldValues(v reflect.Value) []FieldValue { isForced = s.IsForceSend(v, k) } - out = append(out, FieldValue{ - Key: k, - Value: fv, - IsForced: isForced, - }) + if !yield(FieldValue{Key: k, Value: fv, IsForced: isForced}) { + return + } } } - - return out } // isForceSend reports whether the field named k is listed in the ForceSendFields @@ -176,7 +176,15 @@ func (s *StructInfo) IsForceSend(v reflect.Value, k string) bool { if !fsf.IsValid() { return false } - return slices.Contains(fsf.Interface().([]string), s.GolangNames[k]) + // Iterate with reflect: converting the slice to []string via Interface would + // allocate on every call. + name := s.GolangNames[k] + for i := range fsf.Len() { + if fsf.Index(i).String() == name { + return true + } + } + return false } // fieldByIndex resolves the value at the given index path, dereferencing embedded diff --git a/libs/structs/structvar/view.go b/libs/structs/structvar/view.go index ecdc0c73825..d23e481cf2c 100644 --- a/libs/structs/structvar/view.go +++ b/libs/structs/structvar/view.go @@ -291,7 +291,7 @@ func (x View) hasFields(v reflect.Value) bool { func (x View) structFields(v reflect.Value) iter.Seq2[string, View] { return func(yield func(string, View) bool) { info := GetStructInfo(v.Type()) - for _, fv := range info.FieldValues(v) { + for fv := range info.FieldValues(v) { child, ok := x.field(&info, v, fv.Key, fv.Value, fv.IsForced) if ok && !yield(fv.Key, child) { return From 586b4f7396a6bb554ae83545534700a0fc2680cb Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 16 Oct 2026 11:04:22 +0200 Subject: [PATCH 22/22] structvar: traverse views without closures, map lookups or rebuilding fields Walking a configuration examines every field of every struct, so the per-field cost dominates: - eachField/eachMapItem/eachElem take a callback instead of returning an iterator, so walking no longer allocates closures per node (MapItems and Sequence wrap them). - StructInfo keeps a field list with the index, Go name, omitempty, the governing ForceSendFields and whether the key is a dotted path component, so a visit does no lookups by name; GetStructInfo is lock-free. - ForceSendFields are read once per struct visit, not once per zero field. - An unset field that cannot be present (not a struct or interface, no location, no reference) is skipped without building its view. BenchmarkWalkReadOnly (10k jobs): 316ms -> 183ms, 4.77M -> 1.58M allocs. Co-authored-by: Isaac --- bundle/config/mutator/drop_empty_strings.go | 3 +- libs/structs/structvar/decode.go | 2 +- libs/structs/structvar/struct_info.go | 123 ++++++++------ libs/structs/structvar/view.go | 172 ++++++++++++++------ libs/structs/structvar/walk.go | 25 +-- libs/structs/structvar/write.go | 4 +- 6 files changed, 207 insertions(+), 122 deletions(-) diff --git a/bundle/config/mutator/drop_empty_strings.go b/bundle/config/mutator/drop_empty_strings.go index 243a2fb9f58..530f6df101a 100644 --- a/bundle/config/mutator/drop_empty_strings.go +++ b/bundle/config/mutator/drop_empty_strings.go @@ -75,8 +75,7 @@ func collectEmptyStrings(v structvar.View, p *structpath.PathNode, out *[]*struc var info *structvar.StructInfo switch { case r.Kind() == reflect.Struct && !structvar.IsSDKNativeType(r.Type()): - i := structvar.GetStructInfo(r.Type()) - info = &i + info = structvar.GetStructInfo(r.Type()) case r.Kind() == reflect.Map: default: return diff --git a/libs/structs/structvar/decode.go b/libs/structs/structvar/decode.go index 2ea9d008e64..701c6a527ac 100644 --- a/libs/structs/structvar/decode.go +++ b/libs/structs/structvar/decode.go @@ -470,7 +470,7 @@ func (d *decoder) decodeStruct(dst reflect.Value, src source, path *structpath.P // An explicitly set zero value must still serialize: add it to ForceSendFields. if fn.isZeroValue(fv) { - addForceSend(dst, &info, p.key) + addForceSend(dst, info, p.key) } } return n, true, nil diff --git a/libs/structs/structvar/struct_info.go b/libs/structs/structvar/struct_info.go index 12af491fc61..e6c50c59433 100644 --- a/libs/structs/structvar/struct_info.go +++ b/libs/structs/structvar/struct_info.go @@ -1,11 +1,11 @@ package structvar import ( - "iter" "reflect" "slices" "sync" + "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/cli/libs/structs/structtag" sdkduration "github.com/databricks/databricks-sdk-go/common/types/duration" sdkfieldmask "github.com/databricks/databricks-sdk-go/common/types/fieldmask" @@ -36,33 +36,54 @@ type StructInfo struct { // PostgresProject -> PostgresProjectConfig -> ProjectSpec). A field whose // declaring struct has no ForceSendFields has no entry. ForceSendFieldsIndex map[string][]int + + // List holds the fields in FieldNames order, with everything traversal needs, so + // that visiting a struct does no lookups by name. + List []FieldInfo + + // ForceSendGroups holds the distinct ForceSendFields index paths; a struct with + // embedded structs can have several. + ForceSendGroups [][]int + + // ForceSendTargets maps, per ForceSendGroups entry, the Go field names listed in + // that ForceSendFields to their index in List. + ForceSendTargets []map[string]int } -// StructInfoCache caches type information. -var StructInfoCache = make(map[reflect.Type]StructInfo) +// FieldInfo describes a field of a struct. +type FieldInfo struct { + Key string + Index []int + GolangName string + + // ForceEmpty is set for fields without omitempty. + ForceEmpty bool + + // Dot is set if Key can be written as a dotted path component (see + // [structpath.NewDotString]), which is true for every JSON name in practice. + Dot bool + + // ForceSendGroup is the index in [StructInfo.ForceSendGroups] of the + // ForceSendFields governing the field, or -1. + ForceSendGroup int +} -// StructInfoCacheLock guards concurrent access to StructInfoCache. -var StructInfoCacheLock sync.Mutex +var structInfoCache sync.Map // reflect.Type -> *StructInfo -// getStructInfo returns the [StructInfo] for the given type. +// GetStructInfo returns the [StructInfo] for the given type. // It lazily populates a cache, so the first call for a given // type is slower than subsequent calls for that same type. -func GetStructInfo(typ reflect.Type) StructInfo { - StructInfoCacheLock.Lock() - defer StructInfoCacheLock.Unlock() - - si, ok := StructInfoCache[typ] - if !ok { - si = buildStructInfo(typ) - StructInfoCache[typ] = si +func GetStructInfo(typ reflect.Type) *StructInfo { + if si, ok := structInfoCache.Load(typ); ok { + return si.(*StructInfo) } - - return si + si, _ := structInfoCache.LoadOrStore(typ, buildStructInfo(typ)) + return si.(*StructInfo) } // buildStructInfo populates a new [StructInfo] for the given type. -func buildStructInfo(typ reflect.Type) StructInfo { - out := StructInfo{ +func buildStructInfo(typ reflect.Type) *StructInfo { + out := &StructInfo{ Fields: make(map[string][]int), ForceEmpty: make(map[string]bool), GolangNames: make(map[string]string), @@ -126,43 +147,47 @@ func buildStructInfo(typ reflect.Type) StructInfo { // The field is declared directly in this struct, so it is governed by // this struct's ForceSendFields (if it has one). + group := -1 if forceSendFieldsIndex != nil { out.ForceSendFieldsIndex[name] = forceSendFieldsIndex + group = forceSendGroup(out, forceSendFieldsIndex) } + out.List = append(out.List, FieldInfo{ + Key: name, + Index: out.Fields[name], + GolangName: sf.Name, + ForceEmpty: out.ForceEmpty[name], + Dot: structpath.NewStringKey(nil, name).IsDotString(), + ForceSendGroup: group, + }) } } + out.ForceSendTargets = make([]map[string]int, len(out.ForceSendGroups)) + for i, f := range out.List { + if f.ForceSendGroup < 0 { + continue + } + m := out.ForceSendTargets[f.ForceSendGroup] + if m == nil { + m = map[string]int{} + out.ForceSendTargets[f.ForceSendGroup] = m + } + m[f.GolangName] = i + } return out } -type FieldValue struct { - Key string - Value reflect.Value - IsForced bool -} - -// FieldValues yields the fields of the struct v. It is an iterator rather than a slice -// because views call it on every struct they traverse. -func (s *StructInfo) FieldValues(v reflect.Value) iter.Seq[FieldValue] { - return func(yield func(FieldValue) bool) { - for _, k := range s.FieldNames { - fv := FieldByIndex(v, s.Fields[k]) - if !fv.IsValid() { - continue - } - - isForced := true - - // TODO: we should use isEmptyForOmitEmpty instead of IsZero() - if fv.IsZero() { - isForced = s.IsForceSend(v, k) - } - - if !yield(FieldValue{Key: k, Value: fv, IsForced: isForced}) { - return - } +// forceSendGroup returns the index of the ForceSendFields path index in +// out.ForceSendGroups, adding it if needed. +func forceSendGroup(out *StructInfo, index []int) int { + for i, g := range out.ForceSendGroups { + if slices.Equal(g, index) { + return i } } + out.ForceSendGroups = append(out.ForceSendGroups, index) + return len(out.ForceSendGroups) - 1 } // isForceSend reports whether the field named k is listed in the ForceSendFields @@ -176,15 +201,7 @@ func (s *StructInfo) IsForceSend(v reflect.Value, k string) bool { if !fsf.IsValid() { return false } - // Iterate with reflect: converting the slice to []string via Interface would - // allocate on every call. - name := s.GolangNames[k] - for i := range fsf.Len() { - if fsf.Index(i).String() == name { - return true - } - } - return false + return containsString(fsf, s.GolangNames[k]) } // fieldByIndex resolves the value at the given index path, dereferencing embedded diff --git a/libs/structs/structvar/view.go b/libs/structs/structvar/view.go index d23e481cf2c..2e135c903c5 100644 --- a/libs/structs/structvar/view.go +++ b/libs/structs/structvar/view.go @@ -5,6 +5,7 @@ import ( "iter" "reflect" "slices" + "strings" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/structs/structpath" @@ -282,77 +283,140 @@ func (x View) field(info *StructInfo, sv reflect.Value, k string, fv reflect.Val } func (x View) hasFields(v reflect.Value) bool { - for range x.structFields(v) { - return true + return !x.eachField(v, func(string, bool, View) bool { return false }) +} + +// eachField calls fn for the represented fields of the struct v, in declaration order, +// until fn returns false. It returns false if fn did. fn also gets whether the key can +// be a dotted path component, so callers building paths need not check it. +// +// The traversal helpers take a callback rather than returning an iterator so that +// walking a value does not allocate a closure per visited node. +func (x View) eachField(v reflect.Value, fn func(string, bool, View) bool) bool { + info := GetStructInfo(v.Type()) + + // Mark the fields listed in the struct's ForceSendFields, read once per visit. + var forcedBuf [128]bool + var forced []bool + if len(info.List) <= len(forcedBuf) { + forced = forcedBuf[:len(info.List)] + } else { + forced = make([]bool, len(info.List)) + } + for g, index := range info.ForceSendGroups { + names := FieldByIndex(v, index) + if !names.IsValid() { + continue + } + for j := range names.Len() { + if i, ok := info.ForceSendTargets[g][names.Index(j).String()]; ok { + forced[i] = true + } + } + } + + for i := range info.List { + f := &info.List[i] + fv := FieldByIndex(v, f.Index) + if !fv.IsValid() { + continue + } + isForced := forced[i] || !fv.IsZero() + if !isForced && x.isAbsentZeroField(fv, f.Key) { + continue + } + child, ok := x.field(info, v, f.Key, fv, isForced) + if ok && !fn(f.Key, f.Dot, child) { + return false + } + } + return true +} + +// containsString reports whether the []string value s (possibly invalid) contains name. +// It iterates with reflect: converting s with Interface would allocate. +func containsString(s reflect.Value, name string) bool { + if !s.IsValid() { + return false + } + for i := range s.Len() { + if s.Index(i).String() == name { + return true + } } return false } -func (x View) structFields(v reflect.Value) iter.Seq2[string, View] { - return func(yield func(string, View) bool) { - info := GetStructInfo(v.Type()) - for fv := range info.FieldValues(v) { - child, ok := x.field(&info, v, fv.Key, fv.Value, fv.IsForced) - if ok && !yield(fv.Key, child) { - return +// isAbsentZeroField reports whether the zero, unforced field fv (key k) is absent +// without building its view: [View.field] would find it absent, because it is neither +// a struct nor an interface (whose presence depends on their contents), has no +// location and holds no reference. +func (x View) isAbsentZeroField(fv reflect.Value, k string) bool { + switch fv.Kind() { + case reflect.Struct, reflect.Interface: + return false + default: + } + if x.loc.Key(k) != nil { + return false + } + return len(x.refs) == 0 || x.refs[structpath.NewStringKey(x.path, k).String()] == "" +} + +// eachMapItem calls fn for the entries of a map value (struct fields in declaration +// order, map entries in key order) until fn returns false. It returns false if fn did. +func (x View) eachMapItem(fn func(string, bool, View) bool) bool { + if x.ref() != "" { + return true + } + v, _ := x.deref() + switch { + case !v.IsValid(): + case v.Kind() == reflect.Struct && !IsSDKNativeType(v.Type()): + return x.eachField(v, fn) + case v.Kind() == reflect.Map: + keys := v.MapKeys() + slices.SortFunc(keys, func(a, b reflect.Value) int { + return strings.Compare(a.String(), b.String()) + }) + for _, key := range keys { + ev := v.MapIndex(key) + k := key.String() + if !fn(k, false, x.childKey(ev, k, elemOptions(ev))) { + return false } } } + return true +} + +// eachElem calls fn for the elements of a sequence value until fn returns false. +// It returns false if fn did. +func (x View) eachElem(fn func(int, View) bool) bool { + if x.Kind() != KindSequence { + return true + } + v, _ := x.deref() + for i := range v.Len() { + ev := v.Index(i) + if !fn(i, x.childIndex(ev, i, elemOptions(ev))) { + return false + } + } + return true } // MapItems returns the entries of a map value: struct fields in declaration order, // map entries in key order. func (x View) MapItems() iter.Seq2[string, View] { return func(yield func(string, View) bool) { - if x.ref() != "" { - return - } - v, _ := x.deref() - switch { - case !v.IsValid(): - case v.Kind() == reflect.Struct && !IsSDKNativeType(v.Type()): - for k, c := range x.structFields(v) { - if !yield(k, c) { - return - } - } - case v.Kind() == reflect.Map: - keys := v.MapKeys() - slices.SortFunc(keys, func(a, b reflect.Value) int { - switch { - case a.String() < b.String(): - return -1 - case a.String() > b.String(): - return 1 - } - return 0 - }) - for _, key := range keys { - ev := v.MapIndex(key) - child := x.childKey(ev, key.String(), elemOptions(ev)) - if !yield(key.String(), child) { - return - } - } - } + x.eachMapItem(func(k string, _ bool, c View) bool { return yield(k, c) }) } } // Sequence returns the elements of a sequence value. func (x View) Sequence() iter.Seq2[int, View] { - return func(yield func(int, View) bool) { - if x.Kind() != KindSequence { - return - } - v, _ := x.deref() - for i := range v.Len() { - ev := v.Index(i) - child := x.childIndex(ev, i, elemOptions(ev)) - if !yield(i, child) { - return - } - } - } + return func(yield func(int, View) bool) { x.eachElem(yield) } } // Get returns the value of key k of a map value, or an invalid value. @@ -373,7 +437,7 @@ func (x View) Get(k string) View { return View{} } isForced := !fv.IsZero() || info.IsForceSend(v, k) - child, ok := x.field(&info, v, k, fv, isForced) + child, ok := x.field(info, v, k, fv, isForced) if !ok { return View{} } diff --git a/libs/structs/structvar/walk.go b/libs/structs/structvar/walk.go index b7a11a91499..76b1b849a9e 100644 --- a/libs/structs/structvar/walk.go +++ b/libs/structs/structvar/walk.go @@ -24,22 +24,27 @@ func walk(x View, p *structpath.PathNode, fn func(*structpath.PathNode, View) er return err } + var err error switch x.Kind() { case KindMap: - for k, c := range x.MapItems() { - if err := walk(c, structpath.NewStringKey(p, k), fn); err != nil { - return err + x.eachMapItem(func(k string, dot bool, c View) bool { + var cp *structpath.PathNode + if dot { + cp = structpath.NewDotString(p, k) + } else { + cp = structpath.NewStringKey(p, k) } - } + err = walk(c, cp, fn) + return err == nil + }) case KindSequence: - for i, c := range x.Sequence() { - if err := walk(c, structpath.NewIndex(p, i), fn); err != nil { - return err - } - } + x.eachElem(func(i int, c View) bool { + err = walk(c, structpath.NewIndex(p, i), fn) + return err == nil + }) default: } - return nil + return err } // ErrSkip can be returned by the function passed to [Walk] to skip the subtree of the diff --git a/libs/structs/structvar/write.go b/libs/structs/structvar/write.go index 6eba0b8dda6..14ce1494400 100644 --- a/libs/structs/structvar/write.go +++ b/libs/structs/structvar/write.go @@ -107,7 +107,7 @@ func (sv *StructVar) Delete(path *structpath.PathNode) error { if f := FieldByIndex(v, index); f.IsValid() { f.SetZero() } - removeForceSend(v, &info, last.key) + removeForceSend(v, info, last.key) default: return fmt.Errorf("cannot delete %s", path) } @@ -329,7 +329,7 @@ func updateChild(v reflect.Value, c pathComponent, fn func(reflect.Value) (bool, return err } if zero { - addForceSend(v, &info, c.key) + addForceSend(v, info, c.key) } return nil case c.isKey && v.Kind() == reflect.Map: