diff --git a/libs/dyn/convert/from_typed.go b/libs/dyn/convert/from_typed.go index 66451124293..2bc931b7b5d 100644 --- a/libs/dyn/convert/from_typed.go +++ b/libs/dyn/convert/from_typed.go @@ -6,7 +6,7 @@ import ( "slices" "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/dynvar" + "github.com/databricks/cli/libs/structs/structvar" ) type fromTypedOptions int @@ -96,7 +96,7 @@ func fromTypedStruct(src reflect.Value, ref dyn.Value, options ...fromTypedOptio switch ref.Kind() { case dyn.KindString: // Ignore pure variable references (e.g. ${var.foo}). - if dynvar.IsPureVariableReference(ref.MustString()) { + if structvar.IsPureVariableReference(ref.MustString()) { return ref, nil } case dyn.KindMap, dyn.KindNil: @@ -174,7 +174,7 @@ func fromTypedMap(src reflect.Value, ref dyn.Value) (dyn.Value, error) { switch ref.Kind() { case dyn.KindString: // Ignore pure variable references (e.g. ${var.foo}). - if dynvar.IsPureVariableReference(ref.MustString()) { + if structvar.IsPureVariableReference(ref.MustString()) { return ref, nil } case dyn.KindMap, dyn.KindNil: @@ -222,7 +222,7 @@ func fromTypedSlice(src reflect.Value, ref dyn.Value) (dyn.Value, error) { switch ref.Kind() { case dyn.KindString: // Ignore pure variable references (e.g. ${var.foo}). - if dynvar.IsPureVariableReference(ref.MustString()) { + if structvar.IsPureVariableReference(ref.MustString()) { return ref, nil } case dyn.KindSequence, dyn.KindNil: @@ -297,7 +297,7 @@ func fromTypedBool(src reflect.Value, ref dyn.Value, options ...fromTypedOptions return dyn.V(src.Bool()), nil case dyn.KindString: // Ignore pure variable references (e.g. ${var.foo}). - if dynvar.IsPureVariableReference(ref.MustString()) { + if structvar.IsPureVariableReference(ref.MustString()) { return ref, nil } default: @@ -324,7 +324,7 @@ func fromTypedInt(src reflect.Value, ref dyn.Value, options ...fromTypedOptions) return dyn.V(src.Int()), nil case dyn.KindString: // Ignore pure variable references (e.g. ${var.foo}). - if dynvar.IsPureVariableReference(ref.MustString()) { + if structvar.IsPureVariableReference(ref.MustString()) { return ref, nil } default: @@ -351,7 +351,7 @@ func fromTypedFloat(src reflect.Value, ref dyn.Value, options ...fromTypedOption return dyn.V(src.Float()), nil case dyn.KindString: // Ignore pure variable references (e.g. ${var.foo}). - if dynvar.IsPureVariableReference(ref.MustString()) { + if structvar.IsPureVariableReference(ref.MustString()) { return ref, nil } default: diff --git a/libs/dyn/convert/normalize.go b/libs/dyn/convert/normalize.go index e41ed4a0a87..af0c629da87 100644 --- a/libs/dyn/convert/normalize.go +++ b/libs/dyn/convert/normalize.go @@ -8,7 +8,7 @@ import ( "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/dynvar" + "github.com/databricks/cli/libs/structs/structvar" ) // NormalizeOption is the type for options that can be passed to Normalize. @@ -242,7 +242,7 @@ func (n normalizeOptions) normalizeStruct(typ reflect.Type, src dyn.Value, seen case dyn.KindString: // Return verbatim if it's a pure variable reference. - if dynvar.IsPureVariableReference(src.MustString()) { + if structvar.IsPureVariableReference(src.MustString()) { return src, nil } default: @@ -282,7 +282,7 @@ func (n normalizeOptions) normalizeMap(typ reflect.Type, src dyn.Value, seen []r case dyn.KindString: // Return verbatim if it's a pure variable reference. - if dynvar.IsPureVariableReference(src.MustString()) { + if structvar.IsPureVariableReference(src.MustString()) { return src, nil } default: @@ -319,7 +319,7 @@ func (n normalizeOptions) normalizeSlice(typ reflect.Type, src dyn.Value, seen [ case dyn.KindString: // Return verbatim if it's a pure variable reference. - if dynvar.IsPureVariableReference(src.MustString()) { + if structvar.IsPureVariableReference(src.MustString()) { return src, nil } default: @@ -369,7 +369,7 @@ func (n normalizeOptions) normalizeBool(typ reflect.Type, src dyn.Value, path dy return dyn.NewValue(false, src.Locations()), nil default: // Return verbatim if it's a pure variable reference. - if dynvar.IsPureVariableReference(src.MustString()) { + if structvar.IsPureVariableReference(src.MustString()) { return src, nil } @@ -408,7 +408,7 @@ func (n normalizeOptions) normalizeInt(typ reflect.Type, src dyn.Value, path dyn out, err := strconv.ParseInt(src.MustString(), 10, 64) if err != nil { // Return verbatim if it's a pure variable reference. - if dynvar.IsPureVariableReference(src.MustString()) { + if structvar.IsPureVariableReference(src.MustString()) { return src, nil } @@ -452,7 +452,7 @@ func (n normalizeOptions) normalizeFloat(typ reflect.Type, src dyn.Value, path d out, err := strconv.ParseFloat(src.MustString(), 64) if err != nil { // Return verbatim if it's a pure variable reference. - if dynvar.IsPureVariableReference(src.MustString()) { + if structvar.IsPureVariableReference(src.MustString()) { return src, nil } diff --git a/libs/dyn/convert/sdk_native_types.go b/libs/dyn/convert/sdk_native_types.go index b67d9b2b1e3..89c36327d18 100644 --- a/libs/dyn/convert/sdk_native_types.go +++ b/libs/dyn/convert/sdk_native_types.go @@ -7,7 +7,7 @@ import ( "slices" "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/dynvar" + "github.com/databricks/cli/libs/structs/structvar" sdkduration "github.com/databricks/databricks-sdk-go/common/types/duration" sdkfieldmask "github.com/databricks/databricks-sdk-go/common/types/fieldmask" sdktime "github.com/databricks/databricks-sdk-go/common/types/time" @@ -30,7 +30,7 @@ func fromTypedSDKNative(src reflect.Value, ref dyn.Value, options ...fromTypedOp switch ref.Kind() { case dyn.KindString: // Ignore pure variable references (e.g. ${var.foo}). - if dynvar.IsPureVariableReference(ref.MustString()) { + if structvar.IsPureVariableReference(ref.MustString()) { return ref, nil } case dyn.KindNil: @@ -72,7 +72,7 @@ func toTypedSDKNative(dst reflect.Value, src dyn.Value) error { switch src.Kind() { case dyn.KindString: // Ignore pure variable references (e.g. ${var.foo}). - if dynvar.IsPureVariableReference(src.MustString()) { + if structvar.IsPureVariableReference(src.MustString()) { dst.SetZero() return nil } diff --git a/libs/dyn/convert/to_typed.go b/libs/dyn/convert/to_typed.go index 87b0fe7d8ca..29f325aea9e 100644 --- a/libs/dyn/convert/to_typed.go +++ b/libs/dyn/convert/to_typed.go @@ -7,7 +7,7 @@ import ( "strconv" "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/dynvar" + "github.com/databricks/cli/libs/structs/structvar" ) // Populate a destination typed value from a source dynamic value. @@ -118,7 +118,7 @@ func toTypedStruct(dst reflect.Value, src dyn.Value) error { return nil case dyn.KindString: // Ignore pure variable references (e.g. ${var.foo}). - if dynvar.IsPureVariableReference(src.MustString()) { + if structvar.IsPureVariableReference(src.MustString()) { dst.SetZero() return nil } @@ -175,7 +175,7 @@ func toTypedMap(dst reflect.Value, src dyn.Value) error { return nil case dyn.KindString: // Ignore pure variable references (e.g. ${var.foo}). - if dynvar.IsPureVariableReference(src.MustString()) { + if structvar.IsPureVariableReference(src.MustString()) { dst.SetZero() return nil } @@ -208,7 +208,7 @@ func toTypedSlice(dst reflect.Value, src dyn.Value) error { return nil case dyn.KindString: // Ignore pure variable references (e.g. ${var.foo}). - if dynvar.IsPureVariableReference(src.MustString()) { + if structvar.IsPureVariableReference(src.MustString()) { dst.SetZero() return nil } @@ -262,7 +262,7 @@ func toTypedBool(dst reflect.Value, src dyn.Value) error { return nil } // Ignore pure variable references (e.g. ${var.foo}). - if dynvar.IsPureVariableReference(src.MustString()) { + if structvar.IsPureVariableReference(src.MustString()) { dst.SetZero() return nil } @@ -300,7 +300,7 @@ func toTypedInt(dst reflect.Value, src dyn.Value) error { return nil } // Ignore pure variable references (e.g. ${var.foo}). - if dynvar.IsPureVariableReference(src.MustString()) { + if structvar.IsPureVariableReference(src.MustString()) { dst.SetZero() return nil } @@ -325,7 +325,7 @@ func toTypedFloat(dst reflect.Value, src dyn.Value) error { return nil } // Ignore pure variable references (e.g. ${var.foo}). - if dynvar.IsPureVariableReference(src.MustString()) { + if structvar.IsPureVariableReference(src.MustString()) { dst.SetZero() return nil } diff --git a/libs/dyn/dynvar/lookup.go b/libs/dyn/dynvar/lookup.go deleted file mode 100644 index 2bc08f47d47..00000000000 --- a/libs/dyn/dynvar/lookup.go +++ /dev/null @@ -1,21 +0,0 @@ -package dynvar - -import ( - "errors" - - "github.com/databricks/cli/libs/dyn" -) - -// Lookup is the type of lookup functions that can be used with [Resolve]. -type Lookup func(path dyn.Path) (dyn.Value, error) - -// ErrSkipResolution is returned by a lookup function to indicate that the -// resolution of a variable reference should be skipped. -var ErrSkipResolution = errors.New("skip resolution") - -// DefaultLookup is the default lookup function used by [Resolve]. -func DefaultLookup(in dyn.Value) Lookup { - return func(path dyn.Path) (dyn.Value, error) { - return dyn.GetByPath(in, path) - } -} diff --git a/libs/dyn/dynvar/lookup_test.go b/libs/dyn/dynvar/lookup_test.go deleted file mode 100644 index 2341d72084b..00000000000 --- a/libs/dyn/dynvar/lookup_test.go +++ /dev/null @@ -1,27 +0,0 @@ -package dynvar_test - -import ( - "testing" - - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/dynvar" - "github.com/stretchr/testify/assert" -) - -func TestDefaultLookup(t *testing.T) { - lookup := dynvar.DefaultLookup(dyn.V(map[string]dyn.Value{ - "a": dyn.V("a"), - "b": dyn.V("b"), - })) - - v1, err := lookup(dyn.NewPath(dyn.Key("a"))) - assert.NoError(t, err) - assert.Equal(t, dyn.V("a"), v1) - - v2, err := lookup(dyn.NewPath(dyn.Key("b"))) - assert.NoError(t, err) - assert.Equal(t, dyn.V("b"), v2) - - _, err = lookup(dyn.NewPath(dyn.Key("c"))) - assert.True(t, dyn.IsNoSuchKeyError(err)) -} diff --git a/libs/dyn/dynvar/ref.go b/libs/dyn/dynvar/ref.go deleted file mode 100644 index de25f44a212..00000000000 --- a/libs/dyn/dynvar/ref.go +++ /dev/null @@ -1,183 +0,0 @@ -package dynvar - -import ( - "fmt" - "regexp" - "strings" - - "github.com/databricks/cli/libs/dyn" -) - -// BaseVarDef matches a single dot-separated path segment in ${...} references. -// A segment may have leading underscores before the first letter, and may contain -// letters, digits, hyphens and underscores after that. -// Keep in sync with _base_var_def in python/databricks/bundles/core/_transform.py. -// Behavioral parity is enforced by testdata/reference_vectors.json. -const BaseVarDef = `_*\p{L}+([-_]*[\p{L}\p{N}]+)*` - -var re = regexp.MustCompile(fmt.Sprintf(`\$\{(%s(\.%s(\[[0-9]+\])*)*(\[[0-9]+\])*)\}`, BaseVarDef, BaseVarDef)) - -// Ref represents a variable reference. -// It is a string [dyn.Value] contained in a larger [dyn.Value]. -// Its path within the containing [dyn.Value] is also stored. -type Ref struct { - // Original value. - Value dyn.Value - - // String value in the original [dyn.Value]. - Str string - - // Matches of the variable reference in the string. - Matches [][]string - - // Byte offsets [start, end) of each entry in Matches within Str. Interpolation - // must substitute by offset: an escaped occurrence of the same reference text - // may appear earlier in Str, and a search-based replacement would hit that one. - Spans [][2]int -} - -// NewRef returns a new Ref if the given [dyn.Value] contains a string -// with one or more variable references. It returns false if the given -// [dyn.Value] does not contain variable references. -// -// A reference preceded by "$" is treated as an escaped literal and is not -// included in the returned Ref. For example, "$${foo}" contains no references -// because the "${foo}" part is escaped by the leading "$". -// -// Examples of a valid variable references: -// - "${a.b}" -// - "${a.b.c}" -// - "${a.b[0].c}" -// - "${a} ${b} ${c}" -func NewRef(v dyn.Value) (Ref, bool) { - s, ok := v.AsString() - if !ok { - return Ref{}, false - } - - // Find all matches with their byte positions so we can detect escaped references. - indices := re.FindAllStringSubmatchIndex(s, -1) - if len(indices) == 0 { - return Ref{}, false - } - - // Convert position-based matches to string matches, filtering out references - // escaped with a leading "$" (e.g. "$${foo}" should not match "${foo}"). - var m [][]string - var spans [][2]int - for _, idx := range indices { - if isEscapedAt(s, idx[0]) { - continue - } - match := make([]string, len(idx)/2) - for i := range len(idx) / 2 { - if idx[2*i] >= 0 { - match[i] = s[idx[2*i]:idx[2*i+1]] - } - } - m = append(m, match) - spans = append(spans, [2]int{idx[0], idx[1]}) - } - - if len(m) == 0 { - return Ref{}, false - } - - return Ref{ - Value: v, - Str: s, - Matches: m, - Spans: spans, - }, true -} - -// isEscapedAt reports whether the reference starting at byte offset start in s is -// escaped by an immediately preceding "$". -func isEscapedAt(s string, start int) bool { - return start > 0 && s[start-1] == '$' -} - -// Unescape turns escaped references into literal ones ("$${x}" -> "${x}"). Call it -// when building a value to send to the API: the extra "$" exists only to hide the -// reference from bundle variable resolution. -func Unescape(s string) string { - return strings.ReplaceAll(s, "$${", "${") -} - -// ReplaceRef substitutes every unescaped occurrence of reference (a full "${...}" -// string) in s with value. Escaped occurrences are left as they are, so -// strings.ReplaceAll is not a valid substitute: it would also rewrite the literal. -func ReplaceRef(s, reference, value string) string { - var sb strings.Builder - for pos := 0; ; { - i := strings.Index(s[pos:], reference) - if i < 0 { - sb.WriteString(s[pos:]) - return sb.String() - } - start := pos + i - sb.WriteString(s[pos:start]) - if isEscapedAt(s, start) { - sb.WriteString(reference) - } else { - sb.WriteString(value) - } - pos = start + len(reference) - } -} - -// IsPure returns true if the variable reference contains a single -// variable reference and nothing more. We need this so we can -// interpolate values of non-string types (i.e. it can be substituted). -func (v Ref) IsPure() bool { - // Need single match, equal to the incoming string. - if len(v.Matches) == 0 || len(v.Matches[0]) == 0 { - panic("invalid variable reference; expect at least one match") - } - return v.Matches[0][0] == v.Str -} - -func (v Ref) References() []string { - var out []string - for _, m := range v.Matches { - out = append(out, m[1]) - } - return out -} - -func IsPureVariableReference(s string) bool { - return len(s) > 0 && re.FindString(s) == s -} - -// ContainsVariableReference reports whether s contains at least one reference that -// is not escaped. Callers use it to decide whether a string still needs resolution, -// so an escaped "$${foo}" must not count: it is a literal, and treating it as -// pending leaves the value permanently unresolved. -func ContainsVariableReference(s string) bool { - for _, loc := range re.FindAllStringIndex(s, -1) { - if !isEscapedAt(s, loc[0]) { - return true - } - } - return false -} - -// If s is a pure variable reference, this function returns the corresponding -// dyn.Path. Otherwise, it returns false. -func PureReferenceToPath(s string) (dyn.Path, bool) { - ref, ok := NewRef(dyn.V(s)) - if !ok { - return nil, false - } - - if !ref.IsPure() { - return nil, false - } - - p, err := dyn.NewPathFromString(ref.References()[0]) - if err != nil { - return nil, false - } - - return p, true -} diff --git a/libs/dyn/dynvar/ref_test.go b/libs/dyn/dynvar/ref_test.go deleted file mode 100644 index 7b93ac633b9..00000000000 --- a/libs/dyn/dynvar/ref_test.go +++ /dev/null @@ -1,65 +0,0 @@ -package dynvar - -import ( - "testing" - - "github.com/databricks/cli/libs/dyn" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestNewRefNoString(t *testing.T) { - _, ok := NewRef(dyn.V(1)) - require.False(t, ok, "should not match non-string") -} - -func TestNewRefSpans(t *testing.T) { - // Spans must point at the unescaped occurrence, not the escaped one that - // appears earlier with identical text. - ref, ok := NewRef(dyn.V("$${a.b} ${a.b}")) - require.True(t, ok) - require.Len(t, ref.Matches, 1) - require.Len(t, ref.Spans, 1) - assert.Equal(t, "${a.b}", ref.Str[ref.Spans[0][0]:ref.Spans[0][1]]) - assert.Equal(t, 8, ref.Spans[0][0]) -} - -func TestContainsVariableReference(t *testing.T) { - tests := map[string]bool{ - "${a.b}": true, - "x ${a.b} y": true, - "$${a.b}": false, - "$${a.b} $${c.d}": false, - "$${a.b} ${c.d}": true, - "no refs": false, - "": false, - } - for in, want := range tests { - assert.Equal(t, want, ContainsVariableReference(in), "input %q", in) - } -} - -func TestUnescape(t *testing.T) { - assert.Equal(t, "${a.b}", Unescape("$${a.b}")) - assert.Equal(t, "${a} ${b}", Unescape("$${a} $${b}")) - assert.Equal(t, "${a.b}", Unescape("${a.b}"), "already unescaped is unchanged") - assert.Equal(t, "no refs", Unescape("no refs")) -} - -func TestReplaceRef(t *testing.T) { - tests := []struct { - in string - want string - }{ - {"${a.b}", "V"}, - {"$${a.b}", "$${a.b}"}, - {"$${a.b} ${a.b}", "$${a.b} V"}, - {"${a.b} $${a.b}", "V $${a.b}"}, - {"${a.b}-${a.b}", "V-V"}, - {"$${a.b} ${a.b} $${a.b}", "$${a.b} V $${a.b}"}, - {"untouched ${c.d}", "untouched ${c.d}"}, - } - for _, tt := range tests { - assert.Equal(t, tt.want, ReplaceRef(tt.in, "${a.b}", "V"), "input %q", tt.in) - } -} diff --git a/libs/dyn/dynvar/ref_vectors_test.go b/libs/dyn/dynvar/ref_vectors_test.go deleted file mode 100644 index d0084a6ae5c..00000000000 --- a/libs/dyn/dynvar/ref_vectors_test.go +++ /dev/null @@ -1,69 +0,0 @@ -package dynvar - -import ( - "encoding/json" - "os" - "path/filepath" - "testing" - - "github.com/databricks/cli/libs/dyn" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -type referenceVectorsFile struct { - Vectors []referenceVector `json:"vectors"` -} - -type referenceVector struct { - ID string `json:"id"` - Input string `json:"input"` - Match bool `json:"match"` - References []string `json:"references,omitempty"` - Pure *bool `json:"pure,omitempty"` - Path *string `json:"path,omitempty"` - PathOK *bool `json:"path_ok,omitempty"` -} - -func loadReferenceVectors(t *testing.T) []referenceVector { - t.Helper() - - b, err := os.ReadFile(filepath.Join("testdata", "reference_vectors.json")) - require.NoError(t, err) - - var file referenceVectorsFile - require.NoError(t, json.Unmarshal(b, &file)) - require.NotEmpty(t, file.Vectors) - - return file.Vectors -} - -func TestReferenceVectors(t *testing.T) { - for _, v := range loadReferenceVectors(t) { - t.Run(v.ID, func(t *testing.T) { - ref, ok := NewRef(dyn.V(v.Input)) - assert.Equal(t, v.Match, ok, "NewRef match") - - if v.Match { - require.NotEmpty(t, ref.References()) - if v.References != nil { - assert.Equal(t, v.References, ref.References(), "References") - } - } - - if v.Pure != nil { - assert.Equal(t, *v.Pure, IsPureVariableReference(v.Input), "IsPureVariableReference") - } - - switch { - case v.Path != nil: - path, ok := PureReferenceToPath(v.Input) - require.True(t, ok, "PureReferenceToPath") - assert.Equal(t, dyn.MustPathFromString(*v.Path), path) - case v.PathOK != nil: - _, ok := PureReferenceToPath(v.Input) - assert.Equal(t, *v.PathOK, ok, "PureReferenceToPath") - } - }) - } -} diff --git a/libs/dyn/dynvar/resolve.go b/libs/dyn/dynvar/resolve.go deleted file mode 100644 index 99981a2e51b..00000000000 --- a/libs/dyn/dynvar/resolve.go +++ /dev/null @@ -1,255 +0,0 @@ -package dynvar - -import ( - "errors" - "fmt" - "maps" - "slices" - "strings" - - "github.com/databricks/cli/libs/dyn" -) - -// Resolve resolves variable references in the given input value using the provided lookup function. -// It returns the resolved output value and any error encountered during the resolution process. -// -// For example, given the input value: -// -// { -// "a": "a", -// "b": "${a}", -// "c": "${b}${b}", -// } -// -// The output value will be: -// -// { -// "a": "a", -// "b": "a", -// "c": "aa", -// } -// -// If the input value contains a variable reference that cannot be resolved, an error is returned. -// If a cycle is detected in the variable references, an error is returned. -// If for some path the resolution function returns [ErrSkipResolution], the variable reference is left in place. -// This is useful when some variable references are not yet ready to be interpolated. -func Resolve(in dyn.Value, fn Lookup) (out dyn.Value, err error) { - return resolver{in: in, fn: fn}.run() -} - -// ReferenceError is returned for an unresolved variable reference. Suggestions -// are carried as data so callers (which can import libs/diag) format them. -type ReferenceError struct { - Reference string // original reference text, e.g. "var.hst" - Suggestions []string // corrected references, e.g. ["var.host", "var.hosts"] -} - -func (e *ReferenceError) Error() string { - return fmt.Sprintf("reference does not exist: ${%s}", e.Reference) -} - -type lookupResult struct { - v dyn.Value - err error -} - -type resolver struct { //nolint:recvcheck // value receiver for run(), pointer for mutation methods - in dyn.Value - fn Lookup - - refs map[string]Ref - resolved map[string]dyn.Value - - // Memoization for lookups. - lookups map[string]lookupResult -} - -func (r resolver) run() (out dyn.Value, err error) { - err = r.collectVariableReferences() - if err != nil { - return dyn.InvalidValue, err - } - - err = r.resolveVariableReferences() - if err != nil { - return dyn.InvalidValue, err - } - - out, err = r.replaceVariableReferences() - if err != nil { - return dyn.InvalidValue, err - } - - return out, nil -} - -func (r *resolver) collectVariableReferences() (err error) { - r.refs = make(map[string]Ref) - - // First walk the input to gather all values with a variable reference. - _, err = dyn.Walk(r.in, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - ref, ok := NewRef(v) - if !ok { - // Skip values without variable references. - return v, nil - } - - r.refs[p.String()] = ref - return v, nil - }) - - return err -} - -func (r *resolver) resolveVariableReferences() (err error) { - // Initialize cache for lookups. - r.lookups = make(map[string]lookupResult) - - // Initialize cache for resolved variable references. - r.resolved = make(map[string]dyn.Value) - - // Resolve each variable reference (in order). - // We sort the keys here to ensure that we always resolve the same variable reference first. - // This is done such that the cycle detection error is deterministic. If we did not do this, - // we could enter the cycle at any point in the cycle and return varying errors. - keys := slices.Sorted(maps.Keys(r.refs)) - for _, key := range keys { - v, err := r.resolveRef(r.refs[key], []string{key}) - if err != nil { - return err - } - r.resolved[key] = v - } - - return nil -} - -func (r *resolver) resolveRef(ref Ref, seen []string) (dyn.Value, error) { - // This is an unresolved variable reference. - deps := ref.References() - - // Resolve each of the dependencies, then interpolate them in the ref. - resolved := make([]dyn.Value, len(deps)) - complete := true - - for j, dep := range deps { - // Cycle detection. - if slices.Contains(seen, dep) { - return dyn.InvalidValue, fmt.Errorf( - "cycle detected in field resolution: %s", - strings.Join(append(seen, dep), " -> "), - ) - } - - v, err := r.resolveKey(dep, append(seen, dep)) - - // If we should skip resolution of this key, index j will hold an invalid [dyn.Value]. - if errors.Is(err, ErrSkipResolution) { - complete = false - continue - } else if err != nil { - // Otherwise, propagate the error. - return dyn.InvalidValue, err - } - - resolved[j] = v - } - - // Interpolate the resolved values. - if ref.IsPure() && complete { - // If the variable reference is pure, we can substitute it. - // This is useful for interpolating values of non-string types. - // - // Note: we use the location of the variable reference to preserve the information - // of where it is used. This also means that relative path resolution is done - // relative to where a variable is used, not where it is defined. - // - return dyn.NewValue(resolved[0].Value(), ref.Value.Locations()), nil - } - - // Not pure; perform string interpolation. - // - // Substitute by byte offset rather than by searching for the match text: the same - // reference may also appear escaped ("$${foo} ${foo}"), and a search would replace - // that occurrence instead, corrupting the literal and leaving the real one unresolved. - var sb strings.Builder - consumed := 0 - for j := range ref.Matches { - // The value is invalid if resolution returned [ErrSkipResolution]. - // We must skip those and leave the original variable reference in place, - // which happens naturally by not advancing past its span. - if !resolved[j].IsValid() { - continue - } - - // Try to turn the resolved value into a string. - s, ok := resolved[j].AsString() - if !ok { - // Only allow primitive types to be converted to string. - switch resolved[j].Kind() { - case dyn.KindString, dyn.KindBool, dyn.KindInt, dyn.KindFloat, dyn.KindTime, dyn.KindNil: - s = fmt.Sprint(resolved[j].AsAny()) - default: - return dyn.InvalidValue, fmt.Errorf("cannot interpolate non-primitive value of type %s into string", resolved[j].Kind()) - } - } - - start, end := ref.Spans[j][0], ref.Spans[j][1] - sb.WriteString(ref.Str[consumed:start]) - sb.WriteString(s) - consumed = end - } - sb.WriteString(ref.Str[consumed:]) - - return dyn.NewValue(sb.String(), ref.Value.Locations()), nil -} - -func (r *resolver) resolveKey(key string, seen []string) (dyn.Value, error) { - // Check if we have already looked up this key. - if v, ok := r.lookups[key]; ok { - return v.v, v.err - } - - // Parse the key into a path. - p, err := dyn.NewPathFromString(key) - if err != nil { - return dyn.InvalidValue, err - } - - // Look up the value for the given key. - v, err := r.fn(p) - if err != nil { - if dyn.IsNoSuchKeyError(err) { - // Carry suggestions as data; the caller formats them via libs/diag. - err = &ReferenceError{Reference: key, Suggestions: dyn.SuggestedReferences(err, key)} - } - - // Cache the return value and return to the caller. - r.lookups[key] = lookupResult{v: dyn.InvalidValue, err: err} - return dyn.InvalidValue, err - } - - // If the returned value is a valid variable reference, resolve it. - ref, ok := NewRef(v) - if ok { - v, err = r.resolveRef(ref, seen) - } - - // Cache the return value and return to the caller. - r.lookups[key] = lookupResult{v: v, err: err} - return v, err -} - -func (r *resolver) replaceVariableReferences() (dyn.Value, error) { - // Walk the input and replace all variable references. - return dyn.Walk(r.in, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - nv, ok := r.resolved[p.String()] - if !ok { - // No variable reference; return the original value. - return v, nil - } - - // We have a variable reference; return the resolved value. - return nv, nil - }) -} diff --git a/libs/dyn/dynvar/resolve_test.go b/libs/dyn/dynvar/resolve_test.go deleted file mode 100644 index c4b28e88d3f..00000000000 --- a/libs/dyn/dynvar/resolve_test.go +++ /dev/null @@ -1,430 +0,0 @@ -package dynvar_test - -import ( - "testing" - - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/dynvar" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func getByPath(t *testing.T, v dyn.Value, path string) dyn.Value { - v, err := dyn.Get(v, path) - require.NoError(t, err) - return v -} - -func TestResolve(t *testing.T) { - in := dyn.V(map[string]dyn.Value{ - "a": dyn.V("a"), - "b": dyn.V("${a}"), - "c": dyn.V("${a}"), - }) - - out, err := dynvar.Resolve(in, dynvar.DefaultLookup(in)) - require.NoError(t, err) - - assert.Equal(t, "a", getByPath(t, out, "a").MustString()) - assert.Equal(t, "a", getByPath(t, out, "b").MustString()) - assert.Equal(t, "a", getByPath(t, out, "c").MustString()) -} - -func TestResolveNotFound(t *testing.T) { - in := dyn.V(map[string]dyn.Value{ - "b": dyn.V("${a}"), - }) - - _, err := dynvar.Resolve(in, dynvar.DefaultLookup(in)) - require.ErrorContains(t, err, `reference does not exist: ${a}`) -} - -func TestResolveNotFoundSuggestsCloseKey(t *testing.T) { - in := dyn.V(map[string]dyn.Value{ - "host": dyn.V("example.com"), - "b": dyn.V("${hst}"), - }) - - _, err := dynvar.Resolve(in, dynvar.DefaultLookup(in)) - require.ErrorContains(t, err, "reference does not exist: ${hst}") - - var refErr *dynvar.ReferenceError - require.ErrorAs(t, err, &refErr) - assert.Equal(t, []string{"host"}, refErr.Suggestions) -} - -func TestResolveWithNesting(t *testing.T) { - in := dyn.V(map[string]dyn.Value{ - "a": dyn.V("${f.a}"), - "f": dyn.V(map[string]dyn.Value{ - "a": dyn.V("a"), - "b": dyn.V("${f.a}"), - }), - }) - - out, err := dynvar.Resolve(in, dynvar.DefaultLookup(in)) - require.NoError(t, err) - - assert.Equal(t, "a", getByPath(t, out, "a").MustString()) - assert.Equal(t, "a", getByPath(t, out, "f.a").MustString()) - assert.Equal(t, "a", getByPath(t, out, "f.b").MustString()) -} - -func TestResolveWithRecursion(t *testing.T) { - in := dyn.V(map[string]dyn.Value{ - "a": dyn.V("a"), - "b": dyn.V("${a}"), - "c": dyn.V("${b}"), - }) - - out, err := dynvar.Resolve(in, dynvar.DefaultLookup(in)) - require.NoError(t, err) - - assert.Equal(t, "a", getByPath(t, out, "a").MustString()) - assert.Equal(t, "a", getByPath(t, out, "b").MustString()) - assert.Equal(t, "a", getByPath(t, out, "c").MustString()) -} - -func TestResolveWithRecursionLoop(t *testing.T) { - in := dyn.V(map[string]dyn.Value{ - "a": dyn.V("a"), - "b": dyn.V("${c}"), - "c": dyn.V("${d}"), - "d": dyn.V("${b}"), - }) - - _, err := dynvar.Resolve(in, dynvar.DefaultLookup(in)) - assert.ErrorContains(t, err, "cycle detected in field resolution: b -> c -> d -> b") -} - -func TestResolveWithRecursionLoopSelf(t *testing.T) { - in := dyn.V(map[string]dyn.Value{ - "a": dyn.V("${a}"), - }) - - _, err := dynvar.Resolve(in, dynvar.DefaultLookup(in)) - assert.ErrorContains(t, err, "cycle detected in field resolution: a -> a") -} - -func TestResolveWithStringConcatenation(t *testing.T) { - in := dyn.V(map[string]dyn.Value{ - "a": dyn.V("a"), - "b": dyn.V("b"), - "c": dyn.V("${a}${b}${a}"), - }) - - out, err := dynvar.Resolve(in, dynvar.DefaultLookup(in)) - require.NoError(t, err) - - assert.Equal(t, "a", getByPath(t, out, "a").MustString()) - assert.Equal(t, "b", getByPath(t, out, "b").MustString()) - assert.Equal(t, "aba", getByPath(t, out, "c").MustString()) -} - -func TestResolveWithEscapedReference(t *testing.T) { - in := dyn.V(map[string]dyn.Value{ - "a": dyn.V("A"), - "escaped_only": dyn.V("$${a}"), - "escaped_first": dyn.V("$${a} ${a}"), - "real_first": dyn.V("${a} $${a}"), - "surrounded": dyn.V("$${a} ${a} $${a}"), - "other_name": dyn.V("$${zz} ${a}"), - }) - - out, err := dynvar.Resolve(in, dynvar.DefaultLookup(in)) - require.NoError(t, err) - - // The escape is preserved here; it is removed when the API payload is built. - assert.Equal(t, "$${a}", getByPath(t, out, "escaped_only").MustString()) - assert.Equal(t, "$${a} A", getByPath(t, out, "escaped_first").MustString()) - assert.Equal(t, "A $${a}", getByPath(t, out, "real_first").MustString()) - assert.Equal(t, "$${a} A $${a}", getByPath(t, out, "surrounded").MustString()) - assert.Equal(t, "$${zz} A", getByPath(t, out, "other_name").MustString()) -} - -func TestResolveWithTypeInterpolation(t *testing.T) { - in := dyn.V(map[string]dyn.Value{ - "a": dyn.V(1), - "b": dyn.V(2), - "c": dyn.V("${a} ${b}"), - "float_val": dyn.V(3.14), - "bool_true": dyn.V(true), - "bool_false": dyn.V(false), - "time_val": dyn.V(dyn.MustTime("2024-01-01")), - "nil_val": dyn.NilValue, - // Test interpolation of different types in string templates - "float_interp": dyn.V("Value: ${float_val}"), - "bool_true_interp": dyn.V("Enabled: ${bool_true}"), - "bool_false_interp": dyn.V("Disabled: ${bool_false}"), - "time_interp": dyn.V("Date: ${time_val}"), - "nil_interp": dyn.V("Null: ${nil_val}"), - }) - - out, err := dynvar.Resolve(in, dynvar.DefaultLookup(in)) - require.NoError(t, err) - - // Integer interpolation - assert.Equal(t, "1 2", getByPath(t, out, "c").MustString()) - - // Float interpolation - assert.Equal(t, "Value: 3.14", getByPath(t, out, "float_interp").MustString()) - - // Bool interpolation - assert.Equal(t, "Enabled: true", getByPath(t, out, "bool_true_interp").MustString()) - assert.Equal(t, "Disabled: false", getByPath(t, out, "bool_false_interp").MustString()) - - // Time interpolation should convert to string representation of time.Time - // Note: time.Time string representation includes timezone info - timeResult := getByPath(t, out, "time_interp").MustString() - assert.Contains(t, timeResult, "Date: 2024-01-01") - assert.Contains(t, timeResult, "00:00:00") - - // Nil interpolation - assert.Equal(t, "Null: ", getByPath(t, out, "nil_interp").MustString()) -} - -func TestResolveWithTypeRetentionFailure(t *testing.T) { - // Test that mapping interpolation fails with an error - mappingTest := dyn.V(map[string]dyn.Value{ - "mapping": dyn.V(map[string]dyn.Value{"key": dyn.V("value")}), - "interp": dyn.V("Config: ${mapping}"), - }) - _, err := dynvar.Resolve(mappingTest, dynvar.DefaultLookup(mappingTest)) - require.Error(t, err) - assert.Equal(t, "cannot interpolate non-primitive value of type map into string", err.Error()) -} - -func TestResolveWithTypeRetention(t *testing.T) { - in := dyn.V(map[string]dyn.Value{ - "int": dyn.V(1), - "int_var": dyn.V("${int}"), - "bool_true": dyn.V(true), - "bool_true_var": dyn.V("${bool_true}"), - "bool_false": dyn.V(false), - "bool_false_var": dyn.V("${bool_false}"), - "float": dyn.V(1.0), - "float_var": dyn.V("${float}"), - "string": dyn.V("a"), - "string_var": dyn.V("${string}"), - }) - - out, err := dynvar.Resolve(in, dynvar.DefaultLookup(in)) - require.NoError(t, err) - - assert.EqualValues(t, 1, getByPath(t, out, "int").MustInt()) - assert.EqualValues(t, 1, getByPath(t, out, "int_var").MustInt()) - - assert.True(t, getByPath(t, out, "bool_true").MustBool()) - assert.True(t, getByPath(t, out, "bool_true_var").MustBool()) - - assert.False(t, getByPath(t, out, "bool_false").MustBool()) - assert.False(t, getByPath(t, out, "bool_false_var").MustBool()) - - assert.Zero(t, 1.0-getByPath(t, out, "float").MustFloat()) - assert.Zero(t, 1.0-getByPath(t, out, "float_var").MustFloat()) - - assert.Equal(t, "a", getByPath(t, out, "string").MustString()) - assert.Equal(t, "a", getByPath(t, out, "string_var").MustString()) -} - -func TestResolveWithSkip(t *testing.T) { - in := dyn.V(map[string]dyn.Value{ - "a": dyn.V("a"), - "b": dyn.V("b"), - "c": dyn.V("${a}"), - "d": dyn.V("${b}"), - "e": dyn.V("${a} ${b}"), - "f": dyn.V("${b} ${a} ${a} ${b}"), - }) - - fallback := dynvar.DefaultLookup(in) - ignore := func(path dyn.Path) (dyn.Value, error) { - // If the variable reference to look up starts with "b", skip it. - if path.HasPrefix(dyn.NewPath(dyn.Key("b"))) { - return dyn.InvalidValue, dynvar.ErrSkipResolution - } - return fallback(path) - } - - out, err := dynvar.Resolve(in, ignore) - require.NoError(t, err) - - assert.Equal(t, "a", getByPath(t, out, "a").MustString()) - assert.Equal(t, "b", getByPath(t, out, "b").MustString()) - assert.Equal(t, "a", getByPath(t, out, "c").MustString()) - - // Check that the skipped variable references are not interpolated. - assert.Equal(t, "${b}", getByPath(t, out, "d").MustString()) - assert.Equal(t, "a ${b}", getByPath(t, out, "e").MustString()) - assert.Equal(t, "${b} a a ${b}", getByPath(t, out, "f").MustString()) //nolint:dupword -} - -func TestResolveWithSkipEverything(t *testing.T) { - in := dyn.V(map[string]dyn.Value{ - "a": dyn.V("a"), - "b": dyn.V("b"), - "c": dyn.V("${a}"), - "d": dyn.V("${b}"), - "e": dyn.V("${a} ${b}"), - "f": dyn.V("${b} ${a} ${a} ${b}"), - "g": dyn.V("${d} ${c} ${c} ${d}"), - }) - - // The call must not replace anything if the lookup function returns ErrSkipResolution. - out, err := dynvar.Resolve(in, func(path dyn.Path) (dyn.Value, error) { - return dyn.InvalidValue, dynvar.ErrSkipResolution - }) - require.NoError(t, err) - assert.Equal(t, "a", getByPath(t, out, "a").MustString()) - assert.Equal(t, "b", getByPath(t, out, "b").MustString()) - assert.Equal(t, "${a}", getByPath(t, out, "c").MustString()) - assert.Equal(t, "${b}", getByPath(t, out, "d").MustString()) - assert.Equal(t, "${a} ${b}", getByPath(t, out, "e").MustString()) - assert.Equal(t, "${b} ${a} ${a} ${b}", getByPath(t, out, "f").MustString()) - assert.Equal(t, "${d} ${c} ${c} ${d}", getByPath(t, out, "g").MustString()) -} - -func TestResolveWithInterpolateNewRef(t *testing.T) { - in := dyn.V(map[string]dyn.Value{ - "a": dyn.V("a"), - "b": dyn.V("${a}"), - }) - - // The call replaces ${a} with ${foobar} and skips everything else. - out, err := dynvar.Resolve(in, func(path dyn.Path) (dyn.Value, error) { - if path.String() == "a" { - return dyn.V("${foobar}"), nil - } - return dyn.InvalidValue, dynvar.ErrSkipResolution - }) - - require.NoError(t, err) - assert.Equal(t, "a", getByPath(t, out, "a").MustString()) - assert.Equal(t, "${foobar}", getByPath(t, out, "b").MustString()) -} - -func TestResolveWithInterpolateAliasedRef(t *testing.T) { - in := dyn.V(map[string]dyn.Value{ - "a": dyn.V("a"), - "b": dyn.V("${a}"), - "c": dyn.V("${x}"), - }) - - // The call replaces ${x} with ${b} and skips everything else. - out, err := dynvar.Resolve(in, func(path dyn.Path) (dyn.Value, error) { - if path.String() == "x" { - return dyn.V("${b}"), nil - } - return dyn.GetByPath(in, path) - }) - - require.NoError(t, err) - assert.Equal(t, "a", getByPath(t, out, "a").MustString()) - assert.Equal(t, "a", getByPath(t, out, "b").MustString()) - assert.Equal(t, "a", getByPath(t, out, "c").MustString()) -} - -func TestResolveIndexedRefs(t *testing.T) { - in := dyn.V(map[string]dyn.Value{ - "slice": dyn.V([]dyn.Value{dyn.V("a"), dyn.V("b")}), - "a": dyn.V("a: ${slice[0]}"), - }) - - out, err := dynvar.Resolve(in, dynvar.DefaultLookup(in)) - require.NoError(t, err) - - assert.Equal(t, "a: a", getByPath(t, out, "a").MustString()) -} - -func TestResolveIndexedRefsFromMap(t *testing.T) { - in := dyn.V(map[string]dyn.Value{ - "map": dyn.V( - map[string]dyn.Value{ - "slice": dyn.V([]dyn.Value{dyn.V("a")}), - }), - "a": dyn.V("a: ${map.slice[0]}"), - }) - - out, err := dynvar.Resolve(in, dynvar.DefaultLookup(in)) - require.NoError(t, err) - - assert.Equal(t, "a: a", getByPath(t, out, "a").MustString()) -} - -func TestResolveMapFieldFromIndexedRefs(t *testing.T) { - in := dyn.V(map[string]dyn.Value{ - "map": dyn.V( - map[string]dyn.Value{ - "slice": dyn.V([]dyn.Value{ - dyn.V(map[string]dyn.Value{ - "value": dyn.V("a"), - }), - }), - }), - "a": dyn.V("a: ${map.slice[0].value}"), - }) - - out, err := dynvar.Resolve(in, dynvar.DefaultLookup(in)) - require.NoError(t, err) - - assert.Equal(t, "a: a", getByPath(t, out, "a").MustString()) -} - -func TestResolveNestedIndexedRefs(t *testing.T) { - in := dyn.V(map[string]dyn.Value{ - "slice": dyn.V([]dyn.Value{ - dyn.V([]dyn.Value{dyn.V("a")}), - }), - "a": dyn.V("a: ${slice[0][0]}"), - }) - - out, err := dynvar.Resolve(in, dynvar.DefaultLookup(in)) - require.NoError(t, err) - - assert.Equal(t, "a: a", getByPath(t, out, "a").MustString()) -} - -func TestResolveMapVariable(t *testing.T) { - in := dyn.V(map[string]dyn.Value{ - "map": dyn.V(map[string]dyn.Value{ - "key1": dyn.V("value1"), - "key2": dyn.V("value2"), - }), - "var": dyn.V("${map}"), - }) - - out, err := dynvar.Resolve(in, dynvar.DefaultLookup(in)) - require.NoError(t, err) - - // Verify that the map variable was interpolated correctly - mapVal := getByPath(t, out, "var") - _, ok := mapVal.AsMap() - require.True(t, ok, "expected map value") - - // Verify the map contents - assert.Equal(t, "value1", getByPath(t, mapVal, "key1").MustString()) - assert.Equal(t, "value2", getByPath(t, mapVal, "key2").MustString()) -} - -func TestResolveSequenceVariable(t *testing.T) { - in := dyn.V(map[string]dyn.Value{ - "seq": dyn.V([]dyn.Value{ - dyn.V("value1"), - dyn.V("value2"), - }), - "var": dyn.V("${seq}"), - }) - - out, err := dynvar.Resolve(in, dynvar.DefaultLookup(in)) - require.NoError(t, err) - - // Verify that the sequence variable was interpolated correctly - seqVal := getByPath(t, out, "var") - seq, ok := seqVal.AsSequence() - require.True(t, ok, "expected sequence value") - require.Len(t, seq, 2) - - // Verify the sequence contents - assert.Equal(t, "value1", seq[0].MustString()) - assert.Equal(t, "value2", seq[1].MustString()) -} diff --git a/libs/dyn/dynvar/testdata/reference_vectors.json b/libs/dyn/dynvar/testdata/reference_vectors.json deleted file mode 100644 index 2c47b135450..00000000000 --- a/libs/dyn/dynvar/testdata/reference_vectors.json +++ /dev/null @@ -1,368 +0,0 @@ -{ - "comment": "Shared contract for ${...} reference parsing. Consumed by libs/dyn/dynvar and python/databricks/bundles/core/_transform.py tests.", - "vectors": [ - { - "id": "ascii-simple", - "input": "${hello_world.world_world}", - "match": true, - "references": ["hello_world.world_world"], - "pure": true, - "path": "hello_world.world_world" - }, - { - "id": "ascii-hyphen-separators", - "input": "${helloworld.world-world}", - "match": true, - "references": ["helloworld.world-world"], - "pure": true, - "path": "helloworld.world-world" - }, - { - "id": "ascii-hyphen-in-segment", - "input": "${hello-world.world-world}", - "match": true, - "references": ["hello-world.world-world"], - "pure": true, - "path": "hello-world.world-world" - }, - { - "id": "ascii-underscore-separators", - "input": "${hello_world.world__world}", - "match": true, - "references": ["hello_world.world__world"], - "pure": true, - "path": "hello_world.world__world" - }, - { - "id": "ascii-double-hyphen", - "input": "${hello_world.world--world}", - "match": true, - "references": ["hello_world.world--world"], - "pure": true, - "path": "hello_world.world--world" - }, - { - "id": "ascii-mixed-separators-1", - "input": "${hello_world.world-_world}", - "match": true, - "references": ["hello_world.world-_world"], - "pure": true, - "path": "hello_world.world-_world" - }, - { - "id": "ascii-mixed-separators-2", - "input": "${hello_world.world_-world}", - "match": true, - "references": ["hello_world.world_-world"], - "pure": true, - "path": "hello_world.world_-world" - }, - { - "id": "unicode-accent-segment", - "input": "${café.bar}", - "match": true, - "references": ["café.bar"], - "pure": true, - "path": "café.bar" - }, - { - "id": "unicode-accent-trailing-segment", - "input": "${hello.café}", - "match": true, - "references": ["hello.café"], - "pure": true, - "path": "hello.café" - }, - { - "id": "unicode-cjk", - "input": "${变量.世界}", - "match": true, - "references": ["变量.世界"], - "pure": true, - "path": "变量.世界" - }, - { - "id": "unicode-cjk-numeral-letter", - "input": "${变量.九世界}", - "match": true, - "references": ["变量.九世界"], - "pure": true, - "path": "变量.九世界" - }, - { - "id": "unicode-arabic", - "input": "${العربية.key}", - "match": true, - "references": ["العربية.key"], - "pure": true, - "path": "العربية.key" - }, - { - "id": "unicode-multi-ref", - "input": "${变量.世界} ${café.bar}", - "match": true, - "references": ["变量.世界", "café.bar"], - "pure": false - }, - { - "id": "unicode-var-reference", - "input": "${var.变量}", - "match": true, - "references": ["var.变量"], - "pure": true, - "path": "var.变量" - }, - { - "id": "unicode-resource-reference", - "input": "${resources.jobs.工作.name}", - "match": true, - "references": ["resources.jobs.工作.name"], - "pure": true, - "path": "resources.jobs.工作.name" - }, - { - "id": "indexed-path", - "input": "${foo.bar.baz[0]}", - "match": true, - "references": ["foo.bar.baz[0]"], - "pure": true, - "path": "foo.bar.baz[0]" - }, - { - "id": "nested-indexed-path", - "input": "${foo.bar.baz[0][1].qux}", - "match": true, - "references": ["foo.bar.baz[0][1].qux"], - "pure": true, - "path": "foo.bar.baz[0][1].qux" - }, - { - "id": "indexed-unicode-path", - "input": "${变量.世界[0]}", - "match": true, - "references": ["变量.世界[0]"], - "pure": true, - "path": "变量.世界[0]" - }, - { - "id": "python-simple-variable", - "input": "${var.my_var}", - "match": true, - "references": ["var.my_var"], - "pure": true, - "path": "var.my_var" - }, - { - "id": "python-indexed-variable", - "input": "${var.my_var[0].foo}", - "match": true, - "references": ["var.my_var[0].foo"], - "pure": true, - "path": "var.my_var[0].foo" - }, - { - "id": "leading-underscore", - "input": "${_hello_world.world_world}", - "match": true, - "references": ["_hello_world.world_world"], - "pure": true, - "path": "_hello_world.world_world" - }, - { - "id": "resource-key-leading-underscore", - "input": "${resources.jobs._job.id}", - "match": true, - "references": ["resources.jobs._job.id"], - "pure": true, - "path": "resources.jobs._job.id" - }, - { - "id": "resource-key-leading-underscore-and-hyphens", - "input": "${resources.jobs._asset-bundle-deploymentjob.id}", - "match": true, - "references": ["resources.jobs._asset-bundle-deploymentjob.id"], - "pure": true, - "path": "resources.jobs._asset-bundle-deploymentjob.id" - }, - { - "id": "segment-trailing-hyphen", - "input": "${hello_world-.world_world}", - "match": false - }, - { - "id": "segment-trailing-underscore", - "input": "${hello_world-_.world_world}", - "match": false - }, - { - "id": "second-segment-trailing-hyphen", - "input": "${helloworld.world-world-}", - "match": false - }, - { - "id": "first-segment-trailing-hyphen", - "input": "${helloworld-.world-world}", - "match": false - }, - { - "id": "second-segment-leading-hyphen", - "input": "${helloworld.-world-world}", - "match": false - }, - { - "id": "leading-and-trailing-hyphen", - "input": "${-hello-world.-world-world-}", - "match": false - }, - { - "id": "invalid-separator-sequence", - "input": "${_-_._-_.id}", - "match": false - }, - { - "id": "leading-digit-ascii", - "input": "${0helloworld.world-world}", - "match": false - }, - { - "id": "leading-digit-second-segment", - "input": "${helloworld.9world-world}", - "match": false - }, - { - "id": "leading-digit-unicode", - "input": "${0café.bar}", - "match": false - }, - { - "id": "leading-digit-cjk", - "input": "${变量.9世界}", - "match": false - }, - { - "id": "leading-hyphen-cjk", - "input": "${变量.-世界}", - "match": false - }, - { - "id": "trailing-hyphen-accent", - "input": "${café-}", - "match": false - }, - { - "id": "nfd-decomposed-accent", - "comment": "NFD e + combining acute; not equivalent to NFC café", - "input": "${cafe\u0301.bar}", - "match": false - }, - { - "id": "emoji-in-segment", - "input": "${foo.🌍}", - "match": false - }, - { - "id": "emoji-leading-segment", - "input": "${🌍.bar}", - "match": false - }, - { - "id": "symbol-in-segment", - "input": "${foo.bar$}", - "match": false - }, - { - "id": "zero-width-space", - "input": "${foo.bar\u200b}", - "match": false - }, - { - "id": "fullwidth-digit", - "input": "${变量.９世界}", - "match": false - }, - { - "id": "empty-string", - "input": "", - "match": false, - "pure": false - }, - { - "id": "suffix-after-reference", - "input": "${foo.bar} suffix", - "match": true, - "references": ["foo.bar"], - "pure": false, - "path_ok": false - }, - { - "id": "prefix-before-reference", - "input": "prefix ${foo.bar}", - "match": true, - "references": ["foo.bar"], - "pure": false, - "path_ok": false - }, - { - "id": "adjacent-references", - "input": "${foo.one}${foo.two}", - "match": true, - "references": ["foo.one", "foo.two"], - "pure": false, - "path_ok": false - }, - { - "id": "multiple-variables-not-pure", - "input": "${var.my_var} ${var.my_var}", - "match": true, - "references": ["var.my_var", "var.my_var"], - "pure": false, - "path_ok": false - }, - { - "id": "unclosed-reference", - "input": "${foo.bar", - "match": false - }, - { - "id": "stray-closing-brace", - "input": "foo.bar}", - "match": false - }, - { - "id": "bare-path", - "input": "foo.bar", - "match": false - }, - { - "id": "wrong-braces", - "input": "{foo.bar}", - "match": false - }, - { - "id": "escaped-double-dollar", - "input": "$${foo.bar}", - "match": false - }, - { - "id": "escaped-only-multiple", - "input": "$${a.b} $${c.d}", - "match": false - }, - { - "id": "escaped-mixed-with-real", - "input": "${real.ref} $${escaped.ref}", - "match": true, - "references": ["real.ref"], - "pure": false, - "path_ok": false - }, - { - "id": "escaped-prefix-real-suffix", - "input": "$${escaped} text ${real.ref}", - "match": true, - "references": ["real.ref"], - "pure": false, - "path_ok": false - } - ] -} diff --git a/libs/dyn/jsonloader/json.go b/libs/dyn/jsonloader/json.go deleted file mode 100644 index ffb715e9c9c..00000000000 --- a/libs/dyn/jsonloader/json.go +++ /dev/null @@ -1,134 +0,0 @@ -package jsonloader - -import ( - "bytes" - "encoding/json" - "errors" - "fmt" - "io" - - "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" -) - -func LoadJSON(data []byte, source string) (dyn.Value, error) { - offset := BuildLineOffsets(data) - offset.SetSource(source) - - reader := bytes.NewReader(data) - decoder := json.NewDecoder(reader) - - // Use json.Number to avoid losing precision on int64 values above 2^53 (e.g. job and run IDs). - decoder.UseNumber() - - // Start decoding from the top-level value - value, err := decodeValue(decoder, &offset) - if err != nil { - if errors.Is(err, io.EOF) { - err = errors.New("unexpected end of JSON input") - } - // Get the current offset for error reporting - errorOffset := decoder.InputOffset() - errorLocation := offset.GetPosition(errorOffset) - return dyn.InvalidValue, fmt.Errorf("error decoding JSON at %s: %w", errorLocation, err) - } - - // Check if there are any remaining tokens (should not be valid JSON) - if decoder.More() { - errorOffset := decoder.InputOffset() - errorLocation := offset.GetPosition(errorOffset) - return dyn.InvalidValue, fmt.Errorf("error decoding JSON at %s: unexpected additional content", errorLocation) - } - - return value, nil -} - -func decodeValue(decoder *json.Decoder, o *Offset) (dyn.Value, error) { - // Read the next JSON token - token, err := decoder.Token() - if err != nil { - return dyn.InvalidValue, err - } - - // Get the current byte offset and the location. - // We will later use this location to store the location of the value in the file - // For objects and arrays, we will store the location of the opening '{' or '[' - // For primitive types, we will store the location of the value itself (end of the value) - // We can't reliably calculate the beginning of the value for primitive types because - // the decoder doesn't provide the offset of the beginning of the value and the value might or might not be quoted. - offset := decoder.InputOffset() - location := o.GetPosition(offset) - - switch tok := token.(type) { - case json.Delim: - switch tok { - case '{': - location = o.GetPosition(offset - 1) - // Decode JSON object - obj := dyn.NewMapping() - for decoder.More() { - // Decode the key - keyToken, err := decoder.Token() - if err != nil { - return invalidValueWithLocation(decoder, o), err - } - key, ok := keyToken.(string) - if !ok { - return invalidValueWithLocation(decoder, o), errors.New("expected string for object key") - } - - // Get the offset of the key by subtracting the length of the key and the '"' character - keyOffset := decoder.InputOffset() - int64(len(key)+1) - loc := []diag.Location{o.GetPosition(keyOffset)} - - // Decode the value recursively - val, err := decodeValue(decoder, o) - if err != nil { - return invalidValueWithLocation(decoder, o), err - } - - obj.SetLoc(key, loc, val) - } - // Consume the closing '}' - if _, err := decoder.Token(); err != nil { - return invalidValueWithLocation(decoder, o), err - } - return dyn.NewValue(obj, []diag.Location{location}), nil - case '[': - location = o.GetPosition(offset - 1) - // Decode JSON array - var arr []dyn.Value - for decoder.More() { - val, err := decodeValue(decoder, o) - if err != nil { - return invalidValueWithLocation(decoder, o), err - } - arr = append(arr, val) - } - // Consume the closing ']' - if _, err := decoder.Token(); err != nil { - return invalidValueWithLocation(decoder, o), err - } - return dyn.NewValue(arr, []diag.Location{location}), nil - } - case json.Number: - // Integers that overflow int64 fall back to float64, matching the decoder's behavior without UseNumber. - if i64, err := tok.Int64(); err == nil { - return dyn.NewValue(i64, []diag.Location{location}), nil - } - f64, err := tok.Float64() - if err != nil { - return invalidValueWithLocation(decoder, o), fmt.Errorf("invalid number %q: %w", tok.String(), err) - } - return dyn.NewValue(f64, []diag.Location{location}), nil - default: - return dyn.NewValue(tok, []diag.Location{location}), nil - } - - return invalidValueWithLocation(decoder, o), fmt.Errorf("unexpected token: %v", token) -} - -func invalidValueWithLocation(decoder *json.Decoder, o *Offset) dyn.Value { - location := o.GetPosition(decoder.InputOffset()) - return dyn.InvalidValue.WithLocations([]diag.Location{location}) -} diff --git a/libs/dyn/jsonloader/json_test.go b/libs/dyn/jsonloader/json_test.go deleted file mode 100644 index e274de559ce..00000000000 --- a/libs/dyn/jsonloader/json_test.go +++ /dev/null @@ -1,161 +0,0 @@ -package jsonloader - -import ( - "testing" - - "github.com/databricks/cli/libs/dyn/convert" - "github.com/databricks/databricks-sdk-go/service/jobs" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -const jsonData = ` -{ - "job_id": 123, - "new_settings": { - "name": "xxx", - "email_notifications": { - "on_start": [], - "on_success": [], - "on_failure": [] - }, - "webhook_notifications": { - "on_start": [], - "on_failure": [] - }, - "notification_settings": { - "no_alert_for_skipped_runs": true, - "no_alert_for_canceled_runs": true - }, - "timeout_seconds": 0, - "max_concurrent_runs": 1, - "tasks": [ - { - "task_key": "xxx", - "email_notifications": {}, - "notification_settings": {}, - "timeout_seconds": 0, - "max_retries": 0, - "min_retry_interval_millis": 0, - "retry_on_timeout": "true" - } - ] - } -} -` - -func TestJsonLoader(t *testing.T) { - v, err := LoadJSON([]byte(jsonData), "(inline)") - assert.NoError(t, err) - - var r jobs.ResetJob - err = convert.ToTyped(&r, v) - assert.NoError(t, err) -} - -const malformedMap = ` -{ - "job_id": 123, - "new_settings": { - "name": "xxx", - "wrong", - } -} -` - -func TestJsonLoaderMalformedMap(t *testing.T) { - _, err := LoadJSON([]byte(malformedMap), "(inline)") - assert.ErrorContains(t, err, "error decoding JSON at (inline):6:16: invalid character ',' after object key") -} - -const malformedArray = ` -{ - "job_id": 123, - "new_settings": { - "name": "xxx", - "tasks": [1, "asd",] - } -}` - -func TestJsonLoaderMalformedArray(t *testing.T) { - _, err := LoadJSON([]byte(malformedArray), "path/to/file.json") - assert.ErrorContains(t, err, "error decoding JSON at path/to/file.json:6:27: invalid character ',' looking for beginning of value") -} - -const eofData = ` -{ - "job_id": 123, - "new_settings": { - "name": "xxx",` - -func TestJsonLoaderEOF(t *testing.T) { - _, err := LoadJSON([]byte(eofData), "path/to/file.json") - assert.ErrorContains(t, err, "unexpected end of JSON input") -} - -const mapWithNoBraces = ` -"job_id": 123, -"new_settings": { - "name": "xxx", - "wrong": "xxx", -} -` - -func TestJsonMapWithoutBraces(t *testing.T) { - _, err := LoadJSON([]byte(mapWithNoBraces), "path/to/file.json") - assert.ErrorContains(t, err, "error decoding JSON at") -} - -const validInline = `["job_id", 123]` - -func TestJsonValidInline(t *testing.T) { - _, err := LoadJSON([]byte(validInline), "path/to/file.json") - assert.NoError(t, err) -} - -func TestJsonLoaderNumbers(t *testing.T) { - for _, tc := range []struct { - input string - expected any - }{ - {`123`, int64(123)}, - {`-1`, int64(-1)}, - {`123456789012345678`, int64(123456789012345678)}, - {`-123456789012345678`, int64(-123456789012345678)}, - {`9223372036854775807`, int64(9223372036854775807)}, - {`2.0`, 2.0}, - {`2.5`, 2.5}, - {`1e3`, 1000.0}, - {`18446744073709551615`, 1.8446744073709552e+19}, - } { - v, err := LoadJSON([]byte(tc.input), "(inline)") - assert.NoError(t, err, tc.input) - assert.Equal(t, tc.expected, v.AsAny(), tc.input) - } -} - -func TestJsonLoaderNumberOutOfRange(t *testing.T) { - _, err := LoadJSON([]byte(`1e400`), "(inline)") - assert.ErrorContains(t, err, "value out of range") -} - -const mixedNumbersData = ` -{ - "job_id": 123456789012345678, - "new_settings": { - "name": "xxx", - "timeout_seconds": 100 - } -} -` - -func TestJsonLoaderMixedNumbersToTyped(t *testing.T) { - v, err := LoadJSON([]byte(mixedNumbersData), "(inline)") - require.NoError(t, err) - - var r jobs.ResetJob - err = convert.ToTyped(&r, v) - require.NoError(t, err) - assert.Equal(t, int64(123456789012345678), r.JobId) - assert.Equal(t, 100, r.NewSettings.TimeoutSeconds) -} diff --git a/libs/dyn/jsonloader/locations.go b/libs/dyn/jsonloader/locations.go deleted file mode 100644 index 3dbe2da6630..00000000000 --- a/libs/dyn/jsonloader/locations.go +++ /dev/null @@ -1,49 +0,0 @@ -package jsonloader - -import ( - "sort" - - "github.com/databricks/cli/libs/diag" -) - -type LineOffset struct { - Line int - Start int64 -} - -type Offset struct { //nolint:recvcheck // value receiver for read-only GetPosition, pointer for SetSource - offsets []LineOffset - source string -} - -// buildLineOffsets scans the input data and records the starting byte offset of each line. -func BuildLineOffsets(data []byte) Offset { - offsets := []LineOffset{{Line: 1, Start: 0}} - line := 1 - for i, b := range data { - if b == '\n' { - line++ - offsets = append(offsets, LineOffset{Line: line, Start: int64(i + 1)}) - } - } - return Offset{offsets: offsets} -} - -// GetPosition maps a byte offset to its corresponding line and column numbers. -func (o Offset) GetPosition(offset int64) diag.Location { - // Binary search to find the line - idx := max(sort.Search(len(o.offsets), func(i int) bool { - return o.offsets[i].Start > offset - })-1, 0) - - lineOffset := o.offsets[idx] - return diag.Location{ - File: o.source, - Line: lineOffset.Line, - Column: int(offset-lineOffset.Start) + 1, - } -} - -func (o *Offset) SetSource(source string) { - o.source = source -} diff --git a/libs/dyn/mapping.go b/libs/dyn/mapping.go index 8b29181e40f..cd0be01b0e8 100644 --- a/libs/dyn/mapping.go +++ b/libs/dyn/mapping.go @@ -30,21 +30,6 @@ func NewMapping() Mapping { } } -// NewMappingFromPairs computes a [Mapping] from a list of [Pair]s. The index -// map does not need to be provided since that will be computed from the -// key-value pairs provided. -func NewMappingFromPairs(pairs []Pair) Mapping { - index := make(map[string]int) - for i, p := range pairs { - index[p.Key.MustString()] = i - } - - return Mapping{ - pairs: pairs, - index: index, - } -} - // newMappingWithSize creates a new Mapping preallocated to the specified size. func newMappingWithSize(size int) Mapping { return Mapping{ diff --git a/libs/dyn/merge/elements_by_key.go b/libs/dyn/merge/elements_by_key.go deleted file mode 100644 index 7d7e35c8eb5..00000000000 --- a/libs/dyn/merge/elements_by_key.go +++ /dev/null @@ -1,103 +0,0 @@ -package merge - -import ( - "slices" - - "github.com/databricks/cli/libs/dyn" -) - -type elementsByKey struct { - key string - keyFunc func(dyn.Value) string - sortKeys bool -} - -func (e elementsByKey) doMap(_ dyn.Path, v dyn.Value, mergeFunc func(a, b dyn.Value) (dyn.Value, error)) (dyn.Value, error) { - // We know the type of this value is a sequence. - // For additional defence, return self if it is not. - elements, ok := v.AsSequence() - if !ok { - return v, nil - } - - seen := make(map[string]dyn.Value, len(elements)) - keys := make([]string, 0, len(elements)) - - // Iterate in natural order. For a given key, we first see the - // base definition and merge instances that come after it. - for i := range elements { - kv := elements[i].Get(e.key) - key := e.keyFunc(kv) - - // Register element with key if not yet seen before. - ref, ok := seen[key] - if !ok { - keys = append(keys, key) - seen[key] = elements[i] - continue - } - - // Merge this instance into the reference. - nv, err := mergeFunc(ref, elements[i]) - if err != nil { - return v, err - } - - // Overwrite reference. - seen[key] = nv - } - - if e.sortKeys { - slices.Sort(keys) - } - - // Gather resulting elements in natural order. - out := make([]dyn.Value, 0, len(keys)) - for _, key := range keys { - // Preserve the location from the original key value so that - // downstream code can tell this field was defined in the YAML. - keyLoc := seen[key].Get(e.key).Locations() - nv, err := dyn.Set(seen[key], e.key, dyn.NewValue(key, keyLoc)) - if err != nil { - return dyn.InvalidValue, err - } - out = append(out, nv) - } - - return dyn.NewValue(out, v.Locations()), nil -} - -func (e elementsByKey) Map(_ dyn.Path, v dyn.Value) (dyn.Value, error) { - return e.doMap(nil, v, Merge) -} - -func (e elementsByKey) MapWithOverride(p dyn.Path, v dyn.Value) (dyn.Value, error) { - return e.doMap(nil, v, func(a, b dyn.Value) (dyn.Value, error) { - return Override(a, b, OverrideVisitor{ - VisitInsert: func(_ dyn.Path, v dyn.Value) (dyn.Value, error) { - return v, nil - }, - VisitDelete: func(valuePath dyn.Path, left dyn.Value) error { - return nil - }, - VisitUpdate: func(_ dyn.Path, a, b dyn.Value) (dyn.Value, error) { - return b, nil - }, - }) - }) -} - -// ElementsByKey returns a [dyn.MapFunc] that operates on a sequence -// where each element is a map. It groups elements by a key and merges -// elements with the same key. -// -// The function that extracts the key from an element is provided as -// a parameter. The resulting elements get their key field overwritten -// with the value as returned by the key function. -func ElementsByKey(key string, keyFunc func(dyn.Value) string) dyn.MapFunc { - return elementsByKey{key, keyFunc, false}.Map -} - -func ElementsByKeyWithOverride(key string, keyFunc func(dyn.Value) string) dyn.MapFunc { - return elementsByKey{key, keyFunc, false}.MapWithOverride -} diff --git a/libs/dyn/merge/elements_by_key_test.go b/libs/dyn/merge/elements_by_key_test.go deleted file mode 100644 index 2aa19306820..00000000000 --- a/libs/dyn/merge/elements_by_key_test.go +++ /dev/null @@ -1,109 +0,0 @@ -package merge - -import ( - "strings" - "testing" - - "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestElementByKey(t *testing.T) { - vin := dyn.V([]dyn.Value{ - dyn.V(map[string]dyn.Value{ - "key": dyn.V("foo"), - "value": dyn.V(42), - }), - dyn.V(map[string]dyn.Value{ - "key": dyn.V("bar"), - "value": dyn.V(43), - }), - dyn.V(map[string]dyn.Value{ - // Use upper case key to test that the resulting element has its - // key field assigned to the output of the key function. - // The key function in this test returns the lower case version of the key. - "key": dyn.V("FOO"), - "value": dyn.V(44), - }), - }) - - keyFunc := func(v dyn.Value) string { - return strings.ToLower(v.MustString()) - } - - vout, err := dyn.MapByPath(vin, dyn.EmptyPath, ElementsByKey("key", keyFunc)) - require.NoError(t, err) - assert.Len(t, vout.MustSequence(), 2) - assert.Equal(t, - map[string]any{ - "key": "foo", - "value": 44, - }, vout.Index(0).AsAny(), - ) - assert.Equal(t, - map[string]any{ - "key": "bar", - "value": 43, - }, vout.Index(1).AsAny(), - ) -} - -func TestElementByKeyPreservesLocations(t *testing.T) { - loc := diag.Location{File: "config.yml", Line: 10, Column: 5} - vin := dyn.V([]dyn.Value{ - dyn.V(map[string]dyn.Value{ - "key": dyn.NewValue("foo", []diag.Location{loc}), - "value": dyn.V(42), - }), - }) - - keyFunc := func(v dyn.Value) string { - return v.MustString() - } - - vout, err := dyn.MapByPath(vin, dyn.EmptyPath, ElementsByKey("key", keyFunc)) - require.NoError(t, err) - - // Verify the key field retains its original location. - keyValue := vout.Index(0).Get("key") - assert.Equal(t, loc, keyValue.Location()) -} - -func TestElementByKeyWithOverride(t *testing.T) { - vin := dyn.V([]dyn.Value{ - dyn.V(map[string]dyn.Value{ - "key": dyn.V("foo"), - "value": dyn.V(42), - }), - dyn.V(map[string]dyn.Value{ - "key": dyn.V("bar"), - "value": dyn.V(43), - }), - dyn.V(map[string]dyn.Value{ - "key": dyn.V("foo"), - "othervalue": dyn.V(44), - }), - }) - - keyFunc := func(v dyn.Value) string { - return strings.ToLower(v.MustString()) - } - - vout, err := dyn.MapByPath(vin, dyn.EmptyPath, ElementsByKeyWithOverride("key", keyFunc)) - require.NoError(t, err) - assert.Len(t, vout.MustSequence(), 2) - assert.Equal(t, - map[string]any{ - "key": "foo", - "othervalue": 44, - }, vout.Index(0).AsAny(), - ) - assert.Equal(t, - map[string]any{ - "key": "bar", - "value": 43, - }, vout.Index(1).AsAny(), - ) -} diff --git a/libs/dyn/merge/override.go b/libs/dyn/merge/override.go deleted file mode 100644 index 873c38eead1..00000000000 --- a/libs/dyn/merge/override.go +++ /dev/null @@ -1,200 +0,0 @@ -package merge - -import ( - "errors" - "fmt" - - "github.com/databricks/cli/libs/dyn" -) - -// OverrideVisitor is visiting the changes during the override process -// and allows to control what changes are allowed, or update the effective -// value. -// -// For instance, it can disallow changes outside the specific path(s), or update -// the location of the effective value. -// -// Values returned by 'VisitInsert' and 'VisitUpdate' are used as the final value -// of the node. 'VisitDelete' can return ErrOverrideUndoDelete to undo delete. -// -// 'VisitDelete' is called when a value is removed from mapping or sequence -// 'VisitInsert' is called when a new value is added to mapping or sequence -// 'VisitUpdate' is called when a leaf value is updated -type OverrideVisitor struct { - VisitDelete func(valuePath dyn.Path, left dyn.Value) error - VisitInsert func(valuePath dyn.Path, right dyn.Value) (dyn.Value, error) - VisitUpdate func(valuePath dyn.Path, left, right dyn.Value) (dyn.Value, error) -} - -var ErrOverrideUndoDelete = errors.New("undo delete operation") - -// Override overrides value 'leftRoot' with 'rightRoot', keeping 'location' if values -// haven't changed. Preserving 'location' is important to preserve the original source of the value -// for error reporting. -func Override(leftRoot, rightRoot dyn.Value, visitor OverrideVisitor) (dyn.Value, error) { - return override(dyn.EmptyPath, leftRoot, rightRoot, visitor) -} - -func override(basePath dyn.Path, left, right dyn.Value, visitor OverrideVisitor) (dyn.Value, error) { - if left.Kind() != right.Kind() { - return visitor.VisitUpdate(basePath, left, right) - } - - // NB: we only call 'VisitUpdate' on leaf values, and for sequences and mappings - // we don't know if value was updated or not - - switch left.Kind() { - case dyn.KindMap: - merged, err := overrideMapping(basePath, left.MustMap(), right.MustMap(), visitor) - if err != nil { - return dyn.InvalidValue, err - } - - return dyn.NewValue(merged, left.Locations()), nil - - case dyn.KindSequence: - // some sequences are keyed, and we can detect which elements are added/removed/updated, - // but we don't have this information - merged, err := overrideSequence(basePath, left.MustSequence(), right.MustSequence(), visitor) - if err != nil { - return dyn.InvalidValue, err - } - - return dyn.NewValue(merged, left.Locations()), nil - - case dyn.KindString: - if left.MustString() == right.MustString() { - return left, nil - } else { - return visitor.VisitUpdate(basePath, left, right) - } - - case dyn.KindFloat: - // TODO consider comparison with epsilon if normalization doesn't help, where do we use floats? - - if left.MustFloat() == right.MustFloat() { - return left, nil - } else { - return visitor.VisitUpdate(basePath, left, right) - } - - case dyn.KindBool: - if left.MustBool() == right.MustBool() { - return left, nil - } else { - return visitor.VisitUpdate(basePath, left, right) - } - - case dyn.KindTime: - if left.MustTime() == right.MustTime() { - return left, nil - } else { - return visitor.VisitUpdate(basePath, left, right) - } - - case dyn.KindInt: - if left.MustInt() == right.MustInt() { - return left, nil - } else { - return visitor.VisitUpdate(basePath, left, right) - } - case dyn.KindNil: - return left, nil - default: - // Fall through to the error case. - } - - return dyn.InvalidValue, fmt.Errorf("unexpected kind %s at %s", left.Kind(), basePath.String()) -} - -func overrideMapping(basePath dyn.Path, leftMapping, rightMapping dyn.Mapping, visitor OverrideVisitor) (dyn.Mapping, error) { - out := dyn.NewMapping() - - for _, leftPair := range leftMapping.Pairs() { - // detect if key was removed - if _, ok := rightMapping.GetPair(leftPair.Key); !ok { - key := leftPair.Key.MustString() - keyLoc := leftPair.Key.Locations() - path := basePath.Append(dyn.Key(key)) - - err := visitor.VisitDelete(path, leftPair.Value) - - // if 'delete' was undone, add it back - if errors.Is(err, ErrOverrideUndoDelete) { - out.SetLoc(key, keyLoc, leftPair.Value) - } else if err != nil { - return dyn.NewMapping(), err - } - } - } - - // iterating only right mapping will remove keys not present anymore - // and insert new keys - - for _, rightPair := range rightMapping.Pairs() { - key := rightPair.Key.MustString() - keyLoc := rightPair.Key.Locations() - if leftPair, ok := leftMapping.GetPair(rightPair.Key); ok { - path := basePath.Append(dyn.Key(key)) - newValue, err := override(path, leftPair.Value, rightPair.Value, visitor) - if err != nil { - return dyn.NewMapping(), err - } - - // key was there before, so keep its location - out.SetLoc(key, keyLoc, newValue) - } else { - path := basePath.Append(dyn.Key(rightPair.Key.MustString())) - - newValue, err := visitor.VisitInsert(path, rightPair.Value) - if err != nil { - return dyn.NewMapping(), err - } - - out.SetLoc(key, keyLoc, newValue) - } - } - - return out, nil -} - -func overrideSequence(basePath dyn.Path, left, right []dyn.Value, visitor OverrideVisitor) ([]dyn.Value, error) { - minLen := min(len(left), len(right)) - var values []dyn.Value - - for i := range minLen { - path := basePath.Append(dyn.Index(i)) - merged, err := override(path, left[i], right[i], visitor) - if err != nil { - return nil, err - } - - values = append(values, merged) - } - - if len(right) > len(left) { - for i := minLen; i < len(right); i++ { - path := basePath.Append(dyn.Index(i)) - newValue, err := visitor.VisitInsert(path, right[i]) - if err != nil { - return nil, err - } - - values = append(values, newValue) - } - } else if len(left) > len(right) { - for i := minLen; i < len(left); i++ { - path := basePath.Append(dyn.Index(i)) - err := visitor.VisitDelete(path, left[i]) - - // if 'delete' was undone, add it back - if errors.Is(err, ErrOverrideUndoDelete) { - values = append(values, left[i]) - } else if err != nil { - return nil, err - } - } - } - - return values, nil -} diff --git a/libs/dyn/merge/override_test.go b/libs/dyn/merge/override_test.go deleted file mode 100644 index 17602f57ad7..00000000000 --- a/libs/dyn/merge/override_test.go +++ /dev/null @@ -1,520 +0,0 @@ -package merge - -import ( - "errors" - "testing" - "time" - - "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/dynassert" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -type overrideTestCase struct { - name string - left dyn.Value - right dyn.Value - state visitorState - expected dyn.Value -} - -func TestOverride_Primitive(t *testing.T) { - leftLocation := diag.Location{File: "left.yml", Line: 1, Column: 1} - rightLocation := diag.Location{File: "right.yml", Line: 1, Column: 1} - - modifiedTestCases := []overrideTestCase{ - { - name: "string (updated)", - state: visitorState{updated: []string{"root"}}, - left: dyn.NewValue("a", []diag.Location{leftLocation}), - right: dyn.NewValue("b", []diag.Location{rightLocation}), - expected: dyn.NewValue("b", []diag.Location{rightLocation}), - }, - { - name: "string (not updated)", - state: visitorState{}, - left: dyn.NewValue("a", []diag.Location{leftLocation}), - right: dyn.NewValue("a", []diag.Location{rightLocation}), - expected: dyn.NewValue("a", []diag.Location{leftLocation}), - }, - { - name: "bool (updated)", - state: visitorState{updated: []string{"root"}}, - left: dyn.NewValue(true, []diag.Location{leftLocation}), - right: dyn.NewValue(false, []diag.Location{rightLocation}), - expected: dyn.NewValue(false, []diag.Location{rightLocation}), - }, - { - name: "bool (not updated)", - state: visitorState{}, - left: dyn.NewValue(true, []diag.Location{leftLocation}), - right: dyn.NewValue(true, []diag.Location{rightLocation}), - expected: dyn.NewValue(true, []diag.Location{leftLocation}), - }, - { - name: "int (updated)", - state: visitorState{updated: []string{"root"}}, - left: dyn.NewValue(1, []diag.Location{leftLocation}), - right: dyn.NewValue(2, []diag.Location{rightLocation}), - expected: dyn.NewValue(2, []diag.Location{rightLocation}), - }, - { - name: "int (not updated)", - state: visitorState{}, - left: dyn.NewValue(int32(1), []diag.Location{leftLocation}), - right: dyn.NewValue(int64(1), []diag.Location{rightLocation}), - expected: dyn.NewValue(int32(1), []diag.Location{leftLocation}), - }, - { - name: "float (updated)", - state: visitorState{updated: []string{"root"}}, - left: dyn.NewValue(1.0, []diag.Location{leftLocation}), - right: dyn.NewValue(2.0, []diag.Location{rightLocation}), - expected: dyn.NewValue(2.0, []diag.Location{rightLocation}), - }, - { - name: "float (not updated)", - state: visitorState{}, - left: dyn.NewValue(float32(1.0), []diag.Location{leftLocation}), - right: dyn.NewValue(float64(1.0), []diag.Location{rightLocation}), - expected: dyn.NewValue(float32(1.0), []diag.Location{leftLocation}), - }, - { - name: "time (updated)", - state: visitorState{updated: []string{"root"}}, - left: dyn.NewValue(dyn.FromTime(time.UnixMilli(10000)), []diag.Location{leftLocation}), - right: dyn.NewValue(dyn.FromTime(time.UnixMilli(10001)), []diag.Location{rightLocation}), - expected: dyn.NewValue(dyn.FromTime(time.UnixMilli(10001)), []diag.Location{rightLocation}), - }, - { - name: "time (not updated)", - state: visitorState{}, - left: dyn.NewValue(dyn.FromTime(time.UnixMilli(10000)), []diag.Location{leftLocation}), - right: dyn.NewValue(dyn.FromTime(time.UnixMilli(10000)), []diag.Location{rightLocation}), - expected: dyn.NewValue(dyn.FromTime(time.UnixMilli(10000)), []diag.Location{leftLocation}), - }, - { - name: "different types (updated)", - state: visitorState{updated: []string{"root"}}, - left: dyn.NewValue("a", []diag.Location{leftLocation}), - right: dyn.NewValue(42, []diag.Location{rightLocation}), - expected: dyn.NewValue(42, []diag.Location{rightLocation}), - }, - { - name: "map - remove 'a', update 'b'", - state: visitorState{ - removed: []string{"root.a"}, - updated: []string{"root.b"}, - }, - left: dyn.NewValue( - map[string]dyn.Value{ - "a": dyn.NewValue(42, []diag.Location{leftLocation}), - "b": dyn.NewValue(10, []diag.Location{leftLocation}), - }, - []diag.Location{leftLocation}), - - right: dyn.NewValue( - map[string]dyn.Value{ - "b": dyn.NewValue(20, []diag.Location{rightLocation}), - }, - []diag.Location{rightLocation}), - - expected: dyn.NewValue( - map[string]dyn.Value{ - "b": dyn.NewValue(20, []diag.Location{rightLocation}), - }, - []diag.Location{leftLocation}), - }, - { - name: "map - add 'a'", - state: visitorState{ - added: []string{"root.a"}, - }, - left: dyn.NewValue( - map[string]dyn.Value{ - "b": dyn.NewValue(10, []diag.Location{leftLocation}), - }, - []diag.Location{leftLocation}, - ), - - right: dyn.NewValue( - map[string]dyn.Value{ - "a": dyn.NewValue(42, []diag.Location{rightLocation}), - "b": dyn.NewValue(10, []diag.Location{rightLocation}), - }, - []diag.Location{leftLocation}, - ), - - expected: dyn.NewValue( - map[string]dyn.Value{ - "a": dyn.NewValue(42, []diag.Location{rightLocation}), - // location hasn't changed because value hasn't changed - "b": dyn.NewValue(10, []diag.Location{leftLocation}), - }, - []diag.Location{leftLocation}, - ), - }, - { - name: "map - remove 'a'", - state: visitorState{ - removed: []string{"root.a"}, - }, - left: dyn.NewValue( - map[string]dyn.Value{ - "a": dyn.NewValue(42, []diag.Location{leftLocation}), - "b": dyn.NewValue(10, []diag.Location{leftLocation}), - }, - []diag.Location{leftLocation}, - ), - - right: dyn.NewValue( - map[string]dyn.Value{ - "b": dyn.NewValue(10, []diag.Location{rightLocation}), - }, - []diag.Location{leftLocation}, - ), - - expected: dyn.NewValue( - map[string]dyn.Value{ - // location hasn't changed because value hasn't changed - "b": dyn.NewValue(10, []diag.Location{leftLocation}), - }, - []diag.Location{leftLocation}, - ), - }, - { - name: "map - add 'jobs.job_1'", - state: visitorState{ - added: []string{"root.jobs.job_1"}, - }, - left: dyn.NewValue( - map[string]dyn.Value{ - "jobs": dyn.NewValue( - map[string]dyn.Value{ - "job_0": dyn.NewValue(42, []diag.Location{leftLocation}), - }, - []diag.Location{leftLocation}, - ), - }, - []diag.Location{leftLocation}, - ), - - right: dyn.NewValue( - map[string]dyn.Value{ - "jobs": dyn.NewValue( - map[string]dyn.Value{ - "job_0": dyn.NewValue(42, []diag.Location{rightLocation}), - "job_1": dyn.NewValue(1337, []diag.Location{rightLocation}), - }, - []diag.Location{rightLocation}, - ), - }, - []diag.Location{rightLocation}, - ), - - expected: dyn.NewValue( - map[string]dyn.Value{ - "jobs": dyn.NewValue( - map[string]dyn.Value{ - "job_0": dyn.NewValue(42, []diag.Location{leftLocation}), - "job_1": dyn.NewValue(1337, []diag.Location{rightLocation}), - }, - []diag.Location{leftLocation}, - ), - }, - []diag.Location{leftLocation}, - ), - }, - { - name: "map - remove nested key", - state: visitorState{removed: []string{"root.jobs.job_1"}}, - left: dyn.NewValue( - map[string]dyn.Value{ - "jobs": dyn.NewValue( - map[string]dyn.Value{ - "job_0": dyn.NewValue(42, []diag.Location{leftLocation}), - "job_1": dyn.NewValue(1337, []diag.Location{rightLocation}), - }, - []diag.Location{leftLocation}, - ), - }, - []diag.Location{leftLocation}, - ), - right: dyn.NewValue( - map[string]dyn.Value{ - "jobs": dyn.NewValue( - map[string]dyn.Value{ - "job_0": dyn.NewValue(42, []diag.Location{rightLocation}), - }, - []diag.Location{rightLocation}, - ), - }, - []diag.Location{rightLocation}, - ), - expected: dyn.NewValue( - map[string]dyn.Value{ - "jobs": dyn.NewValue( - map[string]dyn.Value{ - "job_0": dyn.NewValue(42, []diag.Location{leftLocation}), - }, - []diag.Location{leftLocation}, - ), - }, - []diag.Location{leftLocation}, - ), - }, - { - name: "sequence - add", - state: visitorState{added: []string{"root[1]"}}, - left: dyn.NewValue( - []dyn.Value{ - dyn.NewValue(42, []diag.Location{leftLocation}), - }, - []diag.Location{leftLocation}, - ), - right: dyn.NewValue( - []dyn.Value{ - dyn.NewValue(42, []diag.Location{rightLocation}), - dyn.NewValue(10, []diag.Location{rightLocation}), - }, - []diag.Location{rightLocation}, - ), - expected: dyn.NewValue( - []dyn.Value{ - dyn.NewValue(42, []diag.Location{leftLocation}), - dyn.NewValue(10, []diag.Location{rightLocation}), - }, - []diag.Location{leftLocation}, - ), - }, - { - name: "sequence - remove", - state: visitorState{removed: []string{"root[1]"}}, - left: dyn.NewValue( - []dyn.Value{ - dyn.NewValue(42, []diag.Location{leftLocation}), - dyn.NewValue(10, []diag.Location{leftLocation}), - }, - []diag.Location{leftLocation}, - ), - right: dyn.NewValue( - []dyn.Value{ - dyn.NewValue(42, []diag.Location{rightLocation}), - }, - []diag.Location{rightLocation}, - ), - expected: dyn.NewValue( - []dyn.Value{ - dyn.NewValue(42, []diag.Location{leftLocation}), - }, - []diag.Location{leftLocation}, - ), - // location hasn't changed because value hasn't changed - }, - { - name: "sequence (not updated)", - state: visitorState{}, - left: dyn.NewValue( - []dyn.Value{ - dyn.NewValue(42, []diag.Location{leftLocation}), - }, - []diag.Location{leftLocation}, - ), - right: dyn.NewValue( - []dyn.Value{ - dyn.NewValue(42, []diag.Location{rightLocation}), - }, - []diag.Location{rightLocation}, - ), - expected: dyn.NewValue( - []dyn.Value{ - dyn.NewValue(42, []diag.Location{leftLocation}), - }, - []diag.Location{leftLocation}, - ), - }, - { - name: "nil (not updated)", - state: visitorState{}, - left: dyn.NilValue.WithLocations([]diag.Location{leftLocation}), - right: dyn.NilValue.WithLocations([]diag.Location{rightLocation}), - expected: dyn.NilValue.WithLocations([]diag.Location{leftLocation}), - }, - { - name: "nil (updated)", - state: visitorState{updated: []string{"root"}}, - left: dyn.NilValue, - right: dyn.NewValue(42, []diag.Location{rightLocation}), - expected: dyn.NewValue(42, []diag.Location{rightLocation}), - }, - { - name: "change kind (updated)", - state: visitorState{updated: []string{"root"}}, - left: dyn.NewValue(42.0, []diag.Location{leftLocation}), - right: dyn.NewValue(42, []diag.Location{rightLocation}), - expected: dyn.NewValue(42, []diag.Location{rightLocation}), - }, - } - - for _, tc := range modifiedTestCases { - t.Run(tc.name, func(t *testing.T) { - s, visitor := createVisitor(visitorOpts{}) - out, err := override(dyn.NewPath(dyn.Key("root")), tc.left, tc.right, visitor) - - assert.NoError(t, err) - dynassert.Equal(t, tc.state, *s) - dynassert.Equal(t, tc.expected, out) - }) - - modified := len(tc.state.removed)+len(tc.state.added)+len(tc.state.updated) > 0 - - // visitor is not used unless there is a change - - if modified { - t.Run(tc.name+" - visitor has error", func(t *testing.T) { - _, visitor := createVisitor(visitorOpts{error: errors.New("unexpected change in test")}) - _, err := override(dyn.EmptyPath, tc.left, tc.right, visitor) - - assert.EqualError(t, err, "unexpected change in test") - }) - - t.Run(tc.name+" - visitor overrides value", func(t *testing.T) { - expected := dyn.V("return value") - s, visitor := createVisitor(visitorOpts{returnValue: &expected}) - out, err := override(dyn.EmptyPath, tc.left, tc.right, visitor) - - assert.NoError(t, err) - - for _, added := range s.added { - actual, err := dyn.GetByPath(out, dyn.MustPathFromString(added)) - - assert.NoError(t, err) - dynassert.Equal(t, expected, actual) - } - - for _, updated := range s.updated { - actual, err := dyn.GetByPath(out, dyn.MustPathFromString(updated)) - - assert.NoError(t, err) - dynassert.Equal(t, expected, actual) - } - }) - - if len(tc.state.removed) > 0 { - t.Run(tc.name+" - visitor can undo delete", func(t *testing.T) { - s, visitor := createVisitor(visitorOpts{deleteError: ErrOverrideUndoDelete}) - out, err := override(dyn.EmptyPath, tc.left, tc.right, visitor) - require.NoError(t, err) - - for _, removed := range s.removed { - expected, err := dyn.GetByPath(tc.left, dyn.MustPathFromString(removed)) - require.NoError(t, err) - - actual, err := dyn.GetByPath(out, dyn.MustPathFromString(removed)) - - assert.NoError(t, err) - dynassert.Equal(t, expected, actual) - } - }) - } - } - } -} - -func TestOverride_PreserveMappingKeys(t *testing.T) { - leftLocation := diag.Location{File: "left.yml", Line: 1, Column: 1} - leftKeyLocation := diag.Location{File: "left.yml", Line: 2, Column: 1} - leftValueLocation := diag.Location{File: "left.yml", Line: 3, Column: 1} - - rightLocation := diag.Location{File: "right.yml", Line: 1, Column: 1} - rightKeyLocation := diag.Location{File: "right.yml", Line: 2, Column: 1} - rightValueLocation := diag.Location{File: "right.yml", Line: 3, Column: 1} - - left := dyn.NewMapping() - left.SetLoc("a", []diag.Location{leftKeyLocation}, dyn.NewValue(42, []diag.Location{leftValueLocation})) - - right := dyn.NewMapping() - right.SetLoc("a", []diag.Location{rightKeyLocation}, dyn.NewValue(7, []diag.Location{rightValueLocation})) - - state, visitor := createVisitor(visitorOpts{}) - - out, err := override( - dyn.EmptyPath, - dyn.NewValue(left, []diag.Location{leftLocation}), - dyn.NewValue(right, []diag.Location{rightLocation}), - visitor, - ) - - assert.NoError(t, err) - - if err != nil { - outPairs := out.MustMap().Pairs() - - assert.Equal(t, visitorState{updated: []string{"a"}}, state) - assert.Len(t, outPairs, 1) - - // mapping was first defined in left, so it should keep its location - assert.Equal(t, leftLocation, out.Location()) - - // if there is a validation error for key value, it should point - // to where it was initially defined - assert.Equal(t, leftKeyLocation, outPairs[0].Key.Location()) - - // the value should have updated location, because it has changed - assert.Equal(t, rightValueLocation, outPairs[0].Value.Location()) - } -} - -type visitorState struct { - added []string - removed []string - updated []string -} - -type visitorOpts struct { - error error - deleteError error - returnValue *dyn.Value -} - -func createVisitor(opts visitorOpts) (*visitorState, OverrideVisitor) { - s := visitorState{} - - return &s, OverrideVisitor{ - VisitUpdate: func(valuePath dyn.Path, left, right dyn.Value) (dyn.Value, error) { - s.updated = append(s.updated, valuePath.String()) - - if opts.error != nil { - return dyn.NilValue, opts.error - } else if opts.returnValue != nil { - return *opts.returnValue, nil - } else { - return right, nil - } - }, - VisitDelete: func(valuePath dyn.Path, left dyn.Value) error { - s.removed = append(s.removed, valuePath.String()) - - if opts.error != nil { - return opts.error - } else if opts.deleteError != nil { - return opts.deleteError - } else { - return nil - } - }, - VisitInsert: func(valuePath dyn.Path, right dyn.Value) (dyn.Value, error) { - s.added = append(s.added, valuePath.String()) - - if opts.error != nil { - return dyn.NilValue, opts.error - } else if opts.returnValue != nil { - return *opts.returnValue, nil - } else { - return right, nil - } - }, - } -} diff --git a/libs/dyn/merge/select.go b/libs/dyn/merge/select.go deleted file mode 100644 index 9b09bfa2162..00000000000 --- a/libs/dyn/merge/select.go +++ /dev/null @@ -1,53 +0,0 @@ -package merge - -import ( - "fmt" - - "github.com/databricks/cli/libs/dyn" -) - -// Select returns a new map that contains only the keys specified in the included list. -func Select(value dyn.Value, included []string) (dyn.Value, error) { - mapping, ok := value.AsMap() - if !ok { - return dyn.InvalidValue, fmt.Errorf("expected a map, but found %s", value.Kind()) - } - - newMapping := dyn.NewMapping() - for _, key := range included { - pair, ok := mapping.GetPairByString(key) - - if ok { - newMapping.SetLoc(key, pair.Key.Locations(), pair.Value) - } - } - - return dyn.NewValue(newMapping, value.Locations()), nil -} - -// AntiSelect returns a new map with all keys from the input map except for the ones in the excluded list. -func AntiSelect(value dyn.Value, excluded []string) (dyn.Value, error) { - mapping, ok := value.AsMap() - if !ok { - return dyn.InvalidValue, fmt.Errorf("expected a map, but found %s", value.Kind()) - } - - excludedSet := make(map[string]struct{}) - for _, key := range excluded { - excludedSet[key] = struct{}{} - } - - included := make([]string, 0, len(mapping.Pairs())) - for _, pair := range mapping.Pairs() { - key, ok := pair.Key.AsString() - if !ok { - return dyn.InvalidValue, fmt.Errorf("expected a string key, but found %s", pair.Key.Kind()) - } - - if _, ok := excludedSet[key]; !ok { - included = append(included, pair.Key.MustString()) - } - } - - return Select(value, included) -} diff --git a/libs/dyn/merge/select_test.go b/libs/dyn/merge/select_test.go deleted file mode 100644 index c621f226849..00000000000 --- a/libs/dyn/merge/select_test.go +++ /dev/null @@ -1,55 +0,0 @@ -package merge - -import ( - "testing" - - "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/stretchr/testify/assert" -) - -func TestSelect(t *testing.T) { - locations := []diag.Location{{File: "foo.yml", Line: 1, Column: 1}} - included := []string{"foo"} - input := dyn.NewValue( - map[string]dyn.Value{ - "foo": dyn.V("bar"), - "baz": dyn.V("qux"), - }, - locations, - ) - expected := dyn.NewValue( - map[string]dyn.Value{ - "foo": dyn.V("bar"), - }, - locations, - ) - - actual, err := Select(input, included) - - assert.NoError(t, err) - assert.Equal(t, expected, actual) -} - -func TestAntiSelect(t *testing.T) { - locations := []diag.Location{{File: "foo.yml", Line: 1, Column: 1}} - excluded := []string{"foo"} - input := dyn.NewValue( - map[string]dyn.Value{ - "foo": dyn.V("bar"), - "baz": dyn.V("qux"), - }, - locations, - ) - expected := dyn.NewValue( - map[string]dyn.Value{ - "baz": dyn.V("qux"), - }, - locations, - ) - - actual, err := AntiSelect(input, excluded) - - assert.NoError(t, err) - assert.Equal(t, expected, actual) -} diff --git a/libs/dyn/pattern_string.go b/libs/dyn/pattern_string.go deleted file mode 100644 index 6d770faeb92..00000000000 --- a/libs/dyn/pattern_string.go +++ /dev/null @@ -1,102 +0,0 @@ -package dyn - -import ( - "fmt" - "strconv" - "strings" -) - -// MustPatternFromString is like NewPatternFromString but panics on error. -func MustPatternFromString(input string) Pattern { - p, err := NewPatternFromString(input) - if err != nil { - panic(err) - } - return p -} - -// NewPatternFromString parses a pattern from a string. -// -// The string must be a sequence of keys and indices separated by dots. -// Indices must be enclosed in square brackets. -// The string may include a leading dot. -// The wildcard character '*' can be used to match any key or index. -// -// Examples: -// - foo.bar -// - foo[1].bar -// - foo.*.bar -// - foo[*].bar -// - . -func NewPatternFromString(input string) (Pattern, error) { - var pattern Pattern - - p := input - - // Trim leading dot. - if p != "" && p[0] == '.' { - p = p[1:] - } - - for p != "" { - // Every component may have a leading dot. - if p[0] == '.' { - p = p[1:] - } - - if p == "" { - return nil, fmt.Errorf("invalid pattern: %s", input) - } - - if p[0] == '[' { - // Find next ] - i := strings.Index(p, "]") - if i < 0 { - return nil, fmt.Errorf("invalid pattern: %s", input) - } - - // Check for wildcard - if p[1:i] == "*" { - pattern = append(pattern, AnyIndex()) - } else { - // Parse index - j, err := strconv.Atoi(p[1:i]) - if err != nil { - return nil, fmt.Errorf("invalid pattern: %s", input) - } - - // Append index - pattern = append(pattern, Index(j)) - } - - p = p[i+1:] - - // The next character must be a . or [ - if p != "" && strings.IndexAny(p, ".[") != 0 { - return nil, fmt.Errorf("invalid pattern: %s", input) - } - } else { - // Find next . or [ - i := strings.IndexAny(p, ".[") - if i < 0 { - i = len(p) - } - - if i == 0 { - return nil, fmt.Errorf("invalid pattern: %s", input) - } - - // Check for wildcard - if p[:i] == "*" { - pattern = append(pattern, AnyKey()) - } else { - // Append key - pattern = append(pattern, Key(p[:i])) - } - - p = p[i:] - } - } - - return pattern, nil -} diff --git a/libs/dyn/pattern_string_test.go b/libs/dyn/pattern_string_test.go deleted file mode 100644 index 85127c12ed1..00000000000 --- a/libs/dyn/pattern_string_test.go +++ /dev/null @@ -1,147 +0,0 @@ -package dyn_test - -import ( - "errors" - "fmt" - "testing" - - . "github.com/databricks/cli/libs/dyn" - "github.com/stretchr/testify/assert" -) - -func TestNewPatternFromString(t *testing.T) { - for ind, tc := range []struct { - input string - output Pattern - err error - }{ - { - input: "", - output: NewPattern(), - }, - { - input: ".", - output: NewPattern(), - }, - { - input: "foo.bar", - output: NewPattern(Key("foo"), Key("bar")), - }, - { - input: "[1]", - output: NewPattern(Index(1)), - }, - { - input: "foo[1].bar", - output: NewPattern(Key("foo"), Index(1), Key("bar")), - }, - { - input: "foo.bar[1]", - output: NewPattern(Key("foo"), Key("bar"), Index(1)), - }, - { - input: "foo.bar[1][2]", - output: NewPattern(Key("foo"), Key("bar"), Index(1), Index(2)), - }, - { - input: "foo.bar[1][2][3]", - output: NewPattern(Key("foo"), Key("bar"), Index(1), Index(2), Index(3)), - }, - { - input: "foo[1234]", - output: NewPattern(Key("foo"), Index(1234)), - }, - { - input: "foo[123", - err: errors.New("invalid pattern: foo[123"), - }, - { - input: "foo[123]]", - err: errors.New("invalid pattern: foo[123]]"), - }, - { - input: "foo[[123]", - err: errors.New("invalid pattern: foo[[123]"), - }, - { - input: "foo[[123]]", - err: errors.New("invalid pattern: foo[[123]]"), - }, - { - input: "foo[foo]", - err: errors.New("invalid pattern: foo[foo]"), - }, - { - input: "foo..bar", - err: errors.New("invalid pattern: foo..bar"), - }, - { - input: "foo.bar.", - err: errors.New("invalid pattern: foo.bar."), - }, - { - // Every component may have a leading dot. - input: ".foo.[1].bar", - output: NewPattern(Key("foo"), Index(1), Key("bar")), - }, - { - // But after an index there must be a dot. - input: "foo[1]bar", - err: errors.New("invalid pattern: foo[1]bar"), - }, - // Wildcard tests - { - input: "foo.*", - output: NewPattern(Key("foo"), AnyKey()), - }, - { - input: "foo.*.bar", - output: NewPattern(Key("foo"), AnyKey(), Key("bar")), - }, - { - input: "foo[*]", - output: NewPattern(Key("foo"), AnyIndex()), - }, - { - input: "foo[*].bar", - output: NewPattern(Key("foo"), AnyIndex(), Key("bar")), - }, - { - input: "*[1]", - output: NewPattern(AnyKey(), Index(1)), - }, - { - input: "*.*", - output: NewPattern(AnyKey(), AnyKey()), - }, - { - input: "*[*]", - output: NewPattern(AnyKey(), AnyIndex()), - }, - } { - t.Run(fmt.Sprintf("%d %s", ind, tc.input), func(t *testing.T) { - p, err := NewPatternFromString(tc.input) - if tc.err != nil { - assert.EqualError(t, err, tc.err.Error(), tc.input) - } else { - assert.NoError(t, err) - assert.Equal(t, tc.output, p) - } - }) - } -} - -func TestMustPatternFromString(t *testing.T) { - // Test valid pattern - p := MustPatternFromString("foo.bar") - assert.Equal(t, NewPattern(Key("foo"), Key("bar")), p) - - // Test with wildcards - p = MustPatternFromString("foo.*.bar[*]") - assert.Equal(t, NewPattern(Key("foo"), AnyKey(), Key("bar"), AnyIndex()), p) - - // Test that invalid pattern panics - assert.Panics(t, func() { - MustPatternFromString("foo[") - }) -} diff --git a/libs/dyn/pattern_test.go b/libs/dyn/pattern_test.go index be78b6df61a..2a2b1e5030e 100644 --- a/libs/dyn/pattern_test.go +++ b/libs/dyn/pattern_test.go @@ -8,28 +8,6 @@ import ( "github.com/stretchr/testify/assert" ) -func TestPatternString(t *testing.T) { - patterns := []string{ - "foo.bar", - "foo.bar.baz", - "foo[1]", - "foo[*]", - "*[1].bar", - "foo.*.bar", - "foo[*].bar", - "", - "foo", - "[1]", - "[*]", - "*", - } - - for _, p := range patterns { - pp := dyn.MustPatternFromString(p) - assert.Equal(t, p, pp.String()) - } -} - func TestNewPattern(t *testing.T) { pat := dyn.NewPattern( dyn.Key("foo"), diff --git a/libs/dyn/walk_read_only.go b/libs/dyn/walk_read_only.go deleted file mode 100644 index 8f8d15e06ea..00000000000 --- a/libs/dyn/walk_read_only.go +++ /dev/null @@ -1,46 +0,0 @@ -package dyn - -import "errors" - -// WalkReadOnly walks the configuration tree in readonly mode and calls the given function on each node. -// The callback may return ErrSkip to skip traversal of a subtree. -// If the callback returns another error, the walk is aborted, and the error is returned. -func WalkReadOnly(v Value, fn func(p Path, v Value) error) error { - return walkReadOnly(v, EmptyPath, fn) -} - -// Unexported counterpart to WalkReadOnly. -// It carries the path leading up to the current node, -// such that it can be passed to the callback function. -func walkReadOnly(v Value, p Path, fn func(p Path, v Value) error) error { - if err := fn(p, v); err != nil { - if errors.Is(err, ErrSkip) { - return nil - } - return err - } - - switch v.Kind() { - case KindMap: - m := v.MustMap() - for _, pair := range m.Pairs() { - pk := pair.Key - pv := pair.Value - if err := walkReadOnly(pv, append(p, Key(pk.MustString())), fn); err != nil { - return err - } - } - case KindSequence: - s := v.MustSequence() - for i := range s { - if err := walkReadOnly(s[i], append(p, Index(i)), fn); err != nil { - return err - } - } - default: - // Other kinds are not walkable. - // This was a leaf node. - } - - return nil -} diff --git a/libs/dyn/walk_read_only_test.go b/libs/dyn/walk_read_only_test.go deleted file mode 100644 index f6df5344d8c..00000000000 --- a/libs/dyn/walk_read_only_test.go +++ /dev/null @@ -1,245 +0,0 @@ -package dyn_test - -import ( - "errors" - "testing" - - "github.com/databricks/cli/libs/dyn" - "github.com/stretchr/testify/assert" -) - -func TestWalkReadOnly(t *testing.T) { - tests := []struct { - name string - input dyn.Value - expectedPaths []dyn.Path - expectedValues []dyn.Value - }{ - { - name: "simple map", - input: dyn.V(dyn.NewMappingFromPairs( - []dyn.Pair{ - {Key: dyn.V("a"), Value: dyn.V("1")}, - {Key: dyn.V("b"), Value: dyn.V("2")}, - }, - )), - expectedPaths: []dyn.Path{ - dyn.EmptyPath, - {dyn.Key("a")}, - {dyn.Key("b")}, - }, - expectedValues: []dyn.Value{ - dyn.V(dyn.NewMappingFromPairs( - []dyn.Pair{ - {Key: dyn.V("a"), Value: dyn.V("1")}, - {Key: dyn.V("b"), Value: dyn.V("2")}, - }, - )), - dyn.V("1"), - dyn.V("2"), - }, - }, - { - name: "nested map", - input: dyn.V(map[string]dyn.Value{ - "a": dyn.V(map[string]dyn.Value{ - "b": dyn.V("1"), - }), - }), - expectedPaths: []dyn.Path{ - dyn.EmptyPath, - {dyn.Key("a")}, - {dyn.Key("a"), dyn.Key("b")}, - }, - expectedValues: []dyn.Value{ - dyn.V(map[string]dyn.Value{ - "a": dyn.V(map[string]dyn.Value{ - "b": dyn.V("1"), - }), - }), - dyn.V(map[string]dyn.Value{ - "b": dyn.V("1"), - }), - dyn.V("1"), - }, - }, - { - name: "sequence", - input: dyn.V([]dyn.Value{ - dyn.V("1"), - dyn.V("2"), - }), - expectedPaths: []dyn.Path{ - dyn.EmptyPath, - {dyn.Index(0)}, - {dyn.Index(1)}, - }, - expectedValues: []dyn.Value{ - dyn.V([]dyn.Value{ - dyn.V("1"), - dyn.V("2"), - }), - dyn.V("1"), - dyn.V("2"), - }, - }, - { - name: "nested sequence", - input: dyn.V([]dyn.Value{ - dyn.V([]dyn.Value{ - dyn.V("1"), - }), - }), - expectedPaths: []dyn.Path{ - dyn.EmptyPath, - {dyn.Index(0)}, - {dyn.Index(0), dyn.Index(0)}, - }, - expectedValues: []dyn.Value{ - dyn.V([]dyn.Value{ - dyn.V([]dyn.Value{ - dyn.V("1"), - }), - }), - dyn.V([]dyn.Value{ - dyn.V("1"), - }), - dyn.V("1"), - }, - }, - { - name: "complex structure", - input: dyn.V(map[string]dyn.Value{ - "a": dyn.V([]dyn.Value{ - dyn.V(map[string]dyn.Value{ - "b": dyn.V("1"), - }), - }), - }), - expectedPaths: []dyn.Path{ - dyn.EmptyPath, - {dyn.Key("a")}, - {dyn.Key("a"), dyn.Index(0)}, - {dyn.Key("a"), dyn.Index(0), dyn.Key("b")}, - }, - expectedValues: []dyn.Value{ - dyn.V(map[string]dyn.Value{ - "a": dyn.V([]dyn.Value{ - dyn.V(map[string]dyn.Value{ - "b": dyn.V("1"), - }), - }), - }), - dyn.V([]dyn.Value{ - dyn.V(map[string]dyn.Value{ - "b": dyn.V("1"), - }), - }), - dyn.V(map[string]dyn.Value{ - "b": dyn.V("1"), - }), - dyn.V("1"), - }, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - visitedPaths := make([]dyn.Path, 0, len(tt.expectedPaths)) - visitedValues := make([]dyn.Value, 0, len(tt.expectedValues)) - - err := dyn.WalkReadOnly(tt.input, func(p dyn.Path, v dyn.Value) error { - visitedPaths = append(visitedPaths, p) - visitedValues = append(visitedValues, v) - return nil - }) - assert.NoError(t, err) - assert.Equal(t, tt.expectedPaths, visitedPaths) - assert.Equal(t, tt.expectedValues, visitedValues) - }) - } -} - -func TestWalkReadOnly_Error(t *testing.T) { - testErr := errors.New("test error") - input := dyn.V(map[string]dyn.Value{ - "a": dyn.V("1"), - }) - - err := dyn.WalkReadOnly(input, func(p dyn.Path, v dyn.Value) error { - if p.Equal(dyn.Path{dyn.Key("a")}) { - return testErr - } - return nil - }) - - assert.Equal(t, err, testErr) -} - -func TestWalkReadOnly_SkipPaths(t *testing.T) { - va := dyn.V(dyn.NewMappingFromPairs( - []dyn.Pair{ - {Key: dyn.V("b"), Value: dyn.V("1")}, - {Key: dyn.V("c"), Value: dyn.V("2")}, - }, - )) - - vd := dyn.V(dyn.NewMappingFromPairs( - []dyn.Pair{ - {Key: dyn.V("e"), Value: dyn.V("3")}, - }, - )) - - input := dyn.V(dyn.NewMappingFromPairs( - []dyn.Pair{ - { - Key: dyn.V("a"), - Value: va, - }, - { - Key: dyn.V("d"), - Value: vd, - }, - { - Key: dyn.V("f"), - Value: dyn.V("4"), - }, - }, - )) - - skipPaths := map[string]bool{ - "a.b": true, - "d": true, - } - - var visitedPaths []dyn.Path - var visitedValues []dyn.Value - - err := dyn.WalkReadOnly(input, func(p dyn.Path, v dyn.Value) error { - _, ok := skipPaths[p.String()] - if ok { - return dyn.ErrSkip - } - - visitedPaths = append(visitedPaths, p) - visitedValues = append(visitedValues, v) - return nil - }) - assert.NoError(t, err) - - expectedPaths := []dyn.Path{ - dyn.EmptyPath, - {dyn.Key("a")}, - {dyn.Key("a"), dyn.Key("c")}, - {dyn.Key("f")}, - } - expectedValues := []dyn.Value{ - input, - va, - dyn.V("2"), - dyn.V("4"), - } - - assert.Equal(t, expectedPaths, visitedPaths) - assert.Equal(t, expectedValues, visitedValues) -}