bundle: delete libs/dyn; configuration is typed structs with structvar refs and locations

.agents/rules/style-guide-go.md

@@ -113,7 +113,7 @@ return fieldPaths
 
 ### Encoding
 
-**RULE: When mutating an API response by round-tripping JSON, decode with `json.Decoder` + `UseNumber()`.** The naive `json.Marshal` → `map[string]any` → `Marshal` path corrupts any int64 larger than 2^53 (it degrades to a float64 mantissa — e.g. a real `spark_context_id`) and alphabetizes object keys. `libs/dyn/jsonloader` preserves key order but also lacks `UseNumber`, so it shares the int64 hazard.
+**RULE: When mutating an API response by round-tripping JSON, decode with `json.Decoder` + `UseNumber()`.** The naive `json.Marshal` → `map[string]any` → `Marshal` path corrupts any int64 larger than 2^53 (it degrades to a float64 mantissa — e.g. a real `spark_context_id`) and alphabetizes object keys.
 
 **RULE: Be careful with `encoding/csv` `Writer.UseCRLF = true`.** It rewrites both record terminators AND embedded newlines inside quoted fields to `\r\n`, so tests for quoted multiline fields must expect `\r\n`, not just the line endings between rows.
 
@@ -191,7 +191,7 @@ func NewService(deps ServiceDeps) *Service { ... }
 
 ### Configuration patterns
 
-- Bundle config uses `dyn.Value` for dynamic typing
+- Bundle config is typed structs; `libs/structs/structvar` carries `${...}` references and source locations
 - Config loading supports includes, variable interpolation, and target overrides
 - Schema generation is automated from Go struct tags
 

.github/workflows/python_push.yml

@@ -39,7 +39,7 @@ jobs:
             acceptance/bundle/python/ \
             go.mod \
             go.sum \
-            libs/dyn/dynvar/testdata/reference_vectors.json \
+            libs/structs/structvar/testdata/reference_vectors.json \
             python/ \
             tools/task/ \
           )

.nextchanges/bundles/6954.md

@@ -0,0 +1 @@
+* Speed up bundle commands on large bundles: configuration is kept as typed structs instead of being converted to and from a dynamic tree (`bundle validate` and `bundle plan` with 1000 jobs are about 10x faster and use about 3.5x less memory). ([#6954](https://github.com/databricks/cli/pull/6954))

AGENTS.md

@@ -14,7 +14,7 @@ This is the Databricks CLI, a command-line interface for interacting with Databr
 
 **RULE: Keep each PR focused on one change.** If you notice an unrelated cleanup, bug fix, or refactor while making your primary change, leave it alone or put it in a separate PR. Reviewers consistently ask to split mixed PRs, especially when a dependency bump or schema diff rides along with a feature change.
 
-**RULE: Before adding a new helper, search the codebase for an existing one.** Common homes: `libs/` (shared utilities), `libs/databrickscfg/` (config), `libs/git/`, `libs/filer/`, `libs/cmdio/` (CLI I/O, spinners, prompts), `libs/env/` (env vars), `libs/testserver/`, `libs/structpath/` and `libs/dyn/` (path / dynamic values), `acceptance/bin/` (acceptance test helpers), `internal/mocks/` (generated mocks). A function that duplicates an existing name and signature in the same package is a compile error waiting to happen; grep before you name.
+**RULE: Before adding a new helper, search the codebase for an existing one.** Common homes: `libs/` (shared utilities), `libs/databrickscfg/` (config), `libs/git/`, `libs/filer/`, `libs/cmdio/` (CLI I/O, spinners, prompts), `libs/env/` (env vars), `libs/testserver/`, `libs/structs/structpath/` and `libs/structs/structvar/` (paths / typed config values), `acceptance/bin/` (acceptance test helpers), `internal/mocks/` (generated mocks). A function that duplicates an existing name and signature in the same package is a compile error waiting to happen; grep before you name.
 
 # Development Commands
 
@@ -73,7 +73,7 @@ GIT_EDITOR=true GIT_SEQUENCE_EDITOR=true VISUAL=true GIT_PAGER=cat git rebase or
 - `bundle/phases/` - High-level deployment phases
 
 **libs/** - Shared libraries and utilities
-- `libs/dyn/` - Dynamic configuration value manipulation
+- `libs/structs/structvar/` - Typed configuration values with references and locations
 - `libs/filer/` - File system abstraction (local, DBFS, workspace)
 - `libs/auth/` - Databricks authentication handling
 - `libs/sync/` - File synchronization between local and remote

acceptance/invariant_test.go

@@ -8,10 +8,9 @@ import (
 	"testing"
 
 	"github.com/databricks/cli/bundle/config"
-	"github.com/databricks/cli/libs/dyn"
-	"github.com/databricks/cli/libs/dyn/yamlloader"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"go.yaml.in/yaml/v3"
 )
 
 const invariantConfigsDir = "bundle/invariant/configs"
@@ -88,31 +87,22 @@ func scanInvariantConfigs(t *testing.T) (present, withPermissions, withGrants ma
 		contents, err := os.ReadFile(path)
 		require.NoError(t, err)
 
-		v, err := yamlloader.LoadYAML(path, strings.NewReader(string(contents)))
-		require.NoError(t, err, "failed to parse %s", path)
-
-		resources := v.Get("resources")
-		if resources.Kind() != dyn.KindMap {
-			// Some configs (e.g. PyDABs) declare resources outside of YAML.
-			continue
+		var doc struct {
+			Resources map[string]any `yaml:"resources"`
 		}
+		require.NoError(t, yaml.Unmarshal(contents, &doc), "failed to parse %s", path)
 
-		for _, group := range resources.MustMap().Pairs() {
-			groupName := group.Key.MustString()
+		// Some configs (e.g. PyDABs) declare resources outside of YAML.
+		for groupName, group := range doc.Resources {
 			present[groupName] = true
 
-			if group.Value.Kind() != dyn.KindMap {
-				continue
-			}
-			for _, resource := range group.Value.MustMap().Pairs() {
-				cfg := resource.Value
-				if cfg.Kind() != dyn.KindMap {
-					continue
-				}
-				if cfg.Get("permissions").Kind() != dyn.KindInvalid {
+			resources, _ := group.(map[string]any)
+			for _, resource := range resources {
+				cfg, _ := resource.(map[string]any)
+				if _, ok := cfg["permissions"]; ok {
 					withPermissions[groupName] = true
 				}
-				if cfg.Get("grants").Kind() != dyn.KindInvalid {
+				if _, ok := cfg["grants"]; ok {
 					withGrants[groupName] = true
 				}
 			}

bundle/apps/validate.go

@@ -8,8 +8,8 @@ import (
 	"github.com/databricks/cli/bundle"
 	"github.com/databricks/cli/bundle/config/resources"
 	"github.com/databricks/cli/libs/diag"
-	"github.com/databricks/cli/libs/dyn"
 	"github.com/databricks/cli/libs/structs/structpath"
+	"github.com/databricks/cli/libs/structs/structvar"
 	"github.com/databricks/databricks-sdk-go/service/apps"
 )
 
@@ -28,7 +28,7 @@ func (v *validate) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics
 				Severity:  diag.Error,
 				Summary:   "Missing app source code path or git source",
 				Detail:    fmt.Sprintf("app resource '%s' should have either source_code_path or git_source field", key),
-				Locations: b.Config.GetLocationsOf(structpath.NewPath(nil, "resources", "apps", key)),
+				Locations: b.Config.NearestLocations(structpath.NewPath(nil, "resources", "apps", key)),
 			})
 			continue
 		}
@@ -38,7 +38,7 @@ func (v *validate) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics
 				Severity:  diag.Error,
 				Summary:   "Both source_code_path and git_source fields are set",
 				Detail:    fmt.Sprintf("app resource '%s' should have either source_code_path or git_source field, not both", key),
-				Locations: b.Config.GetLocationsOf(structpath.NewPath(nil, "resources", "apps", key)),
+				Locations: b.Config.NearestLocations(structpath.NewPath(nil, "resources", "apps", key)),
 			})
 			continue
 		}
@@ -48,7 +48,7 @@ func (v *validate) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics
 				Severity:  diag.Error,
 				Summary:   "Duplicate app source code path",
 				Detail:    fmt.Sprintf("app resource '%s' has the same source code path as app resource '%s', this will lead to the app configuration being overridden by each other", key, usedSourceCodePaths[app.SourceCodePath]),
-				Locations: b.Config.GetLocationsOf(structpath.NewPath(nil, "resources", "apps", key, "source_code_path")),
+				Locations: b.Config.NearestLocations(structpath.NewPath(nil, "resources", "apps", key, "source_code_path")),
 			})
 		}
 		usedSourceCodePaths[app.SourceCodePath] = key
@@ -92,34 +92,20 @@ type appResourceReference struct {
 	permission   string
 }
 
-// hasPermissions checks if a bundle resource at the given dyn path has a non-empty permissions list.
-func hasPermissions(b *bundle.Bundle, resourcePath string) bool {
-	pv, err := dyn.Get(b.Config.Value(), resourcePath+".permissions")
-	if err != nil {
-		return false
+// hasPermissions checks if a bundle resource has a non-empty permissions list.
+func hasPermissions(resource structvar.View) bool {
+	for range resource.Get("permissions").Sequence() {
+		return true
 	}
-	s, ok := pv.AsSequence()
-	return ok && len(s) > 0
+	return false
 }
 
 // hasAppSPInPermissions checks if any permission entry for the given resource
 // references the app's service principal via variable interpolation.
-func hasAppSPInPermissions(b *bundle.Bundle, resourcePath, appKey string) bool {
+func hasAppSPInPermissions(resource structvar.View, appKey string) bool {
 	appSPRef := fmt.Sprintf("${resources.apps.%s.service_principal_client_id}", appKey)
-	pv, err := dyn.Get(b.Config.Value(), resourcePath+".permissions")
-	if err != nil {
-		return false
-	}
-	s, ok := pv.AsSequence()
-	if !ok {
-		return false
-	}
-	for _, entry := range s {
-		spn, err := dyn.Get(entry, "service_principal_name")
-		if err != nil {
-			continue
-		}
-		if str, ok := spn.AsString(); ok && str == appSPRef {
+	for _, entry := range resource.Get("permissions").Sequence() {
+		if str, ok := entry.Get("service_principal_name").AsString(); ok && str == appSPRef {
 			return true
 		}
 	}
@@ -146,12 +132,12 @@ func warnForAppResourcePermissions(b *bundle.Bundle, appKey string, app *resourc
 		}
 		refType, resourceKey := matches[1], matches[2]
 
-		resourcePath := fmt.Sprintf("resources.%s.%s", refType, resourceKey)
-		if !hasPermissions(b, resourcePath) {
+		resource := b.Config.View().Get("resources").Get(refType).Get(resourceKey)
+		if !hasPermissions(resource) {
 			continue
 		}
 
-		if hasAppSPInPermissions(b, resourcePath, appKey) {
+		if hasAppSPInPermissions(resource, appKey) {
 			continue
 		}
 
@@ -174,7 +160,7 @@ func warnForAppResourcePermissions(b *bundle.Bundle, appKey string, app *resourc
 				appKey,
 			),
 			Paths:     []*structpath.PathNode{appPath},
-			Locations: b.Config.GetLocationsOf(appPath),
+			Locations: b.Config.NearestLocations(appPath),
 		})
 	}
 

bundle/artifacts/expand_globs.go

@@ -6,16 +6,17 @@ import (
 	"path/filepath"
 
 	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/bundle/config"
 	"github.com/databricks/cli/libs/diag"
-	"github.com/databricks/cli/libs/dyn"
-	"github.com/databricks/cli/libs/log"
 	"github.com/databricks/cli/libs/patchwheel"
+	"github.com/databricks/cli/libs/structs/structpath"
+	"github.com/databricks/cli/libs/structs/structvar"
 )
 
-func createGlobError(v dyn.Value, p dyn.Path, message string) diag.Diagnostic {
+func createGlobError(v structvar.View, p *structpath.PathNode, message string) diag.Diagnostic {
 	// The pattern contained in v is an absolute path.
 	// Make it relative to the value's location to make it more readable.
-	source := v.MustString()
+	source, _ := v.AsString()
 	if l := v.Location(); l.File != "" {
 		rel, err := filepath.Rel(filepath.Dir(l.File), source)
 		if err == nil {
@@ -27,7 +28,7 @@ func createGlobError(v dyn.Value, p dyn.Path, message string) diag.Diagnostic {
 		Severity:  diag.Error,
 		Summary:   fmt.Sprintf("%s: %s", source, message),
 		Locations: []diag.Location{v.Location()},
-		Paths:     dyn.ToStructPaths(p),
+		Paths:     []*structpath.PathNode{p},
 	}
 }
 
@@ -42,82 +43,78 @@ func (e expandGlobs) Name() string {
 func (e expandGlobs) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
 	// Base path for this mutator.
 	// This path is set with the list of expanded globs when done.
-	base := dyn.NewPath(
-		dyn.Key("artifacts"),
-		dyn.Key(e.name),
-		dyn.Key("files"),
-	)
+	base := structpath.NewPath(nil, "artifacts", e.name, "files")
 
 	// Pattern to match the source key in the files sequence.
-	pattern := dyn.NewPatternFromPath(base).Append(
-		dyn.AnyIndex(),
-		dyn.Key("source"),
-	)
+	pattern := structpath.NewPattern(nil, "artifacts", e.name, "files", structpath.AnyIndex, "source")
+
+	artifact := b.Config.Artifacts[e.name]
+	if artifact == nil {
+		return nil
+	}
 
 	var diags diag.Diagnostics
-	err := b.Config.Mutate(func(rootv dyn.Value) (dyn.Value, error) {
-		var output []dyn.Value
-		_, err := dyn.MapByPattern(rootv, pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) {
-			if v.Kind() != dyn.KindString {
-				return v, nil
-			}
-
-			source := v.MustString()
-
-			// Expand any glob reference in files source path
-			matches, err := filepath.Glob(source)
-			if err != nil {
-				diags = diags.Append(createGlobError(v, p, err.Error()))
-
-				// Drop this value from the list; this does not matter since we've raised an error anyway
-				return v, nil
-			}
-
-			// Note, we're applying this for all artifact types, not just "whl".
-			// Rationale:
-			//  1. type is optional
-			//  2. if you have wheels in other artifact type, maybe you still want the filter logic? impossible to say.
-			matches = patchwheel.FilterLatestWheels(ctx, matches)
-
-			if len(matches) == 1 && matches[0] == source {
-				// No glob expansion was performed.
-				// Keep node unchanged. We need to ensure that "patched" field remains and not wiped out by code below.
-				parent, err := dyn.GetByPath(rootv, p[0:len(p)-1])
-				if err != nil {
-					log.Debugf(ctx, "Failed to get parent of %s", p.String())
-				} else {
-					output = append(output, parent)
-				}
-				return v, nil
-			}
-
-			if len(matches) == 0 {
-				diags = diags.Append(createGlobError(v, p, "no matching files"))
-
-				// Drop this value from the list; this does not matter since we've raised an error anyway
-				return v, nil
-			}
-
-			for _, match := range matches {
-				output = append(output, dyn.V(
-					map[string]dyn.Value{
-						"source": dyn.NewValue(match, v.Locations()),
-					},
-				))
-			}
-
-			return v, nil
-		})
-
-		if err != nil || diags.HasError() {
-			return rootv, err
+	var output []config.ArtifactFile
+	var sources [][]int
+	var locations [][]diag.Location
+	err := structvar.ForEach(b.Config.View(), pattern, func(np *structpath.PathNode, v structvar.View) error {
+		if v.Kind() != structvar.KindString {
+			return nil
+		}
+
+		index, _ := np.Parent().Index()
+		source, _ := v.AsString()
+
+		// Expand any glob reference in files source path
+		matches, err := filepath.Glob(source)
+		if err != nil {
+			diags = diags.Append(createGlobError(v, np, err.Error()))
+
+			// Drop this value from the list; this does not matter since we've raised an error anyway
+			return nil
+		}
+
+		// Note, we're applying this for all artifact types, not just "whl".
+		// Rationale:
+		//  1. type is optional
+		//  2. if you have wheels in other artifact type, maybe you still want the filter logic? impossible to say.
+		matches = patchwheel.FilterLatestWheels(ctx, matches)
+
+		if len(matches) == 1 && matches[0] == source {
+			// No glob expansion was performed.
+			// Keep node unchanged. We need to ensure that "patched" field remains and not wiped out by code below.
+			output = append(output, artifact.Files[index])
+			sources = append(sources, []int{index})
+			locations = append(locations, nil)
+			return nil
 		}
 
-		// Set the expanded globs back into the configuration.
-		return dyn.SetByPath(rootv, base, dyn.V(output))
+		if len(matches) == 0 {
+			diags = diags.Append(createGlobError(v, np, "no matching files"))
+
+			// Drop this value from the list; this does not matter since we've raised an error anyway
+			return nil
+		}
+
+		for _, match := range matches {
+			output = append(output, config.ArtifactFile{Source: match})
+			sources = append(sources, []int{index})
+			locations = append(locations, v.Locations())
+		}
+
+		return nil
 	})
-	if err != nil {
-		diags = diags.Extend(diag.FromErr(err))
+	if err != nil || diags.HasError() {
+		return diags.Extend(diag.FromErr(err))
+	}
+
+	// Set the expanded globs back into the configuration.
+	artifact.Files = output
+	b.Config.UpdateSequence(base, sources)
+	for i, locs := range locations {
+		if locs != nil {
+			b.Config.SetLocations(structpath.NewPath(base, i, "source"), locs)
+		}
 	}
 
 	return diags