Resolve relative paths in variable overrides against the bundle root

.nextchanges/bundles/7049.md

@@ -0,0 +1 @@
+* Relative paths in variable values from `.databricks/bundle/<target>/variable-overrides.json` now resolve against the bundle root. ([#7049](https://github.com/databricks/cli/pull/7049))

acceptance/bundle/validate/variable_overrides_file_complex_path/output.txt

@@ -1,19 +1,20 @@
 
 >>> [CLI] bundle validate -o json
-Error: file .databricks/bundle/default/src/main.py not found
+Warning: unknown field: unknown_field
+  at resources.jobs.my_job.tasks[0]
+  in <variable-overrides>:4:6
 
-
-Exit code: 1
+Warn: unable to get spark version for cluster 0123-456789-abcdefgh, err: Not Found
 [
   {
     "existing_cluster_id": "0123-456789-abcdefgh",
     "libraries": [
       {
-        "whl": ".databricks/bundle/default/src/lib.whl"
+        "whl": "src/lib.whl"
       }
     ],
     "spark_python_task": {
-      "python_file": ".databricks/bundle/default/src/main.py"
+      "python_file": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/src/main.py"
     },
     "task_key": "main"
   }

acceptance/bundle/validate/variable_overrides_file_complex_path/test.toml

@@ -1,5 +1,3 @@
-Badness = "Relative paths in variable-overrides.json resolve inconsistently: task files are resolved against the bundle root (with an error), libraries relative to the overrides file."
-
 [[Repls]]
 Old = '\\'
 New = '/'

acceptance/bundle/validate/variable_overrides_file_complex_path/variable-overrides.json

@@ -1,6 +1,7 @@
 {
   "task": {
     "task_key": "main",
+    "unknown_field": 1,
     "existing_cluster_id": "0123-456789-abcdefgh",
     "spark_python_task": {
       "python_file": "./src/main.py"

bundle/config/mutator/set_variables.go

@@ -16,6 +16,14 @@ import (
 
 const bundleVarPrefix = "BUNDLE_VAR_"
 
+// variableOverridesLocation is the file name in the bundle root that values from the
+// variable overrides file are located at. The file is written by tools, not users, so
+// relative paths in it must resolve against the bundle root. A location serves both
+// diagnostics and path resolution (relative to the directory of its file), so we use a
+// fake file at the bundle root. The limitation is that diagnostics name this label
+// rather than the real file under .databricks.
+const variableOverridesLocation = "<variable-overrides>"
+
 type setVariables struct{}
 
 func SetVariables() bundle.Mutator {
@@ -107,8 +115,10 @@ func readVariablesFromFile(b *bundle.Bundle) (dyn.Value, diag.Diagnostics) {
 		return dyn.InvalidValue, diag.FromErr(fmt.Errorf("failed to read variables file: %w", err))
 	}
 
-	val, err := jsonloader.LoadJSON(f, filePath)
+	val, err := jsonloader.LoadJSON(f, filepath.Join(b.BundleRootPath, variableOverridesLocation))
 	if err != nil {
+		// Report the parse error at the real file.
+		_, err = jsonloader.LoadJSON(f, filePath)
 		return dyn.InvalidValue, diag.FromErr(fmt.Errorf("failed to parse variables file %s: %w", filePath, err))
 	}