Add data_quality_monitors bundle resource (direct engine)

.nextchanges/bundles/data-quality-monitors.md

@@ -0,0 +1 @@
+* Add the `data_quality_monitors` resource for data quality monitors on Unity Catalog schemas and tables (direct engine only). ([#7082](https://github.com/databricks/cli/pull/7082))

acceptance/bundle/deployment/bind/data_quality_monitor/databricks.yml

@@ -0,0 +1,12 @@
+bundle:
+  name: test-bundle
+
+resources:
+  data_quality_monitors:
+    monitor1:
+      object_type: table
+      object_id: 11111111-2222-3333-4444-555555555555
+      data_profiling_config:
+        assets_dir: /Workspace/Users/user@example.com/assets
+        output_schema_id: 66666666-7777-8888-9999-000000000000
+        snapshot: {}

acceptance/bundle/deployment/bind/data_quality_monitor/out.test.toml

@@ -0,0 +1,2 @@
+Cloud = false
+EnvMatrix.DMS = [""]

acceptance/bundle/deployment/bind/data_quality_monitor/output.txt

@@ -0,0 +1,32 @@
+
+>>> [CLI] bundle deployment bind monitor1 table,[UUID] --auto-approve
+Successfully bound data_quality_monitor with an id 'table,[UUID]'
+Run 'bundle deploy' to deploy changes to your workspace
+
+>>> [CLI] bundle summary
+Name: test-bundle
+Target: default
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default
+Resources:
+  Data quality monitors:
+    monitor1:
+      Name: table,[UUID]
+
+=== A malformed id is rejected
+>>> musterr [CLI] bundle deployment bind monitor1 [UUID] --auto-approve
+Error: failed to fetch the resource, err: invalid data quality monitor id "[UUID]" (want <object_type>,<object_id>)
+
+>>> [CLI] bundle deployment unbind monitor1
+
+>>> [CLI] bundle summary
+Name: test-bundle
+Target: default
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default
+Resources:
+  Data quality monitors:
+    monitor1:
+      Name: table,[UUID]

acceptance/bundle/deployment/bind/data_quality_monitor/script

@@ -0,0 +1,8 @@
+trace $CLI bundle deployment bind monitor1 table,11111111-2222-3333-4444-555555555555 --auto-approve
+trace $CLI bundle summary
+
+title "A malformed id is rejected"
+trace musterr $CLI bundle deployment bind monitor1 11111111-2222-3333-4444-555555555555 --auto-approve
+
+trace $CLI bundle deployment unbind monitor1
+trace $CLI bundle summary

acceptance/bundle/deployment/bind/data_quality_monitor/test.toml

@@ -0,0 +1,20 @@
+Cloud = false
+
+Ignore = [
+    ".databricks",
+]
+
+# The bind flow issues a GET to confirm the remote monitor exists before binding.
+[[Server]]
+Pattern = "GET /api/data-quality/v1/monitors/{object_type}/{object_id}"
+Response.Body = '''
+{
+  "object_type": "table",
+  "object_id": "11111111-2222-3333-4444-555555555555",
+  "data_profiling_config": {
+    "assets_dir": "/Workspace/Users/user@example.com/assets",
+    "output_schema_id": "66666666-7777-8888-9999-000000000000",
+    "snapshot": {}
+  }
+}
+'''

acceptance/bundle/invariant/auto-migrate/test.toml

@@ -39,6 +39,8 @@ EnvMatrixExclude.no_cluster_policy = ["INPUT_CONFIG=cluster_policy.yml.tmpl"]
 EnvMatrixExclude.no_cluster_libraries = ["INPUT_CONFIG=cluster_libraries.yml.tmpl"]
 # AI Gateway securables are direct-only; the terraform deploy that seeds the migration fails for them.
 EnvMatrixExclude.no_model_service = ["INPUT_CONFIG=model_service.yml.tmpl"]
+# Data quality monitors are direct-only; the terraform deploy that seeds the migration fails for them.
+EnvMatrixExclude.no_data_quality_monitor = ["INPUT_CONFIG=data_quality_monitor.yml.tmpl"]
 
 # Cross-resource permission references (e.g. ${resources.jobs.job_b.permissions[0].level})
 # don't work in terraform mode: the terraform interpolator converts the path to

acceptance/bundle/invariant/configs/data_quality_monitor.yml.tmpl

@@ -0,0 +1,16 @@
+bundle:
+  name: test-bundle-$UNIQUE_NAME
+
+resources:
+  schemas:
+    foo_schema:
+      catalog_name: main
+      name: test-schema-$UNIQUE_NAME
+
+  data_quality_monitors:
+    foo:
+      object_type: schema
+      object_id: ${resources.schemas.foo_schema.schema_id}
+      anomaly_detection_config:
+        excluded_table_full_names:
+          - main.test-schema-$UNIQUE_NAME.excluded

acceptance/bundle/invariant/continue_293/out.test.toml

@@ -11,6 +11,7 @@ EnvMatrix.INPUT_CONFIG = [
   "job_apply_policy_default_values_job_cluster.yml.tmpl",
   "job_apply_policy_default_values_task_cluster.yml.tmpl",
   "job_apply_policy_default_values_for_each_task.yml.tmpl",
+  "data_quality_monitor.yml.tmpl",
   "database_catalog.yml.tmpl",
   "database_instance.yml.tmpl",
   "experiment.yml.tmpl",

acceptance/bundle/invariant/delete_idempotent/out.test.toml

@@ -13,6 +13,7 @@ EnvMatrix.INPUT_CONFIG = [
   "job_apply_policy_default_values_job_cluster.yml.tmpl",
   "job_apply_policy_default_values_task_cluster.yml.tmpl",
   "job_apply_policy_default_values_for_each_task.yml.tmpl",
+  "data_quality_monitor.yml.tmpl",
   "database_catalog.yml.tmpl",
   "database_instance.yml.tmpl",
   "experiment.yml.tmpl",

acceptance/bundle/invariant/destroy_idempotent/out.test.toml

@@ -13,6 +13,7 @@ EnvMatrix.INPUT_CONFIG = [
   "job_apply_policy_default_values_job_cluster.yml.tmpl",
   "job_apply_policy_default_values_task_cluster.yml.tmpl",
   "job_apply_policy_default_values_for_each_task.yml.tmpl",
+  "data_quality_monitor.yml.tmpl",
   "database_catalog.yml.tmpl",
   "database_instance.yml.tmpl",
   "experiment.yml.tmpl",

acceptance/bundle/invariant/migrate/test.toml

@@ -39,6 +39,8 @@ EnvMatrixExclude.no_cluster_policy = ["INPUT_CONFIG=cluster_policy.yml.tmpl"]
 EnvMatrixExclude.no_cluster_libraries = ["INPUT_CONFIG=cluster_libraries.yml.tmpl"]
 # AI Gateway securables are direct-only; the terraform deploy that seeds the migration fails for them.
 EnvMatrixExclude.no_model_service = ["INPUT_CONFIG=model_service.yml.tmpl"]
+# Data quality monitors are direct-only; the terraform deploy that seeds the migration fails for them.
+EnvMatrixExclude.no_data_quality_monitor = ["INPUT_CONFIG=data_quality_monitor.yml.tmpl"]
 
 # Cross-resource permission references (e.g. ${resources.jobs.job_b.permissions[0].level})
 # don't work in terraform mode: the terraform interpolator converts the path to

acceptance/bundle/invariant/no_drift/out.test.toml

@@ -13,6 +13,7 @@ EnvMatrix.INPUT_CONFIG = [
   "job_apply_policy_default_values_job_cluster.yml.tmpl",
   "job_apply_policy_default_values_task_cluster.yml.tmpl",
   "job_apply_policy_default_values_for_each_task.yml.tmpl",
+  "data_quality_monitor.yml.tmpl",
   "database_catalog.yml.tmpl",
   "database_instance.yml.tmpl",
   "experiment.yml.tmpl",

acceptance/bundle/invariant/test.toml

@@ -27,6 +27,7 @@ EnvMatrix.INPUT_CONFIG = [
     "job_apply_policy_default_values_job_cluster.yml.tmpl",
     "job_apply_policy_default_values_task_cluster.yml.tmpl",
     "job_apply_policy_default_values_for_each_task.yml.tmpl",
+    "data_quality_monitor.yml.tmpl",
     "database_catalog.yml.tmpl",
     "database_instance.yml.tmpl",
     "experiment.yml.tmpl",
@@ -115,6 +116,9 @@ EnvMatrixExclude.no_model_serving_endpoint_telemetry_on_cloud = ["CONFIG_Cloud=t
 # UC connection to create on cloud, which the invariant tests don't provision; the
 # local mock server accepts them, so these run locally only.
 EnvMatrixExclude.no_model_service_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=model_service.yml.tmpl"]
+# Cloud coverage for data quality monitors comes from
+# resources/data_quality_monitors/lifecycle; the invariant matrix runs locally.
+EnvMatrixExclude.no_data_quality_monitor_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=data_quality_monitor.yml.tmpl"]
 # Fake SQL endpoint for local tests
 
 [[Server]]

acceptance/bundle/python/data_quality_monitors-support/databricks.yml

@@ -0,0 +1,19 @@
+bundle:
+  name: my_project
+
+sync: {paths: []} # don't need to copy files
+
+python:
+  resources:
+    - "resources:load_resources"
+  mutators:
+    - "mutators:update_data_quality_monitor"
+
+resources:
+  data_quality_monitors:
+    my_data_quality_monitor_1:
+      object_type: "schema"
+      object_id: "11111111-2222-3333-4444-555555555555"
+      anomaly_detection_config:
+        excluded_table_full_names:
+          - "main.default.excluded"

acceptance/bundle/python/data_quality_monitors-support/mutators.py

@@ -0,0 +1,13 @@
+from dataclasses import replace
+
+from databricks.bundles.data_quality_monitors import DataQualityMonitor
+from databricks.bundles.core import data_quality_monitor_mutator
+
+
+@data_quality_monitor_mutator
+def update_data_quality_monitor(
+    data_quality_monitor: DataQualityMonitor,
+) -> DataQualityMonitor:
+    assert isinstance(data_quality_monitor.object_id, str)
+
+    return replace(data_quality_monitor, object_id=f"{data_quality_monitor.object_id} (updated)")

acceptance/bundle/python/data_quality_monitors-support/out.test.toml

@@ -0,0 +1,3 @@
+Cloud = false
+EnvMatrix.DMS = ["", "true"]
+EnvMatrix.PYDAB_VERSION = ["current"]

acceptance/bundle/python/data_quality_monitors-support/output.txt

@@ -0,0 +1,35 @@
+
+>>> uv run [UV_ARGS] -q [CLI] bundle validate --output json
+{
+  "experimental": {
+    "python": {
+      "mutators": [
+        "mutators:update_data_quality_monitor"
+      ],
+      "resources": [
+        "resources:load_resources"
+      ]
+    }
+  },
+  "resources": {
+    "data_quality_monitors": {
+      "my_data_quality_monitor_1": {
+        "anomaly_detection_config": {
+          "excluded_table_full_names": [
+            "main.default.excluded"
+          ]
+        },
+        "object_id": "[UUID] (updated)",
+        "object_type": "schema"
+      },
+      "my_data_quality_monitor_2": {
+        "data_profiling_config": {
+          "assets_dir": "/Workspace/Users/user@example.com/assets",
+          "output_schema_id": "[UUID]"
+        },
+        "object_id": "[UUID] (updated)",
+        "object_type": "table"
+      }
+    }
+  }
+}

acceptance/bundle/python/data_quality_monitors-support/resources.py

@@ -0,0 +1,20 @@
+from databricks.bundles.core import Resources
+
+
+def load_resources() -> Resources:
+    resources = Resources()
+
+    resources.add_data_quality_monitor(
+        "my_data_quality_monitor_2",
+        {
+            "object_type": "table",
+            "object_id": "66666666-7777-8888-9999-000000000000",
+            "data_profiling_config": {
+                "assets_dir": "/Workspace/Users/user@example.com/assets",
+                "output_schema_id": "11111111-2222-3333-4444-555555555555",
+                "snapshot": {},
+            },
+        },
+    )
+
+    return resources

acceptance/bundle/python/data_quality_monitors-support/script

@@ -0,0 +1,5 @@
+
+trace uv run $UV_ARGS -q $CLI bundle validate --output json | \
+  jq "pick(.experimental.python, .resources)"
+
+rm -fr .databricks __pycache__

acceptance/bundle/python/data_quality_monitors-support/test.toml

@@ -0,0 +1,4 @@
+Cloud = false # tests don't interact with APIs
+
+# data_quality_monitors are only supported in the current version of the wheel
+EnvMatrix.PYDAB_VERSION = ["current"]

acceptance/bundle/refschema/out.fields.txt

@@ -670,6 +670,61 @@ resources.dashboards.*.permissions[*].group_name	string	ALL
 resources.dashboards.*.permissions[*].level	iam.PermissionLevel	ALL
 resources.dashboards.*.permissions[*].service_principal_name	string	ALL
 resources.dashboards.*.permissions[*].user_name	string	ALL
+resources.data_quality_monitors.*.anomaly_detection_config	*dataquality.AnomalyDetectionConfig	ALL
+resources.data_quality_monitors.*.anomaly_detection_config.excluded_table_full_names	[]string	ALL
+resources.data_quality_monitors.*.anomaly_detection_config.excluded_table_full_names[*]	string	ALL
+resources.data_quality_monitors.*.data_profiling_config	*dataquality.DataProfilingConfig	ALL
+resources.data_quality_monitors.*.data_profiling_config.assets_dir	string	ALL
+resources.data_quality_monitors.*.data_profiling_config.baseline_table_name	string	ALL
+resources.data_quality_monitors.*.data_profiling_config.custom_metrics	[]dataquality.DataProfilingCustomMetric	ALL
+resources.data_quality_monitors.*.data_profiling_config.custom_metrics[*]	dataquality.DataProfilingCustomMetric	ALL
+resources.data_quality_monitors.*.data_profiling_config.custom_metrics[*].definition	string	ALL
+resources.data_quality_monitors.*.data_profiling_config.custom_metrics[*].input_columns	[]string	ALL
+resources.data_quality_monitors.*.data_profiling_config.custom_metrics[*].input_columns[*]	string	ALL
+resources.data_quality_monitors.*.data_profiling_config.custom_metrics[*].name	string	ALL
+resources.data_quality_monitors.*.data_profiling_config.custom_metrics[*].output_data_type	string	ALL
+resources.data_quality_monitors.*.data_profiling_config.custom_metrics[*].type	dataquality.DataProfilingCustomMetricType	ALL
+resources.data_quality_monitors.*.data_profiling_config.dashboard_id	string	ALL
+resources.data_quality_monitors.*.data_profiling_config.drift_metrics_table_name	string	ALL
+resources.data_quality_monitors.*.data_profiling_config.effective_warehouse_id	string	ALL
+resources.data_quality_monitors.*.data_profiling_config.inference_log	*dataquality.InferenceLogConfig	ALL
+resources.data_quality_monitors.*.data_profiling_config.inference_log.granularities	[]dataquality.AggregationGranularity	ALL
+resources.data_quality_monitors.*.data_profiling_config.inference_log.granularities[*]	dataquality.AggregationGranularity	ALL
+resources.data_quality_monitors.*.data_profiling_config.inference_log.label_column	string	ALL
+resources.data_quality_monitors.*.data_profiling_config.inference_log.model_id_column	string	ALL
+resources.data_quality_monitors.*.data_profiling_config.inference_log.prediction_column	string	ALL
+resources.data_quality_monitors.*.data_profiling_config.inference_log.problem_type	dataquality.InferenceProblemType	ALL
+resources.data_quality_monitors.*.data_profiling_config.inference_log.timestamp_column	string	ALL
+resources.data_quality_monitors.*.data_profiling_config.latest_monitor_failure_message	string	ALL
+resources.data_quality_monitors.*.data_profiling_config.monitor_version	int64	ALL
+resources.data_quality_monitors.*.data_profiling_config.monitored_table_name	string	ALL
+resources.data_quality_monitors.*.data_profiling_config.notification_settings	*dataquality.NotificationSettings	ALL
+resources.data_quality_monitors.*.data_profiling_config.notification_settings.on_failure	*dataquality.NotificationDestination	ALL
+resources.data_quality_monitors.*.data_profiling_config.notification_settings.on_failure.email_addresses	[]string	ALL
+resources.data_quality_monitors.*.data_profiling_config.notification_settings.on_failure.email_addresses[*]	string	ALL
+resources.data_quality_monitors.*.data_profiling_config.output_schema_id	string	ALL
+resources.data_quality_monitors.*.data_profiling_config.profile_metrics_table_name	string	ALL
+resources.data_quality_monitors.*.data_profiling_config.schedule	*dataquality.CronSchedule	ALL
+resources.data_quality_monitors.*.data_profiling_config.schedule.pause_status	dataquality.CronSchedulePauseStatus	ALL
+resources.data_quality_monitors.*.data_profiling_config.schedule.quartz_cron_expression	string	ALL
+resources.data_quality_monitors.*.data_profiling_config.schedule.timezone_id	string	ALL
+resources.data_quality_monitors.*.data_profiling_config.skip_builtin_dashboard	bool	ALL
+resources.data_quality_monitors.*.data_profiling_config.slicing_exprs	[]string	ALL
+resources.data_quality_monitors.*.data_profiling_config.slicing_exprs[*]	string	ALL
+resources.data_quality_monitors.*.data_profiling_config.snapshot	*dataquality.SnapshotConfig	ALL
+resources.data_quality_monitors.*.data_profiling_config.status	dataquality.DataProfilingStatus	ALL
+resources.data_quality_monitors.*.data_profiling_config.time_series	*dataquality.TimeSeriesConfig	ALL
+resources.data_quality_monitors.*.data_profiling_config.time_series.granularities	[]dataquality.AggregationGranularity	ALL
+resources.data_quality_monitors.*.data_profiling_config.time_series.granularities[*]	dataquality.AggregationGranularity	ALL
+resources.data_quality_monitors.*.data_profiling_config.time_series.timestamp_column	string	ALL
+resources.data_quality_monitors.*.data_profiling_config.warehouse_id	string	ALL
+resources.data_quality_monitors.*.id	string	INPUT
+resources.data_quality_monitors.*.lifecycle	resources.Lifecycle	INPUT
+resources.data_quality_monitors.*.lifecycle.prevent_destroy	bool	INPUT
+resources.data_quality_monitors.*.modified_status	string	INPUT
+resources.data_quality_monitors.*.object_id	string	ALL
+resources.data_quality_monitors.*.object_type	string	ALL
+resources.data_quality_monitors.*.url	string	INPUT
 resources.database_catalogs.*.create_database_if_not_exists	bool	ALL
 resources.database_catalogs.*.database_instance_name	string	ALL
 resources.database_catalogs.*.database_name	string	ALL

acceptance/bundle/resources/data_quality_monitors/basic/databricks.yml

@@ -0,0 +1,16 @@
+bundle:
+  name: test-bundle
+
+resources:
+  schemas:
+    myschema:
+      catalog_name: main
+      name: myschema
+
+  data_quality_monitors:
+    schema_monitor:
+      object_type: schema
+      object_id: ${resources.schemas.myschema.schema_id}
+      anomaly_detection_config:
+        excluded_table_full_names:
+          - main.myschema.table1

acceptance/bundle/resources/data_quality_monitors/basic/out.test.toml

@@ -0,0 +1,2 @@
+Cloud = false
+EnvMatrix.DMS = ["", "true"]