diff --git a/knowledge_base/job_notifications_and_health/README.md b/knowledge_base/job_notifications_and_health/README.md new file mode 100644 index 0000000..fb71be8 --- /dev/null +++ b/knowledge_base/job_notifications_and_health/README.md @@ -0,0 +1,52 @@ +# Job notifications and health + +This Declarative Automation Bundles example defines a serverless job with notifications, health rules, timeouts, and retries. + +The Python task can sleep or fail. Use its job parameters to test alerts. + +For configuration details, see the [job notifications documentation](https://docs.databricks.com/aws/en/jobs/notifications). + +## Prerequisites + +- Databricks CLI v1.20.0 or above +- Configure a Databricks CLI profile for the target workspace. +- Use a workspace that supports serverless jobs environment version `6`. +- Create a webhook notification destination in the workspace. +- Use an email address that can receive notifications. + +## Usage + +1. Set the notification variables. + + Replace both placeholder values with real production values. + + ```bash + export BUNDLE_VAR_notification_email=USER@example.com + export BUNDLE_VAR_notification_destination_id=DESTINATION_UUID + ``` + +2. Deploy the development target without notifications. + + ```bash + databricks bundle deploy -t dev + ``` + +3. Deploy and run the production target. + + ```bash + databricks bundle deploy -t prod + databricks bundle run -t prod notification_health_job + ``` + +4. Test a duration alert by running the task for 90 seconds. + + ```bash + databricks bundle run -t prod notification_health_job --params sleep_seconds=90,fail=false + ``` + +5. Remove the deployed resources after testing. + + ```bash + databricks bundle destroy -t dev + databricks bundle destroy -t prod + ``` diff --git a/knowledge_base/job_notifications_and_health/databricks.yml b/knowledge_base/job_notifications_and_health/databricks.yml new file mode 100644 index 0000000..5aebaeb --- /dev/null +++ b/knowledge_base/job_notifications_and_health/databricks.yml @@ -0,0 +1,42 @@ +bundle: + name: job_notifications_and_health + +include: + - resources/*.job.yml + +variables: + notification_email: + description: Email address that receives job notifications. + default: replace-me@example.com + notification_destination_id: + description: ID of a configured webhook notification destination. + default: 00000000-0000-0000-0000-000000000000 + +targets: + dev: + default: true + mode: development + workspace: + root_path: /Workspace/bundle-examples/${bundle.name}/${bundle.target}/${workspace.current_user.short_name} + + prod: + mode: production + workspace: + root_path: /Workspace/bundle-examples/${bundle.name}/${bundle.target} + resources: + jobs: + notification_health_job: + email_notifications: + on_failure: + - ${var.notification_email} + on_duration_warning_threshold_exceeded: + - ${var.notification_email} + on_streaming_backlog_exceeded: + - ${var.notification_email} + webhook_notifications: + on_failure: + - id: ${var.notification_destination_id} + on_duration_warning_threshold_exceeded: + - id: ${var.notification_destination_id} + on_streaming_backlog_exceeded: + - id: ${var.notification_destination_id} diff --git a/knowledge_base/job_notifications_and_health/resources/notification_health_job.job.yml b/knowledge_base/job_notifications_and_health/resources/notification_health_job.job.yml new file mode 100644 index 0000000..3b13e49 --- /dev/null +++ b/knowledge_base/job_notifications_and_health/resources/notification_health_job.job.yml @@ -0,0 +1,55 @@ +resources: + jobs: + notification_health_job: + name: Job notifications and health + description: Demonstrates notifications, health rules, timeouts, and retries. + + parameters: + - name: sleep_seconds + default: "0" + - name: fail + default: "false" + + email_notifications: + on_failure: [] + on_duration_warning_threshold_exceeded: [] + on_streaming_backlog_exceeded: [] + + webhook_notifications: + on_failure: [] + on_duration_warning_threshold_exceeded: [] + on_streaming_backlog_exceeded: [] + + notification_settings: + no_alert_for_skipped_runs: true + no_alert_for_canceled_runs: true + + health: + rules: + - metric: RUN_DURATION_SECONDS + op: GREATER_THAN + value: 60 + # This preview metric applies only when the job runs a stream. + - metric: STREAMING_BACKLOG_RECORDS + op: GREATER_THAN + value: 1000 + + tasks: + - task_key: controllable_task + spark_python_task: + python_file: ../src/main.py + parameters: + - --sleep-seconds + - "{{job.parameters.sleep_seconds}}" + - --fail + - "{{job.parameters.fail}}" + environment_key: default + timeout_seconds: 120 + max_retries: 1 + min_retry_interval_millis: 10000 + retry_on_timeout: true + + environments: + - environment_key: default + spec: + environment_version: "6" diff --git a/knowledge_base/job_notifications_and_health/src/main.py b/knowledge_base/job_notifications_and_health/src/main.py new file mode 100644 index 0000000..cc3d918 --- /dev/null +++ b/knowledge_base/job_notifications_and_health/src/main.py @@ -0,0 +1,19 @@ +import argparse +import time + + +parser = argparse.ArgumentParser() +parser.add_argument("--sleep-seconds", type=int, default=0) +parser.add_argument("--fail", choices=("true", "false"), default="false") +args = parser.parse_args() + +if args.sleep_seconds < 0: + parser.error("--sleep-seconds must be zero or greater") + +print(f"Sleeping for {args.sleep_seconds} seconds") +time.sleep(args.sleep_seconds) + +if args.fail == "true": + raise RuntimeError("Intentional failure requested") + +print("Task completed successfully")