From df3bc590b25155eb8ae5166a874b9b2c64a1c57a Mon Sep 17 00:00:00 2001 From: Andres Garcia Date: Wed, 21 Oct 2026 22:12:52 +0000 Subject: [PATCH 01/12] Add CI/CD knowledge base examples Co-authored-by: omnigent --- knowledge_base/ci_cd_azure_devops/README.md | 77 ++++++++++++++++++ .../ci_cd_azure_devops/azure-pipelines.yml | 72 +++++++++++++++++ .../ci_cd_azure_devops/databricks.yml | 15 ++++ .../resources/serverless.job.yml | 13 +++ knowledge_base/ci_cd_azure_devops/src/main.py | 1 + .../.github/workflows/bundle.yml | 64 +++++++++++++++ knowledge_base/ci_cd_github_actions/README.md | 80 +++++++++++++++++++ .../ci_cd_github_actions/databricks.yml | 15 ++++ .../resources/serverless.job.yml | 13 +++ .../ci_cd_github_actions/src/main.py | 1 + 10 files changed, 351 insertions(+) create mode 100644 knowledge_base/ci_cd_azure_devops/README.md create mode 100644 knowledge_base/ci_cd_azure_devops/azure-pipelines.yml create mode 100644 knowledge_base/ci_cd_azure_devops/databricks.yml create mode 100644 knowledge_base/ci_cd_azure_devops/resources/serverless.job.yml create mode 100644 knowledge_base/ci_cd_azure_devops/src/main.py create mode 100644 knowledge_base/ci_cd_github_actions/.github/workflows/bundle.yml create mode 100644 knowledge_base/ci_cd_github_actions/README.md create mode 100644 knowledge_base/ci_cd_github_actions/databricks.yml create mode 100644 knowledge_base/ci_cd_github_actions/resources/serverless.job.yml create mode 100644 knowledge_base/ci_cd_github_actions/src/main.py diff --git a/knowledge_base/ci_cd_azure_devops/README.md b/knowledge_base/ci_cd_azure_devops/README.md new file mode 100644 index 0000000..0755b10 --- /dev/null +++ b/knowledge_base/ci_cd_azure_devops/README.md @@ -0,0 +1,77 @@ +# Azure DevOps CI/CD + +## What this example shows + +This example validates and deploys a serverless job with Azure DevOps OIDC and Databricks workload identity federation. + +## Prerequisites + +- Use a Databricks workspace that supports serverless jobs. +- Get permission to create a Databricks service principal and federation policy. +- Get permission to manage Azure DevOps pipelines and environments. +- Install Databricks CLI `0.272.0` for local tests. + +## Set up identity + +1. Create one Databricks service principal for the pipeline. +2. Create a federation policy for the service principal. +3. Set the issuer to `https://vstoken.dev.azure.com/`. +4. Set the subject to `p:////`. +5. Set the audience to your Databricks account ID. +6. Grant the service principal `CAN_USE` access to the workspace. +7. Grant the required Unity Catalog privileges for resources that you add. +8. Do not create or store a personal access token. + +Use the Azure DevOps organization GUID in the issuer. Use names, not IDs, in the subject. + +## Configure the CI system + +Create these Azure DevOps variables. + +| Variable | Location | Value | +|---|---|---| +| `DATABRICKS_HOST` | Pipeline or variable group | The workspace URL | +| `DATABRICKS_CLIENT_ID` | Pipeline or variable group | The Databricks service principal application ID | +| `System.AccessToken` | Built in | The Azure DevOps OIDC token source | + +1. Create Azure DevOps environments named `staging` and `prod`. +2. Add an approval check to the `prod` environment. +3. Restrict each environment to this pipeline. +4. Set the pipeline variables without secret values or personal access tokens. + +## Pipeline flow + +| Trigger | Stage | Command | Target | +|---|---|---|---| +| Pull request | Validate | `databricks bundle validate -t dev` | `dev` | +| Push to `main` | Staging | `databricks bundle deploy -t staging` | `staging` | +| Approved push to `main` | Production | `databricks bundle deploy -t prod` | `prod` | + +## Run it the first time + +1. Replace the identity placeholders in the federation policy. +2. Create a pipeline from `azure-pipelines.yml`. +3. Open a pull request that changes this example. +4. Confirm that the validate stage succeeds. +5. Merge the pull request into `main`. +6. Confirm that the staging deployment succeeds. +7. Approve the pending `prod` environment deployment. +8. Confirm that the production deployment succeeds. + +## Edge cases and common errors + +- Fork pull requests do not receive an OIDC token. Run validation from a trusted branch. +- Azure DevOps environment exclusive locks prevent concurrent deployments. Configure the lock behavior for each environment. +- A wrong subject claim causes an identity federation match error. Check all three names and their letter case. +- CLI version drift causes bundle version errors. Keep the pipeline and bundle versions equal. +- An interrupted deployment can leave a deployment lock. Retry after the active deployment ends. +- Serverless availability, storage URIs, identities, and regions differ by cloud. This example does not validate classic compute. +- Bundle deployment is idempotent. Retry the same target after a safe interruption. + +## Clean up + +1. Run `databricks bundle destroy -t dev` if you deployed the dev target. +2. Run `databricks bundle destroy -t staging`. +3. Run `databricks bundle destroy -t prod`. +4. Delete only the federation policy for this example. +5. Delete the service principal if no other system uses it. diff --git a/knowledge_base/ci_cd_azure_devops/azure-pipelines.yml b/knowledge_base/ci_cd_azure_devops/azure-pipelines.yml new file mode 100644 index 0000000..8db168f --- /dev/null +++ b/knowledge_base/ci_cd_azure_devops/azure-pipelines.yml @@ -0,0 +1,72 @@ +trigger: + branches: + include: [main] + +pr: + branches: + include: [main] + +variables: + cliVersion: 0.272.0 + bundleDirectory: knowledge_base/ci_cd_azure_devops + +stages: + - stage: Validate + condition: eq(variables['Build.Reason'], 'PullRequest') + jobs: + - job: Validate + steps: + - bash: curl -fsSL https://raw.githubusercontent.com/databricks/setup-cli/79467332e7a1a336e5bc86d8a412fdceda7cd62a/install.sh | sh -s -- $(cliVersion) + displayName: Install Databricks CLI + - bash: databricks bundle validate -t dev + displayName: Validate bundle + workingDirectory: $(bundleDirectory) + env: + DATABRICKS_AUTH_TYPE: azure-devops-oidc + DATABRICKS_HOST: $(DATABRICKS_HOST) + DATABRICKS_CLIENT_ID: $(DATABRICKS_CLIENT_ID) + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + + - stage: Staging + dependsOn: [] + condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/main')) + jobs: + - deployment: DeployStaging + environment: staging + strategy: + runOnce: + deploy: + steps: + - checkout: self + - bash: curl -fsSL https://raw.githubusercontent.com/databricks/setup-cli/79467332e7a1a336e5bc86d8a412fdceda7cd62a/install.sh | sh -s -- $(cliVersion) + displayName: Install Databricks CLI + - bash: databricks bundle deploy -t staging + displayName: Deploy staging + workingDirectory: $(bundleDirectory) + env: + DATABRICKS_AUTH_TYPE: azure-devops-oidc + DATABRICKS_HOST: $(DATABRICKS_HOST) + DATABRICKS_CLIENT_ID: $(DATABRICKS_CLIENT_ID) + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + + - stage: Prod + dependsOn: Staging + condition: succeeded() + jobs: + - deployment: DeployProd + environment: prod + strategy: + runOnce: + deploy: + steps: + - checkout: self + - bash: curl -fsSL https://raw.githubusercontent.com/databricks/setup-cli/79467332e7a1a336e5bc86d8a412fdceda7cd62a/install.sh | sh -s -- $(cliVersion) + displayName: Install Databricks CLI + - bash: databricks bundle deploy -t prod + displayName: Deploy production + workingDirectory: $(bundleDirectory) + env: + DATABRICKS_AUTH_TYPE: azure-devops-oidc + DATABRICKS_HOST: $(DATABRICKS_HOST) + DATABRICKS_CLIENT_ID: $(DATABRICKS_CLIENT_ID) + SYSTEM_ACCESSTOKEN: $(System.AccessToken) diff --git a/knowledge_base/ci_cd_azure_devops/databricks.yml b/knowledge_base/ci_cd_azure_devops/databricks.yml new file mode 100644 index 0000000..d1ae881 --- /dev/null +++ b/knowledge_base/ci_cd_azure_devops/databricks.yml @@ -0,0 +1,15 @@ +bundle: + name: ci_cd_azure_devops + databricks_cli_version: "0.272.0" + +include: + - resources/*.job.yml + +targets: + dev: + default: true + mode: development + staging: + mode: production + prod: + mode: production diff --git a/knowledge_base/ci_cd_azure_devops/resources/serverless.job.yml b/knowledge_base/ci_cd_azure_devops/resources/serverless.job.yml new file mode 100644 index 0000000..bcecb56 --- /dev/null +++ b/knowledge_base/ci_cd_azure_devops/resources/serverless.job.yml @@ -0,0 +1,13 @@ +resources: + jobs: + serverless_job: + name: ci-cd-azure-devops-${bundle.target} + tasks: + - task_key: hello + spark_python_task: + python_file: ../src/main.py + environment_key: default + environments: + - environment_key: default + spec: + environment_version: "4" diff --git a/knowledge_base/ci_cd_azure_devops/src/main.py b/knowledge_base/ci_cd_azure_devops/src/main.py new file mode 100644 index 0000000..db7b029 --- /dev/null +++ b/knowledge_base/ci_cd_azure_devops/src/main.py @@ -0,0 +1 @@ +print("Hello from Azure DevOps") diff --git a/knowledge_base/ci_cd_github_actions/.github/workflows/bundle.yml b/knowledge_base/ci_cd_github_actions/.github/workflows/bundle.yml new file mode 100644 index 0000000..5184360 --- /dev/null +++ b/knowledge_base/ci_cd_github_actions/.github/workflows/bundle.yml @@ -0,0 +1,64 @@ +name: Databricks bundle + +on: + pull_request: + push: + branches: [main] + +permissions: + contents: read + id-token: write + +defaults: + run: + working-directory: knowledge_base/ci_cd_github_actions + +jobs: + validate: + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + environment: dev + concurrency: + group: databricks-bundle-dev + cancel-in-progress: false + env: + DATABRICKS_AUTH_TYPE: github-oidc + DATABRICKS_HOST: ${{ vars.DATABRICKS_HOST }} + DATABRICKS_CLIENT_ID: ${{ vars.DATABRICKS_CLIENT_ID }} + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + - uses: databricks/setup-cli@79467332e7a1a336e5bc86d8a412fdceda7cd62a # v0.272.0 + - run: databricks bundle validate -t dev + + staging: + if: github.event_name == 'push' + runs-on: ubuntu-latest + environment: staging + concurrency: + group: databricks-bundle-staging + cancel-in-progress: false + env: + DATABRICKS_AUTH_TYPE: github-oidc + DATABRICKS_HOST: ${{ vars.DATABRICKS_HOST }} + DATABRICKS_CLIENT_ID: ${{ vars.DATABRICKS_CLIENT_ID }} + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + - uses: databricks/setup-cli@79467332e7a1a336e5bc86d8a412fdceda7cd62a # v0.272.0 + - run: databricks bundle deploy -t staging + + prod: + if: github.event_name == 'push' + needs: staging + runs-on: ubuntu-latest + environment: prod + concurrency: + group: databricks-bundle-prod + cancel-in-progress: false + env: + DATABRICKS_AUTH_TYPE: github-oidc + DATABRICKS_HOST: ${{ vars.DATABRICKS_HOST }} + DATABRICKS_CLIENT_ID: ${{ vars.DATABRICKS_CLIENT_ID }} + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + - uses: databricks/setup-cli@79467332e7a1a336e5bc86d8a412fdceda7cd62a # v0.272.0 + - run: databricks bundle deploy -t prod diff --git a/knowledge_base/ci_cd_github_actions/README.md b/knowledge_base/ci_cd_github_actions/README.md new file mode 100644 index 0000000..d282f62 --- /dev/null +++ b/knowledge_base/ci_cd_github_actions/README.md @@ -0,0 +1,80 @@ +# GitHub Actions CI/CD + +## What this example shows + +This example validates and deploys a serverless job with GitHub Actions and Databricks workload identity federation. + +Copy `.github/workflows/bundle.yml` from this example to your repository root. + +## Prerequisites + +- Use a Databricks workspace that supports serverless jobs. +- Get permission to create a Databricks service principal and federation policies. +- Get permission to manage GitHub environments and repository variables. +- Install Databricks CLI `0.272.0` for local tests. + +## Set up identity + +1. Create one Databricks service principal for the pipeline. +2. Create three federation policies for the service principal. +3. Set each issuer to `https://token.actions.githubusercontent.com`. +4. Set the dev subject to `repo:/:environment:dev`. +5. Set the staging subject to `repo:/:environment:staging`. +6. Set the prod subject to `repo:/:environment:prod`. +7. Set each audience to your Databricks account ID. +8. Grant the service principal `CAN_USE` access to the workspace. +9. Grant the required Unity Catalog privileges for resources that you add. +10. Do not create or store a personal access token. + +Use numeric owner and repository IDs in subjects when your GitHub organization requires immutable subjects. + +## Configure the CI system + +Create these GitHub repository or environment variables. + +| Variable | Location | Value | +|---|---|---| +| `DATABRICKS_HOST` | Each environment | The workspace URL for that target | +| `DATABRICKS_CLIENT_ID` | Each environment | The Databricks service principal application ID | + +1. Create GitHub environments named `dev`, `staging`, and `prod`. +2. Add the variables to each environment. +3. Add required reviewers to the `prod` environment. +4. Do not add required reviewers to `dev` or `staging`. + +## Pipeline flow + +| Trigger | Stage | Command | Target | +|---|---|---|---| +| Pull request | Validate | `databricks bundle validate -t dev` | `dev` | +| Push to `main` | Staging | `databricks bundle deploy -t staging` | `staging` | +| Approved push to `main` | Production | `databricks bundle deploy -t prod` | `prod` | + +## Run it the first time + +1. Copy the workflow to your repository root at `.github/workflows/bundle.yml`. +2. Replace `` and `` in the federation policies. +3. Open a pull request that changes this example. +4. Confirm that the validate job succeeds. +5. Merge the pull request into `main`. +6. Confirm that the staging deployment succeeds. +7. Approve the pending `prod` environment deployment. +8. Confirm that the production deployment succeeds. + +## Edge cases and common errors + +- Fork pull requests do not receive an OIDC token. Run validation from a trusted branch. +- The concurrency groups prevent two deployments to the same target. +- A wrong subject claim causes an identity federation match error. Check the exact environment name. +- CLI version drift causes bundle version errors. Keep both pinned versions equal. +- An interrupted deployment can leave a deployment lock. Retry after the active deployment ends. +- Serverless availability, storage URIs, identities, and regions differ by cloud. This example does not validate classic compute. +- Bundle deployment is idempotent. Retry the same target after a safe interruption. + +## Clean up + +1. Run `databricks bundle destroy -t dev` if you deployed the dev target. +2. Run `databricks bundle destroy -t staging`. +3. Run `databricks bundle destroy -t prod`. +4. Delete only the three federation policies for this example. +5. Delete the service principal if no other system uses it. diff --git a/knowledge_base/ci_cd_github_actions/databricks.yml b/knowledge_base/ci_cd_github_actions/databricks.yml new file mode 100644 index 0000000..d3e3c3f --- /dev/null +++ b/knowledge_base/ci_cd_github_actions/databricks.yml @@ -0,0 +1,15 @@ +bundle: + name: ci_cd_github_actions + databricks_cli_version: "0.272.0" + +include: + - resources/*.job.yml + +targets: + dev: + default: true + mode: development + staging: + mode: production + prod: + mode: production diff --git a/knowledge_base/ci_cd_github_actions/resources/serverless.job.yml b/knowledge_base/ci_cd_github_actions/resources/serverless.job.yml new file mode 100644 index 0000000..f8cadac --- /dev/null +++ b/knowledge_base/ci_cd_github_actions/resources/serverless.job.yml @@ -0,0 +1,13 @@ +resources: + jobs: + serverless_job: + name: ci-cd-github-actions-${bundle.target} + tasks: + - task_key: hello + spark_python_task: + python_file: ../src/main.py + environment_key: default + environments: + - environment_key: default + spec: + environment_version: "4" diff --git a/knowledge_base/ci_cd_github_actions/src/main.py b/knowledge_base/ci_cd_github_actions/src/main.py new file mode 100644 index 0000000..a32fe8a --- /dev/null +++ b/knowledge_base/ci_cd_github_actions/src/main.py @@ -0,0 +1 @@ +print("Hello from GitHub Actions") From 5e133adeff3e4c9bcbf9feb89c3b5edddd9fb4eb Mon Sep 17 00:00:00 2001 From: Andres Garcia Date: Wed, 21 Oct 2026 22:17:11 +0000 Subject: [PATCH 02/12] Allow newer Databricks CLI versions Co-authored-by: omnigent --- knowledge_base/ci_cd_azure_devops/README.md | 6 ++++-- knowledge_base/ci_cd_azure_devops/databricks.yml | 6 +++++- knowledge_base/ci_cd_github_actions/README.md | 6 ++++-- knowledge_base/ci_cd_github_actions/databricks.yml | 6 +++++- 4 files changed, 18 insertions(+), 6 deletions(-) diff --git a/knowledge_base/ci_cd_azure_devops/README.md b/knowledge_base/ci_cd_azure_devops/README.md index 0755b10..c913907 100644 --- a/knowledge_base/ci_cd_azure_devops/README.md +++ b/knowledge_base/ci_cd_azure_devops/README.md @@ -9,7 +9,9 @@ This example validates and deploys a serverless job with Azure DevOps OIDC and D - Use a Databricks workspace that supports serverless jobs. - Get permission to create a Databricks service principal and federation policy. - Get permission to manage Azure DevOps pipelines and environments. -- Install Databricks CLI `0.272.0` for local tests. +- Install Databricks CLI `0.272.0` or newer for local tests. + +The bundle accepts compatible newer CLI versions. The pipeline pins `0.272.0` for repeatable CI runs. ## Set up identity @@ -63,7 +65,7 @@ Create these Azure DevOps variables. - Fork pull requests do not receive an OIDC token. Run validation from a trusted branch. - Azure DevOps environment exclusive locks prevent concurrent deployments. Configure the lock behavior for each environment. - A wrong subject claim causes an identity federation match error. Check all three names and their letter case. -- CLI version drift causes bundle version errors. Keep the pipeline and bundle versions equal. +- A CLI older than `0.272.0` causes bundle version errors. Update the local CLI or the pipeline pin. - An interrupted deployment can leave a deployment lock. Retry after the active deployment ends. - Serverless availability, storage URIs, identities, and regions differ by cloud. This example does not validate classic compute. - Bundle deployment is idempotent. Retry the same target after a safe interruption. diff --git a/knowledge_base/ci_cd_azure_devops/databricks.yml b/knowledge_base/ci_cd_azure_devops/databricks.yml index d1ae881..bd57185 100644 --- a/knowledge_base/ci_cd_azure_devops/databricks.yml +++ b/knowledge_base/ci_cd_azure_devops/databricks.yml @@ -1,6 +1,6 @@ bundle: name: ci_cd_azure_devops - databricks_cli_version: "0.272.0" + databricks_cli_version: ">= 0.272.0" include: - resources/*.job.yml @@ -11,5 +11,9 @@ targets: mode: development staging: mode: production + workspace: + root_path: /Workspace/Users/${workspace.current_user.userName}/.bundle/${bundle.name}/${bundle.target} prod: mode: production + workspace: + root_path: /Workspace/Users/${workspace.current_user.userName}/.bundle/${bundle.name}/${bundle.target} diff --git a/knowledge_base/ci_cd_github_actions/README.md b/knowledge_base/ci_cd_github_actions/README.md index d282f62..c4ed756 100644 --- a/knowledge_base/ci_cd_github_actions/README.md +++ b/knowledge_base/ci_cd_github_actions/README.md @@ -11,7 +11,9 @@ Copy `.github/workflows/bundle.yml` from this example to your repository root. - Use a Databricks workspace that supports serverless jobs. - Get permission to create a Databricks service principal and federation policies. - Get permission to manage GitHub environments and repository variables. -- Install Databricks CLI `0.272.0` for local tests. +- Install Databricks CLI `0.272.0` or newer for local tests. + +The bundle accepts compatible newer CLI versions. The workflow pins `0.272.0` for repeatable CI runs. ## Set up identity @@ -66,7 +68,7 @@ Create these GitHub repository or environment variables. - Fork pull requests do not receive an OIDC token. Run validation from a trusted branch. - The concurrency groups prevent two deployments to the same target. - A wrong subject claim causes an identity federation match error. Check the exact environment name. -- CLI version drift causes bundle version errors. Keep both pinned versions equal. +- A CLI older than `0.272.0` causes bundle version errors. Update the local CLI or the workflow pin. - An interrupted deployment can leave a deployment lock. Retry after the active deployment ends. - Serverless availability, storage URIs, identities, and regions differ by cloud. This example does not validate classic compute. - Bundle deployment is idempotent. Retry the same target after a safe interruption. diff --git a/knowledge_base/ci_cd_github_actions/databricks.yml b/knowledge_base/ci_cd_github_actions/databricks.yml index d3e3c3f..c3e1999 100644 --- a/knowledge_base/ci_cd_github_actions/databricks.yml +++ b/knowledge_base/ci_cd_github_actions/databricks.yml @@ -1,6 +1,6 @@ bundle: name: ci_cd_github_actions - databricks_cli_version: "0.272.0" + databricks_cli_version: ">= 0.272.0" include: - resources/*.job.yml @@ -11,5 +11,9 @@ targets: mode: development staging: mode: production + workspace: + root_path: /Workspace/Users/${workspace.current_user.userName}/.bundle/${bundle.name}/${bundle.target} prod: mode: production + workspace: + root_path: /Workspace/Users/${workspace.current_user.userName}/.bundle/${bundle.name}/${bundle.target} From 724bbe53222d9ecb2bd0c5ea08fbd79067c466ff Mon Sep 17 00:00:00 2001 From: Andres Garcia Date: Wed, 21 Oct 2026 22:18:22 +0000 Subject: [PATCH 03/12] Shorten CI/CD example guides Co-authored-by: omnigent --- knowledge_base/ci_cd_azure_devops/README.md | 106 +++++------------ knowledge_base/ci_cd_github_actions/README.md | 110 ++++++------------ 2 files changed, 65 insertions(+), 151 deletions(-) diff --git a/knowledge_base/ci_cd_azure_devops/README.md b/knowledge_base/ci_cd_azure_devops/README.md index c913907..d727b43 100644 --- a/knowledge_base/ci_cd_azure_devops/README.md +++ b/knowledge_base/ci_cd_azure_devops/README.md @@ -1,79 +1,37 @@ # Azure DevOps CI/CD -## What this example shows - -This example validates and deploys a serverless job with Azure DevOps OIDC and Databricks workload identity federation. +This Declarative Automation Bundles example validates and deploys a serverless job with Azure DevOps OIDC. +See [Azure DevOps authentication](https://docs.databricks.com/aws/en/dev-tools/auth/provider-azure-devops). ## Prerequisites -- Use a Databricks workspace that supports serverless jobs. -- Get permission to create a Databricks service principal and federation policy. -- Get permission to manage Azure DevOps pipelines and environments. -- Install Databricks CLI `0.272.0` or newer for local tests. - -The bundle accepts compatible newer CLI versions. The pipeline pins `0.272.0` for repeatable CI runs. - -## Set up identity - -1. Create one Databricks service principal for the pipeline. -2. Create a federation policy for the service principal. -3. Set the issuer to `https://vstoken.dev.azure.com/`. -4. Set the subject to `p:////`. -5. Set the audience to your Databricks account ID. -6. Grant the service principal `CAN_USE` access to the workspace. -7. Grant the required Unity Catalog privileges for resources that you add. -8. Do not create or store a personal access token. - -Use the Azure DevOps organization GUID in the issuer. Use names, not IDs, in the subject. - -## Configure the CI system - -Create these Azure DevOps variables. - -| Variable | Location | Value | -|---|---|---| -| `DATABRICKS_HOST` | Pipeline or variable group | The workspace URL | -| `DATABRICKS_CLIENT_ID` | Pipeline or variable group | The Databricks service principal application ID | -| `System.AccessToken` | Built in | The Azure DevOps OIDC token source | - -1. Create Azure DevOps environments named `staging` and `prod`. -2. Add an approval check to the `prod` environment. -3. Restrict each environment to this pipeline. -4. Set the pipeline variables without secret values or personal access tokens. - -## Pipeline flow - -| Trigger | Stage | Command | Target | -|---|---|---|---| -| Pull request | Validate | `databricks bundle validate -t dev` | `dev` | -| Push to `main` | Staging | `databricks bundle deploy -t staging` | `staging` | -| Approved push to `main` | Production | `databricks bundle deploy -t prod` | `prod` | - -## Run it the first time - -1. Replace the identity placeholders in the federation policy. -2. Create a pipeline from `azure-pipelines.yml`. -3. Open a pull request that changes this example. -4. Confirm that the validate stage succeeds. -5. Merge the pull request into `main`. -6. Confirm that the staging deployment succeeds. -7. Approve the pending `prod` environment deployment. -8. Confirm that the production deployment succeeds. - -## Edge cases and common errors - -- Fork pull requests do not receive an OIDC token. Run validation from a trusted branch. -- Azure DevOps environment exclusive locks prevent concurrent deployments. Configure the lock behavior for each environment. -- A wrong subject claim causes an identity federation match error. Check all three names and their letter case. -- A CLI older than `0.272.0` causes bundle version errors. Update the local CLI or the pipeline pin. -- An interrupted deployment can leave a deployment lock. Retry after the active deployment ends. -- Serverless availability, storage URIs, identities, and regions differ by cloud. This example does not validate classic compute. -- Bundle deployment is idempotent. Retry the same target after a safe interruption. - -## Clean up - -1. Run `databricks bundle destroy -t dev` if you deployed the dev target. -2. Run `databricks bundle destroy -t staging`. -3. Run `databricks bundle destroy -t prod`. -4. Delete only the federation policy for this example. -5. Delete the service principal if no other system uses it. +- A Databricks workspace that supports serverless jobs. +- Permission to manage Databricks service principals and Azure DevOps environments. +- Databricks CLI `0.272.0` or newer for local tests. + +## Usage + +1. Create a Databricks service principal for the pipeline. +2. Grant it workspace access and the required Unity Catalog privileges. +3. Create a federation policy for the service principal. +4. Set the issuer to `https://vstoken.dev.azure.com/`. +5. Set the subject to `p:////`. +6. Set the audience to your Databricks account ID. +7. Set pipeline variables `DATABRICKS_HOST` and `DATABRICKS_CLIENT_ID`. +8. Create `staging` and `prod` Azure DevOps environments. +9. Add an approval check to the `prod` environment. +10. Copy `azure-pipelines.yml` to your repository root. +11. Open a pull request to run validation. + +| Trigger | Command and target | +|---|---| +| Pull request | `databricks bundle validate -t dev` | +| Push to `main` | `databricks bundle deploy -t staging` | +| Production approval | `databricks bundle deploy -t prod` | + +## Notes + +- Do not create or store personal access tokens. +- Fork pull requests receive no OIDC token. Validate them from a trusted branch. +- Environment locks serialize deployments. The `prod` approval check gates production. +- The bundle accepts CLI `>= 0.272.0`. CI pins `0.272.0` for repeatable runs. diff --git a/knowledge_base/ci_cd_github_actions/README.md b/knowledge_base/ci_cd_github_actions/README.md index c4ed756..f8d90cf 100644 --- a/knowledge_base/ci_cd_github_actions/README.md +++ b/knowledge_base/ci_cd_github_actions/README.md @@ -1,82 +1,38 @@ # GitHub Actions CI/CD -## What this example shows - -This example validates and deploys a serverless job with GitHub Actions and Databricks workload identity federation. - -Copy `.github/workflows/bundle.yml` from this example to your repository root. +This Declarative Automation Bundles example validates and deploys a serverless job with GitHub Actions and OIDC. +See [GitHub Actions authentication](https://docs.databricks.com/aws/en/dev-tools/auth/provider-github). ## Prerequisites -- Use a Databricks workspace that supports serverless jobs. -- Get permission to create a Databricks service principal and federation policies. -- Get permission to manage GitHub environments and repository variables. -- Install Databricks CLI `0.272.0` or newer for local tests. - -The bundle accepts compatible newer CLI versions. The workflow pins `0.272.0` for repeatable CI runs. - -## Set up identity - -1. Create one Databricks service principal for the pipeline. -2. Create three federation policies for the service principal. -3. Set each issuer to `https://token.actions.githubusercontent.com`. -4. Set the dev subject to `repo:/:environment:dev`. -5. Set the staging subject to `repo:/:environment:staging`. -6. Set the prod subject to `repo:/:environment:prod`. -7. Set each audience to your Databricks account ID. -8. Grant the service principal `CAN_USE` access to the workspace. -9. Grant the required Unity Catalog privileges for resources that you add. -10. Do not create or store a personal access token. - -Use numeric owner and repository IDs in subjects when your GitHub organization requires immutable subjects. - -## Configure the CI system - -Create these GitHub repository or environment variables. - -| Variable | Location | Value | -|---|---|---| -| `DATABRICKS_HOST` | Each environment | The workspace URL for that target | -| `DATABRICKS_CLIENT_ID` | Each environment | The Databricks service principal application ID | - -1. Create GitHub environments named `dev`, `staging`, and `prod`. -2. Add the variables to each environment. -3. Add required reviewers to the `prod` environment. -4. Do not add required reviewers to `dev` or `staging`. - -## Pipeline flow - -| Trigger | Stage | Command | Target | -|---|---|---|---| -| Pull request | Validate | `databricks bundle validate -t dev` | `dev` | -| Push to `main` | Staging | `databricks bundle deploy -t staging` | `staging` | -| Approved push to `main` | Production | `databricks bundle deploy -t prod` | `prod` | - -## Run it the first time - -1. Copy the workflow to your repository root at `.github/workflows/bundle.yml`. -2. Replace `` and `` in the federation policies. -3. Open a pull request that changes this example. -4. Confirm that the validate job succeeds. -5. Merge the pull request into `main`. -6. Confirm that the staging deployment succeeds. -7. Approve the pending `prod` environment deployment. -8. Confirm that the production deployment succeeds. - -## Edge cases and common errors - -- Fork pull requests do not receive an OIDC token. Run validation from a trusted branch. -- The concurrency groups prevent two deployments to the same target. -- A wrong subject claim causes an identity federation match error. Check the exact environment name. -- A CLI older than `0.272.0` causes bundle version errors. Update the local CLI or the workflow pin. -- An interrupted deployment can leave a deployment lock. Retry after the active deployment ends. -- Serverless availability, storage URIs, identities, and regions differ by cloud. This example does not validate classic compute. -- Bundle deployment is idempotent. Retry the same target after a safe interruption. - -## Clean up - -1. Run `databricks bundle destroy -t dev` if you deployed the dev target. -2. Run `databricks bundle destroy -t staging`. -3. Run `databricks bundle destroy -t prod`. -4. Delete only the three federation policies for this example. -5. Delete the service principal if no other system uses it. +- A Databricks workspace that supports serverless jobs. +- Permission to manage Databricks service principals and GitHub environments. +- Databricks CLI `0.272.0` or newer for local tests. + +## Usage + +1. Create a Databricks service principal for the pipeline. +2. Grant it workspace access and the required Unity Catalog privileges. +3. Create federation policies with issuer `https://token.actions.githubusercontent.com`. +4. Use subject `repo:/:environment:dev` for dev. +5. Use subject `repo:/:environment:staging` for staging. +6. Use subject `repo:/:environment:prod` for prod. +7. Set the audience to your Databricks account ID. +8. Create `dev`, `staging`, and `prod` GitHub environments. +9. Set `DATABRICKS_HOST` and `DATABRICKS_CLIENT_ID` in each environment. +10. Add required reviewers to the `prod` environment. +11. Copy `.github/workflows/bundle.yml` to your repository root. +12. Open a pull request to run validation. + +| Trigger | Command and target | +|---|---| +| Pull request | `databricks bundle validate -t dev` | +| Push to `main` | `databricks bundle deploy -t staging` | +| Production approval | `databricks bundle deploy -t prod` | + +## Notes + +- Do not create or store personal access tokens. +- Fork pull requests receive no OIDC token. Validate them from a trusted branch. +- Concurrency groups serialize each target. Required reviewers gate production. +- The bundle accepts CLI `>= 0.272.0`. CI pins `0.272.0` for repeatable runs. From de4b0dc71b0f992d8fcafc5a7f238088a188b8c5 Mon Sep 17 00:00:00 2001 From: Andres Garcia Date: Wed, 21 Oct 2026 22:19:05 +0000 Subject: [PATCH 04/12] Pin CI to Databricks CLI 1.20.0 Co-authored-by: omnigent --- knowledge_base/ci_cd_azure_devops/README.md | 4 ++-- knowledge_base/ci_cd_azure_devops/azure-pipelines.yml | 8 ++++---- knowledge_base/ci_cd_azure_devops/databricks.yml | 2 +- .../ci_cd_github_actions/.github/workflows/bundle.yml | 6 +++--- knowledge_base/ci_cd_github_actions/README.md | 4 ++-- knowledge_base/ci_cd_github_actions/databricks.yml | 2 +- 6 files changed, 13 insertions(+), 13 deletions(-) diff --git a/knowledge_base/ci_cd_azure_devops/README.md b/knowledge_base/ci_cd_azure_devops/README.md index d727b43..78c198b 100644 --- a/knowledge_base/ci_cd_azure_devops/README.md +++ b/knowledge_base/ci_cd_azure_devops/README.md @@ -7,7 +7,7 @@ See [Azure DevOps authentication](https://docs.databricks.com/aws/en/dev-tools/a - A Databricks workspace that supports serverless jobs. - Permission to manage Databricks service principals and Azure DevOps environments. -- Databricks CLI `0.272.0` or newer for local tests. +- Databricks CLI v1.20.0 or above. ## Usage @@ -34,4 +34,4 @@ See [Azure DevOps authentication](https://docs.databricks.com/aws/en/dev-tools/a - Do not create or store personal access tokens. - Fork pull requests receive no OIDC token. Validate them from a trusted branch. - Environment locks serialize deployments. The `prod` approval check gates production. -- The bundle accepts CLI `>= 0.272.0`. CI pins `0.272.0` for repeatable runs. +- The bundle accepts CLI `>= 1.20.0`. CI pins `1.20.0` for repeatable runs. diff --git a/knowledge_base/ci_cd_azure_devops/azure-pipelines.yml b/knowledge_base/ci_cd_azure_devops/azure-pipelines.yml index 8db168f..4378c21 100644 --- a/knowledge_base/ci_cd_azure_devops/azure-pipelines.yml +++ b/knowledge_base/ci_cd_azure_devops/azure-pipelines.yml @@ -7,7 +7,7 @@ pr: include: [main] variables: - cliVersion: 0.272.0 + cliVersion: 1.20.0 bundleDirectory: knowledge_base/ci_cd_azure_devops stages: @@ -16,7 +16,7 @@ stages: jobs: - job: Validate steps: - - bash: curl -fsSL https://raw.githubusercontent.com/databricks/setup-cli/79467332e7a1a336e5bc86d8a412fdceda7cd62a/install.sh | sh -s -- $(cliVersion) + - bash: curl -fsSL https://raw.githubusercontent.com/databricks/setup-cli/9b64c94a6d2b2e830592d80e9f14c26d32b42ad7/install.sh | sh -s -- $(cliVersion) displayName: Install Databricks CLI - bash: databricks bundle validate -t dev displayName: Validate bundle @@ -38,7 +38,7 @@ stages: deploy: steps: - checkout: self - - bash: curl -fsSL https://raw.githubusercontent.com/databricks/setup-cli/79467332e7a1a336e5bc86d8a412fdceda7cd62a/install.sh | sh -s -- $(cliVersion) + - bash: curl -fsSL https://raw.githubusercontent.com/databricks/setup-cli/9b64c94a6d2b2e830592d80e9f14c26d32b42ad7/install.sh | sh -s -- $(cliVersion) displayName: Install Databricks CLI - bash: databricks bundle deploy -t staging displayName: Deploy staging @@ -60,7 +60,7 @@ stages: deploy: steps: - checkout: self - - bash: curl -fsSL https://raw.githubusercontent.com/databricks/setup-cli/79467332e7a1a336e5bc86d8a412fdceda7cd62a/install.sh | sh -s -- $(cliVersion) + - bash: curl -fsSL https://raw.githubusercontent.com/databricks/setup-cli/9b64c94a6d2b2e830592d80e9f14c26d32b42ad7/install.sh | sh -s -- $(cliVersion) displayName: Install Databricks CLI - bash: databricks bundle deploy -t prod displayName: Deploy production diff --git a/knowledge_base/ci_cd_azure_devops/databricks.yml b/knowledge_base/ci_cd_azure_devops/databricks.yml index bd57185..d2365d8 100644 --- a/knowledge_base/ci_cd_azure_devops/databricks.yml +++ b/knowledge_base/ci_cd_azure_devops/databricks.yml @@ -1,6 +1,6 @@ bundle: name: ci_cd_azure_devops - databricks_cli_version: ">= 0.272.0" + databricks_cli_version: ">= 1.20.0" include: - resources/*.job.yml diff --git a/knowledge_base/ci_cd_github_actions/.github/workflows/bundle.yml b/knowledge_base/ci_cd_github_actions/.github/workflows/bundle.yml index 5184360..b3ca0a2 100644 --- a/knowledge_base/ci_cd_github_actions/.github/workflows/bundle.yml +++ b/knowledge_base/ci_cd_github_actions/.github/workflows/bundle.yml @@ -27,7 +27,7 @@ jobs: DATABRICKS_CLIENT_ID: ${{ vars.DATABRICKS_CLIENT_ID }} steps: - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 - - uses: databricks/setup-cli@79467332e7a1a336e5bc86d8a412fdceda7cd62a # v0.272.0 + - uses: databricks/setup-cli@9b64c94a6d2b2e830592d80e9f14c26d32b42ad7 # v1.20.0 - run: databricks bundle validate -t dev staging: @@ -43,7 +43,7 @@ jobs: DATABRICKS_CLIENT_ID: ${{ vars.DATABRICKS_CLIENT_ID }} steps: - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 - - uses: databricks/setup-cli@79467332e7a1a336e5bc86d8a412fdceda7cd62a # v0.272.0 + - uses: databricks/setup-cli@9b64c94a6d2b2e830592d80e9f14c26d32b42ad7 # v1.20.0 - run: databricks bundle deploy -t staging prod: @@ -60,5 +60,5 @@ jobs: DATABRICKS_CLIENT_ID: ${{ vars.DATABRICKS_CLIENT_ID }} steps: - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 - - uses: databricks/setup-cli@79467332e7a1a336e5bc86d8a412fdceda7cd62a # v0.272.0 + - uses: databricks/setup-cli@9b64c94a6d2b2e830592d80e9f14c26d32b42ad7 # v1.20.0 - run: databricks bundle deploy -t prod diff --git a/knowledge_base/ci_cd_github_actions/README.md b/knowledge_base/ci_cd_github_actions/README.md index f8d90cf..8f0c127 100644 --- a/knowledge_base/ci_cd_github_actions/README.md +++ b/knowledge_base/ci_cd_github_actions/README.md @@ -7,7 +7,7 @@ See [GitHub Actions authentication](https://docs.databricks.com/aws/en/dev-tools - A Databricks workspace that supports serverless jobs. - Permission to manage Databricks service principals and GitHub environments. -- Databricks CLI `0.272.0` or newer for local tests. +- Databricks CLI v1.20.0 or above. ## Usage @@ -35,4 +35,4 @@ See [GitHub Actions authentication](https://docs.databricks.com/aws/en/dev-tools - Do not create or store personal access tokens. - Fork pull requests receive no OIDC token. Validate them from a trusted branch. - Concurrency groups serialize each target. Required reviewers gate production. -- The bundle accepts CLI `>= 0.272.0`. CI pins `0.272.0` for repeatable runs. +- The bundle accepts CLI `>= 1.20.0`. CI pins `1.20.0` for repeatable runs. diff --git a/knowledge_base/ci_cd_github_actions/databricks.yml b/knowledge_base/ci_cd_github_actions/databricks.yml index c3e1999..d8a22fa 100644 --- a/knowledge_base/ci_cd_github_actions/databricks.yml +++ b/knowledge_base/ci_cd_github_actions/databricks.yml @@ -1,6 +1,6 @@ bundle: name: ci_cd_github_actions - databricks_cli_version: ">= 0.272.0" + databricks_cli_version: ">= 1.20.0" include: - resources/*.job.yml From 44734dafcf4ec7b976f2affb787770c80689833c Mon Sep 17 00:00:00 2001 From: Andres Garcia Date: Wed, 21 Oct 2026 22:24:28 +0000 Subject: [PATCH 05/12] Address CI/CD cross-review findings Co-authored-by: omnigent --- knowledge_base/ci_cd_azure_devops/README.md | 7 ++++--- knowledge_base/ci_cd_azure_devops/azure-pipelines.yml | 7 +++---- knowledge_base/ci_cd_azure_devops/databricks.yml | 4 ++-- knowledge_base/ci_cd_github_actions/README.md | 5 +++-- knowledge_base/ci_cd_github_actions/databricks.yml | 4 ++-- 5 files changed, 14 insertions(+), 13 deletions(-) diff --git a/knowledge_base/ci_cd_azure_devops/README.md b/knowledge_base/ci_cd_azure_devops/README.md index 78c198b..1ec485a 100644 --- a/knowledge_base/ci_cd_azure_devops/README.md +++ b/knowledge_base/ci_cd_azure_devops/README.md @@ -16,7 +16,7 @@ See [Azure DevOps authentication](https://docs.databricks.com/aws/en/dev-tools/a 3. Create a federation policy for the service principal. 4. Set the issuer to `https://vstoken.dev.azure.com/`. 5. Set the subject to `p:////`. -6. Set the audience to your Databricks account ID. +6. Set the audience to [`api://AzureADTokenExchange`](https://docs.databricks.com/aws/en/dev-tools/auth/provider-azure-devops). 7. Set pipeline variables `DATABRICKS_HOST` and `DATABRICKS_CLIENT_ID`. 8. Create `staging` and `prod` Azure DevOps environments. 9. Add an approval check to the `prod` environment. @@ -32,6 +32,7 @@ See [Azure DevOps authentication](https://docs.databricks.com/aws/en/dev-tools/a ## Notes - Do not create or store personal access tokens. -- Fork pull requests receive no OIDC token. Validate them from a trusted branch. +- PR validation needs workspace authentication. Fork pull requests receive no OIDC token. - Environment locks serialize deployments. The `prod` approval check gates production. -- The bundle accepts CLI `>= 1.20.0`. CI pins `1.20.0` for repeatable runs. +- The setup-cli SHA fixes CLI `1.20.0`. The bundle accepts CLI `>= 1.20.0`. +- After copying the pipeline, set `bundleDirectory` to your bundle path. diff --git a/knowledge_base/ci_cd_azure_devops/azure-pipelines.yml b/knowledge_base/ci_cd_azure_devops/azure-pipelines.yml index 4378c21..ab0ac7a 100644 --- a/knowledge_base/ci_cd_azure_devops/azure-pipelines.yml +++ b/knowledge_base/ci_cd_azure_devops/azure-pipelines.yml @@ -7,7 +7,6 @@ pr: include: [main] variables: - cliVersion: 1.20.0 bundleDirectory: knowledge_base/ci_cd_azure_devops stages: @@ -16,7 +15,7 @@ stages: jobs: - job: Validate steps: - - bash: curl -fsSL https://raw.githubusercontent.com/databricks/setup-cli/9b64c94a6d2b2e830592d80e9f14c26d32b42ad7/install.sh | sh -s -- $(cliVersion) + - bash: curl -fsSL https://raw.githubusercontent.com/databricks/setup-cli/9b64c94a6d2b2e830592d80e9f14c26d32b42ad7/install.sh | sh displayName: Install Databricks CLI - bash: databricks bundle validate -t dev displayName: Validate bundle @@ -38,7 +37,7 @@ stages: deploy: steps: - checkout: self - - bash: curl -fsSL https://raw.githubusercontent.com/databricks/setup-cli/9b64c94a6d2b2e830592d80e9f14c26d32b42ad7/install.sh | sh -s -- $(cliVersion) + - bash: curl -fsSL https://raw.githubusercontent.com/databricks/setup-cli/9b64c94a6d2b2e830592d80e9f14c26d32b42ad7/install.sh | sh displayName: Install Databricks CLI - bash: databricks bundle deploy -t staging displayName: Deploy staging @@ -60,7 +59,7 @@ stages: deploy: steps: - checkout: self - - bash: curl -fsSL https://raw.githubusercontent.com/databricks/setup-cli/9b64c94a6d2b2e830592d80e9f14c26d32b42ad7/install.sh | sh -s -- $(cliVersion) + - bash: curl -fsSL https://raw.githubusercontent.com/databricks/setup-cli/9b64c94a6d2b2e830592d80e9f14c26d32b42ad7/install.sh | sh displayName: Install Databricks CLI - bash: databricks bundle deploy -t prod displayName: Deploy production diff --git a/knowledge_base/ci_cd_azure_devops/databricks.yml b/knowledge_base/ci_cd_azure_devops/databricks.yml index d2365d8..73fc169 100644 --- a/knowledge_base/ci_cd_azure_devops/databricks.yml +++ b/knowledge_base/ci_cd_azure_devops/databricks.yml @@ -12,8 +12,8 @@ targets: staging: mode: production workspace: - root_path: /Workspace/Users/${workspace.current_user.userName}/.bundle/${bundle.name}/${bundle.target} + root_path: /Workspace/Shared/.bundle/${bundle.name}/${bundle.target} prod: mode: production workspace: - root_path: /Workspace/Users/${workspace.current_user.userName}/.bundle/${bundle.name}/${bundle.target} + root_path: /Workspace/Shared/.bundle/${bundle.name}/${bundle.target} diff --git a/knowledge_base/ci_cd_github_actions/README.md b/knowledge_base/ci_cd_github_actions/README.md index 8f0c127..4cb6d08 100644 --- a/knowledge_base/ci_cd_github_actions/README.md +++ b/knowledge_base/ci_cd_github_actions/README.md @@ -33,6 +33,7 @@ See [GitHub Actions authentication](https://docs.databricks.com/aws/en/dev-tools ## Notes - Do not create or store personal access tokens. -- Fork pull requests receive no OIDC token. Validate them from a trusted branch. +- PR validation needs workspace authentication. Fork pull requests receive no OIDC token. - Concurrency groups serialize each target. Required reviewers gate production. -- The bundle accepts CLI `>= 1.20.0`. CI pins `1.20.0` for repeatable runs. +- `databricks/setup-cli` pins CLI `1.20.0`. The bundle accepts CLI `>= 1.20.0`. +- After copying the workflow, set `working-directory` to your bundle path. diff --git a/knowledge_base/ci_cd_github_actions/databricks.yml b/knowledge_base/ci_cd_github_actions/databricks.yml index d8a22fa..4942e1d 100644 --- a/knowledge_base/ci_cd_github_actions/databricks.yml +++ b/knowledge_base/ci_cd_github_actions/databricks.yml @@ -12,8 +12,8 @@ targets: staging: mode: production workspace: - root_path: /Workspace/Users/${workspace.current_user.userName}/.bundle/${bundle.name}/${bundle.target} + root_path: /Workspace/Shared/.bundle/${bundle.name}/${bundle.target} prod: mode: production workspace: - root_path: /Workspace/Users/${workspace.current_user.userName}/.bundle/${bundle.name}/${bundle.target} + root_path: /Workspace/Shared/.bundle/${bundle.name}/${bundle.target} From 538579e91662ae698d01168dc134f321b604d2f5 Mon Sep 17 00:00:00 2001 From: Andres Garcia Date: Wed, 21 Oct 2026 22:26:05 +0000 Subject: [PATCH 06/12] Use restricted CI deployment paths Co-authored-by: omnigent --- knowledge_base/ci_cd_azure_devops/README.md | 4 ++-- knowledge_base/ci_cd_azure_devops/databricks.yml | 5 +++-- knowledge_base/ci_cd_github_actions/README.md | 6 +++--- knowledge_base/ci_cd_github_actions/databricks.yml | 5 +++-- 4 files changed, 11 insertions(+), 9 deletions(-) diff --git a/knowledge_base/ci_cd_azure_devops/README.md b/knowledge_base/ci_cd_azure_devops/README.md index 1ec485a..2b942c9 100644 --- a/knowledge_base/ci_cd_azure_devops/README.md +++ b/knowledge_base/ci_cd_azure_devops/README.md @@ -31,8 +31,8 @@ See [Azure DevOps authentication](https://docs.databricks.com/aws/en/dev-tools/a ## Notes -- Do not create or store personal access tokens. -- PR validation needs workspace authentication. Fork pull requests receive no OIDC token. +- Do not use personal access tokens. PR validation needs workspace authentication. Fork pull requests receive no OIDC token. - Environment locks serialize deployments. The `prod` approval check gates production. - The setup-cli SHA fixes CLI `1.20.0`. The bundle accepts CLI `>= 1.20.0`. - After copying the pipeline, set `bundleDirectory` to your bundle path. +- Deploy `staging` and `prod` only from CI as the service principal. diff --git a/knowledge_base/ci_cd_azure_devops/databricks.yml b/knowledge_base/ci_cd_azure_devops/databricks.yml index 73fc169..3803e36 100644 --- a/knowledge_base/ci_cd_azure_devops/databricks.yml +++ b/knowledge_base/ci_cd_azure_devops/databricks.yml @@ -12,8 +12,9 @@ targets: staging: mode: production workspace: - root_path: /Workspace/Shared/.bundle/${bundle.name}/${bundle.target} + # In CI, current_user is the SP, so each target has one copy in the SP home. + root_path: /Workspace/Users/${workspace.current_user.userName}/.bundle/${bundle.name}/${bundle.target} prod: mode: production workspace: - root_path: /Workspace/Shared/.bundle/${bundle.name}/${bundle.target} + root_path: /Workspace/Users/${workspace.current_user.userName}/.bundle/${bundle.name}/${bundle.target} diff --git a/knowledge_base/ci_cd_github_actions/README.md b/knowledge_base/ci_cd_github_actions/README.md index 4cb6d08..f8f41ce 100644 --- a/knowledge_base/ci_cd_github_actions/README.md +++ b/knowledge_base/ci_cd_github_actions/README.md @@ -17,7 +17,7 @@ See [GitHub Actions authentication](https://docs.databricks.com/aws/en/dev-tools 4. Use subject `repo:/:environment:dev` for dev. 5. Use subject `repo:/:environment:staging` for staging. 6. Use subject `repo:/:environment:prod` for prod. -7. Set the audience to your Databricks account ID. +7. Set the audience to your [Databricks account ID](https://docs.databricks.com/aws/en/dev-tools/auth/provider-github). 8. Create `dev`, `staging`, and `prod` GitHub environments. 9. Set `DATABRICKS_HOST` and `DATABRICKS_CLIENT_ID` in each environment. 10. Add required reviewers to the `prod` environment. @@ -32,8 +32,8 @@ See [GitHub Actions authentication](https://docs.databricks.com/aws/en/dev-tools ## Notes -- Do not create or store personal access tokens. -- PR validation needs workspace authentication. Fork pull requests receive no OIDC token. +- Do not use personal access tokens. PR validation needs workspace authentication. Fork pull requests receive no OIDC token. - Concurrency groups serialize each target. Required reviewers gate production. - `databricks/setup-cli` pins CLI `1.20.0`. The bundle accepts CLI `>= 1.20.0`. - After copying the workflow, set `working-directory` to your bundle path. +- Deploy `staging` and `prod` only from CI as the service principal. diff --git a/knowledge_base/ci_cd_github_actions/databricks.yml b/knowledge_base/ci_cd_github_actions/databricks.yml index 4942e1d..6672a0e 100644 --- a/knowledge_base/ci_cd_github_actions/databricks.yml +++ b/knowledge_base/ci_cd_github_actions/databricks.yml @@ -12,8 +12,9 @@ targets: staging: mode: production workspace: - root_path: /Workspace/Shared/.bundle/${bundle.name}/${bundle.target} + # In CI, current_user is the SP, so each target has one copy in the SP home. + root_path: /Workspace/Users/${workspace.current_user.userName}/.bundle/${bundle.name}/${bundle.target} prod: mode: production workspace: - root_path: /Workspace/Shared/.bundle/${bundle.name}/${bundle.target} + root_path: /Workspace/Users/${workspace.current_user.userName}/.bundle/${bundle.name}/${bundle.target} From 01a859bc9a658f257702071a436f11c778adcfbd Mon Sep 17 00:00:00 2001 From: Andres Garcia Date: Wed, 21 Oct 2026 22:32:47 +0000 Subject: [PATCH 07/12] Clarify Azure CLI pinning Co-authored-by: omnigent --- knowledge_base/ci_cd_azure_devops/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/knowledge_base/ci_cd_azure_devops/README.md b/knowledge_base/ci_cd_azure_devops/README.md index 2b942c9..0c6a0f7 100644 --- a/knowledge_base/ci_cd_azure_devops/README.md +++ b/knowledge_base/ci_cd_azure_devops/README.md @@ -33,6 +33,6 @@ See [Azure DevOps authentication](https://docs.databricks.com/aws/en/dev-tools/a - Do not use personal access tokens. PR validation needs workspace authentication. Fork pull requests receive no OIDC token. - Environment locks serialize deployments. The `prod` approval check gates production. -- The setup-cli SHA fixes CLI `1.20.0`. The bundle accepts CLI `>= 1.20.0`. +- The setup-cli SHA pins CLI `1.20.0`. The bundle accepts CLI `>= 1.20.0`. - After copying the pipeline, set `bundleDirectory` to your bundle path. - Deploy `staging` and `prod` only from CI as the service principal. From c462fd28284c36b826d1140e3a7ddb318c3766e7 Mon Sep 17 00:00:00 2001 From: Andres Garcia Date: Wed, 21 Oct 2026 22:47:11 +0000 Subject: [PATCH 08/12] Use shared bundle example roots Co-authored-by: omnigent --- knowledge_base/ci_cd_azure_devops/README.md | 1 + knowledge_base/ci_cd_azure_devops/databricks.yml | 9 ++++++--- knowledge_base/ci_cd_github_actions/README.md | 1 + knowledge_base/ci_cd_github_actions/databricks.yml | 9 ++++++--- 4 files changed, 14 insertions(+), 6 deletions(-) diff --git a/knowledge_base/ci_cd_azure_devops/README.md b/knowledge_base/ci_cd_azure_devops/README.md index 0c6a0f7..e033e8a 100644 --- a/knowledge_base/ci_cd_azure_devops/README.md +++ b/knowledge_base/ci_cd_azure_devops/README.md @@ -36,3 +36,4 @@ See [Azure DevOps authentication](https://docs.databricks.com/aws/en/dev-tools/a - The setup-cli SHA pins CLI `1.20.0`. The bundle accepts CLI `>= 1.20.0`. - After copying the pipeline, set `bundleDirectory` to your bundle path. - Deploy `staging` and `prod` only from CI as the service principal. +- The bundle deploys under `/Workspace/bundle-examples`. Grant CI `CAN_MANAGE` there, or change `root_path`. diff --git a/knowledge_base/ci_cd_azure_devops/databricks.yml b/knowledge_base/ci_cd_azure_devops/databricks.yml index 3803e36..558b80e 100644 --- a/knowledge_base/ci_cd_azure_devops/databricks.yml +++ b/knowledge_base/ci_cd_azure_devops/databricks.yml @@ -9,12 +9,15 @@ targets: dev: default: true mode: development + workspace: + # Give each developer a separate dev copy. + root_path: /Workspace/bundle-examples/${bundle.name}/${bundle.target}/${workspace.current_user.short_name} staging: mode: production workspace: - # In CI, current_user is the SP, so each target has one copy in the SP home. - root_path: /Workspace/Users/${workspace.current_user.userName}/.bundle/${bundle.name}/${bundle.target} + # The CI service principal needs CAN_MANAGE on /Workspace/bundle-examples. + root_path: /Workspace/bundle-examples/${bundle.name}/${bundle.target} prod: mode: production workspace: - root_path: /Workspace/Users/${workspace.current_user.userName}/.bundle/${bundle.name}/${bundle.target} + root_path: /Workspace/bundle-examples/${bundle.name}/${bundle.target} diff --git a/knowledge_base/ci_cd_github_actions/README.md b/knowledge_base/ci_cd_github_actions/README.md index f8f41ce..4cfa54b 100644 --- a/knowledge_base/ci_cd_github_actions/README.md +++ b/knowledge_base/ci_cd_github_actions/README.md @@ -37,3 +37,4 @@ See [GitHub Actions authentication](https://docs.databricks.com/aws/en/dev-tools - `databricks/setup-cli` pins CLI `1.20.0`. The bundle accepts CLI `>= 1.20.0`. - After copying the workflow, set `working-directory` to your bundle path. - Deploy `staging` and `prod` only from CI as the service principal. +- The bundle deploys under `/Workspace/bundle-examples`. Grant CI `CAN_MANAGE` there, or change `root_path`. diff --git a/knowledge_base/ci_cd_github_actions/databricks.yml b/knowledge_base/ci_cd_github_actions/databricks.yml index 6672a0e..658580a 100644 --- a/knowledge_base/ci_cd_github_actions/databricks.yml +++ b/knowledge_base/ci_cd_github_actions/databricks.yml @@ -9,12 +9,15 @@ targets: dev: default: true mode: development + workspace: + # Give each developer a separate dev copy. + root_path: /Workspace/bundle-examples/${bundle.name}/${bundle.target}/${workspace.current_user.short_name} staging: mode: production workspace: - # In CI, current_user is the SP, so each target has one copy in the SP home. - root_path: /Workspace/Users/${workspace.current_user.userName}/.bundle/${bundle.name}/${bundle.target} + # The CI service principal needs CAN_MANAGE on /Workspace/bundle-examples. + root_path: /Workspace/bundle-examples/${bundle.name}/${bundle.target} prod: mode: production workspace: - root_path: /Workspace/Users/${workspace.current_user.userName}/.bundle/${bundle.name}/${bundle.target} + root_path: /Workspace/bundle-examples/${bundle.name}/${bundle.target} From b1782e9f91b947ec5248b8feccfec9abb838085a Mon Sep 17 00:00:00 2001 From: Andres Garcia Date: Wed, 21 Oct 2026 22:49:31 +0000 Subject: [PATCH 09/12] Clarify CI/CD root path note Co-authored-by: omnigent --- knowledge_base/ci_cd_azure_devops/README.md | 2 +- knowledge_base/ci_cd_github_actions/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/knowledge_base/ci_cd_azure_devops/README.md b/knowledge_base/ci_cd_azure_devops/README.md index e033e8a..c3331ec 100644 --- a/knowledge_base/ci_cd_azure_devops/README.md +++ b/knowledge_base/ci_cd_azure_devops/README.md @@ -36,4 +36,4 @@ See [Azure DevOps authentication](https://docs.databricks.com/aws/en/dev-tools/a - The setup-cli SHA pins CLI `1.20.0`. The bundle accepts CLI `>= 1.20.0`. - After copying the pipeline, set `bundleDirectory` to your bundle path. - Deploy `staging` and `prod` only from CI as the service principal. -- The bundle deploys under `/Workspace/bundle-examples`. Grant CI `CAN_MANAGE` there, or change `root_path`. +- The bundle deploys under `/Workspace/bundle-examples`. Give the CI service principal `CAN_MANAGE` on that folder. Or, change `root_path` in `databricks.yml`. diff --git a/knowledge_base/ci_cd_github_actions/README.md b/knowledge_base/ci_cd_github_actions/README.md index 4cfa54b..d04cd20 100644 --- a/knowledge_base/ci_cd_github_actions/README.md +++ b/knowledge_base/ci_cd_github_actions/README.md @@ -37,4 +37,4 @@ See [GitHub Actions authentication](https://docs.databricks.com/aws/en/dev-tools - `databricks/setup-cli` pins CLI `1.20.0`. The bundle accepts CLI `>= 1.20.0`. - After copying the workflow, set `working-directory` to your bundle path. - Deploy `staging` and `prod` only from CI as the service principal. -- The bundle deploys under `/Workspace/bundle-examples`. Grant CI `CAN_MANAGE` there, or change `root_path`. +- The bundle deploys under `/Workspace/bundle-examples`. Give the CI service principal `CAN_MANAGE` on that folder. Or, change `root_path` in `databricks.yml`. From 68ffb0f321ef58ab79e666b4410721a56b8adfe1 Mon Sep 17 00:00:00 2001 From: Andres Garcia Date: Wed, 21 Oct 2026 22:51:32 +0000 Subject: [PATCH 10/12] Trim deployment path guidance Co-authored-by: omnigent --- knowledge_base/ci_cd_azure_devops/README.md | 1 - knowledge_base/ci_cd_azure_devops/databricks.yml | 1 - knowledge_base/ci_cd_github_actions/README.md | 1 - knowledge_base/ci_cd_github_actions/databricks.yml | 1 - 4 files changed, 4 deletions(-) diff --git a/knowledge_base/ci_cd_azure_devops/README.md b/knowledge_base/ci_cd_azure_devops/README.md index c3331ec..0c6a0f7 100644 --- a/knowledge_base/ci_cd_azure_devops/README.md +++ b/knowledge_base/ci_cd_azure_devops/README.md @@ -36,4 +36,3 @@ See [Azure DevOps authentication](https://docs.databricks.com/aws/en/dev-tools/a - The setup-cli SHA pins CLI `1.20.0`. The bundle accepts CLI `>= 1.20.0`. - After copying the pipeline, set `bundleDirectory` to your bundle path. - Deploy `staging` and `prod` only from CI as the service principal. -- The bundle deploys under `/Workspace/bundle-examples`. Give the CI service principal `CAN_MANAGE` on that folder. Or, change `root_path` in `databricks.yml`. diff --git a/knowledge_base/ci_cd_azure_devops/databricks.yml b/knowledge_base/ci_cd_azure_devops/databricks.yml index 558b80e..d0a928f 100644 --- a/knowledge_base/ci_cd_azure_devops/databricks.yml +++ b/knowledge_base/ci_cd_azure_devops/databricks.yml @@ -15,7 +15,6 @@ targets: staging: mode: production workspace: - # The CI service principal needs CAN_MANAGE on /Workspace/bundle-examples. root_path: /Workspace/bundle-examples/${bundle.name}/${bundle.target} prod: mode: production diff --git a/knowledge_base/ci_cd_github_actions/README.md b/knowledge_base/ci_cd_github_actions/README.md index d04cd20..f8f41ce 100644 --- a/knowledge_base/ci_cd_github_actions/README.md +++ b/knowledge_base/ci_cd_github_actions/README.md @@ -37,4 +37,3 @@ See [GitHub Actions authentication](https://docs.databricks.com/aws/en/dev-tools - `databricks/setup-cli` pins CLI `1.20.0`. The bundle accepts CLI `>= 1.20.0`. - After copying the workflow, set `working-directory` to your bundle path. - Deploy `staging` and `prod` only from CI as the service principal. -- The bundle deploys under `/Workspace/bundle-examples`. Give the CI service principal `CAN_MANAGE` on that folder. Or, change `root_path` in `databricks.yml`. diff --git a/knowledge_base/ci_cd_github_actions/databricks.yml b/knowledge_base/ci_cd_github_actions/databricks.yml index 658580a..30c07ed 100644 --- a/knowledge_base/ci_cd_github_actions/databricks.yml +++ b/knowledge_base/ci_cd_github_actions/databricks.yml @@ -15,7 +15,6 @@ targets: staging: mode: production workspace: - # The CI service principal needs CAN_MANAGE on /Workspace/bundle-examples. root_path: /Workspace/bundle-examples/${bundle.name}/${bundle.target} prod: mode: production From 0c59ef1e56583ce471767f5ab2c859648db4b35f Mon Sep 17 00:00:00 2001 From: Andres Garcia Date: Wed, 21 Oct 2026 22:54:26 +0000 Subject: [PATCH 11/12] Align README with sibling examples Co-authored-by: omnigent --- knowledge_base/ci_cd_azure_devops/README.md | 10 +--------- knowledge_base/ci_cd_github_actions/README.md | 10 +--------- 2 files changed, 2 insertions(+), 18 deletions(-) diff --git a/knowledge_base/ci_cd_azure_devops/README.md b/knowledge_base/ci_cd_azure_devops/README.md index 0c6a0f7..43e0bcf 100644 --- a/knowledge_base/ci_cd_azure_devops/README.md +++ b/knowledge_base/ci_cd_azure_devops/README.md @@ -20,7 +20,7 @@ See [Azure DevOps authentication](https://docs.databricks.com/aws/en/dev-tools/a 7. Set pipeline variables `DATABRICKS_HOST` and `DATABRICKS_CLIENT_ID`. 8. Create `staging` and `prod` Azure DevOps environments. 9. Add an approval check to the `prod` environment. -10. Copy `azure-pipelines.yml` to your repository root. +10. Copy `azure-pipelines.yml` to your repository root. Set `bundleDirectory` to your bundle path. 11. Open a pull request to run validation. | Trigger | Command and target | @@ -28,11 +28,3 @@ See [Azure DevOps authentication](https://docs.databricks.com/aws/en/dev-tools/a | Pull request | `databricks bundle validate -t dev` | | Push to `main` | `databricks bundle deploy -t staging` | | Production approval | `databricks bundle deploy -t prod` | - -## Notes - -- Do not use personal access tokens. PR validation needs workspace authentication. Fork pull requests receive no OIDC token. -- Environment locks serialize deployments. The `prod` approval check gates production. -- The setup-cli SHA pins CLI `1.20.0`. The bundle accepts CLI `>= 1.20.0`. -- After copying the pipeline, set `bundleDirectory` to your bundle path. -- Deploy `staging` and `prod` only from CI as the service principal. diff --git a/knowledge_base/ci_cd_github_actions/README.md b/knowledge_base/ci_cd_github_actions/README.md index f8f41ce..e7e94c6 100644 --- a/knowledge_base/ci_cd_github_actions/README.md +++ b/knowledge_base/ci_cd_github_actions/README.md @@ -21,7 +21,7 @@ See [GitHub Actions authentication](https://docs.databricks.com/aws/en/dev-tools 8. Create `dev`, `staging`, and `prod` GitHub environments. 9. Set `DATABRICKS_HOST` and `DATABRICKS_CLIENT_ID` in each environment. 10. Add required reviewers to the `prod` environment. -11. Copy `.github/workflows/bundle.yml` to your repository root. +11. Copy `.github/workflows/bundle.yml` to your repository root. Set `working-directory` to your bundle path. 12. Open a pull request to run validation. | Trigger | Command and target | @@ -29,11 +29,3 @@ See [GitHub Actions authentication](https://docs.databricks.com/aws/en/dev-tools | Pull request | `databricks bundle validate -t dev` | | Push to `main` | `databricks bundle deploy -t staging` | | Production approval | `databricks bundle deploy -t prod` | - -## Notes - -- Do not use personal access tokens. PR validation needs workspace authentication. Fork pull requests receive no OIDC token. -- Concurrency groups serialize each target. Required reviewers gate production. -- `databricks/setup-cli` pins CLI `1.20.0`. The bundle accepts CLI `>= 1.20.0`. -- After copying the workflow, set `working-directory` to your bundle path. -- Deploy `staging` and `prod` only from CI as the service principal. From 5566ac2b8b8dd292ee162307f06978dbee467459 Mon Sep 17 00:00:00 2001 From: Andres Garcia Date: Thu, 22 Oct 2026 02:35:24 +0000 Subject: [PATCH 12/12] Update CI/CD serverless environments Co-authored-by: omnigent --- knowledge_base/ci_cd_azure_devops/resources/serverless.job.yml | 2 +- .../ci_cd_github_actions/resources/serverless.job.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/knowledge_base/ci_cd_azure_devops/resources/serverless.job.yml b/knowledge_base/ci_cd_azure_devops/resources/serverless.job.yml index bcecb56..d153991 100644 --- a/knowledge_base/ci_cd_azure_devops/resources/serverless.job.yml +++ b/knowledge_base/ci_cd_azure_devops/resources/serverless.job.yml @@ -10,4 +10,4 @@ resources: environments: - environment_key: default spec: - environment_version: "4" + environment_version: "6" diff --git a/knowledge_base/ci_cd_github_actions/resources/serverless.job.yml b/knowledge_base/ci_cd_github_actions/resources/serverless.job.yml index f8cadac..bff0ad7 100644 --- a/knowledge_base/ci_cd_github_actions/resources/serverless.job.yml +++ b/knowledge_base/ci_cd_github_actions/resources/serverless.job.yml @@ -10,4 +10,4 @@ resources: environments: - environment_key: default spec: - environment_version: "4" + environment_version: "6"