diff --git a/.github/workflows/deploy_prd.yml b/.github/workflows/deploy_prd.yml index edee33e..1945ced 100644 --- a/.github/workflows/deploy_prd.yml +++ b/.github/workflows/deploy_prd.yml @@ -1,10 +1,10 @@ -name: Deploy Databricks Bundle PRD +name: Deploy Databricks Bundle PROD on: workflow_dispatch: push: branches: - - main + - prod jobs: deploy-prd: @@ -35,21 +35,21 @@ jobs: - name: Validar bundle Databricks env: - DATABRICKS_CONFIG_PROFILE: PRD + DATABRICKS_CONFIG_PROFILE: PROD run: | - cd dab_treinamento - databricks bundle validate --target prd + cd dab_test + databricks bundle validate --target prod - name: Deploy bundle Databricks env: - DATABRICKS_CONFIG_PROFILE: PRD + DATABRICKS_CONFIG_PROFILE: PROD run: | - cd dab_treinamento - databricks bundle deploy --target prd + cd dab_test + databricks bundle deploy --target prod - name: Executar job Databricks env: - DATABRICKS_CONFIG_PROFILE: PRD + DATABRICKS_CONFIG_PROFILE: PROD run: | - cd dab_treinamento - databricks bundle run dab_treinamento_job --target prd \ No newline at end of file + cd dab_test + databricks bundle run dab_test_job --target prod \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..5d9096e --- /dev/null +++ b/README.md @@ -0,0 +1,972 @@ +# Databricks Asset Bundle — Python Data Engineering - Databricks platform engineering / CI/CD reference implementation. +A practical Databricks Data Engineering project demonstrating how to develop, test, package, deploy, and execute Python workloads using **Databricks Asset Bundles (DAB)** and **GitHub Actions CI/CD**. + +The project implements separate **DEV and PROD environments**, Python Wheel packaging, automated testing, parameterized Databricks Jobs, OAuth Service Principal authentication, and a reproducible development environment using VS Code Dev Containers. + +--- + +## Architecture + +```text + GitHub Repository + │ + ┌───────────┴───────────┐ + │ │ + dev prod + │ │ + ▼ ▼ + deploy_dev.yml deploy_prd.yml + │ │ + ▼ ▼ + GitHub Actions GitHub Actions + │ │ + ┌─────┴─────┐ ┌─────┴─────┐ + │ │ │ │ + Tests Validate Validate Deploy + │ │ │ │ + └─────┬─────┘ └─────┬─────┘ + │ │ + ▼ ▼ + DAB Bundle DAB Bundle + │ │ + ▼ ▼ + Databricks DEV Databricks PROD + │ │ + └───────────┬───────────┘ + │ + ▼ + dab_test_job + │ + ▼ + demo_notebook.py + │ + ▼ + Unity Catalog + catalog.schema.users +``` + +--- + +# Project Overview + +This project is based on the **Databricks Asset Bundle default Python project structure** and has been extended to demonstrate a more complete Data Engineering deployment workflow. + +The main workload demonstrates: + +1. Python application development. +2. Databricks notebook execution. +3. Parameterized Databricks Jobs. +4. Python Wheel packaging. +5. Databricks Asset Bundle configuration. +6. DEV and PROD environments. +7. Automated testing. +8. GitHub Actions CI/CD. +9. OAuth Service Principal authentication. +10. Containerized local development. + +--- + +# Technology Stack + +| Technology | Purpose | +| ------------------------ | ------------------------------------------- | +| Python 3.10–3.12 | Application development | +| PySpark | Distributed data processing | +| Databricks | Data Engineering platform | +| Databricks Asset Bundles | Deployment and resource management | +| Databricks CLI | Bundle validation, deployment and execution | +| Unity Catalog | Catalog/schema/table organization | +| uv | Python dependency management and build | +| Hatchling | Python Wheel build backend | +| Pytest | Automated testing | +| Ruff | Python linting | +| GitHub Actions | CI/CD | +| Docker | Development environment | +| VS Code Dev Containers | Reproducible development environment | +| YAML | Databricks and CI/CD configuration | +| TOML | Python project configuration | + +--- + +# Repository Structure + +```text +. +├── .devcontainer/ +│ ├── .env +│ ├── Dockerfile +│ ├── devcontainer.json +│ └── requirements.txt +│ +├── .github/ +│ └── workflows/ +│ ├── deploy_dev.yml +│ └── deploy_prd.yml +│ +├── dab_test/ +│ ├── .vscode/ +│ │ +│ ├── fixtures/ +│ │ +│ ├── resources/ +│ │ └── jobs/ +│ │ └── dab_test_job.yml +│ │ +│ ├── src/ +│ │ ├── dab_test/ +│ │ │ ├── __init__.py +│ │ │ └── main.py +│ │ │ +│ │ └── notebooks/ +│ │ └── demo_notebook.py +│ │ +│ ├── tests/ +│ │ ├── job_config_test.py +│ │ └── main_test.py +│ │ +│ ├── AGENTS.md +│ ├── CLAUDE.md +│ ├── README.md +│ ├── databricks.yml +│ └── pyproject.toml +│ +└── README.md +``` + +--- + +# Databricks Asset Bundle + +The core of the project is the `databricks.yml` file. + +The Bundle is named: + +```yaml +bundle: + name: dab_test +``` + +The configuration includes resource definitions from: + +```text +resources/jobs/*.yml +resources/pipelines/*.yml +resources/schemas/*.yml +``` + +It also defines a Python Wheel artifact: + +```yaml +artifacts: + python_artifact: + type: whl + build: uv build --wheel +``` + +This means the deployment process can build the Python application into a `.whl` package and make it available to the Databricks workload. + +--- + +# Environments + +The project defines two Databricks Bundle targets. + +## DEV + +The `dev` target uses: + +```yaml +mode: development +``` + +and is the default Bundle target. + +It uses the following configuration: + +```text +Catalog: dev +Schema: rescue_b +``` + +The DEV workflow is intended for development, testing, validation, deployment, and execution. + +--- + +## PROD + +The `prod` target uses: + +```yaml +mode: production +``` + +with: + +```text +Catalog: prod +Schema: rescue_b +``` + +The production deployment is associated with the `prod` Git branch. + +The production workflow can be triggered by: + +* a push to the `prod` branch; +* manual GitHub Actions execution. + +--- + +# Python Application + +The main Python package is located under: + +```text +dab_test/src/dab_test/ +``` + +The main application contains a simple Spark workload that reads the Databricks sample NYC Taxi dataset: + +```python +def find_all_taxis() -> DataFrame: + return spark.read.table("samples.nyctaxi.trips") +``` + +The `main()` function displays the first five records. + +The project therefore provides a minimal Python/Spark workload that can be packaged and executed through the Databricks environment. + +--- + +# Demo Notebook + +The project also contains: + +```text +dab_test/src/notebooks/demo_notebook.py +``` + +The notebook demonstrates parameterized Databricks execution using widgets: + +```text +catalog +user_id +user_name +``` + +The notebook: + +1. Receives runtime parameters. +2. Creates a `users` table if it does not exist. +3. Inserts sample users. +4. Inserts the parameterized user. +5. Reads the resulting table. +6. Displays the resulting DataFrame. + +The target table follows the pattern: + +```text +.rescue_b.users +``` + +For example: + +```text +dev.rescue_b.users +prod.rescue_b.users +``` + +--- + +# Databricks Job + +The Job is defined in: + +```text +dab_test/resources/jobs/dab_test_job.yml +``` + +The Job is called: + +```text +dab_test_job +``` + +It contains a task named: + +```text +ingestao_usuarios +``` + +which executes: + +```text +src/notebooks/demo_notebook.py +``` + +The Job exposes parameters including: + +```text +catalog_name +user_id +user_name +``` + +This allows the same notebook to be reused with different runtime values instead of hard-coding the input data. + +--- + +# Job Schedule + +The Job configuration defines a Quartz cron schedule: + +```text +Every Tuesday at 08:00 +Timezone: America/Sao_Paulo +``` + +The Job also has a timeout of: + +```text +900 seconds +``` + +Failure notifications are configured through Databricks Job email notifications. + +--- + +# Dynamic Parameters + +One of the main concepts demonstrated by this project is the separation between: + +### Bundle variables + +Defined in: + +```text +databricks.yml +``` + +Example: + +```text +catalog +schema +catalog_name +performance_target +``` + +and: + +### Job parameters + +Defined in: + +```text +resources/jobs/dab_test_job.yml +``` + +Example: + +```text +catalog_name +user_id +user_name +``` + +The values can flow through the deployment configuration into the notebook at runtime. + +Conceptually: + +```text +databricks.yml + │ + │ ${var.catalog_name} + ▼ +Databricks Job + │ + │ {{job.parameters.catalog_name}} + ▼ +demo_notebook.py + │ + ▼ +Unity Catalog +``` + +--- + +# Python Packaging + +The Python project is configured through: + +```text +dab_test/pyproject.toml +``` + +The project supports: + +```text +Python >= 3.10 +Python < 3.13 +``` + +The build system uses: + +```text +Hatchling +``` + +The Wheel package is built using: + +```bash +uv build --wheel +``` + +The resulting package can be used by the Databricks deployment. + +--- + +# Development Dependencies + +The project includes development dependencies for: + +* Pytest +* Ruff +* PyYAML +* Databricks DLT +* Databricks Connect +* IPython Kernel + +Install the development dependencies using: + +```bash +uv sync --dev +``` + +--- + +# Testing + +Tests are located under: + +```text +dab_test/tests/ +``` + +Current test modules include: + +```text +main_test.py +job_config_test.py +``` + +Run the tests locally with: + +```bash +uv run pytest +``` + +The DEV CI/CD workflow also executes: + +```bash +uv run pytest -s +``` + +before Bundle validation and deployment. + +--- + +# CI/CD + +The repository contains two GitHub Actions workflows: + +```text +.github/workflows/ +├── deploy_dev.yml +└── deploy_prd.yml +``` + +## DEV Pipeline + +The DEV workflow is triggered by pushes to: + +```text +dev +``` + +It performs: + +```text +Checkout + │ + ▼ +Install Databricks CLI + │ + ▼ +Install uv + │ + ▼ +Configure Databricks OAuth + │ + ▼ +Run Pytest + │ + ▼ +Validate Bundle + │ + ▼ +Deploy Bundle + │ + ▼ +Run Databricks Job +``` + +--- + +# PROD Pipeline + +The production workflow is triggered by: + +```text +push to prod +``` + +or manually through: + +```text +workflow_dispatch +``` + +The production workflow performs: + +```text +Checkout + │ + ▼ +Install Databricks CLI + │ + ▼ +Install uv + │ + ▼ +Configure PROD OAuth Service Principal + │ + ▼ +Validate Bundle + │ + ▼ +Deploy Bundle + │ + ▼ +Run Databricks Job +``` + +The PROD workflow deploys using: + +```bash +databricks bundle deploy --target prod +``` + +and executes: + +```bash +databricks bundle run dab_test_job --target prod +``` + +--- + +# Authentication + +GitHub Actions authenticates to Databricks using an **OAuth Service Principal**. + +The workflows obtain the following values from GitHub Secrets: + +```text +DATABRICKS_HOST +DATABRICKS_CLIENT_ID +DATABRICKS_CLIENT_SECRET +``` + +Environment-specific secrets are used for DEV and PROD. + +For example: + +```text +DATABRICKS_HOST_DEV +DATABRICKS_CLIENT_ID_DEV +DATABRICKS_CLIENT_SECRET_DEV +``` + +and: + +```text +DATABRICKS_HOST_PRD +DATABRICKS_CLIENT_ID_PRD +DATABRICKS_CLIENT_SECRET_PRD +``` + +No credentials should be committed to the repository. + +--- + +# Local Development + +## Prerequisites + +Recommended tools: + +* Git +* Docker +* Visual Studio Code +* VS Code Dev Containers extension +* Python 3.10–3.12 +* uv +* Databricks CLI + +--- + +## Dev Container + +The project provides a development container under: + +```text +.devcontainer/ +``` + +with: + +```text +Dockerfile +devcontainer.json +requirements.txt +.env +``` + +Open the repository in VS Code and select: + +```text +Dev Containers: Reopen in Container +``` + +This provides a reproducible development environment. + +> Never commit production credentials, access tokens, client secrets, or other sensitive information to `.env` or the repository. + +--- + +# Databricks CLI + +Authenticate to Databricks using your preferred authentication method. + +For a local profile: + +```bash +databricks configure +``` + +Verify the CLI: + +```bash +databricks --version +``` + +--- + +# Validate the Bundle + +From the project directory: + +```bash +cd dab_test +``` + +Validate DEV: + +```bash +databricks bundle validate --target dev +``` + +Validate PROD: + +```bash +databricks bundle validate --target prod +``` + +Validation should be performed before deployment. + +--- + +# Deploy to DEV + +```bash +cd dab_test + +databricks bundle deploy --target dev +``` + +After deployment, execute the Job: + +```bash +databricks bundle run dab_test_job --target dev +``` + +--- + +# Deploy to PROD + +Production deployment should preferably occur through the GitHub Actions workflow. + +The equivalent CLI commands are: + +```bash +cd dab_test + +databricks bundle validate --target prod + +databricks bundle deploy --target prod + +databricks bundle run dab_test_job --target prod +``` + +--- + +# End-to-End Deployment Flow + +The complete workflow is: + +```text +Developer + │ + │ git push + ▼ +GitHub + │ + ├───────────────┐ + │ │ + dev prod + │ │ + ▼ ▼ +DEV Workflow PROD Workflow + │ │ + ▼ ▼ +Pytest Validation + │ │ + ▼ ▼ +Validation Deployment + │ │ + ▼ ▼ +Deployment Job Execution + │ │ + ▼ ▼ +Databricks DEV Databricks PROD +``` + +--- + +# Configuration Files + +## `databricks.yml` + +Defines the Databricks Asset Bundle. + +Responsible for: + +* Bundle name +* Resources +* Variables +* Artifacts +* DEV target +* PROD target +* Workspace configuration + +--- + +## `pyproject.toml` + +Defines the Python project. + +Responsible for: + +* Python version +* Project metadata +* Dependencies +* Development dependencies +* Entry point +* Build system +* Wheel packaging +* Ruff configuration + +--- + +## `resources/jobs/dab_test_job.yml` + +Defines the Databricks Job. + +Responsible for: + +* Job name +* Job parameters +* Schedule +* Timeout +* Notifications +* Notebook task +* Runtime parameters +* Performance configuration + +--- + +## `.github/workflows/deploy_dev.yml` + +Defines the DEV CI/CD pipeline. + +Responsible for: + +* Installing tooling +* Authentication +* Running tests +* Bundle validation +* Deployment +* Job execution + +--- + +## `.github/workflows/deploy_prd.yml` + +Defines the PROD CI/CD pipeline. + +Responsible for: + +* Installing tooling +* PROD authentication +* Bundle validation +* Production deployment +* Job execution + +--- + +# Key Engineering Concepts + +This project demonstrates the following Data Engineering and DevOps practices: + +### Infrastructure as Code + +Databricks resources are defined as code rather than being created manually through the Databricks UI. + +### Environment Separation + +The same Bundle supports: + +```text +DEV +PROD +``` + +using different targets. + +### Python Packaging + +The application is packaged as a Python Wheel before deployment. + +### Automated Testing + +Pytest is integrated into the DEV deployment pipeline. + +### CI/CD + +GitHub Actions automates the deployment lifecycle. + +### Parameterized Workloads + +The Databricks Job passes runtime parameters into the notebook. + +### Service Principal Authentication + +CI/CD uses OAuth-based Service Principal authentication rather than personal credentials. + +### Reproducible Development + +The Dev Container provides a consistent development environment. + +--- + +# Project Status + +Current implementation: + +* [x] Databricks Asset Bundle +* [x] Python project structure +* [x] PySpark application +* [x] Databricks notebook +* [x] Parameterized Databricks Job +* [x] Unity Catalog catalog/schema configuration +* [x] DEV target +* [x] PROD target +* [x] Python Wheel packaging +* [x] uv dependency management +* [x] Hatchling build system +* [x] Pytest tests +* [x] Ruff configuration +* [x] VS Code Dev Container +* [x] GitHub Actions DEV pipeline +* [x] GitHub Actions PROD pipeline +* [x] Databricks OAuth Service Principal authentication +* [x] Automated Databricks Job execution + +--- + +# Future Improvements + +Potential extensions include: + +* [ ] Add Delta Lake ingestion and transformation layers +* [ ] Add Bronze/Silver/Gold architecture +* [ ] Add data quality validation +* [ ] Add integration tests against Databricks +* [ ] Add structured logging +* [ ] Add monitoring and alerting +* [ ] Add Unity Catalog permissions management +* [ ] Add CI quality gates for Ruff +* [ ] Add pull-request validation +* [ ] Add deployment approval gates for PROD +* [ ] Add infrastructure documentation +* [ ] Add job run monitoring +* [ ] Add data lineage documentation + +--- + +# Learning Objectives + +This project was created to demonstrate practical knowledge of: + +```text +Python + │ + ├── Packaging + ├── Testing + └── PySpark + │ + ▼ + Databricks + │ + ├── Asset Bundles + ├── Jobs + ├── Notebooks + └── Unity Catalog + │ + ▼ + DevOps + │ + ├── Git + ├── GitHub Actions + ├── CI/CD + └── Service Principal +``` + +The main objective is to demonstrate how a Data Engineering workload can move from **local development to a controlled production deployment** using modern software engineering practices. + +--- + +# Author + +**Ruben Cruz** + +Data Engineering · Data Integration · Python · PySpark · Databricks · CI/CD diff --git a/dab_test b/dab_test deleted file mode 160000 index 2850ef9..0000000 --- a/dab_test +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 2850ef9a66e0c9446329cbf7c384c69139185c2e diff --git a/dab_test/.vscode/__builtins__.pyi b/dab_test/.vscode/__builtins__.pyi new file mode 100644 index 0000000..0edd518 --- /dev/null +++ b/dab_test/.vscode/__builtins__.pyi @@ -0,0 +1,3 @@ +# Typings for Pylance in Visual Studio Code +# see https://github.com/microsoft/pyright/blob/main/docs/builtins.md +from databricks.sdk.runtime import * diff --git a/dab_test/.vscode/extensions.json b/dab_test/.vscode/extensions.json new file mode 100644 index 0000000..b958aac --- /dev/null +++ b/dab_test/.vscode/extensions.json @@ -0,0 +1,7 @@ +{ + "recommendations": [ + "charliermarsh.ruff", + "databricks.databricks", + "redhat.vscode-yaml" + ] +} diff --git a/dab_test/.vscode/settings.json b/dab_test/.vscode/settings.json new file mode 100644 index 0000000..d73c73b --- /dev/null +++ b/dab_test/.vscode/settings.json @@ -0,0 +1,39 @@ +{ + "jupyter.interactiveWindow.cellMarker.codeRegex": "^# COMMAND ----------|^# Databricks notebook source|^(#\\s*%%|#\\s*\\|#\\s*In\\[\\d*?\\]|#\\s*In\\[ \\])", + "jupyter.interactiveWindow.cellMarker.default": "# COMMAND ----------", + "python.testing.pytestArgs": [ + "." + ], + "files.exclude": { + "**/*.egg-info": true, + "**/__pycache__": true, + ".pytest_cache": true, + "dist": true, + }, + "files.associations": { + "**/.gitkeep": "markdown" + }, + + // Pylance settings (VS Code) + // Set typeCheckingMode to "basic" to enable type checking! + "python.analysis.typeCheckingMode": "off", + "python.analysis.extraPaths": ["src", "lib", "resources"], + "python.analysis.diagnosticMode": "workspace", + "python.analysis.stubPath": ".vscode", + + // Pyright settings (Cursor) + // Set typeCheckingMode to "basic" to enable type checking! + "cursorpyright.analysis.typeCheckingMode": "off", + "cursorpyright.analysis.extraPaths": ["src", "lib", "resources"], + "cursorpyright.analysis.diagnosticMode": "workspace", + "cursorpyright.analysis.stubPath": ".vscode", + + // General Python settings + "python.defaultInterpreterPath": "./.venv/bin/python", + "python.testing.unittestEnabled": false, + "python.testing.pytestEnabled": true, + "[python]": { + "editor.defaultFormatter": "charliermarsh.ruff", + "editor.formatOnSave": true, + }, +} diff --git a/dab_test/AGENTS.md b/dab_test/AGENTS.md new file mode 100644 index 0000000..fdcca98 --- /dev/null +++ b/dab_test/AGENTS.md @@ -0,0 +1,25 @@ +# Declarative Automation Bundles Project + +This project uses Declarative Automation Bundles (DABs) for deployment. Add project-specific instructions below. + +## For AI Agents: Use Databricks AI Tools + +**BEFORE any other action, read the `databricks-core` skill.** + +It sets you up to work with this project reliably: CLI authentication, profile +selection, data discovery, and the bundle deployment workflow. Without it, +results are often slower and less accurate. + +If this skill is not available (Databricks AI Tools are not installed), you can install them for your coding agent in seconds: + +```bash +databricks aitools install +``` + +If the CLI is not installed, see: https://docs.databricks.com/dev-tools/cli/install + +--- + +## Project Instructions + + diff --git a/dab_test/CLAUDE.md b/dab_test/CLAUDE.md new file mode 100644 index 0000000..5612c9b --- /dev/null +++ b/dab_test/CLAUDE.md @@ -0,0 +1,6 @@ +# CLAUDE.md + +Project guidance for AI agents lives in AGENTS.md. +Claude Code loads it via the import below. + +@AGENTS.md diff --git a/dab_test/README.md b/dab_test/README.md new file mode 100644 index 0000000..bdf9c44 --- /dev/null +++ b/dab_test/README.md @@ -0,0 +1,71 @@ +# dab_test + +The 'dab_test' project was generated by using the default-python template. + +* `src/`: Python source code for this project. + * `src/dab_test/`: Shared Python code that can be used by jobs and pipelines. +* `resources/`: Resource configurations (jobs, pipelines, etc.) +* `tests/`: Unit tests for the shared Python code. +* `fixtures/`: Fixtures for data sets (primarily used for testing). + + +## Getting started + +Choose how you want to work on this project: + +(a) Directly in your Databricks workspace, see + https://docs.databricks.com/dev-tools/bundles/workspace. + +(b) Locally with an IDE like Cursor or VS Code, see + https://docs.databricks.com/dev-tools/vscode-ext.html. + +(c) With command line tools, see https://docs.databricks.com/dev-tools/cli/databricks-cli.html + +If you're developing with an IDE, dependencies for this project should be installed using uv: + +* Make sure you have the UV package manager installed. + It's an alternative to tools like pip: https://docs.astral.sh/uv/getting-started/installation/. +* Run `uv sync --dev` to install the project's dependencies. + + +# Using this project using the CLI + +The Databricks workspace and IDE extensions provide a graphical interface for working +with this project. It's also possible to interact with it directly using the CLI: + +1. Authenticate to your Databricks workspace, if you have not done so already: + ``` + $ databricks configure + ``` + +2. To deploy a development copy of this project, type: + ``` + $ databricks bundle deploy --target dev + ``` + (Note that "dev" is the default target, so the `--target` parameter + is optional here.) + + This deploys everything that's defined for this project. + For example, the default template would deploy a pipeline called + `[dev yourname] dab_test_etl` to your workspace. + You can find that resource by opening your workpace and clicking on **Jobs & Pipelines**. + +3. Similarly, to deploy a production copy, type: + ``` + $ databricks bundle deploy --target prod + ``` + Note the default template has a includes a job that runs the pipeline every day + (defined in resources/sample_job.job.yml). The schedule + is paused when deploying in development mode (see + https://docs.databricks.com/dev-tools/bundles/deployment-modes.html). + +4. To run a job or pipeline, use the "run" command: + ``` + $ databricks bundle run + ``` + +5. Finally, to run tests locally, use `pytest`: + ``` + $ uv run pytest + ``` +video 01 diff --git a/dab_test/databricks.yml b/dab_test/databricks.yml new file mode 100644 index 0000000..8cb2b66 --- /dev/null +++ b/dab_test/databricks.yml @@ -0,0 +1,53 @@ +# This is a Declarative Automation Bundle definition for dab_test. +# See https://docs.databricks.com/dev-tools/bundles/index.html for documentation. +bundle: + name: dab_test + uuid: 348afa13-f395-4806-9fcf-f5fc52935756 + +include: + - resources/jobs/*.yml + - resources/pipelines/*.yml + - resources/schemas/*.yml + +artifacts: + python_artifact: + type: whl + build: uv build --wheel + +# Variable declarations. These variables are assigned in the dev/prod targets below. +variables: + catalog: + description: The catalog to use + schema: + description: The schema to use + catalog_name: + description: "Catalog used by the job" + default: dev + performance_target: + description: "Performance target for the job" + default: "STANDARD" +targets: + dev: + # The default target uses 'mode: development' to create a development copy. + # - Deployed resources get prefixed with '[dev my_user_name]' + # - Any job schedules and triggers are paused by default. + # See also https://docs.databricks.com/dev-tools/bundles/deployment-modes.html. + mode: development + default: true + workspace: + host: https://dbc-39ee3252-8e93.cloud.databricks.com + variables: + catalog: dev + schema: dev + prod: + mode: production + workspace: + host: https://dbc-39ee3252-8e93.cloud.databricks.com + # We explicitly deploy to /Workspace/Users/rubencruzh@gmail.com to make sure we only have a single copy. + root_path: /Workspace/Users/rubencruzh@gmail.com/.bundle/${bundle.name}/${bundle.target} + variables: + catalog: prod + schema: prod + permissions: + - user_name: rubencruzh@gmail.com + level: CAN_MANAGE diff --git a/dab_test/fixtures/.gitkeep b/dab_test/fixtures/.gitkeep new file mode 100644 index 0000000..77a9066 --- /dev/null +++ b/dab_test/fixtures/.gitkeep @@ -0,0 +1,9 @@ +# Test fixtures directory + +Add JSON or CSV files here. In tests, use them with `load_fixture()`: + +``` +def test_using_fixture(load_fixture): + data = load_fixture("my_data.json") + assert len(data) >= 1 +``` diff --git a/dab_test/pyproject.toml b/dab_test/pyproject.toml new file mode 100644 index 0000000..24abc54 --- /dev/null +++ b/dab_test/pyproject.toml @@ -0,0 +1,36 @@ +[project] +name = "dab_test" +version = "0.0.1" +authors = [{ name = "rubencruzh@gmail.com" }] +requires-python = ">=3.10,<3.13" +dependencies = [ + # Any dependencies for jobs and pipelines in this project can be added here + # See also https://docs.databricks.com/dev-tools/bundles/library-dependencies + # + # LIMITATION: for pipelines, dependencies are cached during development; + # add dependencies to the 'environment' section of your pipeline.yml file instead +] + +[dependency-groups] +dev = [ + "pytest", + "ruff", + "pyyaml", + "databricks-dlt", + "databricks-connect>=15.4,<15.5", + "ipykernel", +] + +[project.scripts] +main = "dab_test.main:main" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + + +[tool.hatch.build.targets.wheel] +packages = ["src/dab_test"] + +[tool.ruff] +line-length = 120 diff --git a/dab_test/resources/jobs/dab_test_job.yml b/dab_test/resources/jobs/dab_test_job.yml new file mode 100644 index 0000000..4b3191f --- /dev/null +++ b/dab_test/resources/jobs/dab_test_job.yml @@ -0,0 +1,36 @@ +# Job do treinamento para executar o notebook demo. +resources: + jobs: + dab_test_job: + name: dab_test_job + tags: + treinamento: dab + ambiente: dev + area: engenharia + description: Job de treinamento que ingere usuários de exemplo com parâmetros dinâmicos. + parameters: + - name: catalog_name + default: ${var.catalog_name} + - name: user_id + default: "3" + - name: user_name + default: "Anselmo" + email_notifications: + on_failure: + - rubencruzh@gmail.com + timeout_seconds: 900 + schedule: + quartz_cron_expression: "0 0 8 ? * TUE *" + timezone_id: America/Sao_Paulo + tasks: + - task_key: ingestao_usuarios + description: Ingestão de usuários de exemplo via notebook. + notebook_task: + notebook_path: ../../src/notebooks/demo_notebook.py + base_parameters: + catalog_name: "{{job.parameters.catalog_name}}" + user_id: "{{job.parameters.user_id}}" + user_name: "{{job.parameters.user_name}}" + queue: + enabled: true + performance_target: ${var.performance_target} \ No newline at end of file diff --git a/dab_test/src/dab_test/____init__.py b/dab_test/src/dab_test/____init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dab_test/src/dab_test/main.py b/dab_test/src/dab_test/main.py new file mode 100644 index 0000000..aadae50 --- /dev/null +++ b/dab_test/src/dab_test/main.py @@ -0,0 +1,14 @@ +from databricks.sdk.runtime import spark +from pyspark.sql import DataFrame + + +def find_all_taxis() -> DataFrame: + return spark.read.table("samples.nyctaxi.trips") + + +def main(): + find_all_taxis().show(5) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/dab_test/src/notebooks/demo_notebook.py b/dab_test/src/notebooks/demo_notebook.py new file mode 100644 index 0000000..a56304d --- /dev/null +++ b/dab_test/src/notebooks/demo_notebook.py @@ -0,0 +1,30 @@ +# Databricks notebook source +# DBTITLE 1,Configura widgets de entrada +dbutils.widgets.text("catalog", "dev", "Catalog Name") +dbutils.widgets.text("user_id", "3", "User ID") +dbutils.widgets.text("user_name", "Anselmo", "User Name") + +# COMMAND ---------- +# DBTITLE 1,Obtém valores informados +catalog_name = dbutils.widgets.get("catalog") +user_id = int(dbutils.widgets.get("user_id")) +user_name = dbutils.widgets.get("user_name") +print(f"Using Catalog: {catalog_name}") +print(f"Inserting user_id={user_id}, user_name={user_name}") + +# COMMAND ---------- +# DBTITLE 1,Cria tabela e insere dados de exemplo + parâmetro +spark.sql( + f"CREATE TABLE IF NOT EXISTS {catalog_name}.rescue_b.users (id INT, name STRING)" +) +spark.sql( + f"INSERT OVERWRITE {catalog_name}.rescue_b.users VALUES (1, 'Alice'), (2, 'Bob')" +) +spark.sql( + f"INSERT INTO {catalog_name}.rescue_b.users VALUES ({user_id}, '{user_name}')" +) + +# COMMAND ---------- +# DBTITLE 1,Lê os dados inseridos +result_df = spark.sql(f"SELECT * FROM {catalog_name}.rescue_b.users ORDER BY id") +display(result_df) \ No newline at end of file diff --git a/dab_test/tests/job_config_test.py b/dab_test/tests/job_config_test.py new file mode 100644 index 0000000..3421ea0 --- /dev/null +++ b/dab_test/tests/job_config_test.py @@ -0,0 +1,54 @@ +from pathlib import Path +import yaml + +ROOT = Path(__file__).resolve().parents[1] +JOB_FILE = ROOT / "resources" / "jobs" / "dab_test_job.yml" + + +def _load_job(): + data = yaml.safe_load(JOB_FILE.read_text()) + job = data["resources"]["jobs"]["dab_test_job"] + return job + + +def test_job_has_description_and_tags(): + job = _load_job() + print("Validando descrição do job...") + assert job.get("description"), "Job description must be set" + print("Descrição OK!") + tags = job.get("tags", {}) + print(f"Validando tags obrigatórias: {list(tags.keys())}") + for tag_key in ("treinamento", "ambiente", "area"): + assert tag_key in tags, f"Tag '{tag_key}' está ausente" + print("Tags obrigatórias OK!") + + +def test_job_has_schedule_and_timeout(): + job = _load_job() + print("Validando agendamento e timeout...") + schedule = job.get("schedule") + assert schedule, "Job schedule deve estar configurado" + print(f"Agendamento encontrado: {schedule['quartz_cron_expression']} ({schedule['timezone_id']})") + assert schedule["quartz_cron_expression"].lower() == "0 0 8 ? * tue *" + assert schedule["timezone_id"] == "America/Sao_Paulo" + assert job.get("timeout_seconds") == 900 + print("Agendamento e timeout OK!") + + +def test_job_parameters_exposed(): + job = _load_job() + params = {p["name"]: p["default"] for p in job.get("parameters", [])} + print(f"Validando parâmetros expostos: {list(params.keys())}") + for expected in ("catalog_name", "user_id", "user_name"): + assert expected in params, f"Parâmetro '{expected}' não configurado" + print("Parâmetros obrigatórios OK!") + + +def test_job_uses_variable_for_performance_target(): + job = _load_job() + print("Validando uso da variável performance_target...") + assert job.get("performance_target") == "${var.performance_target}" + print("performance_target OK!") + +spark = None + diff --git a/dab_test/tests/main_test.py b/dab_test/tests/main_test.py new file mode 100644 index 0000000..38ed1b0 --- /dev/null +++ b/dab_test/tests/main_test.py @@ -0,0 +1,18 @@ +from unittest.mock import MagicMock, patch + +from dab_test import main + + +def test_find_all_taxis(): + mock_df = MagicMock() + mock_reader = MagicMock() + mock_spark = MagicMock() + + mock_spark.read = mock_reader + mock_reader.table.return_value = mock_df + + with patch.object(main, "spark", mock_spark): + taxis = main.find_all_taxis() + + mock_reader.table.assert_called_once_with("samples.nyctaxi.trips") + assert taxis == mock_df \ No newline at end of file