diff --git a/.github/workflows/deploy_prd.yml b/.github/workflows/deploy_prd.yml index edee33e..8ba25f0 100644 --- a/.github/workflows/deploy_prd.yml +++ b/.github/workflows/deploy_prd.yml @@ -4,7 +4,7 @@ on: workflow_dispatch: push: branches: - - main + - prod jobs: deploy-prd: @@ -35,21 +35,21 @@ jobs: - name: Validar bundle Databricks env: - DATABRICKS_CONFIG_PROFILE: PRD + DATABRICKS_CONFIG_PROFILE: PROD run: | - cd dab_treinamento - databricks bundle validate --target prd + cd dab_test + databricks bundle validate --target prod - name: Deploy bundle Databricks env: - DATABRICKS_CONFIG_PROFILE: PRD + DATABRICKS_CONFIG_PROFILE: PROD run: | - cd dab_treinamento - databricks bundle deploy --target prd + cd dab_test + databricks bundle deploy --target prod - name: Executar job Databricks env: - DATABRICKS_CONFIG_PROFILE: PRD + DATABRICKS_CONFIG_PROFILE: PROD run: | - cd dab_treinamento - databricks bundle run dab_treinamento_job --target prd \ No newline at end of file + cd dab_test + databricks bundle run dab_test_job --target prod \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..4aada1d --- /dev/null +++ b/README.md @@ -0,0 +1 @@ +# repositorio-teste video - p diff --git a/dab_test b/dab_test deleted file mode 160000 index 2850ef9..0000000 --- a/dab_test +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 2850ef9a66e0c9446329cbf7c384c69139185c2e diff --git a/dab_test/.vscode/__builtins__.pyi b/dab_test/.vscode/__builtins__.pyi new file mode 100644 index 0000000..0edd518 --- /dev/null +++ b/dab_test/.vscode/__builtins__.pyi @@ -0,0 +1,3 @@ +# Typings for Pylance in Visual Studio Code +# see https://github.com/microsoft/pyright/blob/main/docs/builtins.md +from databricks.sdk.runtime import * diff --git a/dab_test/.vscode/extensions.json b/dab_test/.vscode/extensions.json new file mode 100644 index 0000000..b958aac --- /dev/null +++ b/dab_test/.vscode/extensions.json @@ -0,0 +1,7 @@ +{ + "recommendations": [ + "charliermarsh.ruff", + "databricks.databricks", + "redhat.vscode-yaml" + ] +} diff --git a/dab_test/.vscode/settings.json b/dab_test/.vscode/settings.json new file mode 100644 index 0000000..d73c73b --- /dev/null +++ b/dab_test/.vscode/settings.json @@ -0,0 +1,39 @@ +{ + "jupyter.interactiveWindow.cellMarker.codeRegex": "^# COMMAND ----------|^# Databricks notebook source|^(#\\s*%%|#\\s*\\|#\\s*In\\[\\d*?\\]|#\\s*In\\[ \\])", + "jupyter.interactiveWindow.cellMarker.default": "# COMMAND ----------", + "python.testing.pytestArgs": [ + "." + ], + "files.exclude": { + "**/*.egg-info": true, + "**/__pycache__": true, + ".pytest_cache": true, + "dist": true, + }, + "files.associations": { + "**/.gitkeep": "markdown" + }, + + // Pylance settings (VS Code) + // Set typeCheckingMode to "basic" to enable type checking! + "python.analysis.typeCheckingMode": "off", + "python.analysis.extraPaths": ["src", "lib", "resources"], + "python.analysis.diagnosticMode": "workspace", + "python.analysis.stubPath": ".vscode", + + // Pyright settings (Cursor) + // Set typeCheckingMode to "basic" to enable type checking! + "cursorpyright.analysis.typeCheckingMode": "off", + "cursorpyright.analysis.extraPaths": ["src", "lib", "resources"], + "cursorpyright.analysis.diagnosticMode": "workspace", + "cursorpyright.analysis.stubPath": ".vscode", + + // General Python settings + "python.defaultInterpreterPath": "./.venv/bin/python", + "python.testing.unittestEnabled": false, + "python.testing.pytestEnabled": true, + "[python]": { + "editor.defaultFormatter": "charliermarsh.ruff", + "editor.formatOnSave": true, + }, +} diff --git a/dab_test/AGENTS.md b/dab_test/AGENTS.md new file mode 100644 index 0000000..fdcca98 --- /dev/null +++ b/dab_test/AGENTS.md @@ -0,0 +1,25 @@ +# Declarative Automation Bundles Project + +This project uses Declarative Automation Bundles (DABs) for deployment. Add project-specific instructions below. + +## For AI Agents: Use Databricks AI Tools + +**BEFORE any other action, read the `databricks-core` skill.** + +It sets you up to work with this project reliably: CLI authentication, profile +selection, data discovery, and the bundle deployment workflow. Without it, +results are often slower and less accurate. + +If this skill is not available (Databricks AI Tools are not installed), you can install them for your coding agent in seconds: + +```bash +databricks aitools install +``` + +If the CLI is not installed, see: https://docs.databricks.com/dev-tools/cli/install + +--- + +## Project Instructions + + diff --git a/dab_test/CLAUDE.md b/dab_test/CLAUDE.md new file mode 100644 index 0000000..5612c9b --- /dev/null +++ b/dab_test/CLAUDE.md @@ -0,0 +1,6 @@ +# CLAUDE.md + +Project guidance for AI agents lives in AGENTS.md. +Claude Code loads it via the import below. + +@AGENTS.md diff --git a/dab_test/README.md b/dab_test/README.md new file mode 100644 index 0000000..bdf9c44 --- /dev/null +++ b/dab_test/README.md @@ -0,0 +1,71 @@ +# dab_test + +The 'dab_test' project was generated by using the default-python template. + +* `src/`: Python source code for this project. + * `src/dab_test/`: Shared Python code that can be used by jobs and pipelines. +* `resources/`: Resource configurations (jobs, pipelines, etc.) +* `tests/`: Unit tests for the shared Python code. +* `fixtures/`: Fixtures for data sets (primarily used for testing). + + +## Getting started + +Choose how you want to work on this project: + +(a) Directly in your Databricks workspace, see + https://docs.databricks.com/dev-tools/bundles/workspace. + +(b) Locally with an IDE like Cursor or VS Code, see + https://docs.databricks.com/dev-tools/vscode-ext.html. + +(c) With command line tools, see https://docs.databricks.com/dev-tools/cli/databricks-cli.html + +If you're developing with an IDE, dependencies for this project should be installed using uv: + +* Make sure you have the UV package manager installed. + It's an alternative to tools like pip: https://docs.astral.sh/uv/getting-started/installation/. +* Run `uv sync --dev` to install the project's dependencies. + + +# Using this project using the CLI + +The Databricks workspace and IDE extensions provide a graphical interface for working +with this project. It's also possible to interact with it directly using the CLI: + +1. Authenticate to your Databricks workspace, if you have not done so already: + ``` + $ databricks configure + ``` + +2. To deploy a development copy of this project, type: + ``` + $ databricks bundle deploy --target dev + ``` + (Note that "dev" is the default target, so the `--target` parameter + is optional here.) + + This deploys everything that's defined for this project. + For example, the default template would deploy a pipeline called + `[dev yourname] dab_test_etl` to your workspace. + You can find that resource by opening your workpace and clicking on **Jobs & Pipelines**. + +3. Similarly, to deploy a production copy, type: + ``` + $ databricks bundle deploy --target prod + ``` + Note the default template has a includes a job that runs the pipeline every day + (defined in resources/sample_job.job.yml). The schedule + is paused when deploying in development mode (see + https://docs.databricks.com/dev-tools/bundles/deployment-modes.html). + +4. To run a job or pipeline, use the "run" command: + ``` + $ databricks bundle run + ``` + +5. Finally, to run tests locally, use `pytest`: + ``` + $ uv run pytest + ``` +video 01 diff --git a/dab_test/databricks.yml b/dab_test/databricks.yml new file mode 100644 index 0000000..8ce9056 --- /dev/null +++ b/dab_test/databricks.yml @@ -0,0 +1,51 @@ +# This is a Declarative Automation Bundle definition for dab_test. +# See https://docs.databricks.com/dev-tools/bundles/index.html for documentation. +bundle: + name: dab_test + uuid: 348afa13-f395-4806-9fcf-f5fc52935756 + +include: + - resources/jobs/*.yml + - resources/pipelines/*.yml + - resources/schemas/*.yml + +artifacts: + python_artifact: + type: whl + build: uv build --wheel + +# Variable declarations. These variables are assigned in the dev/prod targets below. +variables: + catalog: + description: The catalog to use + schema: + description: The schema to use + catalog_name: + description: "Catalog used by the job" + default: dev + performance_target: + description: "Performance target for the job" + default: "STANDARD" +targets: + dev: + # The default target uses 'mode: development' to create a development copy. + # - Deployed resources get prefixed with '[dev my_user_name]' + # - Any job schedules and triggers are paused by default. + # See also https://docs.databricks.com/dev-tools/bundles/deployment-modes.html. + mode: development + default: true + workspace: + host: https://dbc-39ee3252-8e93.cloud.databricks.com + variables: + catalog: dev + schema: rescue_b + prod: + mode: production + workspace: + host: https://dbc-39ee3252-8e93.cloud.databricks.com + variables: + catalog: prod + schema: rescue_b + permissions: + - user_name: rubencruzh@gmail.com + level: CAN_MANAGE diff --git a/dab_test/fixtures/.gitkeep b/dab_test/fixtures/.gitkeep new file mode 100644 index 0000000..77a9066 --- /dev/null +++ b/dab_test/fixtures/.gitkeep @@ -0,0 +1,9 @@ +# Test fixtures directory + +Add JSON or CSV files here. In tests, use them with `load_fixture()`: + +``` +def test_using_fixture(load_fixture): + data = load_fixture("my_data.json") + assert len(data) >= 1 +``` diff --git a/dab_test/pyproject.toml b/dab_test/pyproject.toml new file mode 100644 index 0000000..24abc54 --- /dev/null +++ b/dab_test/pyproject.toml @@ -0,0 +1,36 @@ +[project] +name = "dab_test" +version = "0.0.1" +authors = [{ name = "rubencruzh@gmail.com" }] +requires-python = ">=3.10,<3.13" +dependencies = [ + # Any dependencies for jobs and pipelines in this project can be added here + # See also https://docs.databricks.com/dev-tools/bundles/library-dependencies + # + # LIMITATION: for pipelines, dependencies are cached during development; + # add dependencies to the 'environment' section of your pipeline.yml file instead +] + +[dependency-groups] +dev = [ + "pytest", + "ruff", + "pyyaml", + "databricks-dlt", + "databricks-connect>=15.4,<15.5", + "ipykernel", +] + +[project.scripts] +main = "dab_test.main:main" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + + +[tool.hatch.build.targets.wheel] +packages = ["src/dab_test"] + +[tool.ruff] +line-length = 120 diff --git a/dab_test/resources/jobs/dab_test_job.yml b/dab_test/resources/jobs/dab_test_job.yml new file mode 100644 index 0000000..4b3191f --- /dev/null +++ b/dab_test/resources/jobs/dab_test_job.yml @@ -0,0 +1,36 @@ +# Job do treinamento para executar o notebook demo. +resources: + jobs: + dab_test_job: + name: dab_test_job + tags: + treinamento: dab + ambiente: dev + area: engenharia + description: Job de treinamento que ingere usuários de exemplo com parâmetros dinâmicos. + parameters: + - name: catalog_name + default: ${var.catalog_name} + - name: user_id + default: "3" + - name: user_name + default: "Anselmo" + email_notifications: + on_failure: + - rubencruzh@gmail.com + timeout_seconds: 900 + schedule: + quartz_cron_expression: "0 0 8 ? * TUE *" + timezone_id: America/Sao_Paulo + tasks: + - task_key: ingestao_usuarios + description: Ingestão de usuários de exemplo via notebook. + notebook_task: + notebook_path: ../../src/notebooks/demo_notebook.py + base_parameters: + catalog_name: "{{job.parameters.catalog_name}}" + user_id: "{{job.parameters.user_id}}" + user_name: "{{job.parameters.user_name}}" + queue: + enabled: true + performance_target: ${var.performance_target} \ No newline at end of file diff --git a/dab_test/src/dab_test/____init__.py b/dab_test/src/dab_test/____init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dab_test/src/dab_test/main.py b/dab_test/src/dab_test/main.py new file mode 100644 index 0000000..aadae50 --- /dev/null +++ b/dab_test/src/dab_test/main.py @@ -0,0 +1,14 @@ +from databricks.sdk.runtime import spark +from pyspark.sql import DataFrame + + +def find_all_taxis() -> DataFrame: + return spark.read.table("samples.nyctaxi.trips") + + +def main(): + find_all_taxis().show(5) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/dab_test/src/notebooks/demo_notebook.py b/dab_test/src/notebooks/demo_notebook.py new file mode 100644 index 0000000..a56304d --- /dev/null +++ b/dab_test/src/notebooks/demo_notebook.py @@ -0,0 +1,30 @@ +# Databricks notebook source +# DBTITLE 1,Configura widgets de entrada +dbutils.widgets.text("catalog", "dev", "Catalog Name") +dbutils.widgets.text("user_id", "3", "User ID") +dbutils.widgets.text("user_name", "Anselmo", "User Name") + +# COMMAND ---------- +# DBTITLE 1,Obtém valores informados +catalog_name = dbutils.widgets.get("catalog") +user_id = int(dbutils.widgets.get("user_id")) +user_name = dbutils.widgets.get("user_name") +print(f"Using Catalog: {catalog_name}") +print(f"Inserting user_id={user_id}, user_name={user_name}") + +# COMMAND ---------- +# DBTITLE 1,Cria tabela e insere dados de exemplo + parâmetro +spark.sql( + f"CREATE TABLE IF NOT EXISTS {catalog_name}.rescue_b.users (id INT, name STRING)" +) +spark.sql( + f"INSERT OVERWRITE {catalog_name}.rescue_b.users VALUES (1, 'Alice'), (2, 'Bob')" +) +spark.sql( + f"INSERT INTO {catalog_name}.rescue_b.users VALUES ({user_id}, '{user_name}')" +) + +# COMMAND ---------- +# DBTITLE 1,Lê os dados inseridos +result_df = spark.sql(f"SELECT * FROM {catalog_name}.rescue_b.users ORDER BY id") +display(result_df) \ No newline at end of file diff --git a/dab_test/tests/job_config_test.py b/dab_test/tests/job_config_test.py new file mode 100644 index 0000000..3421ea0 --- /dev/null +++ b/dab_test/tests/job_config_test.py @@ -0,0 +1,54 @@ +from pathlib import Path +import yaml + +ROOT = Path(__file__).resolve().parents[1] +JOB_FILE = ROOT / "resources" / "jobs" / "dab_test_job.yml" + + +def _load_job(): + data = yaml.safe_load(JOB_FILE.read_text()) + job = data["resources"]["jobs"]["dab_test_job"] + return job + + +def test_job_has_description_and_tags(): + job = _load_job() + print("Validando descrição do job...") + assert job.get("description"), "Job description must be set" + print("Descrição OK!") + tags = job.get("tags", {}) + print(f"Validando tags obrigatórias: {list(tags.keys())}") + for tag_key in ("treinamento", "ambiente", "area"): + assert tag_key in tags, f"Tag '{tag_key}' está ausente" + print("Tags obrigatórias OK!") + + +def test_job_has_schedule_and_timeout(): + job = _load_job() + print("Validando agendamento e timeout...") + schedule = job.get("schedule") + assert schedule, "Job schedule deve estar configurado" + print(f"Agendamento encontrado: {schedule['quartz_cron_expression']} ({schedule['timezone_id']})") + assert schedule["quartz_cron_expression"].lower() == "0 0 8 ? * tue *" + assert schedule["timezone_id"] == "America/Sao_Paulo" + assert job.get("timeout_seconds") == 900 + print("Agendamento e timeout OK!") + + +def test_job_parameters_exposed(): + job = _load_job() + params = {p["name"]: p["default"] for p in job.get("parameters", [])} + print(f"Validando parâmetros expostos: {list(params.keys())}") + for expected in ("catalog_name", "user_id", "user_name"): + assert expected in params, f"Parâmetro '{expected}' não configurado" + print("Parâmetros obrigatórios OK!") + + +def test_job_uses_variable_for_performance_target(): + job = _load_job() + print("Validando uso da variável performance_target...") + assert job.get("performance_target") == "${var.performance_target}" + print("performance_target OK!") + +spark = None + diff --git a/dab_test/tests/main_test.py b/dab_test/tests/main_test.py new file mode 100644 index 0000000..38ed1b0 --- /dev/null +++ b/dab_test/tests/main_test.py @@ -0,0 +1,18 @@ +from unittest.mock import MagicMock, patch + +from dab_test import main + + +def test_find_all_taxis(): + mock_df = MagicMock() + mock_reader = MagicMock() + mock_spark = MagicMock() + + mock_spark.read = mock_reader + mock_reader.table.return_value = mock_df + + with patch.object(main, "spark", mock_spark): + taxis = main.find_all_taxis() + + mock_reader.table.assert_called_once_with("samples.nyctaxi.trips") + assert taxis == mock_df \ No newline at end of file