From b4a877080bd6d6aece3fd44c54a6cace01d82673 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 18 Nov 2026 21:07:58 +0000 Subject: [PATCH 01/12] wip: failing E2E for compiling a project that still uses @task (paused) Work in progress, paused for a higher priority. Adds two cases to the compile layout E2E test: a node module importing sqlbuild.tasks (the decorator, and the module with TaskContext) must fail compile and say to use @asset from sqlbuild.assets. Run locally against main's build: both cases fail, because compile still accepts @task. The removal itself is not started. Plan when resumed: after sqlbuild.tasks is deleted, compile already fails with D011 'No module named sqlbuild.tasks'; add the @asset guidance where ProjectDir::load imports node files (python-front imported_modules.rs), then remove the task kind from Rust, Python, UI, docs, templates and skill references. --- .../compile/test_python_project_layout.py | 63 +++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_python_project_layout.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_python_project_layout.py index d36210219a..f1757e6ce9 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_python_project_layout.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_python_project_layout.py @@ -590,3 +590,66 @@ def test_given_unrelated_python_package_on_path_when_building_then_project_pytho if __name__ == "__main__": pytest.main([__file__, "-vv"]) + + +@pytest.mark.parametrize( + "test_case", + ( + PythonProjectLayoutCompileTestCase( + description="a node module importing the removed task decorator", + repo_files={ + "sqlbuild_project.toml": 'name = "task_layout"\nadapter = "duckdb"\n', + "python/fulfillment/refresh_carriers/node.py": ( + "from sqlbuild.tasks import task\n\n\n" + "@task\n" + "def refresh_carriers() -> None:\n" + ' """Refresh carrier rates."""\n' + ), + }, + expected_exit_code=1, + expected_stderr_fragments=( + "python/fulfillment/refresh_carriers/node.py", + "`sqlbuild.tasks` was removed", + "use `@asset` from `sqlbuild.assets`", + ), + ), + PythonProjectLayoutCompileTestCase( + description="a node module importing the removed task context", + repo_files={ + "sqlbuild_project.toml": 'name = "task_layout"\nadapter = "duckdb"\n', + "python/fulfillment/refresh_carriers/node.py": ( + "from sqlbuild import tasks\n\n\n" + "@tasks.task\n" + "def refresh_carriers(context: tasks.TaskContext) -> None:\n" + ' """Refresh carrier rates."""\n' + ), + }, + expected_exit_code=1, + expected_stderr_fragments=( + "python/fulfillment/refresh_carriers/node.py", + "`sqlbuild.tasks` was removed", + "use `@asset` from `sqlbuild.assets`", + ), + ), + ), + ids=lambda case: case.description, +) +def test_given_a_node_using_the_removed_task_decorator_when_compiling_then_command_says_to_use_asset( + test_case: PythonProjectLayoutCompileTestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="task_layout", + repo_files=test_case.repo_files, + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "compile"), + project_dir=project_dir, + ) + + assert result.returncode == test_case.expected_exit_code, result.stdout + result.stderr + assert all(fragment in result.stderr for fragment in test_case.expected_stderr_fragments), ( + result.stderr + ) From e94b6a3b176c9bb2a71ed89cc5264c5fa893374c Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 19 Nov 2026 03:55:14 +0000 Subject: [PATCH 02/12] wip: remove the task node kind in favour of assets (tests in progress) Work in progress, not ready for review. Removes sqlbuild.tasks, @task, TaskContext and the task node kind from discovery, records, compile, selection, planning, rules, the UI selector list and the Python nodes playground (its upstream examples are now assets). Importing sqlbuild.tasks from a node file now fails compile with a message that points to @asset from sqlbuild.assets. Readers of data written by older versions keep accepting the task kind: the integration-result contract, published snapshots in orchestration, change labels for a previous definition snapshot, and UI display maps. Validated so far: cargo check of the workspace with all targets, the UI unit tests (625 passed), and the Python unit suite except about 60 discovery, executor and output tests whose expectations still list task and asset nodes separately. Docs are not updated yet. --- .../edge_cases/loaders/loader_results.py | 4 +- .../project/python/tasks/scoring.py | 4 +- .../loaders/round_trip/refusals/cycle_task.py | 4 +- .../refusals/terminal_loader_reader.py | 4 +- .../round_trip/refusals/unread_task.py | 4 +- .../variants/ingress_gate/rates_task.py | 4 +- .../variants/transitive_gate/packaging.py | 4 +- .../python_execution_inputs.json | 2 +- .../jobs/customer_counts_job.toml | 2 +- .../jobs/order_counts_job.toml | 2 +- .../python_pipeline_carrier_sync.py | 4 +- .../fulfillment/shipment_export/main.py | 4 +- .../fulfillment/shipment_export/node.py | 6 +- .../python/inventory/replenishment.py | 6 +- .../python/inventory/stock_count/node.py | 6 +- .../main/target_error_message.rs | 2 +- .../dag_artifact/tests/test_build_dag_json.rs | 8 +- .../src/definition_changes/constants.rs | 2 +- .../main/relation_remedy.rs | 2 +- .../src/explicit_references/models.rs | 2 +- .../sqlbuild-analysis/src/graph/constants.rs | 6 +- .../src/graph/tests/test_graph.rs | 4 +- .../graph/tests/test_named_selector_atoms.rs | 6 +- .../graph/tests/test_selector_scenarios.rs | 4 +- .../node_set_dependencies/_helpers/sets.rs | 2 +- .../src/node_set_dependencies/constants.rs | 2 +- .../node_set_dependencies/tests/helpers.rs | 14 +-- .../test_resolve_node_set_dependencies.rs | 16 +-- .../src/python_plan/main/run_lifecycle.rs | 8 +- .../src/python_plan/models.rs | 4 +- .../python_plan/tests/test_run_lifecycle.rs | 12 +- .../src/python_plan/tests/test_types.rs | 4 +- .../src/python_selection/_helpers/nodes.rs | 8 +- .../src/python_selection/_helpers/unified.rs | 8 +- .../python_selection/_helpers/validation.rs | 12 +- .../src/python_selection/constants.rs | 1 - .../src/python_selection/tests/helpers.rs | 4 +- .../tests/test_python_selection.rs | 2 +- .../tests/test_round_trips.rs | 36 +++--- .../src/review_command/tests/helpers.rs | 6 +- .../review_command/tests/test_plan_review.rs | 2 +- .../src/hosted_shapes/constants.rs | 10 +- .../sqlbuild-cli/src/adhoc/tests/helpers.rs | 4 +- .../src/adhoc/tests/test_adhoc_planner.rs | 28 ++--- .../adhoc/tests/test_selection_evaluation.rs | 5 +- .../sqlbuild-core/src/build_command/models.rs | 2 +- .../build_command/tests/test_build_command.rs | 4 +- .../tests/test_build_progress.rs | 2 +- .../src/init_command/constants.rs | 2 +- .../src/plan_display/tests/helpers.rs | 2 +- .../project_validation/_helpers/identities.rs | 7 +- .../_helpers/python_nodes.rs | 12 +- .../src/project_validation/models.rs | 4 +- .../tests/test_validate_project_inputs.rs | 8 +- .../src/entries/_helpers/user_code.rs | 5 - crates/sqlbuild-host/src/entries/models.rs | 1 - .../tests/runs/test_group_round_trips.rs | 8 +- .../tests/test_decorated_functions.rs | 8 +- .../node_units/tests/test_node_unit_module.rs | 10 +- .../_helpers/compile_records/constants.rs | 26 ---- .../_helpers/compiler_enums/constants.rs | 1 - .../_helpers/compiler_records/constants.rs | 7 -- .../records/_helpers/native_classes/ids.rs | 2 - .../sqlbuild-python-base/src/records/types.rs | 2 - .../_helpers/assembly/assembly_project.rs | 6 +- .../_helpers/setup/node_results.rs | 9 +- .../_helpers/setup/pipeline_entries.rs | 7 +- .../_helpers/setup/python_plan_entries.rs | 12 +- .../_helpers/stages/discovery.rs | 3 +- .../main/setup/python_node_count.rs | 2 +- .../_helpers/project/project_facts.rs | 4 +- .../python_files/imported_python_nodes.rs | 74 +++++++---- .../discover_python_node_functions.rs | 5 +- .../src/refactor/_helpers/python_strings.rs | 3 +- .../python_nodes/python_node_graph.rs | 11 +- .../_helpers/python_nodes/set_dependencies.rs | 3 +- .../planner_selection/_helpers/unit_tests.rs | 7 +- .../_helpers/explicit_references.rs | 8 +- .../_helpers/build_command/execution.rs | 1 - .../_helpers/build_command/lifecycle.rs | 20 ++- .../change_selection/definition_snapshot.rs | 10 +- .../src/bindings/_helpers/check/check_core.rs | 14 +-- .../location_command/snapshot_providers.rs | 3 +- .../_helpers/sql_test_command/python_tests.rs | 7 +- .../tests/test_named_selector_limit.rs | 2 +- .../src/python_rules/constants.rs | 30 ++--- .../src/python_rules/tests/helpers.rs | 4 +- .../tests/test_compile_purity_rules.rs | 10 +- .../tests/test_evaluate_python_rules.rs | 2 +- .../src/python_rules/tests/test_node_units.rs | 116 +++++++++--------- .../tests/test_python_test_rules.rs | 10 +- .../_helpers/analysis_corpus/failure_cases.py | 4 +- .../corpus/discovery_failure_cases.py | 4 +- .../_helpers/corpus/failure_cases.py | 8 +- .../_helpers/corpus/render_failure_cases.py | 4 +- .../classes/analysis_features.py | 6 +- .../classes/discovery_features.py | 4 +- .../classes/project_builder.py | 10 +- src/sqlbuild/_native.pyi | 47 ------- src/sqlbuild/_native_host.py | 4 - src/sqlbuild/_native_host.pyi | 3 - .../_helpers/diagnostics/python_resources.py | 3 +- src/sqlbuild/compiler/dag/types.py | 1 - .../_helpers/validation/discovery.py | 11 +- .../python_nodes/_helpers/user_inventory.py | 7 -- src/sqlbuild/compiler/python_nodes/types.py | 1 - .../executor/node_results/constants.py | 1 - .../python_nodes/_helpers/execution.py | 26 ++-- .../_helpers/ingress_execution.py | 22 +--- .../read_side_python_execution_tracker.py | 2 +- src/sqlbuild/executor/python_nodes/models.py | 18 --- src/sqlbuild/executor/python_nodes/types.py | 4 +- .../integrations/dagster/constants.py | 4 - src/sqlbuild/integrations/rivers/constants.py | 1 - .../python_nodes/README.md | 2 +- .../python/assets/orders_export.py.txt | 2 +- .../orders_inputs.py.txt} | 10 +- .../python_nodes/python/loaders/orders.py.txt | 6 +- src/sqlbuild/python_nodes/main/apply_task.py | 54 -------- .../python_nodes/main/read_task_definition.py | 16 --- src/sqlbuild/python_nodes/models.py | 13 -- src/sqlbuild/tasks.py | 48 -------- src/sqlbuild/tasks.pyi | 26 ---- .../commands/main/bigquery/test_bigquery.py | 4 +- .../cli/commands/main/build/helpers.py | 10 +- .../test_build_no_tests_no_audits.py | 112 ++++++++--------- .../cli/commands/main/build/test_build.py | 96 +++++++-------- .../main/build/test_dag_json_behavior.py | 6 +- .../main/build/test_explicit_references.py | 30 ++--- .../main/build/test_node_result_location.py | 4 +- .../main/build/test_python_node_identity.py | 4 +- .../main/build/test_python_node_selection.py | 22 ++-- .../main/build/test_python_source_deferral.py | 8 +- .../cli/commands/main/check/helpers.py | 6 +- .../compile/test_required_descriptions.py | 3 +- .../main/databricks/test_databricks.py | 4 +- .../cli/commands/main/init/test_init.py | 4 +- .../cli/commands/main/load/helpers.py | 4 +- .../cli/commands/main/load/test_load.py | 2 +- .../model_migrations/test_old_name_views.py | 8 +- .../commands/main/postgres/test_postgres.py | 4 +- .../main/providers/test_event_exporters.py | 4 +- .../commands/main/providers/test_providers.py | 20 +-- .../cli/commands/main/review/helpers.py | 6 +- .../cli/commands/main/review/test_review.py | 4 +- .../commands/main/snowflake/test_snowflake.py | 4 +- .../commands/main/sqlserver/test_sqlserver.py | 4 +- .../cli/commands/main/test/helpers.py | 4 +- .../commands/main/test/test_python_tests.py | 8 +- .../sqlbuild/integrations/dagster/helpers.py | 4 +- .../hard-coded-relation-in-python.json | 2 +- .../python-node-missing-description.json | 2 +- .../src/sqlbuild/cli/commands/main/helpers.py | 12 +- .../cli/commands/main/test_run_records.py | 8 +- .../src/sqlbuild/cli/recall/helpers.py | 4 +- .../compiled_project/publication/test_dev.py | 4 +- .../test_orchestration_snapshot.py | 30 +++-- .../compiler/explicit_references/helpers.py | 4 +- .../src/sqlbuild/compiler/pipeline/helpers.py | 4 +- .../sqlbuild/compiler/pipeline/test_main.py | 6 +- .../pipeline/test_python_source_reads.py | 6 +- .../pipeline/test_set_dependencies.py | 4 +- .../test_cli_concurrent_history.py | 2 +- .../_helpers/coverage/test_analysis.py | 2 +- .../_helpers/coverage/test_discovery.py | 4 +- .../main/compile/test_compile_command.py | 6 +- .../sqlbuild/cli/commands/main/dag/helpers.py | 4 +- .../cli/commands/main/dag/test_dag_command.py | 12 +- .../main/playground/test_playground.py | 10 +- .../cli/output/main/plan/test_plan_text.py | 12 +- .../output/main/plan_json/test_plan_json.py | 10 +- .../classes/test_native_progress_projector.py | 10 +- .../compiler/compile/_helpers/helpers.py | 4 +- .../compile/_helpers/test_descriptions.py | 1 - .../_helpers/test_python_sql_references.py | 14 +-- .../test_filesystem_audit_factories.py | 10 +- .../_helpers/test_filesystem_factories.py | 16 +-- .../_helpers/test_filesystem_python_nodes.py | 75 +++++------ .../_helpers/test_resource_identities.py | 4 +- .../sqlbuild/compiler/discovery/test_main.py | 88 ++++++------- .../pipeline/main/test_relation_targets.py | 2 +- .../compiler/python_nodes/_helpers/helpers.py | 76 +++--------- .../_helpers/test_identity_sharing.py | 2 +- .../python_nodes/_helpers/test_inventory.py | 25 ++-- .../python_nodes/_helpers/test_selectors.py | 2 +- .../_helpers/test_unified_selectors.py | 23 +--- .../src/sqlbuild/executor/load/helpers.py | 3 +- .../executor/python_nodes/_helpers/helpers.py | 66 +++++----- .../python_nodes/_helpers/test_contexts.py | 17 ++- .../python_nodes/_helpers/test_execution.py | 62 +++++----- .../_helpers/test_ingress_execution.py | 12 +- .../_helpers/test_python_checks.py | 8 +- .../_helpers/test_read_side_execution.py | 4 +- .../python_nodes/_helpers/test_results.py | 12 +- .../lifecycle/test_lifecycle_scheduler.py | 28 +++-- .../sqlbuild/integrations/dagster/helpers.py | 8 +- .../python_nodes/decorators/_test_types.py | 12 -- .../python_nodes/decorators/helpers.py | 4 - .../decorators/test_python_decorators.py | 72 ++--------- .../runtime/observability/test_catalog.py | 4 +- .../test_resource_attempt_lifecycle.py | 12 +- ui/src/lib/api/mock/build-fixtures.ts | 8 +- ui/src/lib/api/mock/build-sql.ts | 2 +- .../api/mock/operate-graph-selection.test.ts | 4 +- ui/src/lib/api/mock/operate-graph.ts | 6 +- ui/src/lib/api/mock/platform.ts | 2 +- ui/src/lib/api/mock/run-detail.ts | 2 +- ui/src/lib/selection/reading.ts | 1 - ui/src/lib/selection/syntax.ts | 2 +- 209 files changed, 941 insertions(+), 1451 deletions(-) rename src/sqlbuild/playground_templates/python_nodes/python/{tasks/orders.py.txt => assets/orders_inputs.py.txt} (72%) delete mode 100644 src/sqlbuild/python_nodes/main/apply_task.py delete mode 100644 src/sqlbuild/python_nodes/main/read_task_definition.py delete mode 100644 src/sqlbuild/tasks.py delete mode 100644 src/sqlbuild/tasks.pyi diff --git a/benchmarks/duckshop/edge_cases/loaders/loader_results.py b/benchmarks/duckshop/edge_cases/loaders/loader_results.py index 4c000683c4..68a53432ee 100644 --- a/benchmarks/duckshop/edge_cases/loaders/loader_results.py +++ b/benchmarks/duckshop/edge_cases/loaders/loader_results.py @@ -5,10 +5,10 @@ the target's schema for the task to find them. """ +from sqlbuild.assets import asset from sqlbuild.executor.load.models import LoaderContext from sqlbuild.loaders import loader from sqlbuild.refs import source -from sqlbuild.tasks import task @loader @@ -23,7 +23,7 @@ def raw_customer_events(ctx: LoaderContext) -> list[dict[str, object]]: return [{"event_id": 1}] -@task(depends_on=[source("raw_order_events"), source("raw_customer_events")]) +@asset(depends_on=[source("raw_order_events"), source("raw_customer_events")]) def announce_events(ctx): """Record the run each loader result came from.""" return ctx.result( diff --git a/benchmarks/duckshop/edge_cases/loaders/round_trip/project/python/tasks/scoring.py b/benchmarks/duckshop/edge_cases/loaders/round_trip/project/python/tasks/scoring.py index 7d7f16b5d4..8059560399 100644 --- a/benchmarks/duckshop/edge_cases/loaders/round_trip/project/python/tasks/scoring.py +++ b/benchmarks/duckshop/edge_cases/loaders/round_trip/project/python/tasks/scoring.py @@ -1,8 +1,8 @@ +from sqlbuild.assets import asset from sqlbuild.refs import model -from sqlbuild.tasks import task -@task(depends_on=model("orders")) +@asset(depends_on=model("orders")) def score_orders(ctx): """Send the orders to the partner and keep the scores it returns.""" relation = ctx.relation(model("orders")) diff --git a/benchmarks/duckshop/edge_cases/loaders/round_trip/refusals/cycle_task.py b/benchmarks/duckshop/edge_cases/loaders/round_trip/refusals/cycle_task.py index 788f0bb748..93f549de01 100644 --- a/benchmarks/duckshop/edge_cases/loaders/round_trip/refusals/cycle_task.py +++ b/benchmarks/duckshop/edge_cases/loaders/round_trip/refusals/cycle_task.py @@ -1,8 +1,8 @@ +from sqlbuild.assets import asset from sqlbuild.refs import model -from sqlbuild.tasks import task -@task(depends_on=model("scored_orders")) +@asset(depends_on=model("scored_orders")) def score_orders(ctx): """Send the scored orders to the partner.""" return ctx.result(payload={}) diff --git a/benchmarks/duckshop/edge_cases/loaders/round_trip/refusals/terminal_loader_reader.py b/benchmarks/duckshop/edge_cases/loaders/round_trip/refusals/terminal_loader_reader.py index c6d5cf6de1..4a7c1bdc47 100644 --- a/benchmarks/duckshop/edge_cases/loaders/round_trip/refusals/terminal_loader_reader.py +++ b/benchmarks/duckshop/edge_cases/loaders/round_trip/refusals/terminal_loader_reader.py @@ -1,9 +1,9 @@ from python.loaders.partner import partner_scores -from sqlbuild.tasks import task +from sqlbuild.assets import asset -@task(depends_on=[partner_scores]) +@asset(depends_on=[partner_scores]) def report_scores(ctx): """Report the scores the partner returned.""" return ctx.result(payload={}) diff --git a/benchmarks/duckshop/edge_cases/loaders/round_trip/refusals/unread_task.py b/benchmarks/duckshop/edge_cases/loaders/round_trip/refusals/unread_task.py index 44d7cc37db..95e55b53e2 100644 --- a/benchmarks/duckshop/edge_cases/loaders/round_trip/refusals/unread_task.py +++ b/benchmarks/duckshop/edge_cases/loaders/round_trip/refusals/unread_task.py @@ -1,7 +1,7 @@ -from sqlbuild.tasks import task +from sqlbuild.assets import asset -@task() +@asset() def score_orders(ctx): """Fetch the partner scores.""" return ctx.result(payload={}) diff --git a/benchmarks/duckshop/edge_cases/loaders/round_trip/variants/ingress_gate/rates_task.py b/benchmarks/duckshop/edge_cases/loaders/round_trip/variants/ingress_gate/rates_task.py index 9fdb868e39..4005b7a684 100644 --- a/benchmarks/duckshop/edge_cases/loaders/round_trip/variants/ingress_gate/rates_task.py +++ b/benchmarks/duckshop/edge_cases/loaders/round_trip/variants/ingress_gate/rates_task.py @@ -1,7 +1,7 @@ -from sqlbuild.tasks import task +from sqlbuild.assets import asset -@task() +@asset() def fetch_rates(ctx): """Fetch the partner's rates; reads no SQL, so it runs before the build's SQL.""" return ctx.result(payload={"rate": 10}) diff --git a/benchmarks/duckshop/edge_cases/loaders/round_trip/variants/transitive_gate/packaging.py b/benchmarks/duckshop/edge_cases/loaders/round_trip/variants/transitive_gate/packaging.py index a1fe66883c..7c3fecb226 100644 --- a/benchmarks/duckshop/edge_cases/loaders/round_trip/variants/transitive_gate/packaging.py +++ b/benchmarks/duckshop/edge_cases/loaders/round_trip/variants/transitive_gate/packaging.py @@ -1,9 +1,9 @@ from python.tasks.scoring import score_orders -from sqlbuild.tasks import task +from sqlbuild.assets import asset -@task(depends_on=[score_orders]) +@asset(depends_on=[score_orders]) def package_scores(ctx): """Package the partner's scores; skips softly when there is nothing new to send.""" return ctx.skip(reason="nothing to package") diff --git a/benchmarks/duckshop/edge_cases/orchestration/python_execution_inputs.json b/benchmarks/duckshop/edge_cases/orchestration/python_execution_inputs.json index 87f831d781..6fc11f3d0f 100644 --- a/benchmarks/duckshop/edge_cases/orchestration/python_execution_inputs.json +++ b/benchmarks/duckshop/edge_cases/orchestration/python_execution_inputs.json @@ -1,6 +1,6 @@ { "initial": "from sqlbuild.checks import check\nfrom sqlbuild.refs import model\n@check(depends_on=model('orders'), description='Orders are present.')\ndef orders_present(ctx):\n return ctx.pass_()\n", "changed": "from sqlbuild.checks import check\nfrom sqlbuild.refs import model\n@check(depends_on=model('orders'), description='Orders are present.')\ndef orders_present(ctx):\n raise RuntimeError('changed check executed after publication')\n", - "task": "from sqlbuild.tasks import task\n@task(description='Notify support.')\ndef notify_support(ctx):\n return None\n", + "task": "from sqlbuild.assets import asset\n@asset(description='Notify support.')\ndef notify_support(ctx):\n return None\n", "asset": "from sqlbuild.assets import asset\n@asset(description='Order export.')\ndef order_export(ctx):\n return []\n" } diff --git a/benchmarks/duckshop/edge_cases/orchestration/python_node_jobs/jobs/customer_counts_job.toml b/benchmarks/duckshop/edge_cases/orchestration/python_node_jobs/jobs/customer_counts_job.toml index e552003d67..85d26ffc60 100644 --- a/benchmarks/duckshop/edge_cases/orchestration/python_node_jobs/jobs/customer_counts_job.toml +++ b/benchmarks/duckshop/edge_cases/orchestration/python_node_jobs/jobs/customer_counts_job.toml @@ -1,4 +1,4 @@ # Jobs whose selections are Python nodes, which `--select job:` must select as published. name = "customer_counts_job" description = "Count customers." -select = "task:count_customers asset:customers_extract" +select = "asset:count_customers asset:customers_extract" diff --git a/benchmarks/duckshop/edge_cases/orchestration/python_node_jobs/jobs/order_counts_job.toml b/benchmarks/duckshop/edge_cases/orchestration/python_node_jobs/jobs/order_counts_job.toml index 3a0104dc8e..1f0c7e1c2b 100644 --- a/benchmarks/duckshop/edge_cases/orchestration/python_node_jobs/jobs/order_counts_job.toml +++ b/benchmarks/duckshop/edge_cases/orchestration/python_node_jobs/jobs/order_counts_job.toml @@ -1,3 +1,3 @@ name = "order_counts_job" description = "Count orders." -select = "+task:count_orders" +select = "+asset:count_orders" diff --git a/benchmarks/duckshop/edge_cases/orchestration/python_pipeline_carrier_sync.py b/benchmarks/duckshop/edge_cases/orchestration/python_pipeline_carrier_sync.py index 221447d1c8..66b18447ad 100644 --- a/benchmarks/duckshop/edge_cases/orchestration/python_pipeline_carrier_sync.py +++ b/benchmarks/duckshop/edge_cases/orchestration/python_pipeline_carrier_sync.py @@ -2,9 +2,9 @@ from python.orders.partner_export.node import partner_orders_file -from sqlbuild.tasks import task +from sqlbuild.assets import asset -@task(depends_on=partner_orders_file, tags=["orders"], description="Book collection.") +@asset(depends_on=partner_orders_file, tags=["orders"], description="Book collection.") def request_carrier_collection(ctx): return None diff --git a/benchmarks/duckshop/edge_cases/python_rules/conforming/python/fulfillment/shipment_export/main.py b/benchmarks/duckshop/edge_cases/python_rules/conforming/python/fulfillment/shipment_export/main.py index fb001a0a9b..7664c575fa 100644 --- a/benchmarks/duckshop/edge_cases/python_rules/conforming/python/fulfillment/shipment_export/main.py +++ b/benchmarks/duckshop/edge_cases/python_rules/conforming/python/fulfillment/shipment_export/main.py @@ -1,11 +1,11 @@ """Write each shipped order as one carrier line.""" from python.utils.currency import amounts +from sqlbuild.assets import AssetContext from sqlbuild.executor.python_nodes.models import PythonNodeResult -from sqlbuild.tasks import TaskContext -def main(ctx: TaskContext) -> PythonNodeResult: +def main(ctx: AssetContext) -> PythonNodeResult: """Write each shipped order as one carrier line.""" label: str = "Expédition" return ctx.result(payload=f"{label} {amounts.cents_text(1250)}") diff --git a/benchmarks/duckshop/edge_cases/python_rules/conforming/python/fulfillment/shipment_export/node.py b/benchmarks/duckshop/edge_cases/python_rules/conforming/python/fulfillment/shipment_export/node.py index 60fe8ac5cf..bf18e586e2 100644 --- a/benchmarks/duckshop/edge_cases/python_rules/conforming/python/fulfillment/shipment_export/node.py +++ b/benchmarks/duckshop/edge_cases/python_rules/conforming/python/fulfillment/shipment_export/node.py @@ -1,10 +1,10 @@ """Export shipped orders to the carrier.""" from python.fulfillment.shipment_export.main import main +from sqlbuild.assets import AssetContext, asset from sqlbuild.executor.python_nodes.models import PythonNodeResult -from sqlbuild.tasks import TaskContext, task -@task(description="Export shipped orders to the carrier.") -def shipment_export(ctx: TaskContext) -> PythonNodeResult: +@asset(description="Export shipped orders to the carrier.") +def shipment_export(ctx: AssetContext) -> PythonNodeResult: return main(ctx=ctx) diff --git a/benchmarks/duckshop/edge_cases/python_rules/violations/python/inventory/replenishment.py b/benchmarks/duckshop/edge_cases/python_rules/violations/python/inventory/replenishment.py index 7accddcff3..38237955b3 100644 --- a/benchmarks/duckshop/edge_cases/python_rules/violations/python/inventory/replenishment.py +++ b/benchmarks/duckshop/edge_cases/python_rules/violations/python/inventory/replenishment.py @@ -1,8 +1,8 @@ """Replenishment declared in a flat module.""" -from sqlbuild.tasks import TaskContext, task +from sqlbuild.assets import AssetContext, asset -@task(description="Reorder products that ran out.") -def replenish(ctx: TaskContext) -> None: +@asset(description="Reorder products that ran out.") +def replenish(ctx: AssetContext) -> None: return None diff --git a/benchmarks/duckshop/edge_cases/python_rules/violations/python/inventory/stock_count/node.py b/benchmarks/duckshop/edge_cases/python_rules/violations/python/inventory/stock_count/node.py index 4d0159b672..19cb7f2be2 100644 --- a/benchmarks/duckshop/edge_cases/python_rules/violations/python/inventory/stock_count/node.py +++ b/benchmarks/duckshop/edge_cases/python_rules/violations/python/inventory/stock_count/node.py @@ -2,12 +2,12 @@ from orchestration.sensors.fulfillment._helpers.hours import CUTOFF_HOURS -from sqlbuild.tasks import TaskContext, task +from sqlbuild.assets import AssetContext, asset WAREHOUSES: tuple[str, ...] = ("north", "south") -@task -def stock_count(context: TaskContext) -> int: +@asset +def stock_count(context: AssetContext) -> int: """Count stock in every warehouse.""" return len(WAREHOUSES) + CUTOFF_HOURS diff --git a/crates/sqlbuild-analysis/src/assembly/target_checks/main/target_error_message.rs b/crates/sqlbuild-analysis/src/assembly/target_checks/main/target_error_message.rs index 7cfb39622b..e771052d18 100644 --- a/crates/sqlbuild-analysis/src/assembly/target_checks/main/target_error_message.rs +++ b/crates/sqlbuild-analysis/src/assembly/target_checks/main/target_error_message.rs @@ -50,7 +50,7 @@ pub fn target_error_message(error: &TargetError) -> String { ), TargetError::PreservedNodeResults(target) => format!( "Target '{target}' keeps each model's own schema, so it has no schema for the \ - results of loaders, tasks, assets and checks. Set targets.{target}.loader_schema; \ + results of loaders, assets and checks. Set targets.{target}.loader_schema; \ Python node results are kept there." ), } diff --git a/crates/sqlbuild-analysis/src/dag_artifact/tests/test_build_dag_json.rs b/crates/sqlbuild-analysis/src/dag_artifact/tests/test_build_dag_json.rs index 4970e8e653..038e568ff4 100644 --- a/crates/sqlbuild-analysis/src/dag_artifact/tests/test_build_dag_json.rs +++ b/crates/sqlbuild-analysis/src/dag_artifact/tests/test_build_dag_json.rs @@ -190,18 +190,18 @@ fn given_a_loader_depending_on_a_task_when_building_dag_json_then_the_edge_reach }, python: DagPythonGraphInput { nodes: vec![ - python_node("task", "score_orders", vec![key("model", "orders")]), + python_node("asset", "score_orders", vec![key("model", "orders")]), python_node("loader", "partner_scores", Vec::new()), ], dependency_edges: vec![("score_orders".to_owned(), "partner_scores".to_owned())], kinds_by_name: vec![ - ("score_orders".to_owned(), "task".to_owned()), + ("score_orders".to_owned(), "asset".to_owned()), ("partner_scores".to_owned(), "loader".to_owned()), ], }, expected_edges: &[ - ("model:orders", "task:score_orders"), - ("task:score_orders", "source:partner_scores"), + ("model:orders", "asset:score_orders"), + ("asset:score_orders", "source:partner_scores"), ], }]; for test_case in test_cases { diff --git a/crates/sqlbuild-analysis/src/definition_changes/constants.rs b/crates/sqlbuild-analysis/src/definition_changes/constants.rs index f17cdf9140..9c84dbf60b 100644 --- a/crates/sqlbuild-analysis/src/definition_changes/constants.rs +++ b/crates/sqlbuild-analysis/src/definition_changes/constants.rs @@ -22,7 +22,7 @@ pub const DEPENDENCIES_KIND: &str = "dependencies"; /// The one node of each project-wide kind. pub const PROJECT_NODE_NAME: &str = "project"; /// Python nodes by decorator. -pub const PYTHON_NODE_KINDS: [&str; 4] = ["loader", "task", "asset", "check"]; +pub const PYTHON_NODE_KINDS: [&str; 3] = ["loader", "asset", "check"]; /// Normalised SQL of a model, a function body or a scenario. pub const SQL_ASPECT: &str = "sql"; diff --git a/crates/sqlbuild-analysis/src/explicit_references/main/relation_remedy.rs b/crates/sqlbuild-analysis/src/explicit_references/main/relation_remedy.rs index 43db711343..21b57459d2 100644 --- a/crates/sqlbuild-analysis/src/explicit_references/main/relation_remedy.rs +++ b/crates/sqlbuild-analysis/src/explicit_references/main/relation_remedy.rs @@ -15,7 +15,7 @@ pub fn relation_remedy( if kind != SOURCE_TYPE { return format!( "loaders run before models and seeds and cannot read them; read \ - {kind}:{name} from a task or asset instead" + {kind}:{name} from an asset instead" ); } return match upstream_loaders.iter().find(|(source, _)| source == name) { diff --git a/crates/sqlbuild-analysis/src/explicit_references/models.rs b/crates/sqlbuild-analysis/src/explicit_references/models.rs index 21021e13a4..eee2732608 100644 --- a/crates/sqlbuild-analysis/src/explicit_references/models.rs +++ b/crates/sqlbuild-analysis/src/explicit_references/models.rs @@ -22,7 +22,7 @@ pub struct ReferenceRequest { pub audits: Vec, /// `project.hook_functions` in order. pub hooks: Vec, - /// Python SQL owners: tasks, assets, checks, loaders, then hooks. + /// Python SQL owners: assets, checks, loaders, then hooks. pub owners: Vec, } diff --git a/crates/sqlbuild-analysis/src/graph/constants.rs b/crates/sqlbuild-analysis/src/graph/constants.rs index 1540090e3f..d1a004f835 100644 --- a/crates/sqlbuild-analysis/src/graph/constants.rs +++ b/crates/sqlbuild-analysis/src/graph/constants.rs @@ -21,13 +21,13 @@ pub(crate) const TEST_KIND: &str = "test"; pub(crate) const JOB_KIND: &str = "job"; pub(crate) const NODE_KIND_KIND: &str = "kind"; /// The node kinds `kind:` selects, as the catalog names them: SQL resources, then Python nodes. -pub(crate) const SELECTABLE_NODE_KINDS: [&str; 10] = [ - "model", "source", "seed", "udf", "table_fn", "function", "task", "asset", "loader", "check", +pub(crate) const SELECTABLE_NODE_KINDS: [&str; 9] = [ + "model", "source", "seed", "udf", "table_fn", "function", "asset", "loader", "check", ]; /// `kind:function` selects both function kinds. pub(crate) const FUNCTION_KIND_ALIAS: &str = "function"; /// Selector kinds Python parses but `sqb compile` cannot map to a compiled resource. -pub(crate) const UNMAPPED_KINDS: [&str; 4] = ["task", "asset", "loader", "check"]; +pub(crate) const UNMAPPED_KINDS: [&str; 3] = ["asset", "loader", "check"]; pub(crate) const SUGGESTION_CUTOFF: f64 = 0.8; pub(crate) const SUGGESTION_LIMIT: usize = 3; pub(crate) const EMPTY_SELECTOR_CODE: &str = "S001"; diff --git a/crates/sqlbuild-analysis/src/graph/tests/test_graph.rs b/crates/sqlbuild-analysis/src/graph/tests/test_graph.rs index 0d4a684eb9..ae9b836ace 100644 --- a/crates/sqlbuild-analysis/src/graph/tests/test_graph.rs +++ b/crates/sqlbuild-analysis/src/graph/tests/test_graph.rs @@ -272,8 +272,8 @@ fn given_invalid_selectors_when_resolving_then_errors_match_python() { ), }, SelectorErrorTestCase { - description: "task:abc", - select: &["task:abc"], + description: "asset:abc", + select: &["asset:abc"], exclude: &[], expected_error: ( "S010", diff --git a/crates/sqlbuild-analysis/src/graph/tests/test_named_selector_atoms.rs b/crates/sqlbuild-analysis/src/graph/tests/test_named_selector_atoms.rs index 20c0594130..8210a92870 100644 --- a/crates/sqlbuild-analysis/src/graph/tests/test_named_selector_atoms.rs +++ b/crates/sqlbuild-analysis/src/graph/tests/test_named_selector_atoms.rs @@ -16,9 +16,9 @@ fn given_a_selection_when_listing_named_atoms_then_only_single_node_names_count( }, NamedAtomsTestCase { description: "typed names count", - selection: "task:refund_review source:raw_orders seed:countries test:orders_unique", + selection: "asset:refund_review source:raw_orders seed:countries test:orders_unique", expected_atoms: &[ - "task:refund_review", + "asset:refund_review", "source:raw_orders", "seed:countries", "test:orders_unique", @@ -46,7 +46,7 @@ fn given_a_selection_when_listing_named_atoms_then_only_single_node_names_count( }, NamedAtomsTestCase { description: "globs do not count", - selection: "fct_orders_* task:refund_? fct_[ab]", + selection: "fct_orders_* asset:refund_? fct_[ab]", expected_atoms: &[], }, NamedAtomsTestCase { diff --git a/crates/sqlbuild-analysis/src/graph/tests/test_selector_scenarios.rs b/crates/sqlbuild-analysis/src/graph/tests/test_selector_scenarios.rs index f618813937..443fae9f60 100644 --- a/crates/sqlbuild-analysis/src/graph/tests/test_selector_scenarios.rs +++ b/crates/sqlbuild-analysis/src/graph/tests/test_selector_scenarios.rs @@ -58,8 +58,8 @@ fn given_raw_selector_when_parsing_then_returns_kind_value_and_expansion() { }, ParseSelectorTestCase { description: "task typed selector", - raw: "task:prepare_orders", - expected_selector: kind("task", "prepare_orders", false, false), + raw: "asset:prepare_orders", + expected_selector: kind("asset", "prepare_orders", false, false), }, ParseSelectorTestCase { description: "asset typed selector", diff --git a/crates/sqlbuild-analysis/src/node_set_dependencies/_helpers/sets.rs b/crates/sqlbuild-analysis/src/node_set_dependencies/_helpers/sets.rs index 8399a32435..ec3b2317b7 100644 --- a/crates/sqlbuild-analysis/src/node_set_dependencies/_helpers/sets.rs +++ b/crates/sqlbuild-analysis/src/node_set_dependencies/_helpers/sets.rs @@ -46,7 +46,7 @@ pub(crate) fn members( if dependant.kind == LOADER_KIND { return Err(failure( format!("{owner} depends on {set}; loaders cannot depend on a job or a tag"), - Some("name the loaders, tasks or assets it depends on one by one".to_owned()), + Some("name the loaders or assets it depends on one by one".to_owned()), )); } if dependency.name.contains(LOCATION_SEPARATOR) { diff --git a/crates/sqlbuild-analysis/src/node_set_dependencies/constants.rs b/crates/sqlbuild-analysis/src/node_set_dependencies/constants.rs index 363915abc0..f0465ff2dd 100644 --- a/crates/sqlbuild-analysis/src/node_set_dependencies/constants.rs +++ b/crates/sqlbuild-analysis/src/node_set_dependencies/constants.rs @@ -9,7 +9,7 @@ pub(crate) const NODE_SET_CODE: &str = "S000"; /// SQL resource types a Python node may depend on. pub(crate) const DEPENDABLE_SQL_KINDS: [&str; 3] = ["model", "seed", "source"]; /// Python node kinds a Python node may depend on. -pub(crate) const DEPENDABLE_PYTHON_KINDS: [&str; 3] = ["loader", "task", "asset"]; +pub(crate) const DEPENDABLE_PYTHON_KINDS: [&str; 2] = ["loader", "asset"]; /// The Python node kind that may depend on Python nodes only. pub(crate) const LOADER_KIND: &str = "loader"; /// The SQL resource type a loader fills. diff --git a/crates/sqlbuild-analysis/src/node_set_dependencies/tests/helpers.rs b/crates/sqlbuild-analysis/src/node_set_dependencies/tests/helpers.rs index 716ac53e18..ad7f2b4b3a 100644 --- a/crates/sqlbuild-analysis/src/node_set_dependencies/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/node_set_dependencies/tests/helpers.rs @@ -25,16 +25,16 @@ type NodeSpec = ( pub(super) type SetSpec = (&'static str, &'static str, Option<&'static str>); const NODES: &[NodeSpec] = &[ - ("report", "task", &["reports"], &[], &[]), - ("export_orders", "task", &[], &[], &[("model", "orders")]), + ("report", "asset", &["reports"], &[], &[]), + ("export_orders", "asset", &[], &[], &[("model", "orders")]), ( "audit_export", - "task", + "asset", &[], &["export_orders", "report"], &[], ), - ("self_tagged", "task", &["finance"], &[], &[]), + ("self_tagged", "asset", &["finance"], &[], &[]), ("sync_events", "loader", &[], &[], &[]), ]; @@ -64,10 +64,10 @@ const JOBS: &[(&str, &str)] = &[ ("nightly", "stg_orders orders order_items"), ("single", "customers"), ("diamond", "stg_orders orders order_items finance_daily"), - ("with_task", "orders task:export_orders"), - ("loop", "task:export_orders task:audit_export"), + ("with_task", "orders asset:export_orders"), + ("loop", "asset:export_orders asset:audit_export"), ("nightly_wide", "orders customer_orders"), - ("reporting", "task:report customers"), + ("reporting", "asset:report customers"), ]; fn key(spec: &str) -> ObjectKey { diff --git a/crates/sqlbuild-analysis/src/node_set_dependencies/tests/test_resolve_node_set_dependencies.rs b/crates/sqlbuild-analysis/src/node_set_dependencies/tests/test_resolve_node_set_dependencies.rs index 9c3fb98dec..49153ced62 100644 --- a/crates/sqlbuild-analysis/src/node_set_dependencies/tests/test_resolve_node_set_dependencies.rs +++ b/crates/sqlbuild-analysis/src/node_set_dependencies/tests/test_resolve_node_set_dependencies.rs @@ -69,7 +69,7 @@ fn given_job_and_tag_dependencies_when_resolving_then_each_reduces_to_its_end_no }, ]; for test_case in test_cases { - let resolved = resolve(("report", "task"), test_case.sets, test_case.groups) + let resolved = resolve(("report", "asset"), test_case.sets, test_case.groups) .expect(test_case.description); assert_eq!( ends(&resolved), @@ -85,7 +85,7 @@ fn given_unresolvable_dependencies_when_resolving_then_compile_fails_naming_the_ let test_cases = [ FailureTestCase { description: "an unknown job", - dependant: ("report", "task"), + dependant: ("report", "asset"), groups: &[], sets: &[("job", "weekly", None)], expected_fragments: &[ @@ -94,7 +94,7 @@ fn given_unresolvable_dependencies_when_resolving_then_compile_fails_naming_the_ }, FailureTestCase { description: "an unknown tag", - dependant: ("report", "task"), + dependant: ("report", "asset"), groups: &[], sets: &[("tag", "marketing", None)], expected_fragments: &[ @@ -103,7 +103,7 @@ fn given_unresolvable_dependencies_when_resolving_then_compile_fails_naming_the_ }, FailureTestCase { description: "a set its exclude empties", - dependant: ("report", "task"), + dependant: ("report", "asset"), groups: &[], sets: &[("job", "single", Some("customers"))], expected_fragments: &[ @@ -113,7 +113,7 @@ fn given_unresolvable_dependencies_when_resolving_then_compile_fails_naming_the_ }, FailureTestCase { description: "another location's job", - dependant: ("report", "task"), + dependant: ("report", "asset"), groups: &[], sets: &[("job", "billing:nightly", None)], expected_fragments: &["a job of another code location"], @@ -129,7 +129,7 @@ fn given_unresolvable_dependencies_when_resolving_then_compile_fails_naming_the_ }, FailureTestCase { description: "a job node reading the dependant's output closes a cycle", - dependant: ("report", "task"), + dependant: ("report", "asset"), groups: &[], sets: &[("job", "loop", None)], expected_fragments: &[ @@ -139,7 +139,7 @@ fn given_unresolvable_dependencies_when_resolving_then_compile_fails_naming_the_ }, FailureTestCase { description: "a tag the dependant carries includes it", - dependant: ("self_tagged", "task"), + dependant: ("self_tagged", "asset"), groups: &[], sets: &[("tag", "finance", None)], expected_fragments: &[ @@ -149,7 +149,7 @@ fn given_unresolvable_dependencies_when_resolving_then_compile_fails_naming_the_ }, FailureTestCase { description: "a group whose jobs would wait for each other", - dependant: ("report", "task"), + dependant: ("report", "asset"), groups: GROUP, sets: &[("job", "nightly_wide", None)], expected_fragments: &[ diff --git a/crates/sqlbuild-analysis/src/python_plan/main/run_lifecycle.rs b/crates/sqlbuild-analysis/src/python_plan/main/run_lifecycle.rs index 0c66855de1..b5e843ef14 100644 --- a/crates/sqlbuild-analysis/src/python_plan/main/run_lifecycle.rs +++ b/crates/sqlbuild-analysis/src/python_plan/main/run_lifecycle.rs @@ -6,7 +6,7 @@ use crate::assembly::project::types::ObjectKey; use crate::python_plan::_helpers::lifecycle::gate_frontier::gate_frontier; use crate::python_plan::main::upstream_closure::upstream_closure; use crate::python_plan::models::{LifecycleNode, RoundTripLoader, RunLifecycle}; -use crate::python_selection::constants::{ASSET_KIND, LOADER_KIND, TASK_KIND}; +use crate::python_selection::constants::{ASSET_KIND, LOADER_KIND}; /// Ingress is each selected loader whose selected upstream reads no SQL, with that upstream; the rest load after the SQL. #[must_use] @@ -16,7 +16,7 @@ pub fn run_lifecycle( upstream: &HashMap>, ) -> RunLifecycle { let kind = |name: &str| nodes.get(name).map_or("", |node| node.kind.as_str()); - let runs = |name: &str| [TASK_KIND, ASSET_KIND].contains(&kind(name)); + let runs = |name: &str| kind(name) == ASSET_KIND; let mut ingress_loaders: BTreeSet = BTreeSet::new(); let mut round_trip_loaders: Vec = Vec::new(); for loader in selected.iter().filter(|name| kind(name) == LOADER_KIND) { @@ -45,9 +45,7 @@ pub fn run_lifecycle( let mut ingress: BTreeSet = ingress_loaders.clone(); for loader in &ingress_loaders { for name in upstream_closure(loader, upstream) { - if selected.contains(&name) - && [TASK_KIND, ASSET_KIND, LOADER_KIND].contains(&kind(&name)) - { + if selected.contains(&name) && [ASSET_KIND, LOADER_KIND].contains(&kind(&name)) { ingress.insert(name); } } diff --git a/crates/sqlbuild-analysis/src/python_plan/models.rs b/crates/sqlbuild-analysis/src/python_plan/models.rs index 5c60f8844a..eaa2de12be 100644 --- a/crates/sqlbuild-analysis/src/python_plan/models.rs +++ b/crates/sqlbuild-analysis/src/python_plan/models.rs @@ -15,9 +15,9 @@ pub struct LifecycleNode { #[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct RoundTripLoader { pub name: String, - /// The SQL its selected upstream tasks and assets read, which its load waits for; sorted. + /// The SQL its selected upstream assets read, which its load waits for; sorted. pub waits_for: Vec, - /// Its selected upstream tasks and assets, none of which may fail or hard-skip for it to load; sorted. + /// Its selected upstream assets, none of which may fail or hard-skip for it to load; sorted. pub gates: Vec, /// The gates it reads directly or through other loaders, one of which must succeed for it to load; sorted. pub frontier: Vec, diff --git a/crates/sqlbuild-analysis/src/python_plan/tests/test_run_lifecycle.rs b/crates/sqlbuild-analysis/src/python_plan/tests/test_run_lifecycle.rs index b52018a631..12dc8006d4 100644 --- a/crates/sqlbuild-analysis/src/python_plan/tests/test_run_lifecycle.rs +++ b/crates/sqlbuild-analysis/src/python_plan/tests/test_run_lifecycle.rs @@ -12,7 +12,7 @@ fn given_selection_when_classifying_lifecycle_then_matches_python_regions() { description: "a loader pulls its selected upstream tasks into ingress", selected: &["fetch_rates", "load_orders", "report", "check_orders"], nodes: &[ - ("fetch_rates", "task", &[]), + ("fetch_rates", "asset", &[]), ("load_orders", "loader", &[]), ("report", "asset", &[("model", "orders")]), ("check_orders", "check", &[]), @@ -40,7 +40,7 @@ fn given_selection_when_classifying_lifecycle_then_matches_python_regions() { description: "a loader whose selected upstream reads SQL loads after it, gated by it", selected: &["score_orders", "package_scores", "partner_scores"], nodes: &[ - ("score_orders", "task", &[("model", "orders")]), + ("score_orders", "asset", &[("model", "orders")]), ("package_scores", "asset", &[]), ("partner_scores", "loader", &[]), ], @@ -59,7 +59,7 @@ fn given_selection_when_classifying_lifecycle_then_matches_python_regions() { )], }, LifecycleTestCase { - description: "only the tasks and assets a loader reads directly can let it run", + description: "only the assets a loader reads directly can let it run", selected: &[ "fetch_rates", "score_orders", @@ -67,8 +67,8 @@ fn given_selection_when_classifying_lifecycle_then_matches_python_regions() { "partner_scores", ], nodes: &[ - ("fetch_rates", "task", &[]), - ("score_orders", "task", &[("model", "orders")]), + ("fetch_rates", "asset", &[]), + ("score_orders", "asset", &[("model", "orders")]), ("package_scores", "asset", &[]), ("partner_scores", "loader", &[]), ], @@ -90,7 +90,7 @@ fn given_selection_when_classifying_lifecycle_then_matches_python_regions() { description: "a loader selected without its SQL-reading upstream is ordinary ingress", selected: &["partner_scores"], nodes: &[ - ("score_orders", "task", &[("model", "orders")]), + ("score_orders", "asset", &[("model", "orders")]), ("partner_scores", "loader", &[]), ], upstream: &[("partner_scores", &["score_orders"])], diff --git a/crates/sqlbuild-analysis/src/python_plan/tests/test_types.rs b/crates/sqlbuild-analysis/src/python_plan/tests/test_types.rs index d23e8c2540..49492b46bf 100644 --- a/crates/sqlbuild-analysis/src/python_plan/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/python_plan/tests/test_types.rs @@ -20,7 +20,7 @@ pub(crate) struct LifecycleTestCase { pub(crate) expected_ingress: &'static [&'static str], pub(crate) expected_loaders: &'static [&'static str], pub(crate) expected_read_side: &'static [&'static str], - /// `(loader, SQL it waits for, tasks and assets that gate it, the gates it reads directly)`. + /// `(loader, SQL it waits for, assets that gate it, the gates it reads directly)`. pub(crate) expected_round_trips: &'static [ExpectedRoundTrip], } @@ -31,7 +31,7 @@ pub(crate) type LifecycleNodeSpec = ( &'static [(&'static str, &'static str)], ); -/// `(loader, SQL it waits for, tasks and assets that gate it)`. +/// `(loader, SQL it waits for, assets that gate it)`. pub(crate) type ExpectedRoundTrip = ( &'static str, &'static [(&'static str, &'static str)], diff --git a/crates/sqlbuild-analysis/src/python_selection/_helpers/nodes.rs b/crates/sqlbuild-analysis/src/python_selection/_helpers/nodes.rs index 7e9fe9a25d..9cc6c33175 100644 --- a/crates/sqlbuild-analysis/src/python_selection/_helpers/nodes.rs +++ b/crates/sqlbuild-analysis/src/python_selection/_helpers/nodes.rs @@ -13,7 +13,7 @@ use crate::graph::models::{EdgeDirection, ParsedSelector, ProjectGraph}; use crate::python_selection::constants::{ ASSET_KIND, CHECK_KIND, INTERSECTION_SEPARATOR, JOB_KIND, LOADER_KIND, NAME_KIND, NODE_KIND_KIND, PATH_BETWEEN_CODE, PATH_KIND, PATH_ROOT_CODE, PATH_ROOT_ERROR, - PATTERN_CHARACTERS, PLANNER_DEFAULT_CODE, SQL_MODEL_PATH_ROOT, TAG_KIND, TASK_KIND, TEST_KIND, + PATTERN_CHARACTERS, PLANNER_DEFAULT_CODE, SQL_MODEL_PATH_ROOT, TAG_KIND, TEST_KIND, UNIT_TEST_CODE, UNIT_TEST_ONLY_TEST_AND_BUILD, UNKNOWN_NAME_CODE, UNKNOWN_PATH_CODE, UNKNOWN_TAG_CODE, UNMAPPED_KIND_CODE, }; @@ -214,9 +214,9 @@ fn tagged(index: &NodeIndex<'_>, tag: &str) -> Resolved { Ok(names) } -/// The Python nodes of one kind, such as every `task`. +/// The Python nodes of one kind, such as every `asset`. fn of_kind(index: &NodeIndex<'_>, kind: &str) -> Resolved { - if ![TASK_KIND, ASSET_KIND, LOADER_KIND, CHECK_KIND].contains(&kind) { + if ![ASSET_KIND, LOADER_KIND, CHECK_KIND].contains(&kind) { return Err(error( UNMAPPED_KIND_CODE, format!("node kind '{kind}' is not a Python node kind"), @@ -278,7 +278,7 @@ fn lookup(index: &NodeIndex<'_>, kind: &str, value: &str) -> Resolved { .map(|(_, node)| node.name.clone()) .collect()); } - if ![TASK_KIND, ASSET_KIND, LOADER_KIND, CHECK_KIND].contains(&kind) { + if ![ASSET_KIND, LOADER_KIND, CHECK_KIND].contains(&kind) { return Err(error( UNMAPPED_KIND_CODE, format!("selector type '{kind}' does not map to a Python node type"), diff --git a/crates/sqlbuild-analysis/src/python_selection/_helpers/unified.rs b/crates/sqlbuild-analysis/src/python_selection/_helpers/unified.rs index c8c7a6c2a4..a7e67676ae 100644 --- a/crates/sqlbuild-analysis/src/python_selection/_helpers/unified.rs +++ b/crates/sqlbuild-analysis/src/python_selection/_helpers/unified.rs @@ -23,7 +23,7 @@ use crate::python_selection::_helpers::validation::{ use crate::python_selection::constants::{ ASSET_KIND, CHECK_KIND, EXPANSION_MARKER, INTERSECTION_SEPARATOR, JOB_KIND, LOADER_KIND, NAME_KIND, NODE_KIND_KIND, PATH_KIND, PATH_ROOT_CODE, PATH_ROOT_ERROR, PLANNER_DEFAULT_CODE, - SEED_KIND, SOURCE_KIND, SOURCE_RESOURCE, SQL_MODEL_PATH_ROOT, TAG_KIND, TASK_KIND, TEST_KIND, + SEED_KIND, SOURCE_KIND, SOURCE_RESOURCE, SQL_MODEL_PATH_ROOT, TAG_KIND, TEST_KIND, UNKNOWN_NAME_CODE, UNKNOWN_TAG_CODE, }; use crate::python_selection::models::{Atom, PythonSelectionNode, RunSelection, SelectionContext}; @@ -344,13 +344,11 @@ fn single_side(context: &SelectionContext<'_>, raw: &str) -> Resolved { PATH_KIND => resolve_path(context, raw, &value)?, SEED_KIND | SOURCE_KIND => resolve_sql(context, raw)?, JOB_KIND => resolve_job(context, raw)?, - TASK_KIND | ASSET_KIND | LOADER_KIND | CHECK_KIND => { + ASSET_KIND | LOADER_KIND | CHECK_KIND => { return resolve_typed(context, raw, (&kind, &value)); } TAG_KIND => resolve_tag(context, raw, &value)?, - NODE_KIND_KIND - if [TASK_KIND, ASSET_KIND, LOADER_KIND, CHECK_KIND].contains(&value.as_str()) => - { + NODE_KIND_KIND if [ASSET_KIND, LOADER_KIND, CHECK_KIND].contains(&value.as_str()) => { resolve_python(context, raw)? } NODE_KIND_KIND => resolve_sql(context, raw)?, diff --git a/crates/sqlbuild-analysis/src/python_selection/_helpers/validation.rs b/crates/sqlbuild-analysis/src/python_selection/_helpers/validation.rs index a87a6a252a..15b2f7373f 100644 --- a/crates/sqlbuild-analysis/src/python_selection/_helpers/validation.rs +++ b/crates/sqlbuild-analysis/src/python_selection/_helpers/validation.rs @@ -11,7 +11,7 @@ use crate::graph::models::EdgeDirection; use crate::python_selection::_helpers::nodes::error; use crate::python_selection::constants::{ ASSET_KIND, CHECK_KIND, LOADER_KIND, MODEL_RESOURCE, PLANNER_DEFAULT_CODE, SOURCE_RESOURCE, - SQL_MODEL_DEPENDENCY_TYPES, TASK_KIND, + SQL_MODEL_DEPENDENCY_TYPES, }; use crate::python_selection::models::{Atom, SelectionContext, SqlSelectionGraph}; @@ -68,7 +68,7 @@ pub(crate) fn validate_boundaries(context: &SelectionContext<'_>) -> Result<(), node.name ))); } - if node.kind == TASK_KIND || node.kind == ASSET_KIND { + if node.kind == ASSET_KIND { return Err(failure(format!( "Python node '{}' depends on terminal loader '{dependency}'; \ depend on source '{source}' instead", @@ -142,7 +142,7 @@ fn dependency_cycle(context: &SelectionContext<'_>) -> Option { cycle.join(" -> ") ), help: Some( - "each arrow points to what a node depends on; a task or asset upstream of a \ + "each arrow points to what a node depends on; an asset upstream of a \ loader cannot read anything built from the source that loader fills" .to_owned(), ), @@ -177,7 +177,7 @@ fn validate_loader_upstreams(context: &SelectionContext<'_>) -> Result<(), Selec let kind: &str = context .nodes .node(reader) - .map_or(TASK_KIND, |node| &node.kind); + .map_or(ASSET_KIND, |node| &node.kind); let through: String = if path.len() > 1 { let names: Vec = path[..path.len() - 1] .iter() @@ -212,7 +212,7 @@ fn python_label(context: &SelectionContext<'_>, name: &str) -> String { format!("{kind} '{name}'") } -/// The nearest task or asset upstream of `loader` that reads SQL, with the Python nodes between. +/// The nearest asset upstream of `loader` that reads SQL, with the Python nodes between. fn sql_reading_upstream<'graph>( context: &SelectionContext<'graph>, loader: &str, @@ -227,7 +227,7 @@ fn sql_reading_upstream<'graph>( } while let Some(name) = queue.pop_front() { if let Some(node) = context.nodes.node(name) - && (node.kind == TASK_KIND || node.kind == ASSET_KIND) + && node.kind == ASSET_KIND && !node.sql_deps.is_empty() { let mut path: Vec<&'graph str> = vec![name]; diff --git a/crates/sqlbuild-analysis/src/python_selection/constants.rs b/crates/sqlbuild-analysis/src/python_selection/constants.rs index c199971bf1..18b286a695 100644 --- a/crates/sqlbuild-analysis/src/python_selection/constants.rs +++ b/crates/sqlbuild-analysis/src/python_selection/constants.rs @@ -1,6 +1,5 @@ //! Python node kinds, selector kinds, error codes and messages of Python's node selectors. -pub(crate) const TASK_KIND: &str = "task"; pub(crate) const ASSET_KIND: &str = "asset"; pub(crate) const LOADER_KIND: &str = "loader"; pub(crate) const CHECK_KIND: &str = "check"; diff --git a/crates/sqlbuild-analysis/src/python_selection/tests/helpers.rs b/crates/sqlbuild-analysis/src/python_selection/tests/helpers.rs index 14a0dace69..64ae70e8e5 100644 --- a/crates/sqlbuild-analysis/src/python_selection/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/python_selection/tests/helpers.rs @@ -38,7 +38,7 @@ const ORDERS_NODES: &[NodeSpec] = &[ ), ( "prepare_orders", - "task", + "asset", "python/tasks/orders.py", &["orders", "daily"], &[], @@ -165,7 +165,7 @@ pub(super) fn published_orders_graph() -> PythonSelectionGraph { let kind: &str = ORDERS_NODES .iter() .find(|node| node.0 == *upstream) - .map_or("task", |node| node.1); + .map_or("asset", |node| node.1); key(&(kind, upstream)) }) .chain(sql_deps.iter().map(key)) diff --git a/crates/sqlbuild-analysis/src/python_selection/tests/test_python_selection.rs b/crates/sqlbuild-analysis/src/python_selection/tests/test_python_selection.rs index 2a2a7e0231..5bba21b8b6 100644 --- a/crates/sqlbuild-analysis/src/python_selection/tests/test_python_selection.rs +++ b/crates/sqlbuild-analysis/src/python_selection/tests/test_python_selection.rs @@ -35,7 +35,7 @@ fn given_python_node_selectors_when_resolving_then_names_match_python() { }, NodeSelectorTestCase { description: "intersection", - select: &["task:prepare_orders,tag:orders"], + select: &["asset:prepare_orders,tag:orders"], exclude: &[], expected_names: Ok(&["prepare_orders"]), }, diff --git a/crates/sqlbuild-analysis/src/python_selection/tests/test_round_trips.rs b/crates/sqlbuild-analysis/src/python_selection/tests/test_round_trips.rs index be5ce9aa49..6d86dad301 100644 --- a/crates/sqlbuild-analysis/src/python_selection/tests/test_round_trips.rs +++ b/crates/sqlbuild-analysis/src/python_selection/tests/test_round_trips.rs @@ -8,11 +8,11 @@ use crate::python_selection::tests::test_types::{ const DIRECT_CYCLE: &str = "loader 'partner_scores' is in a dependency cycle: loader 'partner_scores' -> \ task 'score_orders' -> model 'scored_orders' -> source 'partner_scores' -> \ loader 'partner_scores'"; -const SCORE_ORDERS: RoundTripNodeSpec = ("score_orders", "task", &[], &[("model", "orders")]); +const SCORE_ORDERS: RoundTripNodeSpec = ("score_orders", "asset", &[], &[("model", "orders")]); const PARTNER_SCORES: RoundTripNodeSpec = ("partner_scores", "loader", &["score_orders"], &[]); const HELP: &str = "if this is intended, set `allow_warehouse_round_trips = true` under [settings] \ in sqlbuild_project.toml"; -const CYCLE_HELP: &str = "each arrow points to what a node depends on; a task or asset upstream \ +const CYCLE_HELP: &str = "each arrow points to what a node depends on; an asset upstream \ of a loader cannot read anything built from the source that loader fills"; #[test] @@ -36,7 +36,7 @@ fn given_a_loader_depending_on_python_that_reads_sql_when_selecting_then_the_set description: "a transitive SQL-reading asset is refused by default and named with its path", nodes: &[ ("score_orders", "asset", &[], &[("model", "orders")]), - ("package_scores", "task", &["score_orders"], &[]), + ("package_scores", "asset", &["score_orders"], &[]), ("partner_scores", "loader", &["package_scores"], &[]), ], allow_warehouse_round_trips: false, @@ -68,7 +68,7 @@ fn given_a_loader_depending_on_python_that_reads_sql_when_selecting_then_the_set RoundTripTestCase { description: "a loader depending on a task that reads no SQL is always allowed", nodes: &[ - ("fetch_scores", "task", &[], &[]), + ("fetch_scores", "asset", &[], &[]), ("partner_scores", "loader", &["fetch_scores"], &[]), ], allow_warehouse_round_trips: false, @@ -85,7 +85,7 @@ fn given_a_loader_depending_on_python_that_reads_sql_when_selecting_then_the_set RoundTripTestCase { description: "a task reading what its downstream loader's source builds is a cycle", nodes: &[ - ("score_orders", "task", &[], &[("model", "scored_orders")]), + ("score_orders", "asset", &[], &[("model", "scored_orders")]), PARTNER_SCORES, ], allow_warehouse_round_trips: true, @@ -101,7 +101,7 @@ fn given_a_loader_depending_on_python_that_reads_sql_when_selecting_then_the_set RoundTripTestCase { description: "a cycle is reported before the setting is consulted", nodes: &[ - ("score_orders", "task", &[], &[("model", "scored_orders")]), + ("score_orders", "asset", &[], &[("model", "scored_orders")]), PARTNER_SCORES, ], allow_warehouse_round_trips: false, @@ -136,7 +136,7 @@ fn given_whole_graph_boundaries_when_checked_without_a_selection_then_each_refus CycleTestCase { description: "a loader's depends_on closes the cycle", nodes: &[ - ("score_orders", "task", &[], &[("model", "scored_orders")]), + ("score_orders", "asset", &[], &[("model", "scored_orders")]), ("partner_scores", "loader", &["score_orders"], &[]), ], filled: true, @@ -146,7 +146,7 @@ fn given_whole_graph_boundaries_when_checked_without_a_selection_then_each_refus CycleTestCase { description: "without the loader's depends_on there is no cycle", nodes: &[ - ("score_orders", "task", &[], &[("model", "scored_orders")]), + ("score_orders", "asset", &[], &[("model", "scored_orders")]), ("partner_scores", "loader", &[], &[]), ], filled: true, @@ -156,8 +156,8 @@ fn given_whole_graph_boundaries_when_checked_without_a_selection_then_each_refus CycleTestCase { description: "a task's depends_on on another task closes the cycle", nodes: &[ - ("score_orders", "task", &[], &[("model", "scored_orders")]), - ("package_scores", "task", &["score_orders"], &[]), + ("score_orders", "asset", &[], &[("model", "scored_orders")]), + ("package_scores", "asset", &["score_orders"], &[]), ("partner_scores", "loader", &["package_scores"], &[]), ], filled: true, @@ -171,7 +171,7 @@ fn given_whole_graph_boundaries_when_checked_without_a_selection_then_each_refus CycleTestCase { description: "an asset's depends_on closes the cycle", nodes: &[ - ("score_orders", "task", &[], &[("model", "scored_orders")]), + ("score_orders", "asset", &[], &[("model", "scored_orders")]), ("scores_file", "asset", &["score_orders"], &[]), ("partner_scores", "loader", &["scores_file"], &[]), ], @@ -186,8 +186,8 @@ fn given_whole_graph_boundaries_when_checked_without_a_selection_then_each_refus CycleTestCase { description: "without the task's depends_on there is no cycle", nodes: &[ - ("score_orders", "task", &[], &[("model", "scored_orders")]), - ("package_scores", "task", &[], &[]), + ("score_orders", "asset", &[], &[("model", "scored_orders")]), + ("package_scores", "asset", &[], &[]), ("partner_scores", "loader", &["package_scores"], &[]), ], filled: true, @@ -197,7 +197,7 @@ fn given_whole_graph_boundaries_when_checked_without_a_selection_then_each_refus CycleTestCase { description: "a Python node's dependency model, through a model ref to the source, closes the cycle", nodes: &[ - ("score_orders", "task", &[], &[("model", "scored_orders")]), + ("score_orders", "asset", &[], &[("model", "scored_orders")]), ("partner_scores", "loader", &["score_orders"], &[]), ], filled: true, @@ -207,7 +207,7 @@ fn given_whole_graph_boundaries_when_checked_without_a_selection_then_each_refus CycleTestCase { description: "a dependency model whose refs never reach the source closes no cycle", nodes: &[ - ("score_orders", "task", &[], &[("model", "orders")]), + ("score_orders", "asset", &[], &[("model", "orders")]), ("partner_scores", "loader", &["score_orders"], &[]), ], filled: true, @@ -229,9 +229,9 @@ fn given_whole_graph_boundaries_when_checked_without_a_selection_then_each_refus CycleTestCase { description: "a task depending on the terminal loader instead of its source", nodes: &[ - ("fetch_scores", "task", &[], &[]), + ("fetch_scores", "asset", &[], &[]), ("partner_scores", "loader", &["fetch_scores"], &[]), - ("report_scores", "task", &["partner_scores"], &[]), + ("report_scores", "asset", &["partner_scores"], &[]), ], filled: true, allow_warehouse_round_trips: false, @@ -243,7 +243,7 @@ fn given_whole_graph_boundaries_when_checked_without_a_selection_then_each_refus CycleTestCase { description: "without the source's loader there is no cycle", nodes: &[ - ("score_orders", "task", &[], &[("model", "scored_orders")]), + ("score_orders", "asset", &[], &[("model", "scored_orders")]), ("partner_scores", "loader", &["score_orders"], &[]), ], filled: false, diff --git a/crates/sqlbuild-analysis/src/review_command/tests/helpers.rs b/crates/sqlbuild-analysis/src/review_command/tests/helpers.rs index f6f2273e23..d5d2b4320b 100644 --- a/crates/sqlbuild-analysis/src/review_command/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/review_command/tests/helpers.rs @@ -42,7 +42,7 @@ pub(super) fn waffle_shop() -> ReviewGraph { ("order_totals", "daily_revenue"), ("stg_orders", "order_archive"), ("stg_customers", "customer_lifetime"), - ("daily_revenue", "task:export_revenue"), + ("daily_revenue", "asset:export_revenue"), ("loader:raw_toppings", "stg_toppings"), ("stg_toppings", "topping_totals"), ]; @@ -62,7 +62,7 @@ pub(super) fn waffle_shop() -> ReviewGraph { node("order_archive", "model", false), node("daily_revenue", "model", true), node("customer_lifetime", "model", false), - python_node("task:export_revenue", "task"), + python_node("asset:export_revenue", "asset"), python_node("loader:raw_toppings", "loader"), node("stg_toppings", "model", false), node("topping_totals", "model", true), @@ -81,7 +81,7 @@ pub(super) fn waffle_shop() -> ReviewGraph { "order_totals", "topping_totals", "daily_revenue", - "task:export_revenue", + "asset:export_revenue", ] .iter() .map(|name| (*name).to_owned()) diff --git a/crates/sqlbuild-analysis/src/review_command/tests/test_plan_review.rs b/crates/sqlbuild-analysis/src/review_command/tests/test_plan_review.rs index 5ff3c3d7c7..61d5f7231c 100644 --- a/crates/sqlbuild-analysis/src/review_command/tests/test_plan_review.rs +++ b/crates/sqlbuild-analysis/src/review_command/tests/test_plan_review.rs @@ -10,7 +10,7 @@ use crate::review_command::tests::test_types::{ SchemaNameTestCase, }; -const EXPORT_TASK: (&str, &str) = ("task:export_revenue", "Python task, not run by review"); +const EXPORT_TASK: (&str, &str) = ("asset:export_revenue", "Python task, not run by review"); const ARCHIVE_LEFT_OUT: (&str, &str) = ( "order_archive", "not on a path to a critical model (--downstream critical)", diff --git a/crates/sqlbuild-cache/src/hosted_shapes/constants.rs b/crates/sqlbuild-cache/src/hosted_shapes/constants.rs index 481848e709..f3b2ebaa2a 100644 --- a/crates/sqlbuild-cache/src/hosted_shapes/constants.rs +++ b/crates/sqlbuild-cache/src/hosted_shapes/constants.rs @@ -1,7 +1,7 @@ //! The Python shape names of the natively hosted compiler shapes. /// The Python model type a natively hosted compiler shape is captured as. -pub const HOSTED_SHAPE_TYPES: [(&str, &str); 488] = [ +pub const HOSTED_SHAPE_TYPES: [(&str, &str); 486] = [ ( "sqlbuild._native:AlignedInterval", "sqlbuild.cursor_algebra.models:AlignedInterval", @@ -578,10 +578,6 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 488] = [ "sqlbuild._native:DiscoveredPythonNodeFunctions", "sqlbuild.compiler.discovery.models:DiscoveredPythonNodeFunctions", ), - ( - "sqlbuild._native:DiscoveredPythonTaskMetadata", - "sqlbuild.compiler.python_nodes.models:DiscoveredPythonTaskMetadata", - ), ( "sqlbuild._native:DiscoveredRuntimeExtensions", "sqlbuild.compiler.discovery.models:DiscoveredRuntimeExtensions", @@ -622,10 +618,6 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 488] = [ "sqlbuild._native:DiscoveredSqlTestFile", "sqlbuild.compiler.discovery.models:DiscoveredSqlTestFile", ), - ( - "sqlbuild._native:DiscoveredTaskFunction", - "sqlbuild.compiler.discovery.models:DiscoveredTaskFunction", - ), ( "sqlbuild._native:DiscoveryConflictError", "sqlbuild.compiler.discovery.exceptions:DiscoveryConflictError", diff --git a/crates/sqlbuild-cli/src/adhoc/tests/helpers.rs b/crates/sqlbuild-cli/src/adhoc/tests/helpers.rs index 7ee61af47e..1150df03db 100644 --- a/crates/sqlbuild-cli/src/adhoc/tests/helpers.rs +++ b/crates/sqlbuild-cli/src/adhoc/tests/helpers.rs @@ -46,7 +46,7 @@ pub(super) fn orders_publication() Some("python/orders/partner_export"), ), resource( - "task:request_carrier_collection", + "asset:request_carrier_collection", &["asset:partner_orders_file"], &["orders"], Some("python/fulfillment/carrier_sync"), @@ -71,7 +71,7 @@ pub(super) fn orders_publication() ]; let jobs = std::collections::BTreeMap::from([( "carrier_sync".to_owned(), - std::collections::BTreeSet::from(["task:request_carrier_collection".to_owned()]), + std::collections::BTreeSet::from(["asset:request_carrier_collection".to_owned()]), )]); crate::adhoc::main::adhoc_planner::adhoc_planner().prepare(resources, jobs) } diff --git a/crates/sqlbuild-cli/src/adhoc/tests/test_adhoc_planner.rs b/crates/sqlbuild-cli/src/adhoc/tests/test_adhoc_planner.rs index 64bc896f5f..c50dfd5eaa 100644 --- a/crates/sqlbuild-cli/src/adhoc/tests/test_adhoc_planner.rs +++ b/crates/sqlbuild-cli/src/adhoc/tests/test_adhoc_planner.rs @@ -373,7 +373,7 @@ fn given_published_python_nodes_when_planned_then_selectors_resolve_as_on_the_co Some("python/orders/partner_export"), ), resource( - "task:request_carrier_collection", + "asset:request_carrier_collection", &["asset:partner_orders_file"], &["orders"], Some("python/fulfillment/carrier_sync"), @@ -398,13 +398,13 @@ fn given_published_python_nodes_when_planned_then_selectors_resolve_as_on_the_co ]; let jobs: PublishedJobs = BTreeMap::from([( "carrier_sync".to_owned(), - BTreeSet::from(["task:request_carrier_collection".to_owned()]), + BTreeSet::from(["asset:request_carrier_collection".to_owned()]), )]); let test_cases = [ AdhocPlanTestCase { description: "a published job selects its Python nodes", command: &["build", "--select", "job:carrier_sync"], - expected_plan: Ok((&["task:request_carrier_collection"], json!({}))), + expected_plan: Ok((&["asset:request_carrier_collection"], json!({}))), }, AdhocPlanTestCase { description: "upstream of a published job crosses its Python node's dependencies", @@ -414,7 +414,7 @@ fn given_published_python_nodes_when_planned_then_selectors_resolve_as_on_the_co "asset:partner_orders_file", "model:orders", "model:stg_orders", - "task:request_carrier_collection", + "asset:request_carrier_collection", ], json!({}), )), @@ -424,21 +424,21 @@ fn given_published_python_nodes_when_planned_then_selectors_resolve_as_on_the_co command: &[ "build", "--select", - "task:request_carrier_collection", + "asset:request_carrier_collection", "asset:partner_orders_file", ], expected_plan: Ok(( &[ "asset:partner_orders_file", - "task:request_carrier_collection", + "asset:request_carrier_collection", ], json!({}), )), }, AdhocPlanTestCase { - description: "a task runs without the Python node it depends on, as on the command line", - command: &["build", "--select", "task:request_carrier_collection"], - expected_plan: Ok((&["task:request_carrier_collection"], json!({}))), + description: "an asset runs without the Python node it depends on, as on the command line", + command: &["build", "--select", "asset:request_carrier_collection"], + expected_plan: Ok((&["asset:request_carrier_collection"], json!({}))), }, AdhocPlanTestCase { description: "a python folder selects the nodes under it", @@ -453,20 +453,20 @@ fn given_published_python_nodes_when_planned_then_selectors_resolve_as_on_the_co "asset:partner_orders_file", "model:orders", "model:stg_orders", - "task:request_carrier_collection", + "asset:request_carrier_collection", ], json!({}), )), }, AdhocPlanTestCase { - description: "upstream of a task crosses its Python and SQL dependencies", - command: &["build", "--select", "+task:request_carrier_collection"], + description: "upstream of an asset crosses its Python and SQL dependencies", + command: &["build", "--select", "+asset:request_carrier_collection"], expected_plan: Ok(( &[ "asset:partner_orders_file", "model:orders", "model:stg_orders", - "task:request_carrier_collection", + "asset:request_carrier_collection", ], json!({}), )), @@ -478,7 +478,7 @@ fn given_published_python_nodes_when_planned_then_selectors_resolve_as_on_the_co &[ "asset:partner_orders_file", "model:orders", - "task:request_carrier_collection", + "asset:request_carrier_collection", ], json!({}), )), diff --git a/crates/sqlbuild-cli/src/adhoc/tests/test_selection_evaluation.rs b/crates/sqlbuild-cli/src/adhoc/tests/test_selection_evaluation.rs index 9feffdc741..3767515ec3 100644 --- a/crates/sqlbuild-cli/src/adhoc/tests/test_selection_evaluation.rs +++ b/crates/sqlbuild-cli/src/adhoc/tests/test_selection_evaluation.rs @@ -18,7 +18,7 @@ fn given_typed_selections_when_evaluated_then_keys_or_located_refusals_match_sqb "asset:partner_orders_file", "model:orders", "model:stg_orders", - "task:request_carrier_collection", + "asset:request_carrier_collection", ]), }, SelectionEvaluationTestCase { @@ -140,11 +140,10 @@ fn given_a_publication_when_its_vocabulary_is_read_then_each_value_counts_the_no description: "a kind counts its nodes", entries_of: |vocabulary| &vocabulary.kinds, expected_entries: &[ - ("asset", 1), + ("asset", 2), ("loader", 2), ("model", 4), ("source", 1), - ("task", 1), ], }, ]; diff --git a/crates/sqlbuild-core/src/build_command/models.rs b/crates/sqlbuild-core/src/build_command/models.rs index bf8b0d29e7..3237ab04db 100644 --- a/crates/sqlbuild-core/src/build_command/models.rs +++ b/crates/sqlbuild-core/src/build_command/models.rs @@ -189,7 +189,7 @@ pub struct RunContextFacts { pub selector_files: Vec, } -/// One finished task or asset. +/// One finished asset. #[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct PythonNodeRow { pub kind: String, diff --git a/crates/sqlbuild-core/src/build_command/tests/test_build_command.rs b/crates/sqlbuild-core/src/build_command/tests/test_build_command.rs index f9bb0b89ad..1946bcfcc7 100644 --- a/crates/sqlbuild-core/src/build_command/tests/test_build_command.rs +++ b/crates/sqlbuild-core/src/build_command/tests/test_build_command.rs @@ -170,7 +170,7 @@ fn given_python_node_results_when_rendering_then_shows_status_and_detail() { description: "success, skip reason and failure message", rows: vec![ PythonNodeRow { - kind: "task".to_owned(), + kind: "asset".to_owned(), node_name: "count_orders".to_owned(), status: "success".to_owned(), ..PythonNodeRow::default() @@ -183,7 +183,7 @@ fn given_python_node_results_when_rendering_then_shows_status_and_detail() { ..PythonNodeRow::default() }, PythonNodeRow { - kind: "task".to_owned(), + kind: "asset".to_owned(), node_name: "boom".to_owned(), status: "failed".to_owned(), error_message: Some("boom".to_owned()), diff --git a/crates/sqlbuild-core/src/build_progress/tests/test_build_progress.rs b/crates/sqlbuild-core/src/build_progress/tests/test_build_progress.rs index 74c9447f7c..d3b60b21f4 100644 --- a/crates/sqlbuild-core/src/build_progress/tests/test_build_progress.rs +++ b/crates/sqlbuild-core/src/build_progress/tests/test_build_progress.rs @@ -51,7 +51,7 @@ fn given_build_results_when_formatting_footer_then_matches_python_layout() { }, python: vec![PythonNodeText { name: "refresh".to_owned(), - kind: "task".to_owned(), + kind: "asset".to_owned(), status: "failed".to_owned(), error_message: None, warnings: Vec::new(), diff --git a/crates/sqlbuild-core/src/init_command/constants.rs b/crates/sqlbuild-core/src/init_command/constants.rs index ef080654ac..cce5df996c 100644 --- a/crates/sqlbuild-core/src/init_command/constants.rs +++ b/crates/sqlbuild-core/src/init_command/constants.rs @@ -27,7 +27,7 @@ pub const GITKEEP_FILE: &str = ".gitkeep"; pub const NEXT_STEPS: [&str; 6] = [ " 1. Add sources to sources/", " 2. Add seeds to seeds/", - " 3. Add Python loaders, tasks, assets, checks, and factories to python/", + " 3. Add Python loaders, assets, checks, and factories to python/", " 4. Add SQL hooks to hooks/sql/, Python hooks to hooks/python/, functions to functions/, or macros to macros/sql/ or macros/python/", " 5. Add models to models/staging/ and models/marts/", " 6. Add tests to tests/unit/ or tests/scenarios/, reusable audits to audits/generic/, or standalone audits to audits/singular/", diff --git a/crates/sqlbuild-core/src/plan_display/tests/helpers.rs b/crates/sqlbuild-core/src/plan_display/tests/helpers.rs index d3bf33a6d2..e76f08745f 100644 --- a/crates/sqlbuild-core/src/plan_display/tests/helpers.rs +++ b/crates/sqlbuild-core/src/plan_display/tests/helpers.rs @@ -109,7 +109,7 @@ pub(super) fn full_plan() -> serde_json::Value { {"name": "to_cents", "language": "sql", "reason": "definition_changed"}, {"name": "clean_name", "language": "sql", "reason": "no_change"}, ], - "python_nodes": [{"name": "score_orders", "kind": "task", "phase": "after", "identity_status": "changed"}], + "python_nodes": [{"name": "score_orders", "kind": "asset", "phase": "after", "identity_status": "changed"}], "warnings": [ {"severity": "warning", "model_name": "orders", "message": "orders: already-built dates keep their old results"}, {"severity": "warning", "message": "scenario 'checkout' step 'pay' is stale", "code": "W1"}, diff --git a/crates/sqlbuild-discovery/src/project_validation/_helpers/identities.rs b/crates/sqlbuild-discovery/src/project_validation/_helpers/identities.rs index 44716a3842..5938487a72 100644 --- a/crates/sqlbuild-discovery/src/project_validation/_helpers/identities.rs +++ b/crates/sqlbuild-discovery/src/project_validation/_helpers/identities.rs @@ -131,12 +131,7 @@ fn candidates(facts: &ProjectFacts) -> Vec> { } candidates.push_named(&facts.model_schemas, "model schema"); candidates.push_named(&facts.materializations, "materialization"); - for kind in [ - NodeKind::Loader, - NodeKind::Task, - NodeKind::Asset, - NodeKind::Check, - ] { + for kind in [NodeKind::Loader, NodeKind::Asset, NodeKind::Check] { for node in facts.nodes.iter().filter(|node| node.kind == kind) { candidates.push(&node.name, kind.label(), &node.path); } diff --git a/crates/sqlbuild-discovery/src/project_validation/_helpers/python_nodes.rs b/crates/sqlbuild-discovery/src/project_validation/_helpers/python_nodes.rs index c8c784d9aa..f83ee1a655 100644 --- a/crates/sqlbuild-discovery/src/project_validation/_helpers/python_nodes.rs +++ b/crates/sqlbuild-discovery/src/project_validation/_helpers/python_nodes.rs @@ -26,7 +26,7 @@ fn dependency_key(dependency: &DependencyFact) -> Option> { } } -/// The loaders, tasks and assets a dependency key names, as `node_by_dependency_key` maps them. +/// The loaders and assets a dependency key names, as `node_by_dependency_key` maps them. struct ExecutableNodes<'a> { nodes: &'a [NodeFact], by_name: HashMap<&'a str, usize>, @@ -74,7 +74,7 @@ pub(crate) fn validate_loader_dependencies(nodes: &[NodeFact]) -> Result<(), Val find_cycle(nodes, &roots, next, "Loader dependency cycle detected") } -/// `_validate_python_node_dependencies` over loaders, tasks and assets. +/// `_validate_python_node_dependencies` over loaders and assets. pub(crate) fn validate_python_node_dependencies( nodes: &[NodeFact], ) -> Result<(), ValidationFailure> { @@ -110,7 +110,7 @@ pub(crate) fn validate_python_node_dependencies( } return Err(conflict(format!( "Python node '{}' depends on an unknown Python node; use function \ - references to decorated loaders, tasks, or assets", + references to decorated loaders or assets", node.name ))); } @@ -128,7 +128,7 @@ pub(crate) fn validate_python_node_dependencies( find_cycle(nodes, &roots, next, "Python node dependency cycle detected") } -/// `_validate_check_dependencies`: checks depend on loaders, tasks and assets, never checks. +/// `_validate_check_dependencies`: checks depend on loaders and assets, never checks. pub(crate) fn validate_check_dependencies(nodes: &[NodeFact]) -> Result<(), ValidationFailure> { let allowed: HashSet<&str> = nodes .iter() @@ -151,7 +151,7 @@ pub(crate) fn validate_check_dependencies(nodes: &[NodeFact]) -> Result<(), Vali }; if name.is_some_and(|name| checks.contains(name)) { return Err(conflict(format!( - "Check '{}' depends on another check; checks may depend on loaders, tasks, \ + "Check '{}' depends on another check; checks may depend on loaders, \ assets, models, sources, and seeds only", check.name ))); @@ -159,7 +159,7 @@ pub(crate) fn validate_check_dependencies(nodes: &[NodeFact]) -> Result<(), Vali if !name.is_some_and(|name| allowed.contains(name)) { return Err(conflict(format!( "Check '{}' depends on an unknown Python node; use function references to \ - decorated loaders, tasks, or assets", + decorated loaders or assets", check.name ))); } diff --git a/crates/sqlbuild-discovery/src/project_validation/models.rs b/crates/sqlbuild-discovery/src/project_validation/models.rs index 35e45ea73f..88b31ae896 100644 --- a/crates/sqlbuild-discovery/src/project_validation/models.rs +++ b/crates/sqlbuild-discovery/src/project_validation/models.rs @@ -37,7 +37,6 @@ pub struct SourceFileFact { #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum NodeKind { Loader, - Task, Asset, Check, } @@ -47,7 +46,6 @@ impl NodeKind { pub fn label(self) -> &'static str { match self { Self::Loader => "loader", - Self::Task => "task", Self::Asset => "asset", Self::Check => "check", } @@ -161,7 +159,7 @@ pub struct ProjectFacts { /// Every model schema declaration across the model schema files. pub model_schemas: Vec, pub materializations: Vec, - /// Loaders, tasks, assets, then checks. + /// Loaders, assets, then checks. pub nodes: Vec, pub hook_functions: Vec, pub event_exporters: Vec, diff --git a/crates/sqlbuild-discovery/src/project_validation/tests/test_validate_project_inputs.rs b/crates/sqlbuild-discovery/src/project_validation/tests/test_validate_project_inputs.rs index f07daa347f..ea4cdd05f6 100644 --- a/crates/sqlbuild-discovery/src/project_validation/tests/test_validate_project_inputs.rs +++ b/crates/sqlbuild-discovery/src/project_validation/tests/test_validate_project_inputs.rs @@ -62,11 +62,11 @@ fn given_discovered_facts_when_validating_then_reports_python_first_conflict() { ), }, ValidationTestCase { - description: "a task cycle names its path", + description: "an asset cycle names its path", facts: ProjectFacts { nodes: vec![ - node(NodeKind::Task, "extract", vec![on(1, "load")]), - node(NodeKind::Task, "load", vec![on(0, "extract")]), + node(NodeKind::Asset, "extract", vec![on(1, "load")]), + node(NodeKind::Asset, "load", vec![on(0, "extract")]), ], ..ProjectFacts::default() }, @@ -86,7 +86,7 @@ fn given_discovered_facts_when_validating_then_reports_python_first_conflict() { }, expected_failure: failure( FailureKind::Conflict, - "Check 'second' depends on another check; checks may depend on loaders, tasks, \ + "Check 'second' depends on another check; checks may depend on loaders, \ assets, models, sources, and seeds only", ), }, diff --git a/crates/sqlbuild-host/src/entries/_helpers/user_code.rs b/crates/sqlbuild-host/src/entries/_helpers/user_code.rs index 8e77164e4a..7a1feec3c2 100644 --- a/crates/sqlbuild-host/src/entries/_helpers/user_code.rs +++ b/crates/sqlbuild-host/src/entries/_helpers/user_code.rs @@ -214,11 +214,6 @@ pub(crate) const ENTRIES: &[HostEntry] = &[ kind: HostKind::Function, edge: HostEdge::UserCode, }, - HostEntry { - name: "read_task_definition", - kind: HostKind::Function, - edge: HostEdge::UserCode, - }, HostEntry { name: "AuditCase", kind: HostKind::Class, diff --git a/crates/sqlbuild-host/src/entries/models.rs b/crates/sqlbuild-host/src/entries/models.rs index 4c39f96c72..7d8f0a3da0 100644 --- a/crates/sqlbuild-host/src/entries/models.rs +++ b/crates/sqlbuild-host/src/entries/models.rs @@ -171,7 +171,6 @@ pub enum UserCodeHost { ReadFactoryDefinition, ReadHookDefinition, ReadLoaderDefinition, - ReadTaskDefinition, AuditCase, RelationRenderer, SqlResourceRef, diff --git a/crates/sqlbuild-orchestration/tests/runs/test_group_round_trips.rs b/crates/sqlbuild-orchestration/tests/runs/test_group_round_trips.rs index 01e9b0009a..72b208d82e 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_group_round_trips.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_group_round_trips.rs @@ -64,12 +64,12 @@ fn given_a_round_trip_split_across_jobs_when_the_group_runs_then_members_follow_ snapshot_node("model", json!({"deps": []})), ), ( - "task:score_orders".into(), + "asset:score_orders".into(), snapshot_node("task", json!({"deps": ["model:orders"]})), ), ( "loader:partner_scores".into(), - snapshot_node("loader", json!({"deps": ["task:score_orders"]})), + snapshot_node("loader", json!({"deps": ["asset:score_orders"]})), ), ( "source:partner_scores".into(), @@ -90,7 +90,7 @@ fn given_a_round_trip_split_across_jobs_when_the_group_runs_then_members_follow_ "jobs".into(), json!({ "orders_job": {"description": "Build orders", "selection": ["model:orders"]}, - "score_job": {"description": "Score orders", "selection": ["task:score_orders"]}, + "score_job": {"description": "Score orders", "selection": ["asset:score_orders"]}, "ingest_job": { "description": "Load partner scores", "selection": ["loader:partner_scores", "source:partner_scores"] @@ -107,7 +107,7 @@ fn given_a_round_trip_split_across_jobs_when_the_group_runs_then_members_follow_ "description": "Score orders through the partner and report on them", "selection": [ "loader:partner_scores", "model:orders", "model:scored_orders", - "source:partner_scores", "task:score_orders" + "source:partner_scores", "asset:score_orders" ], "jobs": test_case.expected_waits, }}), diff --git a/crates/sqlbuild-pysource/src/definition_locations/tests/test_decorated_functions.rs b/crates/sqlbuild-pysource/src/definition_locations/tests/test_decorated_functions.rs index ca64b4ae9a..d40eb497e9 100644 --- a/crates/sqlbuild-pysource/src/definition_locations/tests/test_decorated_functions.rs +++ b/crates/sqlbuild-pysource/src/definition_locations/tests/test_decorated_functions.rs @@ -11,12 +11,12 @@ fn given_module_when_listing_decorated_functions_then_only_top_level_functions_c }, DecoratedFunctionsTestCase { description: "a bare imported decorator among others", - contents: "@cache\n@task\ndef run():\n pass\n\n\n@asset\ndef build():\n pass\n", + contents: "@cache\n@asset\ndef run():\n pass\n\n\n@asset\ndef build():\n pass\n", expected_functions: &["run", "build"], }, DecoratedFunctionsTestCase { description: "a decorated method is not a top-level function", - contents: "class Box:\n @task\n def run(self):\n pass\n", + contents: "class Box:\n @asset\n def run(self):\n pass\n", expected_functions: &[], }, DecoratedFunctionsTestCase { @@ -31,13 +31,13 @@ fn given_module_when_listing_decorated_functions_then_only_top_level_functions_c }, DecoratedFunctionsTestCase { description: "a syntax error", - contents: "@task\ndef run(:\n", + contents: "@asset\ndef run(:\n", expected_functions: &[], }, ]; for test_case in test_cases { assert_eq!( - decorated_functions(test_case.contents, &["loader", "task", "asset"], (3, 12)), + decorated_functions(test_case.contents, &["loader", "asset"], (3, 12)), test_case.expected_functions, "{}", test_case.description diff --git a/crates/sqlbuild-pysource/src/node_units/tests/test_node_unit_module.rs b/crates/sqlbuild-pysource/src/node_units/tests/test_node_unit_module.rs index cc9f743606..8897ec6cd8 100644 --- a/crates/sqlbuild-pysource/src/node_units/tests/test_node_unit_module.rs +++ b/crates/sqlbuild-pysource/src/node_units/tests/test_node_unit_module.rs @@ -6,9 +6,9 @@ use crate::node_units::tests::test_types::{ FunctionTestCase, ImportsTestCase, StatementKindsTestCase, }; -const CRLF_NODE_MODULE: &str = "\"\"\"Doc.\"\"\"\r\nfrom .main import main\r\nfrom python.orders.daily.node import daily as upstream\r\nimport python.utils.text\r\n\r\nX = 'é'\r\n\r\n\r\n@sqlbuild.task(depends_on=[upstream], description=\"D.\")\r\ndef run(ctx: TaskContext, /, *, store: \"providers.Store\") -> PythonNodeResult | None:\r\n \"\"\"Doc.\"\"\"\r\n return main(ctx=ctx, store=store)\r\n"; +const CRLF_NODE_MODULE: &str = "\"\"\"Doc.\"\"\"\r\nfrom .main import main\r\nfrom python.orders.daily.node import daily as upstream\r\nimport python.utils.text\r\n\r\nX = 'é'\r\n\r\n\r\n@sqlbuild.task(depends_on=[upstream], description=\"D.\")\r\ndef run(ctx: AssetContext, /, *, store: \"providers.Store\") -> PythonNodeResult | None:\r\n \"\"\"Doc.\"\"\"\r\n return main(ctx=ctx, store=store)\r\n"; -const FACTORY_MODULE: &str = "@factory\ndef tables():\n for table in TABLES:\n for _ in specs():\n pass\n\n @task(name=table)\n def load(ctx, *args, **kwargs) -> Optional[Union[LoaderResult, list]]:\n rows = main(ctx=ctx)\n return rows\n\n def plain():\n return main(ctx, **extra)\n"; +const FACTORY_MODULE: &str = "@factory\ndef tables():\n for table in TABLES:\n for _ in specs():\n pass\n\n @asset(name=table)\n def load(ctx, *args, **kwargs) -> Optional[Union[LoaderResult, list]]:\n rows = main(ctx=ctx)\n return rows\n\n def plain():\n return main(ctx, **extra)\n"; #[test] fn given_module_when_reading_then_each_top_level_statement_has_its_kind() { @@ -90,11 +90,11 @@ fn given_declaration_when_reading_then_signature_body_and_inner_declarations_are name: "run".to_owned(), position: at(10, 5), decorators: vec![decorator( - "task", + "asset", &["depends_on", "description"], &["upstream"], )], - positional: vec![parameter("ctx", Some("TaskContext"))], + positional: vec![parameter("ctx", Some("AssetContext"))], variadic: false, keyword_only: vec![parameter("store", Some("Store"))], keywords_variadic: false, @@ -124,7 +124,7 @@ fn given_declaration_when_reading_then_signature_body_and_inner_declarations_are inner: vec![UnitFunction { name: "load".to_owned(), position: at(8, 13), - decorators: vec![decorator("task", &["name"], &[])], + decorators: vec![decorator("asset", &["name"], &[])], positional: vec![parameter("ctx", None)], variadic: true, keyword_only: vec![], diff --git a/crates/sqlbuild-python-base/src/records/_helpers/compile_records/constants.rs b/crates/sqlbuild-python-base/src/records/_helpers/compile_records/constants.rs index 1eda7ce0f7..befb01b4a1 100644 --- a/crates/sqlbuild-python-base/src/records/_helpers/compile_records/constants.rs +++ b/crates/sqlbuild-python-base/src/records/_helpers/compile_records/constants.rs @@ -949,29 +949,6 @@ const R_DISCOVERED_LOADER_FUNCTION: RecordSpec = RecordSpec { }, ], }; -const R_DISCOVERED_TASK_FUNCTION: RecordSpec = RecordSpec { - name: "DiscoveredTaskFunction", - doc: "A discovered project task function.", - fields: &[ - FieldSpec::required("file_path"), - FieldSpec::required("relative_path"), - FieldSpec::required("name"), - FieldSpec::required("function"), - FieldSpec::new("depends_on", FieldDefault::EmptyTuple), - FieldSpec::new("tags", FieldDefault::EmptyTuple), - FieldSpec::new("group", FieldDefault::None), - FieldSpec::new("description", FieldDefault::None), - FieldSpec::new("meta", FieldDefault::None), - FieldSpec::new("retry", FieldDefault::None), - FieldSpec::new("provider_usages", FieldDefault::EmptyTuple), - FieldSpec { - name: "identity", - default: FieldDefault::None, - compared: false, - shown: false, - }, - ], -}; const R_DISCOVERED_ASSET_FUNCTION: RecordSpec = RecordSpec { name: "DiscoveredAssetFunction", doc: "A discovered project asset function.", @@ -1134,7 +1111,6 @@ const R_DISCOVERED_PYTHON_NODE_FUNCTIONS: RecordSpec = RecordSpec { doc: "Discovered project Python DAG node functions grouped by kind.", fields: &[ FieldSpec::new("loaders", FieldDefault::EmptyTuple), - FieldSpec::new("tasks", FieldDefault::EmptyTuple), FieldSpec::new("assets", FieldDefault::EmptyTuple), FieldSpec::new("checks", FieldDefault::EmptyTuple), FieldSpec::new("audit_factories", FieldDefault::EmptyTuple), @@ -1163,7 +1139,6 @@ const R_DISCOVERED_PROJECT_INPUTS: RecordSpec = RecordSpec { FieldSpec::new("macro_files", FieldDefault::EmptyTuple), FieldSpec::new("materialization_files", FieldDefault::EmptyTuple), FieldSpec::new("loader_functions", FieldDefault::EmptyTuple), - FieldSpec::new("task_functions", FieldDefault::EmptyTuple), FieldSpec::new("asset_functions", FieldDefault::EmptyTuple), FieldSpec::new("check_functions", FieldDefault::EmptyTuple), FieldSpec::new("audit_factories", FieldDefault::EmptyTuple), @@ -1705,7 +1680,6 @@ pub(crate) const COMPILE_VALUE_RECORDS: &[RecordSpec] = &[ R_DISCOVERED_MATERIALIZATION_FILE, R_DISCOVERED_PROVIDER_USAGE, R_DISCOVERED_LOADER_FUNCTION, - R_DISCOVERED_TASK_FUNCTION, R_DISCOVERED_ASSET_FUNCTION, R_DISCOVERED_CHECK_FUNCTION, R_DISCOVERED_HOOK_FUNCTION, diff --git a/crates/sqlbuild-python-base/src/records/_helpers/compiler_enums/constants.rs b/crates/sqlbuild-python-base/src/records/_helpers/compiler_enums/constants.rs index 63f2aa3e7b..c9267e37dc 100644 --- a/crates/sqlbuild-python-base/src/records/_helpers/compiler_enums/constants.rs +++ b/crates/sqlbuild-python-base/src/records/_helpers/compiler_enums/constants.rs @@ -179,7 +179,6 @@ const E_SELECTOR_KIND: EnumSpec = EnumSpec { ("NAME", "name"), ("SEED", "seed"), ("SOURCE", "source"), - ("TASK", "task"), ("ASSET", "asset"), ("LOADER", "loader"), ("CHECK", "check"), diff --git a/crates/sqlbuild-python-base/src/records/_helpers/compiler_records/constants.rs b/crates/sqlbuild-python-base/src/records/_helpers/compiler_records/constants.rs index ca785c24e1..6c7f0981b8 100644 --- a/crates/sqlbuild-python-base/src/records/_helpers/compiler_records/constants.rs +++ b/crates/sqlbuild-python-base/src/records/_helpers/compiler_records/constants.rs @@ -501,11 +501,6 @@ const R_DISCOVERED_PYTHON_LOADER_METADATA: RecordSpec = RecordSpec { FieldSpec::new("contract", FieldDefault::None), ], }; -const R_DISCOVERED_PYTHON_TASK_METADATA: RecordSpec = RecordSpec { - name: "DiscoveredPythonTaskMetadata", - doc: "Task-specific metadata carried by an internal Python task node.", - fields: &[FieldSpec::new("retry", FieldDefault::None)], -}; const R_DISCOVERED_PYTHON_ASSET_METADATA: RecordSpec = RecordSpec { name: "DiscoveredPythonAssetMetadata", doc: "Asset-specific metadata carried by an internal Python asset node.", @@ -792,7 +787,6 @@ const R_DISCOVERED_PYTHON_NODE: RecordSpec = RecordSpec { FieldSpec::new("meta", FieldDefault::None), FieldSpec::new("provider_usages", FieldDefault::EmptyTuple), FieldSpec::new("loader", FieldDefault::None), - FieldSpec::new("task", FieldDefault::None), FieldSpec::new("asset", FieldDefault::None), FieldSpec::new("check", FieldDefault::None), FieldSpec::new("identity", FieldDefault::None), @@ -921,7 +915,6 @@ pub(crate) const COMPILER_RECORDS: &[RecordSpec] = &[ R_CLONE_PIPELINE_CONNECTION, R_CLONE_PIPELINE_RESULT, R_DISCOVERED_PYTHON_LOADER_METADATA, - R_DISCOVERED_PYTHON_TASK_METADATA, R_DISCOVERED_PYTHON_ASSET_METADATA, R_DISCOVERED_PYTHON_CHECK_METADATA, R_PYTHON_NODE_DEPENDENCY_EDGE, diff --git a/crates/sqlbuild-python-base/src/records/_helpers/native_classes/ids.rs b/crates/sqlbuild-python-base/src/records/_helpers/native_classes/ids.rs index ac4ad9478e..51f1ec8673 100644 --- a/crates/sqlbuild-python-base/src/records/_helpers/native_classes/ids.rs +++ b/crates/sqlbuild-python-base/src/records/_helpers/native_classes/ids.rs @@ -224,7 +224,6 @@ native_id_names! { RecordId::DiscoveredPythonLoaderMetadata => "DiscoveredPythonLoaderMetadata", RecordId::DiscoveredPythonNode => "DiscoveredPythonNode", RecordId::DiscoveredPythonNodeFunctions => "DiscoveredPythonNodeFunctions", - RecordId::DiscoveredPythonTaskMetadata => "DiscoveredPythonTaskMetadata", RecordId::DiscoveredRuntimeExtensions => "DiscoveredRuntimeExtensions", RecordId::DiscoveredSchemaFile => "DiscoveredSchemaFile", RecordId::DiscoveredSeedFile => "DiscoveredSeedFile", @@ -235,7 +234,6 @@ native_id_names! { RecordId::DiscoveredSqlTestBlock => "DiscoveredSqlTestBlock", RecordId::DiscoveredSqlTestCase => "DiscoveredSqlTestCase", RecordId::DiscoveredSqlTestFile => "DiscoveredSqlTestFile", - RecordId::DiscoveredTaskFunction => "DiscoveredTaskFunction", RecordId::DiscoveryFileFault => "DiscoveryFileFault", RecordId::DisplayOptions => "DisplayOptions", RecordId::Duration => "Duration", diff --git a/crates/sqlbuild-python-base/src/records/types.rs b/crates/sqlbuild-python-base/src/records/types.rs index ea187f9426..8893ea9fe9 100644 --- a/crates/sqlbuild-python-base/src/records/types.rs +++ b/crates/sqlbuild-python-base/src/records/types.rs @@ -217,7 +217,6 @@ pub enum RecordId { DiscoveredPythonLoaderMetadata, DiscoveredPythonNode, DiscoveredPythonNodeFunctions, - DiscoveredPythonTaskMetadata, DiscoveredRuntimeExtensions, DiscoveredSchemaFile, DiscoveredSeedFile, @@ -228,7 +227,6 @@ pub enum RecordId { DiscoveredSqlTestBlock, DiscoveredSqlTestCase, DiscoveredSqlTestFile, - DiscoveredTaskFunction, DiscoveryFileFault, DisplayOptions, Duration, diff --git a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/assembly/assembly_project.rs b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/assembly/assembly_project.rs index b05f339fc8..26900c7031 100644 --- a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/assembly/assembly_project.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/assembly/assembly_project.rs @@ -311,11 +311,7 @@ fn project_fields<'py>( fields.set_item(name, native_field(&discovered, name)?)?; } let python_inputs: Bound<'py, PyDict> = PyDict::new(py); - for (kind, collection) in [ - ("task", "task_functions"), - ("asset", "asset_functions"), - ("check", "check_functions"), - ] { + for (kind, collection) in [("asset", "asset_functions"), ("check", "check_functions")] { for function in native_field(&discovered, collection)?.try_iter()? { let function = function?; let name: String = native_field(&function, "name")?.extract()?; diff --git a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/node_results.rs b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/node_results.rs index a98fb8e915..c2265f88c1 100644 --- a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/node_results.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/node_results.rs @@ -14,12 +14,7 @@ use sqlbuild_python_base::boundary::main::optional_text::optional_text; use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The discovered inputs' Python node functions, each kind of which keeps node results. -const PYTHON_NODE_FIELDS: [&str; 4] = [ - "loader_functions", - "task_functions", - "asset_functions", - "check_functions", -]; +const PYTHON_NODE_FIELDS: [&str; 3] = ["loader_functions", "asset_functions", "check_functions"]; /// Raise when a `preserve` target has Python nodes but no usable loader schema, after `effective_vars`, to keep their results in. pub(crate) fn require_node_results_schema( @@ -68,7 +63,7 @@ fn resolved_loader_schema( } } -/// How many loaders, tasks, assets and checks the project declares in Python. +/// How many loaders, assets and checks the project declares in Python. pub(crate) fn python_node_count(discovered: &Bound<'_, PyAny>) -> PyResult { PYTHON_NODE_FIELDS .iter() diff --git a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/pipeline_entries.rs b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/pipeline_entries.rs index 05d6928b9f..4a544ce36a 100644 --- a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/pipeline_entries.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/pipeline_entries.rs @@ -10,12 +10,7 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; /// Discovered Python node functions whose selectors need the Python graph. -const PYTHON_NODE_FUNCTIONS: [&str; 4] = [ - "loader_functions", - "task_functions", - "asset_functions", - "check_functions", -]; +const PYTHON_NODE_FUNCTIONS: [&str; 3] = ["loader_functions", "asset_functions", "check_functions"]; /// Plan entry lists that are runtime work. pub(crate) const PLAN_WORK_ENTRIES: [&str; 6] = [ "model_entries", diff --git a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/python_plan_entries.rs b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/python_plan_entries.rs index 10ee6aaf6b..ae44fee3cc 100644 --- a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/python_plan_entries.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/python_plan_entries.rs @@ -25,7 +25,6 @@ const SKIPPED_UPSTREAM_CODE: &str = "P501"; /// The Python node kinds and plan phases entries are built from. pub(crate) struct NodeKinds<'py> { - task: Bound<'py, PyAny>, asset: Bound<'py, PyAny>, } @@ -33,14 +32,13 @@ impl<'py> NodeKinds<'py> { pub(crate) fn load(py: Python<'py>) -> PyResult { let kinds: Bound<'py, PyAny> = host_entry(py, UserCodeHost::PythonNodeKind)?; Ok(Self { - task: native_field(&kinds, "TASK")?, asset: native_field(&kinds, "ASSET")?, }) } - /// Whether `kind` is a task or an asset. + /// Whether `kind` is an asset. fn runs(&self, kind: &Bound<'py, PyAny>) -> PyResult { - Ok(kind.eq(&self.task)? || kind.eq(&self.asset)?) + kind.eq(&self.asset) } } @@ -67,7 +65,7 @@ pub(crate) fn node_graph<'py>(discovered: &Bound<'py, PyAny>) -> PyResult( @@ -96,7 +94,7 @@ pub(crate) fn build_python_plan_entries<'py>( ) } -/// The entries of the lifecycle's ingress and then read-side tasks and assets. +/// The entries of the lifecycle's ingress and then read-side assets. pub(crate) fn plan_entries<'py>( lifecycle: &Bound<'py, PyAny>, (graph, kinds): (&Bound<'py, PyAny>, &NodeKinds<'py>), @@ -201,7 +199,7 @@ fn identity_status<'py>( native_field(&statuses, "CHANGED") } -/// Warnings for skipped task/asset deps of planned source loaders. +/// Warnings for skipped asset deps of planned source loaders. pub(crate) fn skipped_ingress_warnings<'py>( plan_output: &Bound<'py, PyAny>, run_selection: &Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/discovery.rs b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/discovery.rs index ada549062c..1349717631 100644 --- a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/discovery.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/discovery.rs @@ -171,7 +171,7 @@ impl Discoverer { } /// Python node groups whose relative paths count as imported project Python. -const PYTHON_NODE_GROUPS: [&str; 5] = ["loaders", "tasks", "assets", "checks", "audit_factories"]; +const PYTHON_NODE_GROUPS: [&str; 4] = ["loaders", "assets", "checks", "audit_factories"]; /// `OperationLifecycle(operation_kind="project", operation_name=name)`. pub(crate) fn lifecycle<'py>(py: Python<'py>, name: &str) -> PyResult> { @@ -485,7 +485,6 @@ fn project_inputs<'py>( python.get_item("materialization_files")?, )); fields.push(("loader_functions", loader_functions)); - fields.push(("task_functions", tuple_of("tasks")?)); fields.push(("asset_functions", tuple_of("assets")?)); fields.push(("check_functions", tuple_of("checks")?)); fields.push(("audit_factories", tuple_of("audit_factories")?)); diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/setup/python_node_count.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/python_node_count.rs index 92f25771f4..9139254572 100644 --- a/crates/sqlbuild-python-compile/src/compile_driver/main/setup/python_node_count.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/python_node_count.rs @@ -3,7 +3,7 @@ use crate::compile_driver::_helpers::setup::node_results; use pyo3::prelude::{Bound, PyAny, PyResult}; -/// How many loaders, tasks, assets and checks the project declares in Python. +/// How many loaders, assets and checks the project declares in Python. pub fn python_node_count(discovered: &Bound<'_, PyAny>) -> PyResult { node_results::python_node_count(discovered) } diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/project/project_facts.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/project/project_facts.rs index 19d4f79fdf..5bb440093a 100644 --- a/crates/sqlbuild-python-front/src/discovery/_helpers/project/project_facts.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/project/project_facts.rs @@ -20,9 +20,8 @@ const CSV_MODULE: &str = "csv"; /// `DeclarationKind.AUDIT`: a generic audit file. const GENERIC_AUDIT_KIND: &str = "audit"; /// Each Python node kind and the discovered inputs field that lists it, in discovery order. -pub(crate) const PYTHON_NODE_FIELDS: [(&str, &str); 4] = [ +pub(crate) const PYTHON_NODE_FIELDS: [(&str, &str); 3] = [ ("loader", "loader_functions"), - ("task", "task_functions"), ("asset", "asset_functions"), ("check", "check_functions"), ]; @@ -173,7 +172,6 @@ pub(crate) fn declarations_fact(file: &Bound<'_, PyAny>) -> PyResult PyResult { match kind { "loader" => Ok(NodeKind::Loader), - "task" => Ok(NodeKind::Task), "asset" => Ok(NodeKind::Asset), "check" => Ok(NodeKind::Check), other => Err(PyValueError::new_err(format!( diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/imported_python_nodes.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/imported_python_nodes.rs index c9f981204b..984789baa8 100644 --- a/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/imported_python_nodes.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/imported_python_nodes.rs @@ -4,7 +4,8 @@ use crate::discovery::main::python_files::provider_by_name::provider_by_name; use crate::discovery::main::python_files::provider_usages::provider_usages; use std::collections::HashMap; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::exceptions::PyImportError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyBytes, PyDict, PyDictMethods, PyList, PyString, PyTuple}; use pyo3::{IntoPyObjectExt, pyfunction}; use sqlbuild_core::constants::PYTHON_NODE_ROOT; @@ -26,7 +27,9 @@ const AUDIT_CASE: UserCodeHost = UserCodeHost::AuditCase; const READ_FACTORY: UserCodeHost = UserCodeHost::ReadFactoryDefinition; const READ_AUDIT_FACTORY: UserCodeHost = UserCodeHost::ReadAuditFactoryDefinition; const READ_HOOK: UserCodeHost = UserCodeHost::ReadHookDefinition; -const NOT_A_NODE: &str = "that is not a SQLBuild task, asset, loader, or check"; +const NOT_A_NODE: &str = "that is not a SQLBuild asset, loader, or check"; +/// The removed task module, and how `from sqlbuild import tasks` names it. +const REMOVED_TASKS: (&str, (&str, &str)) = ("sqlbuild.tasks", ("sqlbuild", "tasks")); /// One Python-node kind: its reader, record class, bucket and copied definition fields. struct NodeKind { @@ -39,7 +42,7 @@ struct NodeKind { } /// Readers in precedence order; the first matching definition decides the kind. -const NODE_KINDS: [NodeKind; 4] = [ +const NODE_KINDS: [NodeKind; 3] = [ NodeKind { reader: UserCodeHost::ReadLoaderDefinition, record: RecordId::DiscoveredLoaderFunction, @@ -56,24 +59,10 @@ const NODE_KINDS: [NodeKind; 4] = [ "description", ], }, - NodeKind { - reader: UserCodeHost::ReadTaskDefinition, - record: RecordId::DiscoveredTaskFunction, - bucket: 1, - identity: ("TASK", true), - fields: &[ - "depends_on", - "tags", - "group", - "description", - "meta", - "retry", - ], - }, NodeKind { reader: UserCodeHost::ReadAssetDefinition, record: RecordId::DiscoveredAssetFunction, - bucket: 2, + bucket: 1, identity: ("ASSET", true), fields: &[ "depends_on", @@ -89,7 +78,7 @@ const NODE_KINDS: [NodeKind; 4] = [ NodeKind { reader: UserCodeHost::ReadCheckDefinition, record: RecordId::DiscoveredCheckFunction, - bucket: 3, + bucket: 2, identity: ("CHECK", true), fields: &[ "depends_on", @@ -102,9 +91,9 @@ const NODE_KINDS: [NodeKind; 4] = [ }, ]; -/// Discovered loaders, tasks, assets, checks and audit factories, in discovery order. +/// Discovered loaders, assets, checks and audit factories, in discovery order. pub(crate) struct Buckets<'py> { - pub(crate) nodes: [Vec>; 4], + pub(crate) nodes: [Vec>; 3], pub(crate) audit_factories: Vec<(String, String, Bound<'py, PyAny>)>, } @@ -258,8 +247,9 @@ impl<'py> Buckets<'py> { ) -> PyResult<()> { let mut modules: Vec> = Vec::new(); for file_path in project.module_files(PYTHON_NODE_ROOT, false)? { - let module: Bound<'py, PyAny> = - project.load(UserCodeHost::ImportPythonRootModule, &file_path)?; + let module: Bound<'py, PyAny> = project + .load(UserCodeHost::ImportPythonRootModule, &file_path) + .map_err(|error| removed_tasks_guidance(project, &file_path, error))?; modules.push(ImportedModule { file_path, module }); } let mut direct: HashMap> = HashMap::new(); @@ -284,7 +274,7 @@ fn node_definition_names(function: &Bound<'_, PyAny>) -> PyResult( }) } +/// `error` pointing to `@asset` when importing `file_path` failed on the removed `sqlbuild.tasks`. +fn removed_tasks_guidance( + project: &ProjectDir<'_>, + file_path: &Bound<'_, PyAny>, + error: PyErr, +) -> PyErr { + let py: Python<'_> = project.py(); + let Some(cause) = error.cause(py) else { + return error; + }; + if !cause.is_instance_of::(py) + || !imports_removed_tasks(cause.value(py).as_any()) + { + return error; + } + let Ok(path) = project.relative_to(file_path) else { + return error; + }; + let message: String = format!( + "Failed to import Python node file {path}: `sqlbuild.tasks` was removed; \ + use `@asset` from `sqlbuild.assets` instead, with `AssetContext` in place of `TaskContext`" + ); + discovery_error_from(py, (NODE_ERROR, message), error) +} + +/// Whether an `ImportError` names `sqlbuild.tasks`, or `tasks` imported from `sqlbuild`. +fn imports_removed_tasks(error: &Bound<'_, PyAny>) -> bool { + let text = |attribute: &str| -> Option { + error.getattr_opt(attribute).ok().flatten()?.extract().ok() + }; + let (module, (package, name)) = REMOVED_TASKS; + let imported: Option = text("name"); + imported.as_deref() == Some(module) + || (imported.as_deref() == Some(package) && text("name_from").as_deref() == Some(name)) +} + fn audit_cases<'py>( factory: &Bound<'py, PyAny>, (name, relative_path): (&str, &str), diff --git a/crates/sqlbuild-python-front/src/discovery/main/python_files/discover_python_node_functions.rs b/crates/sqlbuild-python-front/src/discovery/main/python_files/discover_python_node_functions.rs index 446f0a8e07..21b42a7f18 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/python_files/discover_python_node_functions.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/python_files/discover_python_node_functions.rs @@ -28,7 +28,7 @@ pub fn discover_python_node_functions<'py>( None => PyDict::new(py), }; let mut buckets = Buckets { - nodes: [Vec::new(), Vec::new(), Vec::new(), Vec::new()], + nodes: [Vec::new(), Vec::new(), Vec::new()], audit_factories: Vec::new(), }; let node_root: Bound<'py, PyAny> = project.child(PYTHON_NODE_ROOT)?; @@ -48,13 +48,12 @@ pub fn discover_python_node_functions<'py>( )?; discovered_nodes?; } - let [loaders, tasks, assets, checks] = buckets.nodes; + let [loaders, assets, checks] = buckets.nodes; discovered( py, RecordId::DiscoveredPythonNodeFunctions, &[ ("loaders", tuple_of(py, loaders)?), - ("tasks", tuple_of(py, tasks)?), ("assets", tuple_of(py, assets)?), ("checks", tuple_of(py, checks)?), ( diff --git a/crates/sqlbuild-python-front/src/refactor/_helpers/python_strings.rs b/crates/sqlbuild-python-front/src/refactor/_helpers/python_strings.rs index bfbd057003..b458ff8a88 100644 --- a/crates/sqlbuild-python-front/src/refactor/_helpers/python_strings.rs +++ b/crates/sqlbuild-python-front/src/refactor/_helpers/python_strings.rs @@ -14,9 +14,8 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; /// The discovered Python declarations whose files a refactoring searches. -const PYTHON_DECLARATIONS: [&str; 8] = [ +const PYTHON_DECLARATIONS: [&str; 7] = [ "loader_functions", - "task_functions", "asset_functions", "check_functions", "hook_functions", diff --git a/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_node_graph.rs b/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_node_graph.rs index 32638fb9af..228875b32c 100644 --- a/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_node_graph.rs +++ b/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_node_graph.rs @@ -26,9 +26,8 @@ use sqlbuild_python_front::discovery::main::python_files::provider_usages::provi const NAME_KEY: &str = "name"; /// One discovered function collection, its node kind and whether it carries documentation. -pub(crate) const KINDS: [(&str, &str, bool); 4] = [ +pub(crate) const KINDS: [(&str, &str, bool); 3] = [ ("loader_functions", "LOADER", false), - ("task_functions", "TASK", true), ("asset_functions", "ASSET", true), ("check_functions", "CHECK", true), ]; @@ -41,7 +40,7 @@ pub(crate) struct Vocabulary<'py> { pub(crate) providers: Bound<'py, PyDict>, /// Each node's resolved `job(...)` and `tag(...)` dependencies; empty for the explicit graph. pub(crate) sets: Vec, - /// Each discovered node's kind, such as `task`, by name. + /// Each discovered node's kind, such as `asset`, by name. pub(crate) node_kinds: HashMap, } @@ -65,7 +64,7 @@ impl Vocabulary<'_> { ) } - /// `model 'orders'` or `task 'export_orders'`. + /// `model 'orders'` or `asset 'export_orders'`. fn end_label(&self, end: &GraphNode) -> String { match end { GraphNode::Sql((kind, name)) => format!("{kind} '{name}'"), @@ -174,10 +173,6 @@ fn kind_metadata<'py>( ], )?, ), - "TASK" => ( - "task", - metadata(RecordId::DiscoveredPythonTaskMetadata, &["retry"])?, - ), "ASSET" => ( "asset", metadata( diff --git a/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/set_dependencies.rs b/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/set_dependencies.rs index 30e2fb4665..95687625d2 100644 --- a/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/set_dependencies.rs +++ b/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/set_dependencies.rs @@ -7,9 +7,8 @@ use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; /// Discovered function collections and the node kind each holds. -const COLLECTIONS: [(&str, &str); 4] = [ +const COLLECTIONS: [(&str, &str); 3] = [ ("loader_functions", "loader"), - ("task_functions", "task"), ("asset_functions", "asset"), ("check_functions", "check"), ]; diff --git a/crates/sqlbuild-python-project/src/planner_selection/_helpers/unit_tests.rs b/crates/sqlbuild-python-project/src/planner_selection/_helpers/unit_tests.rs index 600f939727..7153b0742a 100644 --- a/crates/sqlbuild-python-project/src/planner_selection/_helpers/unit_tests.rs +++ b/crates/sqlbuild-python-project/src/planner_selection/_helpers/unit_tests.rs @@ -37,12 +37,7 @@ const RESOURCE_PATH_FILES: [&str; 4] = [ "sql_function_files", "python_function_files", ]; -const RESOURCE_NODES: [&str; 4] = [ - "loader_functions", - "task_functions", - "asset_functions", - "check_functions", -]; +const RESOURCE_NODES: [&str; 3] = ["loader_functions", "asset_functions", "check_functions"]; fn stem(path: &Bound<'_, PyAny>) -> PyResult { path_stem(path) diff --git a/crates/sqlbuild-python-project/src/references/_helpers/explicit_references.rs b/crates/sqlbuild-python-project/src/references/_helpers/explicit_references.rs index a5ebabaafb..9c5c5e70d2 100644 --- a/crates/sqlbuild-python-project/src/references/_helpers/explicit_references.rs +++ b/crates/sqlbuild-python-project/src/references/_helpers/explicit_references.rs @@ -40,11 +40,7 @@ const REFERENCE_MODELS_MODULE: &str = "sqlbuild.compiler.references.models"; const BUILTINS_MODULE: &str = "builtins"; const VALUE_PLACEHOLDER: &str = "__sqlbuild_python_value_"; const HARD_CODED_ALLOWED: &str = "hard-coded relation names"; -const NODE_OWNERS: [(&str, &str); 3] = [ - ("task", "task_functions"), - ("asset", "asset_functions"), - ("check", "check_functions"), -]; +const NODE_OWNERS: [(&str, &str); 2] = [("asset", "asset_functions"), ("check", "check_functions")]; const NODE_KIND: &str = "node"; const HOOK_KIND: &str = "hook"; const LOADER_KIND: &str = "loader"; @@ -150,7 +146,7 @@ pub(crate) fn explicit_reference_report<'py>( } } -/// Every task, asset, check, loader and hook function, as Python discovery lists them. +/// Every asset, check, loader and hook function, as Python discovery lists them. fn python_sql_owners<'py>( project: &Bound<'py, PyAny>, discovered_inputs: &Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs index 7186236739..1aa8760bea 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs @@ -134,7 +134,6 @@ fn run_context_facts( + count(&project, "seeds")? + count(&project, "functions")? + loaded_sources - + count(&discovered, "task_functions")? + count(&discovered, "asset_functions")?; let resolved: Bound<'_, PyAny> = describe_connection_warehouse( py, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs index f5e23ede68..09c67007f8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs @@ -1,4 +1,4 @@ -//! Ingress Python before the build and read-side tasks and assets dispatched as SQL completes. +//! Ingress Python before the build and read-side assets dispatched as SQL completes. use std::collections::{HashMap, HashSet}; use std::sync::Mutex; @@ -65,7 +65,7 @@ fn node_rows(results: &Bound<'_, PyAny>) -> PyResult> { Ok(rows) } -/// Write human-readable task and asset rows. +/// Write human-readable asset rows. pub(crate) fn write_python_node_results( stream: &Bound<'_, PyAny>, results: &Bound<'_, PyAny>, @@ -101,11 +101,11 @@ pub(crate) struct DirectPythonLifecycle { ingress_python_results: Py, ingress_load_results: Py, ingress_loader_names: Py, - /// Each round-trip loader's selected upstream tasks and assets, `(kind, name)`, and whether each is on its frontier. + /// Each round-trip loader's selected upstream assets, `(kind, name)`, and whether each is on its frontier. round_trip_gates: RoundTripGates, /// Each round-trip loader's source load key and the SQL keys it waits for. round_trip_waits: Vec<(ObjectKey, Vec)>, - /// Whether the build runs read-side tasks and assets; `--no-python` turns this off. + /// Whether the build runs read-side assets; `--no-python` turns this off. include_python: bool, } @@ -439,15 +439,13 @@ fn lifecycle_plan<'py>( }) } -fn task_asset_names( +fn asset_names( discovered: &Bound<'_, PyAny>, pipeline: &Bound<'_, PyAny>, ) -> PyResult> { let mut known: HashSet = HashSet::new(); - for kind in ["task_functions", "asset_functions"] { - for node in native_field(discovered, kind)?.try_iter()? { - known.insert(native_field(&node?, "name")?.extract()?); - } + for node in native_field(discovered, "asset_functions")?.try_iter()? { + known.insert(native_field(&node?, "name")?.extract()?); } let mut names: Vec = Vec::new(); for name in native_field(pipeline, "python_node_names")?.try_iter()? { @@ -473,7 +471,7 @@ pub(crate) fn prepare_direct_python_lifecycle<'py>( let plan_output: Bound<'py, PyAny> = native_field(pipeline, "plan_output")?; let include_python: bool = context.request_flag("include_python")?; let selected: Vec = if include_python { - task_asset_names(&discovered, pipeline)? + asset_names(&discovered, pipeline)? } else { Vec::new() }; @@ -722,7 +720,7 @@ impl SourceLoadGate { None if !lifecycle.include_python => { not_run_reason.get_or_insert_with(|| { format!( - "Upstream {kind} '{name}' did not run; --no-python skips read-side tasks and assets" + "Upstream {kind} '{name}' did not run; --no-python skips read-side assets" ) }); continue; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/definition_snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/definition_snapshot.rs index 10b753b5a2..476f060160 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/definition_snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/definition_snapshot.rs @@ -79,12 +79,8 @@ const RUNTIME_SETTINGS: [&str; 12] = [ ]; const PYTHON_SUFFIX: &str = ".py"; /// Discovered Python node collections, in `PYTHON_NODE_KINDS` order. -const PYTHON_NODE_COLLECTIONS: [&str; 4] = [ - "loader_functions", - "task_functions", - "asset_functions", - "check_functions", -]; +const PYTHON_NODE_COLLECTIONS: [&str; 3] = + ["loader_functions", "asset_functions", "check_functions"]; fn node(kind: &str, name: &str, aspects: Vec<(&str, String)>) -> NodeDefinition { NodeDefinition { @@ -433,7 +429,7 @@ fn python_files(directory: &Path) -> PyResult> { Ok(files) } -/// Loaders, tasks, assets, checks and materializations by their Python modules and imports. +/// Loaders, assets, checks and materializations by their Python modules and imports. fn python_nodes( discovered: &Bound<'_, PyAny>, fingerprints: &Fingerprinting, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/check/check_core.rs b/crates/sqlbuild-python/src/bindings/_helpers/check/check_core.rs index 04442d4156..fd11f33416 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/check/check_core.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/check/check_core.rs @@ -131,14 +131,12 @@ pub(crate) fn record_python_run_state_results<'py>( result_by_name.set_item(native_field(&result, "node_name")?, result)?; } let node_by_name: Bound<'py, PyDict> = PyDict::new(py); - for attribute in ["task_functions", "asset_functions"] { - for node in native_field(discovered_inputs, attribute)?.try_iter()? { - let node: Bound<'py, PyAny> = node?; - node_by_name.set_item( - native_field(&node, "name")?, - native_field(&node, "function")?, - )?; - } + for node in native_field(discovered_inputs, "asset_functions")?.try_iter()? { + let node: Bound<'py, PyAny> = node?; + node_by_name.set_item( + native_field(&node, "name")?, + native_field(&node, "function")?, + )?; } for (name, function) in node_by_name.iter() { if let Some(result) = result_by_name.get_item(name)? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot_providers.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot_providers.rs index b95a1a7b94..dbeb41cbae 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot_providers.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot_providers.rs @@ -25,9 +25,8 @@ enum RunsIn { } /// Discovered collections whose items inject providers, the kind each is published as, and where it runs. -const CONSUMERS: [(&str, &str, RunsIn); 8] = [ +const CONSUMERS: [(&str, &str, RunsIn); 7] = [ ("loader", "loader_functions", RunsIn::SourceLoader), - ("task", "task_functions", RunsIn::Itself), ("asset", "asset_functions", RunsIn::Itself), ("check", "check_functions", RunsIn::Itself), ("hook", "hook_functions", RunsIn::ModelHooks), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/python_tests.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/python_tests.rs index 5b2ab9f168..56c9b0945b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/python_tests.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/python_tests.rs @@ -35,12 +35,7 @@ pub(crate) const PYTHON_TESTS_INPUT_CODE: &str = "S000"; /// The project folder whose `python/`, `providers/` and `orchestration/` folders mirror the project's. const TESTS_FOLDER: &str = "tests"; /// The discovered Python node collections whose folders a selection's tests mirror. -const NODE_COLLECTIONS: [&str; 4] = [ - "loader_functions", - "task_functions", - "asset_functions", - "check_functions", -]; +const NODE_COLLECTIONS: [&str; 3] = ["loader_functions", "asset_functions", "check_functions"]; /// Lets tests import project and `tests.*` modules, and mirrored folders reuse module names. const IMPORT_MODE: &str = "--import-mode=importlib"; /// The JSON section reporting the Python tests a `sqb test` ran. diff --git a/crates/sqlbuild-rules/src/orchestration_rules/tests/test_named_selector_limit.rs b/crates/sqlbuild-rules/src/orchestration_rules/tests/test_named_selector_limit.rs index 699456e581..7142fbd6dd 100644 --- a/crates/sqlbuild-rules/src/orchestration_rules/tests/test_named_selector_limit.rs +++ b/crates/sqlbuild-rules/src/orchestration_rules/tests/test_named_selector_limit.rs @@ -40,7 +40,7 @@ fn given_job_and_sensor_selections_when_evaluating_then_more_named_nodes_than_th "orchestration/jobs/orders/nightly.toml", job( "orders_nightly_job", - "fct_orders,tag:orders task:refund_review source:raw_orders tag:finance", + "fct_orders,tag:orders asset:refund_review source:raw_orders tag:finance", ), ), ], diff --git a/crates/sqlbuild-rules/src/python_rules/constants.rs b/crates/sqlbuild-rules/src/python_rules/constants.rs index 0fd11007f0..4e40981814 100644 --- a/crates/sqlbuild-rules/src/python_rules/constants.rs +++ b/crates/sqlbuild-rules/src/python_rules/constants.rs @@ -52,7 +52,7 @@ pub(crate) const PYTHON_RULES: [PythonRuleSpec; 108] = [ enabled_by_default: true, options: &[], message: "node.py holds imports and exactly one node or factory declaration", - remediation: "Keep one @loader, @task, @asset or @factory function in node.py; move helpers to _helpers/, constants to constants.py and checks to checks.py.", + remediation: "Keep one @loader, @asset or @factory function in node.py; move helpers to _helpers/, constants to constants.py and checks to checks.py.", }, PythonRuleSpec { code: "SQBRPYTHON005", @@ -76,7 +76,7 @@ pub(crate) const PYTHON_RULES: [PythonRuleSpec; 108] = [ enabled_by_default: true, options: &[], message: "main.py holds imports and one public main() whose return annotation fits the node kind", - remediation: "Move other functions and classes into _helpers/ or a role file, and annotate main() with the result it returns: ctx.result(...) or ctx.skip(...) for a task or asset, and rows or a loader result for a loader.", + remediation: "Move other functions and classes into _helpers/ or a role file, and annotate main() with the result it returns: ctx.result(...) or ctx.skip(...) for an asset, and rows or a loader result for a loader.", }, PythonRuleSpec { code: "SQBRPYTHON007", @@ -135,7 +135,7 @@ pub(crate) const PYTHON_RULES: [PythonRuleSpec; 108] = [ applies: PythonFiles::Package, enabled_by_default: true, options: &[], - message: "@task, @asset and @check set description= without a docstring, and @loader sets neither", + message: "@asset and @check set description= without a docstring, and @loader sets neither", remediation: "Move the docstring text into description=; for a loader, describe the managed source in its YAML instead.", }, PythonRuleSpec { @@ -1395,14 +1395,8 @@ pub(crate) const NAMING_CONTRACTS: [(&str, &str); 10] = [ ("iter_*", "returns-iterator"), ]; /// Decorators whose top-level functions SQLBuild loads from `python/`, making their module an entry point. -pub(crate) const LOADED_DECORATORS: [&str; 6] = [ - "loader", - "task", - "asset", - "check", - "factory", - "audit_factory", -]; +pub(crate) const LOADED_DECORATORS: [&str; 5] = + ["loader", "asset", "check", "factory", "audit_factory"]; /// Fensu codes whose own remediation names settings SQLBuild does not have, so the catalogue's is used. pub(crate) const CATALOGUE_REMEDIATION_CODES: [&str; 1] = [fensu::DEAD_DEFINITION_CODE]; /// How Fensu's observation index names the project directory itself. @@ -1412,11 +1406,9 @@ pub(crate) const ENTRYPOINT_KIND: &str = "sqlbuild"; /// Fensu's family of naming-contract rules, which read the naming contracts. pub(crate) const NAMING_FAMILY: &str = "naming"; /// Folders named after node kinds, which belong inside a domain rather than directly under `python/`. -pub(crate) const NODE_KIND_DIRECTORIES: [&str; 10] = [ +pub(crate) const NODE_KIND_DIRECTORIES: [&str; 8] = [ "loaders", "loader", - "tasks", - "task", "assets", "asset", "checks", @@ -1623,11 +1615,8 @@ pub(crate) const CONTEXT_PARAMETER: &str = "ctx"; pub(crate) const DESCRIPTION_KEYWORD: &str = "description"; pub(crate) const NAME_KEYWORD: &str = "name"; /// Node decorators and the context type each node's `ctx` is annotated with. -pub(crate) const NODE_KINDS: [(&str, &str); 3] = [ - ("loader", "LoaderContext"), - ("task", "TaskContext"), - ("asset", "AssetContext"), -]; +pub(crate) const NODE_KINDS: [(&str, &str); 2] = + [("loader", "LoaderContext"), ("asset", "AssetContext")]; /// The check decorator and its context type. pub(crate) const CHECK_KIND: (&str, &str) = ("check", "CheckContext"); /// Decorators that generate nodes or checks. @@ -1638,8 +1627,7 @@ pub(crate) const DEFERRED_CODE_MARKERS: [&str; 2] = ["lambda", "__main__"]; /// The loader decorator, which takes no description because its managed source has one. pub(crate) const LOADER_DECORATOR: &str = "loader"; /// Return annotations `main()` may use, by node decorator. -pub(crate) const MAIN_RESULTS: [(&str, &[&str]); 3] = [ - ("task", &["PythonNodeResult", "PythonNodeSkipResult"]), +pub(crate) const MAIN_RESULTS: [(&str, &[&str]); 2] = [ ("asset", &["PythonNodeResult", "PythonNodeSkipResult"]), ( "loader", diff --git a/crates/sqlbuild-rules/src/python_rules/tests/helpers.rs b/crates/sqlbuild-rules/src/python_rules/tests/helpers.rs index 6f943df76c..07d4bf4333 100644 --- a/crates/sqlbuild-rules/src/python_rules/tests/helpers.rs +++ b/crates/sqlbuild-rules/src/python_rules/tests/helpers.rs @@ -232,12 +232,12 @@ fn unit_files(unit: usize, upstream: &str, arguments: &str) -> [(String, String) ( format!("{dir}/node.py"), format!( - "{upstream}from {module}.main import main\n\n\n@task({arguments})\ndef node_{unit}(ctx: TaskContext) -> None:\n return main(ctx=ctx)\n" + "{upstream}from {module}.main import main\n\n\n@asset({arguments})\ndef node_{unit}(ctx: AssetContext) -> None:\n return main(ctx=ctx)\n" ), ), ( format!("{dir}/main.py"), - "from sqlbuild import PythonNodeResult\n\nfrom python.utils.text.join import joined\n\n\ndef main(ctx: TaskContext) -> PythonNodeResult:\n return ctx.result(joined(\"\"))\n".to_owned(), + "from sqlbuild import PythonNodeResult\n\nfrom python.utils.text.join import joined\n\n\ndef main(ctx: AssetContext) -> PythonNodeResult:\n return ctx.result(joined(\"\"))\n".to_owned(), ), ( format!("{dir}/checks.py"), diff --git a/crates/sqlbuild-rules/src/python_rules/tests/test_compile_purity_rules.rs b/crates/sqlbuild-rules/src/python_rules/tests/test_compile_purity_rules.rs index b9cbbd3e96..772582d4fd 100644 --- a/crates/sqlbuild-rules/src/python_rules/tests/test_compile_purity_rules.rs +++ b/crates/sqlbuild-rules/src/python_rules/tests/test_compile_purity_rules.rs @@ -4,7 +4,7 @@ use serde_json::json; use crate::python_rules::tests::helpers::{owned_faults, python_rule_faults}; use crate::python_rules::tests::test_types::{PythonRunFailureTestCase, PythonRunTestCase}; -const PURE_FACTORY: &str = r#"from sqlbuild.tasks import factory, task +const PURE_FACTORY: &str = r#"from sqlbuild.assets import factory, asset from python.inventory.snapshots.constants import WAREHOUSE_SPECS @@ -14,7 +14,7 @@ def warehouse_snapshots(): nodes = [] for spec in WAREHOUSE_SPECS: - @task(name=spec) + @asset(name=spec) def snapshot(ctx): return requests.get(spec) @@ -24,14 +24,14 @@ def warehouse_snapshots(): const RUNTIME_ONLY: &str = r#"import psycopg import requests -from sqlbuild.tasks import task +from sqlbuild.assets import asset def main(ctx): return psycopg.connect("dsn").execute("SELECT 1") -@task +@asset def export(ctx): return requests.post("https://example.com/orders") @@ -42,7 +42,7 @@ class CarrierApi: "#; const QUERYING_FACTORY: &str = r#"import psycopg -from sqlbuild.tasks import factory +from sqlbuild.assets import factory from ._helpers.spec_loading import load_specs diff --git a/crates/sqlbuild-rules/src/python_rules/tests/test_evaluate_python_rules.rs b/crates/sqlbuild-rules/src/python_rules/tests/test_evaluate_python_rules.rs index 165a83d120..83398c9e41 100644 --- a/crates/sqlbuild-rules/src/python_rules/tests/test_evaluate_python_rules.rs +++ b/crates/sqlbuild-rules/src/python_rules/tests/test_evaluate_python_rules.rs @@ -216,7 +216,7 @@ fn given_project_python_when_evaluating_python_rules_then_reports_fensu_findings ), ( "python/orders/node.py", - "from python.orders import _tables\n\n\n@task\ndef count() -> int:\n return _tables.TABLES\n", + "from python.orders import _tables\n\n\n@asset\ndef count() -> int:\n return _tables.TABLES\n", ), ], config: json!({ "select": ["SQBRPYTHON306"] }), diff --git a/crates/sqlbuild-rules/src/python_rules/tests/test_node_units.rs b/crates/sqlbuild-rules/src/python_rules/tests/test_node_units.rs index 46a5c1d3c8..f2fe527a5d 100644 --- a/crates/sqlbuild-rules/src/python_rules/tests/test_node_units.rs +++ b/crates/sqlbuild-rules/src/python_rules/tests/test_node_units.rs @@ -20,15 +20,15 @@ const UNIT_RULES: [&str; 10] = [ const DAILY_TASK: &str = r#"from .main import main -@task(description="Daily.") -def daily(ctx: TaskContext) -> None: +@asset(description="Daily.") +def daily(ctx: AssetContext) -> None: return main(ctx=ctx) "#; const TASK_MAIN: &str = r#"from sqlbuild import PythonNodeResult -def main(ctx: TaskContext) -> PythonNodeResult: +def main(ctx: AssetContext) -> PythonNodeResult: return ctx.result() "#; @@ -56,8 +56,8 @@ fn given_node_units_when_evaluating_unit_rules_then_reports_each_broken_conventi from .main import main -@task(description="Daily.") -def daily(ctx: TaskContext, *, store: Store) -> None: +@asset(description="Daily.") +def daily(ctx: AssetContext, *, store: Store) -> None: return main(ctx=ctx, store=store) "#, ), @@ -74,7 +74,7 @@ from python.utils.text import clean from ._helpers.rows import rows -def main(ctx: TaskContext, store: Store) -> PythonNodeResult | PythonNodeSkipResult: +def main(ctx: AssetContext, store: Store) -> PythonNodeResult | PythonNodeSkipResult: return ctx.result(rows(today(), clean(), LIMIT, store)) "#, ), @@ -131,8 +131,8 @@ from .main import main def region_exports(): for region in REGIONS: - @task(name=f"export_{region}", description="Export one region.") - def export(ctx: TaskContext) -> None: + @asset(name=f"export_{region}", description="Export one region.") + def export(ctx: AssetContext) -> None: return main(ctx=ctx, region=region) @check(name=f"export_{region}_ok", depends_on=[export], description="Exported.") @@ -146,7 +146,7 @@ def region_exports(): ), ( "python/orders/regions/main.py", - "def main(ctx: TaskContext, region: str) -> PythonNodeResult:\n return ctx.result()\n", + "def main(ctx: AssetContext, region: str) -> PythonNodeResult:\n return ctx.result()\n", ), ], config: json!({ "select": UNIT_RULES }), @@ -184,13 +184,13 @@ def helper() -> int: return 1 -@task(description="Daily.") -def daily(ctx: TaskContext) -> None: +@asset(description="Daily.") +def daily(ctx: AssetContext) -> None: return main(ctx=ctx) -@task(description="Extra.") -def extra(ctx: TaskContext) -> None: +@asset(description="Extra.") +def extra(ctx: AssetContext) -> None: return main(ctx=ctx) "#, ), @@ -213,24 +213,24 @@ def extra(ctx: TaskContext) -> None: from python.orders.weekly.main import main as weekly_main -@task(description="A.") -def by_keyword(ctx: TaskContext) -> None: +@asset(description="A.") +def by_keyword(ctx: AssetContext) -> None: return main(ctx=ctx) -@task(description="B.") -def positional(ctx: TaskContext) -> None: +@asset(description="B.") +def positional(ctx: AssetContext) -> None: return main(ctx) -@task(description="C.") -def computes(ctx: TaskContext) -> None: +@asset(description="C.") +def computes(ctx: AssetContext) -> None: rows = main(ctx=ctx) return rows -@task(description="D.") -def unpacked(ctx: TaskContext) -> None: +@asset(description="D.") +def unpacked(ctx: AssetContext) -> None: return main(**ctx) "#, ), @@ -240,8 +240,8 @@ def unpacked(ctx: TaskContext) -> None: r#"from python.orders.daily.main import main -@task(description="W.") -def weekly(ctx: TaskContext) -> None: +@asset(description="W.") +def weekly(ctx: AssetContext) -> None: return main(ctx=ctx) "#, ), @@ -266,7 +266,7 @@ def weekly(ctx: TaskContext) -> None: LIMIT = 3 -def main(ctx: TaskContext) -> list: +def main(ctx: AssetContext) -> list: return [] @@ -309,23 +309,23 @@ def helper() -> None: r#"from .main import main -@task(description="A.") -def no_ctx(context: TaskContext) -> None: +@asset(description="A.") +def no_ctx(context: AssetContext) -> None: return main(ctx=context) -@task(description="B.") +@asset(description="B.") def wrong_type(ctx: AssetContext, *, store: Store) -> None: return main(ctx=ctx, store=store) -@task(description="C.") -def loose(ctx: TaskContext, *args, store, **kwargs) -> None: +@asset(description="C.") +def loose(ctx: AssetContext, *args, store, **kwargs) -> None: return main(ctx=ctx) -@task(description="D.") -def positional_only(ctx: TaskContext, /) -> None: +@asset(description="D.") +def positional_only(ctx: AssetContext, /) -> None: return main(ctx=ctx) "#, ), @@ -335,7 +335,7 @@ def positional_only(ctx: TaskContext, /) -> None: @check(depends_on=[no_ctx], description="Q.") -def quality(ctx: TaskContext) -> bool: +def quality(ctx: AssetContext) -> bool: return True "#, ), @@ -380,7 +380,7 @@ from python.billing.invoices.node import invoices from providers.store.client import Client -def main(ctx: TaskContext) -> PythonNodeResult: +def main(ctx: AssetContext) -> PythonNodeResult: return invoices "#, ), @@ -390,8 +390,8 @@ def main(ctx: TaskContext) -> PythonNodeResult: from .main import main -@task(depends_on=[daily], description="W.") -def weekly(ctx: TaskContext) -> None: +@asset(depends_on=[daily], description="W.") +def weekly(ctx: AssetContext) -> None: return main(ctx=ctx) "#, ), @@ -417,7 +417,7 @@ def weekly(ctx: TaskContext) -> None: ("python/orders/daily/node.py", DAILY_TASK), ( "python/orders/weekly/node.py", - "@task\ndef weekly(ctx):\n pass\n", + "@asset\ndef weekly(ctx):\n pass\n", ), ( "python/orders/daily/checks.py", @@ -471,8 +471,8 @@ from .main import main def market_exports(): for market in MARKETS: - @task(description="M.") - def export(ctx: TaskContext) -> None: + @asset(description="M.") + def export(ctx: AssetContext) -> None: return main(ctx=ctx) "#, ), @@ -483,8 +483,8 @@ def market_exports(): @factory def once(): - @task(name="once", description="O.") - def run(ctx: TaskContext) -> None: + @asset(name="once", description="O.") + def run(ctx: AssetContext) -> None: return main(ctx=ctx) "#, ), @@ -504,8 +504,8 @@ def once(): r#"from .main import main -@task(description="A.") -def described(ctx: TaskContext) -> None: +@asset(description="A.") +def described(ctx: AssetContext) -> None: return main(ctx=ctx) @@ -560,7 +560,7 @@ def raw_orders_present(ctx: CheckContext) -> bool: ), ( "python/orders/daily/main.py", - "@task(description=\"X.\")\ndef stray(ctx: TaskContext) -> None:\n return None\n", + "@asset(description=\"X.\")\ndef stray(ctx: AssetContext) -> None:\n return None\n", ), ( "python/orders/daily/node.py", @@ -572,8 +572,8 @@ from .main import main def tables(): for table in TABLES: - @task(name=table, description="T.") - def load(ctx: TaskContext) -> None: + @asset(name=table, description="T.") + def load(ctx: AssetContext) -> None: return main(ctx=ctx) @check(name=f"{table}_ok", depends_on=[load], description="C.") @@ -591,7 +591,7 @@ def tables(): ), ( "python/utils/flat.py", - "@task\ndef util_task(ctx):\n return 1\n", + "@asset\ndef util_task(ctx):\n return 1\n", ), ], config: json!({ "select": ["SQBRPYTHON012"] }), @@ -658,14 +658,14 @@ def region_exports(): for region, spec in SPECS.items(): for column in spec.columns: - @task(name=f"export_{region}_{column}", description="E.") - def export(ctx: TaskContext) -> None: + @asset(name=f"export_{region}_{column}", description="E.") + def export(ctx: AssetContext) -> None: return main(ctx=ctx, region=region) for table in constants.TABLES: - @task(name=table, description="T.") - def load(ctx: TaskContext) -> None: + @asset(name=table, description="T.") + def load(ctx: AssetContext) -> None: return main(ctx=ctx) "#, ), @@ -681,8 +681,8 @@ from .main import main def ranked_exports(): for rank, region in enumerate(sorted(SPECS)): - @task(name=f"export_{rank}_{region}", description="E.") - def export(ctx: TaskContext) -> None: + @asset(name=f"export_{rank}_{region}", description="E.") + def export(ctx: AssetContext) -> None: return main(ctx=ctx, region=region) "#, ), @@ -706,8 +706,8 @@ if TYPE_CHECKING: from python.orders.weekly.main import main as weekly_main -@task(description="Daily.") -def daily(ctx: TaskContext, *, store: "Store") -> None: +@asset(description="Daily.") +def daily(ctx: AssetContext, *, store: "Store") -> None: return main(ctx=ctx, store=store) "#, ), @@ -730,8 +730,8 @@ def daily(ctx: TaskContext, *, store: "Store") -> None: from .main import main -@task(depends_on=[python.orders.daily.node.daily], description="W.") -def weekly(ctx: TaskContext) -> None: +@asset(depends_on=[python.orders.daily.node.daily], description="W.") +def weekly(ctx: AssetContext) -> None: return main(ctx=ctx) "#, ), @@ -755,7 +755,7 @@ def weekly(ctx: TaskContext) -> None: ("python/orders/daily/node.py", DAILY_TASK), ( "python/orders/daily/main.py", - "def main(ctx: TaskContext) -> \"PythonNodeResult | PythonNodeSkipResult\":\n return ctx.result()\n", + "def main(ctx: AssetContext) -> \"PythonNodeResult | PythonNodeSkipResult\":\n return ctx.result()\n", ), ( "python/orders/raw/node.py", @@ -774,7 +774,7 @@ def weekly(ctx: TaskContext) -> None: files: &[ ( "python/orders/daily/node.py", - "from . import main\n\n\n@task(description=\"D.\")\ndef daily(ctx: TaskContext) -> None:\n return main(ctx=ctx)\n", + "from . import main\n\n\n@asset(description=\"D.\")\ndef daily(ctx: AssetContext) -> None:\n return main(ctx=ctx)\n", ), ("python/orders/daily/main.py", TASK_MAIN), ], diff --git a/crates/sqlbuild-rules/src/python_rules/tests/test_python_test_rules.rs b/crates/sqlbuild-rules/src/python_rules/tests/test_python_test_rules.rs index e45d9ae02d..37bfeef3b5 100644 --- a/crates/sqlbuild-rules/src/python_rules/tests/test_python_test_rules.rs +++ b/crates/sqlbuild-rules/src/python_rules/tests/test_python_test_rules.rs @@ -7,22 +7,22 @@ use crate::python_rules::tests::test_types::PythonRunTestCase; const RUNTIME_FILES: [(&str, &str); 5] = [ ( "python/orders/storefront_import/node.py", - r#"from sqlbuild.tasks import TaskContext, task + r#"from sqlbuild.assets import AssetContext, asset from python.orders.storefront_import.main import main -@task(description="Import storefront orders.") -def storefront_import(ctx: TaskContext) -> None: +@asset(description="Import storefront orders.") +def storefront_import(ctx: AssetContext) -> None: return main(ctx) "#, ), ( "python/orders/storefront_import/main.py", - r#"from sqlbuild.tasks import PythonNodeResult, TaskContext + r#"from sqlbuild.assets import PythonNodeResult, AssetContext -def main(ctx: TaskContext) -> PythonNodeResult: +def main(ctx: AssetContext) -> PythonNodeResult: return PythonNodeResult() "#, ), diff --git a/scripts/compiler_differential/_helpers/analysis_corpus/failure_cases.py b/scripts/compiler_differential/_helpers/analysis_corpus/failure_cases.py index e42b0a6ff3..e2e61c1072 100644 --- a/scripts/compiler_differential/_helpers/analysis_corpus/failure_cases.py +++ b/scripts/compiler_differential/_helpers/analysis_corpus/failure_cases.py @@ -467,8 +467,8 @@ def _target_cases() -> tuple[FailureCase, ...]: ) + '\n[defaults]\nschema = "mart"\n\n[targets.dev]\nschema = "preserve"\n', "python/tasks/announce.py": ( - "from sqlbuild.tasks import task\n\n\n" - "@task\n" + "from sqlbuild.assets import asset\n\n\n" + "@asset\n" "def announce_orders(ctx):\n" ' """Announce the loaded orders."""\n' " return ctx.result(payload={'announced': True})\n" diff --git a/scripts/compiler_differential/_helpers/corpus/discovery_failure_cases.py b/scripts/compiler_differential/_helpers/corpus/discovery_failure_cases.py index 5d62b97bb1..74c67698f4 100644 --- a/scripts/compiler_differential/_helpers/corpus/discovery_failure_cases.py +++ b/scripts/compiler_differential/_helpers/corpus/discovery_failure_cases.py @@ -18,8 +18,8 @@ from scripts.compiler_differential.models import FailureCase _TASK_MODULE: str = ( - "from sqlbuild.tasks import task\n\n\n" - "@task\ndef export_orders(ctx):\n" + "from sqlbuild.assets import asset\n\n\n" + "@asset\ndef export_orders(ctx):\n" ' """Export orders."""\n' " return None\n" ) diff --git a/scripts/compiler_differential/_helpers/corpus/failure_cases.py b/scripts/compiler_differential/_helpers/corpus/failure_cases.py index c0922abdc4..3789946a37 100644 --- a/scripts/compiler_differential/_helpers/corpus/failure_cases.py +++ b/scripts/compiler_differential/_helpers/corpus/failure_cases.py @@ -164,8 +164,8 @@ def _compile_failure_cases() -> tuple[FailureCase, ...]: expected_code="P010", files={ "python/tasks/exports.py": ( - "from sqlbuild.tasks import TaskContext, task\n\n\n" - "@task\ndef export_orders(ctx: TaskContext):\n return None\n" + "from sqlbuild.assets import AssetContext, asset\n\n\n" + "@asset\ndef export_orders(ctx: AssetContext):\n return None\n" ) }, ), @@ -174,8 +174,8 @@ def _compile_failure_cases() -> tuple[FailureCase, ...]: expected_code="P027", files={ "python/tasks/exports.py": ( - "from sqlbuild.tasks import TaskContext, task\n\n\n" - "@task\ndef export_orders(ctx: TaskContext):\n" + "from sqlbuild.assets import AssetContext, asset\n\n\n" + "@asset\ndef export_orders(ctx: AssetContext):\n" ' """' + "\n\n ".join(f"Orders paragraph {index}." for index in range(6)) + '"""\n return None\n' diff --git a/scripts/compiler_differential/_helpers/corpus/render_failure_cases.py b/scripts/compiler_differential/_helpers/corpus/render_failure_cases.py index 7aa0034670..05e5cd54c0 100644 --- a/scripts/compiler_differential/_helpers/corpus/render_failure_cases.py +++ b/scripts/compiler_differential/_helpers/corpus/render_failure_cases.py @@ -508,8 +508,8 @@ def _reference_cases() -> tuple[FailureCase, ...]: expected_code="P008", files={ "python/tasks/exports.py": ( - "from sqlbuild.tasks import task\n\n\n" - '@task\ndef export_orders(ctx):\n """Export orders."""\n' + "from sqlbuild.assets import asset\n\n\n" + '@asset\ndef export_orders(ctx):\n """Export orders."""\n' ' return ctx.query("SELECT * FROM stg_orders")\n' ) }, diff --git a/scripts/compiler_differential/classes/analysis_features.py b/scripts/compiler_differential/classes/analysis_features.py index a3cd3a6c4e..f56b28b908 100644 --- a/scripts/compiler_differential/classes/analysis_features.py +++ b/scripts/compiler_differential/classes/analysis_features.py @@ -325,9 +325,9 @@ def _analysis_opt_out(self) -> None: def _python_sql(self) -> None: self._features.add("python_sql") self._files["python/tasks/flag_exports.py"] = ( - "from sqlbuild.tasks import TaskContext, task\n\n\n" - "@task\n" - "def count_flagged_events(ctx: TaskContext):\n" + "from sqlbuild.assets import AssetContext, asset\n\n\n" + "@asset\n" + "def count_flagged_events(ctx: AssetContext):\n" ' """Count externally flagged events."""\n' ' ctx.query("SELECT COUNT(*) FROM main.external_event_flags")\n' " return ctx.result()\n" diff --git a/scripts/compiler_differential/classes/discovery_features.py b/scripts/compiler_differential/classes/discovery_features.py index aae00e5bbc..e26eb7c34a 100644 --- a/scripts/compiler_differential/classes/discovery_features.py +++ b/scripts/compiler_differential/classes/discovery_features.py @@ -263,8 +263,8 @@ def _provider(self) -> None: " batch_size: int = 50\n" ) self._files["python/tasks/provided.py"] = ( - "from sqlbuild.tasks import task\n\n\n" - "@task\n" + "from sqlbuild.assets import asset\n\n\n" + "@asset\n" "def provided_export(ctx, export_settings):\n" ' """Export with provider settings."""\n' ' return ctx.result(payload={"region": export_settings.region})\n' diff --git a/scripts/compiler_differential/classes/project_builder.py b/scripts/compiler_differential/classes/project_builder.py index 3e108850b7..4b61b1b6ef 100644 --- a/scripts/compiler_differential/classes/project_builder.py +++ b/scripts/compiler_differential/classes/project_builder.py @@ -563,13 +563,13 @@ def _write_python_nodes(self) -> None: return self._features.add("python_nodes") self._files["python/tasks/exports.py"] = ( - "from sqlbuild.tasks import TaskContext, task\n\n\n" - "@task\n" - "def collect_exports(ctx: TaskContext):\n" + "from sqlbuild.assets import AssetContext, asset\n\n\n" + "@asset\n" + "def collect_exports(ctx: AssetContext):\n" ' """Collect generated exports."""\n' ' return ctx.result(payload={"rows": 3})\n\n\n' - "@task(depends_on=collect_exports)\n" - "def publish_exports(ctx: TaskContext):\n" + "@asset(depends_on=collect_exports)\n" + "def publish_exports(ctx: AssetContext):\n" ' """Publish generated exports."""\n' ' return ctx.result(metadata={"published": True})\n' ) diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index fe94cec490..1122dbaef5 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -9091,7 +9091,6 @@ class SelectorKind(StrEnum): NAME = "name" SEED = "seed" SOURCE = "source" - TASK = "task" ASSET = "asset" LOADER = "loader" CHECK = "check" @@ -10569,15 +10568,6 @@ class DiscoveredPythonLoaderMetadata(NativeRecord): contract: str | None = ..., ) -> None: ... -class DiscoveredPythonTaskMetadata(NativeRecord): - """Task-specific metadata carried by an internal Python task node.""" - - retry: RetryPolicy | None - def __init__( - self, - retry: RetryPolicy | None = ..., - ) -> None: ... - class DiscoveredPythonAssetMetadata(NativeRecord): """Asset-specific metadata carried by an internal Python asset node.""" @@ -13068,7 +13058,6 @@ class DiscoveredPythonNode(NativeRecord): meta: dict[str, object] | None provider_usages: tuple[DiscoveredProviderUsage, ...] loader: DiscoveredPythonLoaderMetadata | None - task: DiscoveredPythonTaskMetadata | None asset: DiscoveredPythonAssetMetadata | None check: DiscoveredPythonCheckMetadata | None identity: PythonNodeIdentity | None @@ -13087,7 +13076,6 @@ class DiscoveredPythonNode(NativeRecord): meta: dict[str, object] | None = ..., provider_usages: tuple[DiscoveredProviderUsage, ...] = ..., loader: DiscoveredPythonLoaderMetadata | None = ..., - task: DiscoveredPythonTaskMetadata | None = ..., asset: DiscoveredPythonAssetMetadata | None = ..., check: DiscoveredPythonCheckMetadata | None = ..., identity: PythonNodeIdentity | None = ..., @@ -14921,37 +14909,6 @@ class DiscoveredLoaderFunction(NativeRecord): identity: PythonNodeIdentity | None = ..., ) -> None: ... -class DiscoveredTaskFunction(NativeRecord): - """A discovered project task function.""" - - file_path: Path - relative_path: Path - name: str - function: Callable[..., object] - depends_on: tuple[Callable[..., object] | SqlResourceRef, ...] - tags: tuple[str, ...] - group: str | None - description: str | None - meta: dict[str, object] | None - retry: RetryPolicy | None - provider_usages: tuple[DiscoveredProviderUsage, ...] - identity: PythonNodeIdentity | None - def __init__( - self, - file_path: Path, - relative_path: Path, - name: str, - function: Callable[..., object], - depends_on: tuple[Callable[..., object] | SqlResourceRef, ...] = ..., - tags: tuple[str, ...] = ..., - group: str | None = ..., - description: str | None = ..., - meta: dict[str, object] | None = ..., - retry: RetryPolicy | None = ..., - provider_usages: tuple[DiscoveredProviderUsage, ...] = ..., - identity: PythonNodeIdentity | None = ..., - ) -> None: ... - class DiscoveredAssetFunction(NativeRecord): """A discovered project asset function.""" @@ -15163,14 +15120,12 @@ class DiscoveredPythonNodeFunctions(NativeRecord): """Discovered project Python DAG node functions grouped by kind.""" loaders: tuple[DiscoveredLoaderFunction, ...] - tasks: tuple[DiscoveredTaskFunction, ...] assets: tuple[DiscoveredAssetFunction, ...] checks: tuple[DiscoveredCheckFunction, ...] audit_factories: tuple[DiscoveredAuditFactory, ...] def __init__( self, loaders: tuple[DiscoveredLoaderFunction, ...] = ..., - tasks: tuple[DiscoveredTaskFunction, ...] = ..., assets: tuple[DiscoveredAssetFunction, ...] = ..., checks: tuple[DiscoveredCheckFunction, ...] = ..., audit_factories: tuple[DiscoveredAuditFactory, ...] = ..., @@ -15198,7 +15153,6 @@ class DiscoveredProjectInputs(NativeRecord): macro_files: tuple[DiscoveredMacroFile, ...] materialization_files: tuple[DiscoveredMaterializationFile, ...] loader_functions: tuple[DiscoveredLoaderFunction, ...] - task_functions: tuple[DiscoveredTaskFunction, ...] asset_functions: tuple[DiscoveredAssetFunction, ...] check_functions: tuple[DiscoveredCheckFunction, ...] audit_factories: tuple[DiscoveredAuditFactory, ...] @@ -15229,7 +15183,6 @@ class DiscoveredProjectInputs(NativeRecord): macro_files: tuple[DiscoveredMacroFile, ...] = ..., materialization_files: tuple[DiscoveredMaterializationFile, ...] = ..., loader_functions: tuple[DiscoveredLoaderFunction, ...] = ..., - task_functions: tuple[DiscoveredTaskFunction, ...] = ..., asset_functions: tuple[DiscoveredAssetFunction, ...] = ..., check_functions: tuple[DiscoveredCheckFunction, ...] = ..., audit_factories: tuple[DiscoveredAuditFactory, ...] = ..., diff --git a/src/sqlbuild/_native_host.py b/src/sqlbuild/_native_host.py index f215c77928..9ebabcdc3d 100644 --- a/src/sqlbuild/_native_host.py +++ b/src/sqlbuild/_native_host.py @@ -276,10 +276,6 @@ "sqlbuild.python_nodes.main.read_loader_definition", "read_loader_definition", ), - "read_task_definition": ( - "sqlbuild.python_nodes.main.read_task_definition", - "read_task_definition", - ), "AuditCase": ("sqlbuild.python_nodes.models", "AuditCase"), "RELATION_RENDERER": ("sqlbuild.python_nodes.models", "RELATION_RENDERER"), "SqlResourceRef": ("sqlbuild.python_nodes.models", "SqlResourceRef"), diff --git a/src/sqlbuild/_native_host.pyi b/src/sqlbuild/_native_host.pyi index 39099b17ac..5e877cd3b0 100644 --- a/src/sqlbuild/_native_host.pyi +++ b/src/sqlbuild/_native_host.pyi @@ -463,9 +463,6 @@ from sqlbuild.python_nodes.main.read_hook_definition import ( from sqlbuild.python_nodes.main.read_loader_definition import ( read_loader_definition as read_loader_definition, ) -from sqlbuild.python_nodes.main.read_task_definition import ( - read_task_definition as read_task_definition, -) from sqlbuild.python_nodes.models import RELATION_RENDERER as RELATION_RENDERER from sqlbuild.python_nodes.models import AuditCase as AuditCase from sqlbuild.python_nodes.models import SqlResourceRef as SqlResourceRef diff --git a/src/sqlbuild/compiler/compile/_helpers/diagnostics/python_resources.py b/src/sqlbuild/compiler/compile/_helpers/diagnostics/python_resources.py index c550729a37..c01c388f40 100644 --- a/src/sqlbuild/compiler/compile/_helpers/diagnostics/python_resources.py +++ b/src/sqlbuild/compiler/compile/_helpers/diagnostics/python_resources.py @@ -19,7 +19,7 @@ def python_resource_rows( *, discovered_inputs: DiscoveredProjectInputs, source_inputs: tuple[SourceInput, ...] ) -> list[PythonResourceRow]: - """User Python loaders, tasks, assets, checks and hooks, then providers, in report order.""" + """User Python loaders, assets, checks and hooks, then providers, in report order.""" return [ *_python_nodes(discovered_inputs=discovered_inputs, source_inputs=source_inputs), @@ -49,7 +49,6 @@ def _python_nodes( function=loader.function, ) for kind, functions in ( - ("task", discovered_inputs.task_functions), ("asset", discovered_inputs.asset_functions), ("check", discovered_inputs.check_functions), ("hook", discovered_inputs.hook_functions), diff --git a/src/sqlbuild/compiler/dag/types.py b/src/sqlbuild/compiler/dag/types.py index 6b7659b424..c60bdeab6f 100644 --- a/src/sqlbuild/compiler/dag/types.py +++ b/src/sqlbuild/compiler/dag/types.py @@ -14,7 +14,6 @@ class NodeKind(StrEnum): UDF = "udf" TABLE_FN = "table_fn" MODEL = "model" - TASK = "task" ASSET = "asset" CHECK = "check" SQL_TEST = "sql_test" diff --git a/src/sqlbuild/compiler/discovery/_helpers/validation/discovery.py b/src/sqlbuild/compiler/discovery/_helpers/validation/discovery.py index 47a53cd8b3..940479ec1e 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/validation/discovery.py +++ b/src/sqlbuild/compiler/discovery/_helpers/validation/discovery.py @@ -11,20 +11,13 @@ DiscoveredCheckFunction, DiscoveredLoaderFunction, DiscoveredProjectInputs, - DiscoveredTaskFunction, ) from sqlbuild.python_nodes.main.read_asset_definition import read_asset_definition from sqlbuild.python_nodes.main.read_check_definition import read_check_definition from sqlbuild.python_nodes.main.read_loader_definition import read_loader_definition -from sqlbuild.python_nodes.main.read_task_definition import read_task_definition from sqlbuild.python_nodes.models import NodeSetRef, SqlResourceRef -type _PythonNode = ( - DiscoveredLoaderFunction - | DiscoveredTaskFunction - | DiscoveredAssetFunction - | DiscoveredCheckFunction -) +type _PythonNode = DiscoveredLoaderFunction | DiscoveredAssetFunction | DiscoveredCheckFunction class _NodeDefinition(Protocol): @@ -34,7 +27,6 @@ def name(self) -> str: ... _DEFINITION_READERS: tuple[Callable[[Callable[..., object]], _NodeDefinition | None], ...] = ( read_loader_definition, - read_task_definition, read_asset_definition, read_check_definition, ) @@ -49,7 +41,6 @@ def validate_discovered_inputs(discovered_inputs: DiscoveredProjectInputs) -> No def _nodes(inputs: DiscoveredProjectInputs) -> list[dict[str, object]]: nodes: tuple[tuple[str, _PythonNode], ...] = ( *(("loader", node) for node in inputs.loader_functions), - *(("task", node) for node in inputs.task_functions), *(("asset", node) for node in inputs.asset_functions), *(("check", node) for node in inputs.check_functions), ) diff --git a/src/sqlbuild/compiler/python_nodes/_helpers/user_inventory.py b/src/sqlbuild/compiler/python_nodes/_helpers/user_inventory.py index 44517a4562..afa2591dde 100644 --- a/src/sqlbuild/compiler/python_nodes/_helpers/user_inventory.py +++ b/src/sqlbuild/compiler/python_nodes/_helpers/user_inventory.py @@ -11,12 +11,10 @@ from sqlbuild.python_nodes.main.read_asset_definition import read_asset_definition from sqlbuild.python_nodes.main.read_check_definition import read_check_definition from sqlbuild.python_nodes.main.read_loader_definition import read_loader_definition -from sqlbuild.python_nodes.main.read_task_definition import read_task_definition from sqlbuild.python_nodes.models import ( AssetDefinition, CheckDefinition, LoaderDefinition, - TaskDefinition, ) @@ -53,11 +51,6 @@ def python_node_dependency_key(dependency: object) -> object | tuple[str, str]: ) if loader_definition is not None: return ("name", loader_definition.name) - task_definition: TaskDefinition | None = ( - read_task_definition(dependency) if callable(dependency) else None - ) - if task_definition is not None: - return ("name", task_definition.name) asset_definition: AssetDefinition | None = ( read_asset_definition(dependency) if callable(dependency) else None ) diff --git a/src/sqlbuild/compiler/python_nodes/types.py b/src/sqlbuild/compiler/python_nodes/types.py index f697d9a70a..336e4ea72e 100644 --- a/src/sqlbuild/compiler/python_nodes/types.py +++ b/src/sqlbuild/compiler/python_nodes/types.py @@ -8,7 +8,6 @@ class PythonNodeKind(StrEnum): """Framework-owned Python node kind.""" - TASK = "task" ASSET = "asset" LOADER = "loader" CHECK = "check" diff --git a/src/sqlbuild/executor/node_results/constants.py b/src/sqlbuild/executor/node_results/constants.py index a4b072a926..03dcc16ed4 100644 --- a/src/sqlbuild/executor/node_results/constants.py +++ b/src/sqlbuild/executor/node_results/constants.py @@ -39,7 +39,6 @@ } RESULT_DEPENDENCY_ATTRIBUTES: tuple[tuple[str, str], ...] = ( - ("__sqlbuild_task__", PythonNodeKind.TASK.value), ("__sqlbuild_asset__", PythonNodeKind.ASSET.value), ("__sqlbuild_loader__", PythonNodeKind.LOADER.value), ) diff --git a/src/sqlbuild/executor/python_nodes/_helpers/execution.py b/src/sqlbuild/executor/python_nodes/_helpers/execution.py index 4501569516..f59b7e25c6 100644 --- a/src/sqlbuild/executor/python_nodes/_helpers/execution.py +++ b/src/sqlbuild/executor/python_nodes/_helpers/execution.py @@ -8,7 +8,7 @@ from dataclasses import replace from typing import Any -from sqlbuild._native import CostContext, DiscoveredAssetFunction +from sqlbuild._native import CostContext from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.compiler.python_nodes.types import ( PythonNodeFanInAction, @@ -31,7 +31,6 @@ PythonNodeFanInDecision, PythonNodeRunState, PythonNodeRuntime, - TaskContext, ) from sqlbuild.executor.python_nodes.types import ExecutablePythonNode, OwnedResultCallback from sqlbuild.provider.main.runtime import ( @@ -107,7 +106,7 @@ def _execute_ready_node( monotonic: Callable[[], float], on_result: OwnedResultCallback | None, ) -> PythonNodeExecutionResult: - node_kind: PythonNodeKind = _node_kind(node) + node_kind: PythonNodeKind = PythonNodeKind.ASSET with CostContext.resource_scope( resource_type=node_kind.value, resource_name=node.name, @@ -181,7 +180,7 @@ def _execute_ready_node_in_cost_scope( _persist_python_node_result(result_store=result_store, result=result, run_id=run_id) return result, None warnings: list[str] = [] - context: TaskContext | AssetContext = _build_context( + context: AssetContext = _build_context( node=node, node_kind=node_kind, runtime=runtime, @@ -270,13 +269,13 @@ def _persist_python_node_result( def _call_node_with_retry( *, node: ExecutablePythonNode, - context: TaskContext | AssetContext, + context: AssetContext, providers: ProviderContainer | None, retry_policy: RetryPolicy | None, sleep: Callable[[float], None], monotonic: Callable[[], float], ) -> PythonNodeExecutionResult: - node_kind: PythonNodeKind = _node_kind(node) + node_kind: PythonNodeKind = PythonNodeKind.ASSET if retry_policy is None: with CostContext.resource_scope( resource_type=node_kind.value, @@ -362,7 +361,7 @@ def _missing_upstream_result( *, node: ExecutablePythonNode, node_kind: PythonNodeKind, - context: TaskContext | AssetContext, + context: AssetContext, upstream_results: tuple[PythonNodeExecutionResult, ...], ) -> str | None: """Why the node cannot run: a Python upstream outside this run that has never succeeded.""" @@ -401,7 +400,7 @@ def _build_context( run_state: PythonNodeRunState, result_store: Any | None, warnings: list[str], -) -> TaskContext | AssetContext: +) -> AssetContext: context_logger: logging.Logger = logger or logging.getLogger( f"sqlbuild.{node_kind.value}.{node.name}" ) @@ -411,10 +410,7 @@ def _build_context( providers: ProviderContainer = ( runtime.providers if runtime.providers is not None else _empty_provider_container() ) - context_class: type[AssetContext] | type[TaskContext] = ( - AssetContext if node_kind == PythonNodeKind.ASSET else TaskContext - ) - return context_class( + return AssetContext( adapter=runtime.adapter, connection_config=runtime.connection_config, connection=runtime.connection, @@ -440,9 +436,3 @@ def _build_context( start_cursor_int=runtime.start_cursor_int, end_cursor_int=runtime.end_cursor_int, ) - - -def _node_kind(node: ExecutablePythonNode) -> PythonNodeKind: - if isinstance(node, DiscoveredAssetFunction): - return PythonNodeKind.ASSET - return PythonNodeKind.TASK diff --git a/src/sqlbuild/executor/python_nodes/_helpers/ingress_execution.py b/src/sqlbuild/executor/python_nodes/_helpers/ingress_execution.py index 383e915f9e..9d4bea46c1 100644 --- a/src/sqlbuild/executor/python_nodes/_helpers/ingress_execution.py +++ b/src/sqlbuild/executor/python_nodes/_helpers/ingress_execution.py @@ -13,7 +13,6 @@ DiscoveredAssetFunction, DiscoveredLoaderFunction, DiscoveredPythonNode, - DiscoveredTaskFunction, PythonNodeGraph, PythonSqlRunLifecyclePlan, SourceEntry, @@ -404,22 +403,7 @@ def _record_scheduler_skips( def _to_executable_python_node(node: DiscoveredPythonNode) -> ExecutablePythonNode: - if node.kind == PythonNodeKind.ASSET: - return DiscoveredAssetFunction( - file_path=node.file_path, - relative_path=node.relative_path, - name=node.name, - function=node.function, - depends_on=node.depends_on, - tags=node.tags, - group=node.group, - description=node.description, - meta=node.meta, - columns=node.asset.columns if node.asset is not None else (), - column_lineage=node.asset.column_lineage if node.asset is not None else None, - retry=node.asset.retry if node.asset is not None else None, - ) - return DiscoveredTaskFunction( + return DiscoveredAssetFunction( file_path=node.file_path, relative_path=node.relative_path, name=node.name, @@ -429,7 +413,9 @@ def _to_executable_python_node(node: DiscoveredPythonNode) -> ExecutablePythonNo group=node.group, description=node.description, meta=node.meta, - retry=node.task.retry if node.task is not None else None, + columns=node.asset.columns if node.asset is not None else (), + column_lineage=node.asset.column_lineage if node.asset is not None else None, + retry=node.asset.retry if node.asset is not None else None, ) diff --git a/src/sqlbuild/executor/python_nodes/classes/read_side_python_execution_tracker.py b/src/sqlbuild/executor/python_nodes/classes/read_side_python_execution_tracker.py index b063f8a60c..3e15c2dbaa 100644 --- a/src/sqlbuild/executor/python_nodes/classes/read_side_python_execution_tracker.py +++ b/src/sqlbuild/executor/python_nodes/classes/read_side_python_execution_tracker.py @@ -121,7 +121,7 @@ def dispatch_ready_python_nodes(self) -> None: if node_name in self._completed_python_names: continue node: DiscoveredPythonNode = self._python_graph.nodes_by_name[node_name] - if node.kind not in {PythonNodeKind.TASK, PythonNodeKind.ASSET}: + if node.kind != PythonNodeKind.ASSET: continue if not self._sql_dependencies_satisfied(node): continue diff --git a/src/sqlbuild/executor/python_nodes/models.py b/src/sqlbuild/executor/python_nodes/models.py index 661af2edc7..70b8280c31 100644 --- a/src/sqlbuild/executor/python_nodes/models.py +++ b/src/sqlbuild/executor/python_nodes/models.py @@ -257,24 +257,6 @@ def skip( ) -@dataclass(frozen=True, kw_only=True) -class TaskContext(BasePythonNodeContext): - """Runtime context passed to a SQLBuild task function.""" - - def result( - self, - *, - payload: object | None = None, - metadata: dict[str, object] | None = None, - ) -> PythonNodeResult: - """Return a successful task result.""" - - return PythonNodeResult( - payload=payload, - metadata={} if metadata is None else metadata, - ) - - @dataclass(frozen=True, kw_only=True) class AssetContext(BasePythonNodeContext): """Runtime context passed to a SQLBuild asset function.""" diff --git a/src/sqlbuild/executor/python_nodes/types.py b/src/sqlbuild/executor/python_nodes/types.py index 222d443440..d376cda39f 100644 --- a/src/sqlbuild/executor/python_nodes/types.py +++ b/src/sqlbuild/executor/python_nodes/types.py @@ -4,9 +4,9 @@ from typing import TYPE_CHECKING, Protocol -from sqlbuild._native import DiscoveredAssetFunction, DiscoveredTaskFunction, PythonNodeIdentity +from sqlbuild._native import DiscoveredAssetFunction, PythonNodeIdentity -type ExecutablePythonNode = DiscoveredTaskFunction | DiscoveredAssetFunction +type ExecutablePythonNode = DiscoveredAssetFunction if TYPE_CHECKING: from sqlbuild.executor.python_nodes.models import PythonNodeExecutionResult diff --git a/src/sqlbuild/integrations/dagster/constants.py b/src/sqlbuild/integrations/dagster/constants.py index 8afa16ddce..4346f31f7b 100644 --- a/src/sqlbuild/integrations/dagster/constants.py +++ b/src/sqlbuild/integrations/dagster/constants.py @@ -19,7 +19,6 @@ NodeKind.MODEL, NodeKind.UDF, NodeKind.TABLE_FN, - NodeKind.TASK, NodeKind.ASSET, } ) @@ -61,7 +60,6 @@ NodeKind.UDF, NodeKind.TABLE_FN, NodeKind.MODEL, - NodeKind.TASK, NodeKind.ASSET, NodeKind.CHECK, NodeKind.SQL_TEST, @@ -79,7 +77,6 @@ DEFAULT_SELECTABLE_NODE_KIND_EXCLUSIONS: frozenset[NodeKind] = frozenset( { NodeKind.LOADER, - NodeKind.TASK, NodeKind.ASSET, NodeKind.CHECK, NodeKind.SQL_TEST, @@ -103,7 +100,6 @@ ) MATERIALIZABLE_NODE_KIND_EXCLUSIONS: frozenset[NodeKind] = frozenset( { - NodeKind.TASK, NodeKind.ASSET, NodeKind.CHECK, NodeKind.SQL_TEST, diff --git a/src/sqlbuild/integrations/rivers/constants.py b/src/sqlbuild/integrations/rivers/constants.py index cf147dd8fd..2b58b5bef9 100644 --- a/src/sqlbuild/integrations/rivers/constants.py +++ b/src/sqlbuild/integrations/rivers/constants.py @@ -17,7 +17,6 @@ NodeKind.MODEL, NodeKind.UDF, NodeKind.TABLE_FN, - NodeKind.TASK, NodeKind.ASSET, } ) diff --git a/src/sqlbuild/playground_templates/python_nodes/README.md b/src/sqlbuild/playground_templates/python_nodes/README.md index afa1154e05..677d7fede6 100644 --- a/src/sqlbuild/playground_templates/python_nodes/README.md +++ b/src/sqlbuild/playground_templates/python_nodes/README.md @@ -10,5 +10,5 @@ sqb build --select +fact_orders --select +orders_export sqb check --select check_orders_export ``` -The project includes a task feeding a loader, a model feeding Python assets through +The project includes an asset feeding a loader, a model feeding Python assets through `ctx.relation(model(...))`, soft skip fan-in, `materialized=False`, and a Python check. diff --git a/src/sqlbuild/playground_templates/python_nodes/python/assets/orders_export.py.txt b/src/sqlbuild/playground_templates/python_nodes/python/assets/orders_export.py.txt index 172b71b1b7..1280ac3c9c 100644 --- a/src/sqlbuild/playground_templates/python_nodes/python/assets/orders_export.py.txt +++ b/src/sqlbuild/playground_templates/python_nodes/python/assets/orders_export.py.txt @@ -3,7 +3,7 @@ from sqlbuild.assets import asset from sqlbuild.refs import model -from python.tasks.orders import export_window, optional_partner_feed +from python.assets.orders_inputs import export_window, optional_partner_feed @asset( diff --git a/src/sqlbuild/playground_templates/python_nodes/python/tasks/orders.py.txt b/src/sqlbuild/playground_templates/python_nodes/python/assets/orders_inputs.py.txt similarity index 72% rename from src/sqlbuild/playground_templates/python_nodes/python/tasks/orders.py.txt rename to src/sqlbuild/playground_templates/python_nodes/python/assets/orders_inputs.py.txt index 04367ff839..8dfae170f2 100644 --- a/src/sqlbuild/playground_templates/python_nodes/python/tasks/orders.py.txt +++ b/src/sqlbuild/playground_templates/python_nodes/python/assets/orders_inputs.py.txt @@ -1,9 +1,9 @@ -"""Task examples for the Python nodes playground.""" +"""Upstream asset examples for the Python nodes playground.""" -from sqlbuild.tasks import SkipMode, task +from sqlbuild.assets import SkipMode, asset -@task(tags=["python", "ingress"], group="ingress") +@asset(tags=["python", "ingress"], group="ingress") def prepare_raw_orders(ctx): """Prepare a tiny in-memory order extract before the loader runs.""" @@ -14,14 +14,14 @@ def prepare_raw_orders(ctx): return ctx.result(payload={"rows": rows}, metadata={"row_count": len(rows)}) -@task(tags=["python", "optional"], group="exports") +@asset(tags=["python", "optional"], group="exports") def optional_partner_feed(ctx): """Soft-skip an optional upstream without blocking sibling fan-in.""" return ctx.skip(reason="partner feed is not configured", mode=SkipMode.SOFT) -@task(tags=["python", "optional"], group="exports") +@asset(tags=["python", "optional"], group="exports") def export_window(ctx): """Successful sibling that lets downstream fan-in continue.""" diff --git a/src/sqlbuild/playground_templates/python_nodes/python/loaders/orders.py.txt b/src/sqlbuild/playground_templates/python_nodes/python/loaders/orders.py.txt index 00d91baa37..f65022bfd1 100644 --- a/src/sqlbuild/playground_templates/python_nodes/python/loaders/orders.py.txt +++ b/src/sqlbuild/playground_templates/python_nodes/python/loaders/orders.py.txt @@ -1,15 +1,15 @@ """Loader examples for the Python nodes playground.""" from sqlbuild.loaders import loader -from python.tasks.orders import prepare_raw_orders +from python.assets.orders_inputs import prepare_raw_orders @loader(depends_on=(prepare_raw_orders,)) def raw_orders(ctx): - """Load source rows after the preparation task has completed.""" + """Load source rows after the preparation asset has completed.""" # Loader contexts intentionally keep the SQL ingress API narrow; the dependency makes - # the task part of the same DAG even though this example returns static rows. + # the asset part of the same DAG even though this example returns static rows. return [ {"order_id": 1, "customer_id": 10, "amount_cents": 1200, "load_batch": 1}, {"order_id": 2, "customer_id": 20, "amount_cents": 800, "load_batch": 1}, diff --git a/src/sqlbuild/python_nodes/main/apply_task.py b/src/sqlbuild/python_nodes/main/apply_task.py deleted file mode 100644 index 83363f2b44..0000000000 --- a/src/sqlbuild/python_nodes/main/apply_task.py +++ /dev/null @@ -1,54 +0,0 @@ -"""Apply SQLBuild task metadata to a Python function.""" - -from __future__ import annotations - -from collections.abc import Callable, Sequence -from typing import Any, cast - -from sqlbuild.python_nodes._helpers.attachment import attach_definition -from sqlbuild.python_nodes._helpers.dependency_normalization import ( - normalize_python_node_dependencies, -) -from sqlbuild.python_nodes._helpers.description_resolution import resolve_description -from sqlbuild.python_nodes.models import NodeSetRef, RetryPolicy, SqlResourceRef, TaskDefinition - - -def apply_task( - *, - function: Callable[..., object] | None = None, - name: str | None = None, - depends_on: Callable[..., object] - | SqlResourceRef - | NodeSetRef - | tuple[Callable[..., object] | SqlResourceRef | NodeSetRef, ...] - | list[Callable[..., object] | SqlResourceRef | NodeSetRef] = (), - tags: Sequence[str] = (), - group: str | None = None, - description: str | None = None, - meta: dict[str, object] | None = None, - retry: RetryPolicy | None = None, -) -> Callable[..., object] | Callable[[Callable[..., object]], Callable[..., object]]: - """Apply SQLBuild task metadata to a Python function.""" - - normalized_deps: tuple[Callable[..., object] | SqlResourceRef | NodeSetRef, ...] = ( - normalize_python_node_dependencies(depends_on) - ) - - def decorate(inner: Callable[..., object]) -> Callable[..., object]: - inner_function: Any = cast(Any, inner) - definition: TaskDefinition = TaskDefinition( - name=name or inner_function.__name__, - depends_on=normalized_deps, - tags=tuple(tags), - group=group, - description=resolve_description(function=inner, description=description), - meta=meta, - retry=retry, - ) - return attach_definition( - function=inner, - attribute_name="__sqlbuild_task__", - definition=definition, - ) - - return decorate(function) if function is not None else decorate diff --git a/src/sqlbuild/python_nodes/main/read_task_definition.py b/src/sqlbuild/python_nodes/main/read_task_definition.py deleted file mode 100644 index 5231ff2051..0000000000 --- a/src/sqlbuild/python_nodes/main/read_task_definition.py +++ /dev/null @@ -1,16 +0,0 @@ -"""Read SQLBuild task metadata from a Python function.""" - -from collections.abc import Callable - -from sqlbuild.python_nodes._helpers.attachment import read_attached_definition -from sqlbuild.python_nodes.models import TaskDefinition - - -def read_task_definition(function: Callable[..., object]) -> TaskDefinition | None: - """Return SQLBuild task metadata from a decorated function, if present.""" - - return read_attached_definition( - function=function, - attribute_name="__sqlbuild_task__", - definition_type=TaskDefinition, - ) diff --git a/src/sqlbuild/python_nodes/models.py b/src/sqlbuild/python_nodes/models.py index 7d47dde0bd..85d5ccc712 100644 --- a/src/sqlbuild/python_nodes/models.py +++ b/src/sqlbuild/python_nodes/models.py @@ -136,19 +136,6 @@ class HookDefinition: reads: tuple[SqlResourceRef, ...] = () -@dataclass(frozen=True) -class TaskDefinition: - """Metadata attached to a decorated SQLBuild task function.""" - - name: str - depends_on: tuple[Callable[..., object] | SqlResourceRef | NodeSetRef, ...] = () - tags: tuple[str, ...] = () - group: str | None = None - description: str | None = None - meta: dict[str, object] | None = None - retry: RetryPolicy | None = None - - @dataclass(frozen=True) class AssetDefinition: """Metadata attached to a decorated SQLBuild asset function.""" diff --git a/src/sqlbuild/tasks.py b/src/sqlbuild/tasks.py deleted file mode 100644 index 5ce20863f4..0000000000 --- a/src/sqlbuild/tasks.py +++ /dev/null @@ -1,48 +0,0 @@ -"""Public decorator API for SQLBuild tasks.""" - -from __future__ import annotations - -from collections.abc import Callable, Sequence - -from sqlbuild.compiler.python_nodes.types import SkipMode -from sqlbuild.executor.python_nodes.models import TaskContext -from sqlbuild.python_nodes.main.apply_task import apply_task -from sqlbuild.python_nodes.main.read_task_definition import read_task_definition -from sqlbuild.python_nodes.models import NodeSetRef, RetryPolicy, SqlResourceRef, TaskDefinition - -__all__ = ("SkipMode", "TaskContext", "get_task_definition", "task") - - -def task( - function: Callable[..., object] | None = None, - *, - name: str | None = None, - depends_on: Callable[..., object] - | SqlResourceRef - | NodeSetRef - | tuple[Callable[..., object] | SqlResourceRef | NodeSetRef, ...] - | list[Callable[..., object] | SqlResourceRef | NodeSetRef] = (), - tags: Sequence[str] = (), - group: str | None = None, - description: str | None = None, - meta: dict[str, object] | None = None, - retry: RetryPolicy | None = None, -) -> Callable[..., object] | Callable[[Callable[..., object]], Callable[..., object]]: - """Mark a Python function as a SQLBuild task.""" - - return apply_task( - function=function, - name=name, - depends_on=depends_on, - tags=tags, - group=group, - description=description, - meta=meta, - retry=retry, - ) - - -def get_task_definition(function: Callable[..., object]) -> TaskDefinition | None: - """Return SQLBuild task metadata from a decorated function, if present.""" - - return read_task_definition(function) diff --git a/src/sqlbuild/tasks.pyi b/src/sqlbuild/tasks.pyi deleted file mode 100644 index ece24034d6..0000000000 --- a/src/sqlbuild/tasks.pyi +++ /dev/null @@ -1,26 +0,0 @@ -from collections.abc import Callable, Sequence -from typing import overload - -from sqlbuild.compiler.python_nodes.types import SkipMode as SkipMode -from sqlbuild.executor.python_nodes.models import TaskContext as TaskContext -from sqlbuild.python_nodes.models import RetryPolicy, SqlResourceRef, TaskDefinition - -__all__ = ("SkipMode", "TaskContext", "get_task_definition", "task") - -@overload -def task(function: Callable[..., object]) -> Callable[..., object]: ... -@overload -def task( - *, - name: str | None = None, - depends_on: Callable[..., object] - | SqlResourceRef - | tuple[Callable[..., object] | SqlResourceRef, ...] - | list[Callable[..., object] | SqlResourceRef] = (), - tags: Sequence[str] = (), - group: str | None = None, - description: str | None = None, - meta: dict[str, object] | None = None, - retry: RetryPolicy | None = None, -) -> Callable[[Callable[..., object]], Callable[..., object]]: ... -def get_task_definition(function: Callable[..., object]) -> TaskDefinition | None: ... diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/bigquery/test_bigquery.py b/tests/e2e/src/sqlbuild/cli/commands/main/bigquery/test_bigquery.py index d2b3d50036..887d021bb4 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/bigquery/test_bigquery.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/bigquery/test_bigquery.py @@ -169,8 +169,8 @@ def test_given_python_result_when_running_check_on_bigquery_then_persists_node_r dataset_name=dataset_name, ), "python/tasks/results.py": ( - "from sqlbuild.tasks import task\n\n" - "@task\n" + "from sqlbuild.assets import asset\n\n" + "@asset\n" "def produce_result(ctx):\n" " '''Test task produce_result.'''\n return ctx.result(payload={'value': 42}, metadata={'source': 'bigquery'})\n" ), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py index 447fecc6c7..1955edec5b 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py @@ -907,7 +907,7 @@ def python_node_selection_project_files(*, orders_sql: str) -> dict[str, str]: " return [{'customer_id': 99}]\n" ), "python/tasks/counts.py": ( - "from sqlbuild.refs import model, source\nfrom sqlbuild.tasks import task\n\n\n" + "from sqlbuild.refs import model, source\nfrom sqlbuild.assets import asset\n\n\n" + task_nodes ), "python/assets/extracts.py": ( @@ -921,8 +921,8 @@ def python_node_selection_project_files(*, orders_sql: str) -> dict[str, str]: ), **duckshop_orchestration("python_node_jobs"), "python/tasks/reports.py": ( - "from python.tasks.counts import count_orders\nfrom sqlbuild.tasks import task\n\n\n" - "@task(depends_on=count_orders)\n" + "from python.tasks.counts import count_orders\nfrom sqlbuild.assets import asset\n\n\n" + "@asset(depends_on=count_orders)\n" "def report_order_count(ctx):\n" " '''Test task report_order_count.'''\n" " upstream = ctx.result_of(node_function=count_orders)\n" @@ -1574,8 +1574,8 @@ def ranking_task_files(body: str) -> dict[str, str]: return { "python/tasks/ranking.py": ( "from sqlbuild.refs import model\n" - "from sqlbuild.tasks import task\n\n\n" - '@task(depends_on=model("orders"))\n' + "from sqlbuild.assets import asset\n\n\n" + '@asset(depends_on=model("orders"))\n' "def rank_orders(ctx):\n" ' """Rank the orders for the partner."""\n' + body ), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/test_build_no_tests_no_audits.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/test_build_no_tests_no_audits.py index 594a573e9b..9d641b1574 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/test_build_no_tests_no_audits.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/test_build_no_tests_no_audits.py @@ -219,7 +219,7 @@ def test_given_factory_generated_nodes_when_running_commands_then_lifecycle_succ from sqlbuild.checks import check from sqlbuild.factories import factory from sqlbuild.loaders import loader -from sqlbuild.tasks import task +from sqlbuild.assets import asset PROJECT_DIR = Path(__file__).parents[2] @@ -227,7 +227,7 @@ def test_given_factory_generated_nodes_when_running_commands_then_lifecycle_succ @factory def generated_pipeline(): - @task(name="prepare_orders", tags=("factory", "runtime")) + @asset(name="prepare_orders", tags=("factory", "runtime")) def prepare(ctx): '''Test task prepare.''' PROJECT_DIR.joinpath("prepare_marker.txt").write_text("ran", encoding="utf-8") @@ -304,7 +304,7 @@ def export_check(ctx): ) dag_payload: dict[str, object] = json.loads(dag_result.stdout) dag_node_ids: set[str] = {str(node["id"]) for node in dag_payload["nodes"]} - assert {"task:prepare_orders", "asset:orders_export", "source:raw_orders"} <= dag_node_ids + assert {"asset:prepare_orders", "asset:orders_export", "source:raw_orders"} <= dag_node_ids assert "factory:generated_pipeline" not in dag_node_ids assert compile_dag_result.returncode == test_case.expected_exit_code, ( compile_dag_result.stdout + compile_dag_result.stderr @@ -421,8 +421,8 @@ def test_given_task_selector_when_running_run_then_task_executes( ), "python/tasks/orders.py": ( "from pathlib import Path\n" - "from sqlbuild.tasks import task\n\n" - "@task\n" + "from sqlbuild.assets import asset\n\n" + "@asset\n" "def prepare_orders(ctx):\n" " '''Test task prepare_orders.'''\n Path(__file__).parents[2].joinpath('task_marker.txt').write_text('ran')\n" " return ctx.result(payload={'status': 'ok'})\n" @@ -437,7 +437,7 @@ def test_given_task_selector_when_running_run_then_task_executes( "--no-tests", "--no-audits", "--select", - "task:prepare_orders", + "asset:prepare_orders", ), project_dir=project_dir, ) @@ -531,8 +531,8 @@ def test_given_task_selector_with_json_output_when_running_run_then_json_include 'database = "python_run_json_project.duckdb"\n' ), "python/tasks/orders.py": ( - "from sqlbuild.tasks import task\n\n" - "@task\n" + "from sqlbuild.assets import asset\n\n" + "@asset\n" "def prepare_orders(ctx):\n" " '''Test task prepare_orders.'''\n return ctx.result(metadata={'rows': 3})\n" ), @@ -548,7 +548,7 @@ def test_given_task_selector_with_json_output_when_running_run_then_json_include "--json-output", str(json_output_path), "--select", - "task:prepare_orders", + "asset:prepare_orders", ), project_dir=project_dir, ) @@ -557,7 +557,7 @@ def test_given_task_selector_with_json_output_when_running_run_then_json_include payload: dict[str, object] = json.loads(json_output_path.read_text(encoding="utf-8")) assets: list[dict[str, object]] = list(payload["assets"]) # type: ignore[arg-type] assert { - "kind": "task", + "kind": "asset", "name": "prepare_orders", "status": "success", "metadata": {"rows": 3}, @@ -602,8 +602,8 @@ def test_given_failing_task_selector_when_running_run_then_command_fails( 'database = "python_failed_run_project.duckdb"\n' ), "python/tasks/orders.py": ( - "from sqlbuild.tasks import task\n\n" - "@task\n" + "from sqlbuild.assets import asset\n\n" + "@asset\n" "def fail_orders(ctx):\n" " '''Test task fail_orders.'''\n raise RuntimeError('API unavailable')\n" ), @@ -619,7 +619,7 @@ def test_given_failing_task_selector_when_running_run_then_command_fails( "--json-output", str(json_output_path), "--select", - "task:fail_orders", + "asset:fail_orders", ), project_dir=project_dir, ) @@ -672,8 +672,8 @@ def test_given_task_loader_source_model_chain_when_running_model_then_task_runs_ ), "python/tasks/orders.py": ( "from pathlib import Path\n" - "from sqlbuild.tasks import task\n\n" - "@task\n" + "from sqlbuild.assets import asset\n\n" + "@asset\n" "def prepare_orders(ctx):\n" " '''Test task prepare_orders.'''\n Path(__file__).parents[2].joinpath('orders_ready.txt').write_text('ready')\n" " return ctx.result(metadata={'prepared': True})\n" @@ -765,8 +765,8 @@ def test_given_model_and_task_selector_when_running_run_then_task_can_read_built "python/tasks/orders.py": ( "from pathlib import Path\n" "from sqlbuild.refs import model\n" - "from sqlbuild.tasks import task\n\n" - "@task(depends_on=model('fact_orders'))\n" + "from sqlbuild.assets import asset\n\n" + "@asset(depends_on=model('fact_orders'))\n" "def summarize_orders(ctx):\n" " '''Test task summarize_orders.'''\n orders = ctx.relation(model('fact_orders'))\n" " rows = ctx.query(f'SELECT COUNT(*) FROM {orders}').fetchall()[0][0]\n" @@ -896,16 +896,16 @@ def test_given_task_asset_task_chain_when_running_final_task_then_chain_executes 'database = "python_task_asset_task_run_project.duckdb"\n' ), "python/tasks/fetch_orders.py": ( - "from sqlbuild.tasks import task\n\n" - "@task\n" + "from sqlbuild.assets import asset\n\n" + "@asset\n" "def fetch_orders(ctx):\n" " '''Test task fetch_orders.'''\n return ctx.result(payload={'rows': 1})\n" ), "python/tasks/notify_orders.py": ( "from pathlib import Path\n" "from python.assets.orders import publish_orders\n" - "from sqlbuild.tasks import task\n\n" - "@task(depends_on=publish_orders)\n" + "from sqlbuild.assets import asset\n\n" + "@asset(depends_on=publish_orders)\n" "def notify_orders(ctx):\n" " '''Test task notify_orders.'''\n metadata = ctx.result_of(node_function=publish_orders).metadata\n" " output = Path(__file__).parents[2].joinpath('notify.txt')\n" @@ -979,8 +979,8 @@ def test_given_source_task_asset_selection_when_running_run_then_task_reads_load ), "python/tasks/orders.py": ( "from sqlbuild.refs import source\n" - "from sqlbuild.tasks import task\n\n" - "@task(depends_on=source('raw_orders'))\n" + "from sqlbuild.assets import asset\n\n" + "@asset(depends_on=source('raw_orders'))\n" "def summarize_loaded_orders(ctx):\n" " '''Test task summarize_loaded_orders.'''\n orders = ctx.relation(source('raw_orders'))\n" " rows = ctx.query(f'SELECT COUNT(*) FROM {orders}').fetchall()[0][0]\n" @@ -1048,8 +1048,8 @@ def test_given_skip_and_asset_selection_with_json_when_running_run_then_json_rec 'database = "python_json_fields_run_project.duckdb"\n' ), "python/tasks/orders.py": ( - "from sqlbuild.tasks import task\n\n" - "@task\n" + "from sqlbuild.assets import asset\n\n" + "@asset\n" "def optional_orders(ctx):\n" " '''Test task optional_orders.'''\n return ctx.skip(reason='no files')\n\n" ), @@ -1138,12 +1138,12 @@ def test_given_independent_python_and_sql_selectors_when_running_run_then_all_br ), "python/tasks/branches.py": ( "from pathlib import Path\n" - "from sqlbuild.tasks import task\n\n" - "@task\n" + "from sqlbuild.assets import asset\n\n" + "@asset\n" "def branch_a(ctx):\n" " '''Test task branch_a.'''\n Path(__file__).parents[2].joinpath('branch_a.txt').write_text('a')\n" " return ctx.result()\n\n" - "@task\n" + "@asset\n" "def branch_b(ctx):\n" " '''Test task branch_b.'''\n Path(__file__).parents[2].joinpath('branch_b.txt').write_text('b')\n" " return ctx.result()\n" @@ -1215,8 +1215,8 @@ def test_given_task_depends_on_model_when_running_run_then_task_runs_before_down ), "python/tasks/profile.py": ( "from sqlbuild.refs import model\n" - "from sqlbuild.tasks import task\n\n" - "@task(depends_on=model('stg_orders'))\n" + "from sqlbuild.assets import asset\n\n" + "@asset(depends_on=model('stg_orders'))\n" "def profile_stg_orders(ctx):\n" " '''Test task profile_stg_orders.'''\n orders = ctx.relation(model('stg_orders'))\n" " rows = ctx.query(f'SELECT COUNT(*) FROM {orders}').fetchall()[0][0]\n" @@ -1291,8 +1291,8 @@ def test_given_task_depends_on_source_when_running_run_then_task_runs_after_sour ), "python/tasks/profile.py": ( "from sqlbuild.refs import source\n" - "from sqlbuild.tasks import task\n\n" - "@task(depends_on=source('raw_orders'))\n" + "from sqlbuild.assets import asset\n\n" + "@asset(depends_on=source('raw_orders'))\n" "def profile_raw_orders(ctx):\n" " '''Test task profile_raw_orders.'''\n orders = ctx.relation(source('raw_orders'))\n" " rows = ctx.query(f'SELECT COUNT(*) FROM {orders}').fetchall()[0][0]\n" @@ -1364,8 +1364,8 @@ def test_given_sql_ready_task_fails_when_running_run_then_footer_json_and_exit_f ), "python/tasks/profile.py": ( "from sqlbuild.refs import model\n" - "from sqlbuild.tasks import task\n\n" - "@task(depends_on=model('stg_orders'))\n" + "from sqlbuild.assets import asset\n\n" + "@asset(depends_on=model('stg_orders'))\n" "def fail_after_stg_orders(ctx):\n" " '''Test task fail_after_stg_orders.'''\n raise RuntimeError('profile failed')\n" ), @@ -1444,8 +1444,8 @@ def test_given_task_depends_on_terminal_loader_when_running_run_then_command_rej ), "python/tasks/orders.py": ( "from python.loaders.orders import raw_orders\n" - "from sqlbuild.tasks import task\n\n" - "@task(depends_on=raw_orders)\n" + "from sqlbuild.assets import asset\n\n" + "@asset(depends_on=raw_orders)\n" "def summarize_orders(ctx):\n" " '''Test task summarize_orders.'''\n return ctx.result()\n" ), @@ -1511,8 +1511,8 @@ def test_given_task_asset_depend_on_intermediate_loader_when_running_run_then_lo ), "python/tasks/orders.py": ( "from python.loaders.orders import stage_orders\n" - "from sqlbuild.tasks import task\n\n" - "@task(depends_on=stage_orders)\n" + "from sqlbuild.assets import asset\n\n" + "@asset(depends_on=stage_orders)\n" "def summarize_stage_orders(ctx):\n" " '''Test task summarize_stage_orders.'''\n rows = ctx.query('SELECT COUNT(*) FROM stage_orders').fetchall()[0][0]\n" " return ctx.result(payload={'rows': rows}, metadata={'rows': rows})\n" @@ -1590,8 +1590,8 @@ def test_given_loader_task_loader_chain_when_running_model_then_ingress_orders_c "python/tasks/orders.py": ( "from pathlib import Path\n" "from python.loaders.window import load_window_orders\n" - "from sqlbuild.tasks import task\n\n" - "@task(depends_on=load_window_orders)\n" + "from sqlbuild.assets import asset\n\n" + "@asset(depends_on=load_window_orders)\n" "def prepare_raw_orders(ctx):\n" " '''Test task prepare_raw_orders.'''\n rows = ctx.query('SELECT COUNT(*) FROM window_orders').fetchall()[0][0]\n" " Path(__file__).parents[2].joinpath('prepared.txt').write_text(str(rows))\n" @@ -1664,8 +1664,8 @@ def test_given_task_asset_loader_chain_when_running_model_then_ingress_orders_ch 'database = "python_task_asset_loader_run_project.duckdb"\n' ), "python/tasks/orders.py": ( - "from sqlbuild.tasks import task\n\n" - "@task\n" + "from sqlbuild.assets import asset\n\n" + "@asset\n" "def fetch_orders(ctx):\n" " '''Test task fetch_orders.'''\n return ctx.result(payload={'order_id': 1})\n" ), @@ -1917,8 +1917,8 @@ def test_given_loader_task_asset_loader_model_task_asset_task_spine_when_running ), "python/tasks/prepare.py": ( "from python.loaders.window import load_window_orders\n" - "from sqlbuild.tasks import task\n\n" - "@task(depends_on=load_window_orders)\n" + "from sqlbuild.assets import asset\n\n" + "@asset(depends_on=load_window_orders)\n" "def prepare_orders(ctx):\n" " '''Test task prepare_orders.'''\n rows = ctx.query('SELECT order_id FROM window_orders').fetchall()\n" " return ctx.result(payload={'order_id': rows[0][0]})\n" @@ -1957,8 +1957,8 @@ def test_given_loader_task_asset_loader_model_task_asset_task_spine_when_running ), "python/tasks/profile.py": ( "from sqlbuild.refs import model\n" - "from sqlbuild.tasks import task\n\n" - "@task(depends_on=model('fact_orders'))\n" + "from sqlbuild.assets import asset\n\n" + "@asset(depends_on=model('fact_orders'))\n" "def profile_fact_orders(ctx):\n" " '''Test task profile_fact_orders.'''\n orders = ctx.relation(model('fact_orders'))\n" " order_id = ctx.query(f'SELECT order_id FROM {orders}').fetchall()[0][0]\n" @@ -1975,8 +1975,8 @@ def test_given_loader_task_asset_loader_model_task_asset_task_spine_when_running "python/tasks/notify.py": ( "from pathlib import Path\n" "from python.assets.export import export_fact_orders\n" - "from sqlbuild.tasks import task\n\n" - "@task(depends_on=export_fact_orders)\n" + "from sqlbuild.assets import asset\n\n" + "@asset(depends_on=export_fact_orders)\n" "def notify_fact_orders(ctx):\n" " '''Test task notify_fact_orders.'''\n payload = ctx.result_of(node_function=export_fact_orders).payload\n" " output = Path(__file__).parents[2].joinpath('notify.txt')\n" @@ -2059,8 +2059,8 @@ def test_given_source_task_loader_chain_when_running_run_then_command_rejects_bo ), "python/tasks/orders.py": ( "from sqlbuild.refs import source\n" - "from sqlbuild.tasks import task\n\n" - "@task(depends_on=source('raw_orders'))\n" + "from sqlbuild.assets import asset\n\n" + "@asset(depends_on=source('raw_orders'))\n" "def prepare_orders(ctx):\n" " '''Test task prepare_orders.'''\n return ctx.result()\n" ), @@ -2138,8 +2138,8 @@ def test_given_model_task_loader_chain_when_running_run_then_command_rejects_bou ), "python/tasks/orders.py": ( "from sqlbuild.refs import model\n" - "from sqlbuild.tasks import task\n\n" - "@task(depends_on=model('stg_orders'))\n" + "from sqlbuild.assets import asset\n\n" + "@asset(depends_on=model('stg_orders'))\n" "def prepare_orders(ctx):\n" " '''Test task prepare_orders.'''\n return ctx.result()\n" ), @@ -2187,7 +2187,7 @@ def test_given_model_task_loader_chain_when_running_run_then_command_rejects_bou expected_table_names=(), expected_view_names=(), expected_output_fragments=( - "Python check 'check_orders_export' checks task:prepare_orders, which the " + "Python check 'check_orders_export' checks asset:prepare_orders, which the " "selection does not include; select it too, or narrow the selector so it no " "longer matches the check.", ), @@ -2199,7 +2199,7 @@ def test_given_model_task_loader_chain_when_running_run_then_command_rejects_bou expected_table_names=(), expected_view_names=(), expected_output_fragments=("check check_orders_export START", "PASS=2"), - select=("check:check_orders_export", "task:prepare_orders"), + select=("check:check_orders_export", "asset:prepare_orders"), ), RunE2ETestCase( description="build rejects a selected check on a source no loader writes", @@ -2238,8 +2238,8 @@ def test_given_check_selector_when_running_build_then_check_needs_its_checked_no " '''Test check check_orders_export.'''\n return ctx.pass_()\n" ), "python/tasks/orders.py": ( - "from sqlbuild.tasks import task\n\n" - "@task\n" + "from sqlbuild.assets import asset\n\n" + "@asset\n" "def prepare_orders(ctx):\n" " '''Test task prepare_orders.'''\n return ctx.result()\n" ), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_build.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_build.py index 3ccb67b8a2..9843f09a97 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_build.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_build.py @@ -676,8 +676,8 @@ def test_given_shared_downstream_failure_when_building_then_blocks_all_source_ap 'database = "python_build_ingress_failure_project.duckdb"\n' ), "python/tasks/prepare.py": ( - "from sqlbuild.tasks import task\n\n" - "@task\n" + "from sqlbuild.assets import asset\n\n" + "@asset\n" "def prepare_orders(ctx):\n" " '''Test task prepare_orders.'''\n raise RuntimeError('prepare failed')\n" ), @@ -723,8 +723,8 @@ def test_given_shared_downstream_failure_when_building_then_blocks_all_source_ap ), "python/tasks/prepare.py": ( "from sqlbuild.compiler.python_nodes.types import SkipMode\n" - "from sqlbuild.tasks import task\n\n" - "@task\n" + "from sqlbuild.assets import asset\n\n" + "@asset\n" "def prepare_orders(ctx):\n" " '''Test task prepare_orders.'''\n return ctx.skip(reason='no input', mode=SkipMode.HARD)\n" ), @@ -769,8 +769,8 @@ def test_given_shared_downstream_failure_when_building_then_blocks_all_source_ap 'database = "python_build_non_json_result_project.duckdb"\n' ), "python/tasks/bad.py": ( - "from sqlbuild.tasks import task\n\n" - "@task\n" + "from sqlbuild.assets import asset\n\n" + "@asset\n" "def produce_bad_result(ctx):\n" " '''Test task produce_bad_result.'''\n return ctx.result(payload={'bad': {'a', 'b'}})\n" ), @@ -793,8 +793,8 @@ def test_given_shared_downstream_failure_when_building_then_blocks_all_source_ap 'database = "python_build_non_json_metadata_project.duckdb"\n' ), "python/tasks/bad.py": ( - "from sqlbuild.tasks import task\n\n" - "@task\n" + "from sqlbuild.assets import asset\n\n" + "@asset\n" "def produce_bad_metadata(ctx):\n" " '''Test task produce_bad_metadata.'''\n return ctx.result(payload={}, metadata={'bad': {'a', 'b'}})\n" ), @@ -829,8 +829,8 @@ def test_given_shared_downstream_failure_when_building_then_blocks_all_source_ap ), "python/tasks/profile.py": ( "from sqlbuild.refs import model\n" - "from sqlbuild.tasks import task\n\n" - "@task(depends_on=model('fact_orders'))\n" + "from sqlbuild.assets import asset\n\n" + "@asset(depends_on=model('fact_orders'))\n" "def fail_after_fact(ctx):\n" " '''Test task fail_after_fact.'''\n raise RuntimeError('profile failed')\n" ), @@ -869,8 +869,8 @@ def test_given_shared_downstream_failure_when_building_then_blocks_all_source_ap "python/tasks/profile.py": ( "from sqlbuild.compiler.python_nodes.types import SkipMode\n" "from sqlbuild.refs import model\n" - "from sqlbuild.tasks import task\n\n" - "@task(depends_on=model('fact_orders'))\n" + "from sqlbuild.assets import asset\n\n" + "@asset(depends_on=model('fact_orders'))\n" "def skip_after_fact(ctx):\n" " '''Test task skip_after_fact.'''\n return ctx.skip(reason='no profile needed', mode=SkipMode.HARD)\n" ), @@ -909,8 +909,8 @@ def test_given_shared_downstream_failure_when_building_then_blocks_all_source_ap "python/tasks/profile.py": ( "from sqlbuild.compiler.python_nodes.types import SkipMode\n" "from sqlbuild.refs import model\n" - "from sqlbuild.tasks import task\n\n" - "@task(depends_on=model('fact_orders'))\n" + "from sqlbuild.assets import asset\n\n" + "@asset(depends_on=model('fact_orders'))\n" "def soft_skip_after_fact(ctx):\n" " '''Test task soft_skip_after_fact.'''\n return ctx.skip(reason='optional profile skipped', mode=SkipMode.SOFT)\n" ), @@ -937,8 +937,8 @@ def test_given_shared_downstream_failure_when_building_then_blocks_all_source_ap ), "python/tasks/prepare.py": ( "from pathlib import Path\n" - "from sqlbuild.tasks import task\n\n" - "@task\n" + "from sqlbuild.assets import asset\n\n" + "@asset\n" "def prepare_orders(ctx):\n" " '''Test task prepare_orders.'''\n marker = Path(__file__).parents[2].joinpath('prepared.txt')\n" " marker.write_text('prepared')\n" @@ -966,8 +966,8 @@ def test_given_shared_downstream_failure_when_building_then_blocks_all_source_ap "python/tasks/profile.py": ( "from pathlib import Path\n" "from sqlbuild.refs import model\n" - "from sqlbuild.tasks import task\n\n" - "@task(depends_on=model('fact_orders'))\n" + "from sqlbuild.assets import asset\n\n" + "@asset(depends_on=model('fact_orders'))\n" "def profile_fact(ctx):\n" " '''Test task profile_fact.'''\n marker = Path(__file__).parents[2].joinpath('profile.txt')\n" " marker.write_text('profiled')\n" @@ -1012,8 +1012,8 @@ def test_given_shared_downstream_failure_when_building_then_blocks_all_source_ap "python/tasks/profile.py": ( "from pathlib import Path\n" "from sqlbuild.refs import source\n" - "from sqlbuild.tasks import task\n\n" - "@task(depends_on=source('raw_orders'))\n" + "from sqlbuild.assets import asset\n\n" + "@asset(depends_on=source('raw_orders'))\n" "def profile_raw_orders(ctx):\n" " '''Test task profile_raw_orders.'''\n relation = ctx.relation(source('raw_orders'))\n" " value = ctx.query(f'SELECT order_id FROM {relation}').fetchall()[0][0]\n" @@ -1113,8 +1113,8 @@ def test_given_python_lifecycle_edge_case_when_building_then_direct_build_harden ), "python/tasks/prepare.py": ( "from pathlib import Path\n" - "from sqlbuild.tasks import task\n\n" - "@task\n" + "from sqlbuild.assets import asset\n\n" + "@asset\n" "def prepare_events(ctx):\n" " '''Test task prepare_events.'''\n marker = Path(__file__).parents[2].joinpath('prepared.txt')\n" " marker.write_text('prepared')\n" @@ -1372,8 +1372,8 @@ def test_given_python_sql_python_spine_when_building_then_orders_python_around_s ), "python/tasks/prepare.py": ( "from python.loaders.window import load_window_orders\n" - "from sqlbuild.tasks import task\n\n" - "@task(depends_on=load_window_orders)\n" + "from sqlbuild.assets import asset\n\n" + "@asset(depends_on=load_window_orders)\n" "def prepare_orders(ctx):\n" " '''Test task prepare_orders.'''\n rows = ctx.query('SELECT order_id FROM window_orders').fetchall()\n" " return ctx.result(payload={'order_id': rows[0][0]})\n" @@ -1412,8 +1412,8 @@ def test_given_python_sql_python_spine_when_building_then_orders_python_around_s ), "python/tasks/profile.py": ( "from sqlbuild.refs import model\n" - "from sqlbuild.tasks import task\n\n" - "@task(depends_on=model('fact_orders'))\n" + "from sqlbuild.assets import asset\n\n" + "@asset(depends_on=model('fact_orders'))\n" "def profile_fact_orders(ctx):\n" " '''Test task profile_fact_orders.'''\n relation = ctx.relation(model('fact_orders'))\n" " order_id = ctx.query(f'SELECT order_id FROM {relation}').fetchall()[0][0]\n" @@ -1430,8 +1430,8 @@ def test_given_python_sql_python_spine_when_building_then_orders_python_around_s "python/tasks/notify.py": ( "from pathlib import Path\n" "from python.assets.export import export_fact_orders\n" - "from sqlbuild.tasks import task\n\n" - "@task(depends_on=export_fact_orders)\n" + "from sqlbuild.assets import asset\n\n" + "@asset(depends_on=export_fact_orders)\n" "def notify_fact_orders(ctx):\n" " '''Test task notify_fact_orders.'''\n payload = ctx.result_of(node_function=export_fact_orders).payload\n" " output = Path(__file__).parents[2].joinpath('notify.txt')\n" @@ -1517,8 +1517,8 @@ def test_given_prior_python_task_result_when_later_task_reads_result_then_uses_p 'database = "python_persisted_result_project.duckdb"\n' ), "python/tasks/produce.py": ( - "from sqlbuild.tasks import task\n\n" - "@task\n" + "from sqlbuild.assets import asset\n\n" + "@asset\n" "def produce_result(ctx):\n" " '''Test task produce_result.'''\n return ctx.result(payload={'value': 42}, metadata={'source': 'first'})\n" ), @@ -1536,8 +1536,8 @@ def test_given_prior_python_task_result_when_later_task_reads_result_then_uses_p first_run_id: str = str(first_run_rows[0][0]) (project_dir / "python" / "tasks" / "produce.py").write_text( ( - "from sqlbuild.tasks import task\n\n" - "@task\n" + "from sqlbuild.assets import asset\n\n" + "@asset\n" "def produce_result(ctx):\n" " '''Test task produce_result.'''\n return ctx.result(payload={'value': 84}, metadata={'source': 'second'})\n" ), @@ -1549,8 +1549,8 @@ def test_given_prior_python_task_result_when_later_task_reads_result_then_uses_p ) (project_dir / "python" / "tasks" / "produce.py").write_text( ( - "from sqlbuild.tasks import task\n\n" - "@task\n" + "from sqlbuild.assets import asset\n\n" + "@asset\n" "def produce_result(ctx):\n" " '''Test task produce_result.'''\n raise RuntimeError('producer failed after previous success')\n" ), @@ -1572,8 +1572,8 @@ def test_given_prior_python_task_result_when_later_task_reads_result_then_uses_p ( "from pathlib import Path\n" "from python.tasks.produce import produce_result\n" - "from sqlbuild.tasks import task\n\n" - "@task\n" + "from sqlbuild.assets import asset\n\n" + "@asset\n" "def consume_result(ctx):\n" " '''Test task consume_result.'''\n latest = ctx.result_of(node_function=produce_result)\n" f" first = ctx.result_of(node_function=produce_result, run_id='{first_run_id}')\n" @@ -1680,8 +1680,8 @@ def test_given_prior_loader_result_when_later_task_reads_result_then_uses_loader "python/tasks/consume.py": ( "from pathlib import Path\n" "from python.loaders.events import raw_events\n" - "from sqlbuild.tasks import task\n\n" - "@task\n" + "from sqlbuild.assets import asset\n\n" + "@asset\n" "def consume_loader_result(ctx):\n" " '''Test task consume_loader_result.'''\n result = ctx.result_of(node_function=raw_events)\n" " output = Path(__file__).parents[2].joinpath('loader_result.txt')\n" @@ -1763,8 +1763,8 @@ def test_given_prior_loader_result_when_later_task_reads_result_then_uses_loader ), "python/tasks/prepare.py": ( "from sqlbuild.compiler.python_nodes.types import SkipMode\n" - "from sqlbuild.tasks import task\n\n" - "@task\n" + "from sqlbuild.assets import asset\n\n" + "@asset\n" "def prepare_events(ctx):\n" " '''Test task prepare_events.'''\n return ctx.skip(reason='no input', mode=SkipMode.HARD)\n" ), @@ -1901,8 +1901,8 @@ def test_given_local_target_override_when_running_python_then_every_context_sees "python/tasks/configured.py": ( "from providers.target_probe import TargetProbe\n" "from python.recorder import record\n" - "from sqlbuild.tasks import task\n\n" - "@task\n" + "from sqlbuild.assets import asset\n\n" + "@asset\n" "def configured_task(ctx, target_probe: TargetProbe):\n" " '''Test task configured_task.'''\n" " record(ctx, 'task')\n" @@ -2092,8 +2092,8 @@ def test_given_ctx_templated_target_schema_when_running_then_python_sees_no_load ), "python/tasks/configured.py": ( "from python.recorder import record\n" - "from sqlbuild.tasks import task\n\n" - "@task\n" + "from sqlbuild.assets import asset\n\n" + "@asset\n" "def configured_task(ctx):\n" ' """Test task configured_task."""\n' " record(ctx, 'task')\n" @@ -2179,16 +2179,16 @@ def test_given_same_node_results_in_multiple_targets_when_reading_then_uses_acti 'schema = "prod"\n' ), "python/tasks/produce.py": ( - "from sqlbuild.tasks import task\n\n" - "@task\n" + "from sqlbuild.assets import asset\n\n" + "@asset\n" "def produce_result(ctx):\n" " '''Test task produce_result.'''\n return ctx.result(payload={'target': ctx.target})\n" ), "python/tasks/consume.py": ( "from pathlib import Path\n" "from python.tasks.produce import produce_result\n" - "from sqlbuild.tasks import task\n\n" - "@task\n" + "from sqlbuild.assets import asset\n\n" + "@asset\n" "def consume_result(ctx):\n" " '''Test task consume_result.'''\n result = ctx.result_of(node_function=produce_result)\n" " output = Path(__file__).parents[2].joinpath('target_result.txt')\n" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_dag_json_behavior.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_dag_json_behavior.py index 66c65da442..b7bd642c92 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_dag_json_behavior.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_dag_json_behavior.py @@ -113,13 +113,13 @@ def test_given_waffle_shop_when_running_dag_json_then_it_reports_static_graph( expected_project_name="partner_scoring", expected_node_ids=( "model:orders", - "task:score_orders", + "asset:score_orders", "source:partner_scores", "model:scored_orders", ), expected_edge_pairs=( - ("model:orders", "task:score_orders"), - ("task:score_orders", "source:partner_scores"), + ("model:orders", "asset:score_orders"), + ("asset:score_orders", "source:partner_scores"), ("source:partner_scores", "model:scored_orders"), ), expected_check_ids=(), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_explicit_references.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_explicit_references.py index f99fbf8f81..901bf36e95 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_explicit_references.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_explicit_references.py @@ -58,15 +58,15 @@ _NULL_AUDIT: str = 'AUDIT ();\nSELECT * FROM __ref("@model") WHERE order_id IS NULL\n' _LITERAL_TASK: str = ( "from sqlbuild.refs import model\n" - "from sqlbuild.tasks import task\n\n\n" - '@task(depends_on=model("all_orders"))\n' + "from sqlbuild.assets import asset\n\n\n" + '@asset(depends_on=model("all_orders"))\n' "def export_orders(ctx):\n" ' """Test task export_orders."""\n ctx.query("SELECT count(*) FROM stg_customers")\n' ) _RUNTIME_TASK: str = ( "from sqlbuild.refs import model\n" - "from sqlbuild.tasks import task\n\n\n" - '@task(depends_on=model("all_orders"))\n' + "from sqlbuild.assets import asset\n\n\n" + '@asset(depends_on=model("all_orders"))\n' "def export_orders(ctx):\n" ' """Test task export_orders."""\n orders = ctx.relation(model("all_orders"))\n' ' ctx.query(f"SELECT count(*) FROM {orders}")\n' @@ -83,8 +83,8 @@ } _SEED_TASK: str = ( "from sqlbuild.refs import seed\n" - "from sqlbuild.tasks import task\n\n\n" - '@task(depends_on=seed("countries"))\n' + "from sqlbuild.assets import asset\n\n\n" + '@asset(depends_on=seed("countries"))\n' "def count_countries(ctx):\n" ' """Test task count_countries."""\n countries = ctx.relation(seed("countries"))\n' " ctx.execute_sql(\n" @@ -123,11 +123,11 @@ ) _SIBLING_TASKS: str = ( "from sqlbuild.refs import model\n" - "from sqlbuild.tasks import task\n\n\n" - '@task(depends_on=model("all_orders"))\n' + "from sqlbuild.assets import asset\n\n\n" + '@asset(depends_on=model("all_orders"))\n' "def export_summary(ctx):\n" ' """Test task export_summary."""\n ctx.execute_sql("CREATE OR REPLACE TABLE export_summary_ran AS SELECT 1 AS n")\n\n\n' - '@task(depends_on=model("stg_orders_eu"))\n' + '@asset(depends_on=model("stg_orders_eu"))\n' "def export_eu(ctx):\n" ' """Test task export_eu."""\n ctx.execute_sql("CREATE OR REPLACE TABLE export_eu_ran AS SELECT 1 AS n")\n' ) @@ -138,9 +138,9 @@ [ ExplicitReferencePythonDependencyE2ETestCase( description="task on a seed and check on a model build, order, select and check", - select=("+task:count_countries",), + select=("+asset:count_countries",), expected_dag_edges=( - ("seed:countries", "task:count_countries"), + ("seed:countries", "asset:count_countries"), ("model:all_orders", "check:orders_present"), ), expected_checked_asset_ids=("model:all_orders",), @@ -257,7 +257,7 @@ def test_given_unrendered_macro_argument_when_building_then_it_is_still_a_depend [ CrossBoundarySelectionE2ETestCase( description="both-way task selection skips a task reading its upstream", - select=("+task:export_summary+",), + select=("+asset:export_summary+",), expected_ran=("export_summary_ran",), expected_not_ran=("export_eu_ran",), ), @@ -386,7 +386,7 @@ def test_given_typed_macro_references_and_hook_reads_when_building_then_graph_an command=("--no-color", "compile"), expected_exit_code=1, expected_output_fragments=( - "error[P008]: task:export_orders names model:stg_customers as 'stg_customers'", + "error[P008]: asset:export_orders names model:stg_customers as 'stg_customers'", "--> python/tasks/export.py:8", "[references]\n enforce_explicit = false", ), @@ -581,7 +581,7 @@ def test_given_check_runtime_hard_coded_relation_when_building_then_it_warns_wit "orders_base()", "error[P006]: model:hidden_customers depends on model:stg_customers through macro " "customers_base()", - "error[P008]: task:export_orders names model:stg_customers as 'stg_customers'", + "error[P008]: asset:export_orders names model:stg_customers as 'stg_customers'", "3 errors", ), expected_json_diagnostics=( @@ -593,7 +593,7 @@ def test_given_check_runtime_hard_coded_relation_when_building_then_it_warns_wit ), ( "P008", - "task:export_orders names model:stg_customers as 'stg_customers' in SQL " + "asset:export_orders names model:stg_customers as 'stg_customers' in SQL " "passed to ctx.query()", ), ), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_node_result_location.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_node_result_location.py index 1a009b12f8..0cc1d05be1 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_node_result_location.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_node_result_location.py @@ -34,8 +34,8 @@ ' """Raw orders."""\n _ = ctx\n return [{"order_id": 1}]\n' ) _TASK: str = ( - "from sqlbuild.refs import source\nfrom sqlbuild.tasks import task\n\n\n" - '@task(depends_on=source("raw_orders"))\n' + "from sqlbuild.refs import source\nfrom sqlbuild.assets import asset\n\n\n" + '@asset(depends_on=source("raw_orders"))\n' "def announce_orders(ctx):\n" ' """Announce the loaded orders."""\n' " return ctx.result(payload={'announced': True})\n" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_identity.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_identity.py index 8d595bfc27..e521a5f57f 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_identity.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_identity.py @@ -196,10 +196,10 @@ def normalize_status(value): "python/orders.py": dedent( """ import python.helpers.clean - from sqlbuild.tasks import task + from sqlbuild.assets import asset - @task + @asset def orders(ctx): '''Test task orders.''' return ctx.result( diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_selection.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_selection.py index 9f94302b15..8b2ba77a65 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_selection.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_selection.py @@ -40,7 +40,7 @@ [ PythonNodeSelectionE2ETestCase( description="task selected alone reads the existing managed source table", - select=("task:count_customers",), + select=("asset:count_customers",), expected_exit_code=0, expected_output_patterns=(r"count_customers\s+OK",), expected_row_counts={"raw_customers": 3}, @@ -56,7 +56,7 @@ ), PythonNodeSelectionE2ETestCase( description="task on an unmanaged source runs when selected alone", - select=("task:count_raw_orders",), + select=("asset:count_raw_orders",), expected_exit_code=0, expected_output_patterns=(r"count_raw_orders\s+OK",), expected_row_counts={}, @@ -64,7 +64,7 @@ ), PythonNodeSelectionE2ETestCase( description="task on an unmanaged source runs when selected with upstreams", - select=("+task:count_raw_orders",), + select=("+asset:count_raw_orders",), expected_exit_code=0, expected_output_patterns=(r"count_raw_orders\s+OK",), expected_row_counts={}, @@ -72,7 +72,7 @@ ), PythonNodeSelectionE2ETestCase( description="task selected alone reads the existing unselected model", - select=("task:count_orders",), + select=("asset:count_orders",), existing_sql=_EXISTING_ORDERS, expected_exit_code=0, expected_output_patterns=(r"count_orders\s+OK",), @@ -90,7 +90,7 @@ ), PythonNodeSelectionE2ETestCase( description="task waits for its selected upstream model", - select=("+task:count_orders",), + select=("+asset:count_orders",), expected_exit_code=0, expected_output_patterns=(r"count_orders\s+OK",), expected_row_counts={"orders": 2}, @@ -130,7 +130,7 @@ ), PythonNodeSelectionE2ETestCase( description="task is skipped when its selected upstream model fails", - select=("+task:count_orders",), + select=("+asset:count_orders",), orders_sql=_FAILING_ORDERS_SQL, expected_exit_code=1, expected_output_patterns=( @@ -154,7 +154,7 @@ ), PythonNodeSelectionE2ETestCase( description="task is skipped when its selected upstream model is skipped", - select=("+task:count_order_summary",), + select=("+asset:count_order_summary",), orders_sql=_FAILING_ORDERS_SQL, expected_exit_code=1, expected_output_patterns=( @@ -206,12 +206,12 @@ def test_given_python_node_selection_when_building_then_it_gates_only_on_schedul [ PythonUpstreamResultE2ETestCase( description="a task reads its unselected upstream task's latest result", - select="task:report_order_count", - upstream_select="task:count_orders", + select="asset:report_order_count", + upstream_select="asset:count_orders", expected_error=( "Upstream task 'count_orders' has no successful result to read. Build it first " - "with 'sqb build --select task:count_orders', or build both with " - "'sqb build --select +task:report_order_count'." + "with 'sqb build --select asset:count_orders', or build both with " + "'sqb build --select +asset:report_order_count'." ), ) ], diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_source_deferral.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_source_deferral.py index 244dbfa4ae..fa8c3e8437 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_source_deferral.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_source_deferral.py @@ -51,8 +51,8 @@ 'MODEL (description "Test model order_marker.", materialized table, post_hooks [python("count_raw")]);\nSELECT 1 AS n\n' ), "python/tasks/count.py": ( - "from sqlbuild.refs import source\nfrom sqlbuild.tasks import task\n\n\n" - '@task(depends_on=source("raw_orders"))\n' + "from sqlbuild.refs import source\nfrom sqlbuild.assets import asset\n\n\n" + '@asset(depends_on=source("raw_orders"))\n' "def count_task(ctx):\n" ' """Test task count_task."""\n raw = ctx.relation(source("raw_orders"))\n' ' ctx.execute_sql(f"CREATE OR REPLACE TABLE main.task_counts AS ' @@ -82,7 +82,7 @@ [ PythonSourceDeferralE2ETestCase( description="hook and task read the deferred source a model reads", - select=("order_count", "task:count_task"), + select=("order_count", "asset:count_task"), expected_counts={"dev.order_count": 2, "main.hook_counts": 2, "main.task_counts": 2}, expected_check_pattern=_CHECK_PASSED, ), @@ -152,7 +152,7 @@ def test_given_only_a_task_selected_when_building_then_it_reads_its_source_like_ execute_duckdb(db_path=database, sql=_ENVIRONMENT_ROWS) build: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build", "--select", "task:count_task"), project_dir=project_dir + command=("--no-color", "build", "--select", "asset:count_task"), project_dir=project_dir ) output: str = build.stdout + build.stderr diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/check/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/check/helpers.py index 0471c3810d..0c01ab6572 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/check/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/check/helpers.py @@ -24,16 +24,16 @@ def prepare_python_check_project(*, tmp_path: Path) -> Path: database = "python_check_project.duckdb" """, "python/tasks/orders.py": """ -from sqlbuild.tasks import task +from sqlbuild.assets import asset -@task() +@asset() def export_orders(ctx): '''Test task export_orders.''' return ctx.result(payload={"rows": 3}, metadata={"rows": 3}) -@task() +@asset() def export_customers(ctx): '''Test task export_customers.''' return ctx.result(payload={"rows": 2}, metadata={"rows": 2}) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_required_descriptions.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_required_descriptions.py index fedac2da3e..987fcf210e 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_required_descriptions.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_required_descriptions.py @@ -39,7 +39,6 @@ "sql_hook_files": "hook", "hook_functions": "hook", "loader_functions": "loader", - "task_functions": "task", "asset_functions": "asset", "check_functions": "check", "providers": "provider", @@ -406,7 +405,7 @@ ), expected_diagnostic=("P010", "python/tasks/refresh_exports.py", 4), expected_message="task 'refresh_exports' has no description", - expected_help_fragments=('@task(description="What refresh_exports does", ...)',), + expected_help_fragments=('@asset(description="What refresh_exports does", ...)',), ), RequiredDescriptionCase( description="asset", diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/databricks/test_databricks.py b/tests/e2e/src/sqlbuild/cli/commands/main/databricks/test_databricks.py index c8ca0e1e46..0201290c7d 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/databricks/test_databricks.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/databricks/test_databricks.py @@ -163,8 +163,8 @@ def test_given_python_result_when_running_check_on_databricks_then_persists_node schema_name=schema_name, ), "python/tasks/results.py": ( - "from sqlbuild.tasks import task\n\n" - "@task\n" + "from sqlbuild.assets import asset\n\n" + "@asset\n" "def produce_result(ctx):\n" " '''Test task produce_result.'''\n return ctx.result(payload={'value': 42}, metadata={'source': 'databricks'})\n" ), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/init/test_init.py b/tests/e2e/src/sqlbuild/cli/commands/main/init/test_init.py index 4706f2ae5f..3b932b8ce0 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/init/test_init.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/init/test_init.py @@ -75,8 +75,8 @@ def test_given_empty_project_directory_when_running_init_then_typed_resource_dir "python/_helpers.py": "def order_count(rows):\n return len(rows)\n", "python/tasks/orders.py": ( "from python._helpers import order_count\n" - "from sqlbuild.tasks import task\n\n\n" - "@task\n" + "from sqlbuild.assets import asset\n\n\n" + "@asset\n" "def count_orders(ctx):\n" " '''Test task count_orders.'''\n return ctx.result(payload={'order_count': order_count([1, 2])})\n" ), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/load/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/load/helpers.py index a35a991aea..86c38f1475 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/load/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/load/helpers.py @@ -55,8 +55,8 @@ def assert_source_only_ingress_dependency_case( ), "python/tasks/prepare.py": ( "from pathlib import Path\n" - "from sqlbuild.tasks import task\n\n" - "@task\n" + "from sqlbuild.assets import asset\n\n" + "@asset\n" "def prepare_events(ctx):\n" " '''Test task prepare_events.'''\n" " Path(__file__).parents[2].joinpath('prepared.txt').write_text('prepared')\n" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/load/test_load.py b/tests/e2e/src/sqlbuild/cli/commands/main/load/test_load.py index 483e1c7853..2f09bd5af3 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/load/test_load.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/load/test_load.py @@ -604,7 +604,7 @@ def test_given_chained_loader_project_when_running_source_model_then_runs_loader expected_return_code=1, expected_error_fragment="No persisted result found for Python node 'prepare_events'", dependency_dir="tasks", - decorator_import="from sqlbuild.tasks import task\n", + decorator_import="from sqlbuild.assets import asset\n", decorator_name="task", ), SourceOnlyComplexIngressE2ETestCase( diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/model_migrations/test_old_name_views.py b/tests/e2e/src/sqlbuild/cli/commands/main/model_migrations/test_old_name_views.py index fad925b216..0dec764406 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/model_migrations/test_old_name_views.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/model_migrations/test_old_name_views.py @@ -41,8 +41,8 @@ _ALL_FACTS: tuple[str, ...] = ("required", "origin_archived", "view_created") _THREE_ORDERS: tuple[tuple[int, int], ...] = ((1, 101), (2, 102), (3, 103)) _REPORT_TASK: str = ( - "from sqlbuild.tasks import task\n\n\n" - "@task\n" + "from sqlbuild.assets import asset\n\n\n" + "@asset\n" "def report(ctx):\n" f' """Test task report."""\n ctx.query("SELECT count(*) FROM {OLD_NAME_SCHEMA}.{OLD_NAME_ORIGIN}").fetchall()\n' ) @@ -425,7 +425,7 @@ def test_given_live_compatibility_view_when_new_model_claims_its_name_then_build header=OLD_NAME_MIGRATE_FROM, command="compile", expected_fragment=( - "error[P008]: task:report names 'analytics.revenue' in SQL passed to ctx.query(); " + "error[P008]: asset:report names 'analytics.revenue' in SQL passed to ctx.query(); " "it is the old name of model:daily_revenue" ), ), @@ -434,7 +434,7 @@ def test_given_live_compatibility_view_when_new_model_claims_its_name_then_build header="", command="build", expected_fragment=( - "task:report names 'analytics.revenue' in SQL passed to ctx.query(); " + "asset:report names 'analytics.revenue' in SQL passed to ctx.query(); " "it is the old name of model:daily_revenue" ), ), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/postgres/test_postgres.py b/tests/e2e/src/sqlbuild/cli/commands/main/postgres/test_postgres.py index 8cdd464dc8..55ab558d6b 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/postgres/test_postgres.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/postgres/test_postgres.py @@ -259,8 +259,8 @@ def test_given_python_result_when_running_check_on_postgres_then_persists_node_r config=postgres_e2e_config, ), "python/tasks/results.py": ( - "from sqlbuild.tasks import task\n\n" - "@task\n" + "from sqlbuild.assets import asset\n\n" + "@asset\n" "def produce_result(ctx):\n" " '''Test task produce_result.'''\n return ctx.result(payload={'value': 42}, metadata={'source': 'postgres'})\n" ), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/providers/test_event_exporters.py b/tests/e2e/src/sqlbuild/cli/commands/main/providers/test_event_exporters.py index 738722509e..3ccbbb722c 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/providers/test_event_exporters.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/providers/test_event_exporters.py @@ -611,10 +611,10 @@ def export_event(event: LifecycleEvent, shared_provider: SharedProvider) -> None "python/tasks/uses_shared.py": dedent( """ from providers.z_shared import SharedProvider - from sqlbuild.tasks import task + from sqlbuild.assets import asset - @task + @asset def uses_shared(ctx, shared_provider: SharedProvider): '''Test task uses_shared.''' del ctx, shared_provider diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/providers/test_providers.py b/tests/e2e/src/sqlbuild/cli/commands/main/providers/test_providers.py index 709b490aa1..f6e30f179f 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/providers/test_providers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/providers/test_providers.py @@ -146,10 +146,10 @@ def raw_orders(marker_provider: MarkerProvider): "python/tasks/publish_orders.py": dedent( """ from providers.marker import MarkerProvider - from sqlbuild.tasks import task + from sqlbuild.assets import asset - @task + @asset def publish_orders(marker_provider: MarkerProvider): '''Test task publish_orders.''' marker_provider.label @@ -326,10 +326,10 @@ def mark(self, value): "python/tasks/provider_task.py": dedent( """ from providers.marker import MarkerProvider - from sqlbuild.tasks import task + from sqlbuild.assets import asset - @task + @asset def provider_task(ctx, marker_provider: MarkerProvider): '''Test task provider_task.''' marker_provider.mark(f"run_task:{marker_provider.label}") @@ -448,10 +448,10 @@ def raw_provider_events(ctx, marker_provider: MarkerProvider): "python/tasks/failing_task.py": dedent( """ from providers.marker import MarkerProvider - from sqlbuild.tasks import task + from sqlbuild.assets import asset - @task + @asset def failing_task(ctx, marker_provider: MarkerProvider): '''Test task failing_task.''' marker_provider.mark("task") @@ -599,10 +599,10 @@ def test_given_provider_project_when_running_compile_or_plan_then_provider_setup "python/tasks/provider_task.py": dedent( """ from providers.marker import MarkerProvider - from sqlbuild.tasks import task + from sqlbuild.assets import asset - @task + @asset def provider_task(ctx, marker_provider: MarkerProvider): '''Test task provider_task.''' marker_provider.mark("task") @@ -653,7 +653,7 @@ def test_given_alias_imported_provider_annotation_when_running_command_then_cli_ import sys from pathlib import Path - from sqlbuild.tasks import task + from sqlbuild.assets import asset provider_path = Path(__file__).parents[2] / "providers" / "marker.py" spec = importlib.util.spec_from_file_location("alias_marker", provider_path) @@ -664,7 +664,7 @@ def test_given_alias_imported_provider_annotation_when_running_command_then_cli_ AliasMarkerProvider = alias_marker.MarkerProvider - @task + @asset def alias_task(ctx, marker_provider: AliasMarkerProvider): '''Test task alias_task.''' marker_provider.mark("task") diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/review/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/review/helpers.py index 9babb9f6f5..9785341dec 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/review/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/review/helpers.py @@ -128,11 +128,11 @@ def export_task_module() -> str: return dedent( """ from sqlbuild.refs import model - from sqlbuild.tasks import TaskContext, task + from sqlbuild.assets import AssetContext, asset - @task(depends_on=model("order_totals")) - def export_totals(ctx: TaskContext): + @asset(depends_on=model("order_totals")) + def export_totals(ctx: AssetContext): \"\"\"Export order totals.\"\"\" return ctx.result(metadata={"rows": 0}) """ diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/review/test_review.py b/tests/e2e/src/sqlbuild/cli/commands/main/review/test_review.py index 3a7dd946e9..f0e3ca3f3a 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/review/test_review.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/review/test_review.py @@ -131,8 +131,8 @@ expected_exit_code=0, expected_status="passed", expected_validated=("stg_orders", "order_totals"), - expected_not_validated=("open_orders", "order_ledger", "task:export_totals"), - expected_node_statuses={"task:export_totals": "not_validated"}, + expected_not_validated=("open_orders", "order_ledger", "asset:export_totals"), + expected_node_statuses={"asset:export_totals": "not_validated"}, ), ReviewOutcomeE2ETestCase( description="a changed seed is built but never passes uncompared", diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/snowflake/test_snowflake.py b/tests/e2e/src/sqlbuild/cli/commands/main/snowflake/test_snowflake.py index 3a5a7c0790..5e537553ea 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/snowflake/test_snowflake.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/snowflake/test_snowflake.py @@ -178,8 +178,8 @@ def test_given_python_result_when_running_check_on_snowflake_then_persists_node_ schema_name=schema_name, ), "python/tasks/results.py": ( - "from sqlbuild.tasks import task\n\n" - "@task\n" + "from sqlbuild.assets import asset\n\n" + "@asset\n" "def produce_result(ctx):\n" " '''Test task produce_result.'''\n return ctx.result(payload={'value': 42}, metadata={'source': 'snowflake'})\n" ), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/sqlserver/test_sqlserver.py b/tests/e2e/src/sqlbuild/cli/commands/main/sqlserver/test_sqlserver.py index e89226e2d3..19cf6e2add 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/sqlserver/test_sqlserver.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/sqlserver/test_sqlserver.py @@ -433,8 +433,8 @@ def test_given_python_result_when_running_check_on_sqlserver_then_persists_node_ config=config, ), "python/tasks/results.py": ( - "from sqlbuild.tasks import task\n\n" - "@task\n" + "from sqlbuild.assets import asset\n\n" + "@asset\n" "def produce_result(ctx):\n" " '''Test task produce_result.'''\n return ctx.result(payload={'value': 42}, metadata={'source': 'sqlserver'})\n" ), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/helpers.py index bfa49fbea8..0d073c5664 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/test/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/helpers.py @@ -2373,8 +2373,8 @@ def build_since_indirect_files() -> dict[str, str]: "__expected__archive_counts AS (SELECT 1 AS archived)\nSELECT 1\n" ), "python/tasks/report.py": ( - "from sqlbuild.tasks import task\n\n\n" - "@task()\ndef report_orders(ctx):\n" + "from sqlbuild.assets import asset\n\n\n" + "@asset()\ndef report_orders(ctx):\n" ' """Report orders."""\n\n' " return ctx.result(payload={})\n" ), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_python_tests.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_python_tests.py index a051644451..5885768399 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_python_tests.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_python_tests.py @@ -38,16 +38,16 @@ "SELECT 1\n" ), "python/orders/storefront_import/node.py": ( - "from sqlbuild.tasks import task\n\n\n" - "@task(description='Import storefront orders.', tags=['ingest'])\n" + "from sqlbuild.assets import asset\n\n\n" + "@asset(description='Import storefront orders.', tags=['ingest'])\n" "def storefront_import(ctx):\n return None\n" ), "python/orders/storefront_import/_helpers/row_mapping.py": ( "def map_row(raw):\n return {'status': str(raw['state']).lower()}\n" ), "python/inventory/stock_sync/node.py": ( - "from sqlbuild.tasks import task\n\n\n" - "@task(description='Sync stock.')\n" + "from sqlbuild.assets import asset\n\n\n" + "@asset(description='Sync stock.')\n" "def stock_sync(ctx):\n return None\n" ), } diff --git a/tests/e2e/src/sqlbuild/integrations/dagster/helpers.py b/tests/e2e/src/sqlbuild/integrations/dagster/helpers.py index 45444992a3..412363d24d 100644 --- a/tests/e2e/src/sqlbuild/integrations/dagster/helpers.py +++ b/tests/e2e/src/sqlbuild/integrations/dagster/helpers.py @@ -69,11 +69,11 @@ def prepare_python_nodes_integration_project(root: Path) -> Path: "from sqlbuild.factories import factory", "from sqlbuild.loaders import loader", "from sqlbuild.refs import model", - "from sqlbuild.tasks import task", + "from sqlbuild.assets import asset", "", "@factory", "def generated_nodes():", - " @task(depends_on=model('orders'), tags=['daily'], group='python')", + " @asset(depends_on=model('orders'), tags=['daily'], group='python')", " def prepare_orders(ctx):", " '''Test task prepare_orders.'''", " return ctx.result(payload={'rows': 1})", diff --git a/tests/goldens/compiler/failure/hard-coded-relation-in-python.json b/tests/goldens/compiler/failure/hard-coded-relation-in-python.json index 6002c32dfd..b4155ad796 100644 --- a/tests/goldens/compiler/failure/hard-coded-relation-in-python.json +++ b/tests/goldens/compiler/failure/hard-coded-relation-in-python.json @@ -8,7 +8,7 @@ "phase": "compile", "severity": "error", "code": "P008", - "message": "task:export_orders names model:stg_orders as 'stg_orders' in SQL passed to ctx.query()", + "message": "asset:export_orders names model:stg_orders as 'stg_orders' in SQL passed to ctx.query()", "path": "python/tasks/exports.py", "line": 7, "column": 1, diff --git a/tests/goldens/compiler/failure/python-node-missing-description.json b/tests/goldens/compiler/failure/python-node-missing-description.json index 139033f6af..ab92ad8d57 100644 --- a/tests/goldens/compiler/failure/python-node-missing-description.json +++ b/tests/goldens/compiler/failure/python-node-missing-description.json @@ -18,7 +18,7 @@ "line": 4, "column": 1 }, - "help": "to describe task 'export_orders', give its function in python/tasks/exports.py a docstring or pass a description to @task:\n @task(description=\"What export_orders does\", ...)" + "help": "to describe task 'export_orders', give its function in python/tasks/exports.py a docstring or pass a description to @asset:\n @asset(description=\"What export_orders does\", ...)" } ] } diff --git a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py index 0be7c535d1..63f4688f68 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py @@ -968,8 +968,8 @@ def write_provider_task_project(*, project_dir: Path, annotation: str) -> None: "providers/orders.py": PROVIDER_SOURCE, "python/orders.py": ( "from providers.orders import OrdersProvider\n" - "from sqlbuild.tasks import task\n\n\n" - "@task\n" + "from sqlbuild.assets import asset\n\n\n" + "@asset\n" f"def refresh_orders(ctx, orders_provider: {annotation}):\n" ' """Refresh the order snapshot."""\n' " return None\n" @@ -1397,16 +1397,16 @@ def write_python_rules_project(*, project_dir: Path, fixture: str, rules_toml: s "python/crash.py": ( '"""A task that kills its process, as a crash or kill -9 would."""\n\n' "import os\n\n" - "from sqlbuild.tasks import task\n\n\n" - "@task()\n" + "from sqlbuild.assets import asset\n\n\n" + "@asset()\n" "def crash(ctx):\n" ' """End the process without unwinding."""\n\n' " os._exit(137)\n" ), "python/fail.py": ( '"""A task that raises, so the build exits non-zero."""\n\n' - "from sqlbuild.tasks import task\n\n\n" - "@task()\n" + "from sqlbuild.assets import asset\n\n\n" + "@asset()\n" "def fail(ctx):\n" ' """Raise."""\n\n' ' raise RuntimeError("task failed")\n' diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_run_records.py b/tests/integration/src/sqlbuild/cli/commands/main/test_run_records.py index 3a20c3a7ad..6b1c7e29a1 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_run_records.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_run_records.py @@ -98,14 +98,14 @@ ), RunRecordTestCase( description="a failing task fails the run even though no model failed", - build_args=("--select", "task:fail"), + build_args=("--select", "asset:fail"), expected_exit_code=1, expected_rows=[ - ("started", "build", '["task:fail"]', "[]", "[]", None, None), + ("started", "build", '["asset:fail"]', "[]", "[]", None, None), ( "finished", "build", - '["task:fail"]', + '["asset:fail"]', "[]", "[]", "failed", @@ -297,7 +297,7 @@ def test_given_a_killed_build_when_reading_its_runs_then_only_started_remains( "build", "--no-tests", "--select", - "task:crash", + "asset:crash", ], capture_output=True, text=True, diff --git a/tests/integration/src/sqlbuild/cli/recall/helpers.py b/tests/integration/src/sqlbuild/cli/recall/helpers.py index fd51e8e343..c54a3e1ef8 100644 --- a/tests/integration/src/sqlbuild/cli/recall/helpers.py +++ b/tests/integration/src/sqlbuild/cli/recall/helpers.py @@ -52,8 +52,8 @@ "import sys\n" "import threading\n" "import time\n\n" - "from sqlbuild.tasks import task\n\n\n" - "@task()\n" + "from sqlbuild.assets import asset\n\n\n" + "@asset()\n" "def emit_bytes(ctx):\n" ' """Write binary stdout, UTF-8 stderr, then more stdout."""\n\n' " sys.stdout.flush()\n" diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_dev.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_dev.py index 184e8771db..4ac098b1db 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_dev.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_dev.py @@ -236,10 +236,10 @@ def test_given_development_start_when_location_is_ready_then_project_python_is_d project: Path = tmp_path / "project" imports: Path = tmp_path / "python_imports.txt" probe: str = ( - "import os\nfrom pathlib import Path\nfrom sqlbuild.tasks import task\n" + "import os\nfrom pathlib import Path\nfrom sqlbuild.assets import asset\n" f"with Path({str(imports)!r}).open('a') as imported:\n" " imported.write(f'{os.getpid()}\\n')\n" - "@task(description='Probe.')\n" + "@asset(description='Probe.')\n" "def import_probe(ctx):\n return None\n" ) write_files( diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py index d38ad1f21e..675d780148 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -456,8 +456,8 @@ def test_given_a_compiled_project_when_publishing_then_its_column_lineage_is_a_s ), "python/tasks/publish_orders.py": ( "from providers.fulfillment import FulfillmentApi\n" - "from sqlbuild.tasks import task\n\n\n" - "@task\n" + "from sqlbuild.assets import asset\n\n\n" + "@asset\n" "def publish_orders(fulfillment_api: FulfillmentApi):\n" ' """Send orders to fulfillment."""\n' ), @@ -537,10 +537,14 @@ def test_given_a_compiled_project_when_publishing_then_its_column_lineage_is_a_s }, ], "used_by": [ - {"kind": "task", "name": "publish_orders"}, + {"kind": "asset", "name": "publish_orders"}, {"kind": "hook", "name": "notify_fulfillment"}, ], - "nodes": ["hook:notify_fulfillment", "model:orders", "task:publish_orders"], + "nodes": [ + "hook:notify_fulfillment", + "model:orders", + "asset:publish_orders", + ], } ] }, @@ -2204,22 +2208,22 @@ def test_given_projects_compiled_in_one_process_when_one_imports_orchestration_t ), PythonJobSelectionTestCase( description="a task selector selects only the task", - select="task:request_carrier_collection", - expected_selection=("task:request_carrier_collection",), + select="asset:request_carrier_collection", + expected_selection=("asset:request_carrier_collection",), ), PythonJobSelectionTestCase( description="typed selectors select the task and the asset it depends on", - select="task:request_carrier_collection asset:partner_orders_file", - expected_selection=("asset:partner_orders_file", "task:request_carrier_collection"), + select="asset:request_carrier_collection asset:partner_orders_file", + expected_selection=("asset:partner_orders_file", "asset:request_carrier_collection"), ), PythonJobSelectionTestCase( description="upstream of a task crosses its Python and SQL dependencies", - select="+task:request_carrier_collection", + select="+asset:request_carrier_collection", expected_selection=( "asset:partner_orders_file", "model:orders", "model:stg_orders", - "task:request_carrier_collection", + "asset:request_carrier_collection", ), ), PythonJobSelectionTestCase( @@ -2234,7 +2238,7 @@ def test_given_projects_compiled_in_one_process_when_one_imports_orchestration_t "asset:partner_orders_file", "model:orders", "model:stg_orders", - "task:request_carrier_collection", + "asset:request_carrier_collection", ), ), ], @@ -2299,14 +2303,14 @@ def test_given_python_dependencies_across_jobs_when_publishing_a_group_then_jobs PythonJobGroupCycleTestCase( description="a Python edge that closes a cycle names both jobs and the edges", jobs=( - ("staging_job", "stg_orders task:request_carrier_collection"), + ("staging_job", "stg_orders asset:request_carrier_collection"), ("export_job", "asset:partner_orders_file"), ), expected_message=( "group dependencies contain a cycle in job group orders_nightly_job_group: " "export_job waits for staging_job (model:orders reads model:stg_orders); " "staging_job waits for export_job " - "(task:request_carrier_collection reads asset:partner_orders_file)" + "(asset:request_carrier_collection reads asset:partner_orders_file)" ), ), ], diff --git a/tests/integration/src/sqlbuild/compiler/explicit_references/helpers.py b/tests/integration/src/sqlbuild/compiler/explicit_references/helpers.py index 6c5975fefe..789c586e04 100644 --- a/tests/integration/src/sqlbuild/compiler/explicit_references/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/explicit_references/helpers.py @@ -57,8 +57,8 @@ def _hook( def _task(body: str, depends_on: str = 'model("orders")') -> str: return ( - "from sqlbuild.refs import model, source\nfrom sqlbuild.tasks import task\n\n\n" - f"@task(depends_on={depends_on})\ndef export_orders(ctx):\n{body}" + "from sqlbuild.refs import model, source\nfrom sqlbuild.assets import asset\n\n\n" + f"@asset(depends_on={depends_on})\ndef export_orders(ctx):\n{body}" ) diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py b/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py index e51e7ecab9..99922d006a 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py @@ -270,8 +270,8 @@ def set_dependency_project( 'select = "stg_orders orders order_items"\n' ), "python/tasks/report.py": ( - "from sqlbuild.refs import job, model, tag\nfrom sqlbuild.tasks import task\n\n\n" - f"@task(depends_on={depends_on})\n" + "from sqlbuild.refs import job, model, tag\nfrom sqlbuild.assets import asset\n\n\n" + f"@asset(depends_on={depends_on})\n" 'def report(ctx):\n """Test task report."""\n return None\n' ), } diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py index 351e5001f9..a4ab39c006 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py @@ -436,8 +436,8 @@ def test_given_project_files_when_running_compile_pipeline_then_produces_valid_o "sqlbuild_project.toml": _PROJECT_TOML, "models/orders.sql": "MODEL (description 'Test model orders.', materialized table);\n\nSELECT 1 AS order_id", "python/tasks/orders.py": ( - "from sqlbuild.tasks import task\n\n" - "@task\n" + "from sqlbuild.assets import asset\n\n" + "@asset\n" "def prepare_orders(ctx):\n" " '''Test task prepare_orders.'''\n return ctx.result(payload={'ok': True})\n" ), @@ -460,7 +460,7 @@ def test_given_python_run_selector_when_running_compile_pipeline_then_tracks_pyt result: CompilePipelineResult = run_compile_pipeline_for_project( project_dir=tmp_path, adapter=DuckDbAdapter(), - select=("task:prepare_orders",), + select=("asset:prepare_orders",), resolve_python_run_selectors=True, ) diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_python_source_reads.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_python_source_reads.py index ff5ca6cf19..9226cd4452 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_python_source_reads.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_python_source_reads.py @@ -33,8 +33,8 @@ ), "models/order_marker.sql": "MODEL (description 'Test model order_marker.', materialized table);\nSELECT 1 AS n\n", "python/tasks/count.py": ( - "from sqlbuild.refs import source\nfrom sqlbuild.tasks import task\n\n\n" - '@task(depends_on=source("raw_orders"))\n' + "from sqlbuild.refs import source\nfrom sqlbuild.assets import asset\n\n\n" + '@asset(depends_on=source("raw_orders"))\n' "def count_task(ctx):\n" " '''Test task count_task.'''\n ctx.query(f\"SELECT count(*) FROM {ctx.relation(source('raw_orders'))}\")\n" ), @@ -73,7 +73,7 @@ def test_given_only_a_task_selected_when_planning_then_its_source_resolves_like_ result: CompilePipelineResult = run_compile_pipeline_for_project( project_dir=tmp_path, adapter=DuckDbAdapter(), - select=("task:count_task",), + select=("asset:count_task",), resolve_python_run_selectors=True, ) diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_set_dependencies.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_set_dependencies.py index ef20d12692..f05798acd3 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_set_dependencies.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_set_dependencies.py @@ -42,7 +42,7 @@ ), ( "orchestration/jobs/reporting_job.toml", - 'name = "reporting_job"\ndescription = "Reporting."\nselect = "task:report customers"\n', + 'name = "reporting_job"\ndescription = "Reporting."\nselect = "asset:report customers"\n', ), ( "orchestration/job_groups/daily_job_group.toml", @@ -93,7 +93,7 @@ def test_given_set_dependency_when_planning_the_tasks_upstream_then_selects_the_ discovered_inputs=discovered, adapter=DuckDbAdapter(), options=CompilePipelineOptions( - no_sql_validation=True, select=("+task:report",), resolve_python_run_selectors=True + no_sql_validation=True, select=("+asset:report",), resolve_python_run_selectors=True ), hooks=ConnectionHooks(), ) diff --git a/tests/integration/src/sqlbuild/runtime/execution_history/test_cli_concurrent_history.py b/tests/integration/src/sqlbuild/runtime/execution_history/test_cli_concurrent_history.py index 120ae60668..aaf5b777c9 100644 --- a/tests/integration/src/sqlbuild/runtime/execution_history/test_cli_concurrent_history.py +++ b/tests/integration/src/sqlbuild/runtime/execution_history/test_cli_concurrent_history.py @@ -58,7 +58,7 @@ def test_given_explicit_cli_history_when_two_workers_publish_then_events_and_pro def execute_attempt(resource_name: str) -> None: with ResourceAttemptLifecycle( - resource_id=f"task:{resource_name}", + resource_id=f"asset:{resource_name}", resource_kind="task", resource_name=resource_name, run_id="concurrent-run", diff --git a/tests/unit/scripts/compiler_differential/_helpers/coverage/test_analysis.py b/tests/unit/scripts/compiler_differential/_helpers/coverage/test_analysis.py index b118c6a866..95de261ad9 100644 --- a/tests/unit/scripts/compiler_differential/_helpers/coverage/test_analysis.py +++ b/tests/unit/scripts/compiler_differential/_helpers/coverage/test_analysis.py @@ -251,7 +251,7 @@ description="literal_sql_relations_need_explicit_references", capture=empty_compiled_capture( enforce_explicit_references=False, - unmatched_literal_sql_relations=[{"owner_label": "task:export"}], + unmatched_literal_sql_relations=[{"owner_label": "asset:export"}], ), expected_present=frozenset({"unmatched_literal_sql_relations"}), expected_absent=frozenset({"python_sql_literal_relation"}), diff --git a/tests/unit/scripts/compiler_differential/_helpers/coverage/test_discovery.py b/tests/unit/scripts/compiler_differential/_helpers/coverage/test_discovery.py index 48096f0322..8e5baa0f0f 100644 --- a/tests/unit/scripts/compiler_differential/_helpers/coverage/test_discovery.py +++ b/tests/unit/scripts/compiler_differential/_helpers/coverage/test_discovery.py @@ -282,14 +282,14 @@ def test_given_discovered_project_inputs_when_listing_required_kinds_then_collec CaptureProblemsTestCase( description="sound_capture", capture=empty_inputs_capture( - task_functions=[{"function": {"__callable__": "python.tasks.orders:export"}}] + asset_functions=[{"function": {"__callable__": "python.assets.orders:export"}}] ), expected_problem_count=0, ), CaptureProblemsTestCase( description="opaque_value_and_unnamed_callable", capture=empty_inputs_capture( - task_functions=[{"function": {"__opaque__": "orders:Exporter"}}] + asset_functions=[{"function": {"__opaque__": "orders:Exporter"}}] ), expected_problem_count=2, ), diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py index 214b5f7956..4c3e972d42 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py @@ -178,15 +178,15 @@ def test_given_dag_flag_when_running_compile_then_writes_dag_artifact( expected_project_name="offline_compile", expected_node_ids={ "model:orders", - "task:prepare_orders", + "asset:prepare_orders", "loader:warehouse_export", "asset:orders_export", "check:check_orders_export", "check:check_loader_export", }, expected_edges={ - ("model:orders", "task:prepare_orders"), - ("task:prepare_orders", "loader:warehouse_export"), + ("model:orders", "asset:prepare_orders"), + ("asset:prepare_orders", "loader:warehouse_export"), ("loader:warehouse_export", "asset:orders_export"), ("loader:warehouse_export", "check:check_loader_export"), }, diff --git a/tests/unit/src/sqlbuild/cli/commands/main/dag/helpers.py b/tests/unit/src/sqlbuild/cli/commands/main/dag/helpers.py index 144dcae048..7cb0c71b02 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/dag/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/dag/helpers.py @@ -89,10 +89,10 @@ def prepare_python_dag_project(root: Path) -> Path: (tasks_dir / "prepare_orders.py").write_text( "\n".join( ( - "from sqlbuild.tasks import task", + "from sqlbuild.assets import asset", "from sqlbuild.refs import model", "", - "@task(", + "@asset(", " depends_on=model('orders'),", " tags=['daily'],", " group='python',", diff --git a/tests/unit/src/sqlbuild/cli/commands/main/dag/test_dag_command.py b/tests/unit/src/sqlbuild/cli/commands/main/dag/test_dag_command.py index 77823c58a4..182d635a2c 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/dag/test_dag_command.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/dag/test_dag_command.py @@ -105,7 +105,7 @@ def test_given_local_project_when_running_dag_json_then_outputs_static_graph_wit PythonDagCommandTestCase( description="emits Python nodes and checks in static dag json", expected_exit_code=0, - expected_task_id="task:prepare_orders", + expected_task_id="asset:prepare_orders", expected_task_tags=["daily"], expected_task_group="python", expected_task_meta={"owner": "data"}, @@ -123,12 +123,12 @@ def test_given_local_project_when_running_dag_json_then_outputs_static_graph_wit expected_check_id="check:check_orders_export", expected_check_group="exports", expected_edges={ - ("model:orders", "task:prepare_orders"), - ("task:prepare_orders", "asset:orders_export"), - ("task:prepare_orders", "loader:warehouse_export"), + ("model:orders", "asset:prepare_orders"), + ("asset:prepare_orders", "asset:orders_export"), + ("asset:prepare_orders", "loader:warehouse_export"), ("loader:warehouse_export", "asset:orders_export"), ("asset:orders_export", "check:check_orders_export"), - ("task:prepare_orders", "check:check_orders_export"), + ("asset:prepare_orders", "check:check_orders_export"), ("loader:warehouse_export", "check:check_orders_export"), ("loader:warehouse_export", "check:check_loader_export"), }, @@ -138,7 +138,7 @@ def test_given_local_project_when_running_dag_json_then_outputs_static_graph_wit "name": "check_orders_export", "checked_asset_ids": [ "asset:orders_export", - "task:prepare_orders", + "asset:prepare_orders", "loader:warehouse_export", ], "path": "python/checks/check_orders_export.py", diff --git a/tests/unit/src/sqlbuild/cli/commands/main/playground/test_playground.py b/tests/unit/src/sqlbuild/cli/commands/main/playground/test_playground.py index 00834d9b30..dc319aaa8b 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/playground/test_playground.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/playground/test_playground.py @@ -176,7 +176,7 @@ Path("sqlbuild_project.toml"), Path("sources/raw.yml"), Path("models/fact_orders.sql"), - Path("python/tasks/orders.py"), + Path("python/assets/orders_inputs.py"), Path("python/loaders/orders.py"), Path("python/assets/orders_export.py"), Path("python/checks/orders_export.py"), @@ -188,9 +188,9 @@ ("[connections.developer]", 'connection = "developer"'), ), ( - Path("python/tasks/orders.py"), + Path("python/assets/orders_inputs.py"), ( - "from sqlbuild.tasks import SkipMode, task", + "from sqlbuild.assets import SkipMode, asset", "def prepare_raw_orders", "SkipMode.SOFT", "def export_window", @@ -216,7 +216,9 @@ ), ), ), - unexpected_file_fragments=((Path("python/tasks/orders.py"), ("sqlbuild.compiler",)),), + unexpected_file_fragments=( + (Path("python/assets/orders_inputs.py"), ("sqlbuild.compiler",)), + ), ), ], ids=lambda case: case.description, diff --git a/tests/unit/src/sqlbuild/cli/output/main/plan/test_plan_text.py b/tests/unit/src/sqlbuild/cli/output/main/plan/test_plan_text.py index 88b5e76b41..88f691af5e 100644 --- a/tests/unit/src/sqlbuild/cli/output/main/plan/test_plan_text.py +++ b/tests/unit/src/sqlbuild/cli/output/main/plan/test_plan_text.py @@ -644,7 +644,7 @@ python_plan_entries=( PythonPlanEntry( name="prepare_orders", - kind=PythonNodeKind.TASK, + kind=PythonNodeKind.ASSET, phase=PythonRunPhase.PRE_SQL_INGRESS, ), PythonPlanEntry( @@ -654,7 +654,7 @@ ), PythonPlanEntry( name="profile_fact_orders", - kind=PythonNodeKind.TASK, + kind=PythonNodeKind.ASSET, phase=PythonRunPhase.READ_SIDE, ), ), @@ -681,7 +681,7 @@ python_plan_entries=( PythonPlanEntry( name="prepare_orders", - kind=PythonNodeKind.TASK, + kind=PythonNodeKind.ASSET, phase=PythonRunPhase.PRE_SQL_INGRESS, identity_status=PythonIdentityStatus.CHANGED, previous_definition_json=json.dumps( @@ -1164,7 +1164,7 @@ python_plan_entries=( PythonPlanEntry( name="publish_orders", - kind=PythonNodeKind.TASK, + kind=PythonNodeKind.ASSET, phase=PythonRunPhase.READ_SIDE, provider_usages=(build_discovered_provider_usage(),), ), @@ -1194,7 +1194,7 @@ python_plan_entries=( PythonPlanEntry( name="publish_orders", - kind=PythonNodeKind.TASK, + kind=PythonNodeKind.ASSET, phase=PythonRunPhase.READ_SIDE, provider_usages=(build_discovered_provider_usage(),), ), @@ -1435,7 +1435,7 @@ def test_given_plan_output_when_formatting_then_contains_expected_fragments( python_plan_entries=( PythonPlanEntry( name="prepare_orders", - kind=PythonNodeKind.TASK, + kind=PythonNodeKind.ASSET, phase=PythonRunPhase.PRE_SQL_INGRESS, identity_status=PythonIdentityStatus.CHANGED, previous_metadata_json=json.dumps( diff --git a/tests/unit/src/sqlbuild/cli/output/main/plan_json/test_plan_json.py b/tests/unit/src/sqlbuild/cli/output/main/plan_json/test_plan_json.py index e2e0f2e647..f0aa36e0e6 100644 --- a/tests/unit/src/sqlbuild/cli/output/main/plan_json/test_plan_json.py +++ b/tests/unit/src/sqlbuild/cli/output/main/plan_json/test_plan_json.py @@ -549,7 +549,7 @@ def test_given_direct_diagnostics_policy_when_formatting_plan_json_then_evidence python_plan_entries=( PythonPlanEntry( name="prepare_orders", - kind=PythonNodeKind.TASK, + kind=PythonNodeKind.ASSET, phase=PythonRunPhase.PRE_SQL_INGRESS, ), PythonPlanEntry( @@ -562,7 +562,7 @@ def test_given_direct_diagnostics_policy_when_formatting_plan_json_then_evidence expected_fragments=( '"python_node_count": 2', '"name": "prepare_orders"', - '"kind": "task"', + '"kind": "asset"', '"phase": "pre_sql_ingress"', '"name": "export_orders"', '"kind": "asset"', @@ -575,7 +575,7 @@ def test_given_direct_diagnostics_policy_when_formatting_plan_json_then_evidence python_plan_entries=( PythonPlanEntry( name="prepare_orders", - kind=PythonNodeKind.TASK, + kind=PythonNodeKind.ASSET, phase=PythonRunPhase.PRE_SQL_INGRESS, identity_status=PythonIdentityStatus.CHANGED, previous_definition_json=json.dumps( @@ -640,7 +640,7 @@ def test_given_direct_diagnostics_policy_when_formatting_plan_json_then_evidence python_plan_entries=( PythonPlanEntry( name="publish_orders", - kind=PythonNodeKind.TASK, + kind=PythonNodeKind.ASSET, phase=PythonRunPhase.READ_SIDE, provider_usages=(build_discovered_provider_usage(),), ), @@ -652,7 +652,7 @@ def test_given_direct_diagnostics_policy_when_formatting_plan_json_then_evidence '"used_by"', '"kind": "loader"', '"name": "raw_orders"', - '"kind": "task"', + '"kind": "asset"', '"name": "publish_orders"', '"parameter": "marker_provider"', '"class_name": "MarkerProvider"', diff --git a/tests/unit/src/sqlbuild/cli/progress/classes/test_native_progress_projector.py b/tests/unit/src/sqlbuild/cli/progress/classes/test_native_progress_projector.py index 1cf91dea05..f076b6b882 100644 --- a/tests/unit/src/sqlbuild/cli/progress/classes/test_native_progress_projector.py +++ b/tests/unit/src/sqlbuild/cli/progress/classes/test_native_progress_projector.py @@ -70,7 +70,7 @@ def test_given_skipped_resource_when_projected_then_skip_terminal_renders_once( run_scope("run-skip-projection"), dispatcher_scope(dispatcher), ResourceAttemptLifecycle( - resource_id="task:skipped", + resource_id="asset:skipped", resource_kind="task", resource_name="skipped", ) as lifecycle, @@ -119,7 +119,7 @@ def test_given_retrying_child_operation_when_projected_then_each_line_flushes_be run_scope("run-retry-projection"), dispatcher_scope(dispatcher), ResourceAttemptLifecycle( - resource_id="task:flaky", + resource_id="asset:flaky", resource_kind="task", resource_name="flaky", ), @@ -290,7 +290,7 @@ def test_given_retried_resource_when_final_result_claims_then_prior_attempt_rend lifecycle_event( "resource_attempt_started", run_id="run", - resource_id="task:orders", + resource_id="asset:orders", resource_attempt_id="orders-attempt-1", payload={ "resource_kind": "task", @@ -309,7 +309,7 @@ def test_given_retried_resource_when_final_result_claims_then_prior_attempt_rend unrelated_start: LifecycleEvent = replace( first_start, event_id="inventory-start", - resource_id="task:inventory", + resource_id="asset:inventory", resource_attempt_id="inventory-attempt", payload={ "resource_kind": "task", @@ -819,7 +819,7 @@ def project_resource() -> None: run_scope("run-lock-order"), dispatcher_scope(dispatcher), ResourceAttemptLifecycle( - resource_id="task:writer", resource_kind="task", resource_name="writer" + resource_id="asset:writer", resource_kind="task", resource_name="writer" ) as lifecycle, ): lifecycle.skipped(skip_code="scheduler") diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py index 680120704a..0ae5d088b6 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py @@ -373,8 +373,8 @@ def python_task_source(*, body: str) -> str: return ( "from sqlbuild.refs import model\n" - "from sqlbuild.tasks import task\n\n\n" - '@task(depends_on=model("orders"))\n' + "from sqlbuild.assets import asset\n\n\n" + '@asset(depends_on=model("orders"))\n' "def export_orders(ctx):\n" f"{body}" ) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_descriptions.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_descriptions.py index acc1df5771..e8de0f63a1 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_descriptions.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_descriptions.py @@ -39,7 +39,6 @@ "sql_hook_files": "hook", "hook_functions": "hook", "loader_functions": "loader", - "task_functions": "task", "asset_functions": "asset", "check_functions": "check", "providers": "provider", diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_python_sql_references.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_python_sql_references.py index 848cb0a319..a27db5e8ef 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_python_sql_references.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_python_sql_references.py @@ -75,7 +75,7 @@ }, expected_error_fragments=( "[P008]", - "task:export_orders names model:customers as 'customers' in SQL passed to " + "asset:export_orders names model:customers as 'customers' in SQL passed to " "ctx.query()", "python/tasks/export.py:7", 'depends_on=model("customers")', @@ -103,11 +103,11 @@ expected_error_fragments=( "[P008] check:orders_present names model:customers as 'customers' in SQL passed " "to ctx.query() --> python/checks/orders.py:7", - "[P008] task:export_orders names model:customers as 'customers' in SQL passed to " + "[P008] asset:export_orders names model:customers as 'customers' in SQL passed to " "ctx.query() --> python/tasks/export.py:7", - "[P008] task:export_orders names model:customers as 'customers' in SQL passed to " + "[P008] asset:export_orders names model:customers as 'customers' in SQL passed to " "ctx.query() --> python/tasks/export.py:8", - "[P008] task:export_orders names model:orders as 'orders' in SQL passed to " + "[P008] asset:export_orders names model:orders as 'orders' in SQL passed to " "ctx.query() --> python/tasks/export.py:8", ), ), @@ -121,7 +121,7 @@ ), }, expected_error_fragments=( - "[P008] task:export_orders names model:customers as 'customers' in SQL passed to " + "[P008] asset:export_orders names model:customers as 'customers' in SQL passed to " "ctx.query() --> python/tasks/export.py:7", ), ), @@ -148,7 +148,7 @@ ), }, expected_error_fragments=( - "[P008] task:export_orders names model:customers as 'customers' in SQL passed to " + "[P008] asset:export_orders names model:customers as 'customers' in SQL passed to " "ctx.query()", ), ), @@ -163,7 +163,7 @@ ), }, expected_error_fragments=( - "task:export_orders names seed:country_codes", + "asset:export_orders names seed:country_codes", 'declare it with depends_on=seed("country_codes") and use ' 'ctx.relation(seed("country_codes"))', ), diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_audit_factories.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_audit_factories.py index d78d981546..6edbb4e71d 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_audit_factories.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_audit_factories.py @@ -26,7 +26,7 @@ "python/factories/quality.py": """ from sqlbuild.audits import AuditCase, audit_factory from sqlbuild.factories import factory -from sqlbuild.tasks import task +from sqlbuild.assets import asset @audit_factory def quality_checks(): @@ -34,7 +34,7 @@ def quality_checks(): @factory def generated_nodes(): - @task(name="refresh_quality") + @asset(name="refresh_quality") def refresh(ctx): return None return [refresh] @@ -48,13 +48,13 @@ def refresh(ctx): files={ "python/quality.py": """ from sqlbuild.audits import AuditCase, audit_factory -from sqlbuild.tasks import task +from sqlbuild.assets import asset @audit_factory def quality_checks(): return [AuditCase(name="positive_amount", definition="expression_is_true", arguments={"expression": "amount > 0"})] -@task(name="refresh_quality") +@asset(name="refresh_quality") def refresh(ctx): return None """, @@ -80,7 +80,7 @@ def test_given_audit_and_node_factories_when_discovering_then_both_are_collected assert discovered.audit_factories[0].cases[0].name == "positive_amount" assert discovered.audit_factories[0].relative_path == Path(next(iter(test_case.files))) assert discovered.audit_factories[0].line > 0 - assert tuple(task.name for task in discovered.tasks) == test_case.expected_task_names + assert tuple(task.name for task in discovered.assets) == test_case.expected_task_names @pytest.mark.parametrize( diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_factories.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_factories.py index 4b248fb85d..8c35fab456 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_factories.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_factories.py @@ -28,11 +28,11 @@ from sqlbuild.checks import check from sqlbuild.factories import factory from sqlbuild.loaders import loader -from sqlbuild.tasks import task +from sqlbuild.assets import asset def make_task(name): - @task(name=name) + @asset(name=name) def generated_task(ctx): return {"name": name} return generated_task @@ -197,10 +197,10 @@ def export(ctx): from python._helpers import REGION from sqlbuild.checks import check from sqlbuild.factories import factory -from sqlbuild.tasks import task +from sqlbuild.assets import asset -@task(name="prepare_orders") +@asset(name="prepare_orders") def prepare_orders(ctx): return {"region": REGION} @@ -233,13 +233,13 @@ def test_given_factory_nodes_when_discovering_python_nodes_then_returns_generate result: DiscoveredPythonNodeFunctions = discover_python_node_functions(project_dir=tmp_path) assert tuple(node.name for node in result.loaders) == test_case.expected_loader_names - assert tuple(node.name for node in result.tasks) == test_case.expected_task_names + assert tuple(node.name for node in result.assets) == test_case.expected_task_names assert tuple(node.name for node in result.assets) == test_case.expected_asset_names assert tuple(node.name for node in result.checks) == test_case.expected_check_names assert tuple(len(node.depends_on) for node in result.loaders) == ( test_case.expected_loader_dependency_counts ) - assert tuple(len(node.depends_on) for node in result.tasks) == ( + assert tuple(len(node.depends_on) for node in result.assets) == ( test_case.expected_task_dependency_counts ) assert tuple(len(node.depends_on) for node in result.assets) == ( @@ -370,10 +370,10 @@ def broken_factory(): files={ "python/tasks/generated.py": """ from sqlbuild.factories import factory -from sqlbuild.tasks import task +from sqlbuild.assets import asset -@task(name="generated_task") +@asset(name="generated_task") def generated_task(ctx): return None diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_python_nodes.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_python_nodes.py index 0588ad2f31..ee0362efb0 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_python_nodes.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_python_nodes.py @@ -14,7 +14,6 @@ DiscoveredAssetFunction, DiscoveredCheckFunction, DiscoveredPythonNodeFunctions, - DiscoveredTaskFunction, PythonNodeDiscoveryError, PythonNodeIdentity, discover_python_node_functions, @@ -48,10 +47,10 @@ def normalize_status(value): """, "python/orders.py": """ import python.helpers.clean -from sqlbuild.tasks import task +from sqlbuild.assets import asset -@task +@asset def orders(ctx): return python.helpers.clean.normalize_status(" Shipped ") """, @@ -65,9 +64,9 @@ def orders(ctx): description="discovers decorated tasks and assets from explicit folders", files={ "python/tasks/windows.py": """ -from sqlbuild.tasks import task +from sqlbuild.assets import asset -@task(tags=("api",), group="ingestion") +@asset(tags=("api",), group="ingestion") def fetch_window(ctx): '''Fetch an API window.''' return {"window": "today"} @@ -128,8 +127,8 @@ def test_given_project_dir_when_discovering_python_nodes_then_returns_expected( file_path.parent.mkdir(parents=True, exist_ok=True) file_path.write_text(contents, encoding="utf-8") - tasks: tuple[DiscoveredTaskFunction, ...] = tuple( - discover_python_node_functions(project_dir=tmp_path).tasks + tasks: tuple[DiscoveredAssetFunction, ...] = tuple( + discover_python_node_functions(project_dir=tmp_path).assets ) assets: tuple[DiscoveredAssetFunction, ...] = tuple( discover_python_node_functions(project_dir=tmp_path).assets @@ -272,9 +271,9 @@ def test_given_project_dir_when_discovering_checks_then_returns_expected( from sqlbuild.assets import asset from sqlbuild.checks import check from sqlbuild.loaders import loader -from sqlbuild.tasks import task +from sqlbuild.assets import asset -@task +@asset def prepare_orders(ctx): return None @@ -307,7 +306,7 @@ def export_uri(name): """, "python/orders/__init__.py": "", "python/orders/utils.py": """ -from sqlbuild.tasks import task +from sqlbuild.assets import asset def order_label(value): return str(value) @@ -323,9 +322,9 @@ def raw_orders(ctx): """, "python/orders/pipeline/tasks.py": """ from python.orders.utils import order_label -from sqlbuild.tasks import task +from sqlbuild.assets import asset -@task +@asset def label_orders(ctx): return order_label(1) """, @@ -380,7 +379,7 @@ def test_given_python_root_when_discovering_python_nodes_then_decorators_define_ discovered: tuple[tuple[str, str, str], ...] = ( *(("loader", node.name, node.relative_path.as_posix()) for node in result.loaders), - *(("task", node.name, node.relative_path.as_posix()) for node in result.tasks), + *(("task", node.name, node.relative_path.as_posix()) for node in result.assets), *(("asset", node.name, node.relative_path.as_posix()) for node in result.assets), *(("check", node.name, node.relative_path.as_posix()) for node in result.checks), ) @@ -406,19 +405,19 @@ def test_given_python_root_when_discovering_python_nodes_then_decorators_define_ """, "python/tasks/first.py": """ import python._counter -from sqlbuild.tasks import task +from sqlbuild.assets import asset -@task +@asset def first_task(ctx): return None """, "python/tasks/second.py": """ from python import _counter -from sqlbuild.tasks import task +from sqlbuild.assets import asset -@task +@asset def second_task(ctx): return None """, @@ -438,8 +437,8 @@ def test_given_python_root_helpers_when_discovering_then_modules_import_under_pa result: DiscoveredPythonNodeFunctions = discover_python_node_functions(project_dir=tmp_path) - assert tuple(node.name for node in result.tasks) == test_case.expected_task_names - assert all(node.function.__module__.startswith("python.") for node in result.tasks) + assert tuple(node.name for node in result.assets) == test_case.expected_task_names + assert all(node.function.__module__.startswith("python.") for node in result.assets) init_lines: list[str] = (tmp_path / "init_count.txt").read_text().splitlines() assert len(init_lines) == test_case.expected_init_count @@ -453,10 +452,10 @@ def test_given_python_root_helpers_when_discovering_then_modules_import_under_pa "python/_values.py": "STATUS = 'first'\n", "python/orders.py": """ from python._values import STATUS -from sqlbuild.tasks import task +from sqlbuild.assets import asset -@task +@asset def orders(ctx): return STATUS """, @@ -465,10 +464,10 @@ def orders(ctx): "python/_values.py": "STATUS = 'second'\n", "python/orders.py": """ from python._values import STATUS -from sqlbuild.tasks import task +from sqlbuild.assets import asset -@task +@asset def orders(ctx): return STATUS """, @@ -492,8 +491,8 @@ def test_given_two_projects_when_discovering_sequentially_then_python_modules_ar first: DiscoveredPythonNodeFunctions = discover_python_node_functions(project_dir=first_dir) second: DiscoveredPythonNodeFunctions = discover_python_node_functions(project_dir=second_dir) - assert first.tasks[0].function(None) == test_case.expected_first_result - assert second.tasks[0].function(None) == test_case.expected_second_result + assert first.assets[0].function(None) == test_case.expected_first_result + assert second.assets[0].function(None) == test_case.expected_second_result @pytest.mark.parametrize( @@ -516,9 +515,9 @@ def test_given_python_root_helper_edit_when_rediscovering_then_node_identity_cha write_repo_files: Callable[[Path, dict[str, str]], None], ) -> None: write_repo_files(tmp_path, test_case.files) - before_task: DiscoveredTaskFunction = discover_python_node_functions( + before_task: DiscoveredAssetFunction = discover_python_node_functions( project_dir=tmp_path - ).tasks[0] + ).assets[0] before: PythonNodeIdentity = build_python_node_identity( node_type="task", node_name=before_task.name, @@ -531,9 +530,9 @@ def test_given_python_root_helper_edit_when_rediscovering_then_node_identity_cha encoding="utf-8", ) - after_task: DiscoveredTaskFunction = discover_python_node_functions(project_dir=tmp_path).tasks[ - 0 - ] + after_task: DiscoveredAssetFunction = discover_python_node_functions( + project_dir=tmp_path + ).assets[0] after: PythonNodeIdentity = build_python_node_identity( node_type="task", node_name=after_task.name, @@ -577,10 +576,10 @@ def run(self, value): """, "python/orders.py": """ import python.helpers.cleaner -from sqlbuild.tasks import task +from sqlbuild.assets import asset -@task +@asset def orders(ctx): return python.helpers.cleaner.Cleaner().run(" Shipped ") """, @@ -613,10 +612,10 @@ def run(self, value): """, "python/orders.py": """ import python.helpers.cleaner -from sqlbuild.tasks import task +from sqlbuild.assets import asset -@task +@asset def orders(ctx): return python.helpers.cleaner.Cleaner().run(" Shipped ") """, @@ -641,7 +640,9 @@ def test_given_unrelated_python_package_restored_when_building_identity_then_hel monkeypatch.setitem(sys.modules, "python", importlib.import_module("python")) monkeypatch.setitem(sys.modules, "python.orders", importlib.import_module("python.orders")) - task: DiscoveredTaskFunction = discover_python_node_functions(project_dir=project_dir).tasks[0] + task: DiscoveredAssetFunction = discover_python_node_functions(project_dir=project_dir).assets[ + 0 + ] identity: PythonNodeIdentity = build_python_node_identity( node_type="task", node_name=task.name, @@ -689,7 +690,7 @@ def test_given_unrelated_python_package_when_discovering_then_project_resolves_a result: DiscoveredPythonNodeFunctions = discover_python_node_functions(project_dir=project_dir) - assert tuple(node.name for node in result.tasks) == test_case.expected_task_names - assert result.tasks[0].function(None) == test_case.expected_task_result + assert tuple(node.name for node in result.assets) == test_case.expected_task_names + assert result.assets[0].function(None) == test_case.expected_task_result assert sys.modules["python"] is unrelated assert sys.modules["python.other"] is unrelated_other diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_resource_identities.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_resource_identities.py index 9bb5032e8f..b84948ac86 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_resource_identities.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_resource_identities.py @@ -195,8 +195,8 @@ def test_given_noncanonical_name_when_validating_resource_identity_then_coded_er description="Python task name", project_files={ "python/tasks/orders.py": ( - "from sqlbuild.tasks import task\n\n" - "@task\n" + "from sqlbuild.assets import asset\n\n" + "@asset\n" "def DailyTask(ctx):\n" " return None\n" ) diff --git a/tests/unit/src/sqlbuild/compiler/discovery/test_main.py b/tests/unit/src/sqlbuild/compiler/discovery/test_main.py index 70a63b7dfc..8e73799523 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/test_main.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/test_main.py @@ -77,9 +77,9 @@ def fetch_orders(ctx): return [] """, "python/tasks/windows.py": """ -from sqlbuild.tasks import task +from sqlbuild.assets import asset -@task(tags=("api",), group="ingestion") +@asset(tags=("api",), group="ingestion") def fetch_window(ctx): return {"window": "today"} """, @@ -266,7 +266,7 @@ def test_given_project_repo_slice_when_discovering_inputs_then_it_returns_expect == test_case.expected_loader_names ) assert ( - tuple(task_function.name for task_function in discovered_inputs.task_functions) + tuple(task_function.name for task_function in discovered_inputs.asset_functions) == test_case.expected_task_names ) assert ( @@ -432,12 +432,12 @@ def load(ctx): "python/factories/generated.py": """ from sqlbuild.assets import asset from sqlbuild.factories import factory -from sqlbuild.tasks import task +from sqlbuild.assets import asset @factory def generated_nodes(): - @task(name="prepare_orders") + @asset(name="prepare_orders") def prepare(ctx): return None @@ -457,12 +457,12 @@ def export(ctx): | { "python/factories/generated_tasks.py": """ from sqlbuild.factories import factory -from sqlbuild.tasks import task +from sqlbuild.assets import asset @factory def generated_tasks(): - @task(name="prepare_orders") + @asset(name="prepare_orders") def prepare(ctx): return None @@ -477,10 +477,10 @@ def prepare(ctx): | { "python/factories/orders.py": """ from sqlbuild.factories import factory -from sqlbuild.tasks import task +from sqlbuild.assets import asset -@task +@asset def orders(ctx): return None @@ -498,14 +498,14 @@ def order_nodes(): | { "python/factories/orders.py": """ from sqlbuild.factories import factory -from sqlbuild.tasks import task +from sqlbuild.assets import asset @factory def order_nodes(): global orders - @task + @asset def orders(ctx): return None @@ -521,21 +521,21 @@ def orders(ctx): "python/a_factories/orders.py": """ from python.tasks.orders import prepare_orders from sqlbuild.factories import factory -from sqlbuild.tasks import task +from sqlbuild.assets import asset @factory def order_nodes(): - @task(name="publish_orders", depends_on=prepare_orders) + @asset(name="publish_orders", depends_on=prepare_orders) def publish(ctx): return None return [prepare_orders, publish] """, "python/tasks/orders.py": """ -from sqlbuild.tasks import task +from sqlbuild.assets import asset -@task +@asset def prepare_orders(ctx): return None """, @@ -550,12 +550,12 @@ def prepare_orders(ctx): "python/factories/orders/generated.py": """ from python.factories.orders._helpers import TASK_NAME from sqlbuild.factories import factory -from sqlbuild.tasks import task +from sqlbuild.assets import asset @factory def generated_tasks(): - @task(name=TASK_NAME) + @asset(name=TASK_NAME) def prepare(ctx): return None @@ -572,12 +572,12 @@ def prepare(ctx): from sqlbuild.checks import check from sqlbuild.factories import factory from sqlbuild.refs import model, seed -from sqlbuild.tasks import task +from sqlbuild.assets import asset @factory def generated_nodes(): - @task(name="export_countries", depends_on=seed("country_codes")) + @asset(name="export_countries", depends_on=seed("country_codes")) def export_countries(ctx): return None @@ -606,7 +606,7 @@ def test_given_generated_factory_nodes_when_discovering_inputs_then_validates_ex assert ( tuple(loader.name for loader in result.loader_functions) == test_case.expected_loader_names ) - assert tuple(task.name for task in result.task_functions) == test_case.expected_task_names + assert tuple(task.name for task in result.asset_functions) == test_case.expected_task_names assert tuple(asset.name for asset in result.asset_functions) == test_case.expected_asset_names assert tuple(check.name for check in result.check_functions) == test_case.expected_check_names @@ -786,11 +786,11 @@ def test_given_non_project_with_python_when_discovering_then_missing_config_erro | { "python/tasks/generated.py": """ from sqlbuild.factories import factory -from sqlbuild.tasks import task +from sqlbuild.assets import asset def make_task(): - @task(name="profile") + @asset(name="profile") def profile(ctx): return None return profile @@ -808,10 +808,10 @@ def generated_tasks(): repo_files=base_repo_files() | { "python/tasks/top_level.py": """ -from sqlbuild.tasks import task +from sqlbuild.assets import asset -@task(name="profile") +@asset(name="profile") def top_level_profile(ctx): return None """, @@ -836,17 +836,17 @@ def generated_profile(ctx): | { "python/factories/orders.py": """ from sqlbuild.factories import factory -from sqlbuild.tasks import task +from sqlbuild.assets import asset -@task +@asset def orders(ctx): return None @factory def order_nodes(): - @task(name="orders") + @asset(name="orders") def generated_orders(ctx): return None return generated_orders @@ -860,7 +860,7 @@ def generated_orders(ctx): | { "python/tasks/generated.py": """ from sqlbuild.factories import factory -from sqlbuild.tasks import task +from sqlbuild.assets import asset @factory @@ -869,9 +869,9 @@ def first_body(ctx): return None def second_body(ctx): return None - first = task(name="first", depends_on=[])(first_body) - second = task(name="second", depends_on=[first])(second_body) - first = task(name="first", depends_on=[second])(first) + first = asset(name="first", depends_on=[])(first_body) + second = asset(name="second", depends_on=[first])(second_body) + first = asset(name="first", depends_on=[second])(first) return [first, second] """, }, @@ -1280,9 +1280,9 @@ def raw_orders(ctx): repo_files=base_repo_files() | { "python/tasks/export_customers.py": """ -from sqlbuild.tasks import task +from sqlbuild.assets import asset -@task +@asset def export_customers(ctx): return None """, @@ -1301,9 +1301,9 @@ def export_customers(ctx): repo_files=base_repo_files() | { "python/tasks/export_customers.py": """ -from sqlbuild.tasks import task +from sqlbuild.assets import asset -@task +@asset def export_customers(ctx): return None """, @@ -1324,9 +1324,9 @@ def export_customers_check(ctx): | { "models/marts/export_customers.sql": "MODEL ();\n\nselect 1\n", "python/tasks/export_customers.py": """ -from sqlbuild.tasks import task +from sqlbuild.assets import asset -@task +@asset def export_customers(ctx): return None """, @@ -1371,9 +1371,9 @@ def export_customers(ctx): """.strip() + "\n", "python/tasks/export_customers.py": """ -from sqlbuild.tasks import task +from sqlbuild.assets import asset -@task +@asset def export_customers(ctx): return None """, @@ -1498,12 +1498,12 @@ def notify(ctx): repo_files=base_repo_files() | { "python/tasks/windows.py": """ -from sqlbuild.tasks import task +from sqlbuild.assets import asset def fetch_window(ctx): return None -@task(depends_on=fetch_window) +@asset(depends_on=fetch_window) def export_window(ctx): return None """, @@ -1515,16 +1515,16 @@ def export_window(ctx): repo_files=base_repo_files() | { "python/tasks/windows.py": """ -from sqlbuild.tasks import task +from sqlbuild.assets import asset def fetch_window(ctx): return None -@task(depends_on=fetch_window) +@asset(depends_on=fetch_window) def enrich_window(ctx): return None -fetch_window = task(depends_on=enrich_window)(fetch_window) +fetch_window = asset(depends_on=enrich_window)(fetch_window) """, }, expected_error_fragment="Python node dependency cycle detected", diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/main/test_relation_targets.py b/tests/unit/src/sqlbuild/compiler/pipeline/main/test_relation_targets.py index 5f4ae4e612..53525b4e63 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/main/test_relation_targets.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/main/test_relation_targets.py @@ -119,7 +119,7 @@ def test_given_selected_python_declarations_when_collecting_refs_then_all_kinds_ sql_deps=(source("loader_source"),), ), DiscoveredPythonNode( - kind=PythonNodeKind.TASK, + kind=PythonNodeKind.ASSET, file_path=project_path, relative_path=project_path, name="score_orders", diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/helpers.py index 820bc643ce..140f04c412 100644 --- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/helpers.py @@ -34,7 +34,6 @@ DiscoveredProjectInputs, DiscoveredPythonNodeFunctions, DiscoveredSourceFile, - DiscoveredTaskFunction, LocalConfig, ProjectConfig, ProjectGraph, @@ -48,6 +47,7 @@ discover_python_node_functions, resolve_python_sql_run_selection, ) +from sqlbuild.assets import asset from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, @@ -58,7 +58,6 @@ from sqlbuild.python_nodes.models import SqlResourceRef from sqlbuild.python_nodes.types import PythonCheckSeverity from sqlbuild.refs import model, source -from sqlbuild.tasks import task from tests.unit.src.sqlbuild.compiler.graph.helpers import project_graph_from_indexes @@ -86,7 +85,7 @@ def summarize_orders(_ctx: object) -> None: return None -@task(name="prepare_orders") +@asset(name="prepare_orders") def imported_prepare_orders(_ctx: object) -> None: return None @@ -143,16 +142,14 @@ def build_orders_python_node_graph() -> PythonNodeGraph: depends_on=(prepare_orders,), ), ), - task_functions=( - DiscoveredTaskFunction( + asset_functions=( + DiscoveredAssetFunction( file_path=Path("/project/tasks/orders.py"), relative_path=Path("python/tasks/orders.py"), name="prepare_orders", function=prepare_orders, tags=("orders", "daily"), ), - ), - asset_functions=( DiscoveredAssetFunction( file_path=Path("/project/assets/orders.py"), relative_path=Path("python/assets/orders.py"), @@ -257,8 +254,8 @@ def build_sql_ref_python_node_graph(*, dependency: SqlResourceRef) -> PythonNode discovered_inputs=DiscoveredProjectInputs( project_config=ProjectConfig(name="demo", adapter="duckdb"), local_config=LocalConfig(), - task_functions=( - DiscoveredTaskFunction( + asset_functions=( + DiscoveredAssetFunction( file_path=Path("/project/tasks/orders.py"), relative_path=Path("python/tasks/orders.py"), name="profile_orders", @@ -271,21 +268,21 @@ def build_sql_ref_python_node_graph(*, dependency: SqlResourceRef) -> PythonNode def build_sibling_sql_ref_python_node_graph() -> PythonNodeGraph: - """Return tasks reading orders and, separately, its upstream raw_orders source.""" + """Return assets reading orders and, separately, its upstream raw_orders source.""" return build_python_node_graph( discovered_inputs=DiscoveredProjectInputs( project_config=ProjectConfig(name="demo", adapter="duckdb"), local_config=LocalConfig(), - task_functions=( - DiscoveredTaskFunction( + asset_functions=( + DiscoveredAssetFunction( file_path=Path("/project/tasks/orders.py"), relative_path=Path("python/tasks/orders.py"), name="export_orders", function=prepare_orders, depends_on=(model_ref("orders"),), ), - DiscoveredTaskFunction( + DiscoveredAssetFunction( file_path=Path("/project/tasks/raw.py"), relative_path=Path("python/tasks/raw.py"), name="export_raw_orders", @@ -297,7 +294,7 @@ def build_sibling_sql_ref_python_node_graph() -> PythonNodeGraph: ) -def build_sql_downstream_task_to_loader_python_node_graph() -> PythonNodeGraph: +def build_sql_downstream_asset_to_loader_python_node_graph() -> PythonNodeGraph: return build_python_node_graph( discovered_inputs=DiscoveredProjectInputs( project_config=ProjectConfig(name="demo", adapter="duckdb"), @@ -311,8 +308,8 @@ def build_sql_downstream_task_to_loader_python_node_graph() -> PythonNodeGraph: depends_on=(prepare_orders,), ), ), - task_functions=( - DiscoveredTaskFunction( + asset_functions=( + DiscoveredAssetFunction( file_path=Path("/project/tasks/orders.py"), relative_path=Path("python/tasks/orders.py"), name="prepare_orders", @@ -332,14 +329,6 @@ def source_ref(name: str) -> SqlResourceRef: return source(name) -def build_terminal_loader_task_dependency_python_node_graph() -> PythonNodeGraph: - return _build_loader_dependency_python_node_graph( - dependency_function=raw_orders, - include_intermediate_loader=False, - dependent_kind="task", - ) - - def build_terminal_loader_asset_dependency_python_node_graph() -> PythonNodeGraph: return _build_loader_dependency_python_node_graph( dependency_function=raw_orders, @@ -356,14 +345,6 @@ def build_terminal_loader_check_dependency_python_node_graph() -> PythonNodeGrap ) -def build_intermediate_loader_task_dependency_python_node_graph() -> PythonNodeGraph: - return _build_loader_dependency_python_node_graph( - dependency_function=fetch_pages, - include_intermediate_loader=True, - dependent_kind="task", - ) - - def build_intermediate_loader_asset_dependency_python_node_graph() -> PythonNodeGraph: return _build_loader_dependency_python_node_graph( dependency_function=fetch_pages, @@ -414,15 +395,6 @@ def _build_loader_dependency_python_node_graph( intermediate_loaders, )[include_intermediate_loader] - task_functions: tuple[DiscoveredTaskFunction, ...] = ( - DiscoveredTaskFunction( - file_path=Path("/project/tasks/orders.py"), - relative_path=Path("python/tasks/orders.py"), - name="summarize_orders", - function=summarize_orders, - depends_on=(dependency_function,), - ), - ) asset_functions: tuple[DiscoveredAssetFunction, ...] = ( DiscoveredAssetFunction( file_path=Path("/project/assets/orders.py"), @@ -442,16 +414,14 @@ def _build_loader_dependency_python_node_graph( severity=PythonCheckSeverity.ERROR, ), ) - task_functions = ((), task_functions)[dependent_kind == "task"] asset_functions = ((), asset_functions)[dependent_kind == "asset"] - check_functions = ((), check_functions)[dependent_kind not in {"task", "asset"}] + check_functions = ((), check_functions)[dependent_kind != "asset"] return build_python_node_graph( discovered_inputs=DiscoveredProjectInputs( project_config=ProjectConfig(name="demo", adapter="duckdb"), local_config=LocalConfig(), loader_functions=loader_functions, - task_functions=task_functions, asset_functions=asset_functions, check_functions=check_functions, ) @@ -461,18 +431,12 @@ def _build_loader_dependency_python_node_graph( def build_python_node_graph_for_case(case_name: str) -> PythonNodeGraph: builders: MappingProxyType[str, Callable[[], PythonNodeGraph]] = MappingProxyType( { - "terminal_loader_task_dependency": ( - build_terminal_loader_task_dependency_python_node_graph - ), "terminal_loader_asset_dependency": ( build_terminal_loader_asset_dependency_python_node_graph ), "terminal_loader_check_dependency": ( build_terminal_loader_check_dependency_python_node_graph ), - "intermediate_loader_task_dependency": ( - build_intermediate_loader_task_dependency_python_node_graph - ), "intermediate_loader_asset_dependency": ( build_intermediate_loader_asset_dependency_python_node_graph ), @@ -609,11 +573,11 @@ def imported_identity_nodes( def identity_summary( *, name: str, node: tuple[Callable[..., object], Path] ) -> tuple[tuple[str, ...], str]: - """A task node's dependency paths and qualnames, and a digest of every identity field.""" + """An asset node's dependency paths and qualnames, and a digest of every identity field.""" function, file_path = node identity: PythonNodeIdentity = python_node_identity( - kind="task", + kind="asset", name=name, function=function, file_path=file_path, @@ -711,10 +675,10 @@ def _node_files(domain: str, index: int) -> dict[str, str]: from providers.warehouse_api.provider import WarehouseApi from python.{domain}._helpers.rows import RowBatch from python.{domain}.{name}.main import main -from sqlbuild.tasks import task +from sqlbuild.assets import asset -@task(description="Sync {domain} batch {index}.") +@asset(description="Sync {domain} batch {index}.") def {name}(ctx, warehouse_api: WarehouseApi): RowBatch([]) return main(ctx=ctx, warehouse_api=warehouse_api) @@ -807,7 +771,7 @@ def _in_package(package: str, module_name: str) -> bool: class DiscoveryWork(NamedTuple): """Reads, parses and hashes counted while discovering a generated project.""" - tasks: int + assets: int most_per_unit: int shared: Counter[str] per_node: float @@ -851,7 +815,7 @@ def counted_sha256(data: bytes = b"", *args: Any, **kwargs: Any) -> Any: shared_units: dict[tuple[str, str], int] = _unit_work(work, shared) node_units: dict[tuple[str, str], int] = _unit_work(work, nodes) return DiscoveryWork( - tasks=len(discovered.tasks), + assets=len(discovered.assets), most_per_unit=max((shared_units | node_units).values()), shared=_work_by_kind(shared_units), per_node=sum(node_units.values()) / node_count, diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_identity_sharing.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_identity_sharing.py index 7f61b20540..231b10b8a7 100644 --- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_identity_sharing.py +++ b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_identity_sharing.py @@ -259,7 +259,7 @@ def test_given_nodes_sharing_helpers_when_discovered_then_each_file_and_object_i project_dir=tmp_path / "large", node_count=large_count, monkeypatch=monkeypatch ) - assert (small.tasks, large.tasks) == test_case.node_counts + assert (small.assets, large.assets) == test_case.node_counts assert (small.most_per_unit, large.most_per_unit) == ( test_case.expected_most_per_unit, test_case.expected_most_per_unit, diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_inventory.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_inventory.py index aa5e96ce3b..d658ce2173 100644 --- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_inventory.py +++ b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_inventory.py @@ -17,7 +17,6 @@ DiscoveredProjectInputs, DiscoveredProvider, DiscoveredPythonNode, - DiscoveredTaskFunction, LocalConfig, ProjectConfig, PythonNodeGraph, @@ -57,14 +56,14 @@ ), expected_kinds=( PythonNodeKind.LOADER, - PythonNodeKind.TASK, + PythonNodeKind.ASSET, PythonNodeKind.ASSET, PythonNodeKind.ASSET, PythonNodeKind.CHECK, ), expected_typed_selectors=( "loader:load_events", - "task:prepare_orders", + "asset:prepare_orders", "asset:export_orders", "asset:notify_orders", "check:check_orders_export", @@ -101,8 +100,8 @@ def test_given_discovered_python_functions_when_building_graph_then_indexes_node depends_on=(prepare_orders,), ), ), - task_functions=( - DiscoveredTaskFunction( + asset_functions=( + DiscoveredAssetFunction( file_path=Path("/project/tasks/orders.py"), relative_path=Path("python/tasks/orders.py"), name="prepare_orders", @@ -113,8 +112,6 @@ def test_given_discovered_python_functions_when_building_graph_then_indexes_node meta={"owner": "data-eng"}, retry=retry, ), - ), - asset_functions=( DiscoveredAssetFunction( file_path=Path("/project/assets/orders.py"), relative_path=Path("python/assets/orders.py"), @@ -177,10 +174,10 @@ def test_given_discovered_python_functions_when_building_graph_then_indexes_node assert task_node.group == "ingestion" assert task_node.description == "Prepare order export inputs." assert task_node.meta == {"owner": "data-eng"} - assert task_node.task is not None - assert task_node.task.retry is retry + assert task_node.asset is not None + assert task_node.asset.retry is retry assert task_node.identity is not None - assert task_node.identity.node_type == "task" + assert task_node.identity.node_type == "asset" assert task_node.identity.node_name == "prepare_orders" assert "Prepare order export inputs." in task_node.identity.definition_json @@ -241,7 +238,7 @@ def test_given_discovered_python_functions_when_building_graph_then_indexes_node "unique_key", "write_strategy", ), - "task:prepare_orders": ("description", "group", "meta", "tags"), + "asset:prepare_orders": ("description", "group", "meta", "tags"), "asset:export_orders": ( "column_lineage", "columns", @@ -285,8 +282,8 @@ def test_given_fully_configured_python_nodes_when_building_then_identity_config_ contract="enforced", ), ), - task_functions=( - DiscoveredTaskFunction( + asset_functions=( + DiscoveredAssetFunction( file_path=Path("/project/tasks/orders.py"), relative_path=Path("python/tasks/orders.py"), name="prepare_orders", @@ -297,8 +294,6 @@ def test_given_fully_configured_python_nodes_when_building_then_identity_config_ meta={"owner": "data-eng"}, retry=retry, ), - ), - asset_functions=( DiscoveredAssetFunction( file_path=Path("/project/assets/orders.py"), relative_path=Path("python/assets/orders.py"), diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_selectors.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_selectors.py index bbebfe503c..7354a289ea 100644 --- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_selectors.py +++ b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_selectors.py @@ -156,7 +156,7 @@ def test_given_python_node_selectors_when_resolving_then_returns_expected_names( [ PythonNodeSelectorErrorTestCase( description="raises when typed selector has wrong Python node kind", - select=("task:export_orders",), + select=("asset:export_orders",), exclude=(), expected_error_type=ValueError, expected_error_fragment="unknown Python node selector 'export_orders'", diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_unified_selectors.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_unified_selectors.py index cb1b756ef5..948db7f8ab 100644 --- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_unified_selectors.py +++ b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_unified_selectors.py @@ -21,7 +21,7 @@ build_orders_python_node_graph, build_python_node_graph_for_case, build_sibling_sql_ref_python_node_graph, - build_sql_downstream_task_to_loader_python_node_graph, + build_sql_downstream_asset_to_loader_python_node_graph, build_sql_ref_python_node_graph, build_terminal_loader_python_node_graph, model_ref, @@ -418,17 +418,6 @@ def test_given_invalid_typed_sql_ref_when_resolving_then_raises_clear_error( @pytest.mark.parametrize( "test_case", [ - PythonSqlSelectorErrorTestCase( - description="raises when task depends on terminal source loader", - select=("summarize_orders",), - exclude=(), - expected_error_type=ValueError, - expected_error_fragment=( - "Python node 'summarize_orders' depends on terminal loader 'raw_orders'; " - "depend on source 'raw_orders' instead" - ), - python_graph_case="terminal_loader_task_dependency", - ), PythonSqlSelectorErrorTestCase( description="raises when asset depends on terminal source loader", select=("export_orders",), @@ -520,7 +509,7 @@ def test_given_sql_downstream_task_feeds_loader_when_validating_then_raises( test_case: PythonSqlSelectorErrorTestCase, ) -> None: project_graph: ProjectGraph = build_orders_project_graph() - python_graph: PythonNodeGraph = build_sql_downstream_task_to_loader_python_node_graph() + python_graph: PythonNodeGraph = build_sql_downstream_asset_to_loader_python_node_graph() with pytest.raises(test_case.expected_error_type, match=test_case.expected_error_fragment): resolve_run_selection( @@ -534,14 +523,6 @@ def test_given_sql_downstream_task_feeds_loader_when_validating_then_raises( @pytest.mark.parametrize( "test_case", [ - PythonSqlSelectorErrorTestCase( - description="allows task dependency on intermediate loader", - select=("summarize_orders",), - exclude=(), - expected_error_type=ValueError, - expected_error_fragment=None, - python_graph_case="intermediate_loader_task_dependency", - ), PythonSqlSelectorErrorTestCase( description="allows asset dependency on intermediate loader", select=("export_orders",), diff --git a/tests/unit/src/sqlbuild/executor/load/helpers.py b/tests/unit/src/sqlbuild/executor/load/helpers.py index 1cd3a8ad89..7edbb7096b 100644 --- a/tests/unit/src/sqlbuild/executor/load/helpers.py +++ b/tests/unit/src/sqlbuild/executor/load/helpers.py @@ -7,7 +7,6 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.assets import asset from sqlbuild.observability import LifecycleEvent -from sqlbuild.tasks import task def operation_events(events: list[LifecycleEvent]) -> tuple[LifecycleEvent, ...]: @@ -79,7 +78,7 @@ def connect(self, config: dict[str, object]) -> object: return super().connect(config) -@task +@asset def score_orders(ctx: object) -> object: """A task a loader may read through `ctx.result_of`, never `ctx.loader`.""" diff --git a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/helpers.py b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/helpers.py index 6a7767909d..b814d17a68 100644 --- a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/helpers.py @@ -18,7 +18,6 @@ DiscoveredLoaderFunction, DiscoveredProjectInputs, DiscoveredPythonNode, - DiscoveredTaskFunction, LocalConfig, ProjectConfig, PythonNodeGraph, @@ -52,7 +51,6 @@ from sqlbuild.providers import Provider from sqlbuild.python_nodes.types import PythonCheckSeverity from sqlbuild.refs import model -from sqlbuild.tasks import TaskContext, task from tests.unit.src.sqlbuild.compiler.python_nodes._helpers.helpers import ( build_intermediate_loader_asset_dependency_python_node_graph, build_orders_python_node_graph, @@ -181,7 +179,7 @@ def results_of( return self._results.get((node_type, node_name), ())[:limit] -def provider_task(ctx: TaskContext, slack_provider: ExecutionSlackProvider) -> dict[str, object]: +def provider_task(ctx: AssetContext, slack_provider: ExecutionSlackProvider) -> dict[str, object]: return {"target": ctx.target, "provider": slack_provider.label} @@ -193,7 +191,7 @@ def provider_check(ctx: CheckContext, slack_provider: ExecutionSlackProvider) -> return ctx.target == "dev" and slack_provider.label == "slack" -def context_provider_task(ctx: TaskContext) -> dict[str, object]: +def context_provider_task(ctx: AssetContext) -> dict[str, object]: return { "attr": cast(ExecutionSlackProvider, ctx.providers.slack_provider).label, "item": cast(ExecutionSlackProvider, ctx.providers["slack_provider"]).label, @@ -214,7 +212,7 @@ def context_provider_check(ctx: CheckContext) -> bool: ) -def missing_context_provider_task(ctx: TaskContext) -> object: +def missing_context_provider_task(ctx: AssetContext) -> object: return ctx.providers["slack_provider"] @@ -225,8 +223,8 @@ def build_task_context( logger_name: str, run_state: PythonNodeRunState | None = None, result_store: object | None = None, -) -> TaskContext: - return TaskContext( +) -> AssetContext: + return AssetContext( adapter=adapter, connection_config={"warehouse": "dev"}, connection=object(), @@ -293,18 +291,18 @@ def build_check_context( ) -@task +@asset def upstream_task(_ctx: object) -> object: return None -@task +@asset def skipped_upstream_task(_ctx: object) -> object: return None -@task -def fetch_orders(ctx: TaskContext) -> object: +@asset +def fetch_orders(ctx: AssetContext) -> object: return ctx.result( payload={"file": "orders.json"}, metadata={"row_count": 3}, @@ -326,11 +324,11 @@ def export_orders(ctx: AssetContext) -> object: ) -def skip_empty_orders(ctx: TaskContext) -> object: +def skip_empty_orders(ctx: AssetContext) -> object: return ctx.skip(reason="No new orders") -def hard_skip_empty_orders(ctx: TaskContext) -> object: +def hard_skip_empty_orders(ctx: AssetContext) -> object: return ctx.skip(reason="No new orders", mode=SkipMode.HARD) @@ -338,7 +336,7 @@ def export_after_skip(ctx: AssetContext) -> object: return ctx.result(payload={"uri": "should-not-run"}, materialized=True) -def successful_sibling(ctx: TaskContext) -> object: +def successful_sibling(ctx: AssetContext) -> object: return ctx.result(payload={"status": "ready"}) @@ -346,11 +344,11 @@ def export_after_mixed_skip(ctx: AssetContext) -> object: return ctx.result(payload={"uri": "s3://exports/orders.json"}, materialized=True) -def fail_orders(_ctx: TaskContext) -> object: +def fail_orders(_ctx: AssetContext) -> object: raise RuntimeError("API unavailable") -def cursor_window(ctx: TaskContext) -> object: +def cursor_window(ctx: AssetContext) -> object: return ctx.result( payload={ "start_cursor_ts": ctx.start_cursor_ts, @@ -379,22 +377,22 @@ def __init__(self, failures_before_success: int, exception_type: type[Exception] self.exception_type: type[Exception] = exception_type self.attempts: int = 0 self.cost_contexts: list[CostResourceContext | None] = [] - failures: Iterator[Callable[[TaskContext], object]] = ( + failures: Iterator[Callable[[AssetContext], object]] = ( self._raise_transient_failure for _attempt in range(self.failures_before_success) ) self._strategies = iter(chain(failures, repeat(self._return_success))) - def __call__(self, ctx: TaskContext) -> object: + def __call__(self, ctx: AssetContext) -> object: self.attempts += 1 cost_context: CostResourceContext | None = CostContext.current() self.cost_contexts.append(cost_context) return next(self._strategies)(ctx) - def _raise_transient_failure(self, ctx: TaskContext) -> object: + def _raise_transient_failure(self, ctx: AssetContext) -> object: del ctx raise self.exception_type("transient failure") - def _return_success(self, ctx: TaskContext) -> object: + def _return_success(self, ctx: AssetContext) -> object: return ctx.result(payload={"attempts": self.attempts}) @@ -454,8 +452,8 @@ def ingress_calls() -> tuple[str, ...]: return tuple(INGRESS_CALLS) -@task -def prepare_ingress_orders(ctx: TaskContext) -> object: +@asset +def prepare_ingress_orders(ctx: AssetContext) -> object: INGRESS_CALLS.append("prepare_ingress_orders") return ctx.result(payload={"prepared": True}) @@ -481,7 +479,7 @@ def build_ingress_task_loader_graph() -> PythonNodeGraph: project_config=ProjectConfig(name="demo", adapter="duckdb"), local_config=LocalConfig(), loader_functions=(ingress_loader_function(),), - task_functions=(ingress_task_function(),), + asset_functions=(ingress_task_function(),), ) ) @@ -491,8 +489,8 @@ def build_ingress_task_asset_graph() -> PythonNodeGraph: discovered_inputs=DiscoveredProjectInputs( project_config=ProjectConfig(name="demo", adapter="duckdb"), local_config=LocalConfig(), - task_functions=(ingress_task_function(),), asset_functions=( + ingress_task_function(), DiscoveredAssetFunction( file_path=Path("/project/assets/orders.py"), relative_path=Path("python/assets/orders.py"), @@ -505,8 +503,8 @@ def build_ingress_task_asset_graph() -> PythonNodeGraph: ) -def ingress_task_function() -> DiscoveredTaskFunction: - return DiscoveredTaskFunction( +def ingress_task_function() -> DiscoveredAssetFunction: + return DiscoveredAssetFunction( file_path=Path("/project/tasks/orders.py"), relative_path=Path("python/tasks/orders.py"), name="prepare_ingress_orders", @@ -544,8 +542,8 @@ def read_side_calls() -> tuple[str, ...]: return tuple(READ_SIDE_CALLS) -@task(depends_on=(model("stg_orders"),)) -def profile_stg_orders(ctx: TaskContext) -> object: +@asset(depends_on=(model("stg_orders"),)) +def profile_stg_orders(ctx: AssetContext) -> object: READ_SIDE_CALLS.append("profile_stg_orders") return ctx.result(payload={"profiled": True}) @@ -564,16 +562,14 @@ def build_read_side_sql_task_asset_graph() -> PythonNodeGraph: discovered_inputs=DiscoveredProjectInputs( project_config=ProjectConfig(name="demo", adapter="duckdb"), local_config=LocalConfig(), - task_functions=( - DiscoveredTaskFunction( + asset_functions=( + DiscoveredAssetFunction( file_path=Path("/project/tasks/orders.py"), relative_path=Path("python/tasks/orders.py"), name="profile_stg_orders", function=profile_stg_orders, depends_on=(model("stg_orders"),), ), - ), - asset_functions=( DiscoveredAssetFunction( file_path=Path("/project/assets/orders.py"), relative_path=Path("python/assets/orders.py"), @@ -586,7 +582,7 @@ def build_read_side_sql_task_asset_graph() -> PythonNodeGraph: ) -@task(name="upstream_task") +@asset(name="upstream_task") def check_upstream_task(_ctx: object) -> object: return {"rows": 3} @@ -631,8 +627,8 @@ def build_python_check_graph(*, check_function: DiscoveredCheckFunction) -> Pyth discovered_inputs=DiscoveredProjectInputs( project_config=ProjectConfig(name="demo", adapter="duckdb"), local_config=LocalConfig(), - task_functions=( - DiscoveredTaskFunction( + asset_functions=( + DiscoveredAssetFunction( file_path=Path("/project/tasks/orders.py"), relative_path=Path("python/tasks/orders.py"), name="upstream_task", diff --git a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_contexts.py b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_contexts.py index 54184ffb9b..7c1f64ff42 100644 --- a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_contexts.py +++ b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_contexts.py @@ -12,6 +12,7 @@ from sqlbuild.adapter.contract.models import LifeCycleEvent from sqlbuild.assets import AssetContext from sqlbuild.assets import SkipMode as AssetSkipMode +from sqlbuild.assets import SkipMode as TaskSkipMode from sqlbuild.compiler.python_nodes.types import PythonNodeKind, PythonNodeStatus, SkipMode from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.node_results.models import NodeResultEnvelope @@ -21,8 +22,6 @@ ) from sqlbuild.python_nodes.models import SqlResourceRef from sqlbuild.refs import model, source -from sqlbuild.tasks import SkipMode as TaskSkipMode -from sqlbuild.tasks import TaskContext from tests.unit.src.sqlbuild.executor.python_nodes._helpers._test_types import ( PublicSkipModeExportTestCase, PythonNodeContextHelperTestCase, @@ -71,7 +70,7 @@ def test_given_task_context_when_using_helpers_then_records_and_qualifies_names( ) -> None: adapter: PythonNodeContextTestAdapter = PythonNodeContextTestAdapter() statement_recorder: StatementRecorder = StatementRecorder() - context: TaskContext = build_task_context( + context: AssetContext = build_task_context( adapter=adapter, statement_recorder=statement_recorder, logger_name=test_case.expected_logger_name, @@ -195,7 +194,7 @@ def test_given_public_task_and_asset_modules_when_importing_skip_mode_then_expor def test_given_context_skip_when_mode_is_string_then_normalizes_to_skip_mode( test_case: PythonNodeSkipModeInputTestCase, ) -> None: - context: TaskContext = build_task_context( + context: AssetContext = build_task_context( adapter=PythonNodeContextTestAdapter(), statement_recorder=StatementRecorder(), logger_name="sqlbuild.task.fetch_orders", @@ -231,7 +230,7 @@ def test_given_task_context_when_inspecting_api_then_loader_only_fields_are_abse test_case: PythonNodeContextHelperTestCase, ) -> None: adapter: PythonNodeContextTestAdapter = PythonNodeContextTestAdapter() - context: TaskContext = build_task_context( + context: AssetContext = build_task_context( adapter=adapter, statement_recorder=StatementRecorder(), logger_name=test_case.expected_logger_name, @@ -285,7 +284,7 @@ def test_given_task_context_when_resolving_sql_relations_then_validates_declared ) -> None: model_ref: SqlResourceRef = model(test_case.raw_name) source_ref: SqlResourceRef = source("orders") - context: TaskContext = TaskContext( + context: AssetContext = AssetContext( adapter=PythonNodeContextTestAdapter(), connection_config={"warehouse": "dev"}, connection=object(), @@ -326,7 +325,7 @@ def test_given_context_with_result_store_when_reading_upstream_outputs_then_retu test_case: PythonNodeRunStateTestCase, ) -> None: expected_result: NodeResultEnvelope = NodeResultEnvelope( - node_type=PythonNodeKind.TASK.value, + node_type=PythonNodeKind.ASSET.value, node_name="upstream_task", run_id="test_run", status=PythonNodeStatus.SUCCESS.value, @@ -341,7 +340,7 @@ def test_given_context_with_result_store_when_reading_upstream_outputs_then_retu statement_recorder=StatementRecorder(), logger_name="sqlbuild.asset.export_customers", result_store=PythonNodeContextTestResultStore( - {(PythonNodeKind.TASK.value, "upstream_task"): (expected_result,)} + {(PythonNodeKind.ASSET.value, "upstream_task"): (expected_result,)} ), ) @@ -370,7 +369,7 @@ def test_given_context_with_result_store_when_reading_upstream_outputs_then_retu def test_given_context_without_result_store_when_reading_result_then_raises_or_returns_default( test_case: PythonNodeRunStateTestCase, ) -> None: - context: TaskContext = build_task_context( + context: AssetContext = build_task_context( adapter=PythonNodeContextTestAdapter(), statement_recorder=StatementRecorder(), logger_name="sqlbuild.task.fetch_orders", diff --git a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_execution.py b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_execution.py index 41439907dd..53f4f219a1 100644 --- a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_execution.py +++ b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_execution.py @@ -7,7 +7,7 @@ import pytest -from sqlbuild._native import CostContext, DiscoveredAssetFunction, DiscoveredTaskFunction +from sqlbuild._native import CostContext, DiscoveredAssetFunction from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.compiler.python_nodes.types import PythonNodeStatus from sqlbuild.cost.models import CostResourceContext @@ -90,7 +90,7 @@ def malformed_task() -> PythonNodeResult: with invocation_scope("inv-malformed-task"), dispatcher_scope(dispatcher): result: tuple[PythonNodeExecutionResult, ...] = execute_ordered_test_nodes( nodes=( - DiscoveredTaskFunction( + DiscoveredAssetFunction( file_path=Path("/project/tasks/bad.py"), relative_path=Path("python/tasks/bad.py"), name="malformed_task", @@ -150,7 +150,7 @@ def test_given_successful_callable_when_result_persistence_fails_then_resource_o "write", lambda record: (_ for _ in ()).throw(RuntimeError("persistence failed")), ) - node: DiscoveredTaskFunction = DiscoveredTaskFunction( + node: DiscoveredAssetFunction = DiscoveredAssetFunction( file_path=Path("/project/tasks/orders.py"), relative_path=Path("python/tasks/orders.py"), name="persist_orders", @@ -203,7 +203,7 @@ def test_given_successful_callable_when_owned_finalizer_fails_then_resource_only events: list[LifecycleEvent] = [] dispatcher: EventDispatcher = EventDispatcher() dispatcher.subscribe_lifecycle(subscriber=events.append, accepts_opaque=False) - node: DiscoveredTaskFunction = DiscoveredTaskFunction( + node: DiscoveredAssetFunction = DiscoveredAssetFunction( file_path=Path("/project/tasks/orders.py"), relative_path=Path("python/tasks/orders.py"), name="finalize_orders", @@ -255,8 +255,8 @@ def test_given_successful_callable_when_owned_finalizer_fails_then_resource_only def test_given_task_asset_chain_when_executing_python_nodes_then_records_results( test_case: PythonNodeExecutorTestCase, ) -> None: - nodes: tuple[DiscoveredTaskFunction | DiscoveredAssetFunction, ...] = ( - DiscoveredTaskFunction( + nodes: tuple[DiscoveredAssetFunction | DiscoveredAssetFunction, ...] = ( + DiscoveredAssetFunction( file_path=Path("/project/tasks/orders.py"), relative_path=Path("python/tasks/orders.py"), name="fetch_orders", @@ -317,8 +317,8 @@ def test_given_task_asset_chain_when_executing_python_nodes_then_records_results def test_given_provider_parameters_when_executing_python_nodes_then_providers_are_injected( test_case: PythonNodeExecutorTestCase, ) -> None: - nodes: tuple[DiscoveredTaskFunction | DiscoveredAssetFunction, ...] = ( - DiscoveredTaskFunction( + nodes: tuple[DiscoveredAssetFunction | DiscoveredAssetFunction, ...] = ( + DiscoveredAssetFunction( file_path=Path("/project/tasks/provider.py"), relative_path=Path("python/tasks/provider.py"), name="provider_task", @@ -375,8 +375,8 @@ def test_given_provider_parameters_when_executing_python_nodes_then_providers_ar def test_given_provider_container_when_executing_python_nodes_then_context_exposes_providers( test_case: PythonNodeExecutorTestCase, ) -> None: - nodes: tuple[DiscoveredTaskFunction | DiscoveredAssetFunction, ...] = ( - DiscoveredTaskFunction( + nodes: tuple[DiscoveredAssetFunction | DiscoveredAssetFunction, ...] = ( + DiscoveredAssetFunction( file_path=Path("/project/tasks/provider.py"), relative_path=Path("python/tasks/provider.py"), name="context_provider_task", @@ -435,8 +435,8 @@ def test_given_provider_container_when_executing_python_nodes_then_context_expos def test_given_missing_provider_container_when_executing_python_nodes_then_failures_are_recorded( test_case: PythonNodeExecutorTestCase, ) -> None: - nodes: tuple[DiscoveredTaskFunction | DiscoveredAssetFunction, ...] = ( - DiscoveredTaskFunction( + nodes: tuple[DiscoveredAssetFunction | DiscoveredAssetFunction, ...] = ( + DiscoveredAssetFunction( file_path=Path("/project/tasks/provider.py"), relative_path=Path("python/tasks/provider.py"), name="provider_task", @@ -494,7 +494,7 @@ def test_given_missing_context_provider_when_executing_python_node_then_failure_ ) -> None: result: tuple[PythonNodeExecutionResult, ...] = execute_ordered_test_nodes( nodes=( - DiscoveredTaskFunction( + DiscoveredAssetFunction( file_path=Path("/project/tasks/provider.py"), relative_path=Path("python/tasks/provider.py"), name="missing_context_provider_task", @@ -542,7 +542,7 @@ def test_given_ready_python_node_when_executing_then_uses_existing_run_state( test_case: PythonNodeExecutorTestCase, ) -> None: run_state: PythonNodeRunState = PythonNodeRunState() - task_node: DiscoveredTaskFunction = DiscoveredTaskFunction( + task_node: DiscoveredAssetFunction = DiscoveredAssetFunction( file_path=Path("/project/tasks/orders.py"), relative_path=Path("python/tasks/orders.py"), name="fetch_orders", @@ -632,8 +632,8 @@ def test_given_ready_python_node_when_executing_then_uses_existing_run_state( def test_given_cursor_overrides_when_executing_python_nodes_then_context_receives_cursors( test_case: PythonNodeExecutorTestCase, ) -> None: - nodes: tuple[DiscoveredTaskFunction | DiscoveredAssetFunction, ...] = ( - DiscoveredTaskFunction( + nodes: tuple[DiscoveredAssetFunction | DiscoveredAssetFunction, ...] = ( + DiscoveredAssetFunction( file_path=Path("/project/tasks/orders.py"), relative_path=Path("python/tasks/orders.py"), name="cursor_window", @@ -684,8 +684,8 @@ def test_given_cursor_overrides_when_executing_python_nodes_then_context_receive def test_given_hard_skipped_upstream_when_executing_python_nodes_then_skips_downstream( test_case: PythonNodeExecutorTestCase, ) -> None: - nodes: tuple[DiscoveredTaskFunction | DiscoveredAssetFunction, ...] = ( - DiscoveredTaskFunction( + nodes: tuple[DiscoveredAssetFunction | DiscoveredAssetFunction, ...] = ( + DiscoveredAssetFunction( file_path=Path("/project/tasks/orders.py"), relative_path=Path("python/tasks/orders.py"), name="skip_empty_orders", @@ -776,14 +776,14 @@ def test_given_mixed_python_skips_when_executing_nodes_then_fan_in_matches_mode( ) -> None: skip_function: Callable[..., object] | None = test_case.skip_function assert skip_function is not None - nodes: tuple[DiscoveredTaskFunction | DiscoveredAssetFunction, ...] = ( - DiscoveredTaskFunction( + nodes: tuple[DiscoveredAssetFunction | DiscoveredAssetFunction, ...] = ( + DiscoveredAssetFunction( file_path=Path("/project/tasks/orders.py"), relative_path=Path("python/tasks/orders.py"), name="skip_empty_orders", function=skip_function, ), - DiscoveredTaskFunction( + DiscoveredAssetFunction( file_path=Path("/project/tasks/sibling.py"), relative_path=Path("python/tasks/sibling.py"), name="successful_sibling", @@ -844,8 +844,8 @@ def test_given_mixed_python_skips_when_executing_nodes_then_fan_in_matches_mode( def test_given_failed_upstream_when_executing_python_nodes_then_blocks_downstream( test_case: PythonNodeExecutorTestCase, ) -> None: - nodes: tuple[DiscoveredTaskFunction | DiscoveredAssetFunction, ...] = ( - DiscoveredTaskFunction( + nodes: tuple[DiscoveredAssetFunction | DiscoveredAssetFunction, ...] = ( + DiscoveredAssetFunction( file_path=Path("/project/tasks/orders.py"), relative_path=Path("python/tasks/orders.py"), name="fail_orders", @@ -906,8 +906,8 @@ def test_given_retry_policy_when_transient_failures_then_retries_and_succeeds( ) -> None: flaky_task: FlakyTask = FlakyTask(failures_before_success=2, exception_type=TimeoutError) sleeps: list[float] = [] - nodes: tuple[DiscoveredTaskFunction | DiscoveredAssetFunction, ...] = ( - DiscoveredTaskFunction( + nodes: tuple[DiscoveredAssetFunction | DiscoveredAssetFunction, ...] = ( + DiscoveredAssetFunction( file_path=Path("/project/tasks/orders.py"), relative_path=Path("python/tasks/orders.py"), name="flaky_task", @@ -1031,8 +1031,8 @@ def test_given_retry_policy_when_attempts_exhausted_then_records_final_exception ) -> None: flaky_task: FlakyTask = FlakyTask(failures_before_success=3, exception_type=TimeoutError) sleeps: list[float] = [] - nodes: tuple[DiscoveredTaskFunction | DiscoveredAssetFunction, ...] = ( - DiscoveredTaskFunction( + nodes: tuple[DiscoveredAssetFunction | DiscoveredAssetFunction, ...] = ( + DiscoveredAssetFunction( file_path=Path("/project/tasks/orders.py"), relative_path=Path("python/tasks/orders.py"), name="flaky_task", @@ -1090,8 +1090,8 @@ def test_given_retry_policy_when_exception_is_not_selected_then_does_not_retry( ) -> None: flaky_task: FlakyTask = FlakyTask(failures_before_success=1, exception_type=ValueError) sleeps: list[float] = [] - nodes: tuple[DiscoveredTaskFunction | DiscoveredAssetFunction, ...] = ( - DiscoveredTaskFunction( + nodes: tuple[DiscoveredAssetFunction | DiscoveredAssetFunction, ...] = ( + DiscoveredAssetFunction( file_path=Path("/project/tasks/orders.py"), relative_path=Path("python/tasks/orders.py"), name="flaky_task", @@ -1144,8 +1144,8 @@ def test_given_retry_policy_when_backoff_exceeds_cap_then_sleep_is_capped( ) -> None: flaky_task: FlakyTask = FlakyTask(failures_before_success=3, exception_type=TimeoutError) sleeps: list[float] = [] - nodes: tuple[DiscoveredTaskFunction | DiscoveredAssetFunction, ...] = ( - DiscoveredTaskFunction( + nodes: tuple[DiscoveredAssetFunction | DiscoveredAssetFunction, ...] = ( + DiscoveredAssetFunction( file_path=Path("/project/tasks/orders.py"), relative_path=Path("python/tasks/orders.py"), name="flaky_task", diff --git a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_ingress_execution.py b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_ingress_execution.py index 573a3696a3..bfe3299788 100644 --- a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_ingress_execution.py +++ b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_ingress_execution.py @@ -209,7 +209,7 @@ def test_given_ingress_task_and_asset_when_executing_then_each_completion_follow ( IngressSchedulerSkipLifecycleTestCase( description="scheduler bypasses downstream loader and task exactly once", - expected_resource_ids=("source:raw_orders", "task:bypassed_task"), + expected_resource_ids=("source:raw_orders", "asset:bypassed_task"), expected_event_types=( "resource_attempt_started", "resource_attempt_skipped", @@ -225,7 +225,7 @@ def test_given_pre_recorded_hard_skip_when_scheduler_bypasses_downstream_then_on ) -> None: base_graph: PythonNodeGraph = build_ingress_task_loader_graph() bypassed_node: DiscoveredPythonNode = DiscoveredPythonNode( - kind=PythonNodeKind.TASK, + kind=PythonNodeKind.ASSET, file_path=Path("/project/tasks/bypassed.py"), relative_path=Path("python/tasks/bypassed.py"), name="bypassed_task", @@ -241,7 +241,7 @@ def test_given_pre_recorded_hard_skip_when_scheduler_bypasses_downstream_then_on name="prepare_ingress_orders", result=PythonNodeExecutionResult( node_name="prepare_ingress_orders", - kind=PythonNodeKind.TASK, + kind=PythonNodeKind.ASSET, status=PythonNodeStatus.SKIPPED, skip_mode=SkipMode.HARD, skip_reason="upstream hard skip", @@ -251,7 +251,7 @@ def test_given_pre_recorded_hard_skip_when_scheduler_bypasses_downstream_then_on results=( LifecycleNodeResult( name="prepare_ingress_orders", - kind="task", + kind="asset", status=LifecycleNodeStatus.SKIPPED, skip_mode=SkipMode.HARD, ), @@ -263,7 +263,7 @@ def test_given_pre_recorded_hard_skip_when_scheduler_bypasses_downstream_then_on ), LifecycleNodeResult( name="bypassed_task", - kind="task", + kind="asset", status=LifecycleNodeStatus.SKIPPED, skip_mode=SkipMode.HARD, ), @@ -333,7 +333,7 @@ def test_given_scheduler_bypassed_loader_when_persistence_fails_then_attempt_fai name="prepare_ingress_orders", result=PythonNodeExecutionResult( node_name="prepare_ingress_orders", - kind=PythonNodeKind.TASK, + kind=PythonNodeKind.ASSET, status=PythonNodeStatus.SKIPPED, skip_mode=SkipMode.HARD, ), diff --git a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_python_checks.py b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_python_checks.py index c735ce8329..f683d4f48d 100644 --- a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_python_checks.py +++ b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_python_checks.py @@ -317,7 +317,7 @@ def test_given_python_check_when_executing_then_returns_expected_result( run_state: PythonNodeRunState = PythonNodeRunState() upstream_result: PythonNodeExecutionResult = PythonNodeExecutionResult( node_name="upstream_task", - kind=PythonNodeKind.TASK, + kind=PythonNodeKind.ASSET, status=test_case.upstream_status, payload={"rows": 3}, metadata={"rows": 3}, @@ -327,9 +327,9 @@ def test_given_python_check_when_executing_then_returns_expected_result( run_state.record_result(node_function=check_upstream_task, result=upstream_result) result_store: PythonNodeContextTestResultStore = PythonNodeContextTestResultStore( { - (PythonNodeKind.TASK.value, "upstream_task"): ( + (PythonNodeKind.ASSET.value, "upstream_task"): ( NodeResultEnvelope( - node_type=PythonNodeKind.TASK.value, + node_type=PythonNodeKind.ASSET.value, node_name="upstream_task", run_id="run_1", status=test_case.upstream_status.value, @@ -410,7 +410,7 @@ def test_given_blocked_python_check_when_persisted_then_terminal_precedes_integr graph: PythonNodeGraph = build_python_check_graph(check_function=check_function) upstream_result: PythonNodeExecutionResult = PythonNodeExecutionResult( node_name="upstream_task", - kind=PythonNodeKind.TASK, + kind=PythonNodeKind.ASSET, status=test_case.upstream_status, skip_mode=test_case.upstream_skip_mode, skip_reason="blocked", diff --git a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_read_side_execution.py b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_read_side_execution.py index a40b19a781..942aed9910 100644 --- a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_read_side_execution.py +++ b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_read_side_execution.py @@ -168,8 +168,8 @@ def test_given_sql_dep_fails_when_finalizing_read_side_then_skips_unrun_python_n assert tuple(event.resource_id for event in events) == ( "asset:export_stg_profile", "asset:export_stg_profile", - "task:profile_stg_orders", - "task:profile_stg_orders", + "asset:profile_stg_orders", + "asset:profile_stg_orders", ) assert len({event.resource_attempt_id for event in events}) == 2 assert all(event.run_id == "test_run" for event in events) diff --git a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_results.py b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_results.py index 9cfd436b80..63c5f44227 100644 --- a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_results.py +++ b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_results.py @@ -179,7 +179,7 @@ def test_given_check_context_when_building_results_then_returns_check_results( [ PythonNodeReturnNormalizationTestCase( description="normalizes plain return as successful payload", - kind=PythonNodeKind.TASK, + kind=PythonNodeKind.ASSET, returned={"rows": 3}, expected_status=PythonNodeStatus.SUCCESS, expected_payload={"rows": 3}, @@ -190,7 +190,7 @@ def test_given_check_context_when_building_results_then_returns_check_results( ), PythonNodeReturnNormalizationTestCase( description="normalizes none return as empty success", - kind=PythonNodeKind.TASK, + kind=PythonNodeKind.ASSET, returned=None, expected_status=PythonNodeStatus.SUCCESS, expected_payload=None, @@ -216,7 +216,7 @@ def test_given_check_context_when_building_results_then_returns_check_results( ), PythonNodeReturnNormalizationTestCase( description="normalizes downstream skip signal", - kind=PythonNodeKind.TASK, + kind=PythonNodeKind.ASSET, returned=PythonNodeSkipResult( reason="No new source rows", mode=SkipMode.HARD, @@ -256,7 +256,7 @@ def test_given_python_node_return_when_normalizing_then_returns_execution_result [ PythonNodeReturnNormalizationErrorTestCase( description="rejects materialized flag on task result", - kind=PythonNodeKind.TASK, + kind=PythonNodeKind.ASSET, returned=PythonNodeResult(materialized=False), expected_error_fragment="Only asset Python nodes", ) @@ -291,7 +291,7 @@ def test_given_python_node_exception_when_building_failure_result_then_records_m ) -> None: result: PythonNodeExecutionResult = build_python_node_failure_result( node_name="fetch_orders", - kind=PythonNodeKind.TASK, + kind=PythonNodeKind.ASSET, error=test_case.error, ) @@ -346,7 +346,7 @@ def test_given_upstream_python_node_results_when_evaluating_fan_in_then_returns_ upstream_results: tuple[PythonNodeExecutionResult, ...] = tuple( PythonNodeExecutionResult( node_name=f"upstream_{index}", - kind=PythonNodeKind.TASK, + kind=PythonNodeKind.ASSET, status=status, skip_mode=test_case.upstream_skip_modes[index], ) diff --git a/tests/unit/src/sqlbuild/executor/scheduling/_helpers/lifecycle/test_lifecycle_scheduler.py b/tests/unit/src/sqlbuild/executor/scheduling/_helpers/lifecycle/test_lifecycle_scheduler.py index 718a5e8057..53007f2e8d 100644 --- a/tests/unit/src/sqlbuild/executor/scheduling/_helpers/lifecycle/test_lifecycle_scheduler.py +++ b/tests/unit/src/sqlbuild/executor/scheduling/_helpers/lifecycle/test_lifecycle_scheduler.py @@ -53,7 +53,9 @@ def test_given_mixed_lifecycle_nodes_when_running_scheduler_then_runs_in_topolog name="load_orders", kind="loader", upstream_names=("summarize_pages",) ), LifecycleExecutionNode(name="load_pages", kind="loader"), - LifecycleExecutionNode(name="summarize_pages", kind="task", upstream_names=("load_pages",)), + LifecycleExecutionNode( + name="summarize_pages", kind="asset", upstream_names=("load_pages",) + ), ) result: LifecycleSchedulerResult = run_lifecycle_scheduler( @@ -93,7 +95,9 @@ def test_given_failed_lifecycle_node_when_running_scheduler_then_skips_downstrea calls: list[str] = [] nodes: tuple[LifecycleExecutionNode, ...] = ( LifecycleExecutionNode(name="load_pages", kind="loader"), - LifecycleExecutionNode(name="summarize_pages", kind="task", upstream_names=("load_pages",)), + LifecycleExecutionNode( + name="summarize_pages", kind="asset", upstream_names=("load_pages",) + ), LifecycleExecutionNode( name="load_orders", kind="loader", upstream_names=("summarize_pages",) ), @@ -136,9 +140,9 @@ def test_given_soft_skip_and_successful_sibling_when_running_scheduler_then_runs ) -> None: calls: list[str] = [] nodes: tuple[LifecycleExecutionNode, ...] = ( - LifecycleExecutionNode(name="A", kind="task"), - LifecycleExecutionNode(name="X", kind="task"), - LifecycleExecutionNode(name="C", kind="task", upstream_names=("A", "X")), + LifecycleExecutionNode(name="A", kind="asset"), + LifecycleExecutionNode(name="X", kind="asset"), + LifecycleExecutionNode(name="C", kind="asset", upstream_names=("A", "X")), ) result: LifecycleSchedulerResult = run_lifecycle_scheduler( @@ -176,9 +180,9 @@ def test_given_all_upstreams_soft_skipped_when_running_scheduler_then_soft_skips ) -> None: calls: list[str] = [] nodes: tuple[LifecycleExecutionNode, ...] = ( - LifecycleExecutionNode(name="A", kind="task"), - LifecycleExecutionNode(name="B", kind="task"), - LifecycleExecutionNode(name="C", kind="task", upstream_names=("A", "B")), + LifecycleExecutionNode(name="A", kind="asset"), + LifecycleExecutionNode(name="B", kind="asset"), + LifecycleExecutionNode(name="C", kind="asset", upstream_names=("A", "B")), ) def handler(node: LifecycleExecutionNode) -> LifecycleNodeResult: @@ -217,9 +221,9 @@ def test_given_hard_skip_and_successful_sibling_when_running_scheduler_then_skip ) -> None: calls: list[str] = [] nodes: tuple[LifecycleExecutionNode, ...] = ( - LifecycleExecutionNode(name="A", kind="task"), - LifecycleExecutionNode(name="X", kind="task"), - LifecycleExecutionNode(name="C", kind="task", upstream_names=("A", "X")), + LifecycleExecutionNode(name="A", kind="asset"), + LifecycleExecutionNode(name="X", kind="asset"), + LifecycleExecutionNode(name="C", kind="asset", upstream_names=("A", "X")), ) result: LifecycleSchedulerResult = run_lifecycle_scheduler( @@ -249,7 +253,7 @@ def test_given_unknown_dependency_when_running_scheduler_then_raises_input_error test_case: LifecycleSchedulerErrorTestCase, ) -> None: nodes: tuple[LifecycleExecutionNode, ...] = ( - LifecycleExecutionNode(name="summarize_pages", kind="task", upstream_names=("missing",)), + LifecycleExecutionNode(name="summarize_pages", kind="asset", upstream_names=("missing",)), ) with pytest.raises(ExecutorInputError) as exc_info: diff --git a/tests/unit/src/sqlbuild/integrations/dagster/helpers.py b/tests/unit/src/sqlbuild/integrations/dagster/helpers.py index 036ed0e904..3769f59567 100644 --- a/tests/unit/src/sqlbuild/integrations/dagster/helpers.py +++ b/tests/unit/src/sqlbuild/integrations/dagster/helpers.py @@ -223,8 +223,8 @@ def build_python_augmented_dagster_test_dag() -> Mapping[str, Any]: dag["nodes"] = [ *dag["nodes"], { - "id": "task:prepare_orders", - "kind": "task", + "id": "asset:prepare_orders", + "kind": "asset", "name": "prepare_orders", "asset_key": ["task", "prepare_orders"], "path": "python/tasks/prepare_orders.py", @@ -255,8 +255,8 @@ def build_python_augmented_dagster_test_dag() -> Mapping[str, Any]: ] dag["edges"] = [ *dag["edges"], - {"from_id": "model:orders", "to_id": "task:prepare_orders"}, - {"from_id": "task:prepare_orders", "to_id": "asset:orders_export"}, + {"from_id": "model:orders", "to_id": "asset:prepare_orders"}, + {"from_id": "asset:prepare_orders", "to_id": "asset:orders_export"}, {"from_id": "asset:orders_export", "to_id": "check:check_orders_export"}, ] dag["checks"] = [ diff --git a/tests/unit/src/sqlbuild/python_nodes/decorators/_test_types.py b/tests/unit/src/sqlbuild/python_nodes/decorators/_test_types.py index 2099112251..91677a4d22 100644 --- a/tests/unit/src/sqlbuild/python_nodes/decorators/_test_types.py +++ b/tests/unit/src/sqlbuild/python_nodes/decorators/_test_types.py @@ -4,18 +4,6 @@ from sqlbuild.python_nodes.types import PythonCheckSeverity -@dataclass(frozen=True) -class TaskDecoratorMetadataTestCase: - description: str - expected_name: str - expected_dep_count: int - expected_tags: tuple[str, ...] - expected_group: str | None - expected_description: str | None - expected_meta: dict[str, object] | None - expected_retry: RetryPolicy | None = None - - @dataclass(frozen=True) class AssetDecoratorMetadataTestCase: description: str diff --git a/tests/unit/src/sqlbuild/python_nodes/decorators/helpers.py b/tests/unit/src/sqlbuild/python_nodes/decorators/helpers.py index ec6d517c8e..c6c3bfd47e 100644 --- a/tests/unit/src/sqlbuild/python_nodes/decorators/helpers.py +++ b/tests/unit/src/sqlbuild/python_nodes/decorators/helpers.py @@ -3,9 +3,5 @@ from __future__ import annotations -def upstream_task(_ctx: object) -> dict[str, object]: - return {"ok": True} - - def upstream_asset(_ctx: object) -> dict[str, object]: return {"uri": "s3://exports/upstream.parquet"} diff --git a/tests/unit/src/sqlbuild/python_nodes/decorators/test_python_decorators.py b/tests/unit/src/sqlbuild/python_nodes/decorators/test_python_decorators.py index 725c615b33..fe1bc60eb5 100644 --- a/tests/unit/src/sqlbuild/python_nodes/decorators/test_python_decorators.py +++ b/tests/unit/src/sqlbuild/python_nodes/decorators/test_python_decorators.py @@ -18,11 +18,9 @@ FactoryDefinition, NodeSetRef, RetryPolicy, - TaskDefinition, ) from sqlbuild.python_nodes.types import NodeSetKind, PythonCheckSeverity from sqlbuild.refs import job, model, seed, tag -from sqlbuild.tasks import get_task_definition, task from tests.unit.src.sqlbuild.python_nodes.decorators._test_types import ( AssetDecoratorMetadataTestCase, CheckDecoratorMetadataTestCase, @@ -30,62 +28,8 @@ FactoryDecoratorMetadataTestCase, HookReadsTestCase, NodeSetDependencyTestCase, - TaskDecoratorMetadataTestCase, ) -from tests.unit.src.sqlbuild.python_nodes.decorators.helpers import upstream_asset, upstream_task - - -@pytest.mark.parametrize( - "test_case", - [ - TaskDecoratorMetadataTestCase( - description="stores task metadata and normalizes single dependency", - expected_name="fetch_window", - expected_dep_count=1, - expected_tags=("api", "hourly"), - expected_group="ingestion", - expected_description="Fetch an API window.", - expected_meta={"owner": "data-eng"}, - expected_retry=RetryPolicy( - max_attempts=3, - retry_on=[TimeoutError, ConnectionError], - initial_delay_seconds=0.5, - jitter=False, - ), - ) - ], - ids=lambda case: case.description, -) -def test_given_task_decorator_when_reading_definition_then_returns_metadata( - test_case: TaskDecoratorMetadataTestCase, -) -> None: - @task( - name=test_case.expected_name, - depends_on=upstream_task, - tags=test_case.expected_tags, - group=test_case.expected_group, - meta=test_case.expected_meta, - retry=test_case.expected_retry, - ) - def fetch_window(_ctx: object) -> dict[str, object]: - """Fetch an API window.""" - - return {"window": "2026-05-30"} - - definition: TaskDefinition | None = get_task_definition(fetch_window) - - assert definition == TaskDefinition( - name=test_case.expected_name, - depends_on=(upstream_task,), - tags=test_case.expected_tags, - group=test_case.expected_group, - description=test_case.expected_description, - meta=test_case.expected_meta, - retry=test_case.expected_retry, - ) - assert definition is not None - assert len(definition.depends_on) == test_case.expected_dep_count - assert get_asset_definition(fetch_window) is None +from tests.unit.src.sqlbuild.python_nodes.decorators.helpers import upstream_asset @pytest.mark.parametrize( @@ -156,7 +100,7 @@ def export_customers(_ctx: object) -> dict[str, object]: test_case.expected_column_descriptions ) assert definition.column_lineage == test_case.expected_column_lineage - assert get_task_definition(export_customers) is None + assert get_asset_definition(export_customers) is None @pytest.mark.parametrize( @@ -205,7 +149,7 @@ def export_customers_exists(_ctx: object) -> bool: assert definition is not None assert len(definition.depends_on) == test_case.expected_dep_count assert get_asset_definition(export_customers_exists) is None - assert get_task_definition(export_customers_exists) is None + assert get_asset_definition(export_customers_exists) is None @pytest.mark.parametrize( @@ -229,7 +173,7 @@ def generated_nodes() -> list[object]: assert decorated is generated_nodes assert definition == FactoryDefinition(name=test_case.expected_name) - assert get_task_definition(generated_nodes) is None + assert get_asset_definition(generated_nodes) is None assert get_asset_definition(generated_nodes) is None assert get_check_definition(generated_nodes) is None @@ -332,14 +276,14 @@ def refresh_orders() -> None: ], ids=lambda case: case.description, ) -def test_given_job_or_tag_dependency_when_decorating_a_task_then_keeps_it_as_declared( +def test_given_job_or_tag_dependency_when_decorating_an_asset_then_keeps_it_as_declared( test_case: NodeSetDependencyTestCase, ) -> None: - @task(depends_on=cast(Any, test_case.dependency)) + @asset(depends_on=cast(Any, test_case.dependency)) def report() -> None: - """Test task report.""" + """Test asset report.""" - definition: TaskDefinition | None = get_task_definition(report) + definition: AssetDefinition | None = get_asset_definition(report) assert definition is not None assert definition.depends_on == (test_case.expected_dependency,) diff --git a/tests/unit/src/sqlbuild/runtime/observability/test_catalog.py b/tests/unit/src/sqlbuild/runtime/observability/test_catalog.py index e14d16718b..0396fc4eb0 100644 --- a/tests/unit/src/sqlbuild/runtime/observability/test_catalog.py +++ b/tests/unit/src/sqlbuild/runtime/observability/test_catalog.py @@ -268,7 +268,7 @@ def test_given_known_scoped_event_without_correlation_when_constructing_then_err description="retry requires safe error type", event_type="retry_scheduled", run_id="run-1", - resource_id="task:orders", + resource_id="asset:orders", payload={ "failed_attempt_number": 1, "next_attempt_number": 2, @@ -280,7 +280,7 @@ def test_given_known_scoped_event_without_correlation_when_constructing_then_err description="retry attempts must be consecutive", event_type="retry_scheduled", run_id="run-1", - resource_id="task:orders", + resource_id="asset:orders", payload={ "failed_attempt_number": 1, "next_attempt_number": 3, diff --git a/tests/unit/src/sqlbuild/runtime/observability/test_resource_attempt_lifecycle.py b/tests/unit/src/sqlbuild/runtime/observability/test_resource_attempt_lifecycle.py index 64be6d0f65..fedac8e2d4 100644 --- a/tests/unit/src/sqlbuild/runtime/observability/test_resource_attempt_lifecycle.py +++ b/tests/unit/src/sqlbuild/runtime/observability/test_resource_attempt_lifecycle.py @@ -72,7 +72,7 @@ def test_given_bounded_skip_when_attempt_marked_skipped_then_reason_is_not_canon with invocation_scope("inv-skip"), run_scope("run-skip"), dispatcher_scope(dispatcher): with ResourceAttemptLifecycle( - resource_id="task:orders", + resource_id="asset:orders", resource_kind="task", resource_name="orders", ) as lifecycle: @@ -103,7 +103,7 @@ def test_given_no_context_when_attempt_runs_then_transient_identity_and_dispatch assert current_event_dispatcher() is None with ResourceAttemptLifecycle( - resource_id="task:direct", + resource_id="asset:direct", resource_kind="task", resource_name="direct", run_id="direct-run", @@ -111,7 +111,7 @@ def test_given_no_context_when_attempt_runs_then_transient_identity_and_dispatch identity: ExecutionIdentity | None = current_execution_identity() assert identity is not None assert identity.run_id == "direct-run" - assert identity.resource_id == "task:direct" + assert identity.resource_id == "asset:direct" assert current_event_dispatcher() is not None with OperationLifecycle(operation_kind="python_node", operation_name="python_task"): nested: ExecutionIdentity | None = current_execution_identity() @@ -141,7 +141,7 @@ def test_given_invalid_direct_attempt_when_start_fails_then_transient_context_is with pytest.raises(ObservabilityValidationError): with ResourceAttemptLifecycle( - resource_id="task:invalid", + resource_id="asset:invalid", resource_kind="task", resource_name="invalid", attempt_number=-1, @@ -171,13 +171,13 @@ def test_given_parent_context_when_nested_start_fails_then_exact_parent_is_resto with ( invocation_scope("parent-inv"), run_scope("parent-run"), - resource_attempt_scope(resource_id="task:parent", resource_attempt_id="parent-attempt"), + resource_attempt_scope(resource_id="asset:parent", resource_attempt_id="parent-attempt"), dispatcher_scope(dispatcher), ): parent: ExecutionIdentity | None = current_execution_identity() with pytest.raises(ObservabilityValidationError): with ResourceAttemptLifecycle( - resource_id="task:invalid", + resource_id="asset:invalid", resource_kind="task", resource_name="invalid", attempt_number=-1, diff --git a/ui/src/lib/api/mock/build-fixtures.ts b/ui/src/lib/api/mock/build-fixtures.ts index b6e014eea5..b3079b91a4 100644 --- a/ui/src/lib/api/mock/build-fixtures.ts +++ b/ui/src/lib/api/mock/build-fixtures.ts @@ -176,7 +176,7 @@ export const NODE_SPECS: Record = { export const PYTHON_SOURCE: Record = { raw__orders: { - file: 'tasks/raw__orders.py', + file: 'python/raw__orders.py', source: `from sqlbuild.loaders import loader from sqlbuild.executor.load.models import LoaderContext @@ -189,13 +189,13 @@ def raw__orders(ctx: LoaderContext): return fetch_since(ctx.current_cursor_value)` }, export_revenue_csv: { - file: 'tasks/export_revenue_csv.py', - source: `from sqlbuild.tasks import task + file: 'python/export_revenue_csv.py', + source: `from sqlbuild.assets import asset from shop.file_drop import upload_csv -@task(upstream=["revenue_report"]) +@asset(upstream=["revenue_report"]) def export_revenue_csv(ctx): """Exports the revenue report as CSV to the finance file drop.""" rows = ctx.query('SELECT * FROM __ref("revenue_report") ORDER BY revenue_date') diff --git a/ui/src/lib/api/mock/build-sql.ts b/ui/src/lib/api/mock/build-sql.ts index a870983f36..1996179566 100644 --- a/ui/src/lib/api/mock/build-sql.ts +++ b/ui/src/lib/api/mock/build-sql.ts @@ -64,7 +64,7 @@ function generated(spec: NodeSpec, sn: SnapshotNode, columns = spec.columns): st /** Current raw SQL in the snapshot; null for sources, seeds and Python nodes. */ export function rawSql(sn: SnapshotNode): string | null { - if (sn.kind === 'source' || sn.kind === 'seed' || sn.kind === 'loader' || sn.kind === 'task') return null; + if (sn.kind === 'source' || sn.kind === 'seed' || sn.kind === 'loader' || sn.kind === 'asset') return null; if (sn.key === 'fact_orders') return FACT_ORDERS_SQL; const spec = NODE_SPECS[sn.key]; return spec ? generated(spec, sn) : null; diff --git a/ui/src/lib/api/mock/operate-graph-selection.test.ts b/ui/src/lib/api/mock/operate-graph-selection.test.ts index f3f76c21a7..767e0d4f83 100644 --- a/ui/src/lib/api/mock/operate-graph-selection.test.ts +++ b/ui/src/lib/api/mock/operate-graph-selection.test.ts @@ -23,11 +23,11 @@ describe('mock selection evaluation', () => { it('counts models of every materialization as one kind', () => { expect(mockVocabulary().kinds).toEqual([ + { value: 'asset', count: 1 }, { value: 'loader', count: 1 }, { value: 'model', count: 12 }, { value: 'seed', count: 1 }, - { value: 'source', count: 2 }, - { value: 'task', count: 1 } + { value: 'source', count: 2 } ]); }); }); diff --git a/ui/src/lib/api/mock/operate-graph.ts b/ui/src/lib/api/mock/operate-graph.ts index 4bc3120723..f479c0e0fc 100644 --- a/ui/src/lib/api/mock/operate-graph.ts +++ b/ui/src/lib/api/mock/operate-graph.ts @@ -17,8 +17,8 @@ function node( ? 'models/sources.yml' : kind === 'seed' ? `seeds/${key}.csv` - : kind === 'task' || kind === 'loader' - ? `tasks/${key}.py` + : kind === 'asset' || kind === 'loader' + ? `python/${key}.py` : `models/${group}/${key}.sql`); return { key, kind, group, tags: options.tags ?? [], path, cursor: options.cursor ?? null, upstream }; } @@ -40,7 +40,7 @@ export const SNAPSHOT_NODES: SnapshotNode[] = [ node('daily_activity_rollup', 'incremental', 'activity', ['hourly_activity_ctx'], { cursor: 'timestamp' }), node('revenue_report', 'table', 'reporting', ['daily_revenue'], { tags: ['finance'] }), node('activity_report', 'table', 'reporting', ['hourly_activity_ctx']), - node('export_revenue_csv', 'task', 'exports', ['revenue_report'], { tags: ['export'] }) + node('export_revenue_csv', 'asset', 'exports', ['revenue_report'], { tags: ['export'] }) ]; const BY_KEY = new Map(SNAPSHOT_NODES.map((n) => [n.key, n])); diff --git a/ui/src/lib/api/mock/platform.ts b/ui/src/lib/api/mock/platform.ts index b8f668a3f9..92655f06b4 100644 --- a/ui/src/lib/api/mock/platform.ts +++ b/ui/src/lib/api/mock/platform.ts @@ -434,7 +434,7 @@ function createPlatformFixtures(now: number, role: BuiltInRole) { setting('secret_access_key', 'SecretStr', 'EXPORT_BUCKET_SECRET_ACCESS_KEY', 'Access key secret', { required: true }) ], usedBy: [ - { kind: 'task', name: 'export_revenue_csv' }, + { kind: 'asset', name: 'export_revenue_csv' }, { kind: 'sink', name: 'run_log_archive' } ], nodes: ['export_revenue_csv'], diff --git a/ui/src/lib/api/mock/run-detail.ts b/ui/src/lib/api/mock/run-detail.ts index 9fc5fb9d11..f5800ea322 100644 --- a/ui/src/lib/api/mock/run-detail.ts +++ b/ui/src/lib/api/mock/run-detail.ts @@ -423,7 +423,7 @@ export function execution(record: RunRecord, nodes: RunNode[], now?: number): Ru if (n.status === 'warn') { ev('audit_completed', e - 60, n.nodeKey, 'not_null(payment_id): 2 failing rows (severity warn)'); out('stdout', e - 60, `WARN ${n.nodeKey} audit not_null(payment_id): 2 rows`); - } else if (n.status === 'success' && k !== 'task' && k !== 'loader') { + } else if (n.status === 'success' && k !== 'asset' && k !== 'loader') { ev('audit_completed', e - 60, n.nodeKey, 'All audits passed'); } const dur = ((e - s) / 1000).toFixed(2); diff --git a/ui/src/lib/selection/reading.ts b/ui/src/lib/selection/reading.ts index 9003ae66ec..4b11d17dee 100644 --- a/ui/src/lib/selection/reading.ts +++ b/ui/src/lib/selection/reading.ts @@ -14,7 +14,6 @@ const KIND_PLURAL: Record = { udf: 'UDFs', table_fn: 'table functions', function: 'functions', - task: 'tasks', asset: 'assets', loader: 'loaders', check: 'checks' diff --git a/ui/src/lib/selection/syntax.ts b/ui/src/lib/selection/syntax.ts index 647498b040..d07fd40aef 100644 --- a/ui/src/lib/selection/syntax.ts +++ b/ui/src/lib/selection/syntax.ts @@ -4,7 +4,7 @@ // the paths from `a` to `b`. Pure: it splits text and never decides what exists. /** Selector types `sqb` accepts before a colon. */ -export const SELECTOR_TYPES = ['tag', 'path', 'job', 'kind', 'source', 'seed', 'test', 'task', 'asset', 'loader', 'check'] as const; +export const SELECTOR_TYPES = ['tag', 'path', 'job', 'kind', 'source', 'seed', 'test', 'asset', 'loader', 'check'] as const; export type SelectorType = (typeof SELECTOR_TYPES)[number]; /** One comma part of a token, such as `+tag:finance`. */ From 19e59684e2b407a89426c74cc81185c9bf82b622 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 19 Nov 2026 04:07:12 +0000 Subject: [PATCH 03/12] wip: move unit tests and the materialized guard off the task kind Unit suite passes (9382). Removes the materialized check that only rejected tasks, the task-only decorator, context and result tests, and re-records the two identity-sharing digests, whose only change is the node type (verified: with kind=task they reproduce the old digests). --- .../python_files/imported_python_nodes.rs | 8 +-- fensu.toml | 11 ---- .../executor/python_nodes/_helpers/results.py | 6 -- .../sqlbuild/cli/commands/main/dag/helpers.py | 2 +- .../cli/commands/main/dag/test_dag_command.py | 2 +- .../cli/commands/main/init/test_run.py | 2 +- .../cli/output/main/plan/test_plan_text.py | 7 +-- .../_helpers/test_python_sql_references.py | 2 +- .../sqlbuild/compiler/dag/main/test_build.py | 1 - .../discovery/_helpers/_test_types.py | 8 +-- .../_helpers/test_filesystem_factories.py | 37 +++-------- .../_helpers/test_filesystem_python_nodes.py | 56 +++++++---------- .../_helpers/test_resource_identities.py | 4 +- .../compiler/discovery/_test_types.py | 2 - .../sqlbuild/compiler/discovery/test_main.py | 27 +++----- .../compiler/pipeline/_helpers/helpers.py | 4 +- .../_helpers/test_identity_sharing.py | 4 +- .../python_nodes/_helpers/test_inventory.py | 9 ++- .../python_nodes/_helpers/test_selectors.py | 2 +- .../_helpers/test_unified_selectors.py | 2 +- .../src/sqlbuild/executor/load/test_models.py | 4 +- .../python_nodes/_helpers/test_contexts.py | 55 ---------------- .../python_nodes/_helpers/test_execution.py | 63 +------------------ .../_helpers/test_ingress_execution.py | 2 +- .../_helpers/test_lifecycle_nodes.py | 2 +- .../python_nodes/_helpers/test_results.py | 24 ------- .../sqlbuild/integrations/dagster/helpers.py | 4 +- .../integrations/dagster/test_assets.py | 8 +-- .../integrations/rivers/test_assets.py | 8 +-- .../decorators/test_python_decorators.py | 1 - 30 files changed, 82 insertions(+), 285 deletions(-) diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/imported_python_nodes.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/imported_python_nodes.rs index 984789baa8..f19fb8c463 100644 --- a/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/imported_python_nodes.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/imported_python_nodes.rs @@ -28,8 +28,8 @@ const READ_FACTORY: UserCodeHost = UserCodeHost::ReadFactoryDefinition; const READ_AUDIT_FACTORY: UserCodeHost = UserCodeHost::ReadAuditFactoryDefinition; const READ_HOOK: UserCodeHost = UserCodeHost::ReadHookDefinition; const NOT_A_NODE: &str = "that is not a SQLBuild asset, loader, or check"; -/// The removed task module, and how `from sqlbuild import tasks` names it. -const REMOVED_TASKS: (&str, (&str, &str)) = ("sqlbuild.tasks", ("sqlbuild", "tasks")); +/// The package and name of the removed task module, as `from sqlbuild import tasks` names it. +const REMOVED_TASKS: (&str, &str) = ("sqlbuild", "tasks"); /// One Python-node kind: its reader, record class, bucket and copied definition fields. struct NodeKind { @@ -343,9 +343,9 @@ fn imports_removed_tasks(error: &Bound<'_, PyAny>) -> bool { let text = |attribute: &str| -> Option { error.getattr_opt(attribute).ok().flatten()?.extract().ok() }; - let (module, (package, name)) = REMOVED_TASKS; + let (package, name) = REMOVED_TASKS; let imported: Option = text("name"); - imported.as_deref() == Some(module) + imported.as_deref() == Some(format!("{package}.{name}").as_str()) || (imported.as_deref() == Some(package) && text("name_from").as_deref() == Some(name)) } diff --git a/fensu.toml b/fensu.toml index 9fec3e7dee..dd6c7d87d2 100644 --- a/fensu.toml +++ b/fensu.toml @@ -667,11 +667,6 @@ modules = ["sqlbuild.python_nodes.main.read_loader_definition"] symbols = ["read_loader_definition"] reason = "Native code resolves these through the sqlbuild._native_host name table." -[[targets.python.dead_code.roots]] -modules = ["sqlbuild.python_nodes.main.read_task_definition"] -symbols = ["read_task_definition"] -reason = "Native code resolves these through the sqlbuild._native_host name table." - [[targets.python.dead_code.roots]] modules = ["sqlbuild.python_nodes.models"] symbols = ["AuditCase", "NodeSetRef", "SqlResourceRef"] @@ -1513,12 +1508,6 @@ path = "src/sqlbuild/hooks.py" symbols = ["hook"] reason = "Bare @hook is evaluated by Python as hook(function), requiring positional function calling." -[[targets.python.rule_exceptions]] -rule = "FFS120" -path = "src/sqlbuild/tasks.py" -symbols = ["task"] -reason = "Bare @task is evaluated by Python as task(function), requiring positional function calling." - [[targets.python.rule_exceptions]] rule = "FFS120" path = "src/sqlbuild/python_nodes/_helpers/loader_authoring.py" diff --git a/src/sqlbuild/executor/python_nodes/_helpers/results.py b/src/sqlbuild/executor/python_nodes/_helpers/results.py index b221e5b9ec..287ab6b0eb 100644 --- a/src/sqlbuild/executor/python_nodes/_helpers/results.py +++ b/src/sqlbuild/executor/python_nodes/_helpers/results.py @@ -54,7 +54,6 @@ def normalize_python_node_return( skip_reason=returned.reason, ) if isinstance(returned, PythonNodeResult): - _validate_materialized_flag(kind=kind, materialized=returned.materialized) return PythonNodeExecutionResult( node_name=node_name, kind=kind, @@ -120,8 +119,3 @@ def evaluate_python_node_fan_in( reason="All upstream Python nodes were skipped", skip_mode=SkipMode.SOFT, ) - - -def _validate_materialized_flag(*, kind: PythonNodeKind, materialized: bool | None) -> None: - if materialized is not None and kind != PythonNodeKind.ASSET: - raise ExecutorInputError("Only asset Python nodes may set materialized") diff --git a/tests/unit/src/sqlbuild/cli/commands/main/dag/helpers.py b/tests/unit/src/sqlbuild/cli/commands/main/dag/helpers.py index 7cb0c71b02..2cc2cf31a5 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/dag/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/dag/helpers.py @@ -75,7 +75,7 @@ def prepare_advisory_placement_dag_project(root: Path) -> Path: def prepare_python_dag_project(root: Path) -> Path: - """Create a local project with Python tasks, assets, and checks.""" + """Create a local project with Python assets and checks.""" project_dir: Path = prepare_static_dag_project(root) tasks_dir: Path = project_dir / "python" / "tasks" diff --git a/tests/unit/src/sqlbuild/cli/commands/main/dag/test_dag_command.py b/tests/unit/src/sqlbuild/cli/commands/main/dag/test_dag_command.py index 182d635a2c..20f420cf25 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/dag/test_dag_command.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/dag/test_dag_command.py @@ -191,7 +191,7 @@ def test_given_python_nodes_when_running_dag_json_then_outputs_artifact_metadata } assert exit_code == test_case.expected_exit_code - assert nodes_by_id[test_case.expected_task_id]["kind"] == "task" + assert nodes_by_id[test_case.expected_task_id]["kind"] == "asset" assert nodes_by_id[test_case.expected_task_id]["tags"] == test_case.expected_task_tags assert nodes_by_id[test_case.expected_task_id]["group"] == test_case.expected_task_group assert nodes_by_id[test_case.expected_task_id]["meta"] == test_case.expected_task_meta diff --git a/tests/unit/src/sqlbuild/cli/commands/main/init/test_run.py b/tests/unit/src/sqlbuild/cli/commands/main/init/test_run.py index b3ed765fdd..61c8678a8b 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/init/test_run.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/init/test_run.py @@ -63,7 +63,7 @@ "Next steps:", "1. Add sources to sources/", "2. Add seeds to seeds/", - "3. Add Python loaders, tasks, assets, checks, and factories to python/", + "3. Add Python loaders, assets, checks, and factories to python/", "4. Add SQL hooks to hooks/sql/, Python hooks to hooks/python/", "reusable audits to audits/generic/", "standalone audits to audits/singular/", diff --git a/tests/unit/src/sqlbuild/cli/output/main/plan/test_plan_text.py b/tests/unit/src/sqlbuild/cli/output/main/plan/test_plan_text.py index 88f691af5e..4cffc3d982 100644 --- a/tests/unit/src/sqlbuild/cli/output/main/plan/test_plan_text.py +++ b/tests/unit/src/sqlbuild/cli/output/main/plan/test_plan_text.py @@ -662,7 +662,6 @@ "Plan ready 1 selected, 1 source to load, 3 Python nodes", "Python ingress (2)", "prepare_orders", - "task", "publish_orders", "asset", "Python read-side (1)", @@ -723,7 +722,7 @@ expected_fragments=( "Python ingress (1)", "prepare_orders", - "task (changed)", + "asset (changed)", "python diff:", "source diff:", "dependency diff:", @@ -1202,9 +1201,9 @@ expected_fragments=( "Providers", "\u2514\u2500\u2500 marker_provider", + " ├── asset publish_orders (MarkerProvider)", " ├── hook mark_pre (MarkerProvider)", - " ├── loader raw_orders (MarkerProvider)", - " └── task publish_orders (MarkerProvider)", + " └── loader raw_orders (MarkerProvider)", ), unexpected_fragments=("used by 3 selected Python surfaces", "parameter"), ), diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_python_sql_references.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_python_sql_references.py index a27db5e8ef..a915dad008 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_python_sql_references.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_python_sql_references.py @@ -226,7 +226,7 @@ expected_error_fragments=( "loader:raw_customers names model:customers as 'customers'", "loaders run before models and seeds and cannot read them; read model:customers " - "from a task or asset instead", + "from an asset instead", ), ), PythonSqlReferenceErrorTestCase( diff --git a/tests/unit/src/sqlbuild/compiler/dag/main/test_build.py b/tests/unit/src/sqlbuild/compiler/dag/main/test_build.py index 606fd31189..d15be0ea64 100644 --- a/tests/unit/src/sqlbuild/compiler/dag/main/test_build.py +++ b/tests/unit/src/sqlbuild/compiler/dag/main/test_build.py @@ -150,7 +150,6 @@ def test_given_project_graph_when_building_dag_artifact_then_includes_assets_edg "udf", "table_fn", "model", - "task", "asset", "check", "sql_test", diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py index 315cff9f79..a0651cd80c 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py @@ -751,25 +751,21 @@ class DiscoverPythonNodeFactoriesTestCase: description: str files: dict[str, str] expected_loader_names: tuple[str, ...] - expected_task_names: tuple[str, ...] expected_asset_names: tuple[str, ...] expected_check_names: tuple[str, ...] expected_loader_dependency_counts: tuple[int, ...] = () - expected_task_dependency_counts: tuple[int, ...] = () expected_asset_dependency_counts: tuple[int, ...] = () expected_check_dependency_counts: tuple[int, ...] = () expected_error_fragment: str = "" @dataclass(frozen=True) -class DiscoverTaskAssetFunctionsTestCase: +class DiscoverAssetFunctionsTestCase: description: str files: dict[str, str] - expected_task_names: tuple[str, ...] - expected_task_dependency_counts: tuple[int, ...] - expected_task_tags: tuple[tuple[str, ...], ...] expected_asset_names: tuple[str, ...] expected_asset_dependency_counts: tuple[int, ...] + expected_asset_tags: tuple[tuple[str, ...], ...] expected_asset_column_names: tuple[tuple[str, ...], ...] expected_asset_lineage_columns: tuple[tuple[str, ...], ...] expected_error_fragment: str = "" diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_factories.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_factories.py index 8c35fab456..f184c69a81 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_factories.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_factories.py @@ -28,7 +28,6 @@ from sqlbuild.checks import check from sqlbuild.factories import factory from sqlbuild.loaders import loader -from sqlbuild.assets import asset def make_task(name): @@ -69,12 +68,10 @@ def regional_pipeline(): """, }, expected_loader_names=("raw_orders",), - expected_task_names=("fetch_orders",), - expected_asset_names=("orders_export",), + expected_asset_names=("fetch_orders", "orders_export"), expected_check_names=("orders_export_exists",), expected_loader_dependency_counts=(1,), - expected_task_dependency_counts=(0,), - expected_asset_dependency_counts=(1,), + expected_asset_dependency_counts=(0, 1), expected_check_dependency_counts=(1,), ), DiscoverPythonNodeFactoriesTestCase( @@ -98,7 +95,6 @@ def generated_loaders(): """, }, expected_loader_names=("raw_orders", "raw_customers"), - expected_task_names=(), expected_asset_names=(), expected_check_names=(), expected_loader_dependency_counts=(0, 0), @@ -120,7 +116,6 @@ def generated_check(ctx): """, }, expected_loader_names=(), - expected_task_names=(), expected_asset_names=(), expected_check_names=("orders_export_exists",), expected_check_dependency_counts=(0,), @@ -142,7 +137,6 @@ def export(ctx): """, }, expected_loader_names=(), - expected_task_names=(), expected_asset_names=("orders_export",), expected_check_names=(), expected_asset_dependency_counts=(0,), @@ -164,7 +158,6 @@ def load(ctx): """, }, expected_loader_names=("raw_orders",), - expected_task_names=(), expected_asset_names=(), expected_check_names=(), expected_loader_dependency_counts=(0,), @@ -182,7 +175,6 @@ def export(ctx): """, }, expected_loader_names=(), - expected_task_names=(), expected_asset_names=("orders_export",), expected_check_names=(), expected_asset_dependency_counts=(0,), @@ -214,10 +206,9 @@ def generated_check(ctx): """, }, expected_loader_names=(), - expected_task_names=("prepare_orders",), - expected_asset_names=(), + expected_asset_names=("prepare_orders",), expected_check_names=("orders_emea_exists",), - expected_task_dependency_counts=(0,), + expected_asset_dependency_counts=(0,), expected_check_dependency_counts=(1,), ), ), @@ -233,15 +224,11 @@ def test_given_factory_nodes_when_discovering_python_nodes_then_returns_generate result: DiscoveredPythonNodeFunctions = discover_python_node_functions(project_dir=tmp_path) assert tuple(node.name for node in result.loaders) == test_case.expected_loader_names - assert tuple(node.name for node in result.assets) == test_case.expected_task_names assert tuple(node.name for node in result.assets) == test_case.expected_asset_names assert tuple(node.name for node in result.checks) == test_case.expected_check_names assert tuple(len(node.depends_on) for node in result.loaders) == ( test_case.expected_loader_dependency_counts ) - assert tuple(len(node.depends_on) for node in result.assets) == ( - test_case.expected_task_dependency_counts - ) assert tuple(len(node.depends_on) for node in result.assets) == ( test_case.expected_asset_dependency_counts ) @@ -266,7 +253,6 @@ def broken_factory(): """, }, expected_loader_names=(), - expected_task_names=(), expected_asset_names=(), expected_check_names=(), expected_error_fragment="must return a SQLBuild node function", @@ -284,7 +270,6 @@ def broken_factory(): """, }, expected_loader_names=(), - expected_task_names=(), expected_asset_names=(), expected_check_names=(), expected_error_fragment="must return a SQLBuild node function", @@ -302,7 +287,6 @@ def broken_factory(): """, }, expected_loader_names=(), - expected_task_names=(), expected_asset_names=(), expected_check_names=(), expected_error_fragment="must return a SQLBuild node function", @@ -320,7 +304,6 @@ def broken_factory(): """, }, expected_loader_names=(), - expected_task_names=(), expected_asset_names=(), expected_check_names=(), expected_error_fragment="must return a SQLBuild node function", @@ -338,7 +321,6 @@ def broken_factory(): """, }, expected_loader_names=(), - expected_task_names=(), expected_asset_names=(), expected_check_names=(), expected_error_fragment="must return a SQLBuild node function", @@ -360,10 +342,9 @@ def broken_factory(): """, }, expected_loader_names=(), - expected_task_names=(), expected_asset_names=(), expected_check_names=(), - expected_error_fragment="returned item 0 that is not a SQLBuild task", + expected_error_fragment="returned item 0 that is not a SQLBuild asset", ), DiscoverPythonNodeFactoriesTestCase( description="raises when factory returns nested structures", @@ -384,10 +365,9 @@ def broken_factory(): """, }, expected_loader_names=(), - expected_task_names=(), expected_asset_names=(), expected_check_names=(), - expected_error_fragment="returned item 0 that is not a SQLBuild task", + expected_error_fragment="returned item 0 that is not a SQLBuild asset", ), DiscoverPythonNodeFactoriesTestCase( description="raises when factory returned item is not decorated", @@ -406,10 +386,9 @@ def broken_factory(): """, }, expected_loader_names=(), - expected_task_names=(), expected_asset_names=(), expected_check_names=(), - expected_error_fragment="returned item 0 that is not a SQLBuild task", + expected_error_fragment="returned item 0 that is not a SQLBuild asset", ), DiscoverPythonNodeFactoriesTestCase( description="raises when factory requires arguments", @@ -424,7 +403,6 @@ def broken_factory(region): """, }, expected_loader_names=(), - expected_task_names=(), expected_asset_names=(), expected_check_names=(), expected_error_fragment="must not require arguments", @@ -442,7 +420,6 @@ def broken_factory(): """, }, expected_loader_names=(), - expected_task_names=(), expected_asset_names=(), expected_check_names=(), expected_error_fragment="failed during discovery: boom", diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_python_nodes.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_python_nodes.py index ee0362efb0..e4ba6f2796 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_python_nodes.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_python_nodes.py @@ -1,4 +1,4 @@ -"""Tests for task and asset discovery.""" +"""Tests for asset discovery.""" from __future__ import annotations @@ -20,10 +20,10 @@ ) from sqlbuild.compiler.python_nodes.main.identity import build_python_node_identity from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( + DiscoverAssetFunctionsTestCase, DiscoverCheckFunctionsTestCase, DiscoverPythonRootImportTestCase, DiscoverPythonRootNodesTestCase, - DiscoverTaskAssetFunctionsTestCase, PythonRootHelperIdentityTestCase, PythonRootProjectIsolationTestCase, RestoredPythonPackageIdentityTestCase, @@ -60,8 +60,8 @@ def orders(ctx): @pytest.mark.parametrize( "test_case", [ - DiscoverTaskAssetFunctionsTestCase( - description="discovers decorated tasks and assets from explicit folders", + DiscoverAssetFunctionsTestCase( + description="discovers decorated assets from explicit folders", files={ "python/tasks/windows.py": """ from sqlbuild.assets import asset @@ -94,22 +94,18 @@ def ignored_init_asset(ctx): return None """, }, - expected_task_names=("fetch_window",), - expected_task_dependency_counts=(0,), - expected_task_tags=(("api",),), - expected_asset_names=("export_customers",), - expected_asset_dependency_counts=(1,), - expected_asset_column_names=(("customer_id",),), - expected_asset_lineage_columns=(("customer_id",),), + expected_asset_names=("export_customers", "fetch_window"), + expected_asset_dependency_counts=(1, 0), + expected_asset_tags=((), ("api",)), + expected_asset_column_names=(("customer_id",), ()), + expected_asset_lineage_columns=(("customer_id",), ()), ), - DiscoverTaskAssetFunctionsTestCase( - description="returns empty tuples when task and asset folders do not exist", + DiscoverAssetFunctionsTestCase( + description="returns empty tuples when asset folders do not exist", files={}, - expected_task_names=(), - expected_task_dependency_counts=(), - expected_task_tags=(), expected_asset_names=(), expected_asset_dependency_counts=(), + expected_asset_tags=(), expected_asset_column_names=(), expected_asset_lineage_columns=(), ), @@ -117,7 +113,7 @@ def ignored_init_asset(ctx): ids=lambda case: case.description, ) def test_given_project_dir_when_discovering_python_nodes_then_returns_expected( - test_case: DiscoverTaskAssetFunctionsTestCase, + test_case: DiscoverAssetFunctionsTestCase, tmp_path: Path, ) -> None: relative_path: str @@ -127,19 +123,12 @@ def test_given_project_dir_when_discovering_python_nodes_then_returns_expected( file_path.parent.mkdir(parents=True, exist_ok=True) file_path.write_text(contents, encoding="utf-8") - tasks: tuple[DiscoveredAssetFunction, ...] = tuple( - discover_python_node_functions(project_dir=tmp_path).assets - ) assets: tuple[DiscoveredAssetFunction, ...] = tuple( discover_python_node_functions(project_dir=tmp_path).assets ) - assert tuple(task.name for task in tasks) == test_case.expected_task_names - assert ( - tuple(len(task.depends_on) for task in tasks) == test_case.expected_task_dependency_counts - ) - assert tuple(task.tags for task in tasks) == test_case.expected_task_tags assert tuple(asset.name for asset in assets) == test_case.expected_asset_names + assert tuple(asset.tags for asset in assets) == test_case.expected_asset_tags assert tuple(len(asset.depends_on) for asset in assets) == ( test_case.expected_asset_dependency_counts ) @@ -159,14 +148,12 @@ def test_given_project_dir_when_discovering_python_nodes_then_returns_expected( @pytest.mark.parametrize( "test_case", [ - DiscoverTaskAssetFunctionsTestCase( - description="raises clear error when task import fails", + DiscoverAssetFunctionsTestCase( + description="raises clear error when a node import fails", files={"python/tasks/broken.py": "import missing_task_dependency\n"}, - expected_task_names=(), - expected_task_dependency_counts=(), - expected_task_tags=(), expected_asset_names=(), expected_asset_dependency_counts=(), + expected_asset_tags=(), expected_asset_column_names=(), expected_asset_lineage_columns=(), expected_error_fragment="Failed to import Python node file", @@ -175,7 +162,7 @@ def test_given_project_dir_when_discovering_python_nodes_then_returns_expected( ids=lambda case: case.description, ) def test_given_python_node_import_error_when_discovering_then_raises_clear_error( - test_case: DiscoverTaskAssetFunctionsTestCase, + test_case: DiscoverAssetFunctionsTestCase, tmp_path: Path, ) -> None: relative_path: str @@ -292,8 +279,8 @@ def orders_export_exists(ctx): }, expected_nodes=( ("loader", "raw_orders", "python/orders.py"), - ("task", "prepare_orders", "python/orders.py"), ("asset", "orders_export", "python/orders.py"), + ("asset", "prepare_orders", "python/orders.py"), ("check", "orders_export_exists", "python/orders.py"), ), ), @@ -311,7 +298,7 @@ def export_uri(name): def order_label(value): return str(value) -LABEL = task +LABEL = asset """, "python/orders/ingest/loaders.py": """ from sqlbuild.loaders import loader @@ -348,8 +335,8 @@ def orders_export_exists(ctx): }, expected_nodes=( ("loader", "raw_orders", "python/orders/ingest/loaders.py"), - ("task", "label_orders", "python/orders/pipeline/tasks.py"), ("asset", "orders_export", "python/exports/assets.py"), + ("asset", "label_orders", "python/orders/pipeline/tasks.py"), ("check", "orders_export_exists", "python/exports/deep/nested/checks.py"), ), ), @@ -379,7 +366,6 @@ def test_given_python_root_when_discovering_python_nodes_then_decorators_define_ discovered: tuple[tuple[str, str, str], ...] = ( *(("loader", node.name, node.relative_path.as_posix()) for node in result.loaders), - *(("task", node.name, node.relative_path.as_posix()) for node in result.assets), *(("asset", node.name, node.relative_path.as_posix()) for node in result.assets), *(("check", node.name, node.relative_path.as_posix()) for node in result.checks), ) diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_resource_identities.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_resource_identities.py index b84948ac86..45bc37a438 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_resource_identities.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_resource_identities.py @@ -192,7 +192,7 @@ def test_given_noncanonical_name_when_validating_resource_identity_then_coded_er expected_error_fragment="Invalid SQL hook identity 'GrantAccess'", ), DiscoveredResourceIdentityErrorTestCase( - description="Python task name", + description="Python asset name", project_files={ "python/tasks/orders.py": ( "from sqlbuild.assets import asset\n\n" @@ -201,7 +201,7 @@ def test_given_noncanonical_name_when_validating_resource_identity_then_coded_er " return None\n" ) }, - expected_error_fragment="Invalid task identity 'DailyTask'", + expected_error_fragment="Invalid asset identity 'DailyTask'", ), ], ids=lambda case: case.description, diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_test_types.py b/tests/unit/src/sqlbuild/compiler/discovery/_test_types.py index 64afc1d3fb..e10ff46209 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_test_types.py @@ -51,7 +51,6 @@ class DiscoverProjectInputsTestCase: expected_macro_paths: tuple[str, ...] expected_loader_names: tuple[str, ...] expected_adapter_path: str | None - expected_task_names: tuple[str, ...] = () expected_asset_names: tuple[str, ...] = () expected_check_names: tuple[str, ...] = () expected_hook_names: tuple[str, ...] = () @@ -72,7 +71,6 @@ class DiscoverFactoryValidationTestCase: description: str repo_files: dict[str, str] expected_loader_names: tuple[str, ...] = () - expected_task_names: tuple[str, ...] = () expected_asset_names: tuple[str, ...] = () expected_check_names: tuple[str, ...] = () expected_error_fragment: str = "" diff --git a/tests/unit/src/sqlbuild/compiler/discovery/test_main.py b/tests/unit/src/sqlbuild/compiler/discovery/test_main.py index 8e73799523..721760acc5 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/test_main.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/test_main.py @@ -143,8 +143,7 @@ class SlackProvider(Provider): expected_macro_paths=("macros/python/name_helpers.py",), expected_loader_names=("raw_orders",), expected_adapter_path="adapter.py", - expected_task_names=("fetch_window",), - expected_asset_names=("export_customers",), + expected_asset_names=("export_customers", "fetch_window"), expected_check_names=("export_customers_exists",), expected_hook_names=("notify_success",), expected_provider_names=("slack_provider",), @@ -265,10 +264,6 @@ def test_given_project_repo_slice_when_discovering_inputs_then_it_returns_expect tuple(loader_function.name for loader_function in discovered_inputs.loader_functions) == test_case.expected_loader_names ) - assert ( - tuple(task_function.name for task_function in discovered_inputs.asset_functions) - == test_case.expected_task_names - ) assert ( tuple(asset_function.name for asset_function in discovered_inputs.asset_functions) == test_case.expected_asset_names @@ -426,7 +421,7 @@ def load(ctx): expected_loader_names=("raw_orders",), ), DiscoverFactoryValidationTestCase( - description="generated task and asset dependency validates", + description="generated asset dependencies validate", repo_files=base_repo_files() | { "python/factories/generated.py": """ @@ -448,8 +443,7 @@ def export(ctx): return [prepare, export] """, }, - expected_task_names=("prepare_orders",), - expected_asset_names=("orders_export",), + expected_asset_names=("prepare_orders", "orders_export"), ), DiscoverFactoryValidationTestCase( description="single-kind factory validates in factories folder", @@ -469,7 +463,7 @@ def prepare(ctx): return prepare """, }, - expected_task_names=("prepare_orders",), + expected_asset_names=("prepare_orders",), ), DiscoverFactoryValidationTestCase( description="factory returning its own module-level node registers it once", @@ -490,7 +484,7 @@ def order_nodes(): return orders """, }, - expected_task_names=("orders",), + expected_asset_names=("orders",), ), DiscoverFactoryValidationTestCase( description="factory binding its generated node to a module global registers it once", @@ -512,7 +506,7 @@ def orders(ctx): return orders """, }, - expected_task_names=("orders",), + expected_asset_names=("orders",), ), DiscoverFactoryValidationTestCase( description="factory returning an imported python module node registers it once", @@ -540,7 +534,7 @@ def prepare_orders(ctx): return None """, }, - expected_task_names=("prepare_orders", "publish_orders"), + expected_asset_names=("prepare_orders", "publish_orders"), ), DiscoverFactoryValidationTestCase( description="factory can import ordinary private helper module", @@ -562,10 +556,10 @@ def prepare(ctx): return prepare """, }, - expected_task_names=("prepare_orders",), + expected_asset_names=("prepare_orders",), ), DiscoverFactoryValidationTestCase( - description="generated check and task may depend on a model and a seed", + description="generated check and asset may depend on a model and a seed", repo_files=base_repo_files() | { "python/factories/generated.py": """ @@ -588,7 +582,7 @@ def check_orders(ctx): return [export_countries, check_orders] """, }, - expected_task_names=("export_countries",), + expected_asset_names=("export_countries",), expected_check_names=("check_orders",), ), ), @@ -606,7 +600,6 @@ def test_given_generated_factory_nodes_when_discovering_inputs_then_validates_ex assert ( tuple(loader.name for loader in result.loader_functions) == test_case.expected_loader_names ) - assert tuple(task.name for task in result.asset_functions) == test_case.expected_task_names assert tuple(asset.name for asset in result.asset_functions) == test_case.expected_asset_names assert tuple(check.name for check in result.check_functions) == test_case.expected_check_names diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/helpers.py index 1a6f59cf26..b736af631c 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/helpers.py @@ -65,8 +65,8 @@ def build_previous_python_identity_map( "current": current_version_hash, }.get(str(previous_version_hash), previous_version_hash) identity_map: dict[tuple[str, str], Fingerprint] = { - ("task", "prepare_orders"): Fingerprint( - node_type="task", + ("asset", "prepare_orders"): Fingerprint( + node_type="asset", node_name="prepare_orders", target_database=None, target_schema="analytics", diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_identity_sharing.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_identity_sharing.py index 231b10b8a7..95f5b54fbc 100644 --- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_identity_sharing.py +++ b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_identity_sharing.py @@ -188,7 +188,7 @@ def customers_node(ctx): "shop_identity/shared/text.py:ping", "shop_identity/shared/text.py:pong", ), - "202337d3a0810b0fb7714781365660d51b6c6f4c3bfc1d0f0bc8cb597ed9baeb", + "9217c1c051274c7c831ba61985ecd6ca8c3130572aada7f0a66423b6993a144b", ), "customers_node": ( ( @@ -198,7 +198,7 @@ def customers_node(ctx): "shop_identity/shared/money.py:to_units", "shop_identity/shared/text.py:normalize_text", ), - "bf109e7d21a57481a09980e75c2e0ab3345fec5f71018da550bcfd4eb4f122e3", + "5553f9d2427656bfa7c2487baa4aee685ec29d9f40ee6051ed28a6c59b9df5c4", ), } diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_inventory.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_inventory.py index d658ce2173..920c0e8a9a 100644 --- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_inventory.py +++ b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_inventory.py @@ -238,7 +238,14 @@ def test_given_discovered_python_functions_when_building_graph_then_indexes_node "unique_key", "write_strategy", ), - "asset:prepare_orders": ("description", "group", "meta", "tags"), + "asset:prepare_orders": ( + "column_lineage", + "columns", + "description", + "group", + "meta", + "tags", + ), "asset:export_orders": ( "column_lineage", "columns", diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_selectors.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_selectors.py index 7354a289ea..1680c2d4b4 100644 --- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_selectors.py +++ b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_selectors.py @@ -156,7 +156,7 @@ def test_given_python_node_selectors_when_resolving_then_returns_expected_names( [ PythonNodeSelectorErrorTestCase( description="raises when typed selector has wrong Python node kind", - select=("asset:export_orders",), + select=("check:export_orders",), exclude=(), expected_error_type=ValueError, expected_error_fragment="unknown Python node selector 'export_orders'", diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_unified_selectors.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_unified_selectors.py index 948db7f8ab..f381604727 100644 --- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_unified_selectors.py +++ b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_unified_selectors.py @@ -499,7 +499,7 @@ def test_given_invalid_sql_model_dependency_when_validating_boundaries_then_rais exclude=(), expected_error_type=ValueError, expected_error_fragment=( - "loader 'load_events' depends on task 'prepare_orders', which reads SQL" + "loader 'load_events' depends on asset 'prepare_orders', which reads SQL" ), ) ], diff --git a/tests/unit/src/sqlbuild/executor/load/test_models.py b/tests/unit/src/sqlbuild/executor/load/test_models.py index 80174e76a2..3beac178a4 100644 --- a/tests/unit/src/sqlbuild/executor/load/test_models.py +++ b/tests/unit/src/sqlbuild/executor/load/test_models.py @@ -169,10 +169,10 @@ def fetch_events(ctx: object) -> object: "test_case", [ LoaderReferenceRejectionTestCase( - description="a task points to ctx.result_of", + description="an asset points to ctx.result_of", reference=score_orders, expected_message=( - "ctx.loader() takes a loader, not the task 'score_orders'; read its result with " + "ctx.loader() takes a loader, not the asset 'score_orders'; read its result with " "ctx.result_of(node_function=score_orders) instead" ), ), diff --git a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_contexts.py b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_contexts.py index 7c1f64ff42..f3b0a28111 100644 --- a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_contexts.py +++ b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_contexts.py @@ -2,7 +2,6 @@ from __future__ import annotations -import inspect import logging from datetime import datetime @@ -34,7 +33,6 @@ assert_base_context_fields, build_asset_context, build_task_context, - loader_only_attribute_names, skipped_upstream_task, upstream_task, ) @@ -205,59 +203,6 @@ def test_given_context_skip_when_mode_is_string_then_normalizes_to_skip_mode( assert skip_result.mode is test_case.expected_mode -@pytest.mark.parametrize( - "test_case", - [ - PythonNodeContextHelperTestCase( - description="task context result omits materialized flag and loader-only fields", - raw_name="scratch_orders", - database=None, - schema=None, - expected_qualified_name="default_db.default_schema.scratch_orders", - expected_execute_result="payload", - expected_query_result="ignored", - expected_recorded_events=(), - expected_logger_name="sqlbuild.task.fetch_orders", - expected_run_id="test_run", - expected_target="dev", - expected_vars={"batch": "hourly"}, - expected_error_fragment="materialized", - ) - ], - ids=lambda case: case.description, -) -def test_given_task_context_when_inspecting_api_then_loader_only_fields_are_absent( - test_case: PythonNodeContextHelperTestCase, -) -> None: - adapter: PythonNodeContextTestAdapter = PythonNodeContextTestAdapter() - context: AssetContext = build_task_context( - adapter=adapter, - statement_recorder=StatementRecorder(), - logger_name=test_case.expected_logger_name, - ) - - result: PythonNodeResult = context.result( - payload=test_case.expected_execute_result, - metadata={"source": "api"}, - ) - - assert result == PythonNodeResult( - payload=test_case.expected_execute_result, - metadata={"source": "api"}, - materialized=None, - ) - assert test_case.expected_error_fragment not in inspect.signature(context.result).parameters - assert context.qualify_name(name=test_case.raw_name) == test_case.expected_qualified_name - assert not hasattr(context, loader_only_attribute_names()[0]) - assert not hasattr(context, loader_only_attribute_names()[1]) - assert not hasattr(context, loader_only_attribute_names()[2]) - assert not hasattr(context, loader_only_attribute_names()[3]) - assert not hasattr(context, loader_only_attribute_names()[4]) - assert not hasattr(context, loader_only_attribute_names()[5]) - assert not hasattr(context, loader_only_attribute_names()[6]) - assert not hasattr(context, loader_only_attribute_names()[7]) - - @pytest.mark.parametrize( "test_case", [ diff --git a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_execution.py b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_execution.py index 53f4f219a1..78d68d6a30 100644 --- a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_execution.py +++ b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_execution.py @@ -18,7 +18,6 @@ ) from sqlbuild.executor.python_nodes.models import ( PythonNodeExecutionResult, - PythonNodeResult, PythonNodeRunState, PythonNodeRuntime, ) @@ -32,7 +31,6 @@ from sqlbuild.provider.classes.session import ProviderSession from sqlbuild.python_nodes.models import RetryPolicy from tests.unit.src.sqlbuild.executor.python_nodes._helpers._test_types import ( - MalformedPythonOperationTestCase, PythonNodeExecutorTestCase, PythonNodeRetryExecutorTestCase, PythonPostCallLifecycleTestCase, @@ -58,70 +56,11 @@ missing_context_provider_task, provider_asset, provider_task, - python_operation_events, skip_empty_orders, successful_sibling, ) -@pytest.mark.parametrize( - "test_case", - ( - MalformedPythonOperationTestCase( - description="task returns invalid materialized result", - operation_name="python_task", - expected_error_fragment="Only asset Python nodes may set materialized", - ), - ), - ids=lambda case: case.description, -) -def test_given_malformed_task_return_when_executing_then_operation_fails_once( - test_case: MalformedPythonOperationTestCase, -) -> None: - events: list[LifecycleEvent] = [] - invocations: list[str] = [] - dispatcher: EventDispatcher = EventDispatcher() - dispatcher.subscribe_lifecycle(subscriber=events.append, accepts_opaque=False) - - def malformed_task() -> PythonNodeResult: - invocations.append("called") - return PythonNodeResult(payload={"invalid": True}, materialized=True) - - with invocation_scope("inv-malformed-task"), dispatcher_scope(dispatcher): - result: tuple[PythonNodeExecutionResult, ...] = execute_ordered_test_nodes( - nodes=( - DiscoveredAssetFunction( - file_path=Path("/project/tasks/bad.py"), - relative_path=Path("python/tasks/bad.py"), - name="malformed_task", - function=malformed_task, - retry=RetryPolicy(max_attempts=3, retry_on=Exception, jitter=False), - ), - ), - statement_recorder=StatementRecorder(), - runtime=PythonNodeRuntime( - adapter=PythonNodeContextTestAdapter(), - connection_config={}, - connection=object(), - run_id="run-malformed", - target="dev", - vars={}, - is_reload=False, - ), - ) - - operation_events: tuple[LifecycleEvent, ...] = python_operation_events(events) - assert result[0].status == PythonNodeStatus.FAILED - assert test_case.expected_error_fragment in (result[0].error_message or "") - assert tuple(event.event_type for event in operation_events) == ( - "operation_started", - "operation_failed", - ) - assert operation_events[0].payload["operation_name"] == test_case.operation_name - assert operation_events[-1].payload["error_type"] == "ExecutorInputError" - assert invocations == ["called"] - - @pytest.mark.parametrize( "test_case", ( @@ -962,7 +901,7 @@ def record_sleep(delay: float) -> None: assert tuple(flaky_task.cost_contexts) == tuple( CostResourceContext( run_id="test_run", - resource_type="task", + resource_type="asset", resource_name="flaky_task", phase="execute", attempt=attempt, diff --git a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_ingress_execution.py b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_ingress_execution.py index bfe3299788..30813069da 100644 --- a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_ingress_execution.py +++ b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_ingress_execution.py @@ -299,7 +299,7 @@ def test_given_pre_recorded_hard_skip_when_scheduler_bypasses_downstream_then_on "hard", ) assert events[0].payload["resource_kind"] == load_resource_kind(source_map["raw_orders"]).value - assert events[2].payload["resource_kind"] == "task" + assert events[2].payload["resource_kind"] == "asset" assert python_operation_events(events) == () assert tuple(accumulator.python_results_by_name) == ( "prepare_ingress_orders", diff --git a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_lifecycle_nodes.py b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_lifecycle_nodes.py index 29ed327271..ef4aafeb8d 100644 --- a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_lifecycle_nodes.py +++ b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_lifecycle_nodes.py @@ -25,7 +25,7 @@ python_graph_case="orders", selected_names=frozenset({"prepare_orders", "load_events"}), expected_names=("load_events", "prepare_orders"), - expected_kinds=("loader", "task"), + expected_kinds=("loader", "asset"), expected_upstream_names=(("prepare_orders",), ()), expected_payload_names=("load_events", "prepare_orders"), ), diff --git a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_results.py b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_results.py index 63c5f44227..7d38269306 100644 --- a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_results.py +++ b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_results.py @@ -33,7 +33,6 @@ PythonCheckReturnNormalizationTestCase, PythonNodeFailureResultTestCase, PythonNodeFanInPolicyTestCase, - PythonNodeReturnNormalizationErrorTestCase, PythonNodeReturnNormalizationTestCase, ) from tests.unit.src.sqlbuild.executor.python_nodes._helpers.helpers import ( @@ -251,29 +250,6 @@ def test_given_python_node_return_when_normalizing_then_returns_execution_result assert result.skip_reason == test_case.expected_skip_reason -@pytest.mark.parametrize( - "test_case", - [ - PythonNodeReturnNormalizationErrorTestCase( - description="rejects materialized flag on task result", - kind=PythonNodeKind.ASSET, - returned=PythonNodeResult(materialized=False), - expected_error_fragment="Only asset Python nodes", - ) - ], - ids=lambda case: case.description, -) -def test_given_non_asset_materialized_result_when_normalizing_then_raises( - test_case: PythonNodeReturnNormalizationErrorTestCase, -) -> None: - with pytest.raises(ValueError, match=test_case.expected_error_fragment): - normalize_python_node_return( - node_name="fetch_orders", - kind=test_case.kind, - returned=test_case.returned, - ) - - @pytest.mark.parametrize( "test_case", [ diff --git a/tests/unit/src/sqlbuild/integrations/dagster/helpers.py b/tests/unit/src/sqlbuild/integrations/dagster/helpers.py index 3769f59567..c2ddd50488 100644 --- a/tests/unit/src/sqlbuild/integrations/dagster/helpers.py +++ b/tests/unit/src/sqlbuild/integrations/dagster/helpers.py @@ -226,8 +226,8 @@ def build_python_augmented_dagster_test_dag() -> Mapping[str, Any]: "id": "asset:prepare_orders", "kind": "asset", "name": "prepare_orders", - "asset_key": ["task", "prepare_orders"], - "path": "python/tasks/prepare_orders.py", + "asset_key": ["asset", "prepare_orders"], + "path": "python/assets/prepare_orders.py", "tags": ["daily"], "group": "python", "meta": {"owner": "data"}, diff --git a/tests/unit/src/sqlbuild/integrations/dagster/test_assets.py b/tests/unit/src/sqlbuild/integrations/dagster/test_assets.py index 31739b05c8..fcf4f2f94f 100644 --- a/tests/unit/src/sqlbuild/integrations/dagster/test_assets.py +++ b/tests/unit/src/sqlbuild/integrations/dagster/test_assets.py @@ -416,7 +416,7 @@ def test_given_sqlbuild_dag_when_building_specs_then_maps_assets_deps_and_checks ("analytics", "normalize_email"), ("analytics", "orders"), ("analytics", "customers"), - ("task", "prepare_orders"), + ("asset", "prepare_orders"), ("asset", "orders_export"), ), expected_check_names=( @@ -427,9 +427,9 @@ def test_given_sqlbuild_dag_when_building_specs_then_maps_assets_deps_and_checks "python_check__check_orders_export", ), expected_task_deps=(("analytics", "orders"),), - expected_asset_deps=(("task", "prepare_orders"),), + expected_asset_deps=(("asset", "prepare_orders"),), expected_python_kinds_by_asset_key=( - (("task", "prepare_orders"), frozenset({"sqlbuild", "task"})), + (("asset", "prepare_orders"), frozenset({"sqlbuild", "asset"})), (("asset", "orders_export"), frozenset({"sqlbuild", "asset"})), ), expected_task_group="python", @@ -455,7 +455,7 @@ def test_given_python_augmented_dag_when_building_specs_then_maps_python_nodes( tuple(spec.key.path): (spec,) for spec in asset_specs } assert len(specs_by_key) == len(asset_specs) - task_spec: Any = next(iter(specs_by_key.get(("task", "prepare_orders"), ()))) + task_spec: Any = next(iter(specs_by_key.get(("asset", "prepare_orders"), ()))) python_asset_spec: Any = next(iter(specs_by_key.get(("asset", "orders_export"), ()))) assert tuple(tuple(spec.key.path) for spec in asset_specs) == test_case.expected_asset_keys diff --git a/tests/unit/src/sqlbuild/integrations/rivers/test_assets.py b/tests/unit/src/sqlbuild/integrations/rivers/test_assets.py index 755d55f294..3cf606cd58 100644 --- a/tests/unit/src/sqlbuild/integrations/rivers/test_assets.py +++ b/tests/unit/src/sqlbuild/integrations/rivers/test_assets.py @@ -36,13 +36,13 @@ "analytics__normalize_email", "analytics__orders", "analytics__customers", - "task__prepare_orders", + "asset__prepare_orders", "asset__orders_export", ), expected_order_deps=("raw__orders", "analytics__normalize_email"), expected_task_deps=("analytics__orders",), - expected_asset_deps=("task__prepare_orders",), - expected_task_kinds=["sqlbuild", "task"], + expected_asset_deps=("asset__prepare_orders",), + expected_task_kinds=["sqlbuild", "asset"], expected_asset_kinds=["sqlbuild", "asset"], expected_task_group="python", expected_asset_group="exports", @@ -69,7 +69,7 @@ def test_given_python_augmented_dag_when_building_asset_defs_then_maps_python_no asset_defs_by_name: dict[str, tuple[Any, ...]] = {asset.name: (asset,) for asset in asset_defs} assert len(asset_defs_by_name) == len(asset_defs) order_def: Any = next(iter(asset_defs_by_name.get("analytics__orders", ()))) - task_def: Any = next(iter(asset_defs_by_name.get("task__prepare_orders", ()))) + task_def: Any = next(iter(asset_defs_by_name.get("asset__prepare_orders", ()))) python_asset_def: Any = next(iter(asset_defs_by_name.get("asset__orders_export", ()))) assert tuple(asset.name for asset in asset_defs) == test_case.expected_asset_names diff --git a/tests/unit/src/sqlbuild/python_nodes/decorators/test_python_decorators.py b/tests/unit/src/sqlbuild/python_nodes/decorators/test_python_decorators.py index fe1bc60eb5..c569195d3a 100644 --- a/tests/unit/src/sqlbuild/python_nodes/decorators/test_python_decorators.py +++ b/tests/unit/src/sqlbuild/python_nodes/decorators/test_python_decorators.py @@ -100,7 +100,6 @@ def export_customers(_ctx: object) -> dict[str, object]: test_case.expected_column_descriptions ) assert definition.column_lineage == test_case.expected_column_lineage - assert get_asset_definition(export_customers) is None @pytest.mark.parametrize( From 99e1230aa8c6baa6ac3810845a362bd3d1c36a55 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 19 Nov 2026 05:09:03 +0000 Subject: [PATCH 04/12] wip: finish moving tests, docs and goldens off the task kind Rust and Python expectations now name assets, selection orders follow the asset sort order, the explicit-reference golden is re-recorded (each changed digest reproduces the old one with the task wording restored), the docs drop tasks, and the docs skill references are regenerated. Validation so far: cargo test of the touched crates passes apart from a launcher SIGINT test that fails in this sandbox only; make test has 4 remaining failures from a stale local sqlbuild version (env-only); fensu check clean. --- .../python_execution_inputs.json | 2 +- clients/typescript/src/schema.d.ts | 4 +- .../src/dag_artifact/_helpers/nodes.rs | 2 +- .../dag_artifact/tests/test_build_dag_json.rs | 6 +- .../src/graph/tests/test_graph.rs | 2 +- .../graph/tests/test_named_selector_atoms.rs | 2 +- .../graph/tests/test_selector_scenarios.rs | 9 +- .../node_set_dependencies/_helpers/labels.rs | 2 +- .../node_set_dependencies/_helpers/sets.rs | 3 +- .../src/node_set_dependencies/models.rs | 2 +- .../test_resolve_node_set_dependencies.rs | 10 +- .../node_set_dependencies/tests/test_types.rs | 2 +- .../src/python_plan/models.rs | 2 +- .../python_plan/tests/test_run_lifecycle.rs | 2 +- .../src/python_selection/models.rs | 4 +- .../tests/test_python_selection.rs | 6 +- .../tests/test_round_trips.rs | 34 ++-- .../src/review_command/models.rs | 2 +- .../src/review_command/tests/helpers.rs | 2 +- .../review_command/tests/test_plan_review.rs | 18 +-- .../src/adhoc/tests/test_adhoc_planner.rs | 8 +- .../adhoc/tests/test_selection_evaluation.rs | 9 +- .../build_command/tests/test_build_command.rs | 4 +- .../tests/test_build_progress.rs | 2 +- .../src/declarations/models.rs | 2 +- .../sqlbuild-orchestration/src/runs/models.rs | 2 +- .../python_files/imported_python_nodes.rs | 28 ++-- fensu.toml | 2 +- python-boundary.toml | 7 +- .../sqlbuild/references/docs/CONTENTS.md | 9 +- .../sqlbuild/references/docs/cli/build.md | 4 +- .../sqlbuild/references/docs/cli/check.md | 8 +- .../sqlbuild/references/docs/cli/plan.md | 2 +- .../references/docs/cli/playground.md | 4 +- .../sqlbuild/references/docs/cli/review.md | 2 +- .../sqlbuild/references/docs/cli/test.md | 2 +- .../docs/concepts/models/configuration.md | 2 +- .../references/docs/concepts/models/hooks.md | 2 +- .../docs/concepts/models/hooks/python.md | 2 +- .../references/docs/concepts/planning.md | 2 +- .../docs/concepts/project-configuration.md | 8 +- .../docs/concepts/python-nodes/assets.md | 147 +++++++++++------ .../docs/concepts/python-nodes/checks.md | 16 +- .../docs/concepts/python-nodes/factories.md | 4 +- .../docs/concepts/python-nodes/loaders.md | 28 ++-- .../docs/concepts/python-nodes/overview.md | 97 ++++++----- .../docs/concepts/python-nodes/providers.md | 17 +- .../concepts/python-nodes/sql-references.md | 34 ++-- .../docs/concepts/python-nodes/tasks.md | 150 ------------------ .../concepts/rules/orchestration-rules.md | 2 +- .../docs/concepts/rules/python-rules.md | 14 +- .../references/docs/concepts/selectors.md | 4 +- .../references/docs/feature-comparison.md | 5 +- .../docs/integrations/dagster-reference.md | 1 - .../references/docs/integrations/dagster.md | 8 +- .../references/docs/integrations/rivers.md | 6 +- .../compile/test_python_project_layout.py | 44 ++--- .../explicit_references_edge_projects.json | 16 +- .../src/sqlbuild/cli/commands/main/helpers.py | 6 +- .../test_provider_annotation_tolerance.py | 6 +- .../test_orchestration_snapshot.py | 18 ++- .../pipeline/test_set_dependencies.py | 10 +- website/src/content/docs/docs/cli/build.mdx | 4 +- website/src/content/docs/docs/cli/check.mdx | 8 +- website/src/content/docs/docs/cli/plan.mdx | 2 +- .../src/content/docs/docs/cli/playground.mdx | 4 +- website/src/content/docs/docs/cli/review.mdx | 2 +- website/src/content/docs/docs/cli/test.mdx | 2 +- .../docs/concepts/models/configuration.mdx | 2 +- .../docs/docs/concepts/models/hooks.mdx | 2 +- .../docs/concepts/models/hooks/python.mdx | 2 +- .../content/docs/docs/concepts/planning.mdx | 2 +- .../docs/concepts/project-configuration.mdx | 8 +- .../docs/concepts/python-nodes/assets.mdx | 142 ++++++++++++----- .../docs/concepts/python-nodes/checks.mdx | 16 +- .../docs/concepts/python-nodes/factories.mdx | 4 +- .../docs/concepts/python-nodes/loaders.mdx | 28 ++-- .../docs/concepts/python-nodes/overview.mdx | 97 ++++++----- .../docs/concepts/python-nodes/providers.mdx | 17 +- .../concepts/python-nodes/sql-references.mdx | 34 ++-- .../docs/docs/concepts/python-nodes/tasks.mdx | 141 ---------------- .../concepts/rules/orchestration-rules.mdx | 2 +- .../docs/docs/concepts/rules/python-rules.mdx | 14 +- .../content/docs/docs/concepts/selectors.mdx | 4 +- .../content/docs/docs/feature-comparison.mdx | 5 +- .../docs/integrations/dagster-reference.mdx | 1 - .../docs/docs/integrations/dagster.mdx | 8 +- .../content/docs/docs/integrations/rivers.mdx | 6 +- website/src/sidebar.json | 1 - 89 files changed, 613 insertions(+), 805 deletions(-) delete mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/tasks.md delete mode 100644 website/src/content/docs/docs/concepts/python-nodes/tasks.mdx diff --git a/benchmarks/duckshop/edge_cases/orchestration/python_execution_inputs.json b/benchmarks/duckshop/edge_cases/orchestration/python_execution_inputs.json index 6fc11f3d0f..43bcac07e8 100644 --- a/benchmarks/duckshop/edge_cases/orchestration/python_execution_inputs.json +++ b/benchmarks/duckshop/edge_cases/orchestration/python_execution_inputs.json @@ -1,6 +1,6 @@ { "initial": "from sqlbuild.checks import check\nfrom sqlbuild.refs import model\n@check(depends_on=model('orders'), description='Orders are present.')\ndef orders_present(ctx):\n return ctx.pass_()\n", "changed": "from sqlbuild.checks import check\nfrom sqlbuild.refs import model\n@check(depends_on=model('orders'), description='Orders are present.')\ndef orders_present(ctx):\n raise RuntimeError('changed check executed after publication')\n", - "task": "from sqlbuild.assets import asset\n@asset(description='Notify support.')\ndef notify_support(ctx):\n return None\n", + "side_effect": "from sqlbuild.assets import asset\n@asset(description='Notify support.')\ndef notify_support(ctx):\n return None\n", "asset": "from sqlbuild.assets import asset\n@asset(description='Order export.')\ndef order_export(ctx):\n return []\n" } diff --git a/clients/typescript/src/schema.d.ts b/clients/typescript/src/schema.d.ts index 63288c6dbc..ec0651c55c 100644 --- a/clients/typescript/src/schema.d.ts +++ b/clients/typescript/src/schema.d.ts @@ -3006,7 +3006,7 @@ export interface components { state_database?: string | null; state_schema: string; }; - /** @description Code that injects a provider: a node (`loader`, `task`, `asset`, `check`, `hook` or `materialization`, keyed `kind:name`), an `event_exporter` or a `sink`. */ + /** @description Code that injects a provider: a node (`loader`, `asset`, `check`, `hook` or `materialization`, keyed `kind:name`), an `event_exporter` or a `sink`. */ ProviderConsumer: { kind: string; name: string; @@ -3700,7 +3700,7 @@ export interface components { index: number; /** @description The part as typed, such as `+tag:finanse`. */ part: string; - /** @description `tag`, `path`, `job`, `source`, `seed`, `kind`, `test`, `name`, `glob`, or a Python node type such as `task`. */ + /** @description `tag`, `path`, `job`, `source`, `seed`, `kind`, `test`, `name`, `glob`, or a Python node type such as `asset`. */ type: string; /** @description The value the part names, without its type prefix or `+` markers, such as `finanse`. */ value: string; diff --git a/crates/sqlbuild-analysis/src/dag_artifact/_helpers/nodes.rs b/crates/sqlbuild-analysis/src/dag_artifact/_helpers/nodes.rs index fbbd296122..799acbc538 100644 --- a/crates/sqlbuild-analysis/src/dag_artifact/_helpers/nodes.rs +++ b/crates/sqlbuild-analysis/src/dag_artifact/_helpers/nodes.rs @@ -394,7 +394,7 @@ pub(crate) fn model_node(model: &DagModelInput) -> DagValue { .value() } -/// The node of a Python task, asset or check. +/// The node of a Python asset or check. pub(crate) fn python_node(node: &DagPythonNodeInput) -> DagValue { let mut fields: NodeFields = NodeFields { id: python_node_id(&node.kind, &node.name), diff --git a/crates/sqlbuild-analysis/src/dag_artifact/tests/test_build_dag_json.rs b/crates/sqlbuild-analysis/src/dag_artifact/tests/test_build_dag_json.rs index 038e568ff4..beda363adc 100644 --- a/crates/sqlbuild-analysis/src/dag_artifact/tests/test_build_dag_json.rs +++ b/crates/sqlbuild-analysis/src/dag_artifact/tests/test_build_dag_json.rs @@ -176,9 +176,9 @@ fn given_project_when_building_dag_json_then_python_artifact_agrees() { } #[test] -fn given_a_loader_depending_on_a_task_when_building_dag_json_then_the_edge_reaches_its_source() { +fn given_a_loader_depending_on_an_asset_when_building_dag_json_then_the_edge_reaches_its_source() { let test_cases = [DagEdgeTestCase { - description: "a task upstream of a declared loader points at the loader's source", + description: "an asset upstream of a declared loader points at the loader's source", project: DagProjectInput { sources: vec![source( "partner_scores", @@ -200,8 +200,8 @@ fn given_a_loader_depending_on_a_task_when_building_dag_json_then_the_edge_reach ], }, expected_edges: &[ - ("model:orders", "asset:score_orders"), ("asset:score_orders", "source:partner_scores"), + ("model:orders", "asset:score_orders"), ], }]; for test_case in test_cases { diff --git a/crates/sqlbuild-analysis/src/graph/tests/test_graph.rs b/crates/sqlbuild-analysis/src/graph/tests/test_graph.rs index ae9b836ace..d0c68e6470 100644 --- a/crates/sqlbuild-analysis/src/graph/tests/test_graph.rs +++ b/crates/sqlbuild-analysis/src/graph/tests/test_graph.rs @@ -277,7 +277,7 @@ fn given_invalid_selectors_when_resolving_then_errors_match_python() { exclude: &[], expected_error: ( "S010", - "selector type 'task' does not map to a resource type yet", + "selector type 'asset' does not map to a resource type yet", None, ), }, diff --git a/crates/sqlbuild-analysis/src/graph/tests/test_named_selector_atoms.rs b/crates/sqlbuild-analysis/src/graph/tests/test_named_selector_atoms.rs index 8210a92870..82343c112e 100644 --- a/crates/sqlbuild-analysis/src/graph/tests/test_named_selector_atoms.rs +++ b/crates/sqlbuild-analysis/src/graph/tests/test_named_selector_atoms.rs @@ -31,7 +31,7 @@ fn given_a_selection_when_listing_named_atoms_then_only_single_node_names_count( }, NamedAtomsTestCase { description: "kinds do not count", - selection: "kind:model kind:task", + selection: "kind:model kind:asset", expected_atoms: &[], }, NamedAtomsTestCase { diff --git a/crates/sqlbuild-analysis/src/graph/tests/test_selector_scenarios.rs b/crates/sqlbuild-analysis/src/graph/tests/test_selector_scenarios.rs index 443fae9f60..330b330bd3 100644 --- a/crates/sqlbuild-analysis/src/graph/tests/test_selector_scenarios.rs +++ b/crates/sqlbuild-analysis/src/graph/tests/test_selector_scenarios.rs @@ -56,11 +56,6 @@ fn given_raw_selector_when_parsing_then_returns_kind_value_and_expansion() { raw: "+seed:country_codes", expected_selector: kind("seed", "country_codes", true, false), }, - ParseSelectorTestCase { - description: "task typed selector", - raw: "asset:prepare_orders", - expected_selector: kind("asset", "prepare_orders", false, false), - }, ParseSelectorTestCase { description: "asset typed selector", raw: "asset:export_orders", @@ -553,10 +548,10 @@ fn given_unresolvable_selectors_when_resolving_then_fails_with_message() { ProjectSelectorErrorTestCase { description: "a Python node kind needs the Python node graph", project: DIAMOND_PROJECT, - select: &["kind:task"], + select: &["kind:asset"], expected_error: ( "S010", - "selector 'kind:task' selects Python nodes; they cannot be selected here", + "selector 'kind:asset' selects Python nodes; they cannot be selected here", None, ), }, diff --git a/crates/sqlbuild-analysis/src/node_set_dependencies/_helpers/labels.rs b/crates/sqlbuild-analysis/src/node_set_dependencies/_helpers/labels.rs index d5614d3884..f8a3372b65 100644 --- a/crates/sqlbuild-analysis/src/node_set_dependencies/_helpers/labels.rs +++ b/crates/sqlbuild-analysis/src/node_set_dependencies/_helpers/labels.rs @@ -5,7 +5,7 @@ use std::collections::HashMap; use crate::node_set_dependencies::constants::NAMED_NODES; use crate::node_set_dependencies::models::{GraphNode, NodeSetDependency}; -/// `model 'orders'` or `task 'export_orders'`. +/// `model 'orders'` or `asset 'export_orders'`. pub(crate) fn node_label(node: &GraphNode, kinds: &HashMap) -> String { match node { GraphNode::Sql((kind, name)) => format!("{kind} '{name}'"), diff --git a/crates/sqlbuild-analysis/src/node_set_dependencies/_helpers/sets.rs b/crates/sqlbuild-analysis/src/node_set_dependencies/_helpers/sets.rs index ec3b2317b7..bac31025b7 100644 --- a/crates/sqlbuild-analysis/src/node_set_dependencies/_helpers/sets.rs +++ b/crates/sqlbuild-analysis/src/node_set_dependencies/_helpers/sets.rs @@ -119,8 +119,7 @@ pub(crate) fn members( }; return Err(failure( format!( - "{owner} depends on {set}, which leaves no model, seed, source, loader, task or \ - asset{after}" + "{owner} depends on {set}, which leaves no model, seed, source, loader or asset{after}" ), None, )); diff --git a/crates/sqlbuild-analysis/src/node_set_dependencies/models.rs b/crates/sqlbuild-analysis/src/node_set_dependencies/models.rs index e8b824ace1..00c60a0f7f 100644 --- a/crates/sqlbuild-analysis/src/node_set_dependencies/models.rs +++ b/crates/sqlbuild-analysis/src/node_set_dependencies/models.rs @@ -16,7 +16,7 @@ pub struct NodeSetDependency { #[derive(Clone, Debug, PartialEq, Eq)] pub struct DependantNode { pub name: String, - /// The node's kind, such as `task`. + /// The node's kind, such as `asset`. pub kind: String, pub sets: Vec, } diff --git a/crates/sqlbuild-analysis/src/node_set_dependencies/tests/test_resolve_node_set_dependencies.rs b/crates/sqlbuild-analysis/src/node_set_dependencies/tests/test_resolve_node_set_dependencies.rs index 49153ced62..fea84dc125 100644 --- a/crates/sqlbuild-analysis/src/node_set_dependencies/tests/test_resolve_node_set_dependencies.rs +++ b/crates/sqlbuild-analysis/src/node_set_dependencies/tests/test_resolve_node_set_dependencies.rs @@ -29,7 +29,7 @@ fn given_job_and_tag_dependencies_when_resolving_then_each_reduces_to_its_end_no description: "a Python node reading the job's model is its end", groups: &[], sets: &[("job", "with_task", None)], - expected_ends: &[&["task 'export_orders'"]], + expected_ends: &[&["asset 'export_orders'"]], }, EndNodesTestCase { description: "exclude removes nodes before reducing to end nodes", @@ -89,7 +89,7 @@ fn given_unresolvable_dependencies_when_resolving_then_compile_fails_naming_the_ groups: &[], sets: &[("job", "weekly", None)], expected_fragments: &[ - "task 'report' depends on job 'weekly', which the project does not declare", + "asset 'report' depends on job 'weekly', which the project does not declare", ], }, FailureTestCase { @@ -98,7 +98,7 @@ fn given_unresolvable_dependencies_when_resolving_then_compile_fails_naming_the_ groups: &[], sets: &[("tag", "marketing", None)], expected_fragments: &[ - "task 'report' depends on tag 'marketing', which no node carries", + "asset 'report' depends on tag 'marketing', which no node carries", ], }, FailureTestCase { @@ -133,7 +133,7 @@ fn given_unresolvable_dependencies_when_resolving_then_compile_fails_naming_the_ groups: &[], sets: &[("job", "loop", None)], expected_fragments: &[ - "task 'report' depends on job 'loop', but job 'loop''s task 'audit_export' reads task 'report''s output", + "asset 'report' depends on job 'loop', but job 'loop''s asset 'audit_export' reads asset 'report''s output", "job(\"loop\", exclude=\"audit_export\")", ], }, @@ -143,7 +143,7 @@ fn given_unresolvable_dependencies_when_resolving_then_compile_fails_naming_the_ groups: &[], sets: &[("tag", "finance", None)], expected_fragments: &[ - "task 'self_tagged' depends on tag 'finance', which includes task 'self_tagged' itself", + "asset 'self_tagged' depends on tag 'finance', which includes asset 'self_tagged' itself", "tag(\"finance\", exclude=\"self_tagged\")", ], }, diff --git a/crates/sqlbuild-analysis/src/node_set_dependencies/tests/test_types.rs b/crates/sqlbuild-analysis/src/node_set_dependencies/tests/test_types.rs index 8bc4c18a1c..93ccdfc446 100644 --- a/crates/sqlbuild-analysis/src/node_set_dependencies/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/node_set_dependencies/tests/test_types.rs @@ -1,4 +1,4 @@ -/// One set dependency of the `report` task and the end nodes it resolves to. +/// One set dependency of the `report` asset and the end nodes it resolves to. pub(super) struct EndNodesTestCase { pub(super) description: &'static str, /// `(kind, name, exclude)` of each `job(...)` or `tag(...)` dependency. diff --git a/crates/sqlbuild-analysis/src/python_plan/models.rs b/crates/sqlbuild-analysis/src/python_plan/models.rs index eaa2de12be..e4b35691eb 100644 --- a/crates/sqlbuild-analysis/src/python_plan/models.rs +++ b/crates/sqlbuild-analysis/src/python_plan/models.rs @@ -5,7 +5,7 @@ use crate::assembly::project::types::ObjectKey; /// One Python node as lifecycle classification reads it. #[derive(Clone, Debug, PartialEq, Eq)] pub struct LifecycleNode { - /// `task`, `asset`, `loader` or `check`. + /// `asset`, `loader` or `check`. pub kind: String, /// The SQL resources the node reads, in authored order. pub sql_deps: Vec, diff --git a/crates/sqlbuild-analysis/src/python_plan/tests/test_run_lifecycle.rs b/crates/sqlbuild-analysis/src/python_plan/tests/test_run_lifecycle.rs index 12dc8006d4..1e99ace95a 100644 --- a/crates/sqlbuild-analysis/src/python_plan/tests/test_run_lifecycle.rs +++ b/crates/sqlbuild-analysis/src/python_plan/tests/test_run_lifecycle.rs @@ -9,7 +9,7 @@ use crate::python_plan::tests::test_types::LifecycleTestCase; fn given_selection_when_classifying_lifecycle_then_matches_python_regions() { let test_cases = [ LifecycleTestCase { - description: "a loader pulls its selected upstream tasks into ingress", + description: "a loader pulls its selected upstream assets into ingress", selected: &["fetch_rates", "load_orders", "report", "check_orders"], nodes: &[ ("fetch_rates", "asset", &[]), diff --git a/crates/sqlbuild-analysis/src/python_selection/models.rs b/crates/sqlbuild-analysis/src/python_selection/models.rs index f4c43a3956..7826e4a2c5 100644 --- a/crates/sqlbuild-analysis/src/python_selection/models.rs +++ b/crates/sqlbuild-analysis/src/python_selection/models.rs @@ -9,7 +9,7 @@ use crate::graph::models::ProjectGraph; #[derive(Clone, Debug, PartialEq, Eq)] pub struct PythonSelectionNode { pub name: String, - /// `task`, `asset`, `loader` or `check`. + /// `asset`, `loader` or `check`. pub kind: String, /// The SQL resource keys the node declares, in authored order. pub sql_deps: Vec, @@ -58,7 +58,7 @@ pub struct InventoryNode { #[derive(Clone, Debug, PartialEq, Eq)] pub struct PublishedPythonNode { pub name: String, - /// `task`, `asset`, `loader` or `check`. + /// `asset`, `loader` or `check`. pub kind: String, pub tags: Vec, /// The folder of the node's file relative to the project, in POSIX form. diff --git a/crates/sqlbuild-analysis/src/python_selection/tests/test_python_selection.rs b/crates/sqlbuild-analysis/src/python_selection/tests/test_python_selection.rs index 5bba21b8b6..26e26be759 100644 --- a/crates/sqlbuild-analysis/src/python_selection/tests/test_python_selection.rs +++ b/crates/sqlbuild-analysis/src/python_selection/tests/test_python_selection.rs @@ -191,9 +191,9 @@ fn given_run_selectors_when_resolving_then_selection_matches_python() { }, RunSelectionTestCase { description: "kind selects every Python node of one kind", - select: &["kind:task"], + select: &["kind:asset"], exclude: &[], - expected_selection: Ok((&[], &["prepare_orders"])), + expected_selection: Ok((&[], &["export_orders", "prepare_orders"])), }, RunSelectionTestCase { description: "+ expands only the comma part it marks: upstream of a node, then only its sources", @@ -211,7 +211,7 @@ fn given_run_selectors_when_resolving_then_selection_matches_python() { description: "kind excludes like tag", select: &["tag:daily"], exclude: &["kind:asset"], - expected_selection: Ok((&[("model", "orders")], &["prepare_orders"])), + expected_selection: Ok((&[("model", "orders")], &[])), }, RunSelectionTestCase { description: "an unknown kind is refused", diff --git a/crates/sqlbuild-analysis/src/python_selection/tests/test_round_trips.rs b/crates/sqlbuild-analysis/src/python_selection/tests/test_round_trips.rs index 6d86dad301..78111029e3 100644 --- a/crates/sqlbuild-analysis/src/python_selection/tests/test_round_trips.rs +++ b/crates/sqlbuild-analysis/src/python_selection/tests/test_round_trips.rs @@ -6,7 +6,7 @@ use crate::python_selection::tests::test_types::{ }; const DIRECT_CYCLE: &str = "loader 'partner_scores' is in a dependency cycle: loader 'partner_scores' -> \ - task 'score_orders' -> model 'scored_orders' -> source 'partner_scores' -> \ + asset 'score_orders' -> model 'scored_orders' -> source 'partner_scores' -> \ loader 'partner_scores'"; const SCORE_ORDERS: RoundTripNodeSpec = ("score_orders", "asset", &[], &[("model", "orders")]); const PARTNER_SCORES: RoundTripNodeSpec = ("partner_scores", "loader", &["score_orders"], &[]); @@ -19,16 +19,16 @@ const CYCLE_HELP: &str = "each arrow points to what a node depends on; an asset fn given_a_loader_depending_on_python_that_reads_sql_when_selecting_then_the_setting_decides() { let test_cases = [ RoundTripTestCase { - description: "a direct SQL-reading task is refused by default", + description: "a direct SQL-reading asset is refused by default", nodes: &[SCORE_ORDERS, PARTNER_SCORES], allow_warehouse_round_trips: false, select: &[], expected_selection: Err(( "S000", - "loader 'partner_scores' depends on task 'score_orders', which reads SQL\n \ + "loader 'partner_scores' depends on asset 'score_orders', which reads SQL\n \ This is a warehouse round trip: data leaves the warehouse and comes back through \ this loader.\n Lineage and column lineage restart at the source \ - 'partner_scores', and builds and job\n groups run the task before the loader.", + 'partner_scores', and builds and job\n groups run the asset before the loader.", HELP, )), }, @@ -43,7 +43,7 @@ fn given_a_loader_depending_on_python_that_reads_sql_when_selecting_then_the_set select: &[], expected_selection: Err(( "S000", - "loader 'partner_scores' depends on asset 'score_orders' (through task \ + "loader 'partner_scores' depends on asset 'score_orders' (through asset \ 'package_scores'), which reads SQL\n This is a warehouse round trip: data \ leaves the warehouse and comes back through this loader.\n Lineage and column \ lineage restart at the source 'partner_scores', and builds and job\n groups run \ @@ -66,7 +66,7 @@ fn given_a_loader_depending_on_python_that_reads_sql_when_selecting_then_the_set )), }, RoundTripTestCase { - description: "a loader depending on a task that reads no SQL is always allowed", + description: "a loader depending on an asset that reads no SQL is always allowed", nodes: &[ ("fetch_scores", "asset", &[], &[]), ("partner_scores", "loader", &["fetch_scores"], &[]), @@ -83,7 +83,7 @@ fn given_a_loader_depending_on_python_that_reads_sql_when_selecting_then_the_set )), }, RoundTripTestCase { - description: "a task reading what its downstream loader's source builds is a cycle", + description: "an asset reading what its downstream loader's source builds is a cycle", nodes: &[ ("score_orders", "asset", &[], &[("model", "scored_orders")]), PARTNER_SCORES, @@ -93,7 +93,7 @@ fn given_a_loader_depending_on_python_that_reads_sql_when_selecting_then_the_set expected_selection: Err(( "S000", "loader 'partner_scores' is in a dependency cycle: loader 'partner_scores' -> \ - task 'score_orders' -> model 'scored_orders' -> source 'partner_scores' -> \ + asset 'score_orders' -> model 'scored_orders' -> source 'partner_scores' -> \ loader 'partner_scores'", CYCLE_HELP, )), @@ -109,7 +109,7 @@ fn given_a_loader_depending_on_python_that_reads_sql_when_selecting_then_the_set expected_selection: Err(( "S000", "loader 'partner_scores' is in a dependency cycle: loader 'partner_scores' -> \ - task 'score_orders' -> model 'scored_orders' -> source 'partner_scores' -> \ + asset 'score_orders' -> model 'scored_orders' -> source 'partner_scores' -> \ loader 'partner_scores'", CYCLE_HELP, )), @@ -154,7 +154,7 @@ fn given_whole_graph_boundaries_when_checked_without_a_selection_then_each_refus expected_error: None, }, CycleTestCase { - description: "a task's depends_on on another task closes the cycle", + description: "an asset's depends_on on another asset closes the cycle", nodes: &[ ("score_orders", "asset", &[], &[("model", "scored_orders")]), ("package_scores", "asset", &["score_orders"], &[]), @@ -164,7 +164,7 @@ fn given_whole_graph_boundaries_when_checked_without_a_selection_then_each_refus allow_warehouse_round_trips: true, expected_error: Some( "loader 'partner_scores' is in a dependency cycle: loader 'partner_scores' -> \ - task 'package_scores' -> task 'score_orders' -> model 'scored_orders' -> \ + asset 'package_scores' -> asset 'score_orders' -> model 'scored_orders' -> \ source 'partner_scores' -> loader 'partner_scores'", ), }, @@ -179,12 +179,12 @@ fn given_whole_graph_boundaries_when_checked_without_a_selection_then_each_refus allow_warehouse_round_trips: true, expected_error: Some( "loader 'partner_scores' is in a dependency cycle: loader 'partner_scores' -> \ - asset 'scores_file' -> task 'score_orders' -> model 'scored_orders' -> \ + asset 'scores_file' -> asset 'score_orders' -> model 'scored_orders' -> \ source 'partner_scores' -> loader 'partner_scores'", ), }, CycleTestCase { - description: "without the task's depends_on there is no cycle", + description: "without the asset's depends_on there is no cycle", nodes: &[ ("score_orders", "asset", &[], &[("model", "scored_orders")]), ("package_scores", "asset", &[], &[]), @@ -215,19 +215,19 @@ fn given_whole_graph_boundaries_when_checked_without_a_selection_then_each_refus expected_error: None, }, CycleTestCase { - description: "a loader downstream of a task that reads SQL, without the setting, is a round trip", + description: "a loader downstream of an asset that reads SQL, without the setting, is a round trip", nodes: &[SCORE_ORDERS, PARTNER_SCORES], filled: true, allow_warehouse_round_trips: false, expected_error: Some( - "loader 'partner_scores' depends on task 'score_orders', which reads SQL\n \ + "loader 'partner_scores' depends on asset 'score_orders', which reads SQL\n \ This is a warehouse round trip: data leaves the warehouse and comes back through \ this loader.\n Lineage and column lineage restart at the source \ - 'partner_scores', and builds and job\n groups run the task before the loader.", + 'partner_scores', and builds and job\n groups run the asset before the loader.", ), }, CycleTestCase { - description: "a task depending on the terminal loader instead of its source", + description: "an asset depending on the terminal loader instead of its source", nodes: &[ ("fetch_scores", "asset", &[], &[]), ("partner_scores", "loader", &["fetch_scores"], &[]), diff --git a/crates/sqlbuild-analysis/src/review_command/models.rs b/crates/sqlbuild-analysis/src/review_command/models.rs index 07af07a94b..70d0dfa1fc 100644 --- a/crates/sqlbuild-analysis/src/review_command/models.rs +++ b/crates/sqlbuild-analysis/src/review_command/models.rs @@ -18,7 +18,7 @@ pub enum DownstreamPolicy { #[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct ReviewNode { pub name: String, - /// `model` or `seed`, or the Python node kind: `loader`, `task`, `asset` or `check`. + /// `model` or `seed`, or the Python node kind: `loader`, `asset` or `check`. pub kind: String, pub critical: bool, pub materialized: String, diff --git a/crates/sqlbuild-analysis/src/review_command/tests/helpers.rs b/crates/sqlbuild-analysis/src/review_command/tests/helpers.rs index d5d2b4320b..a54e6d8330 100644 --- a/crates/sqlbuild-analysis/src/review_command/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/review_command/tests/helpers.rs @@ -34,7 +34,7 @@ fn python_node(name: &str, kind: &str) -> (String, ReviewNode) { ) } -/// A waffle shop with a critical `daily_revenue`, a loader-fed `stg_toppings` and a Python task. +/// A waffle shop with a critical `daily_revenue`, a loader-fed `stg_toppings` and a Python asset. pub(super) fn waffle_shop() -> ReviewGraph { let edges: [(&str, &str); 8] = [ ("waffle_types", "stg_orders"), diff --git a/crates/sqlbuild-analysis/src/review_command/tests/test_plan_review.rs b/crates/sqlbuild-analysis/src/review_command/tests/test_plan_review.rs index 61d5f7231c..537b18162b 100644 --- a/crates/sqlbuild-analysis/src/review_command/tests/test_plan_review.rs +++ b/crates/sqlbuild-analysis/src/review_command/tests/test_plan_review.rs @@ -10,7 +10,7 @@ use crate::review_command::tests::test_types::{ SchemaNameTestCase, }; -const EXPORT_TASK: (&str, &str) = ("asset:export_revenue", "Python task, not run by review"); +const EXPORT_ASSET: (&str, &str) = ("asset:export_revenue", "Python asset, not run by review"); const ARCHIVE_LEFT_OUT: (&str, &str) = ( "order_archive", "not on a path to a critical model (--downstream critical)", @@ -26,7 +26,7 @@ fn given_changed_models_when_planning_then_validated_set_follows_policy_and_sele policy: DownstreamPolicy::Critical, select: None, expected_validated: vec!["stg_orders", "order_totals", "daily_revenue"], - expected_not_validated: vec![ARCHIVE_LEFT_OUT, EXPORT_TASK], + expected_not_validated: vec![ARCHIVE_LEFT_OUT, EXPORT_ASSET], expected_added: vec![], }, PlanTestCase { @@ -40,7 +40,7 @@ fn given_changed_models_when_planning_then_validated_set_follows_policy_and_sele "order_totals", "daily_revenue", ], - expected_not_validated: vec![EXPORT_TASK], + expected_not_validated: vec![EXPORT_ASSET], expected_added: vec![], }, PlanTestCase { @@ -59,7 +59,7 @@ fn given_changed_models_when_planning_then_validated_set_follows_policy_and_sele "daily_revenue", "downstream of a change (--downstream none)", ), - EXPORT_TASK, + EXPORT_ASSET, ], expected_added: vec![], }, @@ -69,7 +69,7 @@ fn given_changed_models_when_planning_then_validated_set_follows_policy_and_sele policy: DownstreamPolicy::Critical, select: None, expected_validated: vec!["daily_revenue"], - expected_not_validated: vec![EXPORT_TASK], + expected_not_validated: vec![EXPORT_ASSET], expected_added: vec![], }, PlanTestCase { @@ -99,7 +99,7 @@ fn given_changed_models_when_planning_then_validated_set_follows_policy_and_sele ARCHIVE_LEFT_OUT, ("order_totals", "not selected (--select)"), ("daily_revenue", "not selected (--select)"), - EXPORT_TASK, + EXPORT_ASSET, ], expected_added: vec![], }, @@ -117,7 +117,7 @@ fn given_changed_models_when_planning_then_validated_set_follows_policy_and_sele ], )]), expected_validated: vec!["stg_orders", "order_totals", "daily_revenue"], - expected_not_validated: vec![ARCHIVE_LEFT_OUT, EXPORT_TASK], + expected_not_validated: vec![ARCHIVE_LEFT_OUT, EXPORT_ASSET], expected_added: vec![], }, PlanTestCase { @@ -129,7 +129,7 @@ fn given_changed_models_when_planning_then_validated_set_follows_policy_and_sele expected_not_validated: vec![ ARCHIVE_LEFT_OUT, ("daily_revenue", "not selected (--select)"), - EXPORT_TASK, + EXPORT_ASSET, ], expected_added: vec![("stg_orders", "order_totals")], }, @@ -139,7 +139,7 @@ fn given_changed_models_when_planning_then_validated_set_follows_policy_and_sele policy: DownstreamPolicy::Critical, select: Some(vec![token("daily_revenue", &["daily_revenue"])]), expected_validated: vec!["stg_orders", "order_totals", "daily_revenue"], - expected_not_validated: vec![ARCHIVE_LEFT_OUT, EXPORT_TASK], + expected_not_validated: vec![ARCHIVE_LEFT_OUT, EXPORT_ASSET], expected_added: vec![ ("stg_orders", "daily_revenue"), ("order_totals", "daily_revenue"), diff --git a/crates/sqlbuild-cli/src/adhoc/tests/test_adhoc_planner.rs b/crates/sqlbuild-cli/src/adhoc/tests/test_adhoc_planner.rs index c50dfd5eaa..5e955a3c09 100644 --- a/crates/sqlbuild-cli/src/adhoc/tests/test_adhoc_planner.rs +++ b/crates/sqlbuild-cli/src/adhoc/tests/test_adhoc_planner.rs @@ -412,9 +412,9 @@ fn given_published_python_nodes_when_planned_then_selectors_resolve_as_on_the_co expected_plan: Ok(( &[ "asset:partner_orders_file", + "asset:request_carrier_collection", "model:orders", "model:stg_orders", - "asset:request_carrier_collection", ], json!({}), )), @@ -451,9 +451,9 @@ fn given_published_python_nodes_when_planned_then_selectors_resolve_as_on_the_co expected_plan: Ok(( &[ "asset:partner_orders_file", + "asset:request_carrier_collection", "model:orders", "model:stg_orders", - "asset:request_carrier_collection", ], json!({}), )), @@ -464,9 +464,9 @@ fn given_published_python_nodes_when_planned_then_selectors_resolve_as_on_the_co expected_plan: Ok(( &[ "asset:partner_orders_file", + "asset:request_carrier_collection", "model:orders", "model:stg_orders", - "asset:request_carrier_collection", ], json!({}), )), @@ -477,8 +477,8 @@ fn given_published_python_nodes_when_planned_then_selectors_resolve_as_on_the_co expected_plan: Ok(( &[ "asset:partner_orders_file", - "model:orders", "asset:request_carrier_collection", + "model:orders", ], json!({}), )), diff --git a/crates/sqlbuild-cli/src/adhoc/tests/test_selection_evaluation.rs b/crates/sqlbuild-cli/src/adhoc/tests/test_selection_evaluation.rs index 3767515ec3..75ecbf32e4 100644 --- a/crates/sqlbuild-cli/src/adhoc/tests/test_selection_evaluation.rs +++ b/crates/sqlbuild-cli/src/adhoc/tests/test_selection_evaluation.rs @@ -16,9 +16,9 @@ fn given_typed_selections_when_evaluated_then_keys_or_located_refusals_match_sqb exclude: "", expected_evaluation: Ok(&[ "asset:partner_orders_file", + "asset:request_carrier_collection", "model:orders", "model:stg_orders", - "asset:request_carrier_collection", ]), }, SelectionEvaluationTestCase { @@ -139,12 +139,7 @@ fn given_a_publication_when_its_vocabulary_is_read_then_each_value_counts_the_no VocabularyTestCase { description: "a kind counts its nodes", entries_of: |vocabulary| &vocabulary.kinds, - expected_entries: &[ - ("asset", 2), - ("loader", 2), - ("model", 4), - ("source", 1), - ], + expected_entries: &[("asset", 2), ("loader", 2), ("model", 4), ("source", 1)], }, ]; let vocabulary = orders_publication().vocabulary(); diff --git a/crates/sqlbuild-core/src/build_command/tests/test_build_command.rs b/crates/sqlbuild-core/src/build_command/tests/test_build_command.rs index 1946bcfcc7..b35c275953 100644 --- a/crates/sqlbuild-core/src/build_command/tests/test_build_command.rs +++ b/crates/sqlbuild-core/src/build_command/tests/test_build_command.rs @@ -192,9 +192,9 @@ fn given_python_node_results_when_rendering_then_shows_status_and_detail() { ], expected_failed: true, expected_text: concat!( - " python task count_orders OK\n", + " python asset count_orders OK\n", " python asset extract SKIP upstream failed\n", - " python task boom FAIL boom\n", + " python asset boom FAIL boom\n", ), }, PythonRowsTestCase { diff --git a/crates/sqlbuild-core/src/build_progress/tests/test_build_progress.rs b/crates/sqlbuild-core/src/build_progress/tests/test_build_progress.rs index d3b60b21f4..e980984f8d 100644 --- a/crates/sqlbuild-core/src/build_progress/tests/test_build_progress.rs +++ b/crates/sqlbuild-core/src/build_progress/tests/test_build_progress.rs @@ -57,7 +57,7 @@ fn given_build_results_when_formatting_footer_then_matches_python_layout() { warnings: Vec::new(), }], use_color: false, - expected_text: "\nWarnings:\n\n orders\n audit not_null returned 2 rows\n\n\u{2717} Completed with errors PASS=1 WARN=1 FAIL=2 INSUFFICIENT=0 SKIP=1 TOTAL=5 (1.50s)\n\nFailures:\n\n customers (staging)\n error [E100] boom\n delta table kept for inspection: orders__delta\n\n\nPython node failures:\n\n task refresh Python node failed\n\nSkipped (1): payments", + expected_text: "\nWarnings:\n\n orders\n audit not_null returned 2 rows\n\n\u{2717} Completed with errors PASS=1 WARN=1 FAIL=2 INSUFFICIENT=0 SKIP=1 TOTAL=5 (1.50s)\n\nFailures:\n\n customers (staging)\n error [E100] boom\n delta table kept for inspection: orders__delta\n\n\nPython node failures:\n\n asset refresh Python node failed\n\nSkipped (1): payments", }, ]; diff --git a/crates/sqlbuild-orchestration/src/declarations/models.rs b/crates/sqlbuild-orchestration/src/declarations/models.rs index bddd2e927a..17f003fa84 100644 --- a/crates/sqlbuild-orchestration/src/declarations/models.rs +++ b/crates/sqlbuild-orchestration/src/declarations/models.rs @@ -539,7 +539,7 @@ pub struct ProviderSetting { pub missing: bool, } -/// Code that injects a provider: a node (`loader`, `task`, `asset`, `check`, `hook` or `materialization`, keyed `kind:name`), an `event_exporter` or a `sink`. +/// Code that injects a provider: a node (`loader`, `asset`, `check`, `hook` or `materialization`, keyed `kind:name`), an `event_exporter` or a `sink`. #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, utoipa::ToSchema)] pub struct ProviderConsumer { pub kind: String, diff --git a/crates/sqlbuild-orchestration/src/runs/models.rs b/crates/sqlbuild-orchestration/src/runs/models.rs index 1d159c9386..4bb322773c 100644 --- a/crates/sqlbuild-orchestration/src/runs/models.rs +++ b/crates/sqlbuild-orchestration/src/runs/models.rs @@ -1030,7 +1030,7 @@ pub struct SelectionRefusal { pub part: String, /// Which comma part of the token it is, from 0. pub index: u32, - /// `tag`, `path`, `job`, `source`, `seed`, `kind`, `test`, `name`, `glob`, or a Python node type such as `task`. + /// `tag`, `path`, `job`, `source`, `seed`, `kind`, `test`, `name`, `glob`, or a Python node type such as `asset`. #[serde(rename = "type")] pub selector_type: String, /// The value the part names, without its type prefix or `+` markers, such as `finanse`. diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/imported_python_nodes.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/imported_python_nodes.rs index f19fb8c463..932044cff4 100644 --- a/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/imported_python_nodes.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/imported_python_nodes.rs @@ -323,11 +323,14 @@ fn removed_tasks_guidance( let Some(cause) = error.cause(py) else { return error; }; - if !cause.is_instance_of::(py) - || !imports_removed_tasks(cause.value(py).as_any()) - { + if !cause.is_instance_of::(py) { return error; } + match imports_removed_tasks(cause.value(py).as_any()) { + Ok(true) => {} + Ok(false) => return error, + Err(unexpected) => return unexpected, + } let Ok(path) = project.relative_to(file_path) else { return error; }; @@ -339,14 +342,21 @@ fn removed_tasks_guidance( } /// Whether an `ImportError` names `sqlbuild.tasks`, or `tasks` imported from `sqlbuild`. -fn imports_removed_tasks(error: &Bound<'_, PyAny>) -> bool { - let text = |attribute: &str| -> Option { - error.getattr_opt(attribute).ok().flatten()?.extract().ok() +fn imports_removed_tasks(error: &Bound<'_, PyAny>) -> PyResult { + let text = |attribute: &str| -> PyResult> { + Ok(error + .getattr_opt(attribute)? + .map(|value| value.extract()) + .transpose()? + .flatten()) }; let (package, name) = REMOVED_TASKS; - let imported: Option = text("name"); - imported.as_deref() == Some(format!("{package}.{name}").as_str()) - || (imported.as_deref() == Some(package) && text("name_from").as_deref() == Some(name)) + let imported: Option = text("name")?; + Ok( + imported.as_deref() == Some(format!("{package}.{name}").as_str()) + || (imported.as_deref() == Some(package) + && text("name_from")?.as_deref() == Some(name)), + ) } fn audit_cases<'py>( diff --git a/fensu.toml b/fensu.toml index dd6c7d87d2..8161913818 100644 --- a/fensu.toml +++ b/fensu.toml @@ -2867,7 +2867,7 @@ pending_budgets = [ "crates/sqlbuild-python-compile/src/compile_driver/_helpers/assembly/assembly_models.rs=2", "crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/inputs.rs=15", "crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/project.rs=8", - "crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/python_plan_entries.rs=2", + "crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/python_plan_entries.rs=1", "crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/relation_targets.rs=3", "crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/target_defaults.rs=4", "crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/analysis.rs=12", diff --git a/python-boundary.toml b/python-boundary.toml index 896154dc6b..8b7d997f89 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -75,7 +75,7 @@ note = "Provider base class users subclass, and injection into user functions." [[stays]] path = "python_nodes" rule = 3 -note = "Public decorators and authoring shapes re-exported by assets.py, tasks.py, loaders.py, hooks.py and retries.py, run when user modules are imported; RetryPolicy and AuditCase validation, retry delays and hook-read normalization are native (validate_retry_policy, retry_delay_seconds, validate_audit_case, normalize_hook_reads)." +note = "Public decorators and authoring shapes re-exported by assets.py, loaders.py, hooks.py and retries.py, run when user modules are imported; RetryPolicy and AuditCase validation, retry delays and hook-read normalization are native (validate_retry_policy, retry_delay_seconds, validate_audit_case, normalize_hook_reads)." [[stays]] path = "microbatches" @@ -667,11 +667,6 @@ path = "sqlite_history.py" rule = 3 note = "Public facade: re-exports only." -[[stays]] -path = "tasks.py" -rule = 3 -note = "Public facade: re-exports plus one-line delegating wrappers (task, get_task_definition)." - [[stays]] path = "cost/models.py" rule = 1 diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md index d68a377f7b..b5c23bfe04 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md @@ -95,11 +95,10 @@ Bundled copies of every page on the SQLBuild documentation site, matching the in - [RuleContext and compiler facts](concepts/rules/custom-rules/rule-context.md) (`concepts/rules/custom-rules/rule-context`) - Inspect stable compiler-owned facts and opt into lazy SQL AST access. - [Testing and deterministic Rules](concepts/rules/custom-rules/testing-and-determinism.md) (`concepts/rules/custom-rules/testing-and-determinism`) - Test custom findings and keep implementations safe for dependency-aware caching. - [Published-layer Rules](concepts/rules/published-layer-rules.md) (`concepts/rules/published-layer-rules`) - Keep each domain's published layer free of consumer-specific variants. -- [Overview](concepts/python-nodes/overview.md) (`concepts/python-nodes/overview`) - Tasks, assets, loaders, and checks as first-class nodes in the SQLBuild graph. +- [Overview](concepts/python-nodes/overview.md) (`concepts/python-nodes/overview`) - Loaders, assets, and checks as first-class nodes in the SQLBuild graph. - [Loaders](concepts/python-nodes/loaders.md) (`concepts/python-nodes/loaders`) - Load external data into source tables with Python functions. -- [Tasks](concepts/python-nodes/tasks.md) (`concepts/python-nodes/tasks`) - Run Python computation and side effects as nodes in the SQLBuild graph. -- [Assets](concepts/python-nodes/assets.md) (`concepts/python-nodes/assets`) - Produce or observe external artifacts as nodes in the SQLBuild graph. -- [Checks](concepts/python-nodes/checks.md) (`concepts/python-nodes/checks`) - Validate tasks, assets, loaders, and SQL relations with Python checks. +- [Assets](concepts/python-nodes/assets.md) (`concepts/python-nodes/assets`) - Run Python computation, side effects and external artifacts as nodes in the SQLBuild graph. +- [Checks](concepts/python-nodes/checks.md) (`concepts/python-nodes/checks`) - Validate assets, loaders, and SQL relations with Python checks. - [Factories](concepts/python-nodes/factories.md) (`concepts/python-nodes/factories`) - Generate Python nodes programmatically with @factory. - [Providers](concepts/python-nodes/providers.md) (`concepts/python-nodes/providers`) - Shared runtime services for Python nodes and hooks. - [SQL References](concepts/python-nodes/sql-references.md) (`concepts/python-nodes/sql-references`) - Read SQL models, sources, and seeds from Python nodes without creating SQL dependencies. @@ -152,7 +151,7 @@ Bundled copies of every page on the SQLBuild documentation site, matching the in - [sqb load](cli/load.md) (`cli/load`) - Load managed sources into the warehouse. - [sqb seed](cli/seed.md) (`cli/seed`) - Load seed CSV files into the warehouse. - [sqb audit](cli/audit.md) (`cli/audit`) - Run data quality audits in isolation. -- [sqb check](cli/check.md) (`cli/check`) - Run Python checks against tasks, assets, and loaders. +- [sqb check](cli/check.md) (`cli/check`) - Run Python checks against assets and loaders. - [sqb freshness](cli/freshness.md) (`cli/freshness`) - Observe source freshness without writing state. - [sqb diff](cli/diff.md) (`cli/diff`) - Compare schemas and data between targets. - [sqb review](cli/review.md) (`cli/review`) - Build what a branch changed in a temporary schema, test it, compare it with a target and report. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md index 129b506831..d0e2b5a1bf 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md @@ -56,7 +56,7 @@ own option. `--version` works only directly after `sqb`, as in `sqb --version`: | `--no-audits` | Skip audits | | `--ignore-audit-failures` | Run every audit, but let failures through: staged tables are promoted, incremental batches applied and downstream nodes built. Failures are listed as ignored; see [Ignoring audit failures](../concepts/audits.md#ignoring-audit-failures). Not allowed with `--no-audits` | | `--no-ignore-audit-failures` | Let audit failures block the build even where `ignore_audit_failures` is set | -| `--no-python` | Skip read-side Python tasks and assets (loader-side Python still runs for selected sources) | +| `--no-python` | Skip read-side Python assets (loader-side Python still runs for selected sources) | | `--no-sql-analysis` | Disable compile-time SQL analysis (`--no-sql-validation` is an alias) | | `--no-cache` | Bypass the reusable compile-analysis cache for this invocation | | `--full-refresh` | Drop and rebuild selected models unless a model sets `full_refresh false`; `full_refresh true` forces a model even without this flag | @@ -153,7 +153,7 @@ A model or seed always runs, whatever counts, when: | Its last build did not finish | The build was killed or failed after the model started writing | | Its relation is missing, or `--full-refresh` or a cursor override applies | `sqb build --changed --full-refresh` | | The planner must rebuild it for a reason other than an edit to its definition | A renamed model | -| What it reads can't be listed | SQL that reads relations not declared as sources or models, or a Python task or asset upstream | +| What it reads can't be listed | SQL that reads relations not declared as sources or models, or a Python asset upstream | | An upstream needs to run, even when it is not selected | `sqb build customer_orders --changed` runs `customer_orders` until a changed `stg_orders` is built | ### How far a change reaches diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/check.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/check.md index c35049ecd0..84a983548d 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/check.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/check.md @@ -2,11 +2,11 @@ # sqb check -> Run Python checks against tasks, assets, and loaders. +> Run Python checks against assets and loaders. Online: https://sqlbuild.com/docs/cli/check/ -Runs Python [checks](../concepts/python-nodes/checks.md) in isolation. Checks validate the output of tasks, assets, and loaders. For SQL relation validation, use [`sqb audit`](audit.md) instead. +Runs Python [checks](../concepts/python-nodes/checks.md) in isolation. Checks validate the output of assets and loaders. For SQL relation validation, use [`sqb audit`](audit.md) instead. ## Usage @@ -29,7 +29,7 @@ Positional `SELECTOR` arguments work like `--select`, and both forms can be comb | `--vars` | Override project variables | | `--warehouse ` | Snowflake warehouse for this invocation; overrides the target's `build` [warehouse group](../concepts/adapters/snowflake.md#command-group-warehouses) and the connection warehouse | -Selecting a non-check node is rejected; use [`sqb build`](build.md) to run tasks and assets. +Selecting a non-check node is rejected; use [`sqb build`](build.md) to run assets. ## Examples @@ -55,4 +55,4 @@ Results are written to `target/run/checks/python_checks.json`. ## Relationship to build -`sqb build` runs relevant Python checks by default after the tasks, assets, loaders, models, sources, and seeds they depend on have run. `sqb build --no-audits` skips checks alongside audits. `sqb check` runs only checks, on demand. +`sqb build` runs relevant Python checks by default after the assets, loaders, models, sources, and seeds they depend on have run. `sqb build --no-audits` skips checks alongside audits. `sqb check` runs only checks, on demand. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/plan.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/plan.md index 66d69eead9..68449c1ea7 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/plan.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/plan.md @@ -32,7 +32,7 @@ Positional `SELECTOR` arguments work like `--select`, and both forms can be comb |------|-------------| | `--no-sql-analysis` | Disable compile-time SQL analysis (`--no-sql-validation` is an alias) | | `--no-cache` | Bypass the reusable compile-analysis cache for this invocation | -| `--no-python` | Exclude read-side Python tasks and assets from the plan | +| `--no-python` | Exclude read-side Python assets from the plan | | `--defer-to` | Resolve unselected model references against another target | | `--plan-from` | Read warehouse state `live`, or from the target's `synced_state` checked against the warehouse first; overrides the target's `plan_from`. See [Planning from synced state](../concepts/project-configuration.md#planning-from-synced-state) | | `--as` | Compile for another target and show its plan, including [model migration](../concepts/models/migrations.md) decisions, using the current connection. Inspection only. | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/playground.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/playground.md index b288863a33..7d8b01c212 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/playground.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/playground.md @@ -23,7 +23,7 @@ The positional argument is the directory to create (default `sqlbuild-playground | `waffle_shop` | Default. DuckDB-backed project with models, tests, scenarios, and macros. | | `dagster` | A Dagster code location: a ready-to-run `definitions.py` next to the waffle shop SQLBuild project in `waffle_shop/`. | | `rivers` | A Rivers code repository: a `definitions.py` repository definition next to the waffle shop SQLBuild project in `waffle_shop/`. | -| `python_nodes` | A small DuckDB project demonstrating [Python nodes](../concepts/python-nodes/overview.md): a task feeding a loader, a model read by a Python asset through `ctx.relation(model(...))`, a soft-skip fan-in, `materialized=False`, and a Python check. | +| `python_nodes` | A small DuckDB project demonstrating [Python nodes](../concepts/python-nodes/overview.md): an asset feeding a loader, a model read by a Python asset through `ctx.relation(model(...))`, a soft-skip fan-in, `materialized=False`, and a Python check. | ## What it creates @@ -52,7 +52,7 @@ The Dagster `definitions.py` uses `sqlbuild_assets`, `sqlbuild_scenario_checks`, The `python_nodes` template instead creates a focused Python-nodes project: -- `python/tasks/orders.py`, `python/loaders/orders.py`, `python/assets/orders_export.py`, `python/checks/orders_export.py` +- `python/assets/orders_inputs.py`, `python/loaders/orders.py`, `python/assets/orders_export.py`, `python/checks/orders_export.py` - A `fact_orders` SQL model over a managed `raw_orders` source - Examples of the SQL boundary, `ctx.relation(model(...))`, soft-skip fan-in, and a Python check diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/review.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/review.md index 3762580046..a47067f818 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/review.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/review.md @@ -79,7 +79,7 @@ Downstream models the policy leaves out are always listed as **not validated**, The impact tree is computed from the whole changed set, whatever the policy. A review never runs [Python nodes](../concepts/python-nodes/overview.md). A changed Python node, -and every task, asset or check downstream of a change, appears in the impact tree as not validated +and every asset or check downstream of a change, appears in the impact tree as not validated with the reason. So does every model that reads a changed loader's source, directly or through other models: the review would build it from the `--against` data rather than from the changed loader, so it is listed as `reads Python node loader:, not rebuilt by review` instead of diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/test.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/test.md index c2c51a1835..25362b1c78 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/test.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/test.md @@ -161,7 +161,7 @@ arguments, result type or body change, or a project variable it reads; formattin description do not count. A node that calls a SQL macro also uses every SQL macro that macro calls. An enum, constant or audit counts as changed when its declaration file changes other than in formatting and comments. A seed counts as changed with its data or column types, a source with its declaration or inline expression, and a function with its definition. -Loaders, tasks, assets and checks are compared by their Python modules and the project modules +Loaders, assets and checks are compared by their Python modules and the project modules they import; they appear in the change summary but no unit test runs them. ### Project-wide inputs diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md index 94cda6a657..243b25f5b9 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md @@ -83,7 +83,7 @@ error[P010]: model 'orders_summary' has no description | SQL function or table function | `FUNCTION (description "...")` | | Python UDF | `@udf(description="...")` or the function docstring | | Named SQL hook | `HOOK (description "...")` | -| Python hook, task, asset, check | `description="..."` on the decorator or the function docstring | +| Python hook, asset, check | `description="..."` on the decorator or the function docstring | | Loader | `@loader(description="...")` or the function docstring; a loader that feeds a declared source may instead rely on the source's YAML description | | Provider | The provider class docstring | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/hooks.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/hooks.md index 4f785a52d5..4ccb251df4 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/hooks.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/hooks.md @@ -124,7 +124,7 @@ A Python post-hook that returns `ctx.skip(...)` stops the remaining post-hooks a ## Names and identity -SQL and Python hook names share the project-wide resource namespace with models, sources, seeds, functions, loaders, tasks, assets, checks, and providers. Any collision fails discovery, including two same-stem SQL hook files in different directories or a SQL and Python hook with the same name. +SQL and Python hook names share the project-wide resource namespace with models, sources, seeds, functions, loaders, assets, checks, and providers. Any collision fails discovery, including two same-stem SQL hook files in different directories or a SQL and Python hook with the same name. The ordered hook list participates in each model's version identity: diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/hooks/python.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/hooks/python.md index d2ad1a345f..e731337cbe 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/hooks/python.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/hooks/python.md @@ -140,7 +140,7 @@ Python hooks declare a `HookContext` parameter named `ctx`, `context`, `_ctx`, o ## Providers -Providers may be injected into Python hook parameters by name or accessed through `ctx.providers`. They are resolved lazily using the same lifecycle as loaders, tasks, assets, and checks. SQL hooks cannot use providers because they are compiled SQL statements rather than Python callables. +Providers may be injected into Python hook parameters by name or accessed through `ctx.providers`. They are resolved lazily using the same lifecycle as loaders, assets, and checks. SQL hooks cannot use providers because they are compiled SQL statements rather than Python callables. ```python from sqlbuild.hooks import hook diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/planning.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/planning.md index 417c91ac0c..23f293e4af 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/planning.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/planning.md @@ -71,7 +71,7 @@ seed from one whose file or loading behavior changed. ### Python nodes -Loaders, tasks, assets, checks, and Python hooks have identities derived from project-owned source, +Loaders, assets, checks, and Python hooks have identities derived from project-owned source, transitive project dependencies, and decorator configuration. Python hook identities also participate in the identity of models that invoke them. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md index 6b7b5893af..801d730a8f 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md @@ -178,7 +178,7 @@ A target is a named build context - the database and schema you build into, plus | Field | Description | |-------|-------------| | `schema` | Schema for all models in this target; required for named targets | -| `loader_schema` | Default write schema for managed source loaders; falls back to `schema`. Required when `schema` is `preserve` and the project has loaders, tasks, assets or checks, because their [results](python-nodes/overview.md#result-persistence) go there | +| `loader_schema` | Default write schema for managed source loaders; falls back to `schema`. Required when `schema` is `preserve` and the project has loaders, assets or checks, because their [results](python-nodes/overview.md#result-persistence) go there | | `database` | Database for all models in this target | | `connection` | Name from `[connections.]` used for endpoint, authentication, and compute; optional when exactly one named connection is defined | | `vars` | Target-specific project variables | @@ -551,7 +551,7 @@ delta_compile = true | `scenario_timeout` | none | Time limit for each scenario, or `"disabled"` | | `delta_compile` | `true` | After an edit to model SQL or SQL test files only, compile just the changed files against the previous compile. See [Compiling after an edit](../cli/compile.md#compiling-after-an-edit). | | `allow_missing_secrets` | `false` | Let `compile`, `rules run`, `format`, `dag`, `lineage` and `scope` run without provider secrets. `sqlbuild_local.toml` only. See [Secrets](python-nodes/providers.md#secrets). | -| `allow_warehouse_round_trips` | `false` | Allow a loader to depend on a task or asset that reads SQL, so data can leave the warehouse and come back through a source. Project file only. See [The SQL boundary](python-nodes/overview.md#warehouse-round-trips). | +| `allow_warehouse_round_trips` | `false` | Allow a loader to depend on an asset that reads SQL, so data can leave the warehouse and come back through a source. Project file only. See [The SQL boundary](python-nodes/overview.md#warehouse-round-trips). | | `ignore_audit_failures` | `false` | Let builds go on past failing audits: they still run and record `fail`, but promote, apply and build downstream as if they passed. A target, a job and `--ignore-audit-failures` / `--no-ignore-audit-failures` override it. See [Ignoring audit failures](audits.md#ignoring-audit-failures). | | `record_run_host` | `true` | Record the machine name in each run's rows in `_sqlbuild_runs`. See [Run records](../cli/build.md#run-records). | @@ -614,7 +614,7 @@ Every reference to a model, source, or seed must be explicit and typed. By defau - A macro whose expanded output contains `__ref()`, `__source()`, or `__seed()` fails to compile with `P006`. Pass the reference to the macro as an argument instead; see [Typed reference arguments](macros/python-macros.md#typed-reference-arguments). -- Literal SQL passed to `ctx.query()` or `ctx.execute_sql()` in a task, asset, check, loader, or +- Literal SQL passed to `ctx.query()` or `ctx.execute_sql()` in an asset, check, loader, or Python hook that names a project relation fails to compile with `P008`. - The same check runs on SQL sent at run time and reports a `P008` warning; the query still runs. See [Hard-coded relation names](python-nodes/sql-references.md#hard-coded-relation-names). @@ -806,7 +806,7 @@ See [skills CLI reference](../cli/skills.md) for usage details. ## Project Python Project-owned Python lives in a small set of roots. Anything that runs as part of the pipeline graph -goes in `python/`: loaders, tasks, assets, checks, factories, and the helper modules they import. +goes in `python/`: loaders, assets, checks, factories, and the helper modules they import. Modules under `python/` are normal Python: constants, classes, undecorated helper functions, and helper-only modules such as `python/_helpers.py` are allowed, while decorators determine which functions become graph nodes. See [Python nodes](python-nodes/overview.md#the-python-folder). diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/assets.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/assets.md index 34a2cf3d07..3a5be2a596 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/assets.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/assets.md @@ -2,71 +2,59 @@ # Assets -> Produce or observe external artifacts as nodes in the SQLBuild graph. +> Run Python computation, side effects and external artifacts as nodes in the SQLBuild graph. Online: https://sqlbuild.com/docs/concepts/python-nodes/assets/ ## Contents - Defining an asset -- Materialization - Dependencies +- Returning results +- Materialization - Columns and column lineage -- Skipping and retries +- Skipping +- Retries - Decorator parameters - Running assets -Assets are Python nodes that produce or observe an external artifact - an exported file, a published dataset, a dashboard refresh, an ML model. They behave like [tasks](tasks.md) but add dataset-like metadata (columns, column lineage) and a materialization flag. See [Python Nodes](overview.md) for the shared model and the SQL boundary rules. +Assets are Python functions that run as part of the DAG. Use them for computation, side effects and external artifacts such as an exported file, a published dataset, a dashboard refresh or an ML model. See [Python Nodes](overview.md) for the shared model and the SQL boundary rules. ## Defining an asset -Place Python files under `python/` (the playgrounds use `python/assets/`) and decorate functions with `@asset`: +Place Python files under `python/` and decorate functions with `@asset`: ```python -# python/assets/exports.py +# python/exports/orders_export.py from sqlbuild.assets import asset, AssetContext @asset def orders_export(ctx: AssetContext): """Orders export.""" path = write_orders_csv() - return ctx.result(metadata={"path": path, "rows": 1200}, materialized=True) + return ctx.result(payload={"path": path}, metadata={"rows": 1200}, materialized=True) ``` -The asset receives an `AssetContext` and returns through `ctx.result(...)`. +The asset receives an `AssetContext` and returns through `ctx.result(...)`. A plain return value or `None` is also accepted and normalized to a successful result. -## Materialization +## Dependencies -Assets record whether they actually produced an artifact via `materialized`: +Assets declare dependencies with `depends_on`, accepting a single function, a tuple, or a list. They may depend on other assets and on loaders: ```python @asset -def orders_export(ctx: AssetContext): - """Orders export.""" - if nothing_changed(): - return ctx.result(metadata={"status": "unchanged"}, materialized=False) - return ctx.result(metadata={"path": export()}, materialized=True) +def fetch_orders(ctx: AssetContext): + """Fetch orders.""" + return ctx.result(payload=download_orders()) + +@asset(depends_on=fetch_orders) +def summarize_orders(ctx: AssetContext): + """Summarize orders.""" + result = ctx.result_of(fetch_orders) + return ctx.result(metadata={"count": len(result.payload)}) ``` -- `materialized=True` - the artifact was produced this run. -- `materialized=False` - the asset ran but produced nothing (e.g. it only observed state). This is **not** a skip; the node still succeeded. - -Only assets have `materialized`; tasks do not. - -## Dependencies - -Assets declare dependencies with `depends_on` (a single function, tuple, or list), and may depend on tasks, assets, and loaders: - -```python -from sqlbuild.assets import asset -from python.tasks.orders import export_orders - -@asset(depends_on=export_orders) -def orders_dashboard(ctx): - """Orders dashboard.""" - result = ctx.result_of(export_orders) - return ctx.result(metadata={"rows": result.payload["rows"]}, materialized=True) -``` +`ctx.result_of(node_fn)` reads the latest persisted result of an upstream node, returning a `NodeResultEnvelope` with `payload`, `metadata`, `status`, and `ts` fields. Results persist across runs. Use `ctx.results_of(node_fn, limit=N)` to read result history. Reading a missing or unsuccessful upstream raises unless you pass `default=`. To read a SQL model or source, declare a typed reference and resolve it at runtime - see [SQL references](sql-references.md): @@ -82,6 +70,40 @@ def orders_extract(ctx): return ctx.result(metadata={"rows": rows}, materialized=True) ``` +## Returning results + +```python +@asset +def build_export(ctx: AssetContext): + """Build export.""" + return ctx.result( + payload={"path": "/exports/orders.csv"}, + metadata={"rows": 1200}, + materialized=True, + ) +``` + +- `payload` is the value downstream nodes read with `ctx.result_of(...)`. +- `metadata` is structured JSON for catalogs and downstream reads. +- `materialized` records whether the asset produced its artifact this run (see below). + +## Materialization + +Set `materialized` when the asset produces or observes an artifact: + +```python +@asset +def orders_export(ctx: AssetContext): + """Orders export.""" + if nothing_changed(): + return ctx.result(metadata={"status": "unchanged"}, materialized=False) + return ctx.result(metadata={"path": export()}, materialized=True) +``` + +- `materialized=True` - the artifact was produced this run. +- `materialized=False` - the asset ran but produced nothing (e.g. it only observed state). This is **not** a skip; the node still succeeded. +- Leave it unset for an asset that runs computation or a side effect without an artifact. + ## Columns and column lineage Assets can declare a schema for catalog and lineage purposes. This does not enforce anything at runtime; it describes the artifact: @@ -105,25 +127,60 @@ def orders_export(ctx): - `columns` - column declarations with `name`, optional `type`, `nullable`, `description`, and `meta`. - `column_lineage` - maps each asset column to upstream `{node, column}` references, surfaced in the DAG artifact and integrations. -Tasks and checks do not support `columns` or `column_lineage`. +Checks do not support `columns` or `column_lineage`. + +## Skipping -## Skipping and retries +Return `ctx.skip(...)` to skip an asset: -Assets support the same `ctx.skip(...)` and `retry` behavior as tasks: +```python +@asset +def export_if_present(ctx: AssetContext): + """Export if present.""" + if not new_files_available(): + return ctx.skip("no new files") + return ctx.result(payload=do_export()) +``` + +- `"soft"` (default) skips only this asset; dependents may still run if another upstream succeeded. +- `"hard"` skips this asset and blocks its dependents. + +`mode` accepts either a plain string or the `SkipMode` enum: ```python from sqlbuild.assets import asset, SkipMode + +@asset +def optional_step(ctx): + """Optional step.""" + return ctx.skip("nothing to do", mode=SkipMode.HARD) # or mode="hard" +``` + +## Retries + +`@asset` accepts a `retry` policy for transient failures: + +```python +from sqlbuild.assets import asset from sqlbuild.retries import RetryPolicy -@asset(retry=RetryPolicy(max_attempts=3, retry_on=(IOError,))) -def export(ctx): - """Export the order summary.""" - if not ready(): - return ctx.skip("upstream not ready", mode="soft") # or SkipMode.SOFT - return ctx.result(metadata={"path": do_export()}, materialized=True) +@asset(retry=RetryPolicy(max_attempts=3, retry_on=(ConnectionError,))) +def call_api(ctx): + """Call api.""" + return ctx.result(payload=fetch_from_flaky_api()) ``` -See [Tasks](tasks.md#retries) for the full retry policy fields. +| Field | Default | Description | +|-------|---------|-------------| +| `max_attempts` | `3` | Total attempts including the first | +| `retry_on` | `Exception` | Exception class, tuple, or list to retry on | +| `initial_delay_seconds` | `1.0` | Delay before the first retry | +| `backoff_multiplier` | `2.0` | Exponential backoff multiplier | +| `max_delay_seconds` | `30.0` | Cap on any single delay | +| `max_elapsed_seconds` | `None` | Overall time bound for retries | +| `jitter` | `True` | Randomize delays to avoid synchronized retries | + +The default is no retry. Set `retry_on` explicitly rather than relying on the broad default when you can. The original exception is preserved if all attempts fail. ## Decorator parameters @@ -149,4 +206,4 @@ sqb build --select orders_export --no-tests --no-audits sqb build --select +orders_export ``` -Assets run during `sqb build`. Use `--no-python` to suppress read-side assets while still loading sources. +Assets run during `sqb build`. Use `--no-python` to suppress read-side assets while still loading sources. They are not validated unless you also write a [check](checks.md) that depends on them. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/checks.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/checks.md index fe54d4880a..f37af2fd34 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/checks.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/checks.md @@ -2,7 +2,7 @@ # Checks -> Validate tasks, assets, loaders, and SQL relations with Python checks. +> Validate assets, loaders, and SQL relations with Python checks. Online: https://sqlbuild.com/docs/concepts/python-nodes/checks/ @@ -16,9 +16,9 @@ Online: https://sqlbuild.com/docs/concepts/python-nodes/checks/ - Running checks - Checks vs audits -Checks are Python nodes that validate the output of tasks, assets, and loaders, or the data in SQL models, sources, and seeds. They are the Python analog of SQL [audits](../audits.md). See [Python Nodes](overview.md) for the shared model. +Checks are Python nodes that validate the output of assets and loaders, or the data in SQL models, sources, and seeds. They are the Python analog of SQL [audits](../audits.md). See [Python Nodes](overview.md) for the shared model. -Checks are separate graph nodes, not callbacks embedded in a task or asset. A check declares what it validates through `depends_on`. +Checks are separate graph nodes, not callbacks embedded in an asset. A check declares what it validates through `depends_on`. ## Defining a check @@ -27,7 +27,7 @@ Place Python files under `python/` (the playgrounds use `python/checks/`) and de ```python # python/checks/orders.py from sqlbuild.checks import check, CheckContext -from python.tasks.orders import export_orders +from python.exports.orders import export_orders @check(depends_on=export_orders) def check_orders_exported(ctx: CheckContext): @@ -97,7 +97,7 @@ def orders_freshness(ctx): ## What checks can depend on -- Checks may depend on **tasks, assets, and loaders**. +- Checks may depend on **assets and loaders**. - Checks may depend on SQL **models, sources, and seeds** through `model()`, `source()`, and `seed()` references. They may **not** depend on functions. - Checks may **not** depend on other checks. - Checks may **not** depend on a terminal source loader directly. Validate loaded source data with a source audit instead. @@ -108,7 +108,7 @@ A check that depends on a single node is displayed grouped under that node. Mult | Parameter | Description | |-----------|-------------| -| `depends_on` | Required. Tasks/assets/loaders to validate, and `model()`/`source()`/`seed()` references (function, reference, tuple, or list) | +| `depends_on` | Required. Assets/loaders to validate, and `model()`/`source()`/`seed()` references (function, reference, tuple, or list) | | `name` | Override the node name (defaults to the function name) | | `severity` | `error` (default) or `warn` | | `tags` | Labels for selection and grouping | @@ -137,13 +137,13 @@ sqb check --select +check_orders_exported sqb check --select tag:exports ``` -`sqb check` rejects selecting non-check nodes; use `sqb build` to run tasks and assets. The models, sources, and seeds a check depends on must already exist; `sqb check` does not build them. Check results are written to `target/run/checks/python_checks.json`, and `sqb check --json` prints them to stdout. +`sqb check` rejects selecting non-check nodes; use `sqb build` to run assets. The models, sources, and seeds a check depends on must already exist; `sqb check` does not build them. Check results are written to `target/run/checks/python_checks.json`, and `sqb check --json` prints them to stdout. ## Checks vs audits | | Checks | Audits | |---|--------|--------| -| Validates | Python tasks, assets, loaders, and SQL relations | SQL relations | +| Validates | Python assets, loaders, and SQL relations | SQL relations | | Authored in | `python/` (Python) | `MODEL()` headers / `audits/` (SQL) | | Run by | `sqb build`, `sqb check` | `sqb build`, `sqb audit` | | Severity | `error`, `warn` | `error`, `warn` | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/factories.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/factories.md index afec5249a9..6d2d78d77b 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/factories.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/factories.md @@ -6,7 +6,7 @@ Online: https://sqlbuild.com/docs/concepts/python-nodes/factories/ -A factory is a function that **generates** Python nodes instead of authoring them one at a time. Use `@factory` when you want to create many similar [tasks](tasks.md), [assets](assets.md), [loaders](loaders.md), or [checks](checks.md) from a list, a config, or a loop, rather than hand-writing each one. +A factory is a function that **generates** Python nodes instead of authoring them one at a time. Use `@factory` when you want to create many similar [assets](assets.md), [loaders](loaders.md), or [checks](checks.md) from a list, a config, or a loop, rather than hand-writing each one. Factories are an advanced feature. Most projects author nodes directly; reach for a factory when you find yourself copy-pasting near-identical node definitions. @@ -35,7 +35,7 @@ def export_assets(): The returned nodes are discovered and added to the graph exactly as if you had written them by hand. They participate in selection, lifecycle, the DAG artifact, and integrations like any other node. -A factory may return a single node or a list, tuple, or set of nodes. Every returned item must be a decorated `@task`, `@asset`, `@loader`, or `@check` function. +A factory may return a single node or a list, tuple, or set of nodes. Every returned item must be a decorated `@asset`, `@loader`, or `@check` function. ## Where factories live diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/loaders.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/loaders.md index fb7deeb529..bcd5033ee0 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/loaders.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/loaders.md @@ -275,9 +275,9 @@ Every loader function receives a `LoaderContext` as its first argument. It provi | `qualify_name(name)` | Return a fully-qualified relation name in the destination database/schema | | `skip(reason, mode=...)` | Skip this loader. `mode` is `"soft"` (default, skip only this loader) or `"hard"` (also block dependents) | | `result(payload=, metadata=, materialized=)` | Return a structured result for a self-managed loader | -| `result_of(node_fn)` | Read the result of an upstream task, asset or loader: its result from this run, else its latest successful one; raises naming the upstream if it never succeeded | +| `result_of(node_fn)` | Read the result of an upstream asset or loader: its result from this run, else its latest successful one; raises naming the upstream if it never succeeded | | `results_of(node_fn, limit=N)` | Read the last N successful results of an upstream node, newest first | -| `loader(loader_fn)` | Return a `LoaderRelationRef` for an upstream loader dependency; a task or asset is refused with a pointer to `result_of` | +| `loader(loader_fn)` | Return a `LoaderRelationRef` for an upstream loader dependency; an asset is refused with a pointer to `result_of` | | `source(source_name)` | Return a `LoaderRelationRef` for a project source by YAML name | ### LoaderRelationRef @@ -292,7 +292,7 @@ Returned by `ctx.loader()` and `ctx.source()`. Provides access to an upstream re ## Loader dependencies -Loaders can depend on other loaders, tasks and assets using `depends_on`. Dependencies are executed first. An upstream loader's destination relation is available via `ctx.loader()`: +Loaders can depend on other loaders and assets using `depends_on`. Dependencies are executed first. An upstream loader's destination relation is available via `ctx.loader()`: ```python from sqlbuild.loaders import loader @@ -317,25 +317,25 @@ def raw_account_metrics(ctx: LoaderContext) -> list[dict[str, object]]: ] ``` -Read upstream relations through `ctx.loader(...)` and `ctx.source(...)`, not by writing their table names in SQL. SQLBuild rejects literal SQL that names a project source, model, or seed with `P008`, and warns at run time when dynamic SQL does; loaders cannot read models and seeds. To use model data in a loader, read it in an upstream task and pass it on through the task's result. See [Hard-coded relation names](sql-references.md#hard-coded-relation-names). +Read upstream relations through `ctx.loader(...)` and `ctx.source(...)`, not by writing their table names in SQL. SQLBuild rejects literal SQL that names a project source, model, or seed with `P008`, and warns at run time when dynamic SQL does; loaders cannot read models and seeds. To use model data in a loader, read it in an upstream asset and pass it on through the asset's result. See [Hard-coded relation names](sql-references.md#hard-coded-relation-names). Dependencies form a DAG. SQLBuild schedules loaders in topological order and executes independent loaders concurrently when `--concurrency` is set. -### Depending on tasks and assets +### Depending on assets -A loader reads an upstream task's or asset's result with `ctx.result_of(...)`; `ctx.loader(...)` -takes loaders only. A failed or hard-skipped upstream task or asset skips the loader and everything -downstream of the source it fills. Otherwise the loader runs when a task or asset it depends on, +A loader reads an upstream asset's result with `ctx.result_of(...)`; `ctx.loader(...)` +takes loaders only. A failed or hard-skipped upstream asset skips the loader and everything +downstream of the source it fills. Otherwise the loader runs when an asset it depends on, directly or through other loaders, succeeded; when all of them skip softly or did not run, only the load is skipped, so the source keeps its last loaded data and its readers still build when another -of their inputs ran. A loader selected without its upstream task or asset reads that upstream's +of their inputs ran. A loader selected without its upstream asset reads that upstream's latest successful result, and fails naming the upstream if it never succeeded. ```python from sqlbuild.loaders import loader -from sqlbuild.tasks import task +from sqlbuild.assets import asset -@task +@asset def fetch_partner_scores(ctx): """Download the scores a partner published.""" return ctx.result(payload={"scores": [[1, 10], [2, 20]]}) @@ -347,9 +347,9 @@ def partner_scores(ctx): return [{"order_id": order_id, "score": score} for order_id, score in scores] ``` -When the task or asset reads SQL, directly or through other Python nodes, the loader makes a +When the asset reads SQL, directly or through other Python nodes, the loader makes a [warehouse round trip](overview.md#warehouse-round-trips), which the project -must allow with `allow_warehouse_round_trips`. The loader then loads after the SQL the task reads +must allow with `allow_warehouse_round_trips`. The loader then loads after the SQL the asset reads and before the models that read its source, in the same `sqb build`. Intermediate loaders (those referenced only via `depends_on`, with no managed source of the same name) are given synthetic source entries and write to `__loader__` tables by default. Only the terminal loader - the one whose name matches a managed source - populates that source; intermediate loaders feed it. Use the `destination` parameter on the decorator to override the intermediate relation: @@ -372,7 +372,7 @@ The `@loader` decorator accepts optional parameters that can also be set in the | Parameter | Description | |-----------|-------------| -| `depends_on` | List of loader, task and asset functions this loader depends on. Depending on one that reads SQL is a [warehouse round trip](overview.md#warehouse-round-trips) | +| `depends_on` | List of loader and asset functions this loader depends on. Depending on one that reads SQL is a [warehouse round trip](overview.md#warehouse-round-trips) | | `destination` | Override the destination relation name (can include schema or database) | | `write_strategy` | `table`, `append`, `delete_insert`, or `merge` | | `cursor_column` | Column used for incremental cursor tracking | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md index 1d108783cb..18af3e1fca 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md @@ -2,7 +2,7 @@ # Overview -> Tasks, assets, loaders, and checks as first-class nodes in the SQLBuild graph. +> Loaders, assets, and checks as first-class nodes in the SQLBuild graph. Online: https://sqlbuild.com/docs/concepts/python-nodes/overview/ @@ -21,20 +21,18 @@ Online: https://sqlbuild.com/docs/concepts/python-nodes/overview/ Python nodes let your project grow beyond warehouse-only SQL while keeping the SQL graph clean. They are ordinary Python functions, decorated to become nodes in the same DAG as your SQL models, and they run as part of `sqb build`. -There are four kinds. The decorator says what each function is, so all of them live under one `python/` folder: +There are three kinds. The decorator says what each function is, so all of them live under one `python/` folder: | Kind | Decorator | Purpose | |------|-----------|---------| | Loader | `@loader` | Load external data into a managed source | -| Task | `@task` | Run Python computation or side effects | -| Asset | `@asset` | Produce or observe an external artifact | -| Check | `@check` | Validate tasks, assets, and loaders | +| Asset | `@asset` | Run Python computation or side effects, or produce an external artifact | +| Check | `@check` | Validate assets and loaders | -All four share the same decorator conventions, dependency model, selection syntax, and runtime context helpers. The pages for each kind cover their specifics: +All three share the same decorator conventions, dependency model, selection syntax, and runtime context helpers. The pages for each kind cover their specifics: - [Loaders](loaders.md) - load data into managed sources -- [Tasks](tasks.md) - Python computation and side effects -- [Assets](assets.md) - external artifacts +- [Assets](assets.md) - Python computation, side effects and external artifacts - [Checks](checks.md) - Python validations - [Factories](factories.md) - generate nodes programmatically with `@factory` - [SQL references](sql-references.md) - read SQL models and sources from Python @@ -43,7 +41,7 @@ Nodes can also be generated programmatically with [`@factory`](factories.md) ins ## The `python/` folder -Anything that runs as part of the pipeline graph goes in `python/`: loaders, tasks, assets, checks, [factories](factories.md), and the helper modules they import. SQLBuild imports every `.py` file under `python/` recursively (except `__init__.py`) and registers each decorated function by its decorator. Undecorated functions and helper-only modules are allowed and never become nodes. +Anything that runs as part of the pipeline graph goes in `python/`: loaders, assets, checks, [factories](factories.md), and the helper modules they import. SQLBuild imports every `.py` file under `python/` recursively (except `__init__.py`) and registers each decorated function by its decorator. Undecorated functions and helper-only modules are allowed and never become nodes. How you organize files inside `python/` is up to you. The playgrounds group nodes by kind: @@ -51,13 +49,13 @@ How you organize files inside `python/` is up to you. The playgrounds group node python/ _helpers.py # shared helpers, not nodes loaders/orders.py - tasks/orders.py + assets/orders_inputs.py assets/orders_export.py checks/orders_export.py factories/regional.py ``` -Grouping by domain (`python/orders/`, `python/customers/`) works just as well, and a single file may mix kinds. A project that selects the [Python Rules](../rules/python-rules.md#laying-out-python) commits to grouping by domain, with each node in its own [unit folder](../rules/python-rules.md#writing-nodes-as-units) holding `node.py` and `main.py`: `SQBRPYTHON001` reports node-kind folders such as `python/loaders/` and modules directly under `python/`, and `SQBRPYTHON012` reports nodes declared outside a unit's `node.py`. Import other project modules through the `python` package, for example `from python.tasks.orders import export_orders` or `from python._helpers import normalize_status`. +Grouping by domain (`python/orders/`, `python/customers/`) works just as well, and a single file may mix kinds. A project that selects the [Python Rules](../rules/python-rules.md#laying-out-python) commits to grouping by domain, with each node in its own [unit folder](../rules/python-rules.md#writing-nodes-as-units) holding `node.py` and `main.py`: `SQBRPYTHON001` reports node-kind folders such as `python/loaders/` and modules directly under `python/`, and `SQBRPYTHON012` reports nodes declared outside a unit's `node.py`. Import other project modules through the `python` package, for example `from python.assets.orders_inputs import prepare_raw_orders` or `from python._helpers import normalize_status`. Code that extends SQLBuild itself keeps its own folder, such as `rules/`, `materializations/`, `adapters/`, `providers/`, `sinks/`, `hooks/python/`, `functions/python/`, and `macros/`. Python outside the supported locations fails compilation with `D016`; see [Project Python](../project-configuration.md#project-python) for the full list. @@ -66,20 +64,20 @@ Code that extends SQLBuild itself keeps its own folder, such as `rules/`, `mater The most important rule: **SQL models never depend on Python nodes.** The dependency direction is strictly one way. - SQL models depend only on other SQL resources (models, sources, seeds, functions). -- Python nodes may depend on other Python nodes; a loader may depend on any Python node, including tasks and assets. +- Python nodes may depend on other Python nodes; a loader may depend on any Python node, including assets. - Python data reaches SQL only through a **loader filling a source**: `loader -> source -> model`. -- Tasks, assets, and checks may **read** SQL models, sources, and seeds at runtime through typed references (see [SQL references](sql-references.md)), but reading a model does not make it a SQL dependency. +- Assets and checks may **read** SQL models, sources, and seeds at runtime through typed references (see [SQL references](sql-references.md)), but reading a model does not make it a SQL dependency. This keeps the SQL graph fully analyzable and testable on its own, while letting Python participate around the edges. ``` depends_on - task ----------------> task + asset ---------------> asset | | | (read only) v | asset v | - source <-- loader | check (validates tasks/assets/loaders/SQL) + source <-- loader | check (validates assets/loaders/SQL) | v model (SQL) ----> model (SQL) @@ -87,19 +85,19 @@ This keeps the SQL graph fully analyzable and testable on its own, while letting ### Warehouse round trips -A loader that depends on a task or asset that reads SQL, directly or through other Python nodes, +A loader that depends on an asset that reads SQL, directly or through other Python nodes, makes a warehouse round trip: data leaves the warehouse and comes back through the loader. For -example, a task reads `orders`, sends them to a partner's scoring service, and a loader loads the +example, an asset reads `orders`, sends them to a partner's scoring service, and a loader loads the scores it returned into the `partner_scores` source that `scored_orders` reads: ``` - model orders --> task score_orders --> loader partner_scores --> source partner_scores --> model scored_orders + model orders --> asset score_orders --> loader partner_scores --> source partner_scores --> model scored_orders ``` Round trips are off by default. Turn them on with `allow_warehouse_round_trips` in `sqlbuild_project.toml` (see [Settings](../project-configuration.md#settings)); without it, -a plan or build that includes such a loader stops with an error naming the loader and the task or -asset that reads SQL. A loader that depends on a task or asset that reads no SQL is ordinary +a plan or build that includes such a loader stops with an error naming the loader and the asset +that reads SQL. A loader that depends on an asset that reads no SQL is ordinary ingestion and is always allowed. ```toml @@ -109,23 +107,23 @@ allow_warehouse_round_trips = true With the setting on: -- One `sqb build` runs the SQL the task reads, then the task, then the loader, then the models that - read the loader's source. The loader follows the usual rule for its upstream tasks and assets: if +- One `sqb build` runs the SQL the asset reads, then the asset, then the loader, then the models that + read the loader's source. The loader follows the usual rule for its upstream assets: if one fails or skips with `mode="hard"`, the loader and everything downstream of its source are skipped; otherwise it runs when any of them succeeded. When all of them skip softly, only the load is skipped: the source keeps its last loaded data, and models reading it still build when another of their inputs ran. - Lineage and column lineage restart at the source: the source's columns come from the loader, not - from the models the task read. -- Job groups run a member that builds the task before the member that runs the loader, and that one + from the models the asset read. +- Job groups run a member that builds the asset before the member that runs the loader, and that one before the members that build models reading its source. -- `--changed` follows the chain: when the SQL the task reads is rebuilt, the task and loader run and - the models reading the source rebuild. When it is not rebuilt, the task does not run and the +- `--changed` follows the chain: when the SQL the asset reads is rebuilt, the asset and loader run and + the models reading the source rebuild. When it is not rebuilt, the asset does not run and the loader is skipped softly, so the source keeps its data and models selected for their own changes still build. -- A task that reads anything built from the source its own downstream loader fills is a cycle, +- An asset that reads anything built from the source its own downstream loader fills is a cycle, which stops the plan or build with the path, such as - `loader 'partner_scores' -> task 'score_orders' -> model 'scored_orders' -> source 'partner_scores' -> loader 'partner_scores'`. + `loader 'partner_scores' -> asset 'score_orders' -> model 'scored_orders' -> source 'partner_scores' -> loader 'partner_scores'`. ## Decorators @@ -139,7 +137,7 @@ Every decorator accepts the same organizational metadata: | `description` | Required human-readable docs; defaults to the function docstring ([Descriptions](../models/configuration.md#descriptions)) | | `meta` | Freeform JSON metadata for catalogs and integrations | -`@task` and `@asset` also accept a `retry` policy. `@asset` additionally accepts `columns` and `column_lineage`. Node kind is inferred from the decorator - you never pass `kind=`. +`@asset` also accepts a `retry` policy, `columns` and `column_lineage`. Node kind is inferred from the decorator - you never pass `kind=`. ## Identity tracking @@ -149,7 +147,7 @@ Unlike SQL models, Python nodes may depend on external inputs the framework cann ## Runtime context -Each node receives a context object as its first argument (`TaskContext`, `AssetContext`, `CheckContext`, or `LoaderContext`). They share these helpers: +Each node receives a context object as its first argument (`AssetContext`, `CheckContext`, or `LoaderContext`). They share these helpers: | Helper | Description | |--------|-------------| @@ -167,7 +165,7 @@ Each node receives a context object as its first argument (`TaskContext`, `Asset | `ctx.results_of(node_fn, limit=N)` | Read the last N successful results of an upstream node, newest first | | `ctx.providers` | Access discovered [providers](providers.md) by name | -Task and asset contexts add `ctx.result(...)` and `ctx.skip(...)`. Check contexts add `ctx.pass_(...)`, `ctx.fail(...)`, and `ctx.warn(...)`. +Asset contexts add `ctx.result(...)` and `ctx.skip(...)`. Check contexts add `ctx.pass_(...)`, `ctx.fail(...)`, and `ctx.warn(...)`. Providers can also be injected directly as function parameters by name. See [Providers](providers.md) for details. @@ -197,17 +195,17 @@ Connection settings and secrets are not part of `ctx.target_config`. ## Returns and skips -Tasks and assets return through `ctx.result(...)`: +Assets return through `ctx.result(...)`: ```python -@task +@asset def export_orders(ctx): """Export orders.""" return ctx.result(payload={"rows": 120}, metadata={"rows": 120}) ``` - A plain value or `None` is also accepted and normalized to a successful result. -- `ctx.skip(reason, mode=...)` skips the node. `mode` accepts `"soft"` (default, skips only this node) or `"hard"` (also blocks dependents), as a string or the `SkipMode` enum from `sqlbuild.tasks`/`sqlbuild.assets`. +- `ctx.skip(reason, mode=...)` skips the node. `mode` accepts `"soft"` (default, skips only this node) or `"hard"` (also blocks dependents), as a string or the `SkipMode` enum from `sqlbuild.assets`. - Assets may pass `materialized=True`/`False` to record whether an artifact was produced. Within one run, downstream nodes run only if at least one upstream succeeded. If all upstreams are skipped, the downstream is skipped. A failed or hard-skipped upstream blocks its dependents. @@ -218,7 +216,7 @@ Node results (payload, metadata, status, errors) are persisted after each execut `_sqlbuild_node_results` table. Results persist across runs, so they are available for observability, debugging, and downstream consumption through `ctx.result_of`. -Results of loaders, tasks, assets and checks all go to one schema, chosen by the target's `schema`: +Results of loaders, assets and checks all go to one schema, chosen by the target's `schema`: | Target `schema` | Results table schema | |-----------------|----------------------| @@ -234,37 +232,36 @@ Python nodes are selected like SQL resources, by bare name or typed selector: ```bash sqb build --select export_orders # bare name -sqb build --select task:export_orders # typed -sqb build --select asset:orders_export +sqb build --select asset:orders_export # typed sqb build --select export_orders check:check_orders_exported # a check with the node it checks sqb build --select tag:exports # by tag sqb build --select +orders_export # with upstreams ``` -Names are globally unique across models, sources, seeds, functions, loaders, tasks, assets, checks, providers, hooks, unit tests, scenarios, jobs, job groups, sensors, and calendars. +Names are globally unique across models, sources, seeds, functions, loaders, assets, checks, providers, hooks, unit tests, scenarios, jobs, job groups, sensors, and calendars. ## Lifecycle: run, build, check Python nodes run in two phases relative to SQL: -- **Ingress (pre-SQL):** loaders, and tasks/assets that feed sources, run before SQL models are built. -- **Read-side (post-SQL):** tasks/assets that read SQL run after their SQL dependencies are built. +- **Ingress (pre-SQL):** loaders, and assets that feed sources, run before SQL models are built. +- **Read-side (post-SQL):** assets that read SQL run after their SQL dependencies are built. A loader in a [warehouse round trip](#warehouse-round-trips) loads after them, and before the models that read its source. -Read-side tasks and assets follow the same selection rule as SQL models. A SQL dependency that is +Read-side assets follow the same selection rule as SQL models. A SQL dependency that is part of the run gates the node: it waits for that model, seed, or source load, and is skipped if the dependency fails or is skipped. A dependency that is not part of the run (not selected, or an -unmanaged source) is read as it currently exists, so `sqb build --select task:export_orders` runs -the task against the relations already in the warehouse. +unmanaged source) is read as it currently exists, so `sqb build --select asset:export_orders` runs +the asset against the relations already in the warehouse. -Python dependencies follow the same rule. A task or asset selected without its Python upstream +Python dependencies follow the same rule. An asset selected without its Python upstream reads that upstream's latest successful result through `ctx.result_of`. If the upstream has never succeeded for the target, the node fails with an error that names the upstream, for example: ```text -Upstream task 'prepare_orders' has no successful result to read. Build it first with -'sqb build --select task:prepare_orders', or build both with 'sqb build --select +asset:export_orders'. +Upstream asset 'prepare_orders' has no successful result to read. Build it first with +'sqb build --select asset:prepare_orders', or build both with 'sqb build --select +asset:export_orders'. ``` Jobs, sensors, roles and ad-hoc runs on the server resolve selectors the same way, so a job can @@ -272,8 +269,8 @@ select one Python node and a job group orders it after the job that builds its u The commands differ in what they include by default: -| Command | SQL | Loaders / tasks / assets | Checks | Audits | -|---------|-----|--------------------------|--------|--------| +| Command | SQL | Loaders / assets | Checks | Audits | +|---------|-----|-----------------|--------|--------| | `sqb build` | Yes | Yes | Yes | Yes | | `sqb build --no-tests --no-audits` | Yes | Yes | No | No | | `sqb check` | No | No | Selected checks only | No | @@ -282,11 +279,11 @@ The commands differ in what they include by default: - `sqb build --no-tests --no-audits` executes the DAG without validation, for fast iteration. - `sqb check` runs Python checks only. See [`sqb check`](../../cli/check.md). -Use `--no-python` on `plan` and `build` to suppress read-side tasks/assets. Loader-side Python required to populate selected sources still runs (use `--no-load` to skip source loading). See the [`sqb build`](../../cli/build.md) reference. +Use `--no-python` on `plan` and `build` to suppress read-side assets. Loader-side Python required to populate selected sources still runs (use `--no-load` to skip source loading). See the [`sqb build`](../../cli/build.md) reference. ## Try it -The `python_nodes` playground is a small working project with a task, loader, model, asset, and check: +The `python_nodes` playground is a small working project with assets, a loader, a model, and a check: ```bash sqb playground --template python_nodes diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/providers.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/providers.md index 23bb7bb11b..1467c2bc3d 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/providers.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/providers.md @@ -63,16 +63,16 @@ class WarehouseClient(Provider): api_key: str ``` -Provider names must be valid Python identifiers (`lower_snake_case`). They share the global project resource namespace with models, sources, seeds, functions, loaders, tasks, assets, checks, and hooks. +Provider names must be valid Python identifiers (`lower_snake_case`). They share the global project resource namespace with models, sources, seeds, functions, loaders, assets, checks, and hooks. ## Using providers in Python nodes Providers are injected into Python node functions by **parameter name**. Add a parameter whose name matches the provider's runtime name: ```python -from sqlbuild.tasks import task +from sqlbuild.assets import asset -@task +@asset def export_orders(ctx, warehouse_client): """Export orders.""" warehouse_client.session.upload(ctx.query("SELECT * FROM orders")) @@ -83,10 +83,10 @@ SQLBuild matches the parameter name `warehouse_client` to the discovered provide You can also type-annotate the parameter for IDE support and compile-time validation: ```python -from sqlbuild.tasks import task +from sqlbuild.assets import asset from providers.warehouse_client.provider import WarehouseClient -@task +@asset def export_orders(ctx, warehouse_client: WarehouseClient): """Export orders.""" warehouse_client.session.upload(ctx.query("SELECT * FROM orders")) @@ -97,7 +97,6 @@ When a type annotation is present, SQLBuild validates that the discovered provid Provider injection works in all Python node types: - **Loaders** (`@loader`) -- **Tasks** (`@task`) - **Assets** (`@asset`) - **Checks** (`@check`) @@ -135,10 +134,10 @@ def notify_complete(ctx): ## Using providers via context -All Python node contexts (`TaskContext`, `AssetContext`, `CheckContext`, `LoaderContext`) and `HookContext` expose a `ctx.providers` container for name-based access: +All Python node contexts (`AssetContext`, `CheckContext`, `LoaderContext`) and `HookContext` expose a `ctx.providers` container for name-based access: ```python -@task +@asset def export_orders(ctx): """Export orders.""" client = ctx.providers.warehouse_client @@ -289,7 +288,7 @@ provider with its class and docstring. A provider's page shows: | Tab | Shows | | --- | --- | | Settings | Each setting's type, environment variable and whether it is required; secret settings are marked, and flagged **missing** when the location's environment doesn't set them | -| Used by | The loaders, tasks, hooks and other code that take the provider, and the jobs that run that code | +| Used by | The loaders, assets, hooks and other code that take the provider, and the jobs that run that code | Setting values, defaults included, are never published. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/sql-references.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/sql-references.md index 44a6114391..bddf27ac70 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/sql-references.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/sql-references.md @@ -19,23 +19,23 @@ Python nodes can **read** SQL models, sources, and seeds at runtime, but SQL can ## Declaring a reference -Use `model()`, `source()`, and `seed()` from `sqlbuild.refs` in the `depends_on` of a task, asset, or check: +Use `model()`, `source()`, and `seed()` from `sqlbuild.refs` in the `depends_on` of an asset or check: ```python from sqlbuild.refs import model, seed, source -from sqlbuild.tasks import task +from sqlbuild.assets import asset -@task(depends_on=model("fact_orders")) +@asset(depends_on=model("fact_orders")) def export_orders(ctx): """Export orders.""" ... -@task(depends_on=source("raw_orders")) +@asset(depends_on=source("raw_orders")) def inspect_raw(ctx): """Inspect raw.""" ... -@task(depends_on=seed("country_codes")) +@asset(depends_on=seed("country_codes")) def export_countries(ctx): """Export countries.""" ... @@ -48,14 +48,14 @@ Declaring the reference does two things: It does **not** make the SQL resource depend on the Python node. The dependency is one-way: Python reads SQL, never the reverse. -The reference is a graph edge. It appears in `sqb dag --json`, and selector expansion follows it: `+task:export_countries` also selects the `country_codes` seed and its upstream, and `country_codes+` also selects the tasks and assets that read it. Checks are never added by expansion; they run when their dependencies run. +The reference is a graph edge. It appears in `sqb dag --json`, and selector expansion follows it: `+asset:export_countries` also selects the `country_codes` seed and its upstream, and `country_codes+` also selects the assets that read it. Checks are never added by expansion; they run when their dependencies run. ## Resolving with ctx.relation Always resolve a reference to its concrete relation with `ctx.relation(...)` instead of hardcoding the table name: ```python -@task(depends_on=model("fact_orders")) +@asset(depends_on=model("fact_orders")) def export_orders(ctx): """Export orders.""" relation = ctx.relation(model("fact_orders")) @@ -69,7 +69,7 @@ Do not hardcode model or source names in raw SQL. A model's relation depends on ## Hard-coded relation names -SQLBuild checks SQL passed to `ctx.query()` and `ctx.execute_sql()` in tasks, assets, checks, +SQLBuild checks SQL passed to `ctx.query()` and `ctx.execute_sql()` in assets, checks, loaders, and [Python hooks](../models/hooks/python.md) for names of project relations: models, sources, and seeds. @@ -78,8 +78,8 @@ literal text, is parsed with the project's SQL dialect. Naming a project relatio with `P008`: ``` -error[P008]: task:export_orders names model:fact_orders as 'fact_orders' in SQL passed to ctx.query() - --> python/tasks/export.py:9 +error[P008]: asset:export_orders names model:fact_orders as 'fact_orders' in SQL passed to ctx.query() + --> python/exports/export.py:9 = help: declare it with depends_on=model("fact_orders") and use ctx.relation(model("fact_orders")) instead of the relation name = help: if 'fact_orders' is an external table that shares the name, qualify it with its schema = help: while migrating a project, allow hard-coded relation names with [references] enforce_explicit = false in sqlbuild_project.toml @@ -127,15 +127,15 @@ Naming every upstream model ties a node to model names that change with each ren ```python from sqlbuild.refs import job, tag -from sqlbuild.tasks import task +from sqlbuild.assets import asset -@task(depends_on=(job("orders_job", exclude="tag:experimental"), tag("finance_inputs"))) +@asset(depends_on=(job("orders_job", exclude="tag:experimental"), tag("finance_inputs"))) def publish_revenue(ctx): """Publish revenue.""" ... ``` -- `job(name)` depends on the job's **end nodes**: the models, seeds, sources, loaders, tasks and +- `job(name)` depends on the job's **end nodes**: the models, seeds, sources, loaders and assets of its selection that nothing else in the selection reads. Everything else in the job is upstream of an end node, so this waits for the whole job without listing every edge. A job group name depends on the union of its jobs. @@ -144,7 +144,7 @@ def publish_revenue(ctx): `kind:`) and removes those nodes before the set is reduced to its end nodes. Compile resolves each into ordinary edges to the end nodes, so planning, `--changed`, selector -expansion (`+task:publish_revenue`), `sqb dag` and job group waits treat them like named +expansion (`+asset:publish_revenue`), `sqb dag` and job group waits treat them like named dependencies. A model named directly and also reached through a job is one edge. Jobs resolve whatever targets they are declared for. @@ -156,7 +156,7 @@ it was last built with: ```text Python read-side (1) -└── publish_revenue task (unchanged) +└── publish_revenue asset (unchanged) └── publish_revenue now also waits on model 'refund_totals' through job 'orders_job' ``` @@ -171,7 +171,7 @@ Compile fails when: by job, so if the dependant's job waits for the job it depends on while that job reads a node of the dependant's job, neither can start. The error names the nodes that close the loop; move them to another job; -- a loader declares one: loaders depend on loaders, tasks and assets named one by one; +- a loader declares one: loaders depend on loaders and assets named one by one; - the name has a code location prefix such as `job("billing:nightly_job")`, which is reserved for depending on other locations' jobs. @@ -187,7 +187,7 @@ Compile fails when: ## Where references are allowed -- **Tasks**, **assets**, and **checks** may declare `model()`, `source()`, and `seed()` references and read them with `ctx.relation(...)`, and depend on a `job()` or `tag()`. +- **Assets** and **checks** may declare `model()`, `source()`, and `seed()` references and read them with `ctx.relation(...)`, and depend on a `job()` or `tag()`. - **Python hooks** declare the models, sources, and seeds they read with `@hook(reads=...)`; see [Python Hooks](../models/hooks/python.md#declared-reads). - **Loaders** may not reference SQL resources; they read sources with `ctx.source(...)` and upstream loaders with `ctx.loader(...)`. - A Python node referencing a SQL resource never turns into a SQL dependency; selector expansion will not pull Python outputs into SQL model dependencies. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/tasks.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/tasks.md deleted file mode 100644 index 73d67cd81d..0000000000 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/tasks.md +++ /dev/null @@ -1,150 +0,0 @@ - - -# Tasks - -> Run Python computation and side effects as nodes in the SQLBuild graph. - -Online: https://sqlbuild.com/docs/concepts/python-nodes/tasks/ - -## Contents - -- Defining a task -- Dependencies -- Returning results -- Skipping -- Retries -- Decorator parameters -- Running tasks - -Tasks are Python functions that run as part of the DAG. Use them for computation, side effects, and orchestration steps that are not loading data into a source and not producing a tracked artifact. See [Python Nodes](overview.md) for the shared model and the SQL boundary rules. - -## Defining a task - -Place Python files under `python/` (the playgrounds use `python/tasks/`) and decorate functions with `@task`: - -```python -# python/tasks/orders.py -from sqlbuild.tasks import task, TaskContext - -@task -def export_orders(ctx: TaskContext): - """Export orders.""" - rows = fetch_orders() - return ctx.result(payload={"rows": len(rows)}, metadata={"rows": len(rows)}) -``` - -The task receives a `TaskContext` and returns through `ctx.result(...)`. A plain return value or `None` is also accepted and normalized to a successful result. - -## Dependencies - -Tasks declare dependencies with `depends_on`, accepting a single function, a tuple, or a list: - -```python -@task -def fetch_orders(ctx: TaskContext): - """Fetch orders.""" - return ctx.result(payload=download_orders()) - -@task(depends_on=fetch_orders) -def summarize_orders(ctx: TaskContext): - """Summarize orders.""" - result = ctx.result_of(fetch_orders) - return ctx.result(metadata={"count": len(result.payload)}) -``` - -`ctx.result_of(node_fn)` reads the latest persisted result of an upstream node, returning a `NodeResultEnvelope` with `payload`, `metadata`, `status`, and `ts` fields. Results persist across runs. Use `ctx.results_of(node_fn, limit=N)` to read result history. Reading a missing or unsuccessful upstream raises unless you pass `default=`. - -Tasks may depend on other tasks, assets, and loaders. They may **not** depend on SQL models or sources as graph dependencies, but they can read them at runtime with typed references - see [SQL references](sql-references.md). - -## Returning results - -```python -@task -def build_export(ctx: TaskContext): - """Build export.""" - return ctx.result( - payload={"path": "/exports/orders.csv"}, - metadata={"rows": 1200}, - ) -``` - -- `payload` is the value downstream nodes read with `ctx.result_of(...)`. -- `metadata` is structured JSON for catalogs and downstream reads. -- Tasks cannot set `materialized` - that is for [assets](assets.md). - -## Skipping - -Return `ctx.skip(...)` to skip a task: - -```python -@task -def export_if_present(ctx: TaskContext): - """Export if present.""" - if not new_files_available(): - return ctx.skip("no new files") - return ctx.result(payload=do_export()) -``` - -- `"soft"` (default) skips only this task; dependents may still run if another upstream succeeded. -- `"hard"` skips this task and blocks its dependents. - -`mode` accepts either a plain string or the `SkipMode` enum: - -```python -from sqlbuild.tasks import task, SkipMode - -@task -def optional_step(ctx): - """Optional step.""" - return ctx.skip("nothing to do", mode=SkipMode.HARD) # or mode="hard" -``` - -## Retries - -`@task` accepts a `retry` policy for transient failures: - -```python -from sqlbuild.retries import RetryPolicy -from sqlbuild.tasks import task - -@task(retry=RetryPolicy(max_attempts=3, retry_on=(ConnectionError,))) -def call_api(ctx): - """Call api.""" - return ctx.result(payload=fetch_from_flaky_api()) -``` - -| Field | Default | Description | -|-------|---------|-------------| -| `max_attempts` | `3` | Total attempts including the first | -| `retry_on` | `Exception` | Exception class, tuple, or list to retry on | -| `initial_delay_seconds` | `1.0` | Delay before the first retry | -| `backoff_multiplier` | `2.0` | Exponential backoff multiplier | -| `max_delay_seconds` | `30.0` | Cap on any single delay | -| `max_elapsed_seconds` | `None` | Overall time bound for retries | -| `jitter` | `True` | Randomize delays to avoid synchronized retries | - -The default is no retry. Set `retry_on` explicitly rather than relying on the broad default when you can. The original exception is preserved if all attempts fail. - -## Decorator parameters - -| Parameter | Description | -|-----------|-------------| -| `name` | Override the node name (defaults to the function name) | -| `depends_on` | Upstream nodes (function, tuple, or list); `model()`/`source()`/`seed()` for read-only SQL refs | -| `tags` | Labels for selection and grouping | -| `group` | Display/catalog grouping | -| `description` | Docs (defaults to docstring) | -| `meta` | Freeform JSON metadata | -| `retry` | A `RetryPolicy` | - -## Running tasks - -```bash -# Run a task and its required graph -sqb build --select export_orders --no-tests --no-audits - -# Include in a full build -sqb build --select export_orders -``` - -Tasks run during `sqb build`. They are not validated by checks unless you also write a [check](checks.md) that depends on them. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/orchestration-rules.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/orchestration-rules.md index faf94c02b1..5792ad5b06 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/orchestration-rules.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/orchestration-rules.md @@ -38,7 +38,7 @@ The Rule reads the `select` of every [job](../orchestration.md#declare-a-job) an | Atom | Counts | | --- | --- | | A bare name, such as `order_totals` | Yes | -| A typed name, such as `source:raw_orders` or `task:refund_review` | Yes | +| A typed name, such as `source:raw_orders` or `asset:refund_review` | Yes | | Each name in an intersection, such as `order_totals,tag:finance` | The name only | | `tag:`, a folder path, `job:`, a job or job group name | No | | A name with `+` before or after it | No | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/python-rules.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/python-rules.md index 061fb110f3..a5b92341ce 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/python-rules.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/python-rules.md @@ -83,7 +83,7 @@ python/ __init__.py fulfillment/ shipment_export/ - node.py # @task shipment_export + node.py # @asset shipment_export main.py # main(), the unit's implementation utils/ currency/ @@ -116,7 +116,7 @@ python/orders/ checks.py # @check declarations for this unit _helpers/ # private functions for this unit refund_review/ - node.py # @task + node.py # @asset main.py ``` @@ -141,21 +141,21 @@ These are reported: | Code | Finding | Fix | |---|---|---| | `SQBRPYTHON003` | A unit without `main.py`, or another module or folder in it | Add `main.py`; move other code into `_helpers/` or a role file | -| `SQBRPYTHON004` | `node.py` holding a helper, a constant, a class or a second declaration | Keep one `@loader`, `@task`, `@asset` or `@factory` function and its imports | +| `SQBRPYTHON004` | `node.py` holding a helper, a constant, a class or a second declaration | Keep one `@loader`, `@asset` or `@factory` function and its imports | | `SQBRPYTHON005` | A declaration that does more than `return main(...)`, or passes a positional argument | Move the logic into `main()` and pass every argument as `name=value` | | `SQBRPYTHON006` | `main.py` holding more than one function, or a `main()` whose return annotation does not fit the node | Move other code into `_helpers/` and annotate `main()` with its result | | `SQBRPYTHON007` | A signature other than `(ctx: Context, *, provider: ProviderClass)` | Take `ctx` positionally and providers after `*`, each annotated | | `SQBRPYTHON008` | An import of another unit's `main.py`, role files or `_helpers/`, or another domain's code | Import shared code from `python/utils/` or the domain's role files | | `SQBRPYTHON009` | `checks.py` holding other code, or a check that does not depend on the unit's node | Name the unit's node in `depends_on` and return directly | | `SQBRPYTHON010` | A factory that does not loop over the unit's specs, or a generated node without `name=` | Loop over specs from the unit's `constants.py` or `models.py` and name each node | -| `SQBRPYTHON011` | A docstring on a node or check, a task, asset or check without `description=`, or a described loader | Use `description=`; a loader takes its description from its managed source | +| `SQBRPYTHON011` | A docstring on a node or check, an asset or check without `description=`, or a described loader | Use `description=`; a loader takes its description from its managed source | | `SQBRPYTHON012` | A node declared outside a unit's `node.py`, or a check outside its `checks.py` | Move it into its own unit folder | `main()` returns: | Node | Return annotation | |---|---| -| `@task`, `@asset` | `PythonNodeResult` or `PythonNodeSkipResult`, from `ctx.result(...)` and `ctx.skip(...)` | +| `@asset` | `PythonNodeResult` or `PythonNodeSkipResult`, from `ctx.result(...)` and `ctx.skip(...)` | | `@loader` | rows such as `list[...]` or `Iterator[...]`, `LoaderResult`, `LoaderSkipResult`, or `None` | A unit may import another unit's `node.py` only to name that node in `depends_on`. It may import @@ -338,7 +338,7 @@ The other size limits check every function. ## Unreachable definitions `SQBRPYTHON306` reports functions, classes and module values that nothing SQLBuild loads can reach. -SQLBuild loads every decorated function under `python/` (`@loader`, `@task`, `@asset`, `@check`, +SQLBuild loads every decorated function under `python/` (`@loader`, `@asset`, `@check`, `@factory` and `@audit_factory`) and everything defined outside `python/`, such as hooks and providers. A definition reached only through them is live. Imports through folders without an `__init__.py` count too. @@ -405,7 +405,7 @@ remediation. | `SQBRPYTHON008` | node-import-boundaries | `python/` | yes | A unit imports only what it owns, its domain's role files, `python/utils/`, provider role files and other nodes for `depends_on` | | `SQBRPYTHON009` | checks-module-shape | `python/` | yes | `checks.py` holds checks of the unit's node that return directly, or one factory | | `SQBRPYTHON010` | factory-shape | `python/` | yes | A factory loops over its unit's specs and names every node it generates | -| `SQBRPYTHON011` | node-description-not-docstring | `python/` | yes | Tasks, assets and checks set `description=` without a docstring; loaders set neither | +| `SQBRPYTHON011` | node-description-not-docstring | `python/` | yes | Assets and checks set `description=` without a docstring; loaders set neither | | `SQBRPYTHON012` | node-declared-in-unit | `python/` | yes | Nodes are declared in a unit's `node.py`, and checks in its `checks.py` | ### Providers diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/selectors.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/selectors.md index 9a25d769ef..3c8f640eb4 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/selectors.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/selectors.md @@ -71,7 +71,7 @@ sqb build --select models/marts sqb build --select path:models/intermediate ``` -Any name containing `/` is treated as a path selector, so `path:models/marts` and a bare `models/marts` work the same way. Path selectors require an explicit root directory: `models/` for SQL models or `python/` for Python nodes. Nested paths work too: `models/staging/orders` or `python/tasks`. +Any name containing `/` is treated as a path selector, so `path:models/marts` and a bare `models/marts` work the same way. Path selectors require an explicit root directory: `models/` for SQL models or `python/` for Python nodes. Nested paths work too: `models/staging/orders` or `python/exports`. ### Seed and source @@ -90,7 +90,7 @@ sqb build --select "kind:model,path:models/marts" sqb build --select +kind:asset ``` -The kinds are `model`, `source`, `seed`, `udf`, `table_fn`, `task`, `asset`, `loader` and +The kinds are `model`, `source`, `seed`, `udf`, `table_fn`, `asset`, `loader` and `check`, as the server's catalog names them, and `function`, which selects both `udf` and `table_fn`. Kind selectors compose with `+`, `,` and `--exclude` like tag selectors. An unknown kind is an error that names the closest kinds, and a kind your diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/feature-comparison.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/feature-comparison.md index 734bcce0d1..d377b7bbdd 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/feature-comparison.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/feature-comparison.md @@ -87,7 +87,7 @@ SQLBuild, dbt, and SQLMesh are all SQL pipeline frameworks. They share common gr | Feature | SQLBuild | dbt | SQLMesh | |---------|----------|-----|---------| | SQL models | `MODEL()` header with inline config | Jinja-templated SQL + YAML sidecar | `MODEL` DDL | -| Python models | Python nodes (tasks, assets, loaders, checks); no dataframe models | Pandas, PySpark, Snowpark, BigFrames | Pandas, PySpark, Snowpark, BigFrames | +| Python models | Python nodes (assets, loaders, checks); no dataframe models | Pandas, PySpark, Snowpark, BigFrames | Pandas, PySpark, Snowpark, BigFrames | | Custom materializations | Python with full framework hooks | Jinja-based | Python-based custom model kinds | | Lifecycle hooks | Typed inline SQL, reusable parameterized SQL resources, and Python hooks with compile-time validation and `HookContext` | Jinja pre/post hooks | Python pre/post hooks | @@ -95,9 +95,8 @@ SQLBuild, dbt, and SQLMesh are all SQL pipeline frameworks. They share common gr | Feature | SQLBuild | dbt | SQLMesh | |---------|----------|-----|---------| -| Tasks | `@task` - Python computation as DAG nodes | No | No | | Assets | `@asset` - external artifact production/observation | No | No | -| Checks | `@check` - Python validation of tasks, assets, and loaders | No | No | +| Checks | `@check` - Python validation of assets and loaders | No | No | | Factories | `@factory` - programmatic node generation | No | No | | Providers | Shared runtime services with name-based injection into nodes and hooks | No | No | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/dagster-reference.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/dagster-reference.md index 62e40aa8fc..4bd4f118fc 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/dagster-reference.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/dagster-reference.md @@ -250,7 +250,6 @@ Python node asset keys use a two-part key with the node kind as prefix: | Kind | Asset key example | |------|-------------------| -| Task | `("task", "prepare_orders")` | | Asset | `("asset", "orders_export")` | | Loader | `("loader", "raw_orders")` | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/dagster.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/dagster.md index f043f730d8..baa106ccf8 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/dagster.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/dagster.md @@ -19,7 +19,7 @@ Online: https://sqlbuild.com/docs/integrations/dagster/ - Project preparation - Translator -SQLBuild includes a Dagster integration that maps your project's models, sources, seeds, functions, loaders, tasks, assets, tests, audits, scenarios, and Python checks into Dagster assets and asset checks. SQLBuild handles the SQL transformation layer. Dagster handles scheduling, retries, alerting, and the asset-centric UI. +SQLBuild includes a Dagster integration that maps your project's models, sources, seeds, functions, loaders, assets, tests, audits, scenarios, and Python checks into Dagster assets and asset checks. SQLBuild handles the SQL transformation layer. Dagster handles scheduling, retries, alerting, and the asset-centric UI. ## Install @@ -44,7 +44,7 @@ This creates a Dagster code location with a `definitions.py` that includes asset ## How it works 1. `sqb compile --dag` generates a static `sqlbuild_dag.json` artifact with your project's full graph (nodes, edges, checks) -2. `@sqlbuild_assets()` reads the artifact and creates one Dagster `AssetSpec` per source, seed, model, function, loader, task, and asset, with dependency edges preserved +2. `@sqlbuild_assets()` reads the artifact and creates one Dagster `AssetSpec` per source, seed, model, function, loader, and asset, with dependency edges preserved 3. `SqlBuildCliResource` shells out to `sqb build`, `sqb test`, `sqb scenario test`, etc. as subprocesses 4. Execution results (materializations, audit pass/fail, scenario outcomes) are read from SQLBuild's structured integration-result stream and emitted as Dagster `MaterializeResult` and `AssetCheckResult` events @@ -80,7 +80,7 @@ dagster dev -f definitions.py `definitions.py` belongs to your Dagster code location and sits next to the SQLBuild project directory (`my_sqlbuild_project/` here), not inside it. SQLBuild projects contain no orchestrator code. -Dagster discovers every SQLBuild model, loader, task, and asset as a Dagster asset. Selecting a subset in the Dagster UI automatically scopes the `sqb build` invocation to those nodes via `--select`. +Dagster discovers every SQLBuild model, loader, and asset as a Dagster asset. Selecting a subset in the Dagster UI automatically scopes the `sqb build` invocation to those nodes via `--select`. For production deployments, use `project.prepare()` or `sqb compile --dag` to generate the DAG artifact explicitly in your CI pipeline. @@ -104,7 +104,7 @@ SQLBuild tests, audits, scenarios, and Python checks are registered as Dagster a - **Unit tests** become checks attached to the models they test - **Audits** become checks attached to the model, source, or seed they audit, with severity mapped to `AssetCheckSeverity.ERROR` or `AssetCheckSeverity.WARN`. A singular audit that runs at the end of the build becomes a check on each model it reads - **Scenarios** become checks attached to the models they exercise -- **Python checks** (`@check`) become checks attached to the tasks, assets, or loaders they validate +- **Python checks** (`@check`) become checks attached to the assets or loaders they validate Check results are emitted with pass/fail status and metadata from the execution JSON. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/rivers.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/rivers.md index 9760133072..bf41cd67ca 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/rivers.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/rivers.md @@ -17,7 +17,7 @@ Online: https://sqlbuild.com/docs/integrations/rivers/ - SqlBuildRiversTranslator - Running locally -SQLBuild includes a Rivers integration that maps your project's models, sources, seeds, loaders, tasks, assets, and functions into Rivers assets with dependency edges preserved. SQLBuild handles the SQL transformation layer. Rivers handles scheduling, execution, and the asset-centric UI. +SQLBuild includes a Rivers integration that maps your project's models, sources, seeds, loaders, assets, and functions into Rivers assets with dependency edges preserved. SQLBuild handles the SQL transformation layer. Rivers handles scheduling, execution, and the asset-centric UI. ## Install @@ -42,7 +42,7 @@ This creates a Rivers code repository with a `definitions.py` that includes asse ## How it works 1. `sqb compile --dag` generates a static `sqlbuild_dag.json` artifact with your project's full graph (nodes, edges) -2. `@sqlbuild_assets()` reads the artifact and creates one Rivers `AssetDef` per source, loader, seed, model, function, task, and asset, with dependency edges preserved +2. `@sqlbuild_assets()` reads the artifact and creates one Rivers `AssetDef` per source, loader, seed, model, function, and asset, with dependency edges preserved 3. The decorated function runs `sqb build` as a subprocess and yields `Materialization` events for each output ## Quickstart @@ -162,7 +162,7 @@ def my_assets(context): | Method | Default behavior | |--------|-----------------| -| `get_asset_name(node)` | Joins the asset key parts with `__` (e.g. `task__prepare_orders`, `asset__orders_export`) | +| `get_asset_name(node)` | Joins the asset key parts with `__` (e.g. `asset__prepare_orders`, `loader__raw_orders`) | | `get_group_name(node)` | Uses the project name | | `get_tags(node)` | Includes `sqlbuild/kind:` plus model tags | | `get_kinds(node)` | Returns `["sqlbuild", ""]` | diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_python_project_layout.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_python_project_layout.py index f1757e6ce9..15a8d405b1 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_python_project_layout.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_python_project_layout.py @@ -141,7 +141,7 @@ description="removed tasks Python root", repo_files={ "sqlbuild_project.toml": 'name = "python_layout"\nadapter = "duckdb"\n', - "tasks/orders.py": "from sqlbuild.tasks import task\n", + "tasks/orders.py": "from sqlbuild.assets import asset\n", }, expected_exit_code=1, expected_stderr_fragments=( @@ -153,7 +153,7 @@ description="removed assets Python root", repo_files={ "sqlbuild_project.toml": 'name = "python_layout"\nadapter = "duckdb"\n', - "assets/orders.py": "from sqlbuild.tasks import task\n", + "assets/orders.py": "from sqlbuild.assets import asset\n", }, expected_exit_code=1, expected_stderr_fragments=( @@ -165,7 +165,7 @@ description="removed checks Python root", repo_files={ "sqlbuild_project.toml": 'name = "python_layout"\nadapter = "duckdb"\n', - "checks/orders.py": "from sqlbuild.tasks import task\n", + "checks/orders.py": "from sqlbuild.assets import asset\n", }, expected_exit_code=1, expected_stderr_fragments=( @@ -177,7 +177,7 @@ description="removed loaders Python root", repo_files={ "sqlbuild_project.toml": 'name = "python_layout"\nadapter = "duckdb"\n', - "loaders/orders.py": "from sqlbuild.tasks import task\n", + "loaders/orders.py": "from sqlbuild.assets import asset\n", }, expected_exit_code=1, expected_stderr_fragments=( @@ -189,7 +189,7 @@ description="removed factories Python root", repo_files={ "sqlbuild_project.toml": 'name = "python_layout"\nadapter = "duckdb"\n', - "factories/orders.py": "from sqlbuild.tasks import task\n", + "factories/orders.py": "from sqlbuild.assets import asset\n", }, expected_exit_code=1, expected_stderr_fragments=( @@ -201,7 +201,7 @@ description="removed libs Python root", repo_files={ "sqlbuild_project.toml": 'name = "python_layout"\nadapter = "duckdb"\n', - "libs/cleaning.py": "from sqlbuild.tasks import task\n", + "libs/cleaning.py": "from sqlbuild.assets import asset\n", }, expected_exit_code=1, expected_stderr_fragments=( @@ -213,7 +213,7 @@ description="removed dagster Python root", repo_files={ "sqlbuild_project.toml": 'name = "python_layout"\nadapter = "duckdb"\n', - "dagster/definitions.py": "from sqlbuild.tasks import task\n", + "dagster/definitions.py": "from sqlbuild.assets import asset\n", }, expected_exit_code=1, expected_stderr_fragments=( @@ -225,7 +225,7 @@ description="removed rivers_pipeline Python root", repo_files={ "sqlbuild_project.toml": 'name = "python_layout"\nadapter = "duckdb"\n', - "rivers_pipeline/definitions.py": "from sqlbuild.tasks import task\n", + "rivers_pipeline/definitions.py": "from sqlbuild.assets import asset\n", }, expected_exit_code=1, expected_stderr_fragments=( @@ -237,7 +237,7 @@ description="removed root definitions.py Python root", repo_files={ "sqlbuild_project.toml": 'name = "python_layout"\nadapter = "duckdb"\n', - "definitions.py": "from sqlbuild.tasks import task\n", + "definitions.py": "from sqlbuild.assets import asset\n", }, expected_exit_code=1, expected_stderr_fragments=( @@ -426,8 +426,8 @@ def test_given_python_root_project_when_compiling_and_building_then_nodes_run_wi ), "python/factories/orders.py": ( "from sqlbuild.factories import factory\n" - "from sqlbuild.tasks import task\n\n\n" - "@task\n" + "from sqlbuild.assets import asset\n\n\n" + "@asset\n" "def orders(ctx):\n" " '''Test task orders.'''\n return None\n\n\n" "@factory\n" @@ -445,11 +445,11 @@ def test_given_python_root_project_when_compiling_and_building_then_nodes_run_wi ), "python/factories/orders.py": ( "from sqlbuild.factories import factory\n" - "from sqlbuild.tasks import task\n\n\n" + "from sqlbuild.assets import asset\n\n\n" "@factory\n" "def order_nodes():\n" " global orders\n\n" - " @task\n" + " @asset\n" " def orders(ctx):\n" " '''Test task orders.'''\n return None\n\n" " return orders\n" @@ -494,13 +494,13 @@ def test_given_factory_returning_module_node_when_planning_then_node_registers_o ), "python/factories/orders.py": ( "from sqlbuild.factories import factory\n" - "from sqlbuild.tasks import task\n\n\n" - "@task\n" + "from sqlbuild.assets import asset\n\n\n" + "@asset\n" "def orders(ctx):\n" " '''Test task orders.'''\n return None\n\n\n" "@factory\n" "def order_nodes():\n" - " @task(name='orders')\n" + " @asset(name='orders')\n" " def generated_orders(ctx):\n" " '''Test task generated_orders.'''\n return None\n" " return generated_orders\n" @@ -551,8 +551,8 @@ def test_given_factory_returning_distinct_same_name_node_when_planning_then_conf ), "python/orders.py": ( "import python.helpers.clean\n" - "from sqlbuild.tasks import task\n\n\n" - "@task\n" + "from sqlbuild.assets import asset\n\n\n" + "@asset\n" "def orders(ctx):\n" " '''Test task orders.'''\n return ctx.result(payload={'status': python.helpers.clean.label()})\n" ), @@ -588,10 +588,6 @@ def test_given_unrelated_python_package_on_path_when_building_then_project_pytho assert all(fragment in output for fragment in test_case.expected_build_fragments) -if __name__ == "__main__": - pytest.main([__file__, "-vv"]) - - @pytest.mark.parametrize( "test_case", ( @@ -653,3 +649,7 @@ def test_given_a_node_using_the_removed_task_decorator_when_compiling_then_comma assert all(fragment in result.stderr for fragment in test_case.expected_stderr_fragments), ( result.stderr ) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/goldens/python_oracle/explicit_references_edge_projects.json b/tests/goldens/python_oracle/explicit_references_edge_projects.json index 5116d3538a..9698d7b89a 100644 --- a/tests/goldens/python_oracle/explicit_references_edge_projects.json +++ b/tests/goldens/python_oracle/explicit_references_edge_projects.json @@ -1,19 +1,19 @@ [ [ "explicit_references", -"b70b8e802a4b67adeb1fc2e3" +"d2b46fdc6296bf91b06813dc" ], [ "explicit_references", -"1f2e788d4349e0a43ed9d99e" +"e33fe6067ae76da5913f184d" ], [ "explicit_references", -"21233f8423253e48e8844b06" +"1379c25c88bdd8284a703c2a" ], [ "explicit_references", -"8a8030c53ec888f2b6a28cc4" +"5b87fefa2541f55bdd6cdaf7" ], [ "explicit_references", @@ -21,15 +21,15 @@ ], [ "explicit_references", -"204c1570007330f19d3d5ef1" +"a6a84faa791220995194fc21" ], [ "explicit_references", -"d41afb90907fa281b173024c" +"f04e803ab1bf78c5b0d874e3" ], [ "explicit_references", -"34b91f2790f83b0b7d3d4820" +"1b7104ea4c42a572777c1ffb" ], [ "explicit_references", @@ -49,6 +49,6 @@ ], [ "explicit_references", -"c16aa298ae2448053bd5e47a" +"7fc02502af1a2ec677aed239" ] ] diff --git a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py index 63f4688f68..82c87b0ec5 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py @@ -957,12 +957,12 @@ class OrdersProvider(Provider): """ -def write_provider_task_project(*, project_dir: Path, annotation: str) -> None: - """Write a DuckDB project whose task takes a provider parameter with `annotation`.""" +def write_provider_asset_project(*, project_dir: Path, annotation: str) -> None: + """Write a DuckDB project whose asset takes a provider parameter with `annotation`.""" files: dict[str, str] = { "sqlbuild_project.toml": ( - 'name = "orders_tasks"\nadapter = "duckdb"\n\n' + 'name = "orders_assets"\nadapter = "duckdb"\n\n' '[connection]\ndatabase = "orders.duckdb"\n' ), "providers/orders.py": PROVIDER_SOURCE, diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_provider_annotation_tolerance.py b/tests/integration/src/sqlbuild/cli/commands/main/test_provider_annotation_tolerance.py index 07ed61665f..38c8589c89 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_provider_annotation_tolerance.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_provider_annotation_tolerance.py @@ -15,7 +15,7 @@ from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( ProviderAnnotationToleranceTestCase, ) -from tests.integration.src.sqlbuild.cli.commands.main.helpers import write_provider_task_project +from tests.integration.src.sqlbuild.cli.commands.main.helpers import write_provider_asset_project @pytest.mark.parametrize( @@ -41,14 +41,14 @@ def test_given_provider_parameter_annotation_when_compiling_then_usage_is_record tmp_path: Path, capsys: pytest.CaptureFixture[str], ) -> None: - write_provider_task_project(project_dir=tmp_path, annotation=test_case.annotation) + write_provider_asset_project(project_dir=tmp_path, annotation=test_case.annotation) discovered: DiscoveredPythonNodeFunctions = discover_python_node_functions( project_dir=tmp_path, providers=discover_provider_classes(project_dir=tmp_path) ) exit_code: int = main(["--project-dir", str(tmp_path), "compile"]) - assert tuple(usage.annotation_class_name for usage in discovered.tasks[0].provider_usages) == ( + assert tuple(usage.annotation_class_name for usage in discovered.assets[0].provider_usages) == ( test_case.expected_annotation_class_name, ) assert exit_code == test_case.expected_exit_code, capsys.readouterr() diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py index 675d780148..550a5e0724 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -147,14 +147,18 @@ "DuckShop Python execution inputs", TYPE_PROOF_FILES | { - f"python/{kind}s/orders.py": json.loads( + path: json.loads( Path( "benchmarks/duckshop/edge_cases/orchestration/python_execution_inputs.json" ).read_text() )[source] - for kind, source in (("task", "task"), ("asset", "asset"), ("check", "initial")) + for path, source in ( + ("python/notify/orders.py", "side_effect"), + ("python/assets/orders.py", "asset"), + ("python/checks/orders.py", "initial"), + ) }, - frozenset({"model", "task", "asset", "check"}), + frozenset({"model", "asset", "check"}), ), SnapshotCatalogTestCase( "SQL hooks", @@ -541,9 +545,9 @@ def test_given_a_compiled_project_when_publishing_then_its_column_lineage_is_a_s {"kind": "hook", "name": "notify_fulfillment"}, ], "nodes": [ + "asset:publish_orders", "hook:notify_fulfillment", "model:orders", - "asset:publish_orders", ], } ] @@ -2217,13 +2221,13 @@ def test_given_projects_compiled_in_one_process_when_one_imports_orchestration_t expected_selection=("asset:partner_orders_file", "asset:request_carrier_collection"), ), PythonJobSelectionTestCase( - description="upstream of a task crosses its Python and SQL dependencies", + description="upstream of an asset crosses its Python and SQL dependencies", select="+asset:request_carrier_collection", expected_selection=( "asset:partner_orders_file", + "asset:request_carrier_collection", "model:orders", "model:stg_orders", - "asset:request_carrier_collection", ), ), PythonJobSelectionTestCase( @@ -2236,9 +2240,9 @@ def test_given_projects_compiled_in_one_process_when_one_imports_orchestration_t select="tag:orders", expected_selection=( "asset:partner_orders_file", + "asset:request_carrier_collection", "model:orders", "model:stg_orders", - "asset:request_carrier_collection", ), ), ], diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_set_dependencies.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_set_dependencies.py index f05798acd3..83d6db7c3e 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_set_dependencies.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_set_dependencies.py @@ -81,7 +81,7 @@ ], ids=lambda case: case.description, ) -def test_given_set_dependency_when_planning_the_tasks_upstream_then_selects_the_set( +def test_given_set_dependency_when_planning_the_assets_upstream_then_selects_the_set( test_case: SetDependencyPlanTestCase, tmp_path: Path, write_repo_files: Callable[[Path, dict[str, str]], None], @@ -113,20 +113,22 @@ def test_given_set_dependency_when_planning_the_tasks_upstream_then_selects_the_ depends_on='job("weekly_job")', job_files=(), expected_fragments=( - "task 'report' depends on job 'weekly_job', which the project does not declare", + "asset 'report' depends on job 'weekly_job', which the project does not declare", ), ), SetDependencyErrorTestCase( description="an unknown tag", depends_on='tag("marketing")', job_files=(), - expected_fragments=("task 'report' depends on tag 'marketing', which no node carries",), + expected_fragments=( + "asset 'report' depends on tag 'marketing', which no node carries", + ), ), SetDependencyErrorTestCase( description="a job its exclude empties", depends_on='job("nightly_job", exclude="stg_orders orders order_items")', job_files=(), - expected_fragments=("which leaves no model, seed, source, loader, task or asset",), + expected_fragments=("which leaves no model, seed, source, loader or asset",), ), SetDependencyErrorTestCase( description="job group members that would wait for each other", diff --git a/website/src/content/docs/docs/cli/build.mdx b/website/src/content/docs/docs/cli/build.mdx index 195c93647a..629443f49a 100644 --- a/website/src/content/docs/docs/cli/build.mdx +++ b/website/src/content/docs/docs/cli/build.mdx @@ -39,7 +39,7 @@ own option. `--version` works only directly after `sqb`, as in `sqb --version`: | `--no-audits` | Skip audits | | `--ignore-audit-failures` | Run every audit, but let failures through: staged tables are promoted, incremental batches applied and downstream nodes built. Failures are listed as ignored; see [Ignoring audit failures](/docs/concepts/audits#ignoring-audit-failures). Not allowed with `--no-audits` | | `--no-ignore-audit-failures` | Let audit failures block the build even where `ignore_audit_failures` is set | -| `--no-python` | Skip read-side Python tasks and assets (loader-side Python still runs for selected sources) | +| `--no-python` | Skip read-side Python assets (loader-side Python still runs for selected sources) | | `--no-sql-analysis` | Disable compile-time SQL analysis (`--no-sql-validation` is an alias) | | `--no-cache` | Bypass the reusable compile-analysis cache for this invocation | | `--full-refresh` | Drop and rebuild selected models unless a model sets `full_refresh false`; `full_refresh true` forces a model even without this flag | @@ -136,7 +136,7 @@ A model or seed always runs, whatever counts, when: | Its last build did not finish | The build was killed or failed after the model started writing | | Its relation is missing, or `--full-refresh` or a cursor override applies | `sqb build --changed --full-refresh` | | The planner must rebuild it for a reason other than an edit to its definition | A renamed model | -| What it reads can't be listed | SQL that reads relations not declared as sources or models, or a Python task or asset upstream | +| What it reads can't be listed | SQL that reads relations not declared as sources or models, or a Python asset upstream | | An upstream needs to run, even when it is not selected | `sqb build customer_orders --changed` runs `customer_orders` until a changed `stg_orders` is built | ### How far a change reaches diff --git a/website/src/content/docs/docs/cli/check.mdx b/website/src/content/docs/docs/cli/check.mdx index 2fa1dff9d5..01141bec8b 100644 --- a/website/src/content/docs/docs/cli/check.mdx +++ b/website/src/content/docs/docs/cli/check.mdx @@ -1,11 +1,11 @@ --- title: "sqb check" -description: "Run Python checks against tasks, assets, and loaders." +description: "Run Python checks against assets and loaders." sidebar: label: "check" --- -Runs Python [checks](/docs/concepts/python-nodes/checks) in isolation. Checks validate the output of tasks, assets, and loaders. For SQL relation validation, use [`sqb audit`](/docs/cli/audit) instead. +Runs Python [checks](/docs/concepts/python-nodes/checks) in isolation. Checks validate the output of assets and loaders. For SQL relation validation, use [`sqb audit`](/docs/cli/audit) instead. ## Usage @@ -28,7 +28,7 @@ Positional `SELECTOR` arguments work like `--select`, and both forms can be comb | `--vars` | Override project variables | | `--warehouse ` | Snowflake warehouse for this invocation; overrides the target's `build` [warehouse group](/docs/concepts/adapters/snowflake#command-group-warehouses) and the connection warehouse | -Selecting a non-check node is rejected; use [`sqb build`](/docs/cli/build) to run tasks and assets. +Selecting a non-check node is rejected; use [`sqb build`](/docs/cli/build) to run assets. ## Examples @@ -54,4 +54,4 @@ Results are written to `target/run/checks/python_checks.json`. ## Relationship to build -`sqb build` runs relevant Python checks by default after the tasks, assets, loaders, models, sources, and seeds they depend on have run. `sqb build --no-audits` skips checks alongside audits. `sqb check` runs only checks, on demand. +`sqb build` runs relevant Python checks by default after the assets, loaders, models, sources, and seeds they depend on have run. `sqb build --no-audits` skips checks alongside audits. `sqb check` runs only checks, on demand. diff --git a/website/src/content/docs/docs/cli/plan.mdx b/website/src/content/docs/docs/cli/plan.mdx index 55f7910385..7c04d3eb39 100644 --- a/website/src/content/docs/docs/cli/plan.mdx +++ b/website/src/content/docs/docs/cli/plan.mdx @@ -21,7 +21,7 @@ Positional `SELECTOR` arguments work like `--select`, and both forms can be comb |------|-------------| | `--no-sql-analysis` | Disable compile-time SQL analysis (`--no-sql-validation` is an alias) | | `--no-cache` | Bypass the reusable compile-analysis cache for this invocation | -| `--no-python` | Exclude read-side Python tasks and assets from the plan | +| `--no-python` | Exclude read-side Python assets from the plan | | `--defer-to` | Resolve unselected model references against another target | | `--plan-from` | Read warehouse state `live`, or from the target's `synced_state` checked against the warehouse first; overrides the target's `plan_from`. See [Planning from synced state](/docs/concepts/project-configuration#planning-from-synced-state) | | `--as` | Compile for another target and show its plan, including [model migration](/docs/concepts/models/migrations) decisions, using the current connection. Inspection only. | diff --git a/website/src/content/docs/docs/cli/playground.mdx b/website/src/content/docs/docs/cli/playground.mdx index 719b76d243..438c1be836 100644 --- a/website/src/content/docs/docs/cli/playground.mdx +++ b/website/src/content/docs/docs/cli/playground.mdx @@ -22,7 +22,7 @@ The positional argument is the directory to create (default `sqlbuild-playground | `waffle_shop` | Default. DuckDB-backed project with models, tests, scenarios, and macros. | | `dagster` | A Dagster code location: a ready-to-run `definitions.py` next to the waffle shop SQLBuild project in `waffle_shop/`. | | `rivers` | A Rivers code repository: a `definitions.py` repository definition next to the waffle shop SQLBuild project in `waffle_shop/`. | -| `python_nodes` | A small DuckDB project demonstrating [Python nodes](/docs/concepts/python-nodes/overview): a task feeding a loader, a model read by a Python asset through `ctx.relation(model(...))`, a soft-skip fan-in, `materialized=False`, and a Python check. | +| `python_nodes` | A small DuckDB project demonstrating [Python nodes](/docs/concepts/python-nodes/overview): an asset feeding a loader, a model read by a Python asset through `ctx.relation(model(...))`, a soft-skip fan-in, `materialized=False`, and a Python check. | ## What it creates @@ -51,7 +51,7 @@ The Dagster `definitions.py` uses `sqlbuild_assets`, `sqlbuild_scenario_checks`, The `python_nodes` template instead creates a focused Python-nodes project: -- `python/tasks/orders.py`, `python/loaders/orders.py`, `python/assets/orders_export.py`, `python/checks/orders_export.py` +- `python/assets/orders_inputs.py`, `python/loaders/orders.py`, `python/assets/orders_export.py`, `python/checks/orders_export.py` - A `fact_orders` SQL model over a managed `raw_orders` source - Examples of the SQL boundary, `ctx.relation(model(...))`, soft-skip fan-in, and a Python check diff --git a/website/src/content/docs/docs/cli/review.mdx b/website/src/content/docs/docs/cli/review.mdx index 8ddf317f65..055d477710 100644 --- a/website/src/content/docs/docs/cli/review.mdx +++ b/website/src/content/docs/docs/cli/review.mdx @@ -65,7 +65,7 @@ Downstream models the policy leaves out are always listed as **not validated**, The impact tree is computed from the whole changed set, whatever the policy. A review never runs [Python nodes](/docs/concepts/python-nodes/overview). A changed Python node, -and every task, asset or check downstream of a change, appears in the impact tree as not validated +and every asset or check downstream of a change, appears in the impact tree as not validated with the reason. So does every model that reads a changed loader's source, directly or through other models: the review would build it from the `--against` data rather than from the changed loader, so it is listed as `reads Python node loader:, not rebuilt by review` instead of diff --git a/website/src/content/docs/docs/cli/test.mdx b/website/src/content/docs/docs/cli/test.mdx index dd71410e7f..a381de6e42 100644 --- a/website/src/content/docs/docs/cli/test.mdx +++ b/website/src/content/docs/docs/cli/test.mdx @@ -151,7 +151,7 @@ arguments, result type or body change, or a project variable it reads; formattin description do not count. A node that calls a SQL macro also uses every SQL macro that macro calls. An enum, constant or audit counts as changed when its declaration file changes other than in formatting and comments. A seed counts as changed with its data or column types, a source with its declaration or inline expression, and a function with its definition. -Loaders, tasks, assets and checks are compared by their Python modules and the project modules +Loaders, assets and checks are compared by their Python modules and the project modules they import; they appear in the change summary but no unit test runs them. ### Project-wide inputs diff --git a/website/src/content/docs/docs/concepts/models/configuration.mdx b/website/src/content/docs/docs/concepts/models/configuration.mdx index ce135807da..b2ea934f05 100644 --- a/website/src/content/docs/docs/concepts/models/configuration.mdx +++ b/website/src/content/docs/docs/concepts/models/configuration.mdx @@ -67,7 +67,7 @@ error[P010]: model 'orders_summary' has no description | SQL function or table function | `FUNCTION (description "...")` | | Python UDF | `@udf(description="...")` or the function docstring | | Named SQL hook | `HOOK (description "...")` | -| Python hook, task, asset, check | `description="..."` on the decorator or the function docstring | +| Python hook, asset, check | `description="..."` on the decorator or the function docstring | | Loader | `@loader(description="...")` or the function docstring; a loader that feeds a declared source may instead rely on the source's YAML description | | Provider | The provider class docstring | diff --git a/website/src/content/docs/docs/concepts/models/hooks.mdx b/website/src/content/docs/docs/concepts/models/hooks.mdx index 575a0e3710..4dcf216e33 100644 --- a/website/src/content/docs/docs/concepts/models/hooks.mdx +++ b/website/src/content/docs/docs/concepts/models/hooks.mdx @@ -112,7 +112,7 @@ A Python post-hook that returns `ctx.skip(...)` stops the remaining post-hooks a ## Names and identity -SQL and Python hook names share the project-wide resource namespace with models, sources, seeds, functions, loaders, tasks, assets, checks, and providers. Any collision fails discovery, including two same-stem SQL hook files in different directories or a SQL and Python hook with the same name. +SQL and Python hook names share the project-wide resource namespace with models, sources, seeds, functions, loaders, assets, checks, and providers. Any collision fails discovery, including two same-stem SQL hook files in different directories or a SQL and Python hook with the same name. The ordered hook list participates in each model's version identity: diff --git a/website/src/content/docs/docs/concepts/models/hooks/python.mdx b/website/src/content/docs/docs/concepts/models/hooks/python.mdx index 009f5e7f13..01547d6380 100644 --- a/website/src/content/docs/docs/concepts/models/hooks/python.mdx +++ b/website/src/content/docs/docs/concepts/models/hooks/python.mdx @@ -128,7 +128,7 @@ Python hooks declare a `HookContext` parameter named `ctx`, `context`, `_ctx`, o ## Providers -Providers may be injected into Python hook parameters by name or accessed through `ctx.providers`. They are resolved lazily using the same lifecycle as loaders, tasks, assets, and checks. SQL hooks cannot use providers because they are compiled SQL statements rather than Python callables. +Providers may be injected into Python hook parameters by name or accessed through `ctx.providers`. They are resolved lazily using the same lifecycle as loaders, assets, and checks. SQL hooks cannot use providers because they are compiled SQL statements rather than Python callables. ```python from sqlbuild.hooks import hook diff --git a/website/src/content/docs/docs/concepts/planning.mdx b/website/src/content/docs/docs/concepts/planning.mdx index c7d6a8baba..7d1ae54eae 100644 --- a/website/src/content/docs/docs/concepts/planning.mdx +++ b/website/src/content/docs/docs/concepts/planning.mdx @@ -60,7 +60,7 @@ seed from one whose file or loading behavior changed. ### Python nodes -Loaders, tasks, assets, checks, and Python hooks have identities derived from project-owned source, +Loaders, assets, checks, and Python hooks have identities derived from project-owned source, transitive project dependencies, and decorator configuration. Python hook identities also participate in the identity of models that invoke them. diff --git a/website/src/content/docs/docs/concepts/project-configuration.mdx b/website/src/content/docs/docs/concepts/project-configuration.mdx index 30f05b29f4..2c56caa49f 100644 --- a/website/src/content/docs/docs/concepts/project-configuration.mdx +++ b/website/src/content/docs/docs/concepts/project-configuration.mdx @@ -152,7 +152,7 @@ A target is a named build context - the database and schema you build into, plus | Field | Description | |-------|-------------| | `schema` | Schema for all models in this target; required for named targets | -| `loader_schema` | Default write schema for managed source loaders; falls back to `schema`. Required when `schema` is `preserve` and the project has loaders, tasks, assets or checks, because their [results](/docs/concepts/python-nodes/overview#result-persistence) go there | +| `loader_schema` | Default write schema for managed source loaders; falls back to `schema`. Required when `schema` is `preserve` and the project has loaders, assets or checks, because their [results](/docs/concepts/python-nodes/overview#result-persistence) go there | | `database` | Database for all models in this target | | `connection` | Name from `[connections.]` used for endpoint, authentication, and compute; optional when exactly one named connection is defined | | `vars` | Target-specific project variables | @@ -525,7 +525,7 @@ delta_compile = true | `scenario_timeout` | none | Time limit for each scenario, or `"disabled"` | | `delta_compile` | `true` | After an edit to model SQL or SQL test files only, compile just the changed files against the previous compile. See [Compiling after an edit](/docs/cli/compile#compiling-after-an-edit). | | `allow_missing_secrets` | `false` | Let `compile`, `rules run`, `format`, `dag`, `lineage` and `scope` run without provider secrets. `sqlbuild_local.toml` only. See [Secrets](/docs/concepts/python-nodes/providers#secrets). | -| `allow_warehouse_round_trips` | `false` | Allow a loader to depend on a task or asset that reads SQL, so data can leave the warehouse and come back through a source. Project file only. See [The SQL boundary](/docs/concepts/python-nodes/overview#warehouse-round-trips). | +| `allow_warehouse_round_trips` | `false` | Allow a loader to depend on an asset that reads SQL, so data can leave the warehouse and come back through a source. Project file only. See [The SQL boundary](/docs/concepts/python-nodes/overview#warehouse-round-trips). | | `ignore_audit_failures` | `false` | Let builds go on past failing audits: they still run and record `fail`, but promote, apply and build downstream as if they passed. A target, a job and `--ignore-audit-failures` / `--no-ignore-audit-failures` override it. See [Ignoring audit failures](/docs/concepts/audits#ignoring-audit-failures). | | `record_run_host` | `true` | Record the machine name in each run's rows in `_sqlbuild_runs`. See [Run records](/docs/cli/build#run-records). | @@ -588,7 +588,7 @@ Every reference to a model, source, or seed must be explicit and typed. By defau - A macro whose expanded output contains `__ref()`, `__source()`, or `__seed()` fails to compile with `P006`. Pass the reference to the macro as an argument instead; see [Typed reference arguments](/docs/concepts/macros/python-macros#typed-reference-arguments). -- Literal SQL passed to `ctx.query()` or `ctx.execute_sql()` in a task, asset, check, loader, or +- Literal SQL passed to `ctx.query()` or `ctx.execute_sql()` in an asset, check, loader, or Python hook that names a project relation fails to compile with `P008`. - The same check runs on SQL sent at run time and reports a `P008` warning; the query still runs. See [Hard-coded relation names](/docs/concepts/python-nodes/sql-references#hard-coded-relation-names). @@ -780,7 +780,7 @@ See [skills CLI reference](/docs/cli/skills) for usage details. ## Project Python Project-owned Python lives in a small set of roots. Anything that runs as part of the pipeline graph -goes in `python/`: loaders, tasks, assets, checks, factories, and the helper modules they import. +goes in `python/`: loaders, assets, checks, factories, and the helper modules they import. Modules under `python/` are normal Python: constants, classes, undecorated helper functions, and helper-only modules such as `python/_helpers.py` are allowed, while decorators determine which functions become graph nodes. See [Python nodes](/docs/concepts/python-nodes/overview#the-python-folder). diff --git a/website/src/content/docs/docs/concepts/python-nodes/assets.mdx b/website/src/content/docs/docs/concepts/python-nodes/assets.mdx index 50fcbb7b43..2f91a3c5c2 100644 --- a/website/src/content/docs/docs/concepts/python-nodes/assets.mdx +++ b/website/src/content/docs/docs/concepts/python-nodes/assets.mdx @@ -1,16 +1,16 @@ --- title: "Assets" -description: "Produce or observe external artifacts as nodes in the SQLBuild graph." +description: "Run Python computation, side effects and external artifacts as nodes in the SQLBuild graph." --- -Assets are Python nodes that produce or observe an external artifact - an exported file, a published dataset, a dashboard refresh, an ML model. They behave like [tasks](/docs/concepts/python-nodes/tasks) but add dataset-like metadata (columns, column lineage) and a materialization flag. See [Python Nodes](/docs/concepts/python-nodes/overview) for the shared model and the SQL boundary rules. +Assets are Python functions that run as part of the DAG. Use them for computation, side effects and external artifacts such as an exported file, a published dataset, a dashboard refresh or an ML model. See [Python Nodes](/docs/concepts/python-nodes/overview) for the shared model and the SQL boundary rules. ## Defining an asset -Place Python files under `python/` (the playgrounds use `python/assets/`) and decorate functions with `@asset`: +Place Python files under `python/` and decorate functions with `@asset`: ```python -# python/assets/exports.py +# python/exports/orders_export.py from sqlbuild.assets import asset, AssetContext @@ -18,45 +18,31 @@ from sqlbuild.assets import asset, AssetContext def orders_export(ctx: AssetContext): """Orders export.""" path = write_orders_csv() - return ctx.result(metadata={"path": path, "rows": 1200}, materialized=True) + return ctx.result(payload={"path": path}, metadata={"rows": 1200}, materialized=True) ``` -The asset receives an `AssetContext` and returns through `ctx.result(...)`. - -## Materialization - -Assets record whether they actually produced an artifact via `materialized`: - -```python -@asset -def orders_export(ctx: AssetContext): - """Orders export.""" - if nothing_changed(): - return ctx.result(metadata={"status": "unchanged"}, materialized=False) - return ctx.result(metadata={"path": export()}, materialized=True) -``` - -- `materialized=True` - the artifact was produced this run. -- `materialized=False` - the asset ran but produced nothing (e.g. it only observed state). This is **not** a skip; the node still succeeded. - -Only assets have `materialized`; tasks do not. +The asset receives an `AssetContext` and returns through `ctx.result(...)`. A plain return value or `None` is also accepted and normalized to a successful result. ## Dependencies -Assets declare dependencies with `depends_on` (a single function, tuple, or list), and may depend on tasks, assets, and loaders: +Assets declare dependencies with `depends_on`, accepting a single function, a tuple, or a list. They may depend on other assets and on loaders: ```python -from sqlbuild.assets import asset -from python.tasks.orders import export_orders +@asset +def fetch_orders(ctx: AssetContext): + """Fetch orders.""" + return ctx.result(payload=download_orders()) -@asset(depends_on=export_orders) -def orders_dashboard(ctx): - """Orders dashboard.""" - result = ctx.result_of(export_orders) - return ctx.result(metadata={"rows": result.payload["rows"]}, materialized=True) +@asset(depends_on=fetch_orders) +def summarize_orders(ctx: AssetContext): + """Summarize orders.""" + result = ctx.result_of(fetch_orders) + return ctx.result(metadata={"count": len(result.payload)}) ``` +`ctx.result_of(node_fn)` reads the latest persisted result of an upstream node, returning a `NodeResultEnvelope` with `payload`, `metadata`, `status`, and `ts` fields. Results persist across runs. Use `ctx.results_of(node_fn, limit=N)` to read result history. Reading a missing or unsuccessful upstream raises unless you pass `default=`. + To read a SQL model or source, declare a typed reference and resolve it at runtime - see [SQL references](/docs/concepts/python-nodes/sql-references): ```python @@ -72,6 +58,40 @@ def orders_extract(ctx): return ctx.result(metadata={"rows": rows}, materialized=True) ``` +## Returning results + +```python +@asset +def build_export(ctx: AssetContext): + """Build export.""" + return ctx.result( + payload={"path": "/exports/orders.csv"}, + metadata={"rows": 1200}, + materialized=True, + ) +``` + +- `payload` is the value downstream nodes read with `ctx.result_of(...)`. +- `metadata` is structured JSON for catalogs and downstream reads. +- `materialized` records whether the asset produced its artifact this run (see below). + +## Materialization + +Set `materialized` when the asset produces or observes an artifact: + +```python +@asset +def orders_export(ctx: AssetContext): + """Orders export.""" + if nothing_changed(): + return ctx.result(metadata={"status": "unchanged"}, materialized=False) + return ctx.result(metadata={"path": export()}, materialized=True) +``` + +- `materialized=True` - the artifact was produced this run. +- `materialized=False` - the asset ran but produced nothing (e.g. it only observed state). This is **not** a skip; the node still succeeded. +- Leave it unset for an asset that runs computation or a side effect without an artifact. + ## Columns and column lineage Assets can declare a schema for catalog and lineage purposes. This does not enforce anything at runtime; it describes the artifact: @@ -95,26 +115,62 @@ def orders_export(ctx): - `columns` - column declarations with `name`, optional `type`, `nullable`, `description`, and `meta`. - `column_lineage` - maps each asset column to upstream `{node, column}` references, surfaced in the DAG artifact and integrations. -Tasks and checks do not support `columns` or `column_lineage`. +Checks do not support `columns` or `column_lineage`. + +## Skipping + +Return `ctx.skip(...)` to skip an asset: -## Skipping and retries +```python +@asset +def export_if_present(ctx: AssetContext): + """Export if present.""" + if not new_files_available(): + return ctx.skip("no new files") + return ctx.result(payload=do_export()) +``` + +- `"soft"` (default) skips only this asset; dependents may still run if another upstream succeeded. +- `"hard"` skips this asset and blocks its dependents. -Assets support the same `ctx.skip(...)` and `retry` behavior as tasks: +`mode` accepts either a plain string or the `SkipMode` enum: ```python from sqlbuild.assets import asset, SkipMode + + +@asset +def optional_step(ctx): + """Optional step.""" + return ctx.skip("nothing to do", mode=SkipMode.HARD) # or mode="hard" +``` + +## Retries + +`@asset` accepts a `retry` policy for transient failures: + +```python +from sqlbuild.assets import asset from sqlbuild.retries import RetryPolicy -@asset(retry=RetryPolicy(max_attempts=3, retry_on=(IOError,))) -def export(ctx): - """Export the order summary.""" - if not ready(): - return ctx.skip("upstream not ready", mode="soft") # or SkipMode.SOFT - return ctx.result(metadata={"path": do_export()}, materialized=True) +@asset(retry=RetryPolicy(max_attempts=3, retry_on=(ConnectionError,))) +def call_api(ctx): + """Call api.""" + return ctx.result(payload=fetch_from_flaky_api()) ``` -See [Tasks](/docs/concepts/python-nodes/tasks#retries) for the full retry policy fields. +| Field | Default | Description | +|-------|---------|-------------| +| `max_attempts` | `3` | Total attempts including the first | +| `retry_on` | `Exception` | Exception class, tuple, or list to retry on | +| `initial_delay_seconds` | `1.0` | Delay before the first retry | +| `backoff_multiplier` | `2.0` | Exponential backoff multiplier | +| `max_delay_seconds` | `30.0` | Cap on any single delay | +| `max_elapsed_seconds` | `None` | Overall time bound for retries | +| `jitter` | `True` | Randomize delays to avoid synchronized retries | + +The default is no retry. Set `retry_on` explicitly rather than relying on the broad default when you can. The original exception is preserved if all attempts fail. ## Decorator parameters @@ -140,4 +196,4 @@ sqb build --select orders_export --no-tests --no-audits sqb build --select +orders_export ``` -Assets run during `sqb build`. Use `--no-python` to suppress read-side assets while still loading sources. +Assets run during `sqb build`. Use `--no-python` to suppress read-side assets while still loading sources. They are not validated unless you also write a [check](/docs/concepts/python-nodes/checks) that depends on them. diff --git a/website/src/content/docs/docs/concepts/python-nodes/checks.mdx b/website/src/content/docs/docs/concepts/python-nodes/checks.mdx index 17a34a1947..ee31694d29 100644 --- a/website/src/content/docs/docs/concepts/python-nodes/checks.mdx +++ b/website/src/content/docs/docs/concepts/python-nodes/checks.mdx @@ -1,11 +1,11 @@ --- title: "Checks" -description: "Validate tasks, assets, loaders, and SQL relations with Python checks." +description: "Validate assets, loaders, and SQL relations with Python checks." --- -Checks are Python nodes that validate the output of tasks, assets, and loaders, or the data in SQL models, sources, and seeds. They are the Python analog of SQL [audits](/docs/concepts/audits). See [Python Nodes](/docs/concepts/python-nodes/overview) for the shared model. +Checks are Python nodes that validate the output of assets and loaders, or the data in SQL models, sources, and seeds. They are the Python analog of SQL [audits](/docs/concepts/audits). See [Python Nodes](/docs/concepts/python-nodes/overview) for the shared model. -Checks are separate graph nodes, not callbacks embedded in a task or asset. A check declares what it validates through `depends_on`. +Checks are separate graph nodes, not callbacks embedded in an asset. A check declares what it validates through `depends_on`. ## Defining a check @@ -14,7 +14,7 @@ Place Python files under `python/` (the playgrounds use `python/checks/`) and de ```python # python/checks/orders.py from sqlbuild.checks import check, CheckContext -from python.tasks.orders import export_orders +from python.exports.orders import export_orders @check(depends_on=export_orders) @@ -86,7 +86,7 @@ def orders_freshness(ctx): ## What checks can depend on -- Checks may depend on **tasks, assets, and loaders**. +- Checks may depend on **assets and loaders**. - Checks may depend on SQL **models, sources, and seeds** through `model()`, `source()`, and `seed()` references. They may **not** depend on functions. - Checks may **not** depend on other checks. - Checks may **not** depend on a terminal source loader directly. Validate loaded source data with a source audit instead. @@ -97,7 +97,7 @@ A check that depends on a single node is displayed grouped under that node. Mult | Parameter | Description | |-----------|-------------| -| `depends_on` | Required. Tasks/assets/loaders to validate, and `model()`/`source()`/`seed()` references (function, reference, tuple, or list) | +| `depends_on` | Required. Assets/loaders to validate, and `model()`/`source()`/`seed()` references (function, reference, tuple, or list) | | `name` | Override the node name (defaults to the function name) | | `severity` | `error` (default) or `warn` | | `tags` | Labels for selection and grouping | @@ -126,13 +126,13 @@ sqb check --select +check_orders_exported sqb check --select tag:exports ``` -`sqb check` rejects selecting non-check nodes; use `sqb build` to run tasks and assets. The models, sources, and seeds a check depends on must already exist; `sqb check` does not build them. Check results are written to `target/run/checks/python_checks.json`, and `sqb check --json` prints them to stdout. +`sqb check` rejects selecting non-check nodes; use `sqb build` to run assets. The models, sources, and seeds a check depends on must already exist; `sqb check` does not build them. Check results are written to `target/run/checks/python_checks.json`, and `sqb check --json` prints them to stdout. ## Checks vs audits | | Checks | Audits | |---|--------|--------| -| Validates | Python tasks, assets, loaders, and SQL relations | SQL relations | +| Validates | Python assets, loaders, and SQL relations | SQL relations | | Authored in | `python/` (Python) | `MODEL()` headers / `audits/` (SQL) | | Run by | `sqb build`, `sqb check` | `sqb build`, `sqb audit` | | Severity | `error`, `warn` | `error`, `warn` | diff --git a/website/src/content/docs/docs/concepts/python-nodes/factories.mdx b/website/src/content/docs/docs/concepts/python-nodes/factories.mdx index 0af21a86e8..196cb60fd0 100644 --- a/website/src/content/docs/docs/concepts/python-nodes/factories.mdx +++ b/website/src/content/docs/docs/concepts/python-nodes/factories.mdx @@ -3,7 +3,7 @@ title: "Factories" description: "Generate Python nodes programmatically with @factory." --- -A factory is a function that **generates** Python nodes instead of authoring them one at a time. Use `@factory` when you want to create many similar [tasks](/docs/concepts/python-nodes/tasks), [assets](/docs/concepts/python-nodes/assets), [loaders](/docs/concepts/python-nodes/loaders), or [checks](/docs/concepts/python-nodes/checks) from a list, a config, or a loop, rather than hand-writing each one. +A factory is a function that **generates** Python nodes instead of authoring them one at a time. Use `@factory` when you want to create many similar [assets](/docs/concepts/python-nodes/assets), [loaders](/docs/concepts/python-nodes/loaders), or [checks](/docs/concepts/python-nodes/checks) from a list, a config, or a loop, rather than hand-writing each one. Factories are an advanced feature. Most projects author nodes directly; reach for a factory when you find yourself copy-pasting near-identical node definitions. @@ -33,7 +33,7 @@ def export_assets(): The returned nodes are discovered and added to the graph exactly as if you had written them by hand. They participate in selection, lifecycle, the DAG artifact, and integrations like any other node. -A factory may return a single node or a list, tuple, or set of nodes. Every returned item must be a decorated `@task`, `@asset`, `@loader`, or `@check` function. +A factory may return a single node or a list, tuple, or set of nodes. Every returned item must be a decorated `@asset`, `@loader`, or `@check` function. ## Where factories live diff --git a/website/src/content/docs/docs/concepts/python-nodes/loaders.mdx b/website/src/content/docs/docs/concepts/python-nodes/loaders.mdx index 4be22259f2..16ff8f98c4 100644 --- a/website/src/content/docs/docs/concepts/python-nodes/loaders.mdx +++ b/website/src/content/docs/docs/concepts/python-nodes/loaders.mdx @@ -255,9 +255,9 @@ Every loader function receives a `LoaderContext` as its first argument. It provi | `qualify_name(name)` | Return a fully-qualified relation name in the destination database/schema | | `skip(reason, mode=...)` | Skip this loader. `mode` is `"soft"` (default, skip only this loader) or `"hard"` (also block dependents) | | `result(payload=, metadata=, materialized=)` | Return a structured result for a self-managed loader | -| `result_of(node_fn)` | Read the result of an upstream task, asset or loader: its result from this run, else its latest successful one; raises naming the upstream if it never succeeded | +| `result_of(node_fn)` | Read the result of an upstream asset or loader: its result from this run, else its latest successful one; raises naming the upstream if it never succeeded | | `results_of(node_fn, limit=N)` | Read the last N successful results of an upstream node, newest first | -| `loader(loader_fn)` | Return a `LoaderRelationRef` for an upstream loader dependency; a task or asset is refused with a pointer to `result_of` | +| `loader(loader_fn)` | Return a `LoaderRelationRef` for an upstream loader dependency; an asset is refused with a pointer to `result_of` | | `source(source_name)` | Return a `LoaderRelationRef` for a project source by YAML name | ### LoaderRelationRef @@ -272,7 +272,7 @@ Returned by `ctx.loader()` and `ctx.source()`. Provides access to an upstream re ## Loader dependencies -Loaders can depend on other loaders, tasks and assets using `depends_on`. Dependencies are executed first. An upstream loader's destination relation is available via `ctx.loader()`: +Loaders can depend on other loaders and assets using `depends_on`. Dependencies are executed first. An upstream loader's destination relation is available via `ctx.loader()`: ```python from sqlbuild.loaders import loader @@ -297,25 +297,25 @@ def raw_account_metrics(ctx: LoaderContext) -> list[dict[str, object]]: ] ``` -Read upstream relations through `ctx.loader(...)` and `ctx.source(...)`, not by writing their table names in SQL. SQLBuild rejects literal SQL that names a project source, model, or seed with `P008`, and warns at run time when dynamic SQL does; loaders cannot read models and seeds. To use model data in a loader, read it in an upstream task and pass it on through the task's result. See [Hard-coded relation names](/docs/concepts/python-nodes/sql-references#hard-coded-relation-names). +Read upstream relations through `ctx.loader(...)` and `ctx.source(...)`, not by writing their table names in SQL. SQLBuild rejects literal SQL that names a project source, model, or seed with `P008`, and warns at run time when dynamic SQL does; loaders cannot read models and seeds. To use model data in a loader, read it in an upstream asset and pass it on through the asset's result. See [Hard-coded relation names](/docs/concepts/python-nodes/sql-references#hard-coded-relation-names). Dependencies form a DAG. SQLBuild schedules loaders in topological order and executes independent loaders concurrently when `--concurrency` is set. -### Depending on tasks and assets +### Depending on assets -A loader reads an upstream task's or asset's result with `ctx.result_of(...)`; `ctx.loader(...)` -takes loaders only. A failed or hard-skipped upstream task or asset skips the loader and everything -downstream of the source it fills. Otherwise the loader runs when a task or asset it depends on, +A loader reads an upstream asset's result with `ctx.result_of(...)`; `ctx.loader(...)` +takes loaders only. A failed or hard-skipped upstream asset skips the loader and everything +downstream of the source it fills. Otherwise the loader runs when an asset it depends on, directly or through other loaders, succeeded; when all of them skip softly or did not run, only the load is skipped, so the source keeps its last loaded data and its readers still build when another -of their inputs ran. A loader selected without its upstream task or asset reads that upstream's +of their inputs ran. A loader selected without its upstream asset reads that upstream's latest successful result, and fails naming the upstream if it never succeeded. ```python from sqlbuild.loaders import loader -from sqlbuild.tasks import task +from sqlbuild.assets import asset -@task +@asset def fetch_partner_scores(ctx): """Download the scores a partner published.""" return ctx.result(payload={"scores": [[1, 10], [2, 20]]}) @@ -327,9 +327,9 @@ def partner_scores(ctx): return [{"order_id": order_id, "score": score} for order_id, score in scores] ``` -When the task or asset reads SQL, directly or through other Python nodes, the loader makes a +When the asset reads SQL, directly or through other Python nodes, the loader makes a [warehouse round trip](/docs/concepts/python-nodes/overview#warehouse-round-trips), which the project -must allow with `allow_warehouse_round_trips`. The loader then loads after the SQL the task reads +must allow with `allow_warehouse_round_trips`. The loader then loads after the SQL the asset reads and before the models that read its source, in the same `sqb build`. Intermediate loaders (those referenced only via `depends_on`, with no managed source of the same name) are given synthetic source entries and write to `__loader__` tables by default. Only the terminal loader - the one whose name matches a managed source - populates that source; intermediate loaders feed it. Use the `destination` parameter on the decorator to override the intermediate relation: @@ -352,7 +352,7 @@ The `@loader` decorator accepts optional parameters that can also be set in the | Parameter | Description | |-----------|-------------| -| `depends_on` | List of loader, task and asset functions this loader depends on. Depending on one that reads SQL is a [warehouse round trip](/docs/concepts/python-nodes/overview#warehouse-round-trips) | +| `depends_on` | List of loader and asset functions this loader depends on. Depending on one that reads SQL is a [warehouse round trip](/docs/concepts/python-nodes/overview#warehouse-round-trips) | | `destination` | Override the destination relation name (can include schema or database) | | `write_strategy` | `table`, `append`, `delete_insert`, or `merge` | | `cursor_column` | Column used for incremental cursor tracking | diff --git a/website/src/content/docs/docs/concepts/python-nodes/overview.mdx b/website/src/content/docs/docs/concepts/python-nodes/overview.mdx index 48b1be1078..4ebed6e1eb 100644 --- a/website/src/content/docs/docs/concepts/python-nodes/overview.mdx +++ b/website/src/content/docs/docs/concepts/python-nodes/overview.mdx @@ -1,24 +1,22 @@ --- title: "Overview" -description: "Tasks, assets, loaders, and checks as first-class nodes in the SQLBuild graph." +description: "Loaders, assets, and checks as first-class nodes in the SQLBuild graph." --- Python nodes let your project grow beyond warehouse-only SQL while keeping the SQL graph clean. They are ordinary Python functions, decorated to become nodes in the same DAG as your SQL models, and they run as part of `sqb build`. -There are four kinds. The decorator says what each function is, so all of them live under one `python/` folder: +There are three kinds. The decorator says what each function is, so all of them live under one `python/` folder: | Kind | Decorator | Purpose | |------|-----------|---------| | Loader | `@loader` | Load external data into a managed source | -| Task | `@task` | Run Python computation or side effects | -| Asset | `@asset` | Produce or observe an external artifact | -| Check | `@check` | Validate tasks, assets, and loaders | +| Asset | `@asset` | Run Python computation or side effects, or produce an external artifact | +| Check | `@check` | Validate assets and loaders | -All four share the same decorator conventions, dependency model, selection syntax, and runtime context helpers. The pages for each kind cover their specifics: +All three share the same decorator conventions, dependency model, selection syntax, and runtime context helpers. The pages for each kind cover their specifics: - [Loaders](/docs/concepts/python-nodes/loaders) - load data into managed sources -- [Tasks](/docs/concepts/python-nodes/tasks) - Python computation and side effects -- [Assets](/docs/concepts/python-nodes/assets) - external artifacts +- [Assets](/docs/concepts/python-nodes/assets) - Python computation, side effects and external artifacts - [Checks](/docs/concepts/python-nodes/checks) - Python validations - [Factories](/docs/concepts/python-nodes/factories) - generate nodes programmatically with `@factory` - [SQL references](/docs/concepts/python-nodes/sql-references) - read SQL models and sources from Python @@ -27,7 +25,7 @@ Nodes can also be generated programmatically with [`@factory`](/docs/concepts/py ## The `python/` folder -Anything that runs as part of the pipeline graph goes in `python/`: loaders, tasks, assets, checks, [factories](/docs/concepts/python-nodes/factories), and the helper modules they import. SQLBuild imports every `.py` file under `python/` recursively (except `__init__.py`) and registers each decorated function by its decorator. Undecorated functions and helper-only modules are allowed and never become nodes. +Anything that runs as part of the pipeline graph goes in `python/`: loaders, assets, checks, [factories](/docs/concepts/python-nodes/factories), and the helper modules they import. SQLBuild imports every `.py` file under `python/` recursively (except `__init__.py`) and registers each decorated function by its decorator. Undecorated functions and helper-only modules are allowed and never become nodes. How you organize files inside `python/` is up to you. The playgrounds group nodes by kind: @@ -35,13 +33,13 @@ How you organize files inside `python/` is up to you. The playgrounds group node python/ _helpers.py # shared helpers, not nodes loaders/orders.py - tasks/orders.py + assets/orders_inputs.py assets/orders_export.py checks/orders_export.py factories/regional.py ``` -Grouping by domain (`python/orders/`, `python/customers/`) works just as well, and a single file may mix kinds. A project that selects the [Python Rules](/docs/concepts/rules/python-rules#laying-out-python) commits to grouping by domain, with each node in its own [unit folder](/docs/concepts/rules/python-rules#writing-nodes-as-units) holding `node.py` and `main.py`: `SQBRPYTHON001` reports node-kind folders such as `python/loaders/` and modules directly under `python/`, and `SQBRPYTHON012` reports nodes declared outside a unit's `node.py`. Import other project modules through the `python` package, for example `from python.tasks.orders import export_orders` or `from python._helpers import normalize_status`. +Grouping by domain (`python/orders/`, `python/customers/`) works just as well, and a single file may mix kinds. A project that selects the [Python Rules](/docs/concepts/rules/python-rules#laying-out-python) commits to grouping by domain, with each node in its own [unit folder](/docs/concepts/rules/python-rules#writing-nodes-as-units) holding `node.py` and `main.py`: `SQBRPYTHON001` reports node-kind folders such as `python/loaders/` and modules directly under `python/`, and `SQBRPYTHON012` reports nodes declared outside a unit's `node.py`. Import other project modules through the `python` package, for example `from python.assets.orders_inputs import prepare_raw_orders` or `from python._helpers import normalize_status`. Code that extends SQLBuild itself keeps its own folder, such as `rules/`, `materializations/`, `adapters/`, `providers/`, `sinks/`, `hooks/python/`, `functions/python/`, and `macros/`. Python outside the supported locations fails compilation with `D016`; see [Project Python](/docs/concepts/project-configuration#project-python) for the full list. @@ -50,20 +48,20 @@ Code that extends SQLBuild itself keeps its own folder, such as `rules/`, `mater The most important rule: **SQL models never depend on Python nodes.** The dependency direction is strictly one way. - SQL models depend only on other SQL resources (models, sources, seeds, functions). -- Python nodes may depend on other Python nodes; a loader may depend on any Python node, including tasks and assets. +- Python nodes may depend on other Python nodes; a loader may depend on any Python node, including assets. - Python data reaches SQL only through a **loader filling a source**: `loader -> source -> model`. -- Tasks, assets, and checks may **read** SQL models, sources, and seeds at runtime through typed references (see [SQL references](/docs/concepts/python-nodes/sql-references)), but reading a model does not make it a SQL dependency. +- Assets and checks may **read** SQL models, sources, and seeds at runtime through typed references (see [SQL references](/docs/concepts/python-nodes/sql-references)), but reading a model does not make it a SQL dependency. This keeps the SQL graph fully analyzable and testable on its own, while letting Python participate around the edges. ``` depends_on - task ----------------> task + asset ---------------> asset | | | (read only) v | asset v | - source <-- loader | check (validates tasks/assets/loaders/SQL) + source <-- loader | check (validates assets/loaders/SQL) | v model (SQL) ----> model (SQL) @@ -71,19 +69,19 @@ This keeps the SQL graph fully analyzable and testable on its own, while letting ### Warehouse round trips -A loader that depends on a task or asset that reads SQL, directly or through other Python nodes, +A loader that depends on an asset that reads SQL, directly or through other Python nodes, makes a warehouse round trip: data leaves the warehouse and comes back through the loader. For -example, a task reads `orders`, sends them to a partner's scoring service, and a loader loads the +example, an asset reads `orders`, sends them to a partner's scoring service, and a loader loads the scores it returned into the `partner_scores` source that `scored_orders` reads: ``` - model orders --> task score_orders --> loader partner_scores --> source partner_scores --> model scored_orders + model orders --> asset score_orders --> loader partner_scores --> source partner_scores --> model scored_orders ``` Round trips are off by default. Turn them on with `allow_warehouse_round_trips` in `sqlbuild_project.toml` (see [Settings](/docs/concepts/project-configuration#settings)); without it, -a plan or build that includes such a loader stops with an error naming the loader and the task or -asset that reads SQL. A loader that depends on a task or asset that reads no SQL is ordinary +a plan or build that includes such a loader stops with an error naming the loader and the asset +that reads SQL. A loader that depends on an asset that reads no SQL is ordinary ingestion and is always allowed. ```toml @@ -93,23 +91,23 @@ allow_warehouse_round_trips = true With the setting on: -- One `sqb build` runs the SQL the task reads, then the task, then the loader, then the models that - read the loader's source. The loader follows the usual rule for its upstream tasks and assets: if +- One `sqb build` runs the SQL the asset reads, then the asset, then the loader, then the models that + read the loader's source. The loader follows the usual rule for its upstream assets: if one fails or skips with `mode="hard"`, the loader and everything downstream of its source are skipped; otherwise it runs when any of them succeeded. When all of them skip softly, only the load is skipped: the source keeps its last loaded data, and models reading it still build when another of their inputs ran. - Lineage and column lineage restart at the source: the source's columns come from the loader, not - from the models the task read. -- Job groups run a member that builds the task before the member that runs the loader, and that one + from the models the asset read. +- Job groups run a member that builds the asset before the member that runs the loader, and that one before the members that build models reading its source. -- `--changed` follows the chain: when the SQL the task reads is rebuilt, the task and loader run and - the models reading the source rebuild. When it is not rebuilt, the task does not run and the +- `--changed` follows the chain: when the SQL the asset reads is rebuilt, the asset and loader run and + the models reading the source rebuild. When it is not rebuilt, the asset does not run and the loader is skipped softly, so the source keeps its data and models selected for their own changes still build. -- A task that reads anything built from the source its own downstream loader fills is a cycle, +- An asset that reads anything built from the source its own downstream loader fills is a cycle, which stops the plan or build with the path, such as - `loader 'partner_scores' -> task 'score_orders' -> model 'scored_orders' -> source 'partner_scores' -> loader 'partner_scores'`. + `loader 'partner_scores' -> asset 'score_orders' -> model 'scored_orders' -> source 'partner_scores' -> loader 'partner_scores'`. ## Decorators @@ -123,7 +121,7 @@ Every decorator accepts the same organizational metadata: | `description` | Required human-readable docs; defaults to the function docstring ([Descriptions](/docs/concepts/models/configuration#descriptions)) | | `meta` | Freeform JSON metadata for catalogs and integrations | -`@task` and `@asset` also accept a `retry` policy. `@asset` additionally accepts `columns` and `column_lineage`. Node kind is inferred from the decorator - you never pass `kind=`. +`@asset` also accepts a `retry` policy, `columns` and `column_lineage`. Node kind is inferred from the decorator - you never pass `kind=`. ## Identity tracking @@ -133,7 +131,7 @@ Unlike SQL models, Python nodes may depend on external inputs the framework cann ## Runtime context -Each node receives a context object as its first argument (`TaskContext`, `AssetContext`, `CheckContext`, or `LoaderContext`). They share these helpers: +Each node receives a context object as its first argument (`AssetContext`, `CheckContext`, or `LoaderContext`). They share these helpers: | Helper | Description | |--------|-------------| @@ -151,7 +149,7 @@ Each node receives a context object as its first argument (`TaskContext`, `Asset | `ctx.results_of(node_fn, limit=N)` | Read the last N successful results of an upstream node, newest first | | `ctx.providers` | Access discovered [providers](/docs/concepts/python-nodes/providers) by name | -Task and asset contexts add `ctx.result(...)` and `ctx.skip(...)`. Check contexts add `ctx.pass_(...)`, `ctx.fail(...)`, and `ctx.warn(...)`. +Asset contexts add `ctx.result(...)` and `ctx.skip(...)`. Check contexts add `ctx.pass_(...)`, `ctx.fail(...)`, and `ctx.warn(...)`. Providers can also be injected directly as function parameters by name. See [Providers](/docs/concepts/python-nodes/providers) for details. @@ -181,17 +179,17 @@ Connection settings and secrets are not part of `ctx.target_config`. ## Returns and skips -Tasks and assets return through `ctx.result(...)`: +Assets return through `ctx.result(...)`: ```python -@task +@asset def export_orders(ctx): """Export orders.""" return ctx.result(payload={"rows": 120}, metadata={"rows": 120}) ``` - A plain value or `None` is also accepted and normalized to a successful result. -- `ctx.skip(reason, mode=...)` skips the node. `mode` accepts `"soft"` (default, skips only this node) or `"hard"` (also blocks dependents), as a string or the `SkipMode` enum from `sqlbuild.tasks`/`sqlbuild.assets`. +- `ctx.skip(reason, mode=...)` skips the node. `mode` accepts `"soft"` (default, skips only this node) or `"hard"` (also blocks dependents), as a string or the `SkipMode` enum from `sqlbuild.assets`. - Assets may pass `materialized=True`/`False` to record whether an artifact was produced. Within one run, downstream nodes run only if at least one upstream succeeded. If all upstreams are skipped, the downstream is skipped. A failed or hard-skipped upstream blocks its dependents. @@ -202,7 +200,7 @@ Node results (payload, metadata, status, errors) are persisted after each execut `_sqlbuild_node_results` table. Results persist across runs, so they are available for observability, debugging, and downstream consumption through `ctx.result_of`. -Results of loaders, tasks, assets and checks all go to one schema, chosen by the target's `schema`: +Results of loaders, assets and checks all go to one schema, chosen by the target's `schema`: | Target `schema` | Results table schema | |-----------------|----------------------| @@ -218,37 +216,36 @@ Python nodes are selected like SQL resources, by bare name or typed selector: ```bash sqb build --select export_orders # bare name -sqb build --select task:export_orders # typed -sqb build --select asset:orders_export +sqb build --select asset:orders_export # typed sqb build --select export_orders check:check_orders_exported # a check with the node it checks sqb build --select tag:exports # by tag sqb build --select +orders_export # with upstreams ``` -Names are globally unique across models, sources, seeds, functions, loaders, tasks, assets, checks, providers, hooks, unit tests, scenarios, jobs, job groups, sensors, and calendars. +Names are globally unique across models, sources, seeds, functions, loaders, assets, checks, providers, hooks, unit tests, scenarios, jobs, job groups, sensors, and calendars. ## Lifecycle: run, build, check Python nodes run in two phases relative to SQL: -- **Ingress (pre-SQL):** loaders, and tasks/assets that feed sources, run before SQL models are built. -- **Read-side (post-SQL):** tasks/assets that read SQL run after their SQL dependencies are built. +- **Ingress (pre-SQL):** loaders, and assets that feed sources, run before SQL models are built. +- **Read-side (post-SQL):** assets that read SQL run after their SQL dependencies are built. A loader in a [warehouse round trip](#warehouse-round-trips) loads after them, and before the models that read its source. -Read-side tasks and assets follow the same selection rule as SQL models. A SQL dependency that is +Read-side assets follow the same selection rule as SQL models. A SQL dependency that is part of the run gates the node: it waits for that model, seed, or source load, and is skipped if the dependency fails or is skipped. A dependency that is not part of the run (not selected, or an -unmanaged source) is read as it currently exists, so `sqb build --select task:export_orders` runs -the task against the relations already in the warehouse. +unmanaged source) is read as it currently exists, so `sqb build --select asset:export_orders` runs +the asset against the relations already in the warehouse. -Python dependencies follow the same rule. A task or asset selected without its Python upstream +Python dependencies follow the same rule. An asset selected without its Python upstream reads that upstream's latest successful result through `ctx.result_of`. If the upstream has never succeeded for the target, the node fails with an error that names the upstream, for example: ```text -Upstream task 'prepare_orders' has no successful result to read. Build it first with -'sqb build --select task:prepare_orders', or build both with 'sqb build --select +asset:export_orders'. +Upstream asset 'prepare_orders' has no successful result to read. Build it first with +'sqb build --select asset:prepare_orders', or build both with 'sqb build --select +asset:export_orders'. ``` Jobs, sensors, roles and ad-hoc runs on the server resolve selectors the same way, so a job can @@ -256,8 +253,8 @@ select one Python node and a job group orders it after the job that builds its u The commands differ in what they include by default: -| Command | SQL | Loaders / tasks / assets | Checks | Audits | -|---------|-----|--------------------------|--------|--------| +| Command | SQL | Loaders / assets | Checks | Audits | +|---------|-----|-----------------|--------|--------| | `sqb build` | Yes | Yes | Yes | Yes | | `sqb build --no-tests --no-audits` | Yes | Yes | No | No | | `sqb check` | No | No | Selected checks only | No | @@ -266,11 +263,11 @@ The commands differ in what they include by default: - `sqb build --no-tests --no-audits` executes the DAG without validation, for fast iteration. - `sqb check` runs Python checks only. See [`sqb check`](/docs/cli/check). -Use `--no-python` on `plan` and `build` to suppress read-side tasks/assets. Loader-side Python required to populate selected sources still runs (use `--no-load` to skip source loading). See the [`sqb build`](/docs/cli/build) reference. +Use `--no-python` on `plan` and `build` to suppress read-side assets. Loader-side Python required to populate selected sources still runs (use `--no-load` to skip source loading). See the [`sqb build`](/docs/cli/build) reference. ## Try it -The `python_nodes` playground is a small working project with a task, loader, model, asset, and check: +The `python_nodes` playground is a small working project with assets, a loader, a model, and a check: ```bash sqb playground --template python_nodes diff --git a/website/src/content/docs/docs/concepts/python-nodes/providers.mdx b/website/src/content/docs/docs/concepts/python-nodes/providers.mdx index 05e8e8eab6..b26a4c195c 100644 --- a/website/src/content/docs/docs/concepts/python-nodes/providers.mdx +++ b/website/src/content/docs/docs/concepts/python-nodes/providers.mdx @@ -45,16 +45,16 @@ class WarehouseClient(Provider): api_key: str ``` -Provider names must be valid Python identifiers (`lower_snake_case`). They share the global project resource namespace with models, sources, seeds, functions, loaders, tasks, assets, checks, and hooks. +Provider names must be valid Python identifiers (`lower_snake_case`). They share the global project resource namespace with models, sources, seeds, functions, loaders, assets, checks, and hooks. ## Using providers in Python nodes Providers are injected into Python node functions by **parameter name**. Add a parameter whose name matches the provider's runtime name: ```python -from sqlbuild.tasks import task +from sqlbuild.assets import asset -@task +@asset def export_orders(ctx, warehouse_client): """Export orders.""" warehouse_client.session.upload(ctx.query("SELECT * FROM orders")) @@ -65,10 +65,10 @@ SQLBuild matches the parameter name `warehouse_client` to the discovered provide You can also type-annotate the parameter for IDE support and compile-time validation: ```python -from sqlbuild.tasks import task +from sqlbuild.assets import asset from providers.warehouse_client.provider import WarehouseClient -@task +@asset def export_orders(ctx, warehouse_client: WarehouseClient): """Export orders.""" warehouse_client.session.upload(ctx.query("SELECT * FROM orders")) @@ -79,7 +79,6 @@ When a type annotation is present, SQLBuild validates that the discovered provid Provider injection works in all Python node types: - **Loaders** (`@loader`) -- **Tasks** (`@task`) - **Assets** (`@asset`) - **Checks** (`@check`) @@ -117,10 +116,10 @@ def notify_complete(ctx): ## Using providers via context -All Python node contexts (`TaskContext`, `AssetContext`, `CheckContext`, `LoaderContext`) and `HookContext` expose a `ctx.providers` container for name-based access: +All Python node contexts (`AssetContext`, `CheckContext`, `LoaderContext`) and `HookContext` expose a `ctx.providers` container for name-based access: ```python -@task +@asset def export_orders(ctx): """Export orders.""" client = ctx.providers.warehouse_client @@ -272,7 +271,7 @@ provider with its class and docstring. A provider's page shows: | Tab | Shows | | --- | --- | | Settings | Each setting's type, environment variable and whether it is required; secret settings are marked, and flagged **missing** when the location's environment doesn't set them | -| Used by | The loaders, tasks, hooks and other code that take the provider, and the jobs that run that code | +| Used by | The loaders, assets, hooks and other code that take the provider, and the jobs that run that code | Setting values, defaults included, are never published. diff --git a/website/src/content/docs/docs/concepts/python-nodes/sql-references.mdx b/website/src/content/docs/docs/concepts/python-nodes/sql-references.mdx index b485d4eb8e..65b3599f8c 100644 --- a/website/src/content/docs/docs/concepts/python-nodes/sql-references.mdx +++ b/website/src/content/docs/docs/concepts/python-nodes/sql-references.mdx @@ -9,26 +9,26 @@ Python nodes can **read** SQL models, sources, and seeds at runtime, but SQL can ## Declaring a reference -Use `model()`, `source()`, and `seed()` from `sqlbuild.refs` in the `depends_on` of a task, asset, or check: +Use `model()`, `source()`, and `seed()` from `sqlbuild.refs` in the `depends_on` of an asset or check: ```python from sqlbuild.refs import model, seed, source -from sqlbuild.tasks import task +from sqlbuild.assets import asset -@task(depends_on=model("fact_orders")) +@asset(depends_on=model("fact_orders")) def export_orders(ctx): """Export orders.""" ... -@task(depends_on=source("raw_orders")) +@asset(depends_on=source("raw_orders")) def inspect_raw(ctx): """Inspect raw.""" ... -@task(depends_on=seed("country_codes")) +@asset(depends_on=seed("country_codes")) def export_countries(ctx): """Export countries.""" ... @@ -41,14 +41,14 @@ Declaring the reference does two things: It does **not** make the SQL resource depend on the Python node. The dependency is one-way: Python reads SQL, never the reverse. -The reference is a graph edge. It appears in `sqb dag --json`, and selector expansion follows it: `+task:export_countries` also selects the `country_codes` seed and its upstream, and `country_codes+` also selects the tasks and assets that read it. Checks are never added by expansion; they run when their dependencies run. +The reference is a graph edge. It appears in `sqb dag --json`, and selector expansion follows it: `+asset:export_countries` also selects the `country_codes` seed and its upstream, and `country_codes+` also selects the assets that read it. Checks are never added by expansion; they run when their dependencies run. ## Resolving with ctx.relation Always resolve a reference to its concrete relation with `ctx.relation(...)` instead of hardcoding the table name: ```python -@task(depends_on=model("fact_orders")) +@asset(depends_on=model("fact_orders")) def export_orders(ctx): """Export orders.""" relation = ctx.relation(model("fact_orders")) @@ -64,7 +64,7 @@ Do not hardcode model or source names in raw SQL. A model's relation depends on ## Hard-coded relation names -SQLBuild checks SQL passed to `ctx.query()` and `ctx.execute_sql()` in tasks, assets, checks, +SQLBuild checks SQL passed to `ctx.query()` and `ctx.execute_sql()` in assets, checks, loaders, and [Python hooks](/docs/concepts/models/hooks/python) for names of project relations: models, sources, and seeds. @@ -73,8 +73,8 @@ literal text, is parsed with the project's SQL dialect. Naming a project relatio with `P008`: ``` -error[P008]: task:export_orders names model:fact_orders as 'fact_orders' in SQL passed to ctx.query() - --> python/tasks/export.py:9 +error[P008]: asset:export_orders names model:fact_orders as 'fact_orders' in SQL passed to ctx.query() + --> python/exports/export.py:9 = help: declare it with depends_on=model("fact_orders") and use ctx.relation(model("fact_orders")) instead of the relation name = help: if 'fact_orders' is an external table that shares the name, qualify it with its schema = help: while migrating a project, allow hard-coded relation names with [references] enforce_explicit = false in sqlbuild_project.toml @@ -122,16 +122,16 @@ Naming every upstream model ties a node to model names that change with each ren ```python from sqlbuild.refs import job, tag -from sqlbuild.tasks import task +from sqlbuild.assets import asset -@task(depends_on=(job("orders_job", exclude="tag:experimental"), tag("finance_inputs"))) +@asset(depends_on=(job("orders_job", exclude="tag:experimental"), tag("finance_inputs"))) def publish_revenue(ctx): """Publish revenue.""" ... ``` -- `job(name)` depends on the job's **end nodes**: the models, seeds, sources, loaders, tasks and +- `job(name)` depends on the job's **end nodes**: the models, seeds, sources, loaders and assets of its selection that nothing else in the selection reads. Everything else in the job is upstream of an end node, so this waits for the whole job without listing every edge. A job group name depends on the union of its jobs. @@ -140,7 +140,7 @@ def publish_revenue(ctx): `kind:`) and removes those nodes before the set is reduced to its end nodes. Compile resolves each into ordinary edges to the end nodes, so planning, `--changed`, selector -expansion (`+task:publish_revenue`), `sqb dag` and job group waits treat them like named +expansion (`+asset:publish_revenue`), `sqb dag` and job group waits treat them like named dependencies. A model named directly and also reached through a job is one edge. Jobs resolve whatever targets they are declared for. @@ -152,7 +152,7 @@ it was last built with: ```text Python read-side (1) -└── publish_revenue task (unchanged) +└── publish_revenue asset (unchanged) └── publish_revenue now also waits on model 'refund_totals' through job 'orders_job' ``` @@ -167,7 +167,7 @@ Compile fails when: by job, so if the dependant's job waits for the job it depends on while that job reads a node of the dependant's job, neither can start. The error names the nodes that close the loop; move them to another job; -- a loader declares one: loaders depend on loaders, tasks and assets named one by one; +- a loader declares one: loaders depend on loaders and assets named one by one; - the name has a code location prefix such as `job("billing:nightly_job")`, which is reserved for depending on other locations' jobs. @@ -183,7 +183,7 @@ Compile fails when: ## Where references are allowed -- **Tasks**, **assets**, and **checks** may declare `model()`, `source()`, and `seed()` references and read them with `ctx.relation(...)`, and depend on a `job()` or `tag()`. +- **Assets** and **checks** may declare `model()`, `source()`, and `seed()` references and read them with `ctx.relation(...)`, and depend on a `job()` or `tag()`. - **Python hooks** declare the models, sources, and seeds they read with `@hook(reads=...)`; see [Python Hooks](/docs/concepts/models/hooks/python#declared-reads). - **Loaders** may not reference SQL resources; they read sources with `ctx.source(...)` and upstream loaders with `ctx.loader(...)`. - A Python node referencing a SQL resource never turns into a SQL dependency; selector expansion will not pull Python outputs into SQL model dependencies. diff --git a/website/src/content/docs/docs/concepts/python-nodes/tasks.mdx b/website/src/content/docs/docs/concepts/python-nodes/tasks.mdx deleted file mode 100644 index f7f4c4860a..0000000000 --- a/website/src/content/docs/docs/concepts/python-nodes/tasks.mdx +++ /dev/null @@ -1,141 +0,0 @@ ---- -title: "Tasks" -description: "Run Python computation and side effects as nodes in the SQLBuild graph." ---- - -Tasks are Python functions that run as part of the DAG. Use them for computation, side effects, and orchestration steps that are not loading data into a source and not producing a tracked artifact. See [Python Nodes](/docs/concepts/python-nodes/overview) for the shared model and the SQL boundary rules. - -## Defining a task - -Place Python files under `python/` (the playgrounds use `python/tasks/`) and decorate functions with `@task`: - -```python -# python/tasks/orders.py -from sqlbuild.tasks import task, TaskContext - - -@task -def export_orders(ctx: TaskContext): - """Export orders.""" - rows = fetch_orders() - return ctx.result(payload={"rows": len(rows)}, metadata={"rows": len(rows)}) -``` - -The task receives a `TaskContext` and returns through `ctx.result(...)`. A plain return value or `None` is also accepted and normalized to a successful result. - -## Dependencies - -Tasks declare dependencies with `depends_on`, accepting a single function, a tuple, or a list: - -```python -@task -def fetch_orders(ctx: TaskContext): - """Fetch orders.""" - return ctx.result(payload=download_orders()) - - -@task(depends_on=fetch_orders) -def summarize_orders(ctx: TaskContext): - """Summarize orders.""" - result = ctx.result_of(fetch_orders) - return ctx.result(metadata={"count": len(result.payload)}) -``` - -`ctx.result_of(node_fn)` reads the latest persisted result of an upstream node, returning a `NodeResultEnvelope` with `payload`, `metadata`, `status`, and `ts` fields. Results persist across runs. Use `ctx.results_of(node_fn, limit=N)` to read result history. Reading a missing or unsuccessful upstream raises unless you pass `default=`. - -Tasks may depend on other tasks, assets, and loaders. They may **not** depend on SQL models or sources as graph dependencies, but they can read them at runtime with typed references - see [SQL references](/docs/concepts/python-nodes/sql-references). - -## Returning results - -```python -@task -def build_export(ctx: TaskContext): - """Build export.""" - return ctx.result( - payload={"path": "/exports/orders.csv"}, - metadata={"rows": 1200}, - ) -``` - -- `payload` is the value downstream nodes read with `ctx.result_of(...)`. -- `metadata` is structured JSON for catalogs and downstream reads. -- Tasks cannot set `materialized` - that is for [assets](/docs/concepts/python-nodes/assets). - -## Skipping - -Return `ctx.skip(...)` to skip a task: - -```python -@task -def export_if_present(ctx: TaskContext): - """Export if present.""" - if not new_files_available(): - return ctx.skip("no new files") - return ctx.result(payload=do_export()) -``` - -- `"soft"` (default) skips only this task; dependents may still run if another upstream succeeded. -- `"hard"` skips this task and blocks its dependents. - -`mode` accepts either a plain string or the `SkipMode` enum: - -```python -from sqlbuild.tasks import task, SkipMode - - -@task -def optional_step(ctx): - """Optional step.""" - return ctx.skip("nothing to do", mode=SkipMode.HARD) # or mode="hard" -``` - -## Retries - -`@task` accepts a `retry` policy for transient failures: - -```python -from sqlbuild.retries import RetryPolicy -from sqlbuild.tasks import task - - -@task(retry=RetryPolicy(max_attempts=3, retry_on=(ConnectionError,))) -def call_api(ctx): - """Call api.""" - return ctx.result(payload=fetch_from_flaky_api()) -``` - -| Field | Default | Description | -|-------|---------|-------------| -| `max_attempts` | `3` | Total attempts including the first | -| `retry_on` | `Exception` | Exception class, tuple, or list to retry on | -| `initial_delay_seconds` | `1.0` | Delay before the first retry | -| `backoff_multiplier` | `2.0` | Exponential backoff multiplier | -| `max_delay_seconds` | `30.0` | Cap on any single delay | -| `max_elapsed_seconds` | `None` | Overall time bound for retries | -| `jitter` | `True` | Randomize delays to avoid synchronized retries | - -The default is no retry. Set `retry_on` explicitly rather than relying on the broad default when you can. The original exception is preserved if all attempts fail. - -## Decorator parameters - -| Parameter | Description | -|-----------|-------------| -| `name` | Override the node name (defaults to the function name) | -| `depends_on` | Upstream nodes (function, tuple, or list); `model()`/`source()`/`seed()` for read-only SQL refs | -| `tags` | Labels for selection and grouping | -| `group` | Display/catalog grouping | -| `description` | Docs (defaults to docstring) | -| `meta` | Freeform JSON metadata | -| `retry` | A `RetryPolicy` | - -## Running tasks - -```bash -# Run a task and its required graph -sqb build --select export_orders --no-tests --no-audits - -# Include in a full build -sqb build --select export_orders -``` - -Tasks run during `sqb build`. They are not validated by checks unless you also write a [check](/docs/concepts/python-nodes/checks) that depends on them. diff --git a/website/src/content/docs/docs/concepts/rules/orchestration-rules.mdx b/website/src/content/docs/docs/concepts/rules/orchestration-rules.mdx index 1e5038edee..4564307eef 100644 --- a/website/src/content/docs/docs/concepts/rules/orchestration-rules.mdx +++ b/website/src/content/docs/docs/concepts/rules/orchestration-rules.mdx @@ -35,7 +35,7 @@ The Rule reads the `select` of every [job](/docs/concepts/orchestration#declare- | Atom | Counts | | --- | --- | | A bare name, such as `order_totals` | Yes | -| A typed name, such as `source:raw_orders` or `task:refund_review` | Yes | +| A typed name, such as `source:raw_orders` or `asset:refund_review` | Yes | | Each name in an intersection, such as `order_totals,tag:finance` | The name only | | `tag:`, a folder path, `job:`, a job or job group name | No | | A name with `+` before or after it | No | diff --git a/website/src/content/docs/docs/concepts/rules/python-rules.mdx b/website/src/content/docs/docs/concepts/rules/python-rules.mdx index 7e7402a029..e14afda84a 100644 --- a/website/src/content/docs/docs/concepts/rules/python-rules.mdx +++ b/website/src/content/docs/docs/concepts/rules/python-rules.mdx @@ -64,7 +64,7 @@ python/ __init__.py fulfillment/ shipment_export/ - node.py # @task shipment_export + node.py # @asset shipment_export main.py # main(), the unit's implementation utils/ currency/ @@ -97,7 +97,7 @@ python/orders/ checks.py # @check declarations for this unit _helpers/ # private functions for this unit refund_review/ - node.py # @task + node.py # @asset main.py ``` @@ -123,21 +123,21 @@ These are reported: | Code | Finding | Fix | |---|---|---| | `SQBRPYTHON003` | A unit without `main.py`, or another module or folder in it | Add `main.py`; move other code into `_helpers/` or a role file | -| `SQBRPYTHON004` | `node.py` holding a helper, a constant, a class or a second declaration | Keep one `@loader`, `@task`, `@asset` or `@factory` function and its imports | +| `SQBRPYTHON004` | `node.py` holding a helper, a constant, a class or a second declaration | Keep one `@loader`, `@asset` or `@factory` function and its imports | | `SQBRPYTHON005` | A declaration that does more than `return main(...)`, or passes a positional argument | Move the logic into `main()` and pass every argument as `name=value` | | `SQBRPYTHON006` | `main.py` holding more than one function, or a `main()` whose return annotation does not fit the node | Move other code into `_helpers/` and annotate `main()` with its result | | `SQBRPYTHON007` | A signature other than `(ctx: Context, *, provider: ProviderClass)` | Take `ctx` positionally and providers after `*`, each annotated | | `SQBRPYTHON008` | An import of another unit's `main.py`, role files or `_helpers/`, or another domain's code | Import shared code from `python/utils/` or the domain's role files | | `SQBRPYTHON009` | `checks.py` holding other code, or a check that does not depend on the unit's node | Name the unit's node in `depends_on` and return directly | | `SQBRPYTHON010` | A factory that does not loop over the unit's specs, or a generated node without `name=` | Loop over specs from the unit's `constants.py` or `models.py` and name each node | -| `SQBRPYTHON011` | A docstring on a node or check, a task, asset or check without `description=`, or a described loader | Use `description=`; a loader takes its description from its managed source | +| `SQBRPYTHON011` | A docstring on a node or check, an asset or check without `description=`, or a described loader | Use `description=`; a loader takes its description from its managed source | | `SQBRPYTHON012` | A node declared outside a unit's `node.py`, or a check outside its `checks.py` | Move it into its own unit folder | `main()` returns: | Node | Return annotation | |---|---| -| `@task`, `@asset` | `PythonNodeResult` or `PythonNodeSkipResult`, from `ctx.result(...)` and `ctx.skip(...)` | +| `@asset` | `PythonNodeResult` or `PythonNodeSkipResult`, from `ctx.result(...)` and `ctx.skip(...)` | | `@loader` | rows such as `list[...]` or `Iterator[...]`, `LoaderResult`, `LoaderSkipResult`, or `None` | A unit may import another unit's `node.py` only to name that node in `depends_on`. It may import @@ -322,7 +322,7 @@ The other size limits check every function. ## Unreachable definitions `SQBRPYTHON306` reports functions, classes and module values that nothing SQLBuild loads can reach. -SQLBuild loads every decorated function under `python/` (`@loader`, `@task`, `@asset`, `@check`, +SQLBuild loads every decorated function under `python/` (`@loader`, `@asset`, `@check`, `@factory` and `@audit_factory`) and everything defined outside `python/`, such as hooks and providers. A definition reached only through them is live. Imports through folders without an `__init__.py` count too. @@ -389,7 +389,7 @@ remediation. | `SQBRPYTHON008` | node-import-boundaries | `python/` | yes | A unit imports only what it owns, its domain's role files, `python/utils/`, provider role files and other nodes for `depends_on` | | `SQBRPYTHON009` | checks-module-shape | `python/` | yes | `checks.py` holds checks of the unit's node that return directly, or one factory | | `SQBRPYTHON010` | factory-shape | `python/` | yes | A factory loops over its unit's specs and names every node it generates | -| `SQBRPYTHON011` | node-description-not-docstring | `python/` | yes | Tasks, assets and checks set `description=` without a docstring; loaders set neither | +| `SQBRPYTHON011` | node-description-not-docstring | `python/` | yes | Assets and checks set `description=` without a docstring; loaders set neither | | `SQBRPYTHON012` | node-declared-in-unit | `python/` | yes | Nodes are declared in a unit's `node.py`, and checks in its `checks.py` | ### Providers diff --git a/website/src/content/docs/docs/concepts/selectors.mdx b/website/src/content/docs/docs/concepts/selectors.mdx index e8ec7d9eaa..9ac52edc5e 100644 --- a/website/src/content/docs/docs/concepts/selectors.mdx +++ b/website/src/content/docs/docs/concepts/selectors.mdx @@ -58,7 +58,7 @@ sqb build --select models/marts sqb build --select path:models/intermediate ``` -Any name containing `/` is treated as a path selector, so `path:models/marts` and a bare `models/marts` work the same way. Path selectors require an explicit root directory: `models/` for SQL models or `python/` for Python nodes. Nested paths work too: `models/staging/orders` or `python/tasks`. +Any name containing `/` is treated as a path selector, so `path:models/marts` and a bare `models/marts` work the same way. Path selectors require an explicit root directory: `models/` for SQL models or `python/` for Python nodes. Nested paths work too: `models/staging/orders` or `python/exports`. ### Seed and source @@ -77,7 +77,7 @@ sqb build --select "kind:model,path:models/marts" sqb build --select +kind:asset ``` -The kinds are `model`, `source`, `seed`, `udf`, `table_fn`, `task`, `asset`, `loader` and +The kinds are `model`, `source`, `seed`, `udf`, `table_fn`, `asset`, `loader` and `check`, as the server's catalog names them, and `function`, which selects both `udf` and `table_fn`. Kind selectors compose with `+`, `,` and `--exclude` like tag selectors. An unknown kind is an error that names the closest kinds, and a kind your diff --git a/website/src/content/docs/docs/feature-comparison.mdx b/website/src/content/docs/docs/feature-comparison.mdx index 7e6778372b..97ef74f791 100644 --- a/website/src/content/docs/docs/feature-comparison.mdx +++ b/website/src/content/docs/docs/feature-comparison.mdx @@ -79,7 +79,7 @@ SQLBuild, dbt, and SQLMesh are all SQL pipeline frameworks. They share common gr | Feature | SQLBuild | dbt | SQLMesh | |---------|----------|-----|---------| | SQL models | `MODEL()` header with inline config | Jinja-templated SQL + YAML sidecar | `MODEL` DDL | -| Python models | Python nodes (tasks, assets, loaders, checks); no dataframe models | Pandas, PySpark, Snowpark, BigFrames | Pandas, PySpark, Snowpark, BigFrames | +| Python models | Python nodes (assets, loaders, checks); no dataframe models | Pandas, PySpark, Snowpark, BigFrames | Pandas, PySpark, Snowpark, BigFrames | | Custom materializations | Python with full framework hooks | Jinja-based | Python-based custom model kinds | | Lifecycle hooks | Typed inline SQL, reusable parameterized SQL resources, and Python hooks with compile-time validation and `HookContext` | Jinja pre/post hooks | Python pre/post hooks | @@ -87,9 +87,8 @@ SQLBuild, dbt, and SQLMesh are all SQL pipeline frameworks. They share common gr | Feature | SQLBuild | dbt | SQLMesh | |---------|----------|-----|---------| -| Tasks | `@task` - Python computation as DAG nodes | No | No | | Assets | `@asset` - external artifact production/observation | No | No | -| Checks | `@check` - Python validation of tasks, assets, and loaders | No | No | +| Checks | `@check` - Python validation of assets and loaders | No | No | | Factories | `@factory` - programmatic node generation | No | No | | Providers | Shared runtime services with name-based injection into nodes and hooks | No | No | diff --git a/website/src/content/docs/docs/integrations/dagster-reference.mdx b/website/src/content/docs/docs/integrations/dagster-reference.mdx index a0b44dc6b9..98e4738123 100644 --- a/website/src/content/docs/docs/integrations/dagster-reference.mdx +++ b/website/src/content/docs/docs/integrations/dagster-reference.mdx @@ -238,7 +238,6 @@ Python node asset keys use a two-part key with the node kind as prefix: | Kind | Asset key example | |------|-------------------| -| Task | `("task", "prepare_orders")` | | Asset | `("asset", "orders_export")` | | Loader | `("loader", "raw_orders")` | diff --git a/website/src/content/docs/docs/integrations/dagster.mdx b/website/src/content/docs/docs/integrations/dagster.mdx index c64e10b955..e0f34788a1 100644 --- a/website/src/content/docs/docs/integrations/dagster.mdx +++ b/website/src/content/docs/docs/integrations/dagster.mdx @@ -3,7 +3,7 @@ title: "Overview" description: "Orchestrate SQLBuild pipelines with Dagster scheduling, retries, and asset UI." --- -SQLBuild includes a Dagster integration that maps your project's models, sources, seeds, functions, loaders, tasks, assets, tests, audits, scenarios, and Python checks into Dagster assets and asset checks. SQLBuild handles the SQL transformation layer. Dagster handles scheduling, retries, alerting, and the asset-centric UI. +SQLBuild includes a Dagster integration that maps your project's models, sources, seeds, functions, loaders, assets, tests, audits, scenarios, and Python checks into Dagster assets and asset checks. SQLBuild handles the SQL transformation layer. Dagster handles scheduling, retries, alerting, and the asset-centric UI. ## Install @@ -28,7 +28,7 @@ This creates a Dagster code location with a `definitions.py` that includes asset ## How it works 1. `sqb compile --dag` generates a static `sqlbuild_dag.json` artifact with your project's full graph (nodes, edges, checks) -2. `@sqlbuild_assets()` reads the artifact and creates one Dagster `AssetSpec` per source, seed, model, function, loader, task, and asset, with dependency edges preserved +2. `@sqlbuild_assets()` reads the artifact and creates one Dagster `AssetSpec` per source, seed, model, function, loader, and asset, with dependency edges preserved 3. `SqlBuildCliResource` shells out to `sqb build`, `sqb test`, `sqb scenario test`, etc. as subprocesses 4. Execution results (materializations, audit pass/fail, scenario outcomes) are read from SQLBuild's structured integration-result stream and emitted as Dagster `MaterializeResult` and `AssetCheckResult` events @@ -64,7 +64,7 @@ dagster dev -f definitions.py `definitions.py` belongs to your Dagster code location and sits next to the SQLBuild project directory (`my_sqlbuild_project/` here), not inside it. SQLBuild projects contain no orchestrator code. -Dagster discovers every SQLBuild model, loader, task, and asset as a Dagster asset. Selecting a subset in the Dagster UI automatically scopes the `sqb build` invocation to those nodes via `--select`. +Dagster discovers every SQLBuild model, loader, and asset as a Dagster asset. Selecting a subset in the Dagster UI automatically scopes the `sqb build` invocation to those nodes via `--select`. For production deployments, use `project.prepare()` or `sqb compile --dag` to generate the DAG artifact explicitly in your CI pipeline. @@ -88,7 +88,7 @@ SQLBuild tests, audits, scenarios, and Python checks are registered as Dagster a - **Unit tests** become checks attached to the models they test - **Audits** become checks attached to the model, source, or seed they audit, with severity mapped to `AssetCheckSeverity.ERROR` or `AssetCheckSeverity.WARN`. A singular audit that runs at the end of the build becomes a check on each model it reads - **Scenarios** become checks attached to the models they exercise -- **Python checks** (`@check`) become checks attached to the tasks, assets, or loaders they validate +- **Python checks** (`@check`) become checks attached to the assets or loaders they validate Check results are emitted with pass/fail status and metadata from the execution JSON. diff --git a/website/src/content/docs/docs/integrations/rivers.mdx b/website/src/content/docs/docs/integrations/rivers.mdx index c8a17a11a0..a19494d217 100644 --- a/website/src/content/docs/docs/integrations/rivers.mdx +++ b/website/src/content/docs/docs/integrations/rivers.mdx @@ -3,7 +3,7 @@ title: "Rivers" description: "Orchestrate SQLBuild pipelines with Rivers scheduling, jobs, and asset tracking." --- -SQLBuild includes a Rivers integration that maps your project's models, sources, seeds, loaders, tasks, assets, and functions into Rivers assets with dependency edges preserved. SQLBuild handles the SQL transformation layer. Rivers handles scheduling, execution, and the asset-centric UI. +SQLBuild includes a Rivers integration that maps your project's models, sources, seeds, loaders, assets, and functions into Rivers assets with dependency edges preserved. SQLBuild handles the SQL transformation layer. Rivers handles scheduling, execution, and the asset-centric UI. ## Install @@ -28,7 +28,7 @@ This creates a Rivers code repository with a `definitions.py` that includes asse ## How it works 1. `sqb compile --dag` generates a static `sqlbuild_dag.json` artifact with your project's full graph (nodes, edges) -2. `@sqlbuild_assets()` reads the artifact and creates one Rivers `AssetDef` per source, loader, seed, model, function, task, and asset, with dependency edges preserved +2. `@sqlbuild_assets()` reads the artifact and creates one Rivers `AssetDef` per source, loader, seed, model, function, and asset, with dependency edges preserved 3. The decorated function runs `sqb build` as a subprocess and yields `Materialization` events for each output ## Quickstart @@ -150,7 +150,7 @@ def my_assets(context): | Method | Default behavior | |--------|-----------------| -| `get_asset_name(node)` | Joins the asset key parts with `__` (e.g. `task__prepare_orders`, `asset__orders_export`) | +| `get_asset_name(node)` | Joins the asset key parts with `__` (e.g. `asset__prepare_orders`, `loader__raw_orders`) | | `get_group_name(node)` | Uses the project name | | `get_tags(node)` | Includes `sqlbuild/kind:` plus model tags | | `get_kinds(node)` | Returns `["sqlbuild", ""]` | diff --git a/website/src/sidebar.json b/website/src/sidebar.json index ea80e3d9da..00db60622d 100644 --- a/website/src/sidebar.json +++ b/website/src/sidebar.json @@ -175,7 +175,6 @@ "items": [ "docs/concepts/python-nodes/overview", "docs/concepts/python-nodes/loaders", - "docs/concepts/python-nodes/tasks", "docs/concepts/python-nodes/assets", "docs/concepts/python-nodes/checks", "docs/concepts/python-nodes/factories", From cf4bd481b0d1b3886f22c6ee485a4c14a25ebecc Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 20 Nov 2026 22:14:15 +0000 Subject: [PATCH 05/12] feat!: remove the task kind from the UI, events and remaining tests Drop task from the Lineage, Catalog, change and provider views, from the publication, integration-result and definition-change kind tables, and from the python_task lifecycle and progress operations. Move the tests, helpers and docs that still used @task or the task kind to @asset. --- .../edge_cases/loaders/loader_results.py | 4 +- .../variants/ingress_gate/partner.py | 2 +- .../orchestration/node_result_history.json | 2 +- .../src/definition_changes/constants.rs | 3 +- .../src/integration_results/constants.rs | 4 +- .../src/lifecycle_events/constants.rs | 3 +- .../sqlbuild-core/src/progress/constants.rs | 6 +- .../runs/_helpers/lifecycle/publications.rs | 6 +- .../tests/runs/test_group_round_trips.rs | 2 +- .../sqlbuild/references/docs/cli/build.md | 2 +- .../docs/concepts/python-nodes/assets.md | 4 +- .../commands/main/bigquery/test_bigquery.py | 2 +- .../cli/commands/main/build/test_build.py | 8 +-- .../main/build/test_node_result_location.py | 12 ++-- .../compile/test_required_descriptions.py | 45 ++++----------- .../main/databricks/test_databricks.py | 2 +- .../cli/commands/main/load/test_load.py | 9 --- .../commands/main/postgres/test_postgres.py | 2 +- .../commands/main/providers/test_providers.py | 2 +- .../commands/main/snowflake/test_snowflake.py | 2 +- .../commands/main/sqlserver/test_sqlserver.py | 2 +- .../dagster/test_sqlbuild_assets.py | 4 +- .../src/sqlbuild/cli/commands/main/helpers.py | 10 ++-- .../fingerprints/main/test_read_write.py | 12 ++-- .../executor/node_results/main/helpers.py | 2 +- .../node_results/main/test_read_write.py | 6 +- .../test_cli_concurrent_history.py | 2 +- .../classes/test_native_progress_projector.py | 56 +++++++++---------- .../_helpers/test_filesystem_python_nodes.py | 6 +- .../python_nodes/_helpers/test_identity.py | 28 +++++----- tests/unit/src/sqlbuild/cost/test_context.py | 2 +- .../node_results/_helpers/test_sql.py | 8 +-- .../executor/node_results/classes/helpers.py | 2 +- .../classes/test_node_result_store.py | 14 ++--- .../executor/node_results/main/test_read.py | 4 +- .../_helpers/test_fingerprinting.py | 4 +- .../_helpers/test_node_warehouse_failures.py | 2 +- .../observability/test_operation_lifecycle.py | 6 +- .../test_resource_attempt_lifecycle.py | 10 ++-- ui/src/lib/api/http-catalog.test.ts | 6 +- ui/src/lib/api/http-catalog.ts | 2 +- ui/src/lib/api/http-changes.ts | 2 +- ui/src/lib/api/http-providers.test.ts | 12 ++-- ui/src/lib/api/types/build.ts | 2 +- ui/src/lib/api/types/operate.ts | 4 +- ui/src/lib/api/types/platform.ts | 2 +- ui/src/lib/build-area/kind-tabs.test.ts | 2 +- ui/src/lib/build-area/labels.ts | 9 +-- .../lib/components/lineage/asset-node.svelte | 2 - ui/src/lib/format/plural.test.ts | 1 - ui/src/lib/format/plural.ts | 1 - ui/src/routes/providers/utils.test.ts | 4 +- ui/src/routes/providers/utils.ts | 4 +- website/src/content/docs/docs/cli/build.mdx | 2 +- .../docs/concepts/python-nodes/assets.mdx | 4 +- 55 files changed, 159 insertions(+), 202 deletions(-) diff --git a/benchmarks/duckshop/edge_cases/loaders/loader_results.py b/benchmarks/duckshop/edge_cases/loaders/loader_results.py index 68a53432ee..98c5229655 100644 --- a/benchmarks/duckshop/edge_cases/loaders/loader_results.py +++ b/benchmarks/duckshop/edge_cases/loaders/loader_results.py @@ -1,8 +1,8 @@ -"""Two loaders and a task that reads both of their results. +"""Two loaders and an asset that reads both of their results. A build that selects only one loader still reads the other loader's latest result, which may come from an earlier build. On Snowflake the adapter has no default schema, so the results must go to -the target's schema for the task to find them. +the target's schema for the asset to find them. """ from sqlbuild.assets import asset diff --git a/benchmarks/duckshop/edge_cases/loaders/round_trip/variants/ingress_gate/partner.py b/benchmarks/duckshop/edge_cases/loaders/round_trip/variants/ingress_gate/partner.py index c7d7520d10..a3cf9f5bc7 100644 --- a/benchmarks/duckshop/edge_cases/loaders/round_trip/variants/ingress_gate/partner.py +++ b/benchmarks/duckshop/edge_cases/loaders/round_trip/variants/ingress_gate/partner.py @@ -6,6 +6,6 @@ @loader(depends_on=[score_orders, fetch_rates]) def partner_scores(ctx): - """Load the scores, gated by a read-side task and an ingress task.""" + """Load the scores, gated by a read-side asset and an ingress asset.""" result = ctx.result_of(node_function=score_orders) return [{"order_id": o, "score": s} for o, s in result.payload["scores"]] diff --git a/benchmarks/duckshop/edge_cases/orchestration/node_result_history.json b/benchmarks/duckshop/edge_cases/orchestration/node_result_history.json index db3a8f2dae..8bf13c74b2 100644 --- a/benchmarks/duckshop/edge_cases/orchestration/node_result_history.json +++ b/benchmarks/duckshop/edge_cases/orchestration/node_result_history.json @@ -2,7 +2,7 @@ "record": { "state_database": null, "state_schema": "main", - "node_type": "task", + "node_type": "asset", "node_name": "result-000", "target_database": null, "target_schema": "main", diff --git a/crates/sqlbuild-analysis/src/definition_changes/constants.rs b/crates/sqlbuild-analysis/src/definition_changes/constants.rs index 9c84dbf60b..a96c993aad 100644 --- a/crates/sqlbuild-analysis/src/definition_changes/constants.rs +++ b/crates/sqlbuild-analysis/src/definition_changes/constants.rs @@ -54,7 +54,7 @@ pub(crate) const ASPECT_LABELS: [(&str, &str); 8] = [ ]; /// How each node kind reads in summaries: singular, plural. -pub(crate) const KIND_LABELS: [(&str, &str, &str); 23] = [ +pub(crate) const KIND_LABELS: [(&str, &str, &str); 22] = [ (SETTINGS_KIND, "project settings", "project settings"), ( CONNECTION_KIND, @@ -81,7 +81,6 @@ pub(crate) const KIND_LABELS: [(&str, &str, &str); 23] = [ ("python_hook", "Python hook", "Python hooks"), (MATERIALIZATION_KIND, "materialization", "materializations"), ("loader", "loader", "loaders"), - ("task", "task", "tasks"), ("asset", "asset", "assets"), ("check", "check", "checks"), (TEST_FILE_KIND, "test file", "test files"), diff --git a/crates/sqlbuild-core/src/integration_results/constants.rs b/crates/sqlbuild-core/src/integration_results/constants.rs index 9bdf314da0..839116b821 100644 --- a/crates/sqlbuild-core/src/integration_results/constants.rs +++ b/crates/sqlbuild-core/src/integration_results/constants.rs @@ -22,9 +22,9 @@ pub(crate) const MAX_STRING_CHARS: usize = 2_048; pub(crate) const MAX_IDENTIFIER_CHARS: usize = 512; pub(crate) const MAX_COLLECTION_ITEMS: usize = 128; pub(crate) const MAX_NESTING_DEPTH: usize = 6; -pub(crate) const ASSET_KINDS: [&str; 12] = [ +pub(crate) const ASSET_KINDS: [&str; 11] = [ "asset", "custom", "loader", "model", "seed", "snapshot", "source", "table", "table_fn", - "task", "udf", "view", + "udf", "view", ]; pub(crate) const CHECK_KINDS: [&str; 3] = [AUDIT_KIND, "python_check", "sql_test"]; pub(crate) const EXTRA_RESOURCE_KINDS: [&str; 3] = ["audit", "check", "test"]; diff --git a/crates/sqlbuild-core/src/lifecycle_events/constants.rs b/crates/sqlbuild-core/src/lifecycle_events/constants.rs index b55046b356..14bf878aca 100644 --- a/crates/sqlbuild-core/src/lifecycle_events/constants.rs +++ b/crates/sqlbuild-core/src/lifecycle_events/constants.rs @@ -224,7 +224,7 @@ pub const OPERATION_KINDS: [&str; 10] = [ ]; /// Includes retired names, such as the removed dlt and ingestr loads, so stored history still decodes. -pub const OPERATION_NAMES: [&str; 57] = [ +pub const OPERATION_NAMES: [&str; 56] = [ "adbc_driver_install", "audit_evaluation", "clone_execution", @@ -258,7 +258,6 @@ pub const OPERATION_NAMES: [&str; 57] = [ "python_check", "python_hook", "python_materialization", - "python_task", "relation_promotion", "retention_application", "retention_inspection", diff --git a/crates/sqlbuild-core/src/progress/constants.rs b/crates/sqlbuild-core/src/progress/constants.rs index 17b9d155ed..f96409d8bb 100644 --- a/crates/sqlbuild-core/src/progress/constants.rs +++ b/crates/sqlbuild-core/src/progress/constants.rs @@ -1,7 +1,7 @@ //! Lifecycle event types, operation labels and layout of CLI progress lines. /// Operations with visible progress lines, and the label each renders with. -pub(crate) const VISIBLE_OPERATION_LABELS: [(&str, &str); 30] = [ +pub(crate) const VISIBLE_OPERATION_LABELS: [(&str, &str); 29] = [ ("clone_finalization", "Clone finalization"), ("discovery_declaration_parse", "Declaration parsing"), ("discovery_filesystem_walk", "Filesystem discovery"), @@ -17,7 +17,6 @@ pub(crate) const VISIBLE_OPERATION_LABELS: [(&str, &str); 30] = [ ("python_asset", "Python asset attempt"), ("python_check", "Python check attempt"), ("python_hook", "Python hook"), - ("python_task", "Python task attempt"), ("sql_hook", "SQL hook"), ("audit_evaluation", "Audit evaluation"), ("sql_test_setup", "SQL test setup"), @@ -37,7 +36,7 @@ pub(crate) const VISIBLE_OPERATION_LABELS: [(&str, &str); 30] = [ ("relation_promotion", "Promote relation"), ]; /// Operations a live transient owner shows on a terminal when no resource attempt owns them. -pub(crate) const TTY_TRANSIENT_OWNER_OPERATIONS: [&str; 17] = [ +pub(crate) const TTY_TRANSIENT_OWNER_OPERATIONS: [&str; 16] = [ "adbc_driver_install", "dbt_command", "discovery_declaration_parse", @@ -52,7 +51,6 @@ pub(crate) const TTY_TRANSIENT_OWNER_OPERATIONS: [&str; 17] = [ "python_asset", "python_check", "python_hook", - "python_task", "sql_hook", "sql_test_setup", ]; diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/publications.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/publications.rs index 97cd1902bf..a089a57b79 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/publications.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/publications.rs @@ -26,12 +26,12 @@ use crate::store::models::{ControlStore, ControlTransaction, TransactionMode}; /// Graph kinds in the order the CLI builds its project graph: models, sources, seeds, then functions. const GRAPH_KINDS: [&str; 5] = ["model", "source", "seed", "udf", "table_fn"]; /// Python node kinds, in the order the CLI discovers them. -const PYTHON_NODE_KINDS: [&str; 4] = ["task", "asset", "loader", "check"]; +const PYTHON_NODE_KINDS: [&str; 3] = ["asset", "loader", "check"]; /// SQL unit tests, which `test:` selects; each lists the models it tests. const SQL_TEST_KIND: &str = "sql_test"; /// Every kind planning reads a definition of; audits, macros and scenarios are neither selected nor written. -const PLANNED_KINDS: [&str; 10] = [ - "model", "source", "seed", "udf", "table_fn", "task", "asset", "loader", "check", "sql_test", +const PLANNED_KINDS: [&str; 9] = [ + "model", "source", "seed", "udf", "table_fn", "asset", "loader", "check", "sql_test", ]; /// Where a SQL unit test's definition lists the models it tests. const SQL_TEST_MODELS_POINTER: &str = "/test/payload/expected_model_names"; diff --git a/crates/sqlbuild-orchestration/tests/runs/test_group_round_trips.rs b/crates/sqlbuild-orchestration/tests/runs/test_group_round_trips.rs index 72b208d82e..128552fe3b 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_group_round_trips.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_group_round_trips.rs @@ -65,7 +65,7 @@ fn given_a_round_trip_split_across_jobs_when_the_group_runs_then_members_follow_ ), ( "asset:score_orders".into(), - snapshot_node("task", json!({"deps": ["model:orders"]})), + snapshot_node("asset", json!({"deps": ["model:orders"]})), ), ( "loader:partner_scores".into(), diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md index 12e24f9563..ea140d0c9e 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md @@ -356,7 +356,7 @@ each schema carry the same run ID, so a role that can build in a schema can also there, with no other grants. A run that writes to any schema never writes these rows anywhere else, such as the target's default schema. -A run whose nodes write no relation, such as one that only runs Python tasks or checks, has no +A run whose nodes write no relation, such as one that only runs Python assets or checks, has no schema of its own. It records in the target's default schema instead, where audit results go. A run's status is its `finished` row, in any of its schemas. A run with only `started` rows was diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/assets.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/assets.md index 3a5be2a596..ffed412280 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/assets.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/assets.md @@ -18,7 +18,9 @@ Online: https://sqlbuild.com/docs/concepts/python-nodes/assets/ - Decorator parameters - Running assets -Assets are Python functions that run as part of the DAG. Use them for computation, side effects and external artifacts such as an exported file, a published dataset, a dashboard refresh or an ML model. See [Python Nodes](overview.md) for the shared model and the SQL boundary rules. +Assets are Python functions that run as part of the DAG, for computation, side effects and external artifacts such as an exported file, a published dataset, a dashboard refresh or an ML model. If you are coming from a task-based tool, an asset is a task that may not describe its output. + +See [Python Nodes](overview.md) for the shared model and the SQL boundary rules. ## Defining an asset diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/bigquery/test_bigquery.py b/tests/e2e/src/sqlbuild/cli/commands/main/bigquery/test_bigquery.py index 887d021bb4..d9e6ff4b2a 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/bigquery/test_bigquery.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/bigquery/test_bigquery.py @@ -149,7 +149,7 @@ def test_given_bigquery_dbt_profile_when_running_dbt_init_then_builds_profile_li description="direct node results persist and read on bigquery", expected_rows=( ("check", "check_produce_result", "success"), - ("task", "produce_result", "success"), + ("asset", "produce_result", "success"), ), ) ], diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_build.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_build.py index 9843f09a97..c7762fa324 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_build.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_build.py @@ -1905,7 +1905,7 @@ def test_given_local_target_override_when_running_python_then_every_context_sees "@asset\n" "def configured_task(ctx, target_probe: TargetProbe):\n" " '''Test task configured_task.'''\n" - " record(ctx, 'task')\n" + " record(ctx, 'asset')\n" " return ctx.result()\n" ), "python/loaders/raw.py": ( @@ -1963,7 +1963,7 @@ def test_given_local_target_override_when_running_python_then_every_context_sees # Each command (build, load, check) sets the provider up once, before its first node runs. assert [str(line.pop("node")) for line in recorded] == [ "provider", - "task", + "asset", "loader", "check", "provider", @@ -2096,7 +2096,7 @@ def test_given_ctx_templated_target_schema_when_running_then_python_sees_no_load "@asset\n" "def configured_task(ctx):\n" ' """Test task configured_task."""\n' - " record(ctx, 'task')\n" + " record(ctx, 'asset')\n" " return ctx.result()\n" ), "sources/raw.yml": ( @@ -2130,7 +2130,7 @@ def test_given_ctx_templated_target_schema_when_running_then_python_sees_no_load ] assert [(line["node"], line["name"], line["loader_schema"]) for line in recorded] == [ ("loader", "dev", test_case.expected_loader_schema), - ("task", "dev", test_case.expected_loader_schema), + ("asset", "dev", test_case.expected_loader_schema), ("loader", "dev", test_case.expected_loader_schema), ], recorded # Python node results land in the adapter's default schema; no schema keeps the template. diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_node_result_location.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_node_result_location.py index 0cc1d05be1..a1a2b0267d 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_node_result_location.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_node_result_location.py @@ -152,10 +152,10 @@ def test_given_target_location_when_building_and_loading_then_node_results_stay_ "GROUP BY ALL ORDER BY ALL" ), ) == [ + ("asset", "announce_orders", test_case.expected_runs_per_node), ("check", "check_orders_announced", test_case.expected_runs_per_node), ("loader", "raw_order_copies", test_case.expected_runs_per_node), ("loader", "raw_orders", test_case.expected_runs_per_node), - ("task", "announce_orders", test_case.expected_runs_per_node), ] assert query_duckdb(db_path=db_path, sql="SELECT order_id FROM raw.raw_order_copies") == [(1,)] @@ -221,18 +221,18 @@ def test_given_preserved_schema_without_loader_schema_when_compiling_then_python PlanAfterBuildsE2ETestCase( description="a preserved schema reads the task's fingerprint from the loader schema", target_lines='schema = "preserve"\nloader_schema = "raw"\n', - expected_plan_line="announce_orders task (unchanged)", + expected_plan_line="announce_orders asset (unchanged)", ), PlanAfterBuildsE2ETestCase( description="a plan of the task alone reads its fingerprint from the loader schema", target_lines='schema = "preserve"\nloader_schema = "raw"\n', - expected_plan_line="announce_orders task (unchanged)", + expected_plan_line="announce_orders asset (unchanged)", plan_args=("--select", "announce_orders"), ), PlanAfterBuildsE2ETestCase( description="a named schema reads the task's fingerprint from the target schema", target_lines='schema = "analytics"\n', - expected_plan_line="announce_orders task (unchanged)", + expected_plan_line="announce_orders asset (unchanged)", ), ], ids=lambda case: case.description, @@ -270,7 +270,7 @@ def test_given_two_builds_when_planning_then_python_nodes_are_not_new( description="older task results in a later-sorting model schema do not hide current ones", earlier_target_lines='schema = "staging"\n', target_lines='schema = "preserve"\nloader_schema = "raw"\n', - expected_plan_line="announce_orders task (unchanged)", + expected_plan_line="announce_orders asset (unchanged)", ), ], ids=lambda case: case.description, @@ -384,7 +384,7 @@ def test_given_results_in_the_adapter_default_when_one_loader_rebuilds_then_both assert set(new_runs) == {test_case.rebuilt_loader} assert new_runs[test_case.rebuilt_loader] != old_runs[test_case.rebuilt_loader] [(_, task_run, payload)] = node_results_in( - db_path=db_path, schema=test_case.target_schema, node_type="task" + db_path=db_path, schema=test_case.target_schema, node_type="asset" ) assert task_run == new_runs[test_case.rebuilt_loader] rebuilt_key, kept_key = test_case.expected_payload_keys diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_required_descriptions.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_required_descriptions.py index 987fcf210e..2c6fa4ffdc 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_required_descriptions.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_required_descriptions.py @@ -384,29 +384,6 @@ expected_message="loader 'raw_returns' has no description", expected_help_fragments=('@loader(description="What raw_returns does", ...)',), ), - RequiredDescriptionCase( - description="task", - missing_files=( - ( - "python/tasks/refresh_exports.py", - python_node_source(module="tasks", decorator="task", name="refresh_exports"), - ), - ), - described_files=( - ( - "python/tasks/refresh_exports.py", - python_node_source( - module="tasks", - decorator="task", - name="refresh_exports", - docstring_line=' """Refresh the order exports."""\n', - ), - ), - ), - expected_diagnostic=("P010", "python/tasks/refresh_exports.py", 4), - expected_message="task 'refresh_exports' has no description", - expected_help_fragments=('@asset(description="What refresh_exports does", ...)',), - ), RequiredDescriptionCase( description="asset", missing_files=( @@ -511,31 +488,31 @@ "test_case", [ RequiredDescriptionCase( - description="task docstring", + description="asset docstring", missing_files=( ( - "python/tasks/refresh_exports.py", + "python/assets/order_report.py", python_node_source( - module="tasks", - decorator="task", - name="refresh_exports", + module="assets", + decorator="asset", + name="order_report", docstring_line=description_paragraphs(6), ), ), ), described_files=( ( - "python/tasks/refresh_exports.py", + "python/assets/order_report.py", python_node_source( - module="tasks", - decorator="task", - name="refresh_exports", + module="assets", + decorator="asset", + name="order_report", docstring_line=description_paragraphs(5), ), ), ), - expected_diagnostic=("P027", "python/tasks/refresh_exports.py", 4), - expected_message="task 'refresh_exports' description exceeds 10 lines", + expected_diagnostic=("P027", "python/assets/order_report.py", 4), + expected_message="asset 'order_report' description exceeds 10 lines", expected_help_fragments=("Shorten the description to 10 lines",), ), RequiredDescriptionCase( diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/databricks/test_databricks.py b/tests/e2e/src/sqlbuild/cli/commands/main/databricks/test_databricks.py index 0201290c7d..52564e6334 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/databricks/test_databricks.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/databricks/test_databricks.py @@ -143,7 +143,7 @@ def test_given_databricks_dbt_profile_when_running_dbt_init_then_builds_profile_ description="direct node results persist and read on databricks", expected_rows=( ("check", "check_produce_result", "success"), - ("task", "produce_result", "success"), + ("asset", "produce_result", "success"), ), ) ], diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/load/test_load.py b/tests/e2e/src/sqlbuild/cli/commands/main/load/test_load.py index 2f09bd5af3..98444d09a5 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/load/test_load.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/load/test_load.py @@ -598,15 +598,6 @@ def test_given_chained_loader_project_when_running_source_model_then_runs_loader @pytest.mark.parametrize( "test_case", [ - SourceOnlyComplexIngressE2ETestCase( - description="source-only load fails when loader reads unselected task payload", - command=("--no-color", "load", "--select", "raw_events"), - expected_return_code=1, - expected_error_fragment="No persisted result found for Python node 'prepare_events'", - dependency_dir="tasks", - decorator_import="from sqlbuild.assets import asset\n", - decorator_name="task", - ), SourceOnlyComplexIngressE2ETestCase( description="source-only load fails when loader reads unselected asset payload", command=("--no-color", "load", "--select", "raw_events"), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/postgres/test_postgres.py b/tests/e2e/src/sqlbuild/cli/commands/main/postgres/test_postgres.py index 55ab558d6b..36ad5735dc 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/postgres/test_postgres.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/postgres/test_postgres.py @@ -237,7 +237,7 @@ def test_given_postgres_dbt_profile_when_running_dbt_init_then_plain_build_uses_ description="direct node results persist and read on postgres", expected_rows=( ("check", "check_produce_result", "success"), - ("task", "produce_result", "success"), + ("asset", "produce_result", "success"), ), ) ], diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/providers/test_providers.py b/tests/e2e/src/sqlbuild/cli/commands/main/providers/test_providers.py index f6e30f179f..b6cfc922c5 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/providers/test_providers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/providers/test_providers.py @@ -90,7 +90,7 @@ def mark(self, value): ("custom materialization", "copy_table", "marker_provider"), ("hook", "mark_pre", "marker_provider"), ("loader", "raw_orders", "marker_provider"), - ("task", "publish_orders", "marker_provider"), + ("asset", "publish_orders", "marker_provider"), ), ) ], diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/snowflake/test_snowflake.py b/tests/e2e/src/sqlbuild/cli/commands/main/snowflake/test_snowflake.py index 5e537553ea..83ab0f1299 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/snowflake/test_snowflake.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/snowflake/test_snowflake.py @@ -158,7 +158,7 @@ def test_given_snowflake_dbt_profile_when_running_dbt_init_then_builds_profile_l description="direct node results persist and read on snowflake", expected_rows=( ("check", "check_produce_result", "success"), - ("task", "produce_result", "success"), + ("asset", "produce_result", "success"), ), ) ], diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/sqlserver/test_sqlserver.py b/tests/e2e/src/sqlbuild/cli/commands/main/sqlserver/test_sqlserver.py index 19cf6e2add..564babcd22 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/sqlserver/test_sqlserver.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/sqlserver/test_sqlserver.py @@ -411,7 +411,7 @@ def test_given_sqlserver_dbt_profile_when_running_dbt_init_then_builds_profile_l description="direct node results persist and read on SQL Server", expected_rows=( ("check", "check_produce_result", "success"), - ("task", "produce_result", "success"), + ("asset", "produce_result", "success"), ), ) ], diff --git a/tests/e2e/src/sqlbuild/integrations/dagster/test_sqlbuild_assets.py b/tests/e2e/src/sqlbuild/integrations/dagster/test_sqlbuild_assets.py index c761d1140c..818090c212 100644 --- a/tests/e2e/src/sqlbuild/integrations/dagster/test_sqlbuild_assets.py +++ b/tests/e2e/src/sqlbuild/integrations/dagster/test_sqlbuild_assets.py @@ -156,7 +156,7 @@ def sqlbuild_orders(context: AssetExecutionContext) -> Iterator[object]: description="dagster consumes real Python-node dag artifact", expected_asset_keys=( ("main", "orders"), - ("task", "prepare_orders"), + ("asset", "prepare_orders"), ("warehouse_export",), ("asset", "orders_export"), ), @@ -191,7 +191,7 @@ def sqlbuild_python_nodes(context: AssetExecutionContext) -> Iterator[object]: tuple(spec.key.path): (spec,) for spec in sqlbuild_python_nodes.specs } assert len(specs_by_key) == len(sqlbuild_python_nodes.specs) - task_spec: object = next(iter(specs_by_key.get(("task", "prepare_orders"), ()))) + task_spec: object = next(iter(specs_by_key.get(("asset", "prepare_orders"), ()))) asset_spec: object = next(iter(specs_by_key.get(("asset", "orders_export"), ()))) assert sqlbuild_project.dag_path.exists() diff --git a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py index 94ba7a8010..172f4bc8e6 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py @@ -1463,9 +1463,9 @@ def write_python_rules_project(*, project_dir: Path, fixture: str, rules_toml: s " enabled false, schema archive);\nSELECT 1 AS order_id\n" ), "python/noop.py": ( - '"""A task that writes no relation."""\n\n' - "from sqlbuild.tasks import task\n\n\n" - "@task()\n" + '"""An asset that writes no relation."""\n\n' + "from sqlbuild.assets import asset\n\n\n" + "@asset()\n" "def noop(ctx):\n" ' """Do nothing."""\n' ), @@ -1655,8 +1655,8 @@ def write_job_warehouse_project(*, project_dir: Path) -> None: "models/orders_daily.sql": _job_warehouse_model(' FROM __ref("orders")'), "models/refunds.sql": _job_warehouse_model(' FROM __ref("orders_daily")'), "models/summary.sql": _job_warehouse_model(' FROM __ref("refunds")'), - "python/tasks.py": ( - "from sqlbuild.tasks import task\n\n\n@task\ndef refresh_marker(ctx):\n" + "python/assets.py": ( + "from sqlbuild.assets import asset\n\n\n@asset\ndef refresh_marker(ctx):\n" ' """Refresh."""\n ctx.execute_sql("SELECT \'task marker\'")\n' ), "orchestration/jobs/orders_wh_job.toml": _job_file( diff --git a/tests/integration/src/sqlbuild/compiler/fingerprints/main/test_read_write.py b/tests/integration/src/sqlbuild/compiler/fingerprints/main/test_read_write.py index 577698c6a8..a92c52aaf2 100644 --- a/tests/integration/src/sqlbuild/compiler/fingerprints/main/test_read_write.py +++ b/tests/integration/src/sqlbuild/compiler/fingerprints/main/test_read_write.py @@ -225,7 +225,7 @@ ts=datetime(2026, 1, 15, 11, 0, 0), ), Fingerprint( - node_type="task", + node_type="asset", node_name="orders", target_database=None, target_schema=None, @@ -243,7 +243,7 @@ expected_latest_target_names={"orders": "orders"}, expected_identity_definition_hashes={ ("model", "orders"): "model_hash_new", - ("task", "orders"): "task_hash_new", + ("asset", "orders"): "task_hash_new", }, ), ], @@ -725,7 +725,7 @@ def test_given_multiple_fingerprints_when_reading_then_resolves_latest( ts=datetime(2026, 1, 15, 11, 0, 0), ), Fingerprint( - node_type="task", + node_type="asset", node_name="orders", target_database=None, target_schema=None, @@ -738,7 +738,7 @@ def test_given_multiple_fingerprints_when_reading_then_resolves_latest( ts=datetime(2026, 1, 15, 9, 0, 0), ), Fingerprint( - node_type="task", + node_type="asset", node_name="orders", target_database=None, target_schema=None, @@ -751,7 +751,7 @@ def test_given_multiple_fingerprints_when_reading_then_resolves_latest( ts=datetime(2026, 1, 15, 11, 0, 0), ), Fingerprint( - node_type="task", + node_type="asset", node_name="orders", target_database=None, target_schema=None, @@ -767,7 +767,7 @@ def test_given_multiple_fingerprints_when_reading_then_resolves_latest( expected_run_ids_by_identity={ ("model", "orders"): ("run_003", "run_002"), ("model", "customers"): ("run_011", "run_010"), - ("task", "orders"): ("run_task_002", "run_task_001"), + ("asset", "orders"): ("run_task_002", "run_task_001"), }, expected_latest_run_id="run_003", ) diff --git a/tests/integration/src/sqlbuild/executor/node_results/main/helpers.py b/tests/integration/src/sqlbuild/executor/node_results/main/helpers.py index 19ddf21d7f..a1a54b07d6 100644 --- a/tests/integration/src/sqlbuild/executor/node_results/main/helpers.py +++ b/tests/integration/src/sqlbuild/executor/node_results/main/helpers.py @@ -9,7 +9,7 @@ def task_result_record(*, run_id: str, schema: str) -> NodeResultRecord: """A successful `produce_result` task result recorded in `schema`.""" return NodeResultRecord( - node_type="task", + node_type="asset", node_name="produce_result", target_database=None, target_schema=schema, diff --git a/tests/integration/src/sqlbuild/executor/node_results/main/test_read_write.py b/tests/integration/src/sqlbuild/executor/node_results/main/test_read_write.py index d433c4082d..322b683473 100644 --- a/tests/integration/src/sqlbuild/executor/node_results/main/test_read_write.py +++ b/tests/integration/src/sqlbuild/executor/node_results/main/test_read_write.py @@ -49,7 +49,7 @@ def test_given_node_result_record_when_writing_and_reading_then_round_trips_enve database=None, schema="main", record=NodeResultRecord( - node_type="task", + node_type="asset", node_name="produce_result", target_database=None, target_schema="main", @@ -71,7 +71,7 @@ def test_given_node_result_record_when_writing_and_reading_then_round_trips_enve database=None, schema="main", query=NodeResultQuery( - node_type="task", + node_type="asset", node_name="produce_result", target_database=None, target_schema="main", @@ -152,7 +152,7 @@ def test_given_result_in_earlier_location_when_reading_then_store_falls_back_onl schema="raw", target_config=TargetSettings(schema_preserved=test_case.schema_preserved), ) - found: object = reader.result_of(node_type="task", node_name="produce_result", default=None) + found: object = reader.result_of(node_type="asset", node_name="produce_result", default=None) finally: adapter.close(connection) diff --git a/tests/integration/src/sqlbuild/runtime/execution_history/test_cli_concurrent_history.py b/tests/integration/src/sqlbuild/runtime/execution_history/test_cli_concurrent_history.py index aaf5b777c9..019c44ab30 100644 --- a/tests/integration/src/sqlbuild/runtime/execution_history/test_cli_concurrent_history.py +++ b/tests/integration/src/sqlbuild/runtime/execution_history/test_cli_concurrent_history.py @@ -59,7 +59,7 @@ def test_given_explicit_cli_history_when_two_workers_publish_then_events_and_pro def execute_attempt(resource_name: str) -> None: with ResourceAttemptLifecycle( resource_id=f"asset:{resource_name}", - resource_kind="task", + resource_kind="asset", resource_name=resource_name, run_id="concurrent-run", ): diff --git a/tests/unit/src/sqlbuild/cli/progress/classes/test_native_progress_projector.py b/tests/unit/src/sqlbuild/cli/progress/classes/test_native_progress_projector.py index f076b6b882..c2bf9bc7fa 100644 --- a/tests/unit/src/sqlbuild/cli/progress/classes/test_native_progress_projector.py +++ b/tests/unit/src/sqlbuild/cli/progress/classes/test_native_progress_projector.py @@ -71,7 +71,7 @@ def test_given_skipped_resource_when_projected_then_skip_terminal_renders_once( dispatcher_scope(dispatcher), ResourceAttemptLifecycle( resource_id="asset:skipped", - resource_kind="task", + resource_kind="asset", resource_name="skipped", ) as lifecycle, ): @@ -80,8 +80,8 @@ def test_given_skipped_resource_when_projected_then_skip_terminal_renders_once( assert tuple(event.event_type for event in events) == test_case.expected_event_types assert stream.flush_count == test_case.expected_flush_count_before_block lines: tuple[str, ...] = tuple(stream.getvalue().splitlines()) - assert lines[0] == " task skipped START" - assert lines[1].startswith(" task skipped SKIP 0.") + assert lines[0] == " asset skipped START" + assert lines[1].startswith(" asset skipped SKIP 0.") @pytest.mark.parametrize( @@ -90,13 +90,13 @@ def test_given_skipped_resource_when_projected_then_skip_terminal_renders_once( RetryProjectionCase( description="retry attempts flush before each blocking boundary", expected_lines=( - " task flaky START", - " Python task attempt 1 START", - " Python task attempt 1 FAIL (0.00s)", + " asset flaky START", + " Python asset attempt 1 START", + " Python asset attempt 1 FAIL (0.00s)", " retry 1->2 in 0.25s", - " Python task attempt 2 START", - " Python task attempt 2 OK (0.00s)", - " task flaky OK 0.00s", + " Python asset attempt 2 START", + " Python asset attempt 2 OK (0.00s)", + " asset flaky OK 0.00s", ), expected_unrelated_duration_ms=0.0, expected_final_duration_ms=0.0, @@ -120,16 +120,16 @@ def test_given_retrying_child_operation_when_projected_then_each_line_flushes_be dispatcher_scope(dispatcher), ResourceAttemptLifecycle( resource_id="asset:flaky", - resource_kind="task", + resource_kind="asset", resource_name="flaky", ), ): with OperationLifecycle( operation_kind="python_node", - operation_name="python_task", + operation_name="python_asset", metadata={"attempt_number": 1}, ) as first_attempt: - assert stream.getvalue().endswith("Python task attempt 1 START\n") + assert stream.getvalue().endswith("Python asset attempt 1 START\n") first_attempt.failed(error=TimeoutError()) publish_retry_scheduled( failed_attempt_number=1, @@ -140,18 +140,18 @@ def test_given_retrying_child_operation_when_projected_then_each_line_flushes_be assert stream.getvalue().endswith(" retry 1->2 in 0.25s\n") with OperationLifecycle( operation_kind="python_node", - operation_name="python_task", + operation_name="python_asset", metadata={"attempt_number": 2}, ): - assert stream.getvalue().endswith("Python task attempt 2 START\n") + assert stream.getvalue().endswith("Python asset attempt 2 START\n") assert stream.flush_count == len(stream.getvalue().splitlines()) rendered_lines: tuple[str, ...] = tuple(stream.getvalue().splitlines()) assert rendered_lines[0:2] == test_case.expected_lines[0:2] - assert rendered_lines[2].startswith(" Python task attempt 1 FAIL (") + assert rendered_lines[2].startswith(" Python asset attempt 1 FAIL (") assert rendered_lines[3:5] == test_case.expected_lines[3:5] - assert rendered_lines[5].startswith(" Python task attempt 2 OK (") - assert rendered_lines[-1].startswith(" task flaky OK 0.") + assert rendered_lines[5].startswith(" Python asset attempt 2 OK (") + assert rendered_lines[-1].startswith(" asset flaky OK 0.") @pytest.mark.parametrize( @@ -267,12 +267,12 @@ def test_given_interleaved_and_duplicate_events_when_projected_then_ids_remain_i RetryProjectionCase( description="failed retry followed by successful final attempt", expected_lines=( - " 1/2 task orders START", - " 2/2 task inventory START", - " task orders FAIL 0.10s", - " 1/2 task orders START", - " task inventory OK 0.30s rich", - " task orders OK 0.25s rich", + " 1/2 asset orders START", + " 2/2 asset inventory START", + " asset orders FAIL 0.10s", + " 1/2 asset orders START", + " asset inventory OK 0.30s rich", + " asset orders OK 0.25s rich", ), expected_unrelated_duration_ms=300.0, expected_final_duration_ms=250.0, @@ -293,7 +293,7 @@ def test_given_retried_resource_when_final_result_claims_then_prior_attempt_rend resource_id="asset:orders", resource_attempt_id="orders-attempt-1", payload={ - "resource_kind": "task", + "resource_kind": "asset", "resource_name": "orders", "attempt_number": 1, }, @@ -312,7 +312,7 @@ def test_given_retried_resource_when_final_result_claims_then_prior_attempt_rend resource_id="asset:inventory", resource_attempt_id="inventory-attempt", payload={ - "resource_kind": "task", + "resource_kind": "asset", "resource_name": "inventory", "attempt_number": 1, }, @@ -345,11 +345,11 @@ def test_given_retried_resource_when_final_result_claims_then_prior_attempt_rend resource_name="inventory" ) stream.write( - f" task inventory OK {cast(float, unrelated_duration) / 1000.0:.2f}s rich\n" + f" asset inventory OK {cast(float, unrelated_duration) / 1000.0:.2f}s rich\n" ) projector.consume(final_terminal) final_duration: float | None = projector.consume_resource_terminal(resource_name="orders") - stream.write(f" task orders OK {cast(float, final_duration) / 1000.0:.2f}s rich\n") + stream.write(f" asset orders OK {cast(float, final_duration) / 1000.0:.2f}s rich\n") projector.consume(first_failed) projector.consume(final_terminal) projector.close() @@ -819,7 +819,7 @@ def project_resource() -> None: run_scope("run-lock-order"), dispatcher_scope(dispatcher), ResourceAttemptLifecycle( - resource_id="asset:writer", resource_kind="task", resource_name="writer" + resource_id="asset:writer", resource_kind="asset", resource_name="writer" ) as lifecycle, ): lifecycle.skipped(skip_code="scheduler") diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_python_nodes.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_python_nodes.py index e4ba6f2796..bc7cf7e81d 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_python_nodes.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_python_nodes.py @@ -505,7 +505,7 @@ def test_given_python_root_helper_edit_when_rediscovering_then_node_identity_cha project_dir=tmp_path ).assets[0] before: PythonNodeIdentity = build_python_node_identity( - node_type="task", + node_type="asset", node_name=before_task.name, function=before_task.function, project_dir=tmp_path, @@ -520,7 +520,7 @@ def test_given_python_root_helper_edit_when_rediscovering_then_node_identity_cha project_dir=tmp_path ).assets[0] after: PythonNodeIdentity = build_python_node_identity( - node_type="task", + node_type="asset", node_name=after_task.name, function=after_task.function, project_dir=tmp_path, @@ -630,7 +630,7 @@ def test_given_unrelated_python_package_restored_when_building_identity_then_hel 0 ] identity: PythonNodeIdentity = build_python_node_identity( - node_type="task", + node_type="asset", node_name=task.name, function=task.function, project_dir=project_dir, diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_identity.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_identity.py index 0f68b1caee..2cef3e4ea4 100644 --- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_identity.py +++ b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_identity.py @@ -38,7 +38,7 @@ def build_orders(ctx): }, entry_module_path="python/tasks/orders.py", function_name="build_orders", - node_type="task", + node_type="asset", expected_source_path="python/tasks/orders.py", expected_dependency_qualnames=("normalize_order",), expected_definition_fragments=("def build_orders(ctx):", "source_text"), @@ -71,7 +71,7 @@ def clean_status(value): }, entry_module_path="python/tasks/orders.py", function_name="build_orders", - node_type="task", + node_type="asset", expected_source_path="python/tasks/orders.py", expected_dependency_qualnames=("normalize_orders", "clean_status"), expected_definition_fragments=("def build_orders(ctx):", "source_text"), @@ -100,7 +100,7 @@ def normalize_order(value): }, entry_module_path="python/tasks/orders.py", function_name="build_orders", - node_type="task", + node_type="asset", expected_source_path="python/tasks/orders.py", expected_dependency_qualnames=(), expected_definition_fragments=("def build_orders(ctx):", "source_text"), @@ -127,7 +127,7 @@ def normalize_order(value): }, entry_module_path="python/tasks/orders.py", function_name="build_orders", - node_type="task", + node_type="asset", expected_source_path="python/tasks/orders.py", expected_dependency_qualnames=(), expected_definition_fragments=("def build_orders(ctx):", "source_text"), @@ -155,7 +155,7 @@ def build_orders(ctx): }, entry_module_path="python/tasks/orders.py", function_name="build_orders", - node_type="task", + node_type="asset", expected_source_path="python/tasks/orders.py", expected_dependency_qualnames=(), expected_definition_fragments=("def build_orders(ctx):", "source_text"), @@ -185,7 +185,7 @@ def helper_b(value): }, entry_module_path="python/tasks/orders.py", function_name="build_orders", - node_type="task", + node_type="asset", expected_source_path="python/tasks/orders.py", expected_dependency_qualnames=("helper_a", "helper_b"), expected_definition_fragments=("def build_orders(ctx):", "source_text"), @@ -210,7 +210,7 @@ def normalize_order(value): }, entry_module_path="python/tasks/orders.py", function_name="build_orders", - node_type="task", + node_type="asset", expected_source_path="python/tasks/orders.py", expected_dependency_qualnames=("normalize_order",), expected_definition_fragments=("def build_orders(ctx):", "source_text"), @@ -232,7 +232,7 @@ def build_orders(ctx): }, entry_module_path="python/tasks/orders.py", function_name="build_orders", - node_type="task", + node_type="asset", expected_source_path="python/tasks/orders.py", expected_dependency_qualnames=("Cleaner",), expected_definition_fragments=("def build_orders(ctx):", "source_text"), @@ -255,7 +255,7 @@ def build_orders(ctx): }, entry_module_path="python/tasks/orders.py", function_name="build_orders", - node_type="task", + node_type="asset", expected_source_path="python/tasks/orders.py", expected_dependency_qualnames=("Cleaner.clean",), expected_definition_fragments=("def build_orders(ctx):", "source_text"), @@ -286,7 +286,7 @@ def a_clean(value): }, entry_module_path="python/tasks/orders.py", function_name="build_orders", - node_type="task", + node_type="asset", expected_source_path="python/tasks/orders.py", expected_dependency_qualnames=("a_clean", "z_clean"), expected_definition_fragments=("def build_orders(ctx):", "source_text"), @@ -303,7 +303,7 @@ def build_orders(ctx): }, entry_module_path="python/tasks/orders.py", function_name="build_orders", - node_type="task", + node_type="asset", expected_source_path="python/tasks/orders.py", expected_dependency_qualnames=(), expected_definition_fragments=("orders", "daily", "source_text"), @@ -401,7 +401,7 @@ def normalize_order(value): }, entry_module_path="python/tasks/orders.py", function_name="build_orders", - node_type="task", + node_type="asset", expected_definition_hash_changed=True, expected_version_hash_changed=True, ), @@ -439,7 +439,7 @@ def normalize_order(value): }, entry_module_path="python/tasks/orders.py", function_name="build_orders", - node_type="task", + node_type="asset", expected_definition_hash_changed=False, expected_version_hash_changed=True, ), @@ -461,7 +461,7 @@ def build_orders(ctx): }, entry_module_path="python/tasks/orders.py", function_name="build_orders", - node_type="task", + node_type="asset", expected_definition_hash_changed=True, expected_version_hash_changed=True, before_decorator_config={"tags": ["orders"]}, diff --git a/tests/unit/src/sqlbuild/cost/test_context.py b/tests/unit/src/sqlbuild/cost/test_context.py index 28ad5accbb..8f87ec45f6 100644 --- a/tests/unit/src/sqlbuild/cost/test_context.py +++ b/tests/unit/src/sqlbuild/cost/test_context.py @@ -13,7 +13,7 @@ CostResourceScopeTestCase( description="inherits run ledger while overriding resource phase and attempt", ledger_path=Path("target/executions/run-1/statements.jsonl"), - expected_resource_type="task", + expected_resource_type="asset", expected_resource_name="refresh_orders", expected_phase="execute", expected_attempt=3, diff --git a/tests/unit/src/sqlbuild/executor/node_results/_helpers/test_sql.py b/tests/unit/src/sqlbuild/executor/node_results/_helpers/test_sql.py index 9c8dd96180..e0651ece58 100644 --- a/tests/unit/src/sqlbuild/executor/node_results/_helpers/test_sql.py +++ b/tests/unit/src/sqlbuild/executor/node_results/_helpers/test_sql.py @@ -23,7 +23,7 @@ description="builds scoped read SQL with deterministic ordering", expected_fragments=( "FROM analytics._sqlbuild_node_results", - "node_type = 'task'", + "node_type = 'asset'", "node_name = 'produce_result'", "target_schema = 'analytics'", "status IN ('success')", @@ -40,7 +40,7 @@ def test_given_node_result_read_request_when_building_sql_then_scopes_and_orders database=None, schema="analytics", query=NodeResultQuery( - node_type="task", + node_type="asset", node_name="produce_result", target_database=None, target_schema="analytics", @@ -66,7 +66,7 @@ def test_given_node_result_read_request_when_building_sql_then_scopes_and_orders "INSERT INTO analytics._sqlbuild_node_results (node_type, node_name, " "target_database, target_schema, target_name, run_id, status, " "payload_json_b64, metadata_json_b64, error_message, materialized, ts) VALUES " - "('task', 'node''name', NULL, 'analytics', NULL, 'run-1', 'success', 'e30=', " + "('asset', 'node''name', NULL, 'analytics', NULL, 'run-1', 'success', 'e30=', " "'e30=', 'it''s fine', 'true', '2026-01-01T00:00:00+00:00')" ), ) @@ -80,7 +80,7 @@ def test_given_representative_node_result_when_rendering_insert_then_sql_matches database=None, schema="analytics", record=NodeResultRecord( - node_type="task", + node_type="asset", node_name="node'name", target_database=None, target_schema="analytics", diff --git a/tests/unit/src/sqlbuild/executor/node_results/classes/helpers.py b/tests/unit/src/sqlbuild/executor/node_results/classes/helpers.py index b54c9825ed..7ac79f73c9 100644 --- a/tests/unit/src/sqlbuild/executor/node_results/classes/helpers.py +++ b/tests/unit/src/sqlbuild/executor/node_results/classes/helpers.py @@ -33,7 +33,7 @@ def build_node_result_record( materialized: bool | None = None, ) -> NodeResultRecord: return NodeResultRecord( - node_type="task", + node_type="asset", node_name="orders", target_database="analytics", target_schema="state", diff --git a/tests/unit/src/sqlbuild/executor/node_results/classes/test_node_result_store.py b/tests/unit/src/sqlbuild/executor/node_results/classes/test_node_result_store.py index c6d34f3b0f..75046e2382 100644 --- a/tests/unit/src/sqlbuild/executor/node_results/classes/test_node_result_store.py +++ b/tests/unit/src/sqlbuild/executor/node_results/classes/test_node_result_store.py @@ -69,7 +69,7 @@ expected_result=_PERSISTED_SUCCESS, expected_queries=( NodeResultQuery( - node_type="task", + node_type="asset", node_name="orders", target_database="analytics", target_schema="state", @@ -88,7 +88,7 @@ expected_result=_PERSISTED_SUCCESS, expected_queries=( NodeResultQuery( - node_type="task", + node_type="asset", node_name="orders", target_database="analytics", target_schema="state", @@ -113,7 +113,7 @@ def test_given_cached_and_persisted_results_when_reading_one_then_contract_is_pr store.write(record) result: NodeResultEnvelope | object = store.result_of( - node_type="task", + node_type="asset", node_name="orders", run_id=test_case.run_id, ) @@ -141,7 +141,7 @@ def test_given_cached_and_persisted_results_when_reading_one_then_contract_is_pr expected_run_ids=("persisted-success",), expected_queries=( NodeResultQuery( - node_type="task", + node_type="asset", node_name="orders", target_database="analytics", target_schema="state", @@ -166,7 +166,7 @@ def test_given_cached_and_persisted_results_when_reading_history_then_contract_i store.write(record) results: tuple[NodeResultEnvelope, ...] = store.results_of( - node_type="task", + node_type="asset", node_name="orders", limit=test_case.limit, ) @@ -194,11 +194,11 @@ def test_given_missing_result_when_reading_then_default_and_error_contract_are_p assert ( store.result_of( - node_type="task", + node_type="asset", node_name="missing", default=test_case.default, ) == test_case.expected_default ) with pytest.raises(ExecutorInputError, match=test_case.expected_error_fragment): - store.result_of(node_type="task", node_name="missing") + store.result_of(node_type="asset", node_name="missing") diff --git a/tests/unit/src/sqlbuild/executor/node_results/main/test_read.py b/tests/unit/src/sqlbuild/executor/node_results/main/test_read.py index 0aeeebf457..2eae6d18f7 100644 --- a/tests/unit/src/sqlbuild/executor/node_results/main/test_read.py +++ b/tests/unit/src/sqlbuild/executor/node_results/main/test_read.py @@ -33,7 +33,7 @@ def execute(*, connection: object, sql: str) -> NodeResultReadFakeResult: return NodeResultReadFakeResult( [ ( - "task", + "asset", "produce_result", None, "main", @@ -56,7 +56,7 @@ def execute(*, connection: object, sql: str) -> NodeResultReadFakeResult: database=None, schema="main", query=NodeResultQuery( - node_type="task", + node_type="asset", node_name="produce_result", target_database=None, target_schema="main", diff --git a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_fingerprinting.py b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_fingerprinting.py index f81724f4f0..d856b33a50 100644 --- a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_fingerprinting.py +++ b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_fingerprinting.py @@ -25,7 +25,7 @@ expected_sql_count=2, expected_fragments=( "_sqlbuild_fingerprints", - "task", + "asset", "build_orders", "definition_b64", "metadata_json_b64", @@ -50,7 +50,7 @@ def test_given_python_identity_when_writing_fingerprint_then_uses_fingerprint_st try_write_python_node_identity_fingerprint( identity=PythonNodeIdentity( - node_type="task", + node_type="asset", node_name="build_orders", source_path="python/tasks/orders.py", source_hash="source-hash", diff --git a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_node_warehouse_failures.py b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_node_warehouse_failures.py index 68a05f2839..13d2c8688c 100644 --- a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_node_warehouse_failures.py +++ b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_node_warehouse_failures.py @@ -116,7 +116,7 @@ def test_given_an_unusable_job_warehouse_when_running_a_check_then_it_fails_alon upstream_python_results=( PythonNodeExecutionResult( node_name="upstream_task", - kind=PythonNodeKind.TASK, + kind=PythonNodeKind.ASSET, status=test_case.upstream_status, ), ), diff --git a/tests/unit/src/sqlbuild/runtime/observability/test_operation_lifecycle.py b/tests/unit/src/sqlbuild/runtime/observability/test_operation_lifecycle.py index 79fcc95e58..8419d82160 100644 --- a/tests/unit/src/sqlbuild/runtime/observability/test_operation_lifecycle.py +++ b/tests/unit/src/sqlbuild/runtime/observability/test_operation_lifecycle.py @@ -220,7 +220,7 @@ def test_given_pending_future_when_result_owned_then_terminal_follows_result( context: Context = copy_context() result: list[str] = [] lifecycle: OperationLifecycle = OperationLifecycle( - operation_kind="python_node", operation_name="python_task" + operation_kind="python_node", operation_name="python_asset" ) thread: Thread = Thread( target=lambda: result.append(cast(str, context.run(lifecycle.result, future))) @@ -260,7 +260,7 @@ def test_given_cancelled_future_when_result_owned_then_one_failure_is_published( dispatcher_scope(dispatcher), pytest.raises(CancelledError), ): - _ = OperationLifecycle(operation_kind="python_node", operation_name="python_task").result( + _ = OperationLifecycle(operation_kind="python_node", operation_name="python_asset").result( future ) @@ -352,7 +352,7 @@ def test_given_nested_operations_when_exiting_then_parent_and_siblings_do_not_le ): with OperationLifecycle( operation_kind="python_node", - operation_name="python_task", + operation_name="python_asset", operation_id="child-one", ): with StatementLifecycle(adapter="duckdb", sql="SELECT 1", intent="execute"): diff --git a/tests/unit/src/sqlbuild/runtime/observability/test_resource_attempt_lifecycle.py b/tests/unit/src/sqlbuild/runtime/observability/test_resource_attempt_lifecycle.py index fedac8e2d4..e3078b2f10 100644 --- a/tests/unit/src/sqlbuild/runtime/observability/test_resource_attempt_lifecycle.py +++ b/tests/unit/src/sqlbuild/runtime/observability/test_resource_attempt_lifecycle.py @@ -73,7 +73,7 @@ def test_given_bounded_skip_when_attempt_marked_skipped_then_reason_is_not_canon with invocation_scope("inv-skip"), run_scope("run-skip"), dispatcher_scope(dispatcher): with ResourceAttemptLifecycle( resource_id="asset:orders", - resource_kind="task", + resource_kind="asset", resource_name="orders", ) as lifecycle: lifecycle.skipped(skip_code="explicit", skip_mode="hard") @@ -104,7 +104,7 @@ def test_given_no_context_when_attempt_runs_then_transient_identity_and_dispatch with ResourceAttemptLifecycle( resource_id="asset:direct", - resource_kind="task", + resource_kind="asset", resource_name="direct", run_id="direct-run", ): @@ -113,7 +113,7 @@ def test_given_no_context_when_attempt_runs_then_transient_identity_and_dispatch assert identity.run_id == "direct-run" assert identity.resource_id == "asset:direct" assert current_event_dispatcher() is not None - with OperationLifecycle(operation_kind="python_node", operation_name="python_task"): + with OperationLifecycle(operation_kind="python_node", operation_name="python_asset"): nested: ExecutionIdentity | None = current_execution_identity() assert nested is not None assert nested.resource_attempt_id == identity.resource_attempt_id @@ -142,7 +142,7 @@ def test_given_invalid_direct_attempt_when_start_fails_then_transient_context_is with pytest.raises(ObservabilityValidationError): with ResourceAttemptLifecycle( resource_id="asset:invalid", - resource_kind="task", + resource_kind="asset", resource_name="invalid", attempt_number=-1, ): @@ -178,7 +178,7 @@ def test_given_parent_context_when_nested_start_fails_then_exact_parent_is_resto with pytest.raises(ObservabilityValidationError): with ResourceAttemptLifecycle( resource_id="asset:invalid", - resource_kind="task", + resource_kind="asset", resource_name="invalid", attempt_number=-1, ): diff --git a/ui/src/lib/api/http-catalog.test.ts b/ui/src/lib/api/http-catalog.test.ts index f39ca1fe70..249aabb956 100644 --- a/ui/src/lib/api/http-catalog.test.ts +++ b/ui/src/lib/api/http-catalog.test.ts @@ -34,7 +34,7 @@ describe('catalog node mapping', () => { const nodes = toCatalogNodes([ node('asset:order_feed'), node('loader:raw_customers'), - node('task:refresh_inventory', { dependencies: ['model:orders'] }), + node('asset:refresh_inventory', { dependencies: ['model:orders'] }), node('udf:cents_to_amount'), node('table_fn:orders_between'), node('model:orders', { materialization: 'table', dependencies: ['asset:order_feed', 'udf:cents_to_amount', 'audit:orders_positive'] }), @@ -48,13 +48,13 @@ describe('catalog node mapping', () => { expect(nodes.map((n) => [n.key, n.kind, n.materialization])).toEqual([ ['asset:order_feed', 'asset', null], ['loader:raw_customers', 'loader', null], - ['task:refresh_inventory', 'task', null], + ['asset:refresh_inventory', 'asset', null], ['udf:cents_to_amount', 'udf', null], ['table_fn:orders_between', 'table_fn', null], ['model:orders', 'table', 'table'] ]); expect(nodes.find((n) => n.key === 'model:orders')?.upstream).toEqual(['asset:order_feed', 'udf:cents_to_amount']); - expect(nodes.find((n) => n.key === 'task:refresh_inventory')?.upstream).toEqual(['model:orders']); + expect(nodes.find((n) => n.key === 'asset:refresh_inventory')?.upstream).toEqual(['model:orders']); }); it('keeps only upstream nodes that are listed, so graphs never point at a hidden node', () => { diff --git a/ui/src/lib/api/http-catalog.ts b/ui/src/lib/api/http-catalog.ts index 7c673cf03e..e8b0d1e306 100644 --- a/ui/src/lib/api/http-catalog.ts +++ b/ui/src/lib/api/http-catalog.ts @@ -8,7 +8,7 @@ import type { NodeKind } from './types/operate'; type Wire = components['schemas']; /** Snapshot kinds that are graph nodes under their own name; models take theirs from the materialization. */ -const GRAPH_KINDS: ReadonlySet = new Set(['source', 'seed', 'asset', 'loader', 'task', 'udf', 'table_fn']); +const GRAPH_KINDS: ReadonlySet = new Set(['source', 'seed', 'asset', 'loader', 'udf', 'table_fn']); /** * Every graph node kind is a catalog node: models, sources, seeds, Python nodes and functions. diff --git a/ui/src/lib/api/http-changes.ts b/ui/src/lib/api/http-changes.ts index 1b50115883..fee6418e5c 100644 --- a/ui/src/lib/api/http-changes.ts +++ b/ui/src/lib/api/http-changes.ts @@ -7,7 +7,7 @@ type Wire = components['schemas']; /** How many runs of each deploy the feed lists. */ const FOLLOWING_RUNS = 8; -const NODE_KINDS: Record = { seed: 'seed', source: 'source', asset: 'asset', loader: 'loader', task: 'task', udf: 'udf', table_fn: 'table_fn' }; +const NODE_KINDS: Record = { seed: 'seed', source: 'source', asset: 'asset', loader: 'loader', udf: 'udf', table_fn: 'table_fn' }; function opOf(kind: string): ChangeOp { return kind === 'added' || kind === 'removed' ? kind : 'changed'; diff --git a/ui/src/lib/api/http-providers.test.ts b/ui/src/lib/api/http-providers.test.ts index fe00ecfc74..3debdca514 100644 --- a/ui/src/lib/api/http-providers.test.ts +++ b/ui/src/lib/api/http-providers.test.ts @@ -16,23 +16,23 @@ describe('toProviders', () => { path: 'providers/fulfillment.py', settings: [{ name: 'api_key', type: 'SecretStr', required: true, secret: true, env: 'FULFILLMENT_API_KEY', description: null, missing: true }], used_by: [ - { kind: 'task', name: 'publish_orders' }, + { kind: 'asset', name: 'publish_orders' }, { kind: 'sink', name: 'order_events' }, { kind: 'event_exporter', name: 'run_events' } ], - nodes: ['task:publish_orders'], + nodes: ['asset:publish_orders'], jobs: ['order_exports'], - job_nodes: { order_exports: ['task:publish_orders'] } + job_nodes: { order_exports: ['asset:publish_orders'] } } ] }; const [provider] = toProviders(wire); expect(provider.className).toBe('providers.fulfillment.FulfillmentApi'); expect(provider.settings[0]).toEqual({ name: 'api_key', type: 'SecretStr', required: true, secret: true, env: 'FULFILLMENT_API_KEY', description: null, missing: true }); - expect(provider.usedBy.map((c) => `${c.kind}:${c.name}`)).toEqual(['task:publish_orders', 'sink:order_events', 'event_exporter:run_events']); + expect(provider.usedBy.map((c) => `${c.kind}:${c.name}`)).toEqual(['asset:publish_orders', 'sink:order_events', 'event_exporter:run_events']); expect(provider.jobs).toEqual(['order_exports']); - expect(provider.nodes).toEqual(['task:publish_orders']); - expect(provider.jobNodes).toEqual({ order_exports: ['task:publish_orders'] }); + expect(provider.nodes).toEqual(['asset:publish_orders']); + expect(provider.jobNodes).toEqual({ order_exports: ['asset:publish_orders'] }); }); it('defaults the nodes to none when the server leaves them out', () => { diff --git a/ui/src/lib/api/types/build.ts b/ui/src/lib/api/types/build.ts index a9e7da26be..88f0fa1517 100644 --- a/ui/src/lib/api/types/build.ts +++ b/ui/src/lib/api/types/build.ts @@ -56,7 +56,7 @@ export type CatalogNode = { liveStatus?: 'run' | 'ok' | 'warn' | 'fail'; key: string; kind: NodeKind; - /** e.g. `table`, `view`, `incremental · merge`, `incremental · microbatch`; null for sources and tasks. */ + /** e.g. `table`, `view`, `incremental · merge`, `incremental · microbatch`; null for sources and Python nodes. */ materialization: string | null; /** Null when the server does not report groups. */ group: string | null; diff --git a/ui/src/lib/api/types/operate.ts b/ui/src/lib/api/types/operate.ts index 5a9e8e6f77..f6685e9b19 100644 --- a/ui/src/lib/api/types/operate.ts +++ b/ui/src/lib/api/types/operate.ts @@ -196,8 +196,8 @@ export type RunNodeStatus = /** The run ended (lost worker, never started) before this node ran. */ | 'not_run'; -/** Graph node kinds: SQL models by materialization, sources and seeds, Python nodes (asset, loader, task) and functions (udf, table_fn). */ -export type NodeKind = 'source' | 'loader' | 'seed' | 'view' | 'table' | 'incremental' | 'task' | 'asset' | 'udf' | 'table_fn'; +/** Graph node kinds: SQL models by materialization, sources and seeds, Python nodes (asset, loader) and functions (udf, table_fn). */ +export type NodeKind = 'source' | 'loader' | 'seed' | 'view' | 'table' | 'incremental' | 'asset' | 'udf' | 'table_fn'; export type SkippedBecauseOf = { nodeKey: string; diff --git a/ui/src/lib/api/types/platform.ts b/ui/src/lib/api/types/platform.ts index 77d935ad2a..cb9ea47868 100644 --- a/ui/src/lib/api/types/platform.ts +++ b/ui/src/lib/api/types/platform.ts @@ -302,7 +302,7 @@ export type ProviderSetting = { /** Python code that injects a provider. None of it is a catalog node, so it has no page to open. */ export type ProviderConsumer = { - kind: 'loader' | 'task' | 'asset' | 'check' | 'hook' | 'materialization' | 'event_exporter' | 'sink'; + kind: 'loader' | 'asset' | 'check' | 'hook' | 'materialization' | 'event_exporter' | 'sink'; name: string; }; diff --git a/ui/src/lib/build-area/kind-tabs.test.ts b/ui/src/lib/build-area/kind-tabs.test.ts index f25b0f028d..8a637d61a5 100644 --- a/ui/src/lib/build-area/kind-tabs.test.ts +++ b/ui/src/lib/build-area/kind-tabs.test.ts @@ -3,7 +3,7 @@ import { NODE_KINDS, kindTabOf, nodeIcon, nodeIconKey, parseKindTab } from './la describe('catalog kind tabs', () => { it('puts Python nodes and functions in their own tabs', () => { - expect(['asset', 'loader', 'task'].map((k) => kindTabOf(k as never))).toEqual(['python', 'python', 'python']); + expect(['asset', 'loader'].map((k) => kindTabOf(k as never))).toEqual(['python', 'python']); expect(['udf', 'table_fn'].map((k) => kindTabOf(k as never))).toEqual(['functions', 'functions']); expect(kindTabOf('view')).toBe('view'); }); diff --git a/ui/src/lib/build-area/labels.ts b/ui/src/lib/build-area/labels.ts index 2c6d569801..0bf90108bf 100644 --- a/ui/src/lib/build-area/labels.ts +++ b/ui/src/lib/build-area/labels.ts @@ -10,7 +10,6 @@ import SquareFunction from '@lucide/svelte/icons/square-function'; import TableProperties from '@lucide/svelte/icons/table-properties'; import Sprout from '@lucide/svelte/icons/sprout'; import Table2 from '@lucide/svelte/icons/table-2'; -import Workflow from '@lucide/svelte/icons/workflow'; import type { BuiltDefinition, CatalogNode, @@ -33,7 +32,6 @@ export const nodeKindLabel: Record = { view: 'View', table: 'Table', incremental: 'Incremental', - task: 'Task', asset: 'Asset', udf: 'Function', table_fn: 'Table function' @@ -46,20 +44,19 @@ export const nodeKindPlural: Record = { view: 'Views', table: 'Tables', incremental: 'Incremental', - task: 'Tasks', asset: 'Assets', udf: 'Functions', table_fn: 'Table functions' }; -export const NODE_KINDS: NodeKind[] = ['source', 'loader', 'seed', 'view', 'table', 'incremental', 'task', 'asset', 'udf', 'table_fn']; +export const NODE_KINDS: NodeKind[] = ['source', 'loader', 'seed', 'view', 'table', 'incremental', 'asset', 'udf', 'table_fn']; /** The Catalog's kind tabs: SQL kinds one by one, Python nodes together, and functions together. */ export type KindTab = 'source' | 'seed' | 'view' | 'table' | 'incremental' | 'python' | 'functions'; export const KIND_TABS: KindTab[] = ['source', 'seed', 'view', 'table', 'incremental', 'python', 'functions']; -export const PYTHON_KINDS: readonly NodeKind[] = ['asset', 'loader', 'task']; +export const PYTHON_KINDS: readonly NodeKind[] = ['asset', 'loader']; export const FUNCTION_KINDS: readonly NodeKind[] = ['udf', 'table_fn']; export function kindTabOf(kind: NodeKind): KindTab { @@ -88,7 +85,6 @@ export function parseKindTab(value: string | null): KindTab | null { /** Icon key used by the lineage cards, from the node kind and materialization. */ export function nodeIconKey(node: Pick): string { if (node.kind === 'incremental') return node.materialization?.includes('microbatch') ? 'microbatch' : node.materialization?.includes('merge') ? 'merge' : 'delete_insert'; - if (node.kind === 'task') return 'task'; return node.kind; } @@ -102,7 +98,6 @@ export const nodeIcon: Record = { merge: GitMerge, delete_insert: Layers, microbatch: Layers, - task: Workflow, asset: Package, udf: SquareFunction, table_fn: TableProperties diff --git a/ui/src/lib/components/lineage/asset-node.svelte b/ui/src/lib/components/lineage/asset-node.svelte index 1cf17e1527..7ff09418ac 100644 --- a/ui/src/lib/components/lineage/asset-node.svelte +++ b/ui/src/lib/components/lineage/asset-node.svelte @@ -12,7 +12,6 @@ GitMerge, Repeat, Columns3, - Workflow, Package, TableProperties, type Icon as IconType @@ -82,7 +81,6 @@ merge: GitMerge, delete_insert: Repeat, microbatch: Layers, - task: Workflow, asset: Package, udf: FunctionSquare, table_fn: TableProperties, diff --git a/ui/src/lib/format/plural.test.ts b/ui/src/lib/format/plural.test.ts index 7804a2a597..81fc465f16 100644 --- a/ui/src/lib/format/plural.test.ts +++ b/ui/src/lib/format/plural.test.ts @@ -12,7 +12,6 @@ describe('counts', () => { it('name node kinds, telling SQL audits from Python checks', () => { expect(nodeCount(407, 'asset')).toBe('407 assets'); expect(nodeCount(1, 'asset')).toBe('1 asset'); - expect(nodeCount(26, 'task')).toBe('26 tasks'); expect(nodeCount(35, 'check')).toBe('35 Python checks'); expect(nodeCount(4668, 'audit')).toBe('4,668 audits'); expect(nodeNoun('table_fn', 2)).toBe('table functions'); diff --git a/ui/src/lib/format/plural.ts b/ui/src/lib/format/plural.ts index 238705248f..5c0d2631fb 100644 --- a/ui/src/lib/format/plural.ts +++ b/ui/src/lib/format/plural.ts @@ -23,7 +23,6 @@ export const NODE_NOUNS: Record = { snapshot: ['snapshot', 'snapshots'], loader: ['loader', 'loaders'], asset: ['asset', 'assets'], - task: ['task', 'tasks'], check: ['Python check', 'Python checks'], python_check: ['Python check', 'Python checks'], audit: ['audit', 'audits'], diff --git a/ui/src/routes/providers/utils.test.ts b/ui/src/routes/providers/utils.test.ts index 84452d9c9a..c88ef42c05 100644 --- a/ui/src/routes/providers/utils.test.ts +++ b/ui/src/routes/providers/utils.test.ts @@ -37,8 +37,8 @@ describe('usedByGraph', () => { }); it('falls back to the consumer of the same name for keys without a kind', () => { - const graph = usedByGraph({ usedBy: [{ kind: 'task', name: 'export_orders' }], nodes: [], jobs: ['exports'], jobNodes: { exports: ['export_orders'] } }); - expect(graph.groups.map((g) => [g.kind, g.nodes, g.jobs])).toEqual([['task', ['export_orders'], ['exports']]]); + const graph = usedByGraph({ usedBy: [{ kind: 'asset', name: 'export_orders' }], nodes: [], jobs: ['exports'], jobNodes: { exports: ['export_orders'] } }); + expect(graph.groups.map((g) => [g.kind, g.nodes, g.jobs])).toEqual([['asset', ['export_orders'], ['exports']]]); }); it('is empty for an unused provider', () => { diff --git a/ui/src/routes/providers/utils.ts b/ui/src/routes/providers/utils.ts index e94a356969..ab1fd04bdf 100644 --- a/ui/src/routes/providers/utils.ts +++ b/ui/src/routes/providers/utils.ts @@ -4,7 +4,6 @@ import { plural } from '$lib/format/plural'; export const consumerKindLabel: Record = { loader: 'loader', - task: 'task', asset: 'asset', check: 'check', hook: 'hook', @@ -38,14 +37,13 @@ const KIND_WORDS: Record = { table: ['table', 'tables'], incremental: ['incremental model', 'incremental models'], model: ['model', 'models'], - task: ['task', 'tasks'], asset: ['asset', 'assets'], check: ['check', 'checks'], udf: ['function', 'functions'], table_fn: ['table function', 'table functions'] }; /** Groups in the order the Catalog lists kinds; any other kind follows, alphabetically. */ -const KIND_ORDER = ['loader', 'source', 'seed', 'view', 'table', 'incremental', 'model', 'asset', 'task', 'check', 'udf', 'table_fn']; +const KIND_ORDER = ['loader', 'source', 'seed', 'view', 'table', 'incremental', 'model', 'asset', 'check', 'udf', 'table_fn']; /** Consumers that run inside other nodes or outside jobs, so they have no node of their own in the graph. */ const OTHER_CONSUMERS: ReadonlySet = new Set(['hook', 'materialization', 'sink', 'event_exporter']); diff --git a/website/src/content/docs/docs/cli/build.mdx b/website/src/content/docs/docs/cli/build.mdx index 2aeda7d811..3686481639 100644 --- a/website/src/content/docs/docs/cli/build.mdx +++ b/website/src/content/docs/docs/cli/build.mdx @@ -339,7 +339,7 @@ each schema carry the same run ID, so a role that can build in a schema can also there, with no other grants. A run that writes to any schema never writes these rows anywhere else, such as the target's default schema. -A run whose nodes write no relation, such as one that only runs Python tasks or checks, has no +A run whose nodes write no relation, such as one that only runs Python assets or checks, has no schema of its own. It records in the target's default schema instead, where audit results go. A run's status is its `finished` row, in any of its schemas. A run with only `started` rows was diff --git a/website/src/content/docs/docs/concepts/python-nodes/assets.mdx b/website/src/content/docs/docs/concepts/python-nodes/assets.mdx index 2f91a3c5c2..908cc89136 100644 --- a/website/src/content/docs/docs/concepts/python-nodes/assets.mdx +++ b/website/src/content/docs/docs/concepts/python-nodes/assets.mdx @@ -3,7 +3,9 @@ title: "Assets" description: "Run Python computation, side effects and external artifacts as nodes in the SQLBuild graph." --- -Assets are Python functions that run as part of the DAG. Use them for computation, side effects and external artifacts such as an exported file, a published dataset, a dashboard refresh or an ML model. See [Python Nodes](/docs/concepts/python-nodes/overview) for the shared model and the SQL boundary rules. +Assets are Python functions that run as part of the DAG, for computation, side effects and external artifacts such as an exported file, a published dataset, a dashboard refresh or an ML model. If you are coming from a task-based tool, an asset is a task that may not describe its output. + +See [Python Nodes](/docs/concepts/python-nodes/overview) for the shared model and the SQL boundary rules. ## Defining an asset From 21e413b9250daf8bf114d74d7e5b3bd084b55418 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 20 Nov 2026 22:33:28 +0000 Subject: [PATCH 06/12] fix: keep stored Python task attempts readable and drop task from messages Keep python_task in the lifecycle operation catalogue as a retired name so history written by earlier versions still decodes. Remove task from the remaining validation and executor messages, and regenerate the compiler goldens that carried the old wording. --- crates/sqlbuild-cli/src/adhoc/_helpers/publication.rs | 2 +- crates/sqlbuild-core/src/lifecycle_events/constants.rs | 6 ++++-- .../src/project_validation/_helpers/loaders.rs | 2 +- .../src/project_validation/_helpers/python_nodes.rs | 2 +- .../src/project_validation/_helpers/resource_names.rs | 2 +- crates/sqlbuild-discovery/src/project_validation/models.rs | 4 ++-- .../src/project_validation/tests/helpers.rs | 2 +- .../src/compiled_views/_helpers/project/project_view.rs | 2 +- .../node_results/classes/dependency_result_reader.py | 4 ++-- src/sqlbuild/executor/python_nodes/_helpers/execution.py | 4 ++-- .../executor/python_nodes/_helpers/ingress_execution.py | 2 +- .../executor/python_nodes/_helpers/relation_guard.py | 2 +- src/sqlbuild/executor/python_nodes/main/_ingress.py | 2 +- src/sqlbuild/executor/python_nodes/main/_ready.py | 4 ++-- tests/goldens/compiler/failure/check-depends-on-check.json | 2 +- tests/goldens/compiler/failure/duplicate-python-node.json | 2 +- .../compiler/failure/python-node-description-too-long.json | 2 +- .../compiler/failure/python-node-missing-description.json | 4 ++-- ...target-preserved-schema-without-node-results-schema.json | 2 +- .../src/sqlbuild/cli/commands/main/test_run_records.py | 4 ++-- .../compiled_project/test_orchestration_snapshot.py | 2 +- .../sqlbuild/runtime/observability/test_serialization.py | 3 +++ 22 files changed, 33 insertions(+), 28 deletions(-) diff --git a/crates/sqlbuild-cli/src/adhoc/_helpers/publication.rs b/crates/sqlbuild-cli/src/adhoc/_helpers/publication.rs index 396f6f7316..2533962232 100644 --- a/crates/sqlbuild-cli/src/adhoc/_helpers/publication.rs +++ b/crates/sqlbuild-cli/src/adhoc/_helpers/publication.rs @@ -42,7 +42,7 @@ pub(crate) struct PythonIndex { /// Node names, which a bare selector may name instead of a job. pub(crate) names: Vec, pub(crate) graph: PythonSelectionGraph, - /// Node name to kind, such as `task`. + /// Node name to kind, such as `asset`. pub(crate) kinds: BTreeMap, } diff --git a/crates/sqlbuild-core/src/lifecycle_events/constants.rs b/crates/sqlbuild-core/src/lifecycle_events/constants.rs index 14bf878aca..0e63513715 100644 --- a/crates/sqlbuild-core/src/lifecycle_events/constants.rs +++ b/crates/sqlbuild-core/src/lifecycle_events/constants.rs @@ -223,8 +223,9 @@ pub const OPERATION_KINDS: [&str; 10] = [ "warehouse", ]; -/// Includes retired names, such as the removed dlt and ingestr loads, so stored history still decodes. -pub const OPERATION_NAMES: [&str; 56] = [ +/// Includes retired names, such as the removed dlt and ingestr loads and Python tasks, so stored history +/// still decodes. +pub const OPERATION_NAMES: [&str; 57] = [ "adbc_driver_install", "audit_evaluation", "clone_execution", @@ -258,6 +259,7 @@ pub const OPERATION_NAMES: [&str; 56] = [ "python_check", "python_hook", "python_materialization", + "python_task", "relation_promotion", "retention_application", "retention_inspection", diff --git a/crates/sqlbuild-discovery/src/project_validation/_helpers/loaders.rs b/crates/sqlbuild-discovery/src/project_validation/_helpers/loaders.rs index 4af09a2dea..5b693fcb65 100644 --- a/crates/sqlbuild-discovery/src/project_validation/_helpers/loaders.rs +++ b/crates/sqlbuild-discovery/src/project_validation/_helpers/loaders.rs @@ -25,7 +25,7 @@ pub(crate) fn validate_loaders(facts: &ProjectFacts) -> Result<(), ValidationFai unique_named(&node_paths, |name, existing, path| { format!( "Duplicate Python node found for '{name}' in {existing} and {path}; loader, \ - task, asset, and check names must be globally unique" + asset, and check names must be globally unique" ) })?; source_loader_name_collisions(&facts.sources, &loaders)?; diff --git a/crates/sqlbuild-discovery/src/project_validation/_helpers/python_nodes.rs b/crates/sqlbuild-discovery/src/project_validation/_helpers/python_nodes.rs index f83ee1a655..69bec0f843 100644 --- a/crates/sqlbuild-discovery/src/project_validation/_helpers/python_nodes.rs +++ b/crates/sqlbuild-discovery/src/project_validation/_helpers/python_nodes.rs @@ -90,7 +90,7 @@ pub(crate) fn validate_python_node_dependencies( if node.kind == NodeKind::Loader { return Err(conflict(format!( "Loader '{}' depends on SQL resource '{name}'; loaders may depend on \ - Python loader/task/asset functions only", + Python loader/asset functions only", node.name ))); } diff --git a/crates/sqlbuild-discovery/src/project_validation/_helpers/resource_names.rs b/crates/sqlbuild-discovery/src/project_validation/_helpers/resource_names.rs index 6191263dd9..7543a4a28b 100644 --- a/crates/sqlbuild-discovery/src/project_validation/_helpers/resource_names.rs +++ b/crates/sqlbuild-discovery/src/project_validation/_helpers/resource_names.rs @@ -141,7 +141,7 @@ fn name_conflict(first: &DeclaredName<'_>, second: &DeclaredName<'_>) -> Validat "" }; conflict(format!( - "Project resource name '{}' is declared as both {} in {} and {} in {}; model, source, seed, function, loader, task, asset, check, provider, \ + "Project resource name '{}' is declared as both {} in {} and {} in {}; model, source, seed, function, loader, asset, check, provider, \ hook, unit test, scenario, job, job group, sensor, and calendar names must be \ globally unique; rename one of them{advice}", first.name, first.kind, first.path, second.kind, second.path diff --git a/crates/sqlbuild-discovery/src/project_validation/models.rs b/crates/sqlbuild-discovery/src/project_validation/models.rs index 88b31ae896..5f72f81c88 100644 --- a/crates/sqlbuild-discovery/src/project_validation/models.rs +++ b/crates/sqlbuild-discovery/src/project_validation/models.rs @@ -61,12 +61,12 @@ pub enum DependencyFact { Object { /// The index of the discovered node whose function this object is. owner: Option, - /// The name a loader, task, asset or check decorator records on the object. + /// The name a loader, asset or check decorator records on the object. decorated_name: Option, }, } -/// A decorated Python loader, task, asset or check function. +/// A decorated Python loader, asset or check function. #[derive(Clone, Debug, PartialEq, Eq)] pub struct NodeFact { pub kind: NodeKind, diff --git a/crates/sqlbuild-discovery/src/project_validation/tests/helpers.rs b/crates/sqlbuild-discovery/src/project_validation/tests/helpers.rs index d7b42c6708..8a9482cd03 100644 --- a/crates/sqlbuild-discovery/src/project_validation/tests/helpers.rs +++ b/crates/sqlbuild-discovery/src/project_validation/tests/helpers.rs @@ -105,7 +105,7 @@ pub(super) fn collision(message: &str, advice: &str) -> Result<(), String> { failure( FailureKind::Conflict, &format!( - "{message}; model, source, seed, function, loader, task, asset, check, provider, \ + "{message}; model, source, seed, function, loader, asset, check, provider, \ hook, unit test, scenario, job, job group, sensor, and calendar names must \ be globally unique; rename one of them{advice}" ), diff --git a/crates/sqlbuild-python-project/src/compiled_views/_helpers/project/project_view.rs b/crates/sqlbuild-python-project/src/compiled_views/_helpers/project/project_view.rs index eb60838df9..d95fe8b8ac 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/_helpers/project/project_view.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/_helpers/project/project_view.rs @@ -35,7 +35,7 @@ pub(crate) struct BoundaryInputs { pub(crate) compile_cache_dir: Py, /// Discovered loader functions, user callables. pub(crate) loader_functions: Py, - /// Task/asset/check source locations, without retaining additional Python callables. + /// Asset and check source locations, without retaining additional Python callables. pub(crate) python_execution_inputs: BTreeMap, /// Discovered Python hooks, user callables. pub(crate) hook_functions: Py, diff --git a/src/sqlbuild/executor/node_results/classes/dependency_result_reader.py b/src/sqlbuild/executor/node_results/classes/dependency_result_reader.py index 3f21b2b6c6..8d846ef303 100644 --- a/src/sqlbuild/executor/node_results/classes/dependency_result_reader.py +++ b/src/sqlbuild/executor/node_results/classes/dependency_result_reader.py @@ -53,12 +53,12 @@ def result_dependency_identity(self, node_function: Callable[..., object]) -> tu identity: tuple[str, str] | None = self.result_node_identity(node_function) if identity is None: raise ExecutorInputError( - "Python node result dependency must be a task, asset, or loader" + "Python node result dependency must be an asset or loader" ) return identity def result_node_identity(self, node_function: Callable[..., object]) -> tuple[str, str] | None: - """Return the `(kind, name)` of a decorated task, asset or loader, else None.""" + """Return the `(kind, name)` of a decorated asset or loader, else None.""" for attribute, kind in RESULT_DEPENDENCY_ATTRIBUTES: definition: object = getattr(node_function, attribute, None) diff --git a/src/sqlbuild/executor/python_nodes/_helpers/execution.py b/src/sqlbuild/executor/python_nodes/_helpers/execution.py index 7aa365557d..52f77589d2 100644 --- a/src/sqlbuild/executor/python_nodes/_helpers/execution.py +++ b/src/sqlbuild/executor/python_nodes/_helpers/execution.py @@ -1,4 +1,4 @@ -"""Serial executor helpers for Python task and asset nodes.""" +"""Serial executor helpers for Python asset nodes.""" from __future__ import annotations @@ -66,7 +66,7 @@ def execute_ready_python_node( monotonic: Callable[[], float] = time.monotonic, on_result: OwnedResultCallback | None = None, ) -> PythonNodeExecutionResult: - """Execute one scheduler-ready task/asset node.""" + """Execute one scheduler-ready asset node.""" return _execute_ready_node( node=node, diff --git a/src/sqlbuild/executor/python_nodes/_helpers/ingress_execution.py b/src/sqlbuild/executor/python_nodes/_helpers/ingress_execution.py index 6e8508526e..ad244d1b69 100644 --- a/src/sqlbuild/executor/python_nodes/_helpers/ingress_execution.py +++ b/src/sqlbuild/executor/python_nodes/_helpers/ingress_execution.py @@ -67,7 +67,7 @@ def execute_ingress_python_loader_nodes( runtime: PythonNodeRuntime, callbacks: IngressCallbacks | None = None, ) -> PythonIngressLoaderExecutorResult: - """Execute Python ingress task/asset/loader nodes in lifecycle topological order.""" + """Execute Python ingress asset and loader nodes in lifecycle topological order.""" resolved_callbacks: IngressCallbacks = IngressCallbacks() if callbacks is None else callbacks run_state: PythonNodeRunState = PythonNodeRunState() diff --git a/src/sqlbuild/executor/python_nodes/_helpers/relation_guard.py b/src/sqlbuild/executor/python_nodes/_helpers/relation_guard.py index fd755d2af9..e311880a18 100644 --- a/src/sqlbuild/executor/python_nodes/_helpers/relation_guard.py +++ b/src/sqlbuild/executor/python_nodes/_helpers/relation_guard.py @@ -1,4 +1,4 @@ -"""Runtime relation guards for task, asset, and check contexts.""" +"""Runtime relation guards for asset and check contexts.""" from __future__ import annotations diff --git a/src/sqlbuild/executor/python_nodes/main/_ingress.py b/src/sqlbuild/executor/python_nodes/main/_ingress.py index ea1d03a5f5..3ba062152b 100644 --- a/src/sqlbuild/executor/python_nodes/main/_ingress.py +++ b/src/sqlbuild/executor/python_nodes/main/_ingress.py @@ -24,7 +24,7 @@ def run_ingress_python_loader_nodes( runtime: PythonNodeRuntime, callbacks: IngressCallbacks | None = None, ) -> PythonIngressLoaderExecutorResult: - """Execute Python ingress task/asset/loader nodes in lifecycle order.""" + """Execute Python ingress asset and loader nodes in lifecycle order.""" return execute_ingress_python_loader_nodes( python_graph=python_graph, diff --git a/src/sqlbuild/executor/python_nodes/main/_ready.py b/src/sqlbuild/executor/python_nodes/main/_ready.py index d84acd8144..a615ad5bdb 100644 --- a/src/sqlbuild/executor/python_nodes/main/_ready.py +++ b/src/sqlbuild/executor/python_nodes/main/_ready.py @@ -1,4 +1,4 @@ -"""Public executor entrypoint for one ready task/asset Python node.""" +"""Public executor entrypoint for one ready asset Python node.""" from __future__ import annotations @@ -21,7 +21,7 @@ def run_ready_python_node( run_state: PythonNodeRunState, on_result: OwnedResultCallback | None = None, ) -> PythonNodeExecutionResult: - """Execute one scheduler-ready task/asset Python node.""" + """Execute one scheduler-ready asset Python node.""" return execute_ready_python_node( node=node, diff --git a/tests/goldens/compiler/failure/check-depends-on-check.json b/tests/goldens/compiler/failure/check-depends-on-check.json index 79039c3ed2..c124751801 100644 --- a/tests/goldens/compiler/failure/check-depends-on-check.json +++ b/tests/goldens/compiler/failure/check-depends-on-check.json @@ -8,7 +8,7 @@ "phase": "compile", "severity": "error", "code": "D007", - "message": "Check 'orders_checked' depends on another check; checks may depend on loaders, tasks, assets, models, sources, and seeds only" + "message": "Check 'orders_checked' depends on another check; checks may depend on loaders, assets, models, sources, and seeds only" } ] } diff --git a/tests/goldens/compiler/failure/duplicate-python-node.json b/tests/goldens/compiler/failure/duplicate-python-node.json index 7b8d648d20..d8090a36ba 100644 --- a/tests/goldens/compiler/failure/duplicate-python-node.json +++ b/tests/goldens/compiler/failure/duplicate-python-node.json @@ -8,7 +8,7 @@ "phase": "compile", "severity": "error", "code": "D007", - "message": "Duplicate Python node found for 'export_orders' in python/tasks/first.py and python/tasks/second.py; loader, task, asset, and check names must be globally unique" + "message": "Duplicate Python node found for 'export_orders' in python/tasks/first.py and python/tasks/second.py; loader, asset, and check names must be globally unique" } ] } diff --git a/tests/goldens/compiler/failure/python-node-description-too-long.json b/tests/goldens/compiler/failure/python-node-description-too-long.json index 34bd8ae483..2ae93a0356 100644 --- a/tests/goldens/compiler/failure/python-node-description-too-long.json +++ b/tests/goldens/compiler/failure/python-node-description-too-long.json @@ -8,7 +8,7 @@ "phase": "compile", "severity": "error", "code": "P027", - "message": "task 'export_orders' description exceeds 10 lines", + "message": "asset 'export_orders' description exceeds 10 lines", "resource_name": "export_orders", "path": "python/tasks/exports.py", "line": 4, diff --git a/tests/goldens/compiler/failure/python-node-missing-description.json b/tests/goldens/compiler/failure/python-node-missing-description.json index ab92ad8d57..eef733411c 100644 --- a/tests/goldens/compiler/failure/python-node-missing-description.json +++ b/tests/goldens/compiler/failure/python-node-missing-description.json @@ -8,7 +8,7 @@ "phase": "compile", "severity": "error", "code": "P010", - "message": "task 'export_orders' has no description", + "message": "asset 'export_orders' has no description", "resource_name": "export_orders", "path": "python/tasks/exports.py", "line": 4, @@ -18,7 +18,7 @@ "line": 4, "column": 1 }, - "help": "to describe task 'export_orders', give its function in python/tasks/exports.py a docstring or pass a description to @asset:\n @asset(description=\"What export_orders does\", ...)" + "help": "to describe asset 'export_orders', give its function in python/tasks/exports.py a docstring or pass a description to @asset:\n @asset(description=\"What export_orders does\", ...)" } ] } diff --git a/tests/goldens/compiler/failure/target-preserved-schema-without-node-results-schema.json b/tests/goldens/compiler/failure/target-preserved-schema-without-node-results-schema.json index 3b47051dde..214b94686b 100644 --- a/tests/goldens/compiler/failure/target-preserved-schema-without-node-results-schema.json +++ b/tests/goldens/compiler/failure/target-preserved-schema-without-node-results-schema.json @@ -8,7 +8,7 @@ "phase": "compile", "severity": "error", "code": "S105", - "message": "Target 'dev' keeps each model's own schema, so it has no schema for the results of loaders, tasks, assets and checks. Set targets.dev.loader_schema; Python node results are kept there.", + "message": "Target 'dev' keeps each model's own schema, so it has no schema for the results of loaders, assets and checks. Set targets.dev.loader_schema; Python node results are kept there.", "help": "to keep Python node results in a schema, set this in sqlbuild_project.toml:\n [targets.dev]\n loader_schema = \"my_schema\"" } ] diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_run_records.py b/tests/integration/src/sqlbuild/cli/commands/main/test_run_records.py index 1f60a386a4..1b981a7540 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_run_records.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_run_records.py @@ -520,8 +520,8 @@ def test_given_a_killed_build_when_reading_its_runs_then_only_started_remains( expected_finished_status="failed", ), RunRecordSchemasTestCase( - description="a task-only run records in the target's default schema", - command_args=("build", "--no-tests", "--select", "task:noop"), + description="an asset-only run records in the target's default schema", + command_args=("build", "--no-tests", "--select", "asset:noop"), settings='\n[defaults]\nschema = "analytics"\n' + RUN_RECORD_MARTS_SCHEMA, warehouse_setup="", expected_exit_code=0, diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py index 1091076192..ba4af8097d 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -58,7 +58,7 @@ from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files ORCHESTRATION_NAME_CONFLICT: str = ( - "; model, source, seed, function, loader, task, asset, check, provider, hook, unit test, " + "; model, source, seed, function, loader, asset, check, provider, hook, unit test, " "scenario, job, job group, sensor, and calendar names must be globally unique; rename one of " "them (a job or job group takes its name from `name` in its TOML file, and a calendar or " "sensor from its `name` argument)" diff --git a/tests/unit/src/sqlbuild/runtime/observability/test_serialization.py b/tests/unit/src/sqlbuild/runtime/observability/test_serialization.py index 4fe58e6704..b270b50ed8 100644 --- a/tests/unit/src/sqlbuild/runtime/observability/test_serialization.py +++ b/tests/unit/src/sqlbuild/runtime/observability/test_serialization.py @@ -162,6 +162,9 @@ def test_given_known_event_when_serializing_and_decoding_then_round_trips( RetiredOperationCase( "a stored ingestr subprocess", "subprocess", "ingestr_command", "ingestr_command" ), + RetiredOperationCase( + "a stored Python task attempt", "python_node", "python_task", "python_task" + ), ], ids=lambda case: case.description, ) From ee6fb979f50a3cf4de5b4171cd08be67fc231f8a Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 20 Nov 2026 22:40:40 +0000 Subject: [PATCH 07/12] chore: record the Python node kind parser as an intentional mirror --- fensu.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/fensu.toml b/fensu.toml index 77361d53d2..536c46bedf 100644 --- a/fensu.toml +++ b/fensu.toml @@ -4016,3 +4016,7 @@ reason = "Each reads a two-field key from a different record type (a SqlResource [[dupes.allowlist]] paths = ["crates/sqlbuild-orchestration/src/credentials/main/locations/location_named.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/read_development_location.rs"] reason = "One reads the active location with a name, the other the development location of a target whose project path matches; each is one control-store read with its own SQL and filter, and only the read-then-decode-the-identity shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-compile/src/rules/_helpers/sql/sql_test_objects.rs", "crates/sqlbuild-python-front/src/discovery/_helpers/project/project_facts.rs"] +reason = "Each parses one closed string set into its own enum: SQL test modes, directly tested resource kinds, and Python node kinds. Only the match-or-ValueError shape is alike." From 75327216c58e8993bfb600fcef174dc6d293e434 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 20 Nov 2026 22:46:08 +0000 Subject: [PATCH 08/12] style: format the integration result kinds --- crates/sqlbuild-core/src/integration_results/constants.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/sqlbuild-core/src/integration_results/constants.rs b/crates/sqlbuild-core/src/integration_results/constants.rs index 839116b821..b1c80964da 100644 --- a/crates/sqlbuild-core/src/integration_results/constants.rs +++ b/crates/sqlbuild-core/src/integration_results/constants.rs @@ -23,8 +23,8 @@ pub(crate) const MAX_IDENTIFIER_CHARS: usize = 512; pub(crate) const MAX_COLLECTION_ITEMS: usize = 128; pub(crate) const MAX_NESTING_DEPTH: usize = 6; pub(crate) const ASSET_KINDS: [&str; 11] = [ - "asset", "custom", "loader", "model", "seed", "snapshot", "source", "table", "table_fn", - "udf", "view", + "asset", "custom", "loader", "model", "seed", "snapshot", "source", "table", "table_fn", "udf", + "view", ]; pub(crate) const CHECK_KINDS: [&str; 3] = [AUDIT_KIND, "python_check", "sql_test"]; pub(crate) const EXTRA_RESOURCE_KINDS: [&str; 3] = ["audit", "check", "test"]; From ae427d3673d63c0acce9fc85d26ea7cef1b670f4 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 20 Nov 2026 23:02:23 +0000 Subject: [PATCH 09/12] style: format and keep the retired-name doc comment on one line --- crates/sqlbuild-core/src/lifecycle_events/constants.rs | 3 +-- .../executor/node_results/classes/dependency_result_reader.py | 4 +--- .../sqlbuild/executor/node_results/main/test_read_write.py | 4 +++- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/crates/sqlbuild-core/src/lifecycle_events/constants.rs b/crates/sqlbuild-core/src/lifecycle_events/constants.rs index 0e63513715..0091735078 100644 --- a/crates/sqlbuild-core/src/lifecycle_events/constants.rs +++ b/crates/sqlbuild-core/src/lifecycle_events/constants.rs @@ -223,8 +223,7 @@ pub const OPERATION_KINDS: [&str; 10] = [ "warehouse", ]; -/// Includes retired names, such as the removed dlt and ingestr loads and Python tasks, so stored history -/// still decodes. +/// Includes retired names, such as removed loads and Python tasks, so stored history still decodes. pub const OPERATION_NAMES: [&str; 57] = [ "adbc_driver_install", "audit_evaluation", diff --git a/src/sqlbuild/executor/node_results/classes/dependency_result_reader.py b/src/sqlbuild/executor/node_results/classes/dependency_result_reader.py index 8d846ef303..a68ff14d86 100644 --- a/src/sqlbuild/executor/node_results/classes/dependency_result_reader.py +++ b/src/sqlbuild/executor/node_results/classes/dependency_result_reader.py @@ -52,9 +52,7 @@ def result_dependency_identity(self, node_function: Callable[..., object]) -> tu identity: tuple[str, str] | None = self.result_node_identity(node_function) if identity is None: - raise ExecutorInputError( - "Python node result dependency must be an asset or loader" - ) + raise ExecutorInputError("Python node result dependency must be an asset or loader") return identity def result_node_identity(self, node_function: Callable[..., object]) -> tuple[str, str] | None: diff --git a/tests/integration/src/sqlbuild/executor/node_results/main/test_read_write.py b/tests/integration/src/sqlbuild/executor/node_results/main/test_read_write.py index 322b683473..c64ba37a17 100644 --- a/tests/integration/src/sqlbuild/executor/node_results/main/test_read_write.py +++ b/tests/integration/src/sqlbuild/executor/node_results/main/test_read_write.py @@ -152,7 +152,9 @@ def test_given_result_in_earlier_location_when_reading_then_store_falls_back_onl schema="raw", target_config=TargetSettings(schema_preserved=test_case.schema_preserved), ) - found: object = reader.result_of(node_type="asset", node_name="produce_result", default=None) + found: object = reader.result_of( + node_type="asset", node_name="produce_result", default=None + ) finally: adapter.close(connection) From d56c0af37bcba52b2912a908dbecac5f32021027 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 21 Nov 2026 00:07:27 +0000 Subject: [PATCH 10/12] test: move the node-unit fixture from @sqlbuild.task to @sqlbuild.asset --- .../src/node_units/tests/test_node_unit_module.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/sqlbuild-pysource/src/node_units/tests/test_node_unit_module.rs b/crates/sqlbuild-pysource/src/node_units/tests/test_node_unit_module.rs index 8897ec6cd8..7c3488f0ac 100644 --- a/crates/sqlbuild-pysource/src/node_units/tests/test_node_unit_module.rs +++ b/crates/sqlbuild-pysource/src/node_units/tests/test_node_unit_module.rs @@ -6,7 +6,7 @@ use crate::node_units::tests::test_types::{ FunctionTestCase, ImportsTestCase, StatementKindsTestCase, }; -const CRLF_NODE_MODULE: &str = "\"\"\"Doc.\"\"\"\r\nfrom .main import main\r\nfrom python.orders.daily.node import daily as upstream\r\nimport python.utils.text\r\n\r\nX = 'é'\r\n\r\n\r\n@sqlbuild.task(depends_on=[upstream], description=\"D.\")\r\ndef run(ctx: AssetContext, /, *, store: \"providers.Store\") -> PythonNodeResult | None:\r\n \"\"\"Doc.\"\"\"\r\n return main(ctx=ctx, store=store)\r\n"; +const CRLF_NODE_MODULE: &str = "\"\"\"Doc.\"\"\"\r\nfrom .main import main\r\nfrom python.orders.daily.node import daily as upstream\r\nimport python.utils.text\r\n\r\nX = 'é'\r\n\r\n\r\n@sqlbuild.asset(depends_on=[upstream], description=\"D.\")\r\ndef run(ctx: AssetContext, /, *, store: \"providers.Store\") -> PythonNodeResult | None:\r\n \"\"\"Doc.\"\"\"\r\n return main(ctx=ctx, store=store)\r\n"; const FACTORY_MODULE: &str = "@factory\ndef tables():\n for table in TABLES:\n for _ in specs():\n pass\n\n @asset(name=table)\n def load(ctx, *args, **kwargs) -> Optional[Union[LoaderResult, list]]:\n rows = main(ctx=ctx)\n return rows\n\n def plain():\n return main(ctx, **extra)\n"; From 708b5b2137d9ea07d842a5da91d54bfe3d168785 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 21 Nov 2026 00:33:23 +0000 Subject: [PATCH 11/12] test: restore the mistyped ctx in the node-signature rule fixture and rename task fixtures --- .../node_set_dependencies/tests/helpers.rs | 2 +- .../test_resolve_node_set_dependencies.rs | 2 +- .../tests/runs/test_group_round_trips.rs | 6 +-- .../src/python_rules/tests/test_node_units.rs | 46 +++++++++---------- 4 files changed, 28 insertions(+), 28 deletions(-) diff --git a/crates/sqlbuild-analysis/src/node_set_dependencies/tests/helpers.rs b/crates/sqlbuild-analysis/src/node_set_dependencies/tests/helpers.rs index ad7f2b4b3a..c2d4eb1e27 100644 --- a/crates/sqlbuild-analysis/src/node_set_dependencies/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/node_set_dependencies/tests/helpers.rs @@ -64,7 +64,7 @@ const JOBS: &[(&str, &str)] = &[ ("nightly", "stg_orders orders order_items"), ("single", "customers"), ("diamond", "stg_orders orders order_items finance_daily"), - ("with_task", "orders asset:export_orders"), + ("with_asset", "orders asset:export_orders"), ("loop", "asset:export_orders asset:audit_export"), ("nightly_wide", "orders customer_orders"), ("reporting", "asset:report customers"), diff --git a/crates/sqlbuild-analysis/src/node_set_dependencies/tests/test_resolve_node_set_dependencies.rs b/crates/sqlbuild-analysis/src/node_set_dependencies/tests/test_resolve_node_set_dependencies.rs index fea84dc125..5a3f102d1e 100644 --- a/crates/sqlbuild-analysis/src/node_set_dependencies/tests/test_resolve_node_set_dependencies.rs +++ b/crates/sqlbuild-analysis/src/node_set_dependencies/tests/test_resolve_node_set_dependencies.rs @@ -28,7 +28,7 @@ fn given_job_and_tag_dependencies_when_resolving_then_each_reduces_to_its_end_no EndNodesTestCase { description: "a Python node reading the job's model is its end", groups: &[], - sets: &[("job", "with_task", None)], + sets: &[("job", "with_asset", None)], expected_ends: &[&["asset 'export_orders'"]], }, EndNodesTestCase { diff --git a/crates/sqlbuild-orchestration/tests/runs/test_group_round_trips.rs b/crates/sqlbuild-orchestration/tests/runs/test_group_round_trips.rs index 128552fe3b..c3e544bbc1 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_group_round_trips.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_group_round_trips.rs @@ -1,4 +1,4 @@ -//! A group orders a warehouse round trip by its nearest lineage, SQL to task to loader to models, on both stores. +//! A group orders a warehouse round trip by its nearest lineage, SQL to asset to loader to models, on both stores. use std::collections::BTreeMap; @@ -25,7 +25,7 @@ use crate::store::tests::store_test_types::Backend; fn given_a_round_trip_split_across_jobs_when_the_group_runs_then_members_follow_the_loader_edge() { let test_cases = [ RoundTripGroupTestCase { - description: "SQLite: the task waits for SQL, the loader for the task, the model for the loader", + description: "SQLite: the asset waits for SQL, the loader for the asset, the model for the loader", backend: Backend::Sqlite, expected_waits: json!({ "orders_job": [], @@ -36,7 +36,7 @@ fn given_a_round_trip_split_across_jobs_when_the_group_runs_then_members_follow_ expected_order: &["orders_job", "score_job", "ingest_job", "report_job"], }, RoundTripGroupTestCase { - description: "PostgreSQL: the task waits for SQL, the loader for the task, the model for the loader", + description: "PostgreSQL: the asset waits for SQL, the loader for the asset, the model for the loader", backend: Backend::Postgres, expected_waits: json!({ "orders_job": [], diff --git a/crates/sqlbuild-rules/src/python_rules/tests/test_node_units.rs b/crates/sqlbuild-rules/src/python_rules/tests/test_node_units.rs index f2fe527a5d..9e2ffee3fc 100644 --- a/crates/sqlbuild-rules/src/python_rules/tests/test_node_units.rs +++ b/crates/sqlbuild-rules/src/python_rules/tests/test_node_units.rs @@ -17,7 +17,7 @@ const UNIT_RULES: [&str; 10] = [ "SQBRPYTHON012", ]; -const DAILY_TASK: &str = r#"from .main import main +const DAILY_ASSET: &str = r#"from .main import main @asset(description="Daily.") @@ -25,7 +25,7 @@ def daily(ctx: AssetContext) -> None: return main(ctx=ctx) "#; -const TASK_MAIN: &str = r#"from sqlbuild import PythonNodeResult +const ASSET_MAIN: &str = r#"from sqlbuild import PythonNodeResult def main(ctx: AssetContext) -> PythonNodeResult: @@ -155,7 +155,7 @@ def region_exports(): PythonRunTestCase { description: "unit layout: a missing main.py, a stray module and a folder that is not _helpers/", files: &[ - ("python/orders/daily/node.py", DAILY_TASK), + ("python/orders/daily/node.py", DAILY_ASSET), ("python/orders/weekly/node.py", ""), ("python/orders/weekly/main.py", ""), ("python/orders/weekly/report.py", ""), @@ -234,7 +234,7 @@ def unpacked(ctx: AssetContext) -> None: return main(**ctx) "#, ), - ("python/orders/daily/main.py", TASK_MAIN), + ("python/orders/daily/main.py", ASSET_MAIN), ( "python/orders/weekly/node.py", r#"from python.orders.daily.main import main @@ -245,7 +245,7 @@ def weekly(ctx: AssetContext) -> None: return main(ctx=ctx) "#, ), - ("python/orders/weekly/main.py", TASK_MAIN), + ("python/orders/weekly/main.py", ASSET_MAIN), ], config: json!({ "select": ["SQBRPYTHON005"] }), expected_faults: vec![ @@ -258,7 +258,7 @@ def weekly(ctx: AssetContext) -> None: PythonRunTestCase { description: "main.py: other statements, a result that does not fit the node kind, and no main()", files: &[ - ("python/orders/daily/node.py", DAILY_TASK), + ("python/orders/daily/node.py", DAILY_ASSET), ( "python/orders/daily/main.py", r#"from sqlbuild import PythonNodeResult @@ -315,7 +315,7 @@ def no_ctx(context: AssetContext) -> None: @asset(description="B.") -def wrong_type(ctx: AssetContext, *, store: Store) -> None: +def wrong_type(ctx: LoaderContext, *, store: Store) -> None: return main(ctx=ctx, store=store) @@ -361,7 +361,7 @@ def quality(ctx: AssetContext) -> bool: "from python.orders.weekly.main import main\n", ), ("python/orders/_helpers/dates.py", ""), - ("python/orders/daily/node.py", DAILY_TASK), + ("python/orders/daily/node.py", DAILY_ASSET), ("python/orders/daily/_helpers/rows.py", ""), ( "python/orders/daily/main.py", @@ -395,7 +395,7 @@ def weekly(ctx: AssetContext) -> None: return main(ctx=ctx) "#, ), - ("python/orders/weekly/main.py", TASK_MAIN), + ("python/orders/weekly/main.py", ASSET_MAIN), ("python/orders/weekly/constants.py", ""), ("python/orders/weekly/_helpers/x.py", ""), ("python/billing/invoices/node.py", ""), @@ -414,7 +414,7 @@ def weekly(ctx: AssetContext) -> None: PythonRunTestCase { description: "checks.py: statements, a check of another unit's node, a check that computes, and a plain function report", files: &[ - ("python/orders/daily/node.py", DAILY_TASK), + ("python/orders/daily/node.py", DAILY_ASSET), ( "python/orders/weekly/node.py", "@asset\ndef weekly(ctx):\n pass\n", @@ -591,7 +591,7 @@ def tables(): ), ( "python/utils/flat.py", - "@asset\ndef util_task(ctx):\n return 1\n", + "@asset\ndef util_asset(ctx):\n return 1\n", ), ], config: json!({ "select": ["SQBRPYTHON012"] }), @@ -625,8 +625,8 @@ fn given_valid_shapes_and_hidden_mistakes_when_evaluating_unit_rules_then_report files: &[ ("python/utils/graph/node.py", "class Node:\n pass\n"), ("python/orders/models/node.py", "class Node:\n pass\n"), - ("python/orders/daily/node.py", DAILY_TASK), - ("python/orders/daily/main.py", TASK_MAIN), + ("python/orders/daily/node.py", DAILY_ASSET), + ("python/orders/daily/main.py", ASSET_MAIN), ( "python/orders/daily/_helpers/node.py", "class Node:\n pass\n", @@ -669,7 +669,7 @@ def region_exports(): return main(ctx=ctx) "#, ), - ("python/orders/regions/main.py", TASK_MAIN), + ("python/orders/regions/main.py", ASSET_MAIN), ("python/orders/ranked/constants.py", "SPECS: dict = {}\n"), ( "python/orders/ranked/node.py", @@ -686,7 +686,7 @@ def ranked_exports(): return main(ctx=ctx, region=region) "#, ), - ("python/orders/ranked/main.py", TASK_MAIN), + ("python/orders/ranked/main.py", ASSET_MAIN), ], config: json!({ "select": ["SQBRPYTHON010"] }), expected_faults: vec![], @@ -711,9 +711,9 @@ def daily(ctx: AssetContext, *, store: "Store") -> None: return main(ctx=ctx, store=store) "#, ), - ("python/orders/daily/main.py", TASK_MAIN), - ("python/orders/weekly/node.py", DAILY_TASK), - ("python/orders/weekly/main.py", TASK_MAIN), + ("python/orders/daily/main.py", ASSET_MAIN), + ("python/orders/weekly/node.py", DAILY_ASSET), + ("python/orders/weekly/main.py", ASSET_MAIN), ], config: json!({ "select": ["SQBRPYTHON004", "SQBRPYTHON007", "SQBRPYTHON008"] }), expected_faults: vec![("SQBRPYTHON008", "python/orders/daily/node.py", 7, 43)], @@ -721,8 +721,8 @@ def daily(ctx: AssetContext, *, store: "Store") -> None: PythonRunTestCase { description: "a node named through import a.b in depends_on passes; an unused import of a node does not", files: &[ - ("python/orders/daily/node.py", DAILY_TASK), - ("python/orders/daily/main.py", TASK_MAIN), + ("python/orders/daily/node.py", DAILY_ASSET), + ("python/orders/daily/main.py", ASSET_MAIN), ( "python/orders/weekly/node.py", r#"import python.orders.daily.node @@ -735,7 +735,7 @@ def weekly(ctx: AssetContext) -> None: return main(ctx=ctx) "#, ), - ("python/orders/weekly/main.py", TASK_MAIN), + ("python/orders/weekly/main.py", ASSET_MAIN), ( "python/orders/weekly/_helpers/rows.py", "from python.orders.daily.node import daily\n", @@ -752,7 +752,7 @@ def weekly(ctx: AssetContext) -> None: PythonRunTestCase { description: "a quoted union result and a loader's tuple of rows fit their node", files: &[ - ("python/orders/daily/node.py", DAILY_TASK), + ("python/orders/daily/node.py", DAILY_ASSET), ( "python/orders/daily/main.py", "def main(ctx: AssetContext) -> \"PythonNodeResult | PythonNodeSkipResult\":\n return ctx.result()\n", @@ -776,7 +776,7 @@ def weekly(ctx: AssetContext) -> None: "python/orders/daily/node.py", "from . import main\n\n\n@asset(description=\"D.\")\ndef daily(ctx: AssetContext) -> None:\n return main(ctx=ctx)\n", ), - ("python/orders/daily/main.py", TASK_MAIN), + ("python/orders/daily/main.py", ASSET_MAIN), ], config: json!({ "select": ["SQBRPYTHON005"] }), expected_faults: vec![("SQBRPYTHON005", "python/orders/daily/node.py", 5, 5)], From d3981f1b05dabe277fb3100ea897bdcb3cad72fa Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 21 Nov 2026 04:11:45 +0000 Subject: [PATCH 12/12] test: move E2E expectations from the task kind to assets The end-to-end shards still asserted task labels, kind ordering and decorators that no longer exist. They now assert the same behavior against @asset nodes. --- .../commands/main/bigquery/test_bigquery.py | 2 +- .../cli/commands/main/build/helpers.py | 4 +-- .../test_build_no_tests_no_audits.py | 8 ++--- .../cli/commands/main/build/test_build.py | 22 ++++++------- .../main/build/test_explicit_references.py | 2 +- .../main/build/test_python_node_identity.py | 2 +- .../main/build/test_python_node_selection.py | 2 +- .../main/build/test_warehouse_round_trips.py | 32 +++++++++---------- .../test_python_sql_graph_boundaries.py | 4 +-- .../main/databricks/test_databricks.py | 2 +- .../cli/commands/main/init/test_init.py | 2 +- .../cli/commands/main/plan/helpers.py | 2 +- .../commands/main/postgres/test_postgres.py | 2 +- .../main/providers/test_provider_secrets.py | 4 +-- .../commands/main/providers/test_providers.py | 6 ++-- .../commands/main/snowflake/test_snowflake.py | 2 +- .../commands/main/sqlserver/test_sqlserver.py | 2 +- .../cli/commands/main/test/test_since.py | 2 +- .../rivers/test_sqlbuild_assets.py | 6 ++-- .../python_nodes/classes/_test_types.py | 2 +- .../classes/test_runtime_relation_guard.py | 2 +- 21 files changed, 56 insertions(+), 56 deletions(-) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/bigquery/test_bigquery.py b/tests/e2e/src/sqlbuild/cli/commands/main/bigquery/test_bigquery.py index d9e6ff4b2a..b78d64c4f1 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/bigquery/test_bigquery.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/bigquery/test_bigquery.py @@ -148,8 +148,8 @@ def test_given_bigquery_dbt_profile_when_running_dbt_init_then_builds_profile_li BigQueryNodeResultE2ETestCase( description="direct node results persist and read on bigquery", expected_rows=( - ("check", "check_produce_result", "success"), ("asset", "produce_result", "success"), + ("check", "check_produce_result", "success"), ), ) ], diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py index 1955edec5b..27d3545fdf 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py @@ -867,10 +867,10 @@ def _counting_python_node(*, decorator: str, name: str, ref: str, returned: str) def python_node_selection_project_files(*, orders_sql: str) -> dict[str, str]: - """Return a DuckDB project whose tasks and assets count managed, unmanaged, and model rows.""" + """Return a DuckDB project whose assets count managed, unmanaged, and model rows.""" task_nodes: str = "".join( - _counting_python_node(decorator="task", name=name, ref=ref, returned="None") + _counting_python_node(decorator="asset", name=name, ref=ref, returned="None") for name, ref in ( ("count_customers", 'source("raw_customers")'), ("count_raw_orders", 'source("raw_orders")'), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/test_build_no_tests_no_audits.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/test_build_no_tests_no_audits.py index 9d641b1574..2652952160 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/test_build_no_tests_no_audits.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/test_build_no_tests_no_audits.py @@ -443,7 +443,7 @@ def test_given_task_selector_when_running_run_then_task_executes( ) assert result.returncode == test_case.expected_exit_code, result.stdout + result.stderr - assert "python task prepare_orders" in result.stdout + assert "python asset prepare_orders" in result.stdout assert "OK" in result.stdout assert (project_dir / "task_marker.txt").read_text(encoding="utf-8") == "ran" @@ -626,7 +626,7 @@ def test_given_failing_task_selector_when_running_run_then_command_fails( assert result.returncode == test_case.expected_exit_code combined_output: str = result.stdout + result.stderr - assert "python task fail_orders" in combined_output + assert "python asset fail_orders" in combined_output assert "FAIL" in combined_output assert "\u2717 Completed with errors" in combined_output assert "Python node failures:" in combined_output @@ -713,7 +713,7 @@ def test_given_task_loader_source_model_chain_when_running_model_then_task_runs_ ) assert result.returncode == test_case.expected_exit_code, result.stdout + result.stderr - assert "python task prepare_orders" in result.stdout + assert "python asset prepare_orders" in result.stdout execution_output: str = result.stdout[result.stdout.index("Execution sqb build") :] assert execution_output.index("prepare_orders") < execution_output.index("fact_orders") db_path: Path = project_dir / "python_loader_run_project.duckdb" @@ -1388,7 +1388,7 @@ def test_given_sql_ready_task_fails_when_running_run_then_footer_json_and_exit_f assert result.returncode == test_case.expected_exit_code combined_output: str = result.stdout + result.stderr - assert "python task fail_after_stg_orders" in combined_output + assert "python asset fail_after_stg_orders" in combined_output assert "FAIL" in combined_output assert "Python node failures:" in combined_output payload: dict[str, object] = json.loads(json_output_path.read_text(encoding="utf-8")) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_build.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_build.py index c7762fa324..b1dc964223 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_build.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_build.py @@ -704,7 +704,7 @@ def test_given_shared_downstream_failure_when_building_then_blocks_all_source_ap expected_exit_code=1, expected_output_fragments=( "Python ingress (1)", - "python task prepare_orders", + "python asset prepare_orders", "FAIL", "Python node failures:", ), @@ -751,7 +751,7 @@ def test_given_shared_downstream_failure_when_building_then_blocks_all_source_ap expected_exit_code=0, expected_output_fragments=( "Python ingress (1)", - "python task prepare_orders", + "python asset prepare_orders", "SKIP", "no input", ), @@ -838,7 +838,7 @@ def test_given_shared_downstream_failure_when_building_then_blocks_all_source_ap expected_exit_code=1, expected_output_fragments=( "fact_orders", - "python task fail_after_fact", + "python asset fail_after_fact", "FAIL", "Python node failures:", ), @@ -878,7 +878,7 @@ def test_given_shared_downstream_failure_when_building_then_blocks_all_source_ap expected_exit_code=0, expected_output_fragments=( "fact_orders", - "python task skip_after_fact", + "python asset skip_after_fact", "SKIP", "no profile needed", ), @@ -918,7 +918,7 @@ def test_given_shared_downstream_failure_when_building_then_blocks_all_source_ap expected_exit_code=0, expected_output_fragments=( "fact_orders", - "python task soft_skip_after_fact", + "python asset soft_skip_after_fact", "SKIP", "optional profile skipped", ), @@ -976,7 +976,7 @@ def test_given_shared_downstream_failure_when_building_then_blocks_all_source_ap }, expected_exit_code=0, expected_output_fragments=( - "python task prepare_orders", + "python asset prepare_orders", "OK", ), expected_present_tables=("raw_orders", "fact_orders"), @@ -1024,7 +1024,7 @@ def test_given_shared_downstream_failure_when_building_then_blocks_all_source_ap }, expected_exit_code=0, expected_output_fragments=( - "python task profile_raw_orders", + "python asset profile_raw_orders", "OK", ), expected_present_tables=("raw_orders",), @@ -1138,7 +1138,7 @@ def test_given_python_lifecycle_edge_case_when_building_then_direct_build_harden ), }, expected_exit_code=0, - expected_output_fragments=("source raw_events", "unselected upstream task"), + expected_output_fragments=("source raw_events", "unselected upstream asset"), expected_present_tables=("raw_events",), expected_absent_paths=("prepared.txt",), ), @@ -1331,11 +1331,11 @@ def test_given_python_asset_with_json_output_when_building_then_json_includes_py expected_execution_fragments=( "Python ingress (2)", "Python read-side (3)", - "python task prepare_orders", + "python asset prepare_orders", "python asset publish_prepared_orders", - "python task profile_fact_orders", + "python asset profile_fact_orders", "python asset export_fact_orders", - "python task notify_fact_orders", + "python asset notify_fact_orders", ), expected_table_names=("window_orders", "raw_orders", "fact_orders"), expected_notify_text="7", diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_explicit_references.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_explicit_references.py index 901bf36e95..1d8b7ef2a2 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_explicit_references.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_explicit_references.py @@ -459,7 +459,7 @@ def test_given_implicit_reference_when_running_command_then_it_fails_with_guidan enforce_explicit=True, expected_exit_code=0, expected_warning_fragments=( - "[P008] task 'export_orders' named model:stg_customers as 'stg_customers'", + "[P008] asset 'export_orders' named model:stg_customers as 'stg_customers'", ), expected_final_line_prefix="\u2713 Completed with warnings", ), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_identity.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_identity.py index e521a5f57f..6644a71605 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_identity.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_identity.py @@ -237,7 +237,7 @@ def orders(ctx): db_path=project_dir / "warehouse.duckdb", sql=( "SELECT COUNT(DISTINCT version_hash) FROM main._sqlbuild_fingerprints " - "WHERE node_type = 'task' AND node_name = 'orders'" + "WHERE node_type = 'asset' AND node_name = 'orders'" ), ) == [(test_case.expected_version_count,)] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_selection.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_selection.py index 8b2ba77a65..d9269c06c2 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_selection.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_selection.py @@ -209,7 +209,7 @@ def test_given_python_node_selection_when_building_then_it_gates_only_on_schedul select="asset:report_order_count", upstream_select="asset:count_orders", expected_error=( - "Upstream task 'count_orders' has no successful result to read. Build it first " + "Upstream asset 'count_orders' has no successful result to read. Build it first " "with 'sqb build --select asset:count_orders', or build both with " "'sqb build --select +asset:report_order_count'." ), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_warehouse_round_trips.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_warehouse_round_trips.py index 3531ad691a..aa2b9c67e4 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_warehouse_round_trips.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_warehouse_round_trips.py @@ -42,7 +42,7 @@ expected_exit_code=0, expected_output_order=( "table orders OK", - "task score_orders OK", + "asset score_orders OK", "source partner_scores OK", "table scored_orders OK", ), @@ -79,11 +79,11 @@ def test_given_a_warehouse_round_trip_when_building_then_the_loader_runs_between expected_exit_code=1, expected_output_order=( "table orders OK", - "task score_orders FAIL", + "asset score_orders FAIL", "source partner_scores SKIP", "table scored_orders SKIP", ), - expected_output_fragments=("hard skip: Upstream task 'score_orders' failed",), + expected_output_fragments=("hard skip: Upstream asset 'score_orders' failed",), expected_scored_rows=None, expected_scored_table_exists=False, ), @@ -94,7 +94,7 @@ def test_given_a_warehouse_round_trip_when_building_then_the_loader_runs_between expected_exit_code=1, expected_output_order=(), expected_output_fragments=( - "loader 'partner_scores' depends on task 'score_orders', which reads SQL", + "loader 'partner_scores' depends on asset 'score_orders', which reads SQL", "set `allow_warehouse_round_trips = true` under [settings]", ), expected_scored_rows=None, @@ -125,7 +125,7 @@ def test_given_a_round_trip_that_cannot_complete_when_building_then_its_readers_ "SELECT 1 AS order_id UNION ALL SELECT 2 UNION ALL SELECT 3\n" ), expected_output_fragments=( - "task score_orders OK", + "asset score_orders OK", "source partner_scores OK", "table scored_orders OK", ), @@ -138,7 +138,7 @@ def test_given_a_round_trip_that_cannot_complete_when_building_then_its_readers_ "SELECT 1 AS order_id UNION ALL SELECT 2 AS order_id\n" ), expected_output_fragments=( - "soft skip: Upstream task 'score_orders' did not run; the SQL it reads was not " + "soft skip: Upstream asset 'score_orders' did not run; the SQL it reads was not " "rebuilt", "Completed successfully", ), @@ -151,8 +151,8 @@ def test_given_a_round_trip_that_cannot_complete_when_building_then_its_readers_ "SELECT 1 AS order_id UNION ALL SELECT 2 UNION ALL SELECT 3\n" ), expected_output_fragments=( - "soft skip: Upstream task 'score_orders' did not run; --no-python skips " - "read-side tasks and assets", + "soft skip: Upstream asset 'score_orders' did not run; --no-python skips " + "read-side assets", "Completed successfully", ), expected_scored_rows=((1, 10), (2, 20)), @@ -166,7 +166,7 @@ def test_given_a_round_trip_that_cannot_complete_when_building_then_its_readers_ ), extra_files=scoring_task_files(' return ctx.skip(reason="partner offline")\n'), expected_output_fragments=( - "soft skip: Upstream task 'score_orders' was skipped", + "soft skip: Upstream asset 'score_orders' was skipped", "table scored_orders OK", "Completed successfully", ), @@ -182,7 +182,7 @@ def test_given_a_round_trip_that_cannot_complete_when_building_then_its_readers_ ' return ctx.skip(reason="partner offline", mode="hard")\n' ), expected_output_fragments=( - "hard skip: Upstream task 'score_orders' was skipped", + "hard skip: Upstream asset 'score_orders' was skipped", "table scored_orders SKIP", ), expected_scored_rows=((1, 10), (2, 20)), @@ -195,7 +195,7 @@ def test_given_a_round_trip_that_cannot_complete_when_building_then_its_readers_ **ranking_task_files(' return ctx.result(payload={"ranked": True})\n'), }, expected_output_fragments=( - "task rank_orders OK", + "asset rank_orders OK", "source partner_scores OK", "table scored_orders OK", ), @@ -210,7 +210,7 @@ def test_given_a_round_trip_that_cannot_complete_when_building_then_its_readers_ }, expected_output_fragments=( "partner_scores SKIP", - "soft skip: Upstream task '", + "soft skip: Upstream asset '", "table scored_orders OK", ), expected_scored_rows=((1, 10), (2, 20)), @@ -225,7 +225,7 @@ def test_given_a_round_trip_that_cannot_complete_when_building_then_its_readers_ **ranking_task_files(' return ctx.result(payload={"ranked": True})\n'), }, expected_output_fragments=( - "hard skip: Upstream task 'score_orders' was skipped", + "hard skip: Upstream asset 'score_orders' was skipped", "table scored_orders SKIP", ), expected_scored_rows=((1, 10), (2, 20)), @@ -241,7 +241,7 @@ def test_given_a_round_trip_that_cannot_complete_when_building_then_its_readers_ }, ), expected_output_fragments=( - "soft skip: Upstream task 'package_scores'", + "soft skip: Upstream asset 'package_scores'", "table scored_orders OK", "Completed successfully", ), @@ -260,8 +260,8 @@ def test_given_a_round_trip_that_cannot_complete_when_building_then_its_readers_ }, ), expected_output_fragments=( - "soft skip: Upstream task 'score_orders' did not run; --no-python skips " - "read-side tasks and assets", + "soft skip: Upstream asset 'score_orders' did not run; --no-python skips " + "read-side assets", "Completed successfully", ), expected_scored_rows=((1, 10), (2, 20)), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_python_sql_graph_boundaries.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_python_sql_graph_boundaries.py index b27cc1d738..bb94927713 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_python_sql_graph_boundaries.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_python_sql_graph_boundaries.py @@ -40,7 +40,7 @@ expected_output_fragments=( "S000", "loader 'partner_scores' is in a dependency cycle: loader 'partner_scores' -> " - "task 'score_orders' -> model 'scored_orders' -> source 'partner_scores' -> " + "asset 'score_orders' -> model 'scored_orders' -> source 'partner_scores' -> " "loader 'partner_scores'", ), ), @@ -63,7 +63,7 @@ edited_files={}, expected_exit_code=1, expected_output_fragments=( - "loader 'partner_scores' depends on task 'score_orders', which reads SQL", + "loader 'partner_scores' depends on asset 'score_orders', which reads SQL", "set `allow_warehouse_round_trips = true` under [settings]", ), ), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/databricks/test_databricks.py b/tests/e2e/src/sqlbuild/cli/commands/main/databricks/test_databricks.py index 52564e6334..8c0a6f278f 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/databricks/test_databricks.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/databricks/test_databricks.py @@ -142,8 +142,8 @@ def test_given_databricks_dbt_profile_when_running_dbt_init_then_builds_profile_ DatabricksNodeResultE2ETestCase( description="direct node results persist and read on databricks", expected_rows=( - ("check", "check_produce_result", "success"), ("asset", "produce_result", "success"), + ("check", "check_produce_result", "success"), ), ) ], diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/init/test_init.py b/tests/e2e/src/sqlbuild/cli/commands/main/init/test_init.py index 3b932b8ce0..195cf62368 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/init/test_init.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/init/test_init.py @@ -35,7 +35,7 @@ ), expected_output_fragments=( "SQLBuild project created", - "Add Python loaders, tasks, assets, checks, and factories to python/", + "Add Python loaders, assets, checks, and factories to python/", "Add SQL hooks to hooks/sql/, Python hooks to hooks/python/", "reusable audits to audits/generic/", "standalone audits to audits/singular/", diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/plan/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/plan/helpers.py index b203e0b3da..4f0d334c39 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/plan/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/plan/helpers.py @@ -379,7 +379,7 @@ def published_keys(control: Path) -> list[str]: for row in database.execute( "SELECT n.node_key FROM sqb_snapshot_nodes n JOIN sqb_current_snapshots c " "ON c.snapshot_id=n.snapshot_id WHERE n.node_kind IN " - "('model','source','seed','udf','table_fn','task','asset','loader')" + "('model','source','seed','udf','table_fn','asset','loader')" ) ] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/postgres/test_postgres.py b/tests/e2e/src/sqlbuild/cli/commands/main/postgres/test_postgres.py index 36ad5735dc..c8acd594a2 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/postgres/test_postgres.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/postgres/test_postgres.py @@ -236,8 +236,8 @@ def test_given_postgres_dbt_profile_when_running_dbt_init_then_plain_build_uses_ PostgresNodeResultE2ETestCase( description="direct node results persist and read on postgres", expected_rows=( - ("check", "check_produce_result", "success"), ("asset", "produce_result", "success"), + ("check", "check_produce_result", "success"), ), ) ], diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/providers/test_provider_secrets.py b/tests/e2e/src/sqlbuild/cli/commands/main/providers/test_provider_secrets.py index 25494f8648..1cf8c33935 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/providers/test_provider_secrets.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/providers/test_provider_secrets.py @@ -63,10 +63,10 @@ class ApiProvider(Provider): ).strip() + "\n", "python/tasks/uses_api.py": MARKING_NODE.format( - decorator="task", arguments="", name="uses_api", parameter=USES_API, body="" + decorator="asset", arguments="", name="uses_api", parameter=USES_API, body="" ), "python/tasks/no_api.py": MARKING_NODE.format( - decorator="task", arguments="", name="no_api", parameter="", body="" + decorator="asset", arguments="", name="no_api", parameter="", body="" ), "python/checks/api_check.py": MARKING_NODE.format( decorator="check", diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/providers/test_providers.py b/tests/e2e/src/sqlbuild/cli/commands/main/providers/test_providers.py index b6cfc922c5..ab7f0b5a33 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/providers/test_providers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/providers/test_providers.py @@ -80,17 +80,17 @@ def mark(self, value): expected_text_fragments=( "Providers", "\u2514\u2500\u2500 marker_provider", + " ├── asset publish_orders (MarkerProvider)", " ├── custom materialization copy_table (MarkerProvider)", " ├── hook mark_pre (MarkerProvider)", - " ├── loader raw_orders (MarkerProvider)", - " └── task publish_orders (MarkerProvider)", + " └── loader raw_orders (MarkerProvider)", ), expected_provider_name="marker_provider", expected_used_by=( + ("asset", "publish_orders", "marker_provider"), ("custom materialization", "copy_table", "marker_provider"), ("hook", "mark_pre", "marker_provider"), ("loader", "raw_orders", "marker_provider"), - ("asset", "publish_orders", "marker_provider"), ), ) ], diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/snowflake/test_snowflake.py b/tests/e2e/src/sqlbuild/cli/commands/main/snowflake/test_snowflake.py index 83ab0f1299..b7f212dbf9 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/snowflake/test_snowflake.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/snowflake/test_snowflake.py @@ -157,8 +157,8 @@ def test_given_snowflake_dbt_profile_when_running_dbt_init_then_builds_profile_l SnowflakeNodeResultE2ETestCase( description="direct node results persist and read on snowflake", expected_rows=( - ("check", "check_produce_result", "success"), ("asset", "produce_result", "success"), + ("check", "check_produce_result", "success"), ), ) ], diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/sqlserver/test_sqlserver.py b/tests/e2e/src/sqlbuild/cli/commands/main/sqlserver/test_sqlserver.py index 564babcd22..1b50f67e26 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/sqlserver/test_sqlserver.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/sqlserver/test_sqlserver.py @@ -410,8 +410,8 @@ def test_given_sqlserver_dbt_profile_when_running_dbt_init_then_builds_profile_l SqlServerNodeResultE2ETestCase( description="direct node results persist and read on SQL Server", expected_rows=( - ("check", "check_produce_result", "success"), ("asset", "produce_result", "success"), + ("check", "check_produce_result", "success"), ), ) ], diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_since.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_since.py index 64ca9488ce..6166906e38 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_since.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_since.py @@ -553,7 +553,7 @@ def test_given_changes_since_ref_when_testing_then_runs_only_affected_tests( }, command=_TEST_COMMAND, expected_exit_code=0, - expected_fragments=("Changed since main: 1 task changed\n",), + expected_fragments=("Changed since main: 1 asset changed\n",), ), ), ids=lambda case: case.description, diff --git a/tests/e2e/src/sqlbuild/integrations/rivers/test_sqlbuild_assets.py b/tests/e2e/src/sqlbuild/integrations/rivers/test_sqlbuild_assets.py index 53435d376d..2cb12468f5 100644 --- a/tests/e2e/src/sqlbuild/integrations/rivers/test_sqlbuild_assets.py +++ b/tests/e2e/src/sqlbuild/integrations/rivers/test_sqlbuild_assets.py @@ -44,13 +44,13 @@ expected_asset_names=frozenset( { "main__orders", - "task__prepare_orders", + "asset__prepare_orders", "warehouse_export", "asset__orders_export", } ), expected_task_deps=("main__orders",), - expected_asset_deps=("warehouse_export", "task__prepare_orders"), + expected_asset_deps=("asset__prepare_orders", "warehouse_export"), expected_task_group="python", expected_asset_group="exports", ) @@ -84,7 +84,7 @@ def sqlbuild_python_nodes(context: Any) -> Iterator[Any]: asset.name: (asset,) for asset in output_defs } assert len(output_defs_by_name) == len(output_defs) - task_def: Any = next(iter(output_defs_by_name.get("task__prepare_orders", ()))) + task_def: Any = next(iter(output_defs_by_name.get("asset__prepare_orders", ()))) asset_def: Any = next(iter(output_defs_by_name.get("asset__orders_export", ()))) assert sqlbuild_project.dag_path.exists() diff --git a/tests/unit/src/sqlbuild/executor/python_nodes/classes/_test_types.py b/tests/unit/src/sqlbuild/executor/python_nodes/classes/_test_types.py index a6f0961b84..3322b5f7f9 100644 --- a/tests/unit/src/sqlbuild/executor/python_nodes/classes/_test_types.py +++ b/tests/unit/src/sqlbuild/executor/python_nodes/classes/_test_types.py @@ -16,6 +16,6 @@ class RuntimeRelationGuardTestCase: expected_warning_count: int resolved_refs: tuple[SqlResourceRef, ...] = field(default_factory=tuple) own_refs: frozenset[SqlResourceRef] = frozenset() - owner_label: str = "task 'export_orders'" + owner_label: str = "asset 'export_orders'" owner_kind: HardCodedRelationOwnerKind = HardCodedRelationOwnerKind.NODE upstream_loader_by_source: dict[str, str] = field(default_factory=dict) diff --git a/tests/unit/src/sqlbuild/executor/python_nodes/classes/test_runtime_relation_guard.py b/tests/unit/src/sqlbuild/executor/python_nodes/classes/test_runtime_relation_guard.py index d178f0f5bd..099ed24a57 100644 --- a/tests/unit/src/sqlbuild/executor/python_nodes/classes/test_runtime_relation_guard.py +++ b/tests/unit/src/sqlbuild/executor/python_nodes/classes/test_runtime_relation_guard.py @@ -26,7 +26,7 @@ description="unqualified model name resolved against the current schema warns once", statements=("SELECT * FROM customers", "SELECT count(*) FROM main.customers"), expected_warning_fragments=( - "[P008] task 'export_orders' named model:customers as 'customers' in SQL; " + "[P008] asset 'export_orders' named model:customers as 'customers' in SQL; " 'declare it with depends_on=model("customers") and use ' 'ctx.relation(model("customers")) instead of the relation name', ),