diff --git a/acceptance/bin/add_repl.py b/acceptance/bin/add_repl.py index 4b5df5c530e..5f97d9cee80 100755 --- a/acceptance/bin/add_repl.py +++ b/acceptance/bin/add_repl.py @@ -37,6 +37,16 @@ def add_repl(value, repl): break +def id_repl_name(key): + """Replacement name for the id of the resource with the given key: "foo" -> "FOO_ID". + + The name is upper-cased, so a $UNIQUE_NAME inside the key would escape the harness's + (case-sensitive) [UNIQUE_NAME] replacement and make the name differ on every run. + Substitute it first: "app_${UNIQUE_NAME}_job" -> "APP_UNIQUE_NAME_JOB_ID". + """ + return key.replace(os.environ["UNIQUE_NAME"], "UNIQUE_NAME").upper() + "_ID" + + def main(): parser = argparse.ArgumentParser() parser.add_argument("value") diff --git a/acceptance/bin/read_id.py b/acceptance/bin/read_id.py index 9663a05a8ae..1b596d3d92f 100755 --- a/acceptance/bin/read_id.py +++ b/acceptance/bin/read_id.py @@ -15,7 +15,7 @@ from pathlib import Path sys.path.insert(0, str(Path(__file__).parent)) -from add_repl import add_repl +from add_repl import add_repl, id_repl_name from print_state import get_resources, get_state_file @@ -82,7 +82,7 @@ def main(): if id: print(id) - add_repl(str(id), args.name.upper() + "_ID") + add_repl(str(id), id_repl_name(args.name)) if __name__ == "__main__": diff --git a/acceptance/bin/replace_ids.py b/acceptance/bin/replace_ids.py index 5665c705de7..4bd13bf94b2 100755 --- a/acceptance/bin/replace_ids.py +++ b/acceptance/bin/replace_ids.py @@ -10,7 +10,7 @@ from pathlib import Path sys.path.insert(0, str(Path(__file__).parent)) -from add_repl import add_repl +from add_repl import add_repl, id_repl_name from print_state import get_resources, get_state_file @@ -61,7 +61,7 @@ def main(): it = iter_ids_direct(filename) for name, id in it: - add_repl(id, name.upper() + "_ID") + add_repl(id, id_repl_name(name)) if __name__ == "__main__": diff --git a/acceptance/bundle/templates/default-python/integration_classic/output.txt b/acceptance/bundle/templates/default-python/integration_classic/output.txt index 982e0564939..f3a56f4b4ae 100644 --- a/acceptance/bundle/templates/default-python/integration_classic/output.txt +++ b/acceptance/bundle/templates/default-python/integration_classic/output.txt @@ -48,11 +48,11 @@ Resources: Jobs: sample_job: Name: [dev [USERNAME]] sample_job - URL: [DATABRICKS_URL]/jobs/[JOB_ID] + URL: [DATABRICKS_URL]/jobs/[SAMPLE_JOB_ID] Pipelines: project_name_[UNIQUE_NAME]_etl: Name: [dev [USERNAME]] project_name_[UNIQUE_NAME]_etl - URL: [DATABRICKS_URL]/pipelines/[UUID] + URL: [DATABRICKS_URL]/pipelines/[PROJECT_NAME_UNIQUE_NAME_ETL_ID] >>> [CLI] bundle summary -t dev -o json @@ -62,7 +62,7 @@ Resources: @@ -51,4 +51,5 @@ "edit_mode": "UI_LOCKED", "format": "MULTI_TASK", -+ "id": "[JOB_ID]", ++ "id": "[SAMPLE_JOB_ID]", "job_clusters": [ { @@ -138,5 +139,6 @@ @@ -70,13 +70,13 @@ Resources: } - } + }, -+ "url": "[DATABRICKS_URL]/jobs/[JOB_ID]" ++ "url": "[DATABRICKS_URL]/jobs/[SAMPLE_JOB_ID]" } }, @@ -155,4 +157,5 @@ ] }, -+ "id": "[UUID]", ++ "id": "[PROJECT_NAME_UNIQUE_NAME_ETL_ID]", "libraries": [ { @@ -167,5 +170,6 @@ @@ -84,7 +84,7 @@ Resources: "dev": "[USERNAME]" - } + }, -+ "url": "[DATABRICKS_URL]/pipelines/[UUID]" ++ "url": "[DATABRICKS_URL]/pipelines/[PROJECT_NAME_UNIQUE_NAME_ETL_ID]" } } @@ -308,11 +308,11 @@ Resources: Jobs: sample_job: Name: sample_job - URL: [DATABRICKS_URL]/jobs/[JOB_ID] + URL: [DATABRICKS_URL]/jobs/[SAMPLE_JOB_ID_2] Pipelines: project_name_[UNIQUE_NAME]_etl: Name: project_name_[UNIQUE_NAME]_etl - URL: [DATABRICKS_URL]/pipelines/[UUID] + URL: [DATABRICKS_URL]/pipelines/[PROJECT_NAME_UNIQUE_NAME_ETL_ID_2] >>> [CLI] bundle summary -t prod -o json @@ -366,13 +366,18 @@ Resources: + ], "resources": { "jobs": { -@@ -47,5 +37,5 @@ +@@ -47,9 +37,9 @@ "deployment": { "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/state/metadata.json" + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/prod/state/metadata.json" }, "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", +- "id": "[SAMPLE_JOB_ID]", ++ "id": "[SAMPLE_JOB_ID_2]", + "job_clusters": [ + { @@ -66,6 +56,6 @@ } ], @@ -424,7 +429,14 @@ Resources: + "pause_status": "UNPAUSED", "periodic": { "interval": 1, -@@ -148,11 +141,10 @@ +@@ -140,5 +133,5 @@ + } + }, +- "url": "[DATABRICKS_URL]/jobs/[SAMPLE_JOB_ID]" ++ "url": "[DATABRICKS_URL]/jobs/[SAMPLE_JOB_ID_2]" + } + }, +@@ -148,28 +141,30 @@ "deployment": { "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/state/metadata.json" @@ -438,7 +450,9 @@ Resources: + "--editable /Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/prod/files" ] }, -@@ -161,14 +153,17 @@ +- "id": "[PROJECT_NAME_UNIQUE_NAME_ETL_ID]", ++ "id": "[PROJECT_NAME_UNIQUE_NAME_ETL_ID_2]", + "libraries": [ { "glob": { - "include": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/files/src/project_name_[UNIQUE_NAME]_etl/transformations/**" @@ -452,6 +466,7 @@ Resources: - "tags": { - "dev": "[USERNAME]" - }, +- "url": "[DATABRICKS_URL]/pipelines/[PROJECT_NAME_UNIQUE_NAME_ETL_ID]" + "name": "project_name_[UNIQUE_NAME]_etl", + "permissions": [ + { @@ -461,8 +476,9 @@ Resources: + ], + "root_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/prod/files/src/project_name_[UNIQUE_NAME]_etl", + "schema": "prod", - "url": "[DATABRICKS_URL]/pipelines/[UUID]" ++ "url": "[DATABRICKS_URL]/pipelines/[PROJECT_NAME_UNIQUE_NAME_ETL_ID_2]" } + } @@ -187,16 +182,16 @@ }, "schema": { diff --git a/acceptance/bundle/templates/default-python/integration_classic/script b/acceptance/bundle/templates/default-python/integration_classic/script index f68e675832c..d47fcaf1786 100644 --- a/acceptance/bundle/templates/default-python/integration_classic/script +++ b/acceptance/bundle/templates/default-python/integration_classic/script @@ -19,6 +19,7 @@ trace $CLI bundle validate -t dev trace $CLI bundle validate -t dev -o json | jq -S "del($JQ_DELETE)" > ../out.validate.dev.json $CLI bundle plan -o json -t dev | nostamp > ../out.plan_dev.json trace $CLI bundle deploy -t dev +replace_ids.py -t dev trace $CLI bundle summary -t dev trace $CLI bundle summary -t dev -o json | jq -S "del($JQ_DELETE)" > ../out.summary.dev.json trace diff.py ../out.validate.dev.json ../out.summary.dev.json @@ -32,6 +33,7 @@ trace diff.py ../out.validate.dev.json ../out.validate.prod.json rm ../out.validate.prod.json $CLI bundle plan -o json -t prod | nostamp > ../out.plan_prod.json trace $CLI bundle deploy -t prod +replace_ids.py -t prod trace $CLI bundle summary -t prod trace $CLI bundle summary -t prod -o json | jq -S "del($JQ_DELETE)" > ../out.summary.prod.json trace diff.py ../out.summary.dev.json ../out.summary.prod.json diff --git a/acceptance/bundle/templates/default-python/integration_classic/test.toml b/acceptance/bundle/templates/default-python/integration_classic/test.toml index bcdbe2b610e..541c9919997 100644 --- a/acceptance/bundle/templates/default-python/integration_classic/test.toml +++ b/acceptance/bundle/templates/default-python/integration_classic/test.toml @@ -20,15 +20,6 @@ EnvRepl.UV_PYTHON = true Old = '0\.0\.1\+\d+' New = '0.0.1+[WHEEL_BUILD]' -# Job ids; the resource keys embed $UNIQUE_NAME, so replace_ids.py names would vary per run. -[[Repls]] -Old = '/jobs/\d+' -New = '/jobs/[JOB_ID]' - -[[Repls]] -Old = '"id": "\d+"' -New = '"id": "[JOB_ID]"' - [[Repls]] Old = 'Standard_D3_v2' New = '[NODE_TYPE_ID]'