diff --git a/README.md b/README.md index 27ee953..f07932f 100644 --- a/README.md +++ b/README.md @@ -79,9 +79,13 @@ structurally impossible. **Only the dbt template's `ci` target is prefixed.** `g3dt config dbt-env` emits, alongside the real names, the CI-isolation variants the template's -`ci` target consumes: `G3DT_DB_SILVER_CI` / `G3DT_DB_GOLD_CI` -(`ci_` + the real database name) and `G3DT_S3_SILVER_DATA_DIR_CI` / -`G3DT_S3_GOLD_DATA_DIR_CI` (`dbt_ci/` under the same buckets). Toolkit +`ci` target consumes: `G3DT_DB_BRONZE_CI` / `G3DT_DB_SILVER_CI` / +`G3DT_DB_GOLD_CI` (`ci_` + the real database name) and +`G3DT_S3_BRONZE_DATA_DIR_CI` / `G3DT_S3_SILVER_DATA_DIR_CI` / +`G3DT_S3_GOLD_DATA_DIR_CI` (`dbt_ci/` under the same buckets). Bronze can be +dbt-managed from gen3-dbt-template's synthetic-data revision onwards; a +pipeline deployment >= v2.1.0 provides the matching `ci_..._bronze_db` Glue +database. Toolkit releases >= 3 read the raw-free medallion SSM keys and therefore require a pipeline deployment >= v2.0.0, which publishes them. Commit- triggered CI builds land there; every other target (default, local) and the diff --git a/pyproject.toml b/pyproject.toml index dda10fe..3f2d92b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "gen3-dataops-toolkit" -version = "3.0.0" +version = "3.1.0" description = "Gen3 DataOps toolkit (g3dt): operate SSM-published Gen3 data pipeline environments" authors = ["JoshuaHarris391 "] readme = "README.md" diff --git a/src/g3dt/cli/config_cmds.py b/src/g3dt/cli/config_cmds.py index 265a9ec..49808c9 100644 --- a/src/g3dt/cli/config_cmds.py +++ b/src/g3dt/cli/config_cmds.py @@ -233,6 +233,7 @@ def dbt_env( gold_db = _req_key(rc, "glue/db/gold") silver_bucket = _req_key(rc, "buckets/silver") gold_bucket = _req_key(rc, "buckets/gold") + bronze_bucket = _req_key(rc, "buckets/bronze") bronze_db = _req_key(rc, "glue/db/bronze") except config.ConfigError as exc: typer.secho(str(exc), fg=typer.colors.RED, err=True) @@ -245,13 +246,16 @@ def dbt_env( "G3DT_DB_BRONZE": bronze_db, "G3DT_DB_SILVER": silver_db, "G3DT_DB_GOLD": gold_db, + "G3DT_S3_BRONZE_DATA_DIR": f"s3://{bronze_bucket}/dbt/", "G3DT_S3_SILVER_DATA_DIR": f"s3://{silver_bucket}/dbt/", "G3DT_S3_GOLD_DATA_DIR": f"s3://{gold_bucket}/dbt/", # CI isolation: the dbt template's `ci` target builds into these # instead — same grammar as the CDK's ci_ databases, same buckets # under a dbt_ci/ prefix. Real names above are never prefixed. + "G3DT_DB_BRONZE_CI": f"ci_{bronze_db}", "G3DT_DB_SILVER_CI": f"ci_{silver_db}", "G3DT_DB_GOLD_CI": f"ci_{gold_db}", + "G3DT_S3_BRONZE_DATA_DIR_CI": f"s3://{bronze_bucket}/dbt_ci/", "G3DT_S3_SILVER_DATA_DIR_CI": f"s3://{silver_bucket}/dbt_ci/", "G3DT_S3_GOLD_DATA_DIR_CI": f"s3://{gold_bucket}/dbt_ci/", } diff --git a/tests/test_release_cmds.py b/tests/test_release_cmds.py index 51f3a16..648112e 100644 --- a/tests/test_release_cmds.py +++ b/tests/test_release_cmds.py @@ -52,6 +52,7 @@ def _seed(project="etl", env="test"): "buckets/metadata": f"{project}-{env}-metadata-{ACCOUNT}-{REGION}", "buckets/silver": f"{project}-{env}-silver-{ACCOUNT}-{REGION}", "buckets/gold": f"{project}-{env}-gold-{ACCOUNT}-{REGION}", + "buckets/bronze": f"{project}-{env}-bronze-{ACCOUNT}-{REGION}", "glue/db/bronze": f"{project}_{env}_bronze_db", "glue/db/silver": f"{project}_{env}_silver_db", "glue/db/gold": f"{project}_{env}_gold_db", @@ -161,7 +162,7 @@ def test_config_dbt_env_emits_every_dbt_setting(): """ Inputs: g3dt config dbt-env --env test Expected Output: shell-evaluable `export` lines carrying the workgroup, - Athena output, region, bronze/silver/gold DBs and the silver/gold + Athena output, region, bronze/silver/gold DBs and the bronze/silver/gold s3_data_dir values — the full env_var() contract of the dbt template. """ _seed() @@ -174,6 +175,7 @@ def test_config_dbt_env_emits_every_dbt_setting(): assert "export G3DT_DB_BRONZE=etl_test_bronze_db" in out assert "export G3DT_DB_SILVER=etl_test_silver_db" in out assert "export G3DT_DB_GOLD=etl_test_gold_db" in out + assert f"export G3DT_S3_BRONZE_DATA_DIR=s3://etl-test-bronze-{ACCOUNT}-{REGION}/dbt/" in out assert f"export G3DT_S3_SILVER_DATA_DIR=s3://etl-test-silver-{ACCOUNT}-{REGION}/dbt/" in out assert f"export G3DT_S3_GOLD_DATA_DIR=s3://etl-test-gold-{ACCOUNT}-{REGION}/dbt/" in out # no profile configured -> ambient credentials and the default dbt target @@ -228,11 +230,14 @@ def test_config_dbt_env_emits_ci_isolation_vars(): result = runner.invoke(app, ["config", "dbt-env", "--env", "test"]) assert result.exit_code == 0, result.output out = result.output + assert "export G3DT_DB_BRONZE_CI=ci_etl_test_bronze_db" in out assert "export G3DT_DB_SILVER_CI=ci_etl_test_silver_db" in out assert "export G3DT_DB_GOLD_CI=ci_etl_test_gold_db" in out + assert f"export G3DT_S3_BRONZE_DATA_DIR_CI=s3://etl-test-bronze-{ACCOUNT}-{REGION}/dbt_ci/" in out assert f"export G3DT_S3_SILVER_DATA_DIR_CI=s3://etl-test-silver-{ACCOUNT}-{REGION}/dbt_ci/" in out assert f"export G3DT_S3_GOLD_DATA_DIR_CI=s3://etl-test-gold-{ACCOUNT}-{REGION}/dbt_ci/" in out # the real names remain unprefixed + assert "export G3DT_DB_BRONZE=etl_test_bronze_db" in out assert "export G3DT_DB_SILVER=etl_test_silver_db" in out assert "export G3DT_DB_GOLD=etl_test_gold_db" in out