diff --git a/README.md b/README.md index 5268f37..27ee953 100644 --- a/README.md +++ b/README.md @@ -79,9 +79,11 @@ structurally impossible. **Only the dbt template's `ci` target is prefixed.** `g3dt config dbt-env` emits, alongside the real names, the CI-isolation variants the template's -`ci` target consumes: `G3DT_DB_RAW_SILVER_CI` / `G3DT_DB_RAW_GOLD_CI` +`ci` target consumes: `G3DT_DB_SILVER_CI` / `G3DT_DB_GOLD_CI` (`ci_` + the real database name) and `G3DT_S3_SILVER_DATA_DIR_CI` / -`G3DT_S3_GOLD_DATA_DIR_CI` (`dbt_ci/` under the same buckets). Commit- +`G3DT_S3_GOLD_DATA_DIR_CI` (`dbt_ci/` under the same buckets). Toolkit +releases >= 3 read the raw-free medallion SSM keys and therefore require a +pipeline deployment >= v2.0.0, which publishes them. Commit- triggered CI builds land there; every other target (default, local) and the release build keep the real, unprefixed names — so CI can never advance the warehouse's Iceberg snapshots that releases pin. The library enforces the diff --git a/pyproject.toml b/pyproject.toml index 5b0541a..dda10fe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "gen3-dataops-toolkit" -version = "2.3.0" +version = "3.0.0" description = "Gen3 DataOps toolkit (g3dt): operate SSM-published Gen3 data pipeline environments" authors = ["JoshuaHarris391 "] readme = "README.md" diff --git a/src/g3dt/cli/config_cmds.py b/src/g3dt/cli/config_cmds.py index a0b0b1b..265a9ec 100644 --- a/src/g3dt/cli/config_cmds.py +++ b/src/g3dt/cli/config_cmds.py @@ -25,6 +25,27 @@ ) +def _req_key(rc, key: str) -> str: + """Return the SSM leaf ``key`` from ``rc``, failing loudly if absent. + + The medallion names (``buckets/silver|gold``, ``glue/db/bronze|silver|gold``) + are published under these raw-free keys by pipeline deployments >= v2.0.0. + Before this guard, a missing key propagated as ``None``: dbt-env silently + dropped the G3DT_DB_* vars and emitted ``s3://None/dbt/`` data dirs, and + the release search silently fell back to an account-wide catalog walk. + """ + value = rc.get(key) + if value is None: + raise config.ConfigError( + f"SSM parameter /{rc.project}/{rc.env}/{key} is missing. " + f"gen3-dataops-toolkit >= 3 reads the raw-free medallion keys " + f"published by gen3-aws-data-pipeline >= v2.0.0; a pipeline " + f"deployment older than v2.0.0 still publishes raw-prefixed keys. " + f"Upgrade the pipeline deployment (or pin gen3-dataops-toolkit<3)." + ) + return value + + @app.command() def envs() -> None: """List the environments with a deployed SSM tree for this project.""" @@ -208,28 +229,31 @@ def dbt_env( profile = None if env.endswith("_ec2") else config.aws_profile_for(base, marker) try: rc = resolver.resolve(project, base, profile=profile) + silver_db = _req_key(rc, "glue/db/silver") + gold_db = _req_key(rc, "glue/db/gold") + silver_bucket = _req_key(rc, "buckets/silver") + gold_bucket = _req_key(rc, "buckets/gold") + bronze_db = _req_key(rc, "glue/db/bronze") except config.ConfigError as exc: typer.secho(str(exc), fg=typer.colors.RED, err=True) raise typer.Exit(1) - raw_silver_db = rc.get("glue/db/rawSilver") - raw_gold_db = rc.get("glue/db/rawGold") values = { "G3DT_REGION": rc.region, "G3DT_ATHENA_WORKGROUP": rc.athena_workgroup, "G3DT_ATHENA_OUTPUT": rc.athena_output_location, - "G3DT_DB_RAW_BRONZE": rc.get("glue/db/rawBronze"), - "G3DT_DB_RAW_SILVER": raw_silver_db, - "G3DT_DB_RAW_GOLD": raw_gold_db, - "G3DT_S3_SILVER_DATA_DIR": f"s3://{rc.get('buckets/rawSilver')}/dbt/", - "G3DT_S3_GOLD_DATA_DIR": f"s3://{rc.get('buckets/rawGold')}/dbt/", + "G3DT_DB_BRONZE": bronze_db, + "G3DT_DB_SILVER": silver_db, + "G3DT_DB_GOLD": gold_db, + "G3DT_S3_SILVER_DATA_DIR": f"s3://{silver_bucket}/dbt/", + "G3DT_S3_GOLD_DATA_DIR": f"s3://{gold_bucket}/dbt/", # CI isolation: the dbt template's `ci` target builds into these # instead — same grammar as the CDK's ci_ databases, same buckets # under a dbt_ci/ prefix. Real names above are never prefixed. - "G3DT_DB_RAW_SILVER_CI": f"ci_{raw_silver_db}" if raw_silver_db else None, - "G3DT_DB_RAW_GOLD_CI": f"ci_{raw_gold_db}" if raw_gold_db else None, - "G3DT_S3_SILVER_DATA_DIR_CI": f"s3://{rc.get('buckets/rawSilver')}/dbt_ci/", - "G3DT_S3_GOLD_DATA_DIR_CI": f"s3://{rc.get('buckets/rawGold')}/dbt_ci/", + "G3DT_DB_SILVER_CI": f"ci_{silver_db}", + "G3DT_DB_GOLD_CI": f"ci_{gold_db}", + "G3DT_S3_SILVER_DATA_DIR_CI": f"s3://{silver_bucket}/dbt_ci/", + "G3DT_S3_GOLD_DATA_DIR_CI": f"s3://{gold_bucket}/dbt_ci/", } if profile: # A named profile means a laptop run: select the dbt target that diff --git a/src/g3dt/cli/release_cmds.py b/src/g3dt/cli/release_cmds.py index 4873fc2..4af7c17 100644 --- a/src/g3dt/cli/release_cmds.py +++ b/src/g3dt/cli/release_cmds.py @@ -57,10 +57,14 @@ def write( ) # The env's own silver/gold DBs (from SSM) scope the model->DB search: # deterministic in shared accounts, no account-wide Glue perms needed. + # _req_key fails loudly if the raw-free keys are absent (pre-v2.0.0 + # pipeline) instead of silently widening to an account-wide walk. + from g3dt.cli.config_cmds import _req_key + search_databases = [ - db for db in (rc.get("glue/db/rawSilver"), rc.get("glue/db/rawGold")) - if db - ] or None + _req_key(rc, "glue/db/silver"), + _req_key(rc, "glue/db/gold"), + ] release_writer.run( dbt_schema_path=dbt_schema_path, release_db=rc.release_db, diff --git a/tests/test_athena_utils.py b/tests/test_athena_utils.py index 2e84815..2d5a45f 100644 --- a/tests/test_athena_utils.py +++ b/tests/test_athena_utils.py @@ -1675,7 +1675,7 @@ def test_find_db_for_model_skips_ci_databases(mock_boto_session, mock_wr, scoped paths: the scoped databases= list and the account-wide catalog walk. Steps: - 1. Present ['ci_proj_test_raw_gold_db', 'proj_test_raw_gold_db'], + 1. Present ['ci_proj_test_gold_db', 'proj_test_gold_db'], both containing 'gold_model', with the ci_ database FIRST. 2. Call find_db_for_model, scoped and unscoped. @@ -1685,7 +1685,7 @@ def test_find_db_for_model_skips_ci_databases(mock_boto_session, mock_wr, scoped """ mock_session_instance = MagicMock() mock_boto_session.return_value = mock_session_instance - dbs = ['ci_proj_test_raw_gold_db', 'proj_test_raw_gold_db'] + dbs = ['ci_proj_test_gold_db', 'proj_test_gold_db'] mock_wr.catalog.databases.return_value = {'Database': dbs} mock_wr.catalog.get_tables.return_value = [{'Name': 'gold_model'}] @@ -1693,10 +1693,10 @@ def test_find_db_for_model_skips_ci_databases(mock_boto_session, mock_wr, scoped 'gold_model', databases=dbs if scoped else None ) - assert result == 'proj_test_raw_gold_db' + assert result == 'proj_test_gold_db' assert mock_wr.catalog.get_tables.call_count == 1 mock_wr.catalog.get_tables.assert_called_once_with( - database='proj_test_raw_gold_db', boto3_session=mock_session_instance + database='proj_test_gold_db', boto3_session=mock_session_instance ) diff --git a/tests/test_release_cmds.py b/tests/test_release_cmds.py index 1f47233..51f3a16 100644 --- a/tests/test_release_cmds.py +++ b/tests/test_release_cmds.py @@ -50,11 +50,11 @@ def _seed(project="etl", env="test"): leaves = { "meta/region": REGION, "buckets/metadata": f"{project}-{env}-metadata-{ACCOUNT}-{REGION}", - "buckets/rawSilver": f"{project}-{env}-raw-silver-{ACCOUNT}-{REGION}", - "buckets/rawGold": f"{project}-{env}-raw-gold-{ACCOUNT}-{REGION}", - "glue/db/rawBronze": f"{project}_{env}_raw_bronze_db", - "glue/db/rawSilver": f"{project}_{env}_raw_silver_db", - "glue/db/rawGold": f"{project}_{env}_raw_gold_db", + "buckets/silver": f"{project}-{env}-silver-{ACCOUNT}-{REGION}", + "buckets/gold": f"{project}-{env}-gold-{ACCOUNT}-{REGION}", + "glue/db/bronze": f"{project}_{env}_bronze_db", + "glue/db/silver": f"{project}_{env}_silver_db", + "glue/db/gold": f"{project}_{env}_gold_db", "release/db": f"{project}_{env}_dataops_metadata_db", "release/table": "releases", "athena/workgroup": f"{project}-{env}", @@ -96,7 +96,7 @@ def test_release_write_resolves_names_from_ssm(mock_run): # env's own DBs so shared accounts can't cross-match assert kwargs["workgroup"] == "etl-test" assert kwargs["search_databases"] == [ - "etl_test_raw_silver_db", "etl_test_raw_gold_db" + "etl_test_silver_db", "etl_test_gold_db" ] # the resolved target is echoed so a build log always shows where rows go assert "etl_test_dataops_metadata_db.releases" in result.output @@ -144,7 +144,7 @@ def test_release_writer_dry_run_writes_nothing(): insert_release_row( athena_config=athena_config, model_name="silver_x", - db_name="etl_test_raw_silver_db", + db_name="etl_test_silver_db", snapshot_id=1, committed_at="2026-01-01 00:00:00", release_db="etl_test_dataops_metadata_db", @@ -171,11 +171,11 @@ def test_config_dbt_env_emits_every_dbt_setting(): assert "export G3DT_ATHENA_WORKGROUP=etl-test" in out assert f"export G3DT_ATHENA_OUTPUT=s3://etl-test-athena-results-{ACCOUNT}-{REGION}/" in out assert f"export G3DT_REGION={REGION}" in out - assert "export G3DT_DB_RAW_BRONZE=etl_test_raw_bronze_db" in out - assert "export G3DT_DB_RAW_SILVER=etl_test_raw_silver_db" in out - assert "export G3DT_DB_RAW_GOLD=etl_test_raw_gold_db" in out - assert f"export G3DT_S3_SILVER_DATA_DIR=s3://etl-test-raw-silver-{ACCOUNT}-{REGION}/dbt/" in out - assert f"export G3DT_S3_GOLD_DATA_DIR=s3://etl-test-raw-gold-{ACCOUNT}-{REGION}/dbt/" in out + assert "export G3DT_DB_BRONZE=etl_test_bronze_db" in out + assert "export G3DT_DB_SILVER=etl_test_silver_db" in out + assert "export G3DT_DB_GOLD=etl_test_gold_db" in out + assert f"export G3DT_S3_SILVER_DATA_DIR=s3://etl-test-silver-{ACCOUNT}-{REGION}/dbt/" in out + assert f"export G3DT_S3_GOLD_DATA_DIR=s3://etl-test-gold-{ACCOUNT}-{REGION}/dbt/" in out # no profile configured -> ambient credentials and the default dbt target assert "G3DT_AWS_PROFILE" not in out assert "G3DT_DBT_TARGET" not in out @@ -228,13 +228,63 @@ def test_config_dbt_env_emits_ci_isolation_vars(): result = runner.invoke(app, ["config", "dbt-env", "--env", "test"]) assert result.exit_code == 0, result.output out = result.output - assert "export G3DT_DB_RAW_SILVER_CI=ci_etl_test_raw_silver_db" in out - assert "export G3DT_DB_RAW_GOLD_CI=ci_etl_test_raw_gold_db" in out - assert f"export G3DT_S3_SILVER_DATA_DIR_CI=s3://etl-test-raw-silver-{ACCOUNT}-{REGION}/dbt_ci/" in out - assert f"export G3DT_S3_GOLD_DATA_DIR_CI=s3://etl-test-raw-gold-{ACCOUNT}-{REGION}/dbt_ci/" in out + assert "export G3DT_DB_SILVER_CI=ci_etl_test_silver_db" in out + assert "export G3DT_DB_GOLD_CI=ci_etl_test_gold_db" in out + assert f"export G3DT_S3_SILVER_DATA_DIR_CI=s3://etl-test-silver-{ACCOUNT}-{REGION}/dbt_ci/" in out + assert f"export G3DT_S3_GOLD_DATA_DIR_CI=s3://etl-test-gold-{ACCOUNT}-{REGION}/dbt_ci/" in out # the real names remain unprefixed - assert "export G3DT_DB_RAW_SILVER=etl_test_raw_silver_db" in out - assert "export G3DT_DB_RAW_GOLD=etl_test_raw_gold_db" in out + assert "export G3DT_DB_SILVER=etl_test_silver_db" in out + assert "export G3DT_DB_GOLD=etl_test_gold_db" in out + + +@mock_aws +@patch("g3dt.utils.release_writer.run") +def test_missing_medallion_keys_fail_loudly(mock_run): + """ + Background: + gen3-aws-data-pipeline < v2.0.0 published the medallion names under + SSM keys carrying a legacy raw prefix in the leaf name. Toolkit + versions < 3 read them with rc.get(), which returns None for + an absent key — so a key-name mismatch FAILED SILENTLY: dbt-env + dropped the G3DT_DB_* export lines entirely and emitted + `s3://None/dbt/` data dirs, and `release write` quietly fell back to + an account-wide Glue catalog walk. Toolkit >= 3 reads only the + raw-free keys and must fail loudly when they are missing. + + Inputs: an SSM tree seeded WITHOUT the medallion keys — exactly what a + pipeline deployment older than v2.0.0 presents to toolkit >= 3. + Expected: `config dbt-env` and `release write` both exit 1 with a + ConfigError naming the missing SSM key and pointing at the + pipeline upgrade (>= v2.0.0); no partial exports, no s3://None, + and the release writer is never invoked. + """ + ssm = boto3.client("ssm", region_name=REGION) + leaves = { # everything _seed publishes EXCEPT the medallion keys + "meta/region": REGION, + "buckets/metadata": f"etl-test-metadata-{ACCOUNT}-{REGION}", + "release/db": "etl_test_dataops_metadata_db", + "release/table": "releases", + "athena/workgroup": "etl-test", + "athena/outputLocation": f"s3://etl-test-athena-results-{ACCOUNT}-{REGION}/", + } + for rel, value in leaves.items(): + ssm.put_parameter(Name=f"/etl/test/{rel}", Value=value, Type="String") + + result = runner.invoke(app, ["config", "dbt-env", "--env", "test"]) + assert result.exit_code == 1 + assert "/etl/test/glue/db/silver is missing" in result.output + assert "v2.0.0" in result.output + assert "export" not in result.output # all-or-nothing: no partial env + assert "s3://None" not in result.output + + result = runner.invoke( + app, + ["release", "write", "--env", "test", "--data-release-version", "1.0.0"], + ) + assert result.exit_code == 1 + assert "/etl/test/glue/db/silver is missing" in result.output + assert "v2.0.0" in result.output + mock_run.assert_not_called() def test_release_writer_run_aggregates_model_failures(): @@ -267,7 +317,7 @@ def fake_snapshot(self, return_commit_datetime=False): with patch.object(rw, "get_model_names", return_value=["silver_a", "silver_bad", "silver_b"]), \ patch.object(rw, "insert_release_row", side_effect=fake_insert), \ patch.object(rw.AthenaQuery, "create_release_table", MagicMock()), \ - patch.object(rw.AthenaQuery, "find_db_for_model", lambda self, m, databases=None: "etl_test_raw_silver_db"), \ + patch.object(rw.AthenaQuery, "find_db_for_model", lambda self, m, databases=None: "etl_test_silver_db"), \ patch.object(rw.AthenaValidationWriter, "_get_latest_snapshot_id", fake_snapshot): with pytest.raises(RuntimeError, match="silver_bad"): rw.run(