diff --git a/bigquery/biglake/bigquery_upgrade_manifest_biglake_table.py b/bigquery/biglake/bigquery_upgrade_manifest_biglake_table.py new file mode 100644 index 0000000000..fb5c4cd11e --- /dev/null +++ b/bigquery/biglake/bigquery_upgrade_manifest_biglake_table.py @@ -0,0 +1,96 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START bigquery_upgrade_manifest_biglake_table] +import sys + +from google.api_core import exceptions +from google.cloud import bigquery + + +def upgrade_manifest_biglake_table( + project_id: str, + dataset_id: str, + table_id: str, + connection_id: str, + manifest_uri: str, +) -> None: + """Upgrades a manifest external table to a BigLake table with metadata caching. + + Args: + project_id: Google Cloud project ID (for example, 'example-project'). + dataset_id: BigQuery dataset ID containing the external table + (for example, 'example_dataset'). + table_id: BigQuery external table ID to upgrade + (for example, 'example_manifest_table'). + connection_id: Fully qualified BigQuery Cloud Resource connection ID + (for example, 'example-project.us.example_biglake_conn'). + manifest_uri: Cloud Storage URI of the SymLinkTextInputFormat manifest file + (for example, 'gs://example-bucket/manifests/symlink.manifest'). + """ + client = bigquery.Client(project=project_id) + table_ref = f"{project_id}.{dataset_id}.{table_id}" + # Upgrading the manifest external table with a Cloud Resource connection and AUTOMATIC metadata + # caching accelerates open-table-format reads and enforces BigLake governance. + sql = f""" + CREATE OR REPLACE EXTERNAL TABLE `{table_ref}` + WITH CONNECTION `{connection_id}` + OPTIONS ( + format = 'PARQUET', + uris = ['{manifest_uri}'], + file_set_spec_type = 'NEW_LINE_DELIMITED_MANIFEST', + max_staleness = INTERVAL 1 HOUR, + metadata_cache_mode = 'AUTOMATIC' + ); + SELECT + id, + label, + score + FROM `{table_ref}` + ORDER BY id ASC + LIMIT 10; + """ + + try: + print( + f"Upgrading manifest external table to BigLake with metadata caching: {table_ref}" + ) + query_job = client.query(sql) + rows = list(query_job.result()) + except exceptions.NotFound as err: + print( + f"error: manifest external table '{table_id}' or connection '{connection_id}' " + f"not found in dataset '{dataset_id}' for project '{project_id}' " + f"(manifest '{manifest_uri}'): {err.message}", + file=sys.stderr, + ) + return + except exceptions.GoogleAPICallError as err: + print( + f"error: failed to upgrade manifest external table '{table_id}' in dataset " + f"'{dataset_id}' for project '{project_id}' (connection '{connection_id}', " + f"manifest '{manifest_uri}'): {err.message}", + file=sys.stderr, + ) + return + + print(f"Resource ID: {table_ref}") + print(f"Connection ID: {connection_id}") + print(f"Job ID: {query_job.job_id}") + print(f"Manifest URI: {manifest_uri}") + print("Metadata Cache Mode: AUTOMATIC") + print(f"Rows Returned: {len(rows)}") + + +# [END bigquery_upgrade_manifest_biglake_table] diff --git a/bigquery/biglake/bigquery_upgrade_manifest_biglake_table_test.py b/bigquery/biglake/bigquery_upgrade_manifest_biglake_table_test.py new file mode 100644 index 0000000000..d037359b3b --- /dev/null +++ b/bigquery/biglake/bigquery_upgrade_manifest_biglake_table_test.py @@ -0,0 +1,73 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from unittest import mock + +import bigquery_upgrade_manifest_biglake_table +from google.api_core import exceptions +import pytest + + +def test_upgrade_manifest_biglake_table_success( + capsys: pytest.CaptureFixture[str], +) -> None: + with mock.patch("google.cloud.bigquery.Client") as mock_client_cls: + mock_client = mock_client_cls.return_value + mock_job = mock.MagicMock() + mock_job.job_id = "example-job-123" + mock_job.result.return_value = [ + mock.MagicMock(), + mock.MagicMock(), + mock.MagicMock(), + ] + mock_client.query.return_value = mock_job + + bigquery_upgrade_manifest_biglake_table.upgrade_manifest_biglake_table( + project_id="example-project", + dataset_id="example_dataset", + table_id="example_manifest_table", + connection_id="example-project.us.example_biglake_conn", + manifest_uri="gs://example-bucket/manifests/symlink.manifest", + ) + + mock_client.query.assert_called_once() + captured = capsys.readouterr() + assert ( + "Resource ID: example-project.example_dataset.example_manifest_table" + in captured.out + ) + assert "Metadata Cache Mode: AUTOMATIC" in captured.out + assert "Rows Returned: 3" in captured.out + + +def test_upgrade_manifest_biglake_table_not_found( + capsys: pytest.CaptureFixture[str], +) -> None: + with mock.patch("google.cloud.bigquery.Client") as mock_client_cls: + mock_client = mock_client_cls.return_value + mock_job = mock.MagicMock() + mock_job.result.side_effect = exceptions.NotFound("Connection not found") + mock_client.query.return_value = mock_job + + bigquery_upgrade_manifest_biglake_table.upgrade_manifest_biglake_table( + project_id="example-project", + dataset_id="example_dataset", + table_id="example_manifest_table", + connection_id="example-project.us.example_biglake_conn", + manifest_uri="gs://example-bucket/manifests/symlink.manifest", + ) + + captured = capsys.readouterr() + assert "error:" in captured.err + assert "example_manifest_table" in captured.err diff --git a/bigquery/biglake/noxfile_config.py b/bigquery/biglake/noxfile_config.py new file mode 100644 index 0000000000..63902aa8c7 --- /dev/null +++ b/bigquery/biglake/noxfile_config.py @@ -0,0 +1,23 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +TEST_CONFIG_OVERRIDE = { + # You can opt out from the test for specific Python versions. + "ignored_versions": ["2.7", "3.7", "3.8", "3.9", "3.10", "3.11"], + # Enforce type hints on new samples. + "enforce_type_hints": True, + "gcloud_project_env": "GOOGLE_CLOUD_PROJECT", + "pip_version_override": None, + "envs": {}, +} diff --git a/bigquery/biglake/requirements-test.txt b/bigquery/biglake/requirements-test.txt new file mode 100644 index 0000000000..c9e154ba44 --- /dev/null +++ b/bigquery/biglake/requirements-test.txt @@ -0,0 +1 @@ +pytest==9.0.3; python_version >= "3.10" diff --git a/bigquery/biglake/requirements.txt b/bigquery/biglake/requirements.txt new file mode 100644 index 0000000000..af6312b7e7 --- /dev/null +++ b/bigquery/biglake/requirements.txt @@ -0,0 +1,2 @@ +google-cloud-bigquery==3.47.0 +google-cloud-bigquery-storage==2.43.0