From 27459f343085a72c2453976ed80902a0591dbdff Mon Sep 17 00:00:00 2001 From: Jennifer Davis Date: Tue, 3 Nov 2026 23:31:54 +0000 Subject: [PATCH] feat(bigquery): add bigquery_query_external_bigtable sample to query Cloud Bigtable data as an external table Problem: The BigQuery documentation (https://cloud.google.com/bigquery/docs/external-data-bigtable) needs a standalone, tested Python code sample demonstrating how to query Cloud Bigtable data as an external table (region tag `bigquery_query_external_bigtable`). Solution: - Add `query_external_bigtable(project_id: str, bigtable_instance_id: str, bigtable_table_id: str) -> None` in `bigquery/biglake/bigquery_query_external_bigtable.py` (Configure a temporary Cloud Bigtable external table mapping column families and qualifiers and query it from BigQuery.) - Add unit tests (`bigquery/biglake/bigquery_query_external_bigtable_test.py`) covering both the success path and API error handling (`GoogleAPICallError`) --- .../bigquery_query_external_bigtable.py | 90 +++++++++++++++++++ .../bigquery_query_external_bigtable_test.py | 77 ++++++++++++++++ bigquery/biglake/noxfile_config.py | 23 +++++ bigquery/biglake/requirements-test.txt | 1 + bigquery/biglake/requirements.txt | 2 + 5 files changed, 193 insertions(+) create mode 100644 bigquery/biglake/bigquery_query_external_bigtable.py create mode 100644 bigquery/biglake/bigquery_query_external_bigtable_test.py create mode 100644 bigquery/biglake/noxfile_config.py create mode 100644 bigquery/biglake/requirements-test.txt create mode 100644 bigquery/biglake/requirements.txt diff --git a/bigquery/biglake/bigquery_query_external_bigtable.py b/bigquery/biglake/bigquery_query_external_bigtable.py new file mode 100644 index 00000000000..a401936cf90 --- /dev/null +++ b/bigquery/biglake/bigquery_query_external_bigtable.py @@ -0,0 +1,90 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START bigquery_query_external_bigtable] +import sys + +from google.api_core import exceptions +from google.cloud import bigquery + + +def query_external_bigtable( + project_id: str, + bigtable_instance_id: str, + bigtable_table_id: str, +) -> None: + """Configures and queries a temporary BigQuery external table backed by Cloud Bigtable. + + Args: + project_id: Google Cloud project ID (for example, 'example-project'). + bigtable_instance_id: Cloud Bigtable instance ID hosting the source table + (for example, 'example-bt-instance'). + bigtable_table_id: Cloud Bigtable table ID containing the column family + (for example, 'example_bt_table'). + """ + client = bigquery.Client(project=project_id) + temp_table_name = "example_bigtable_temp" + source_uri = ( + f"https://googleapis.com/bigtable/projects/{project_id}" + f"/instances/{bigtable_instance_id}/tables/{bigtable_table_id}" + ) + + # Map the Cloud Bigtable column family into a temporary table definition for a single query job. + external_config = bigquery.ExternalConfig("BIGTABLE") + external_config.source_uris = [source_uri] + external_config.bigtable_options.read_rowkey_as_string = True + family = bigquery.BigtableColumnFamily() + family.family_id = "stats_summary" + family.type_ = "STRING" + family.encoding = "TEXT" + external_config.bigtable_options.column_families = [family] + + job_config = bigquery.QueryJobConfig( + table_definitions={temp_table_name: external_config} + ) + sql = f""" + SELECT + rowkey, + stats_summary + FROM `{temp_table_name}` + ORDER BY rowkey ASC + LIMIT 10; + """ + + try: + print(f"Querying temporary Cloud Bigtable table mapped to {source_uri}") + query_job = client.query(sql, job_config=job_config) + rows = list(query_job.result()) + except exceptions.NotFound as err: + print( + f"error: Cloud Bigtable table '{bigtable_table_id}' in instance " + f"'{bigtable_instance_id}' not found for project '{project_id}': {err.message}", + file=sys.stderr, + ) + return + except exceptions.GoogleAPICallError as err: + print( + f"error: failed to query Cloud Bigtable table '{bigtable_table_id}' in " + f"instance '{bigtable_instance_id}' for project '{project_id}': {err.message}", + file=sys.stderr, + ) + return + + print(f"Resource ID: {source_uri}") + print(f"Job ID: {query_job.job_id}") + print(f"Rows Returned: {len(rows)}") + print(f"Temporary Table: {temp_table_name}") + + +# [END bigquery_query_external_bigtable] diff --git a/bigquery/biglake/bigquery_query_external_bigtable_test.py b/bigquery/biglake/bigquery_query_external_bigtable_test.py new file mode 100644 index 00000000000..907beee8660 --- /dev/null +++ b/bigquery/biglake/bigquery_query_external_bigtable_test.py @@ -0,0 +1,77 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from unittest import mock + +import bigquery_query_external_bigtable +from google.api_core import exceptions +import pytest + + +def test_query_external_bigtable_success( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + mock_client = mock.MagicMock() + mock_job = mock.MagicMock() + mock_job.job_id = "example-job-123" + mock_row = mock.MagicMock() + mock_row.rowkey = "row-001" + mock_job.result.return_value = [mock_row] + mock_client.query.return_value = mock_job + monkeypatch.setattr( + bigquery_query_external_bigtable.bigquery, + "Client", + lambda project=None: mock_client, + ) + + bigquery_query_external_bigtable.query_external_bigtable( + project_id="example-project", + bigtable_instance_id="example-bt-instance", + bigtable_table_id="example_bt_table", + ) + + captured = capsys.readouterr() + assert ( + "Resource ID: https://googleapis.com/bigtable/projects/example-project" + in captured.out + ) + assert "Rows Returned: 1" in captured.out + assert "Temporary Table: example_bigtable_temp" in captured.out + assert mock_client.query.called + + +def test_query_external_bigtable_error( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + mock_client = mock.MagicMock() + mock_job = mock.MagicMock() + mock_job.result.side_effect = exceptions.NotFound("resource error") + mock_client.query.return_value = mock_job + monkeypatch.setattr( + bigquery_query_external_bigtable.bigquery, + "Client", + lambda project=None: mock_client, + ) + + bigquery_query_external_bigtable.query_external_bigtable( + project_id="example-project", + bigtable_instance_id="example-bt-instance", + bigtable_table_id="example_bt_table", + ) + + captured = capsys.readouterr() + assert "error:" in captured.err + assert "example-bt-instance" in captured.err diff --git a/bigquery/biglake/noxfile_config.py b/bigquery/biglake/noxfile_config.py new file mode 100644 index 00000000000..63902aa8c7f --- /dev/null +++ b/bigquery/biglake/noxfile_config.py @@ -0,0 +1,23 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +TEST_CONFIG_OVERRIDE = { + # You can opt out from the test for specific Python versions. + "ignored_versions": ["2.7", "3.7", "3.8", "3.9", "3.10", "3.11"], + # Enforce type hints on new samples. + "enforce_type_hints": True, + "gcloud_project_env": "GOOGLE_CLOUD_PROJECT", + "pip_version_override": None, + "envs": {}, +} diff --git a/bigquery/biglake/requirements-test.txt b/bigquery/biglake/requirements-test.txt new file mode 100644 index 00000000000..c9e154ba440 --- /dev/null +++ b/bigquery/biglake/requirements-test.txt @@ -0,0 +1 @@ +pytest==9.0.3; python_version >= "3.10" diff --git a/bigquery/biglake/requirements.txt b/bigquery/biglake/requirements.txt new file mode 100644 index 00000000000..af6312b7e7f --- /dev/null +++ b/bigquery/biglake/requirements.txt @@ -0,0 +1,2 @@ +google-cloud-bigquery==3.47.0 +google-cloud-bigquery-storage==2.43.0