Skip to content

Commit 43ac8a3

Browse files
committed
feat(bigquery): add bigquery_query_iceberg_table sample to refresh and query an Apache Iceberg BigLake table
Problem: The BigQuery documentation (https://cloud.google.com/bigquery/docs/query-iceberg-data) needs a standalone, tested Python code sample demonstrating how to refresh and query an Apache Iceberg BigLake table (region tag `bigquery_query_iceberg_table`). Solution: - Add `query_iceberg_table(project_id: str, dataset_id: str, table_id: str, connection_id: str, metadata_uri: str) -> None` in `bigquery/biglake/bigquery_query_iceberg_table.py` (Update an Apache Iceberg external table definition to point to the latest metadata snapshot on Cloud Storage and query its rows.) - Add unit tests (`bigquery/biglake/bigquery_query_iceberg_table_test.py`) covering both the success path and API error handling (`GoogleAPICallError`)
1 parent 186be98 commit 43ac8a3

5 files changed

Lines changed: 187 additions & 0 deletions

File tree

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
# Copyright 2026 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# https://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
# [START bigquery_query_iceberg_table]
16+
import sys
17+
18+
from google.api_core import exceptions
19+
from google.cloud import bigquery
20+
21+
22+
def query_iceberg_table(
23+
project_id: str,
24+
dataset_id: str,
25+
table_id: str,
26+
connection_id: str,
27+
metadata_uri: str,
28+
) -> None:
29+
"""Refreshes an Apache Iceberg external table metadata snapshot and queries it.
30+
31+
Args:
32+
project_id: Google Cloud project ID (for example, 'example-project').
33+
dataset_id: BigQuery dataset ID containing the Iceberg external table
34+
(for example, 'example_dataset').
35+
table_id: BigQuery Iceberg external table ID
36+
(for example, 'example_iceberg_table').
37+
connection_id: Fully qualified BigQuery Cloud Resource connection ID
38+
(for example, 'example-project.us.example_iceberg_conn').
39+
metadata_uri: Cloud Storage URI of the Apache Iceberg metadata JSON file
40+
(for example, 'gs://example-bucket/warehouse/db/table/metadata/v2.metadata.json').
41+
"""
42+
client = bigquery.Client(project=project_id)
43+
table_ref = f"{project_id}.{dataset_id}.{table_id}"
44+
# BigQuery Iceberg external tables point to a specific Iceberg metadata JSON snapshot;
45+
# re-running CREATE OR REPLACE EXTERNAL TABLE updates the snapshot URI with schema autodetection.
46+
query = (
47+
f"CREATE OR REPLACE EXTERNAL TABLE `{table_ref}` "
48+
f"WITH CONNECTION `{connection_id}` "
49+
"OPTIONS ("
50+
" format = 'ICEBERG',"
51+
f" uris = ['{metadata_uri}']"
52+
"); "
53+
"SELECT order_id, user_id, status, num_of_item "
54+
f"FROM `{table_ref}` "
55+
"ORDER BY order_id ASC;"
56+
)
57+
58+
try:
59+
print(f"Refreshing Iceberg metadata and querying table: {table_ref}")
60+
query_job = client.query(query)
61+
rows = list(query_job.result())
62+
except exceptions.NotFound as err:
63+
print(
64+
f"error: Apache Iceberg table '{table_id}' in dataset '{dataset_id}' "
65+
f"for project '{project_id}' (connection '{connection_id}', "
66+
f"metadata '{metadata_uri}') was not found: {err.message}",
67+
file=sys.stderr,
68+
)
69+
return
70+
except exceptions.GoogleAPICallError as err:
71+
print(
72+
f"error: could not refresh or query Apache Iceberg table '{table_id}' "
73+
f"in dataset '{dataset_id}' for project '{project_id}' "
74+
f"(connection '{connection_id}', metadata '{metadata_uri}'): {err.message}",
75+
file=sys.stderr,
76+
)
77+
return
78+
79+
print(f"Resource ID: {table_ref}")
80+
print(f"Job ID: {query_job.job_id}")
81+
print(f"Metadata URI: {metadata_uri}")
82+
print(f"Rows Returned: {len(rows)}")
83+
84+
85+
# [END bigquery_query_iceberg_table]
Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
# Copyright 2026 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# https://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
from unittest import mock
16+
17+
from google.api_core import exceptions
18+
import pytest
19+
20+
import bigquery_query_iceberg_table
21+
22+
23+
def test_query_iceberg_table_success(
24+
monkeypatch: pytest.MonkeyPatch,
25+
capsys: pytest.CaptureFixture[str],
26+
) -> None:
27+
mock_client = mock.MagicMock()
28+
mock_job = mock.MagicMock()
29+
mock_job.job_id = "example-job-123"
30+
mock_job.statement_type = "SCRIPT"
31+
mock_row = mock.MagicMock()
32+
mock_job.result.return_value = [mock_row]
33+
mock_client.query.return_value = mock_job
34+
monkeypatch.setattr(
35+
bigquery_query_iceberg_table.bigquery,
36+
"Client",
37+
lambda project=None: mock_client,
38+
)
39+
40+
bigquery_query_iceberg_table.query_iceberg_table(
41+
project_id="example-project",
42+
dataset_id="example_dataset",
43+
table_id="example_iceberg_table",
44+
connection_id="example-project.us.example_iceberg_conn",
45+
metadata_uri="gs://example-bucket/warehouse/orders_ns/customer_orders/metadata/v2.metadata.json",
46+
)
47+
48+
captured = capsys.readouterr()
49+
assert "Resource ID:" in captured.out
50+
assert "example-project" in captured.out
51+
assert mock_client.query.called
52+
53+
54+
def test_query_iceberg_table_error(
55+
monkeypatch: pytest.MonkeyPatch,
56+
capsys: pytest.CaptureFixture[str],
57+
) -> None:
58+
mock_client = mock.MagicMock()
59+
mock_client.query.side_effect = exceptions.NotFound("resource error")
60+
monkeypatch.setattr(
61+
bigquery_query_iceberg_table.bigquery,
62+
"Client",
63+
lambda project=None: mock_client,
64+
)
65+
66+
bigquery_query_iceberg_table.query_iceberg_table(
67+
project_id="example-project",
68+
dataset_id="example_dataset",
69+
table_id="example_iceberg_table",
70+
connection_id="example-project.us.example_iceberg_conn",
71+
metadata_uri="gs://example-bucket/warehouse/orders_ns/customer_orders/metadata/v2.metadata.json",
72+
)
73+
74+
captured = capsys.readouterr()
75+
assert "error:" in captured.err
76+
assert "example-project" in captured.err

‎bigquery/biglake/noxfile_config.py‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
# Copyright 2026 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# https://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
TEST_CONFIG_OVERRIDE = {
16+
# You can opt out from the test for specific Python versions.
17+
"ignored_versions": ["2.7", "3.7", "3.8", "3.9", "3.10", "3.11"],
18+
# Enforce type hints on new samples.
19+
"enforce_type_hints": True,
20+
"gcloud_project_env": "GOOGLE_CLOUD_PROJECT",
21+
"pip_version_override": None,
22+
"envs": {},
23+
}
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
pytest==9.0.3; python_version >= "3.10"

‎bigquery/biglake/requirements.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
google-cloud-bigquery==3.47.0
2+
google-cloud-bigquery-storage==2.43.0

0 commit comments

Comments
 (0)