Skip to content

Commit 49d301c

Browse files
authored
feat(bigquery): add bigquery_query_manifest_external_table sample to query an external table defined by a manifest file (#14715)
Problem: The BigQuery documentation (https://cloud.google.com/bigquery/docs/query-open-table-format-using-manifest-files) needs a standalone, tested Python code sample demonstrating how to query an external table defined by a manifest file (region tag `bigquery_query_manifest_external_table`). Solution: - Add `query_manifest_external_table(project_id: str, dataset_id: str, table_id: str, manifest_uri: str) -> None` in `bigquery/biglake/bigquery_query_manifest_external_table.py` (Run a GoogleSQL query against an open-table-format external table backed by a manifest file on Cloud Storage.) - Add unit tests (`bigquery/biglake/bigquery_query_manifest_external_table_test.py`) covering both the success path and API error handling (`GoogleAPICallError`)
1 parent 4233b24 commit 49d301c

5 files changed

Lines changed: 182 additions & 0 deletions

File tree

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
# Copyright 2026 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# https://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
# [START bigquery_query_manifest_external_table]
16+
import sys
17+
18+
from google.api_core import exceptions
19+
from google.cloud import bigquery
20+
21+
22+
def query_manifest_external_table(
23+
project_id: str,
24+
dataset_id: str,
25+
table_id: str,
26+
manifest_uri: str,
27+
) -> None:
28+
"""Configures a newline-delimited manifest on an external table and queries it.
29+
30+
Args:
31+
project_id: Google Cloud project ID (for example, 'example-project').
32+
dataset_id: BigQuery dataset ID containing the external table
33+
(for example, 'example_dataset').
34+
table_id: BigQuery external table ID
35+
(for example, 'example_manifest_table').
36+
manifest_uri: Cloud Storage URI of the newline-delimited manifest file
37+
(for example, 'gs://example-bucket/manifests/symlink.manifest').
38+
"""
39+
client = bigquery.Client(project=project_id)
40+
table_ref = f"{project_id}.{dataset_id}.{table_id}"
41+
42+
# Setting FILE_SET_SPEC_TYPE_NEW_LINE_DELIMITED_MANIFEST tells BigQuery to treat each line
43+
# in manifest_uri as a Cloud Storage data file URI rather than reading the manifest as CSV data.
44+
external_config = bigquery.ExternalConfig("CSV")
45+
external_config.source_uris = [manifest_uri]
46+
external_config.file_set_spec_type = (
47+
"FILE_SET_SPEC_TYPE_NEW_LINE_DELIMITED_MANIFEST"
48+
)
49+
50+
table = bigquery.Table(table_ref)
51+
table.external_data_configuration = external_config
52+
53+
query = (
54+
"SELECT id, label, score " f"FROM `{table_ref}` " "ORDER BY id ASC " "LIMIT 10"
55+
)
56+
57+
try:
58+
print(f"Configuring manifest URI and querying external table: {table_ref}")
59+
client.update_table(table, ["external_data_configuration"])
60+
query_job = client.query(query)
61+
rows = list(query_job.result())
62+
except exceptions.NotFound as err:
63+
print(
64+
f"error: manifest external table '{table_id}' in dataset '{dataset_id}' "
65+
f"for project '{project_id}' (manifest '{manifest_uri}') was not found: {err.message}",
66+
file=sys.stderr,
67+
)
68+
return
69+
except exceptions.GoogleAPICallError as err:
70+
print(
71+
f"error: could not query manifest external table '{table_id}' in dataset "
72+
f"'{dataset_id}' for project '{project_id}' (manifest '{manifest_uri}'): {err.message}",
73+
file=sys.stderr,
74+
)
75+
return
76+
77+
print(f"Resource ID: {table_ref}")
78+
print(f"Job ID: {query_job.job_id}")
79+
print(f"Manifest URI: {manifest_uri}")
80+
print(f"Rows Returned: {len(rows)}")
81+
82+
83+
# [END bigquery_query_manifest_external_table]
Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
# Copyright 2026 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# https://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
from unittest import mock
16+
17+
import bigquery_query_manifest_external_table
18+
from google.api_core import exceptions
19+
import pytest
20+
21+
22+
def test_query_manifest_external_table_success(
23+
monkeypatch: pytest.MonkeyPatch,
24+
capsys: pytest.CaptureFixture[str],
25+
) -> None:
26+
mock_client = mock.MagicMock()
27+
mock_job = mock.MagicMock()
28+
mock_job.job_id = "example-job-123"
29+
mock_job.statement_type = "SCRIPT"
30+
mock_row = mock.MagicMock()
31+
mock_job.result.return_value = [mock_row]
32+
mock_client.query.return_value = mock_job
33+
monkeypatch.setattr(
34+
bigquery_query_manifest_external_table.bigquery,
35+
"Client",
36+
lambda project=None: mock_client,
37+
)
38+
39+
bigquery_query_manifest_external_table.query_manifest_external_table(
40+
project_id="example-project",
41+
dataset_id="example_dataset",
42+
table_id="example_manifest_table",
43+
manifest_uri="gs://example-bucket/manifests/symlink.manifest",
44+
)
45+
46+
captured = capsys.readouterr()
47+
assert "Resource ID:" in captured.out
48+
assert "example-project" in captured.out
49+
assert mock_client.query.called
50+
51+
52+
def test_query_manifest_external_table_error(
53+
monkeypatch: pytest.MonkeyPatch,
54+
capsys: pytest.CaptureFixture[str],
55+
) -> None:
56+
mock_client = mock.MagicMock()
57+
mock_client.query.side_effect = exceptions.NotFound("resource error")
58+
monkeypatch.setattr(
59+
bigquery_query_manifest_external_table.bigquery,
60+
"Client",
61+
lambda project=None: mock_client,
62+
)
63+
64+
bigquery_query_manifest_external_table.query_manifest_external_table(
65+
project_id="example-project",
66+
dataset_id="example_dataset",
67+
table_id="example_manifest_table",
68+
manifest_uri="gs://example-bucket/manifests/symlink.manifest",
69+
)
70+
71+
captured = capsys.readouterr()
72+
assert "error:" in captured.err
73+
assert "example-project" in captured.err

‎bigquery/biglake/noxfile_config.py‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
# Copyright 2026 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# https://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
TEST_CONFIG_OVERRIDE = {
16+
# You can opt out from the test for specific Python versions.
17+
"ignored_versions": ["2.7", "3.7", "3.8", "3.9", "3.10", "3.11"],
18+
# Enforce type hints on new samples.
19+
"enforce_type_hints": True,
20+
"gcloud_project_env": "GOOGLE_CLOUD_PROJECT",
21+
"pip_version_override": None,
22+
"envs": {},
23+
}
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
pytest==9.0.3; python_version >= "3.10"

‎bigquery/biglake/requirements.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
google-cloud-bigquery==3.47.0
2+
google-cloud-bigquery-storage==2.43.0

0 commit comments

Comments
 (0)