From 1802442818ffad640dd429c528848074d685f8b8 Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Mon, 14 Sep 2026 15:44:20 -0400 Subject: [PATCH 1/6] test(bigquery): add ITBigQueryTest integration tests for queryArrow --- .../cloud/bigquery/it/ITBigQueryTest.java | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/java-bigquery/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/java-bigquery/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index 87a9ffcfd238..3e79618a5b41 100644 --- a/java-bigquery/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/java-bigquery/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -45,6 +45,7 @@ import com.google.cloud.bigquery.Acl.DatasetAclEntity; import com.google.cloud.bigquery.Acl.Expr; import com.google.cloud.bigquery.Acl.User; +import com.google.cloud.bigquery.ArrowQueryResult; import com.google.cloud.bigquery.BigQuery; import com.google.cloud.bigquery.BigQuery.DatasetField; import com.google.cloud.bigquery.BigQuery.DatasetListOption; @@ -118,6 +119,7 @@ import com.google.cloud.bigquery.QueryJobConfiguration.JobCreationMode; import com.google.cloud.bigquery.QueryJobConfiguration.Priority; import com.google.cloud.bigquery.QueryParameterValue; +import com.google.cloud.bigquery.QueryResultsFormat; import com.google.cloud.bigquery.Range; import com.google.cloud.bigquery.RangePartitioning; import com.google.cloud.bigquery.Routine; @@ -203,6 +205,7 @@ import java.util.concurrent.TimeoutException; import java.util.logging.Level; import java.util.logging.Logger; +import org.apache.arrow.vector.VectorSchemaRoot; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @@ -7499,6 +7502,50 @@ void testQueryWithTimeout() throws InterruptedException { assertTrue(millis < 1_000_000 * 2); } + @Test + void testQueryResultsFormatArrow() throws InterruptedException { + RemoteBigQueryHelper bigqueryHelper = RemoteBigQueryHelper.create(); + BigQuery bigQuery = bigqueryHelper.getOptions().getService(); + String query = "SELECT 1 as id, 'hello' as name, TIMESTAMP('2026-08-10T12:00:00Z') as ts"; + QueryJobConfiguration config = + QueryJobConfiguration.newBuilder(query) + .setQueryResultsFormat(QueryResultsFormat.ARROW) + .setJobCreationMode(JobCreationMode.JOB_CREATION_OPTIONAL) + .build(); + try (ArrowQueryResult result = bigQuery.queryArrow(config)) { + assertNotNull(result); + int batchCount = 0; + long totalRows = 0; + for (VectorSchemaRoot root : result) { + batchCount++; + totalRows += root.getRowCount(); + assertEquals(1, root.getRowCount()); + } + assertTrue(batchCount > 0); + assertEquals(1, totalRows); + } + } + + @Test + void testQueryResultsFormatArrowMultiPage() throws InterruptedException { + RemoteBigQueryHelper bigqueryHelper = RemoteBigQueryHelper.create(); + BigQuery bigQuery = bigqueryHelper.getOptions().getService(); + String query = "SELECT x FROM UNNEST(GENERATE_ARRAY(1, 15000)) AS x"; + QueryJobConfiguration config = + QueryJobConfiguration.newBuilder(query) + .setQueryResultsFormat(QueryResultsFormat.ARROW) + .setJobCreationMode(JobCreationMode.JOB_CREATION_OPTIONAL) + .build(); + try (ArrowQueryResult result = bigQuery.queryArrow(config)) { + assertNotNull(result); + long totalRows = 0; + for (VectorSchemaRoot root : result) { + totalRows += root.getRowCount(); + } + assertEquals(15000, totalRows); + } + } + @Test void testUniverseDomainWithInvalidUniverseDomain() { RemoteBigQueryHelper bigqueryHelper = RemoteBigQueryHelper.create(); From 39a4b5cf7abf2ce6ff22c318ad39340e069bf98d Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Mon, 14 Sep 2026 17:15:37 -0400 Subject: [PATCH 2/6] test(bigquery): use shared BigQuery instance in Arrow integration tests --- .../java/com/google/cloud/bigquery/it/ITBigQueryTest.java | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/java-bigquery/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/java-bigquery/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index 3e79618a5b41..c752279219a5 100644 --- a/java-bigquery/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/java-bigquery/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -7504,15 +7504,13 @@ void testQueryWithTimeout() throws InterruptedException { @Test void testQueryResultsFormatArrow() throws InterruptedException { - RemoteBigQueryHelper bigqueryHelper = RemoteBigQueryHelper.create(); - BigQuery bigQuery = bigqueryHelper.getOptions().getService(); String query = "SELECT 1 as id, 'hello' as name, TIMESTAMP('2026-08-10T12:00:00Z') as ts"; QueryJobConfiguration config = QueryJobConfiguration.newBuilder(query) .setQueryResultsFormat(QueryResultsFormat.ARROW) .setJobCreationMode(JobCreationMode.JOB_CREATION_OPTIONAL) .build(); - try (ArrowQueryResult result = bigQuery.queryArrow(config)) { + try (ArrowQueryResult result = bigquery.queryArrow(config)) { assertNotNull(result); int batchCount = 0; long totalRows = 0; @@ -7528,15 +7526,13 @@ void testQueryResultsFormatArrow() throws InterruptedException { @Test void testQueryResultsFormatArrowMultiPage() throws InterruptedException { - RemoteBigQueryHelper bigqueryHelper = RemoteBigQueryHelper.create(); - BigQuery bigQuery = bigqueryHelper.getOptions().getService(); String query = "SELECT x FROM UNNEST(GENERATE_ARRAY(1, 15000)) AS x"; QueryJobConfiguration config = QueryJobConfiguration.newBuilder(query) .setQueryResultsFormat(QueryResultsFormat.ARROW) .setJobCreationMode(JobCreationMode.JOB_CREATION_OPTIONAL) .build(); - try (ArrowQueryResult result = bigQuery.queryArrow(config)) { + try (ArrowQueryResult result = bigquery.queryArrow(config)) { assertNotNull(result); long totalRows = 0; for (VectorSchemaRoot root : result) { From 91e6ea94d390970e0f49cb42017bb9e81cd98b1d Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Wed, 16 Sep 2026 21:29:52 -0400 Subject: [PATCH 3/6] test(bigquery): verify batch count in arrow integration tests --- .../java/com/google/cloud/bigquery/it/ITBigQueryTest.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/java-bigquery/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/java-bigquery/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index c752279219a5..36821d7ed83f 100644 --- a/java-bigquery/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/java-bigquery/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -7519,7 +7519,7 @@ void testQueryResultsFormatArrow() throws InterruptedException { totalRows += root.getRowCount(); assertEquals(1, root.getRowCount()); } - assertTrue(batchCount > 0); + assertEquals(1, batchCount); assertEquals(1, totalRows); } } @@ -7534,10 +7534,13 @@ void testQueryResultsFormatArrowMultiPage() throws InterruptedException { .build(); try (ArrowQueryResult result = bigquery.queryArrow(config)) { assertNotNull(result); + int batchCount = 0; long totalRows = 0; for (VectorSchemaRoot root : result) { + batchCount++; totalRows += root.getRowCount(); } + assertTrue(batchCount > 1); assertEquals(15000, totalRows); } } From ac87fc3ce9e26ecdda7560384abe97a8402c6c73 Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Fri, 18 Sep 2026 13:19:03 -0400 Subject: [PATCH 4/6] chore(bigquery): document multi-batch row count expectation in ITBigQueryTest --- .../test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/java-bigquery/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/java-bigquery/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index 36821d7ed83f..1f2344ac4c0d 100644 --- a/java-bigquery/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/java-bigquery/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -7526,6 +7526,8 @@ void testQueryResultsFormatArrow() throws InterruptedException { @Test void testQueryResultsFormatArrowMultiPage() throws InterruptedException { + // The BigQuery REST API and Java SDK default to 10,000 rows per page, and the Storage Read API + // defaults to ~1,024 rows per batch. Querying 15,000 rows ensures multiple Arrow batches. String query = "SELECT x FROM UNNEST(GENERATE_ARRAY(1, 15000)) AS x"; QueryJobConfiguration config = QueryJobConfiguration.newBuilder(query) From 67c5fcf22771e39c7544895bec71d93a9caf77ed Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Fri, 18 Sep 2026 14:16:12 -0400 Subject: [PATCH 5/6] chore(bigquery): set maxResults to enforce multi-batch Arrow streaming in ITBigQueryTest --- .../java/com/google/cloud/bigquery/it/ITBigQueryTest.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/java-bigquery/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/java-bigquery/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index 1f2344ac4c0d..40a3f344f1df 100644 --- a/java-bigquery/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/java-bigquery/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -7526,13 +7526,15 @@ void testQueryResultsFormatArrow() throws InterruptedException { @Test void testQueryResultsFormatArrowMultiPage() throws InterruptedException { - // The BigQuery REST API and Java SDK default to 10,000 rows per page, and the Storage Read API - // defaults to ~1,024 rows per batch. Querying 15,000 rows ensures multiple Arrow batches. + // Under fast-query execution, the initial REST response defaults to a 10 MB payload limit. + // Setting maxResults limits the initial page to 5,000 rows, forcing the remaining 10,000 rows + // to stream across multiple batches via the BigQuery Storage Read API. String query = "SELECT x FROM UNNEST(GENERATE_ARRAY(1, 15000)) AS x"; QueryJobConfiguration config = QueryJobConfiguration.newBuilder(query) .setQueryResultsFormat(QueryResultsFormat.ARROW) .setJobCreationMode(JobCreationMode.JOB_CREATION_OPTIONAL) + .setMaxResults(5000L) .build(); try (ArrowQueryResult result = bigquery.queryArrow(config)) { assertNotNull(result); From 15d4bf0f49b08d7a9e4844ef8ff4fec5194d482d Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Fri, 18 Sep 2026 14:52:50 -0400 Subject: [PATCH 6/6] fix(bigquery): pass NoHeaderProvider to avoid NPE in configureReadSettings --- .../main/java/com/google/cloud/bigquery/BigQueryImpl.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryImpl.java b/java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryImpl.java index 1e76d3f1fa74..9661c1b85025 100644 --- a/java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryImpl.java +++ b/java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryImpl.java @@ -24,8 +24,10 @@ import com.google.api.core.BetaApi; import com.google.api.core.InternalApi; import com.google.api.gax.core.FixedCredentialsProvider; +import com.google.api.gax.core.NoCredentialsProvider; import com.google.api.gax.paging.Page; import com.google.api.gax.rpc.HeaderProvider; +import com.google.api.gax.rpc.NoHeaderProvider; import com.google.api.services.bigquery.model.ErrorProto; import com.google.api.services.bigquery.model.GetQueryResultsResponse; import com.google.api.services.bigquery.model.ProjectList; @@ -371,8 +373,10 @@ private static void configureReadSettings( if (options.getCredentials() != null) { settingsBuilder.setCredentialsProvider( FixedCredentialsProvider.create(options.getCredentials())); + } else { + settingsBuilder.setCredentialsProvider(NoCredentialsProvider.create()); } - HeaderProvider headerProvider = options.getMergedHeaderProvider(null); + HeaderProvider headerProvider = options.getMergedHeaderProvider(new NoHeaderProvider()); if (headerProvider != null) { settingsBuilder.setHeaderProvider(headerProvider); }