diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryDatabaseMetaData.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryDatabaseMetaData.java index e8c3f3353ccc..dbf80fbd43f6 100644 --- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryDatabaseMetaData.java +++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryDatabaseMetaData.java @@ -828,7 +828,8 @@ public ResultSet getProcedures( (rt) -> rt.getRoutineId().getRoutine(), procedureNamePattern, procedureNameRegex, - LOG); + LOG, + false); Future> apiFuture = apiExecutor.submit(apiCallable); apiFutures.add(apiFuture); } @@ -1130,7 +1131,8 @@ List listMatchingProcedureIdsFromDatasets( (rt) -> rt.getRoutineId().getRoutine(), procedureNamePattern, procedureNameRegex, - logger); + logger, + false); listRoutineFutures.add(listRoutinesExecutor.submit(listCallable)); } logger.fine( @@ -1629,7 +1631,8 @@ public ResultSet getTables( (tbl) -> tbl.getTableId().getTable(), tableNamePattern, tableNameRegex, - LOG); + LOG, + false); Future> apiFuture = apiExecutor.submit(apiCallable); apiFutures.add(apiFuture); } @@ -1943,7 +1946,8 @@ public ResultSet getColumns( (tbl) -> tbl.getTableId().getTable(), tableNamePattern, tableNameRegex, - LOG); + LOG, + false); for (Table table : tablesToScan) { if (Thread.currentThread().isInterrupted()) { @@ -2420,13 +2424,11 @@ public ResultSet getPrimaryKeys(String catalog, String schema, String table) thr final List collectedResults = Collections.synchronizedList(new ArrayList<>()); List targetDatasets = getTargetDatasets(catalog, schema); - boolean ignoreAccessErrors = (catalog == null); processTargetTablesConcurrently( targetDatasets, table, collectedResults, resultSchemaFields, - ignoreAccessErrors, (bqTable, results, fields) -> { TableConstraints constraints = bqTable.getTableConstraints(); processPrimaryKey(constraints, bqTable.getTableId(), results, fields); @@ -2517,13 +2519,11 @@ public ResultSet getImportedKeys(String catalog, String schema, String table) final List collectedResults = Collections.synchronizedList(new ArrayList<>()); List targetDatasets = getTargetDatasets(catalog, schema); - boolean ignoreAccessErrors = (catalog == null); processTargetTablesConcurrently( targetDatasets, table, collectedResults, resultSchemaFields, - ignoreAccessErrors, (bqTable, results, fields) -> { TableConstraints constraints = bqTable.getTableConstraints(); if (constraints == null || constraints.getForeignKeys() == null) { @@ -2562,13 +2562,11 @@ public ResultSet getExportedKeys(String catalog, String schema, String table) final List collectedResults = Collections.synchronizedList(new ArrayList<>()); List targetDatasets = getTargetDatasets(catalog, null); - boolean ignoreAccessErrors = (catalog == null); processTargetTablesConcurrently( targetDatasets, null, collectedResults, resultSchemaFields, - ignoreAccessErrors, (bqTable, results, fields) -> { TableConstraints constraints = bqTable.getTableConstraints(); if (constraints == null || constraints.getForeignKeys() == null) { @@ -2623,13 +2621,11 @@ public ResultSet getCrossReference( final List collectedResults = Collections.synchronizedList(new ArrayList<>()); List targetDatasets = getTargetDatasets(foreignCatalog, foreignSchema); - boolean ignoreAccessErrors = (foreignCatalog == null); processTargetTablesConcurrently( targetDatasets, foreignTable, collectedResults, resultSchemaFields, - ignoreAccessErrors, (bqTable, results, fields) -> { TableConstraints constraints = bqTable.getTableConstraints(); if (constraints == null || constraints.getForeignKeys() == null) { @@ -3836,7 +3832,8 @@ public ResultSet getFunctions(String catalog, String schemaPattern, String funct (rt) -> rt.getRoutineId().getRoutine(), functionNamePattern, functionNameRegex, - LOG); + LOG, + false); }; Future> apiFuture = apiExecutor.submit(apiCallable); apiFutures.add(apiFuture); @@ -4187,7 +4184,8 @@ List listMatchingFunctionIdsFromDatasets( (rt) -> rt.getRoutineId().getRoutine(), functionNamePattern, functionNameRegex, - logger); + logger, + false); listRoutineFutures.add(listRoutinesExecutor.submit(listCallable)); } logger.fine( @@ -4623,7 +4621,8 @@ List findMatchingBigQueryObjects( Function nameExtractor, String pattern, Pattern regex, - BigQueryJdbcCustomLogger logger) + BigQueryJdbcCustomLogger logger, + boolean throwOn404) throws InterruptedException { boolean needsList = needsListing(pattern); @@ -4672,8 +4671,9 @@ List findMatchingBigQueryObjects( } } catch (BigQueryException e) { - if (!needsList && e.getCode() == 404) { + if (e.getCode() == 404 && !throwOn404) { logger.info("%s '%s' not found (API error 404).", objectTypeName, pattern); + return Collections.emptyList(); } else { logger.warning( "BigQueryException finding %ss for pattern '%s': %s (Code: %d)", @@ -4989,7 +4989,8 @@ private void signalEndOfData( } private List fetchDatasetsForProject( - String project, String schemaPattern, Pattern schemaRegex) throws SQLException { + String project, String schemaPattern, Pattern schemaRegex, boolean throwOn404) + throws SQLException { try { List datasets = findMatchingBigQueryObjects( @@ -4999,7 +5000,8 @@ private List fetchDatasetsForProject( (ds) -> ds.getDatasetId().getDataset(), schemaPattern, schemaRegex, - LOG); + LOG, + throwOn404); return datasets != null ? datasets : Collections.emptyList(); } catch (InterruptedException e) { Thread.currentThread().interrupt(); @@ -5019,9 +5021,11 @@ private List fetchMatchingDatasets( return Collections.emptyList(); } + boolean isBroadScan = (catalog == null); + // Single project path if (projects.size() == 1) { - return fetchDatasetsForProject(projects.get(0), schemaPattern, schemaRegex); + return fetchDatasetsForProject(projects.get(0), schemaPattern, schemaRegex, isBroadScan); } // Multi-project path @@ -5033,7 +5037,8 @@ private List fetchMatchingDatasets( for (String project : projects) { Callable task = () -> { - List datasets = fetchDatasetsForProject(project, schemaPattern, schemaRegex); + List datasets = + fetchDatasetsForProject(project, schemaPattern, schemaRegex, isBroadScan); allDatasets.addAll(datasets); return null; }; @@ -5146,7 +5151,6 @@ private void processSingleTable( String tableName, List collectedResults, FieldList resultSchemaFields, - boolean ignoreAccessErrors, TableProcessor processor) throws SQLException { Table bqTable; @@ -5154,10 +5158,10 @@ private void processSingleTable( bqTable = bigquery.getTable(TableId.of(datasetId.getProject(), datasetId.getDataset(), tableName)); } catch (BigQueryException e) { - if (ignoreAccessErrors && (e.getCode() == 404 || e.getCode() == 403)) { + if (e.getCode() == 404) { LOG.info( - "Table '%s' or dataset '%s' not found/accessible in project '%s' (API error %d). Skipping.", - tableName, datasetId.getDataset(), datasetId.getProject(), e.getCode()); + "Table '%s' or dataset '%s' not found in project '%s' (API error 404). Skipping.", + tableName, datasetId.getDataset(), datasetId.getProject()); bqTable = null; } else { throw new SQLException("Error while fetching table metadata: " + e.getMessage(), e); @@ -5175,17 +5179,11 @@ private void processTargetTablesConcurrently( String tableName, List collectedResults, FieldList resultSchemaFields, - boolean ignoreAccessErrors, TableProcessor processor) throws SQLException { if (targetDatasets.size() == 1 && tableName != null) { processSingleTable( - targetDatasets.get(0), - tableName, - collectedResults, - resultSchemaFields, - ignoreAccessErrors, - processor); + targetDatasets.get(0), tableName, collectedResults, resultSchemaFields, processor); return; } @@ -5199,12 +5197,7 @@ private void processTargetTablesConcurrently( tasks.add( () -> { processSingleTable( - datasetId, - tableName, - collectedResults, - resultSchemaFields, - ignoreAccessErrors, - processor); + datasetId, tableName, collectedResults, resultSchemaFields, processor); return null; }); continue; @@ -5228,16 +5221,15 @@ private void processTargetTablesConcurrently( table.getTableId().getTable(), collectedResults, resultSchemaFields, - ignoreAccessErrors, processor); return null; }); } } catch (BigQueryException e) { - if (ignoreAccessErrors && (e.getCode() == 404 || e.getCode() == 403)) { + if (e.getCode() == 404) { LOG.info( - "Dataset '%s' not found/accessible in project '%s' (API error %d). Skipping.", - datasetId.getDataset(), datasetId.getProject(), e.getCode()); + "Dataset '%s' not found in project '%s' (API error 404). Skipping.", + datasetId.getDataset(), datasetId.getProject()); continue; } throw new SQLException("Error while listing tables: " + e.getMessage(), e); diff --git a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryDatabaseMetaDataTest.java b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryDatabaseMetaDataTest.java index 7264b3349772..47bd0b2ff207 100644 --- a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryDatabaseMetaDataTest.java +++ b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryDatabaseMetaDataTest.java @@ -1027,7 +1027,8 @@ public void testFindMatchingBigQueryObjects_Routines_ListWithPattern() throws Ex (rt) -> rt.getRoutineId().getRoutine(), pattern, regex, - dbMetadata.LOG); + dbMetadata.LOG, + false); verify(bigqueryClient, times(1)) .listRoutines(eq(datasetId), any(BigQuery.RoutineListOption[].class)); @@ -1069,7 +1070,8 @@ public void testFindMatchingBigQueryObjects_Routines_ListNoPattern() throws Exce (rt) -> rt.getRoutineId().getRoutine(), pattern, regex, - dbMetadata.LOG); + dbMetadata.LOG, + false); verify(bigqueryClient, times(1)) .listRoutines(eq(datasetId), any(BigQuery.RoutineListOption[].class)); @@ -1104,7 +1106,8 @@ public void testFindMatchingBigQueryObjects_Routines_GetSpecific() throws Except (rt) -> rt.getRoutineId().getRoutine(), procNameExact, regex, - dbMetadata.LOG); + dbMetadata.LOG, + false); verify(bigqueryClient, times(1)).getRoutine(eq(routineId)); verify(bigqueryClient, never()) @@ -1116,6 +1119,58 @@ public void testFindMatchingBigQueryObjects_Routines_GetSpecific() throws Except assertSame(mockRoutine, resultList.get(0)); } + private List invokeFindMatchingObjectsWithException( + BigQueryException bqe, String pattern, boolean throwOn404) throws Exception { + return dbMetadata.findMatchingBigQueryObjects( + "Table", + () -> { + throw bqe; + }, + (name) -> { + throw bqe; + }, + (table) -> "name", + pattern, + dbMetadata.compileSqlLikePattern(pattern), + dbMetadata.LOG, + throwOn404); + } + + @Test + public void testFindMatchingBigQueryObjects_Swallows404_TargetedScan() throws Exception { + List
results = + invokeFindMatchingObjectsWithException( + new BigQueryException(404, "Not Found"), "exact_match", false); + assertTrue(results.isEmpty()); + } + + @Test + public void testFindMatchingBigQueryObjects_Throws404_BroadScan() { + assertThrows( + BigQueryException.class, + () -> + invokeFindMatchingObjectsWithException( + new BigQueryException(404, "Not Found"), "%", true)); + } + + @Test + public void testFindMatchingBigQueryObjects_Throws403_BroadScan() { + assertThrows( + BigQueryException.class, + () -> + invokeFindMatchingObjectsWithException( + new BigQueryException(403, "Access Denied"), "%", true)); + } + + @Test + public void testFindMatchingBigQueryObjects_Throws403_TargetedScan() { + assertThrows( + BigQueryException.class, + () -> + invokeFindMatchingObjectsWithException( + new BigQueryException(403, "Access Denied"), "exact_match", false)); + } + @Test public void testDefineGetProcedureColumnsSchema() { Schema schema = dbMetadata.defineGetProcedureColumnsSchema(); diff --git a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/it/ITDatabaseMetadataTest.java b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/it/ITDatabaseMetadataTest.java index 046f49991dbe..cfba5f2f81e8 100644 --- a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/it/ITDatabaseMetadataTest.java +++ b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/it/ITDatabaseMetadataTest.java @@ -850,10 +850,9 @@ public void testDatabaseMetadataGetSchemas() throws SQLException { Assertions.assertFalse(rsNoMatch.next()); // Test case 4: Get schemas with non-existent catalog - Assertions.assertThrows( - SQLException.class, - () -> databaseMetaData.getSchemas("invalid-catalog", null), - "Should throw SQLException for non-existent catalog"); + ResultSet rsInvalid = databaseMetaData.getSchemas("invalid-catalog", null); + Assertions.assertFalse( + rsInvalid.next(), "Should return empty ResultSet for non-existent catalog"); connection.close(); }