diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/auth/AuthorityChecker.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/auth/AuthorityChecker.java index 0593db10840ba..951444d20283c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/auth/AuthorityChecker.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/auth/AuthorityChecker.java @@ -131,13 +131,15 @@ public static IAuthorityFetcher getAuthorityFetcher() { } public static boolean invalidateCache(String username, String roleName) { + final boolean invalidated = + authorityFetcher.get().getAuthorCache().invalidateCache(username, roleName); PipeInsertionDataNodeListener.getInstance().invalidateAllCache(); - return authorityFetcher.get().getAuthorCache().invalidateCache(username, roleName); + return invalidated; } public static void invalidateAllCache() { - PipeInsertionDataNodeListener.getInstance().invalidateAllCache(); authorityFetcher.get().getAuthorCache().invalidAllCache(); + PipeInsertionDataNodeListener.getInstance().invalidateAllCache(); } public static User getUser(String username) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/matcher/CachedSchemaPatternMatcher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/matcher/CachedSchemaPatternMatcher.java index e9d2e98c5d0e6..d8f427868f04e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/matcher/CachedSchemaPatternMatcher.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/matcher/CachedSchemaPatternMatcher.java @@ -61,7 +61,7 @@ public class CachedSchemaPatternMatcher implements PipeDataRegionMatcher { // Use full cache to avoid queue stuck and block insertion protected final Map> deviceToSourcesCache; - protected final Map, Set> + protected final Map, Set> databaseAndTableToSourcesCache; public CachedSchemaPatternMatcher() { @@ -102,7 +102,7 @@ public void deregister(final PipeRealtimeDataRegionSource source) { public void invalidateCache() { lock.writeLock().lock(); try { - // Will invalidate device cache + // The table-model cache also depends on access control, so it must be invalidated separately. databaseAndTableToSourcesCache.clear(); } finally { lock.writeLock().unlock(); @@ -144,6 +144,11 @@ public Pair, Set return new Pair<>(matchedSources, findUnmatchedSources(matchedSources)); } + // tableNames is also used for privilege checks on table-model TsFile events, so it must be + // complete even after every source has already matched. + final boolean isTableModelTsFileEvent = + event.getEvent() instanceof PipeTsFileInsertionEvent + && ((PipeTsFileInsertionEvent) event.getEvent()).isTableModelEvent(); final Set tableNames = new HashSet<>(); for (final Map.Entry entry : event.getSchemaInfo().entrySet()) { final IDeviceID deviceID = entry.getKey(); @@ -154,16 +159,17 @@ public Pair, Set || deviceID.getTableName().equals(PATH_ROOT)) { matchTreeModelEvent(deviceID, entry.getValue(), matchedSources); } else { - tableNames.add(deviceID.getTableName()); - matchTableModelEvent( - event.getEvent() instanceof PipeInsertionEvent - ? ((PipeInsertionEvent) event.getEvent()).getTableModelDatabaseName() - : null, - deviceID, - matchedSources); + final String tableName = deviceID.getTableName(); + if (tableNames.add(tableName) && matchedSources.size() < sources.size()) { + final String tableModelDatabaseName = + event.getEvent() instanceof PipeInsertionEvent + ? ((PipeInsertionEvent) event.getEvent()).getTableModelDatabaseName() + : null; + matchTableModelEvent(tableModelDatabaseName, tableName, matchedSources); + } } - if (matchedSources.size() == sources.size()) { + if (matchedSources.size() == sources.size() && !isTableModelTsFileEvent) { break; } } @@ -171,7 +177,7 @@ public Pair, Set if (event.getEvent() instanceof PipeTsFileInsertionEvent) { final PipeTsFileInsertionEvent tsFileInsertionEvent = (PipeTsFileInsertionEvent) event.getEvent(); - if (tsFileInsertionEvent.isTableModelEvent()) { + if (isTableModelTsFileEvent) { tsFileInsertionEvent.setTableNames(tableNames); } else { tsFileInsertionEvent.setTreeSchemaMap(event.getSchemaInfo()); @@ -273,7 +279,7 @@ protected Set filterSourcesByDevice(final IDeviceI protected void matchTableModelEvent( final String databaseName, - final IDeviceID tableName, + final String tableName, final Set matchedSources) { // this would not happen if (databaseName == null) { @@ -294,7 +300,7 @@ protected void matchTableModelEvent( } protected Set filterSourcesByDatabaseAndTable( - final Pair databaseNameAndTableName) { + final Pair databaseNameAndTableName) { final Set filteredSources = new HashSet<>(); for (final PipeRealtimeDataRegionSource source : sources) { @@ -317,21 +323,20 @@ protected Set filterSourcesByDatabaseAndTable( } private boolean matchesTablePattern( - final TablePattern tablePattern, final Pair databaseNameAndTableName) { + final TablePattern tablePattern, final Pair databaseNameAndTableName) { return Objects.isNull(tablePattern) || (tablePattern.isTableModelDataAllowedToBeCaptured() && tablePattern.matchesDatabase(databaseNameAndTableName.getLeft()) - && tablePattern.matchesTable(databaseNameAndTableName.getRight().getTableName())); + && tablePattern.matchesTable(databaseNameAndTableName.getRight())); } private boolean notFilteredByAccess( - final UserEntity userEntity, final Pair databaseNameAndTableName) { + final UserEntity userEntity, final Pair databaseNameAndTableName) { return AuthorityChecker.getAccessControl() .checkCanSelectFromTable4Pipe( userEntity.getUsername(), new QualifiedObjectName( - databaseNameAndTableName.getLeft(), - databaseNameAndTableName.getRight().getTableName()), + databaseNameAndTableName.getLeft(), databaseNameAndTableName.getRight()), userEntity); } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/pattern/CachedSchemaPatternMatcherTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/pattern/CachedSchemaPatternMatcherTest.java index 61d5232a3d858..b9d7ca5db4c25 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/pattern/CachedSchemaPatternMatcherTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/pattern/CachedSchemaPatternMatcherTest.java @@ -25,6 +25,8 @@ import org.apache.iotdb.commons.pipe.datastructure.pattern.PrefixTreePattern; import org.apache.iotdb.commons.pipe.event.EnrichedEvent; import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.pipe.event.common.PipeInsertionEvent; +import org.apache.iotdb.db.pipe.event.common.tsfile.PipeTsFileInsertionEvent; import org.apache.iotdb.db.pipe.event.realtime.PipeRealtimeEvent; import org.apache.iotdb.db.pipe.source.dataregion.realtime.PipeRealtimeDataRegionSource; import org.apache.iotdb.db.pipe.source.dataregion.realtime.epoch.TsFileEpoch; @@ -39,12 +41,18 @@ import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.Mockito; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; @@ -73,6 +81,25 @@ public boolean shouldParsePattern() { } } + private static class CountingCachedSchemaPatternMatcher extends CachedSchemaPatternMatcher { + + private int tableMatchCount; + + @Override + protected void matchTableModelEvent( + final String databaseName, + final String tableName, + final Set matchedSources) { + ++tableMatchCount; + // Simulate a successful table-level match so this test focuses on match orchestration. + matchedSources.addAll(sources); + } + + private int getTableMatchCount() { + return tableMatchCount; + } + } + private CachedSchemaPatternMatcher matcher; private ExecutorService executorService; private List extractors; @@ -178,6 +205,51 @@ public void testCachedMatcher() throws Exception { future.get(); } + @Test + public void testTableModelMatchesEachTableOncePerEvent() throws Exception { + final CountingCachedSchemaPatternMatcher countingMatcher = + new CountingCachedSchemaPatternMatcher(); + final PipeRealtimeDataRegionSource source = new PipeRealtimeDataRegionFakeSource(); + countingMatcher.register(source); + + final PipeInsertionEvent insertionEvent = Mockito.mock(PipeInsertionEvent.class); + Mockito.when(insertionEvent.getTableModelDatabaseName()).thenReturn("db"); + final Map schemaInfo = new LinkedHashMap<>(); + schemaInfo.put(new StringArrayDeviceID("table1", "tag1"), new String[0]); + schemaInfo.put(new StringArrayDeviceID("table1", "tag2"), new String[0]); + + Assert.assertTrue( + countingMatcher + .match(new MockedPipeRealtimeEvent(insertionEvent, null, schemaInfo)) + .getLeft() + .contains(source)); + Assert.assertEquals(1, countingMatcher.getTableMatchCount()); + } + + @Test + public void testMultiTableTsFileCollectsAllTableNamesAfterAllSourcesMatched() throws Exception { + final CountingCachedSchemaPatternMatcher countingMatcher = + new CountingCachedSchemaPatternMatcher(); + final PipeRealtimeDataRegionSource source = new PipeRealtimeDataRegionFakeSource(); + countingMatcher.register(source); + + final PipeTsFileInsertionEvent tsFileInsertionEvent = + Mockito.mock(PipeTsFileInsertionEvent.class); + Mockito.when(tsFileInsertionEvent.isTableModelEvent()).thenReturn(true); + Mockito.when(tsFileInsertionEvent.getTableModelDatabaseName()).thenReturn("db"); + final Map schemaInfo = new LinkedHashMap<>(); + schemaInfo.put(new StringArrayDeviceID("table1", "tag1"), new String[0]); + schemaInfo.put(new StringArrayDeviceID("table2", "tag2"), new String[0]); + + countingMatcher.match(new MockedPipeRealtimeEvent(tsFileInsertionEvent, null, schemaInfo)); + + final ArgumentCaptor> tableNamesCaptor = ArgumentCaptor.forClass(Set.class); + Mockito.verify(tsFileInsertionEvent).setTableNames(tableNamesCaptor.capture()); + Assert.assertEquals( + new HashSet<>(Arrays.asList("table1", "table2")), tableNamesCaptor.getValue()); + Assert.assertEquals(1, countingMatcher.getTableMatchCount()); + } + public static class PipeRealtimeDataRegionFakeSource extends PipeRealtimeDataRegionSource { public PipeRealtimeDataRegionFakeSource() {