diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/consensus/local/tablemodel/ConsensusSubscriptionTableITSupport.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/consensus/local/tablemodel/ConsensusSubscriptionTableITSupport.java index 29a859d3011ee..6f4e828fb9c88 100644 --- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/consensus/local/tablemodel/ConsensusSubscriptionTableITSupport.java +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/consensus/local/tablemodel/ConsensusSubscriptionTableITSupport.java @@ -125,6 +125,16 @@ static void createConsensusTopic( final String tablePattern, final String columnFilter) throws Exception { + createConsensusTopic(topicName, databasePattern, tablePattern, columnFilter, null); + } + + static void createConsensusTopic( + final String topicName, + final String databasePattern, + final String tablePattern, + final String columnFilter, + final String tagFilter) + throws Exception { final String host = EnvFactory.getEnv().getIP(); final int port = Integer.parseInt(EnvFactory.getEnv().getPort()); @@ -141,6 +151,9 @@ static void createConsensusTopic( if (columnFilter != null) { config.put(TopicConstant.COLUMN_FILTER_KEY, columnFilter); } + if (tagFilter != null) { + config.put(TopicConstant.TAG_FILTER_KEY, tagFilter); + } session.createTopic(topicName, config); } } @@ -160,6 +173,21 @@ static void alterConsensusTopicColumnFilter(final String topicName, final String } } + static void alterConsensusTopicTagFilter(final String topicName, final String tagFilter) + throws Exception { + final String host = EnvFactory.getEnv().getIP(); + final int port = Integer.parseInt(EnvFactory.getEnv().getPort()); + + try (final ISubscriptionTableSession session = + new SubscriptionTableSessionBuilder().host(host).port(port).build()) { + session.open(); + + final Properties config = new Properties(); + config.put(TopicConstant.TAG_FILTER_KEY, tagFilter); + session.alterTopic(topicName, config); + } + } + static SubscriptionTablePullConsumer createConsumer( final String consumerId, final String consumerGroupId) throws Exception { final SubscriptionTablePullConsumer consumer = diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/consensus/local/tablemodel/IoTDBConsensusSubscriptionFilterTableIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/consensus/local/tablemodel/IoTDBConsensusSubscriptionFilterTableIT.java index 96193a005587d..f311d00406a9c 100644 --- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/consensus/local/tablemodel/IoTDBConsensusSubscriptionFilterTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/consensus/local/tablemodel/IoTDBConsensusSubscriptionFilterTableIT.java @@ -282,6 +282,80 @@ public void testColumnFilteringWithNoMatchedColumnsReturnsNothing() throws Excep } } + @Test + public void testTagFilteringAndAlterInConsensusQueue() throws Exception { + final ConsensusSubscriptionTableITSupport.TestIdentifiers ids = + ConsensusSubscriptionTableITSupport.newIdentifiers("table_filter_tags"); + final String database = ids.getDatabase(); + final String table = "t1"; + final String schema = "tag1 STRING TAG, s1 INT64 FIELD, s2 DOUBLE FIELD"; + final String expectedColumnSignature = + ConsensusSubscriptionTableITSupport.normalizeColumnSignature("tag1", "s2"); + SubscriptionTablePullConsumer consumer = null; + + try { + ConsensusSubscriptionTableITSupport.createDatabaseAndTable(database, table, schema); + try (final ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("use " + database); + session.executeNonQueryStatement( + "insert into " + table + "(tag1, s1, s2, time) values ('bootstrap', 0, 0.0, 0)"); + session.executeNonQueryStatement("flush"); + } + + ConsensusSubscriptionTableITSupport.createConsensusTopic( + ids.getTopic(), database, table, "column_name = \"s2\"", "tag1 = \"keep\""); + + consumer = + ConsensusSubscriptionTableITSupport.createConsumer( + ids.getConsumerId(), ids.getConsumerGroupId()); + consumer.subscribe(ids.getTopic()); + + final Set expectedBeforeAlter = new LinkedHashSet<>(); + try (final ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("use " + database); + for (int i = 1; i <= 5; i++) { + final long timestamp = 100L + i; + final String tag = i % 2 == 1 ? "keep" : "drop"; + session.executeNonQueryStatement( + String.format( + Locale.ROOT, + "insert into %s(tag1, s1, s2, time) values ('%s', %d, %.1f, %d)", + table, + tag, + i * 10L, + i + 0.5d, + timestamp)); + if ("keep".equals(tag)) { + expectedBeforeAlter.add( + ConsensusSubscriptionTableITSupport.rowKey(database, table, timestamp)); + } + } + session.executeNonQueryStatement("flush"); + } + + final ConsensusSubscriptionTableITSupport.ConsumedRecords consumedBeforeAlter = + ConsensusSubscriptionTableITSupport.pollAndCommitUntilContains( + consumer, expectedBeforeAlter, 50); + ConsensusSubscriptionTableITSupport.assertExactRowKeys( + expectedBeforeAlter, consumedBeforeAlter); + Assert.assertEquals( + Collections.singleton(expectedColumnSignature), + consumedBeforeAlter.getSeenColumnSignatures()); + + ConsensusSubscriptionTableITSupport.alterConsensusTopicTagFilter( + ids.getTopic(), "tag1 LIKE \"t1_tag_%\""); + final ConsensusSubscriptionTableITSupport.ConsumedRecords consumedAfterAlter = + ConsensusSubscriptionTableITSupport.insertRowsAndPollUntilColumnSignature( + consumer, database, table, 200L, 3, true, false, expectedColumnSignature, 60); + Assert.assertTrue(consumedAfterAlter.getRowCount() > 0); + Assert.assertTrue( + consumedAfterAlter.getSeenColumnSignatures().contains(expectedColumnSignature)); + ConsensusSubscriptionTableITSupport.assertNoMoreMessages(consumer, 3, Duration.ofMillis(500)); + } finally { + ConsensusSubscriptionTableITSupport.cleanup(consumer, ids.getTopic(), database); + } + } + @Test public void testColumnFilteringComplexOperatorsInConsensusQueue() throws Exception { final ConsensusSubscriptionTableITSupport.TestIdentifiers ids = diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/tablemodel/IoTDBSubscriptionColumnFilterIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/tablemodel/IoTDBSubscriptionColumnFilterIT.java index bfeb7e6b7aa7b..15dff38564824 100644 --- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/tablemodel/IoTDBSubscriptionColumnFilterIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/tablemodel/IoTDBSubscriptionColumnFilterIT.java @@ -452,6 +452,73 @@ public void testLiveTsFileColumnFilter() throws Exception { } } + @Test + public void testLiveRecordHandlerTagFilterBeforeColumnFilter() throws Exception { + final String database = databaseName("tag_record"); + final String topicName = topicName("tag_record"); + final String consumerId = consumerName("tag_record"); + final String consumerGroupId = consumerGroupName("tag_record"); + + try { + createDatabaseAndTable(senderEnv, database, TABLE_NAME, TABLE_SCHEMA); + createTopicWithFilters( + topicName, + database, + TABLE_NAME, + TopicConstant.MODE_INITIAL_VALUE, + TopicConstant.FORMAT_RECORD_HANDLER_VALUE, + COLUMN_FILTER, + "tag1 IN (\"tag_1\", \"tag_3\")"); + + try (final ISubscriptionTablePullConsumer consumer = + createConsumer(consumerId, consumerGroupId)) { + consumer.subscribe(topicName); + insertRows(senderEnv, database, TABLE_NAME, 1, 5); + + final Set expectedTimestamps = new LinkedHashSet<>(Arrays.asList(1L, 3L)); + final ConsumedRecordStats stats = + pollRecordMessagesForTimestamps(consumer, expectedTimestamps, true); + + Assert.assertEquals(expectedTimestamps, stats.timestamps); + Assert.assertEquals(EXPECTED_COLUMNS, stats.columnNames); + } + } finally { + cleanup(topicName, database); + } + } + + @Test + public void testSnapshotTsFileTagFilter() throws Exception { + final String database = databaseName("tag_snapshot_tsfile"); + final String topicName = topicName("tag_snapshot_tsfile"); + final String consumerId = consumerName("tag_snapshot_tsfile"); + final String consumerGroupId = consumerGroupName("tag_snapshot_tsfile"); + + try { + createDatabaseAndTable(senderEnv, database, TABLE_NAME, TABLE_SCHEMA); + createDatabaseAndTable(receiverEnv, database, TABLE_NAME, TABLE_SCHEMA); + insertRows(senderEnv, database, TABLE_NAME, 10, 14); + createTopicWithFilters( + topicName, + database, + TABLE_NAME, + TopicConstant.MODE_SNAPSHOT_VALUE, + TopicConstant.FORMAT_TS_FILE_VALUE, + TopicConstant.COLUMN_FILTER_DEFAULT_VALUE, + "tag1 IN (\"tag_10\", \"tag_12\")"); + + try (final ISubscriptionTablePullConsumer consumer = + createConsumer(consumerId, consumerGroupId)) { + consumer.subscribe(topicName); + pollTsFileMessagesAndLoad(consumer, database, 2); + } + + assertLoadedTimestamps(database, new LinkedHashSet<>(Arrays.asList(10L, 12L))); + } finally { + cleanup(topicName, database); + } + } + @Test public void testSnapshotTsFileColumnFilter() throws Exception { final String database = databaseName("snapshot_tsfile"); @@ -1036,6 +1103,34 @@ private void createTopicWithoutColumnFilter( topicName, database, tableName, TopicConstant.MODE_INITIAL_VALUE, format, "", false); } + private void createTopicWithFilters( + final String topicName, + final String database, + final String tableName, + final String mode, + final String format, + final String columnFilter, + final String tagFilter) + throws Exception { + try (final ISubscriptionTableSession session = + new SubscriptionTableSessionBuilder() + .host(senderEnv.getIP()) + .port(Integer.parseInt(senderEnv.getPort())) + .build()) { + session.open(); + session.dropTopicIfExists(topicName); + + final Properties config = new Properties(); + config.put(TopicConstant.MODE_KEY, mode); + config.put(TopicConstant.FORMAT_KEY, format); + config.put(TopicConstant.DATABASE_KEY, database); + config.put(TopicConstant.TABLE_KEY, tableName); + config.put(TopicConstant.COLUMN_FILTER_KEY, columnFilter); + config.put(TopicConstant.TAG_FILTER_KEY, tagFilter); + session.createTopic(topicName, config); + } + } + private void createTopic( final String topicName, final String database, @@ -1455,6 +1550,22 @@ private void assertLoadedTsFileRowsWithAllColumns(final String database, final i } } + private void assertLoadedTimestamps(final String database, final Set expectedTimestamps) + throws Exception { + final Set actualTimestamps = new LinkedHashSet<>(); + try (final Connection connection = receiverEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute("use " + database); + try (final java.sql.ResultSet resultSet = + statement.executeQuery("select time from " + TABLE_NAME + " order by time")) { + while (resultSet.next()) { + actualTimestamps.add(resultSet.getLong("time")); + } + } + } + Assert.assertEquals(expectedTimestamps, actualTimestamps); + } + private void cleanup(final String topicName, final String database) { try (final ISubscriptionTableSession session = new SubscriptionTableSessionBuilder() diff --git a/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/config/TopicConfig.java b/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/config/TopicConfig.java index 3d107476b85b1..76b86a6240ed6 100644 --- a/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/config/TopicConfig.java +++ b/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/config/TopicConfig.java @@ -235,6 +235,23 @@ public boolean isColumnFilterTrivial() { return TopicConstant.COLUMN_FILTER_DEFAULT_VALUE.equalsIgnoreCase(getColumnFilter().trim()); } + public Map getAttributesWithSourceTagFilter() { + return Collections.singletonMap(TopicConstant.TAG_FILTER_KEY, getTagFilter()); + } + + public String getTagFilter() { + return getStringIgnoreCase( + TopicConstant.TAG_FILTER_KEY, TopicConstant.TAG_FILTER_DEFAULT_VALUE); + } + + public boolean hasTagFilter() { + return containsKeyIgnoreCase(TopicConstant.TAG_FILTER_KEY); + } + + public boolean isTagFilterTrivial() { + return TopicConstant.TAG_FILTER_DEFAULT_VALUE.equalsIgnoreCase(getTagFilter().trim()); + } + public Map getAttributesWithSourceTimeRange() { final Map attributesWithTimeRange = new HashMap<>(); diff --git a/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/config/TopicConstant.java b/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/config/TopicConstant.java index 4d11f2fe8a27a..27355454494ed 100644 --- a/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/config/TopicConstant.java +++ b/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/config/TopicConstant.java @@ -31,11 +31,13 @@ public class TopicConstant { public static final String DATABASE_KEY = "database"; public static final String TABLE_KEY = "table"; public static final String COLUMN_FILTER_KEY = "column-filter"; + public static final String TAG_FILTER_KEY = "tag-filter"; public static final String RETENTION_BYTES_KEY = "retention.bytes"; public static final String RETENTION_MS_KEY = "retention.ms"; public static final String DATABASE_DEFAULT_VALUE = ".*"; public static final String TABLE_DEFAULT_VALUE = ".*"; public static final String COLUMN_FILTER_DEFAULT_VALUE = "true"; + public static final String TAG_FILTER_DEFAULT_VALUE = "true"; public static final String START_TIME_KEY = "start-time"; public static final String END_TIME_KEY = "end-time"; diff --git a/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/AbstractSubscriptionSession.java b/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/AbstractSubscriptionSession.java index c09ebdfab657c..e4b1e1c76a561 100644 --- a/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/AbstractSubscriptionSession.java +++ b/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/AbstractSubscriptionSession.java @@ -241,7 +241,7 @@ private Set convertDataSetToTopics(final SessionDataSet dataSet) while (dataSet.hasNext()) { final RowRecord record = dataSet.next(); final List fields = record.getFields(); - if (fields.size() != 2) { + if (fields.size() < 2) { throw new SubscriptionException( String.format( SubscriptionMessages diff --git a/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionConsumer.java b/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionConsumer.java index 38b16ed7cf90f..2bf6bf426e392 100644 --- a/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionConsumer.java +++ b/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionConsumer.java @@ -1180,7 +1180,8 @@ private Optional pollTablets( } private Optional pollTabletsInternal( - final SubscriptionPollResponse initialResponse, final PollTimer timer) { + final SubscriptionPollResponse initialResponse, final PollTimer timer) + throws SubscriptionException { final Map> tablets = ((TabletsPayload) initialResponse.getPayload()).getTabletsWithDBInfo(); final SubscriptionCommitContext commitContext = initialResponse.getCommitContext(); @@ -1200,6 +1201,10 @@ private Optional pollTabletsInternal( LOGGER.warn(errorMessage); throw new SubscriptionRuntimeNonCriticalException(errorMessage); } + if (tabletsSize == 0) { + ack(Collections.singletonList(new SubscriptionMessage(commitContext, tablets))); + return Optional.empty(); + } return Optional.of( new SubscriptionMessage(commitContext, tablets, timeSelected, timeSelectedByTable)); } diff --git a/iotdb-client/subscription/src/test/java/org/apache/iotdb/rpc/subscription/config/TopicConfigTest.java b/iotdb-client/subscription/src/test/java/org/apache/iotdb/rpc/subscription/config/TopicConfigTest.java index d308ef123cf68..64a1272fd4f27 100644 --- a/iotdb-client/subscription/src/test/java/org/apache/iotdb/rpc/subscription/config/TopicConfigTest.java +++ b/iotdb-client/subscription/src/test/java/org/apache/iotdb/rpc/subscription/config/TopicConfigTest.java @@ -110,6 +110,35 @@ public void testColumnFilterTrivialWithMixedCaseKeyAndValue() { Assert.assertTrue(new TopicConfig(attributes).isColumnFilterTrivial()); } + @Test + public void testTagFilterKeyIsCaseInsensitive() { + final TopicConfig topicConfig = + new TopicConfig(Collections.singletonMap("Tag-Filter", "region = \"north\"")); + + Assert.assertTrue(topicConfig.hasTagFilter()); + Assert.assertEquals("region = \"north\"", topicConfig.getTagFilter()); + Assert.assertEquals( + "region = \"north\"", + topicConfig.getAttributesWithSourceTagFilter().get(TopicConstant.TAG_FILTER_KEY)); + } + + @Test + public void testTagFilterDefaultsToTrivialWhenAbsent() { + final TopicConfig topicConfig = new TopicConfig(new HashMap<>()); + + Assert.assertFalse(topicConfig.hasTagFilter()); + Assert.assertTrue(topicConfig.isTagFilterTrivial()); + Assert.assertEquals(TopicConstant.TAG_FILTER_DEFAULT_VALUE, topicConfig.getTagFilter()); + } + + @Test + public void testTagFilterTrivialWithMixedCaseKeyAndValue() { + final Map attributes = new HashMap<>(); + attributes.put("TAG-FILTER", " TRUE "); + + Assert.assertTrue(new TopicConfig(attributes).isTagFilterTrivial()); + } + private static TopicConfig topicConfigWithMode(final String mode) { return new TopicConfig(Collections.singletonMap(TopicConstant.MODE_KEY, mode)); } diff --git a/iotdb-client/subscription/src/test/java/org/apache/iotdb/session/subscription/consumer/base/SubscriptionConsumerMultiProviderPollTest.java b/iotdb-client/subscription/src/test/java/org/apache/iotdb/session/subscription/consumer/base/SubscriptionConsumerMultiProviderPollTest.java index 34ef7a8fc4840..61f15a9362467 100644 --- a/iotdb-client/subscription/src/test/java/org/apache/iotdb/session/subscription/consumer/base/SubscriptionConsumerMultiProviderPollTest.java +++ b/iotdb-client/subscription/src/test/java/org/apache/iotdb/session/subscription/consumer/base/SubscriptionConsumerMultiProviderPollTest.java @@ -39,6 +39,7 @@ import org.junit.Assert; import org.junit.Test; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -93,14 +94,41 @@ public void testPollBacksOffAfterAllProvidersAreEmpty() throws SubscriptionExcep } } + @Test + public void testFilteredTabletBatchIsAckedWithoutReturningEmptyMessage() + throws SubscriptionException { + final TestPullConsumer consumer = new TestPullConsumer(1, 0, true); + try { + consumer.open(); + consumer.subscribeTopic(); + + final List messages = consumer.pollForTest(1_000L); + + Assert.assertTrue(messages.isEmpty()); + Assert.assertEquals(1, consumer.getAcknowledgedCommitContexts().size()); + Assert.assertEquals( + new SubscriptionCommitContext(1, 0, TOPIC, CONSUMER_GROUP_ID, 0L), + consumer.getAcknowledgedCommitContexts().get(0)); + } finally { + consumer.close(); + } + } + private static class TestPullConsumer extends AbstractSubscriptionPullConsumer { private final Map pollCounts = new HashMap<>(); + private final List acknowledgedCommitContexts = new ArrayList<>(); private final int dataProviderId; private final int emptyPollsBeforeData; + private final boolean emptyTabletBatch; private int backoffCount; private TestPullConsumer(final int dataProviderId, final int emptyPollsBeforeData) { + this(dataProviderId, emptyPollsBeforeData, false); + } + + private TestPullConsumer( + final int dataProviderId, final int emptyPollsBeforeData, final boolean emptyTabletBatch) { super( new AbstractSubscriptionPullConsumerBuilder() .host(HOST) @@ -112,6 +140,7 @@ private TestPullConsumer(final int dataProviderId, final int emptyPollsBeforeDat .autoCommit(false)); this.dataProviderId = dataProviderId; this.emptyPollsBeforeData = emptyPollsBeforeData; + this.emptyTabletBatch = emptyTabletBatch; } @Override @@ -141,7 +170,8 @@ protected AbstractSubscriptionProvider constructSubscriptionProvider( connectionTimeoutInMs, pollCounts, dataProviderId, - emptyPollsBeforeData); + emptyPollsBeforeData, + emptyTabletBatch); } private void subscribeTopic() { @@ -161,6 +191,17 @@ private int getBackoffCount() { return backoffCount; } + private List getAcknowledgedCommitContexts() { + return acknowledgedCommitContexts; + } + + @Override + protected void ack(final Iterable messages) throws SubscriptionException { + for (final SubscriptionMessage message : messages) { + acknowledgedCommitContexts.add(message.getCommitContext()); + } + } + @Override void sleepAfterEmptyPollRound() { backoffCount++; @@ -173,6 +214,7 @@ private static class TestSubscriptionProvider extends AbstractSubscriptionProvid private final Map pollCounts; private final int dataProviderId; private final int emptyPollsBeforeData; + private final boolean emptyTabletBatch; private TestSubscriptionProvider( final TEndPoint endPoint, @@ -188,7 +230,8 @@ private TestSubscriptionProvider( final int connectionTimeoutInMs, final Map pollCounts, final int dataProviderId, - final int emptyPollsBeforeData) { + final int emptyPollsBeforeData, + final boolean emptyTabletBatch) { super( endPoint, username, @@ -205,6 +248,7 @@ private TestSubscriptionProvider( this.pollCounts = pollCounts; this.dataProviderId = dataProviderId; this.emptyPollsBeforeData = emptyPollsBeforeData; + this.emptyTabletBatch = emptyTabletBatch; } @Override @@ -265,6 +309,18 @@ List poll( } final SubscriptionCommitContext commitContext = new SubscriptionCommitContext(dataNodeId, 0, TOPIC, CONSUMER_GROUP_ID, 0L); + if (emptyTabletBatch) { + final int pollCount = pollCounts.get(dataNodeId); + if (pollCount != emptyPollsBeforeData + 1) { + return Collections.emptyList(); + } + return new ArrayList<>( + Collections.singletonList( + new SubscriptionPollResponse( + SubscriptionPollResponseType.TABLETS.getType(), + new TabletsPayload(Collections.emptyMap(), 1), + commitContext))); + } final List schemas = Collections.singletonList(new MeasurementSchema("s1", TSDataType.INT64)); final Tablet tablet = new Tablet("root.sg.d1", schemas, 1); @@ -277,5 +333,19 @@ List poll( new TabletsPayload(Collections.singletonList(tablet), -1), commitContext)); } + + @Override + List pollTablets( + final SubscriptionCommitContext commitContext, final int offset, final long timeoutMs) + throws SubscriptionException { + if (emptyTabletBatch && dataNodeId == dataProviderId && offset == 1) { + return Collections.singletonList( + new SubscriptionPollResponse( + SubscriptionPollResponseType.TABLETS.getType(), + new TabletsPayload(Collections.emptyMap(), 0), + commitContext)); + } + return Collections.emptyList(); + } } } diff --git a/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ConfigNodeMessages.java b/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ConfigNodeMessages.java index 800d59bf727ba..487cfbeeb7364 100644 --- a/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ConfigNodeMessages.java +++ b/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ConfigNodeMessages.java @@ -675,6 +675,15 @@ private ConfigNodeMessages() {} public static final String EXCEPTION_FAILED_TO_CREATE_OR_ALTER_TOPIC_ARG_AND_ARG_ARE_ONLY_SUPPORTED_FOR_INCREMENTAL_TOPICS_D86CEA8E = "Failed to create or alter topic, %s and %s are only supported for incremental topics"; + public static final String + EXCEPTION_FAILED_TO_CREATE_OR_ALTER_TOPIC_DUPLICATE_ARG_ATTRIBUTES_ARE_NOT_ALLOWED_27315578 = + "Failed to create or alter topic, duplicate %s attributes are not allowed"; + public static final String + EXCEPTION_FAILED_TO_CREATE_OR_ALTER_TOPIC_ARG_IS_ONLY_SUPPORTED_FOR_TABLE_TOPICS_A5126607 = + "Failed to create or alter topic, %s is only supported for table topics"; + public static final String + EXCEPTION_FAILED_TO_CREATE_OR_ALTER_TOPIC_ARG_SHOULD_NOT_BE_EMPTY_767B1148 = + "Failed to create or alter topic, %s should not be empty"; public static final String EXCEPTION_FAILED_TO_CREATE_OR_ALTER_TOPIC_SUBSCRIBING_ONLY_TO_THE_AUDIT_DATABASE_OR_PATHS_UNDER_IT_IS_NOT_ALLOWED_3E96A6BA = "Failed to create or alter topic, subscribing only to the __audit database or paths under it is not allowed"; diff --git a/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ConfigNodeMessages.java b/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ConfigNodeMessages.java index 14713cbb8011b..c6eee8466b3ae 100644 --- a/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ConfigNodeMessages.java +++ b/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ConfigNodeMessages.java @@ -718,6 +718,15 @@ private ConfigNodeMessages() {} public static final String EXCEPTION_FAILED_TO_CREATE_OR_ALTER_TOPIC_ARG_AND_ARG_ARE_ONLY_SUPPORTED_FOR_INCREMENTAL_TOPICS_D86CEA8E = "创建或修改 topic 失败，%s 和 %s 仅支持 incremental 模式的 topic"; + public static final String + EXCEPTION_FAILED_TO_CREATE_OR_ALTER_TOPIC_DUPLICATE_ARG_ATTRIBUTES_ARE_NOT_ALLOWED_27315578 = + "\u521b\u5efa\u6216\u4fee\u6539 Topic \u5931\u8d25\uff0c\u4e0d\u5141\u8bb8\u91cd\u590d\u7684 %s \u5c5e\u6027"; + public static final String + EXCEPTION_FAILED_TO_CREATE_OR_ALTER_TOPIC_ARG_IS_ONLY_SUPPORTED_FOR_TABLE_TOPICS_A5126607 = + "\u521b\u5efa\u6216\u4fee\u6539 Topic \u5931\u8d25\uff0c%s \u4ec5\u652f\u6301\u8868\u6a21\u578b Topic"; + public static final String + EXCEPTION_FAILED_TO_CREATE_OR_ALTER_TOPIC_ARG_SHOULD_NOT_BE_EMPTY_767B1148 = + "\u521b\u5efa\u6216\u4fee\u6539 Topic \u5931\u8d25\uff0c%s \u4e0d\u5e94\u4e3a\u7a7a"; public static final String EXCEPTION_FAILED_TO_CREATE_OR_ALTER_TOPIC_SUBSCRIBING_ONLY_TO_THE_AUDIT_DATABASE_OR_PATHS_UNDER_IT_IS_NOT_ALLOWED_3E96A6BA = "创建或修改 topic 失败，不允许仅订阅 __audit 数据库或其下的路径"; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/subscription/SubscriptionInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/subscription/SubscriptionInfo.java index 36b6f35c31d66..9d33fee155db9 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/subscription/SubscriptionInfo.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/subscription/SubscriptionInfo.java @@ -54,6 +54,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TUnsubscribeReq; import org.apache.iotdb.consensus.ConsensusFactory; import org.apache.iotdb.consensus.common.DataSet; +import org.apache.iotdb.db.subscription.tagfilter.TagFilterParser; import org.apache.iotdb.mpp.rpc.thrift.TTopicOwnerLeaseEntry; import org.apache.iotdb.pipe.api.exception.PipeException; import org.apache.iotdb.rpc.TSStatusCode; @@ -92,6 +93,7 @@ public class SubscriptionInfo implements SnapshotProcessor { private static final Logger LOGGER = LoggerFactory.getLogger(SubscriptionInfo.class); + private static final TagFilterParser TAG_FILTER_PARSER = new TagFilterParser(); private static final ConfigNodeConfig CONF = ConfigNodeDescriptor.getInstance().getConf(); @@ -106,6 +108,7 @@ public class SubscriptionInfo implements SnapshotProcessor { TopicConstant.DATABASE_KEY, TopicConstant.TABLE_KEY, TopicConstant.COLUMN_FILTER_KEY, + TopicConstant.TAG_FILTER_KEY, TopicConstant.RETENTION_BYTES_KEY, TopicConstant.RETENTION_MS_KEY, TopicConstant.START_TIME_KEY, @@ -127,6 +130,7 @@ public class SubscriptionInfo implements SnapshotProcessor { TopicConstant.DATABASE_KEY, TopicConstant.TABLE_KEY, TopicConstant.COLUMN_FILTER_KEY, + TopicConstant.TAG_FILTER_KEY, TopicConstant.RETENTION_BYTES_KEY, TopicConstant.RETENTION_MS_KEY, TopicConstant.MODE_KEY, @@ -325,6 +329,11 @@ public void validateBeforeAlteringTopic(TopicMeta topicMeta) throws Subscription } } + public void validateUpdatedTopicAttributes(final Map updatedAttributes) + throws SubscriptionException { + validateDuplicateTopicAttributes(new TopicConfig(safeTopicAttributes(updatedAttributes))); + } + private void checkBeforeAlteringTopicInternal(TopicMeta topicMeta) throws SubscriptionException { validateTopicConfig(topicMeta.getConfig()); @@ -396,6 +405,7 @@ private void validateTopicConfig(final TopicConfig topicConfig) throws Subscript } validateColumnFilter(topicConfig); + validateTagFilter(topicConfig); validateIncrementalTopicRetentionConfig(topicConfig); final Long ownerLeaseDurationMs = @@ -548,7 +558,8 @@ private void validateDuplicateTopicAttributes(final TopicConfig topicConfig) if (!seenKeys.add(normalizedKey)) { final String exceptionMessage = String.format( - "Failed to create or alter topic, duplicate %s attributes are not allowed", + ConfigNodeMessages + .EXCEPTION_FAILED_TO_CREATE_OR_ALTER_TOPIC_DUPLICATE_ARG_ATTRIBUTES_ARE_NOT_ALLOWED_27315578, normalizedKey); LOGGER.warn(exceptionMessage); throw new SubscriptionException(exceptionMessage); @@ -557,24 +568,50 @@ private void validateDuplicateTopicAttributes(final TopicConfig topicConfig) } private void validateColumnFilter(final TopicConfig topicConfig) throws SubscriptionException { - if (!topicConfig.hasColumnFilter()) { + validateTableOnlyFilter( + topicConfig, + TopicConstant.COLUMN_FILTER_KEY, + topicConfig.hasColumnFilter(), + topicConfig.getColumnFilter()); + } + + private void validateTagFilter(final TopicConfig topicConfig) throws SubscriptionException { + validateTableOnlyFilter( + topicConfig, + TopicConstant.TAG_FILTER_KEY, + topicConfig.hasTagFilter(), + topicConfig.getTagFilter()); + if (topicConfig.hasTagFilter()) { + TAG_FILTER_PARSER.parseAndValidate(topicConfig.getTagFilter()); + } + } + + private void validateTableOnlyFilter( + final TopicConfig topicConfig, + final String filterKey, + final boolean hasFilter, + final String filter) + throws SubscriptionException { + if (!hasFilter) { return; } if (!topicConfig.isTableTopic()) { final String exceptionMessage = String.format( - "Failed to create or alter topic, %s is only supported for table topics", - TopicConstant.COLUMN_FILTER_KEY); + ConfigNodeMessages + .EXCEPTION_FAILED_TO_CREATE_OR_ALTER_TOPIC_ARG_IS_ONLY_SUPPORTED_FOR_TABLE_TOPICS_A5126607, + filterKey); LOGGER.warn(exceptionMessage); throw new SubscriptionException(exceptionMessage); } - if (topicConfig.getColumnFilter().trim().isEmpty()) { + if (filter.trim().isEmpty()) { final String exceptionMessage = String.format( - "Failed to create or alter topic, %s should not be empty", - TopicConstant.COLUMN_FILTER_KEY); + ConfigNodeMessages + .EXCEPTION_FAILED_TO_CREATE_OR_ALTER_TOPIC_ARG_SHOULD_NOT_BE_EMPTY_767B1148, + filterKey); LOGGER.warn(exceptionMessage); throw new SubscriptionException(exceptionMessage); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/subscription/topic/AlterTopicProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/subscription/topic/AlterTopicProcedure.java index 1f6cc40cb0956..fb7ff2f6332b5 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/subscription/topic/AlterTopicProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/subscription/topic/AlterTopicProcedure.java @@ -130,6 +130,7 @@ public boolean executeFromValidate(ConfigNodeProcedureEnv env) throws Subscripti .deepCopyTopicMeta( updatedTopicMeta.getTopicName(), updatedTopicMeta.visibleUnderTableModel()); if (Objects.nonNull(updatedTopicAttributes) && Objects.nonNull(existedTopicMeta)) { + subscriptionInfo.get().validateUpdatedTopicAttributes(updatedTopicAttributes); updatedTopicMeta = existedTopicMeta.deepCopyWithUpdatedAttributes(updatedTopicAttributes); } diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/subscription/SubscriptionInfoTopicValidationTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/subscription/SubscriptionInfoTopicValidationTest.java index 48523171345dd..fc1c9abb30f84 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/subscription/SubscriptionInfoTopicValidationTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/subscription/SubscriptionInfoTopicValidationTest.java @@ -86,6 +86,75 @@ public void testRejectMixedCaseColumnFilterOnTreeTopic() { assertCreateRejected(subscriptionInfo, attributes, "only supported for table topics"); } + @Test + public void testValidateTagFilterOnCreate() throws Exception { + final SubscriptionInfo subscriptionInfo = new SubscriptionInfo(); + final Map attributes = newIncrementalTableTopicAttributes(); + attributes.put(TopicConstant.TAG_FILTER_KEY, "region IN (\"north\", \"south\")"); + + Assert.assertTrue( + subscriptionInfo.validateBeforeCreatingTopic( + new TCreateTopicReq("table_topic").setTopicAttributes(attributes))); + } + + @Test + public void testRejectInvalidTagFilterFromSessionApiOnCreate() { + final SubscriptionInfo subscriptionInfo = new SubscriptionInfo(); + final Map attributes = newIncrementalTableTopicAttributes(); + attributes.put(TopicConstant.TAG_FILTER_KEY, "region = north"); + + assertCreateRejected(subscriptionInfo, attributes, "Invalid tag-filter"); + } + + @Test + public void testRejectInvalidTagFilterFromSessionApiOnAlter() throws Exception { + final SubscriptionInfo subscriptionInfo = new SubscriptionInfo(); + final Map originalAttributes = newIncrementalTableTopicAttributes(); + originalAttributes.put(TopicConstant.TAG_FILTER_KEY, "region = \"north\""); + subscriptionInfo.createTopic( + new CreateTopicPlan(new TopicMeta("table_topic", 1L, originalAttributes))); + + final Map updatedAttributes = newIncrementalTableTopicAttributes(); + updatedAttributes.put(TopicConstant.TAG_FILTER_KEY, "region IN ()"); + try { + subscriptionInfo.validateBeforeAlteringTopic( + new TopicMeta("table_topic", 2L, updatedAttributes)); + Assert.fail("Expected invalid tag-filter alteration to fail"); + } catch (final SubscriptionException e) { + Assert.assertTrue(e.getMessage().contains("Invalid tag-filter")); + } + } + + @Test + public void testRejectTagFilterOnTreeTopic() { + final SubscriptionInfo subscriptionInfo = new SubscriptionInfo(); + final Map attributes = new HashMap<>(); + attributes.put(TopicConstant.TAG_FILTER_KEY, "region = \"north\""); + + assertCreateRejected(subscriptionInfo, attributes, "only supported for table topics"); + } + + @Test + public void testTagFilterKeyIsCaseInsensitiveOnCreate() throws Exception { + final SubscriptionInfo subscriptionInfo = new SubscriptionInfo(); + final Map attributes = newInitialTableTopicAttributes(); + attributes.put("Tag-Filter", "region = \"north\""); + + Assert.assertTrue( + subscriptionInfo.validateBeforeCreatingTopic( + new TCreateTopicReq("table_topic").setTopicAttributes(attributes))); + } + + @Test + public void testRejectDuplicateTagFilterKeys() { + final SubscriptionInfo subscriptionInfo = new SubscriptionInfo(); + final Map attributes = newInitialTableTopicAttributes(); + attributes.put(TopicConstant.TAG_FILTER_KEY, "region = \"north\""); + attributes.put("Tag-Filter", "region = \"south\""); + + assertCreateRejected(subscriptionInfo, attributes, "duplicate tag-filter"); + } + @Test public void testRejectTopicThatOnlySelectsAuditDatabase() { final SubscriptionInfo subscriptionInfo = new SubscriptionInfo(); @@ -250,6 +319,15 @@ public void testRejectEmptyColumnFilter() { assertCreateRejected(subscriptionInfo, attributes, "column-filter should not be empty"); } + @Test + public void testRejectEmptyTagFilter() { + final SubscriptionInfo subscriptionInfo = new SubscriptionInfo(); + final Map attributes = newIncrementalTableTopicAttributes(); + attributes.put(TopicConstant.TAG_FILTER_KEY, " "); + + assertCreateRejected(subscriptionInfo, attributes, "tag-filter should not be empty"); + } + @Test public void testAcceptAlteringColumnFilter() throws Exception { final SubscriptionInfo subscriptionInfo = new SubscriptionInfo(); diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/subscription/topic/AlterTopicProcedureTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/subscription/topic/AlterTopicProcedureTest.java index 3f553cb60135b..2d2fa671c03a1 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/subscription/topic/AlterTopicProcedureTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/subscription/topic/AlterTopicProcedureTest.java @@ -25,6 +25,8 @@ import org.apache.iotdb.confignode.persistence.subscription.SubscriptionInfo; import org.apache.iotdb.confignode.procedure.store.ProcedureFactory; import org.apache.iotdb.rpc.TSStatusCode; +import org.apache.iotdb.rpc.subscription.config.TopicConstant; +import org.apache.iotdb.rpc.subscription.exception.SubscriptionException; import org.apache.tsfile.utils.PublicBAOS; import org.junit.Test; @@ -36,6 +38,8 @@ import java.util.concurrent.atomic.AtomicReference; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; public class AlterTopicProcedureTest { @@ -105,4 +109,33 @@ public void testRebaseUpdatedAttributesDuringValidate() throws Exception { assertEquals("processor1", procedure.getUpdatedTopicMeta().getConfig().getString("processor")); assertEquals("source1", procedure.getUpdatedTopicMeta().getConfig().getString("source")); } + + @Test + public void testRejectCaseInsensitiveDuplicateAttributesBeforeMerge() throws Exception { + final String topicName = "test_table_topic"; + final SubscriptionInfo subscriptionInfo = new SubscriptionInfo(); + final Map initialAttributes = new HashMap<>(); + initialAttributes.put("__system.sql-dialect", "table"); + assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + subscriptionInfo + .createTopic(new CreateTopicPlan(new TopicMeta(topicName, 1, initialAttributes))) + .getCode()); + + final Map requestAttributes = new HashMap<>(); + requestAttributes.put(TopicConstant.TAG_FILTER_KEY, "region = \"north\""); + requestAttributes.put("TAG-FILTER", "region = \"south\""); + final AlterTopicProcedure procedure = + new AlterTopicProcedure( + subscriptionInfo.deepCopyTopicMetaWithUpdatedAttributes(topicName, requestAttributes), + requestAttributes, + new AtomicReference<>(subscriptionInfo)); + + try { + procedure.executeFromValidate(null); + fail("Expected duplicate tag-filter attributes to be rejected"); + } catch (final SubscriptionException e) { + assertTrue(e.getMessage(), e.getMessage().contains("duplicate tag-filter")); + } + } } diff --git a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java index 12f4e374ca7a7..9449936c716f7 100644 --- a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java +++ b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java @@ -118,6 +118,78 @@ public final class DataNodeMiscMessages { public static final String COLUMN_FILTER_LIKE_PATTERN = "LIKE pattern"; public static final String COLUMN_FILTER_LIKE_ESCAPE = "LIKE escape"; public static final String COLUMN_FILTER_REGEXP_PATTERN = "REGEXP pattern"; + public static final String EXCEPTION_INVALID_TAG_FILTER_ARG_E4B1C1C6 = + "Invalid tag-filter: %s"; + public static final String EXCEPTION_TAG_FILTER_SHOULD_NOT_BE_EMPTY_507CA5B0 = + "tag-filter should not be empty"; + public static final String + EXCEPTION_ONLY_AND_COMPARISONS_ARE_SUPPORTED_IN_TAG_FILTER_19946958 = + "only =, !=, and <> comparisons are supported in tag-filter"; + public static final String EXCEPTION_LEFT_OPERAND_MUST_BE_A_TAG_COLUMN_F9D4548B = + "left operand must be a TAG column"; + public static final String EXCEPTION_TAG_FILTER_EVALUATION_FAILED_ARG_1B239B2F = + "Tag-filter evaluation failed: %s"; + public static final String EXCEPTION_TAG_FILTER_SCHEMA_BINDING_FAILED_ARG_7A5D2D47 = + "Tag-filter schema binding failed: %s"; + public static final String EXCEPTION_TAG_FILTER_EXCEEDS_MAXIMUM_UTF_8_LENGTH_OF_ARG_BYTES_9C8400F2 = + "tag-filter exceeds maximum UTF-8 length of %s bytes"; + public static final String EXCEPTION_TAG_FILTER_AST_DEPTH_EXCEEDS_MAXIMUM_OF_ARG_64C92731 = + "tag-filter AST depth exceeds maximum of %s"; + public static final String EXCEPTION_TAG_FILTER_AST_NODE_COUNT_EXCEEDS_MAXIMUM_OF_ARG_40CCE694 = + "tag-filter AST node count exceeds maximum of %s"; + public static final String EXCEPTION_TAG_FILTER_IN_LIST_EXCEEDS_MAXIMUM_OF_ARG_VALUES_07E7FC6C = + "tag-filter IN list exceeds maximum of %s values"; + public static final String EXCEPTION_TAG_FILTER_REGEXP_PATTERN_EXCEEDS_MAXIMUM_LENGTH_OF_ARG_CHARACTERS_67590971 = + "tag-filter REGEXP pattern exceeds maximum length of %s characters"; + public static final String EXCEPTION_TABLE_SCHEMA_IS_NOT_AVAILABLE_FOR_TAG_FILTER_993AB728 = + "table schema is not available for tag-filter"; + public static final String EXCEPTION_REFERENCED_TAG_COLUMN_IS_MISSING_ARG_F5300BEA = + "referenced TAG column is missing: %s"; + public static final String EXCEPTION_REFERENCED_COLUMN_IS_NOT_A_TAG_COLUMN_ARG_34D60881 = + "referenced column is not a TAG column: %s"; + public static final String EXCEPTION_UNCOMPILED_LIKE_PREDICATE_01AEE439 = + "uncompiled LIKE predicate"; + public static final String EXCEPTION_UNCOMPILED_REGEXP_PREDICATE_2B0DD646 = + "uncompiled REGEXP predicate"; + public static final String EXCEPTION_MATCHER_IS_UNAVAILABLE_1A659D47 = "matcher is unavailable"; + public static final String EXCEPTION_TABLET_SCHEMA_IS_MISSING_164075B0 = "tablet schema is missing"; + public static final String EXCEPTION_TABLET_VALUE_COLUMNS_ARE_INCOMPLETE_DCD09F3C = + "tablet value columns are incomplete"; + public static final String EXCEPTION_FAILED_TO_EVALUATE_A_TABLET_ROW_7E4E94CE = + "failed to evaluate a tablet row"; + public static final String EXCEPTION_TABLE_BINDING_IS_NOT_AVAILABLE_882C5F2F = + "table binding is not available"; + public static final String EXCEPTION_REFERENCED_COLUMN_IS_NOT_A_TAG_COLUMN_33FA34BC = + "referenced column is not a TAG column"; + public static final String EXCEPTION_REFERENCED_COLUMN_CATEGORY_CHANGED_564594D8 = + "referenced column category changed"; + public static final String EXCEPTION_REFERENCED_TAG_COLUMN_IS_MISSING_55E07377 = + "referenced TAG column is missing"; + public static final String EXCEPTION_TABLET_MEASUREMENT_SCHEMA_IS_INCOMPLETE_6A813472 = + "tablet measurement schema is incomplete"; + public static final String EXCEPTION_FAILED_TO_COMPACT_FILTERED_TABLET_5D4AD8AA = + "failed to compact filtered tablet"; + public static final String EXCEPTION_TABLET_COLUMN_CATEGORIES_ARE_MISSING_2C660532 = + "tablet column categories are missing"; + public static final String EXCEPTION_TABLET_COLUMN_CATEGORY_IS_MISSING_A812B500 = + "tablet column category is missing"; + public static final String EXCEPTION_TABLET_TIMESTAMPS_ARE_INCOMPLETE_24F8CE6F = + "tablet timestamps are incomplete"; + public static final String EXCEPTION_TABLET_BITMAPS_ARE_INCOMPLETE_7BBC8035 = + "tablet bitmaps are incomplete"; + public static final String EXCEPTION_TABLET_VALUE_COLUMN_IS_INCOMPLETE_845721FE = + "tablet value column is incomplete"; + public static final String EXCEPTION_TREE_VIEW_PROJECTOR_IS_UNAVAILABLE_FOR_FILTERED_SUBSCRIPTION_DATA_B5F396A5 = + "Tree View projector is unavailable for filtered subscription data"; + public static final String + EXCEPTION_TOPIC_CONFIGURATION_IS_NOT_AVAILABLE_FOR_TAG_FILTER_1E023E3A = + "topic configuration is not available for tag-filter"; + public static final String + EXCEPTION_TOPIC_CONFIGURATION_CHANGED_WHILE_CAPTURING_TAG_FILTER_SNAPSHOT_984CEB1B = + "topic configuration changed while capturing tag-filter snapshot"; + public static final String + LOG_SUBSCRIPTION_TABLE_SCHEMA_IS_NOT_AVAILABLE_FOR_TAG_FILTER_TOPIC_ARG_E864C98F = + "Subscription: table schema is not available for tag-filter topic [{}]"; public static final String CREATE_NEW_REGION_ERROR_FMT = "create new region %s error, exception:%s"; public static final String CREATE_NEW_REGION_SUCCEED_FMT = "create new region %s succeed"; @@ -783,6 +855,14 @@ private DataNodeMiscMessages() {} "Subscription: failed to lazily refresh column-filter matcher for topic [{}]"; public static final String SUBSCRIPTION_DROP_COLUMN_FILTER = "Subscription: dropped column-filter matcher for topic [{}]"; + public static final String + LOG_SUBSCRIPTION_FAILED_TO_REFRESH_TAG_FILTER_MATCHER_FOR_TOPIC_ARG_USE_EMPTY_MATCHER_TO_FAIL_CLOSED_3CB76D70 = + "Subscription: failed to refresh tag-filter matcher for topic [{}], use empty matcher to fail closed"; + public static final String + LOG_SUBSCRIPTION_REFRESHED_TAG_FILTER_MATCHER_FOR_TOPIC_ARG_71598849 = + "Subscription: refreshed tag-filter matcher for topic [{}]"; + public static final String LOG_SUBSCRIPTION_DROPPED_TAG_FILTER_MATCHER_FOR_TOPIC_ARG_9EA52458 = + "Subscription: dropped tag-filter matcher for topic [{}]"; public static final String SUBSCRIPTION_UNSUPPORTED_CONSENSUS_PROGRESS_FILE_VERSION_FMT = "Unsupported consensus subscription progress file version %s"; @@ -823,6 +903,8 @@ private DataNodeMiscMessages() {} "Exception occurred when sealing events from batch {}"; public static final String EXCEPTION_CONSTRUCT_NEW_BATCH = "Exception occurred when construct new batch"; + public static final String EXCEPTION_FAILED_TO_SEAL_SUBSCRIPTION_EVENT_BATCH_1FB7E92C = + "Failed to seal subscription event batch"; // --------------------------------------------------------------------------- // subscription – SubscriptionPrefetchingQueue diff --git a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java index bf7a36010030a..1492cd9b1f11c 100644 --- a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java +++ b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java @@ -3877,6 +3877,12 @@ private DataNodeQueryMessages() {} "No materializer is available to enforce the DeviceEntry memory limit"; public static final String EXCEPTION_NO_MORE_DEVICEENTRY_RECORDS_ARE_AVAILABLE_8D51C199 = "No more DeviceEntry records are available"; + public static final String + EXCEPTION_FAILED_TO_CREATE_OR_ALTER_TOPIC_DUPLICATE_ARG_ATTRIBUTES_ARE_NOT_ALLOWED_27315578 = + "Failed to create or alter topic, duplicate %s attributes are not allowed"; + public static final String + EXCEPTION_FAILED_TO_CREATE_OR_ALTER_TOPIC_ARG_IS_ONLY_SUPPORTED_FOR_TABLE_TOPICS_A5126607 = + "Failed to create or alter topic, %s is only supported for table topics"; public static final String EXCEPTION_ONLY_INMEMORYDEVICEENTRYDATASET_SUPPORTS_GET_INLINE_DEVICE_ENTRIES_07A52CAB = "Only InMemoryDeviceEntryDataSet supports get inline device entries"; diff --git a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java index d809fea199259..af4324e053d10 100644 --- a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java +++ b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java @@ -118,6 +118,80 @@ public final class DataNodeMiscMessages { public static final String COLUMN_FILTER_LIKE_PATTERN = "LIKE 模式"; public static final String COLUMN_FILTER_LIKE_ESCAPE = "LIKE 转义符"; public static final String COLUMN_FILTER_REGEXP_PATTERN = "REGEXP 模式"; + public static final String EXCEPTION_INVALID_TAG_FILTER_ARG_E4B1C1C6 = + "\u65e0\u6548\u7684 tag-filter\uff1a%s"; + public static final String EXCEPTION_TAG_FILTER_SHOULD_NOT_BE_EMPTY_507CA5B0 = + "tag-filter \u4e0d\u5e94\u4e3a\u7a7a"; + public static final String + EXCEPTION_ONLY_AND_COMPARISONS_ARE_SUPPORTED_IN_TAG_FILTER_19946958 = + "tag-filter \u4ec5\u652f\u6301 =\u3001!= \u548c <> \u6bd4\u8f83"; + public static final String EXCEPTION_LEFT_OPERAND_MUST_BE_A_TAG_COLUMN_F9D4548B = + "\u5de6\u64cd\u4f5c\u6570\u5fc5\u987b\u662f TAG \u5217"; + public static final String EXCEPTION_TAG_FILTER_EVALUATION_FAILED_ARG_1B239B2F = + "tag-filter \u6c42\u503c\u5931\u8d25\uff1a%s"; + public static final String EXCEPTION_TAG_FILTER_SCHEMA_BINDING_FAILED_ARG_7A5D2D47 = + "tag-filter Schema \u7ed1\u5b9a\u5931\u8d25\uff1a%s"; + public static final String EXCEPTION_TAG_FILTER_EXCEEDS_MAXIMUM_UTF_8_LENGTH_OF_ARG_BYTES_9C8400F2 = + "tag-filter \u8d85\u8fc7 UTF-8 \u6700\u5927\u957f\u5ea6 %s \u5b57\u8282"; + public static final String EXCEPTION_TAG_FILTER_AST_DEPTH_EXCEEDS_MAXIMUM_OF_ARG_64C92731 = + "tag-filter AST \u6df1\u5ea6\u8d85\u8fc7\u6700\u5927\u503c %s"; + public static final String EXCEPTION_TAG_FILTER_AST_NODE_COUNT_EXCEEDS_MAXIMUM_OF_ARG_40CCE694 = + "tag-filter AST \u8282\u70b9\u6570\u8d85\u8fc7\u6700\u5927\u503c %s"; + public static final String EXCEPTION_TAG_FILTER_IN_LIST_EXCEEDS_MAXIMUM_OF_ARG_VALUES_07E7FC6C = + "tag-filter IN \u5217\u8868\u8d85\u8fc7\u6700\u5927\u503c %s"; + public static final String EXCEPTION_TAG_FILTER_REGEXP_PATTERN_EXCEEDS_MAXIMUM_LENGTH_OF_ARG_CHARACTERS_67590971 = + "tag-filter REGEXP \u6a21\u5f0f\u8d85\u8fc7\u6700\u5927\u957f\u5ea6 %s \u5b57\u7b26"; + public static final String EXCEPTION_TABLE_SCHEMA_IS_NOT_AVAILABLE_FOR_TAG_FILTER_993AB728 = + "tag-filter \u6240\u9700\u8868\u7ed3\u6784\u4e0d\u53ef\u7528"; + public static final String EXCEPTION_REFERENCED_TAG_COLUMN_IS_MISSING_ARG_F5300BEA = + "\u5f15\u7528\u7684 TAG \u5217\u4e0d\u5b58\u5728\uff1a%s"; + public static final String EXCEPTION_REFERENCED_COLUMN_IS_NOT_A_TAG_COLUMN_ARG_34D60881 = + "\u5f15\u7528\u7684\u5217\u4e0d\u662f TAG \u5217\uff1a%s"; + public static final String EXCEPTION_UNCOMPILED_LIKE_PREDICATE_01AEE439 = + "LIKE \u8c13\u8bcd\u672a\u7f16\u8bd1"; + public static final String EXCEPTION_UNCOMPILED_REGEXP_PREDICATE_2B0DD646 = + "REGEXP \u8c13\u8bcd\u672a\u7f16\u8bd1"; + public static final String EXCEPTION_MATCHER_IS_UNAVAILABLE_1A659D47 = + "tag-filter \u5339\u914d\u5668\u4e0d\u53ef\u7528"; + public static final String EXCEPTION_TABLET_SCHEMA_IS_MISSING_164075B0 = + "Tablet Schema \u7f3a\u5931"; + public static final String EXCEPTION_TABLET_VALUE_COLUMNS_ARE_INCOMPLETE_DCD09F3C = + "Tablet \u503c\u5217\u4e0d\u5b8c\u6574"; + public static final String EXCEPTION_FAILED_TO_EVALUATE_A_TABLET_ROW_7E4E94CE = + "Tablet \u884c\u6c42\u503c\u5931\u8d25"; + public static final String EXCEPTION_TABLE_BINDING_IS_NOT_AVAILABLE_882C5F2F = + "\u8868\u7ed1\u5b9a\u4e0d\u53ef\u7528"; + public static final String EXCEPTION_REFERENCED_COLUMN_IS_NOT_A_TAG_COLUMN_33FA34BC = + "\u5f15\u7528\u7684\u5217\u4e0d\u662f TAG \u5217"; + public static final String EXCEPTION_REFERENCED_COLUMN_CATEGORY_CHANGED_564594D8 = + "\u5f15\u7528\u5217\u7684\u7c7b\u522b\u5df2\u53d8\u66f4"; + public static final String EXCEPTION_REFERENCED_TAG_COLUMN_IS_MISSING_55E07377 = + "\u5f15\u7528\u7684 TAG \u5217\u7f3a\u5931"; + public static final String EXCEPTION_TABLET_MEASUREMENT_SCHEMA_IS_INCOMPLETE_6A813472 = + "Tablet \u6d4b\u91cf\u7ed3\u6784\u4e0d\u5b8c\u6574"; + public static final String EXCEPTION_FAILED_TO_COMPACT_FILTERED_TABLET_5D4AD8AA = + "\u538b\u7f29\u8fc7\u6ee4\u540e\u7684 Tablet \u5931\u8d25"; + public static final String EXCEPTION_TABLET_COLUMN_CATEGORIES_ARE_MISSING_2C660532 = + "Tablet \u5217\u7c7b\u522b\u7f3a\u5931"; + public static final String EXCEPTION_TABLET_COLUMN_CATEGORY_IS_MISSING_A812B500 = + "Tablet \u5217\u7c7b\u522b\u7f3a\u5931"; + public static final String EXCEPTION_TABLET_TIMESTAMPS_ARE_INCOMPLETE_24F8CE6F = + "Tablet \u65f6\u95f4\u6233\u4e0d\u5b8c\u6574"; + public static final String EXCEPTION_TABLET_BITMAPS_ARE_INCOMPLETE_7BBC8035 = + "Tablet bitmap \u4e0d\u5b8c\u6574"; + public static final String EXCEPTION_TABLET_VALUE_COLUMN_IS_INCOMPLETE_845721FE = + "Tablet \u503c\u5217\u4e0d\u5b8c\u6574"; + public static final String EXCEPTION_TREE_VIEW_PROJECTOR_IS_UNAVAILABLE_FOR_FILTERED_SUBSCRIPTION_DATA_B5F396A5 = + "\u8fc7\u6ee4\u8ba2\u9605\u6570\u636e\u7684 Tree View \u6295\u5f71\u5668\u4e0d\u53ef\u7528"; + public static final String + EXCEPTION_TOPIC_CONFIGURATION_IS_NOT_AVAILABLE_FOR_TAG_FILTER_1E023E3A = + "\u65e0\u6cd5\u83b7\u53d6 tag-filter \u7684 Topic \u914d\u7f6e"; + public static final String + EXCEPTION_TOPIC_CONFIGURATION_CHANGED_WHILE_CAPTURING_TAG_FILTER_SNAPSHOT_984CEB1B = + "\u6355\u83b7 tag-filter \u5feb\u7167\u65f6 Topic \u914d\u7f6e\u53d1\u751f\u53d8\u5316"; + public static final String + LOG_SUBSCRIPTION_TABLE_SCHEMA_IS_NOT_AVAILABLE_FOR_TAG_FILTER_TOPIC_ARG_E864C98F = + "\u8ba2\u9605：tag-filter Topic \u7684\u8868 Schema \u4e0d\u53ef\u7528 [{}]"; public static final String CREATE_NEW_REGION_ERROR_FMT = "创建新 region %s 错误，异常：%s"; public static final String CREATE_NEW_REGION_SUCCEED_FMT = "创建新 region %s 成功"; @@ -783,6 +857,14 @@ private DataNodeMiscMessages() {} "\u8ba2\u9605\uff1a\u61d2\u52a0\u8f7d\u5237\u65b0\u4e3b\u9898 [{}] \u7684 column-filter \u5339\u914d\u5668\u5931\u8d25"; public static final String SUBSCRIPTION_DROP_COLUMN_FILTER = "\u8ba2\u9605\uff1a\u5220\u9664\u4e3b\u9898 [{}] \u7684 column-filter \u5339\u914d\u5668"; + public static final String + LOG_SUBSCRIPTION_FAILED_TO_REFRESH_TAG_FILTER_MATCHER_FOR_TOPIC_ARG_USE_EMPTY_MATCHER_TO_FAIL_CLOSED_3CB76D70 = + "\u8ba2\u9605\uff1a\u5237\u65b0\u4e3b\u9898 [{}] \u7684 tag-filter \u5339\u914d\u5668\u5931\u8d25\uff0c\u4f7f\u7528\u7a7a\u5339\u914d\u5668\u4ee5\u5b89\u5168\u5931\u8d25"; + public static final String + LOG_SUBSCRIPTION_REFRESHED_TAG_FILTER_MATCHER_FOR_TOPIC_ARG_71598849 = + "\u8ba2\u9605\uff1a\u5df2\u5237\u65b0\u4e3b\u9898 [{}] \u7684 tag-filter \u5339\u914d\u5668"; + public static final String LOG_SUBSCRIPTION_DROPPED_TAG_FILTER_MATCHER_FOR_TOPIC_ARG_9EA52458 = + "\u8ba2\u9605\uff1a\u5220\u9664\u4e3b\u9898 [{}] \u7684 tag-filter \u5339\u914d\u5668"; public static final String SUBSCRIPTION_UNSUPPORTED_CONSENSUS_PROGRESS_FILE_VERSION_FMT = "不支持的共识订阅进度文件版本 %s"; @@ -823,6 +905,8 @@ private DataNodeMiscMessages() {} "从批次 {} 封存事件时发生异常"; public static final String EXCEPTION_CONSTRUCT_NEW_BATCH = "构造新批次时发生异常"; + public static final String EXCEPTION_FAILED_TO_SEAL_SUBSCRIPTION_EVENT_BATCH_1FB7E92C = + "\u5c01\u5b58\u8ba2\u9605\u4e8b\u4ef6\u6279\u6b21\u5931\u8d25"; // --------------------------------------------------------------------------- // subscription – SubscriptionPrefetchingQueue diff --git a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java index f26f2306be2fa..800b04c2a0ef6 100644 --- a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java +++ b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java @@ -4634,6 +4634,12 @@ private DataNodeQueryMessages() {} "没有可用于执行 DeviceEntry 内存限制的 materializer"; public static final String EXCEPTION_NO_MORE_DEVICEENTRY_RECORDS_ARE_AVAILABLE_8D51C199 = "没有更多可用的 DeviceEntry 记录"; + public static final String + EXCEPTION_FAILED_TO_CREATE_OR_ALTER_TOPIC_DUPLICATE_ARG_ATTRIBUTES_ARE_NOT_ALLOWED_27315578 = + "\u521b\u5efa\u6216\u4fee\u6539 Topic \u5931\u8d25\uff0c\u4e0d\u5141\u8bb8\u91cd\u590d\u7684 %s \u5c5e\u6027"; + public static final String + EXCEPTION_FAILED_TO_CREATE_OR_ALTER_TOPIC_ARG_IS_ONLY_SUPPORTED_FOR_TABLE_TOPICS_A5126607 = + "\u521b\u5efa\u6216\u4fee\u6539 Topic \u5931\u8d25\uff0c%s \u4ec5\u652f\u6301\u8868\u6a21\u578b Topic"; public static final String EXCEPTION_ONLY_INMEMORYDEVICEENTRYDATASET_SUPPORTS_GET_INLINE_DEVICE_ENTRIES_07A52CAB = "只有 InMemoryDeviceEntryDataSet 支持获取内存中的设备条目"; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/batch/PipeTabletEventTsFileBatch.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/batch/PipeTabletEventTsFileBatch.java index 4793f0cbfd1c6..b395c825b000e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/batch/PipeTabletEventTsFileBatch.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/batch/PipeTabletEventTsFileBatch.java @@ -58,26 +58,33 @@ public class PipeTabletEventTsFileBatch extends PipeTabletEventBatch { private final PipeTsFileBuilder treeModeTsFileBuilder; private final PipeTsFileBuilder tableModeTsFileBuilder; - private final BiFunction tableModelTabletPruner; + private final TabletTransformer tabletTransformer; private final Map, Double> pipeName2WeightMap = new HashMap<>(); public PipeTabletEventTsFileBatch(final int maxDelayInMs, final long requestMaxBatchSizeInBytes) { - this(maxDelayInMs, requestMaxBatchSizeInBytes, null, null); + this(maxDelayInMs, requestMaxBatchSizeInBytes, null, null, null); } public PipeTabletEventTsFileBatch( final int maxDelayInMs, final long requestMaxBatchSizeInBytes, final TriLongConsumer recordMetric) { - this(maxDelayInMs, requestMaxBatchSizeInBytes, recordMetric, null); + this(maxDelayInMs, requestMaxBatchSizeInBytes, recordMetric, null, null); } public PipeTabletEventTsFileBatch( final int maxDelayInMs, final long requestMaxBatchSizeInBytes, final BiFunction tableModelTabletPruner) { - this(maxDelayInMs, requestMaxBatchSizeInBytes, null, tableModelTabletPruner); + this(maxDelayInMs, requestMaxBatchSizeInBytes, null, tableModelTabletPruner, null); + } + + public PipeTabletEventTsFileBatch( + final int maxDelayInMs, + final long requestMaxBatchSizeInBytes, + final TabletTransformer tabletTransformer) { + this(maxDelayInMs, requestMaxBatchSizeInBytes, null, null, tabletTransformer); } public PipeTabletEventTsFileBatch( @@ -85,12 +92,31 @@ public PipeTabletEventTsFileBatch( final long requestMaxBatchSizeInBytes, final TriLongConsumer recordMetric, final BiFunction tableModelTabletPruner) { + this(maxDelayInMs, requestMaxBatchSizeInBytes, recordMetric, tableModelTabletPruner, null); + } + + private PipeTabletEventTsFileBatch( + final int maxDelayInMs, + final long requestMaxBatchSizeInBytes, + final TriLongConsumer recordMetric, + final BiFunction tableModelTabletPruner, + final TabletTransformer tabletTransformer) { super(maxDelayInMs, requestMaxBatchSizeInBytes, recordMetric); final AtomicLong tsFileIdGenerator = new AtomicLong(0); treeModeTsFileBuilder = new PipeTreeModelTsFileBuilderV2(currentBatchId, tsFileIdGenerator); tableModeTsFileBuilder = new PipeTableModelTsFileBuilderV2(currentBatchId, tsFileIdGenerator); - this.tableModelTabletPruner = tableModelTabletPruner; + this.tabletTransformer = + Objects.nonNull(tabletTransformer) + ? tabletTransformer + : (databaseName, tablet, isTableModel, isAligned) -> + new TabletTransformResult( + Objects.nonNull(tableModelTabletPruner) && isTableModel + ? tableModelTabletPruner.apply(databaseName, tablet) + : tablet, + databaseName, + isTableModel, + isAligned); } @Override @@ -100,25 +126,22 @@ protected boolean constructBatch(final TabletInsertionEvent event) { (PipeInsertNodeTabletInsertionEvent) event; final boolean isTableModel = insertNodeTabletInsertionEvent.isTableModelEvent(); final List tablets = insertNodeTabletInsertionEvent.convertToTablets(); - final List retainedTablets = new ArrayList<>(tablets.size()); - final List retainedAlignedFlags = new ArrayList<>(tablets.size()); + final List retainedTablets = new ArrayList<>(tablets.size()); for (int i = 0; i < tablets.size(); ++i) { - Tablet tablet = tablets.get(i); + final Tablet tablet = tablets.get(i); if (isTabletEmpty(tablet)) { continue; } - if (isTableModel) { - tablet = - pruneTableModelTablet( - tablet, insertNodeTabletInsertionEvent.getTableModelDatabaseName()); - if (isTabletEmpty(tablet)) { - continue; - } - } - retainedTablets.add(tablet); - if (!isTableModel) { - retainedAlignedFlags.add(insertNodeTabletInsertionEvent.isAligned(i)); + final TabletTransformResult transformedTablet = + transformTablet( + isTableModel ? insertNodeTabletInsertionEvent.getTableModelDatabaseName() : null, + tablet, + isTableModel, + !isTableModel && insertNodeTabletInsertionEvent.isAligned(i)); + if (Objects.isNull(transformedTablet) || isTabletEmpty(transformedTablet.getTablet())) { + continue; } + retainedTablets.add(transformedTablet); } // Pruning can remove all rows/columns from a tablet. Account only for data that is @@ -127,50 +150,58 @@ protected boolean constructBatch(final TabletInsertionEvent event) { if (retainedTablets.isEmpty()) { return false; } - increaseTotalBufferSizeAndUpdateMemoryBlock(calculateTabletsSizeInBytes(retainedTablets)); - for (int i = 0; i < retainedTablets.size(); ++i) { - final Tablet tablet = retainedTablets.get(i); - if (isTableModel) { + increaseTotalBufferSizeAndUpdateMemoryBlock( + calculateTabletsSizeInBytes( + retainedTablets.stream() + .map(TabletTransformResult::getTablet) + .collect(java.util.stream.Collectors.toList()))); + for (final TabletTransformResult transformedTablet : retainedTablets) { + if (transformedTablet.isTableModel()) { bufferTableModelTablet( insertNodeTabletInsertionEvent.getPipeName(), insertNodeTabletInsertionEvent.getCreationTime(), - tablet, - insertNodeTabletInsertionEvent.getTableModelDatabaseName()); + transformedTablet.getTablet(), + transformedTablet.getDatabaseName()); } else { bufferTreeModelTablet( insertNodeTabletInsertionEvent.getPipeName(), insertNodeTabletInsertionEvent.getCreationTime(), - tablet, - retainedAlignedFlags.get(i)); + transformedTablet.getTablet(), + transformedTablet.isAligned()); } } return true; } else if (event instanceof PipeRawTabletInsertionEvent) { final PipeRawTabletInsertionEvent rawTabletInsertionEvent = (PipeRawTabletInsertionEvent) event; - Tablet tablet = rawTabletInsertionEvent.convertToTablet(); + final Tablet tablet = rawTabletInsertionEvent.convertToTablet(); if (isTabletEmpty(tablet)) { return false; } - if (rawTabletInsertionEvent.isTableModelEvent()) { - tablet = pruneTableModelTablet(tablet, rawTabletInsertionEvent.getTableModelDatabaseName()); - if (isTabletEmpty(tablet)) { - return false; - } + final boolean isTableModel = rawTabletInsertionEvent.isTableModelEvent(); + final TabletTransformResult transformedTablet = + transformTablet( + isTableModel ? rawTabletInsertionEvent.getTableModelDatabaseName() : null, + tablet, + isTableModel, + !isTableModel && rawTabletInsertionEvent.isAligned()); + if (Objects.isNull(transformedTablet) || isTabletEmpty(transformedTablet.getTablet())) { + return false; } - increaseTotalBufferSizeAndUpdateMemoryBlock(calculateTabletSizeInBytes(tablet)); - if (rawTabletInsertionEvent.isTableModelEvent()) { + increaseTotalBufferSizeAndUpdateMemoryBlock( + calculateTabletSizeInBytes(transformedTablet.getTablet())); + if (transformedTablet.isTableModel()) { bufferTableModelTablet( rawTabletInsertionEvent.getPipeName(), rawTabletInsertionEvent.getCreationTime(), - tablet, - rawTabletInsertionEvent.getTableModelDatabaseName()); + transformedTablet.getTablet(), + transformedTablet.getDatabaseName()); } else { bufferTreeModelTablet( rawTabletInsertionEvent.getPipeName(), rawTabletInsertionEvent.getCreationTime(), - tablet, - rawTabletInsertionEvent.isAligned()); + transformedTablet.getTablet(), + transformedTablet.isAligned()); } return true; } else { @@ -183,10 +214,12 @@ protected boolean constructBatch(final TabletInsertionEvent event) { return false; } - private Tablet pruneTableModelTablet(final Tablet tablet, final String databaseName) { - return Objects.nonNull(tableModelTabletPruner) - ? tableModelTabletPruner.apply(databaseName, tablet) - : tablet; + private TabletTransformResult transformTablet( + final String databaseName, + final Tablet tablet, + final boolean isTableModel, + final boolean isAligned) { + return tabletTransformer.transform(databaseName, tablet, isTableModel, isAligned); } private long calculateTabletsSizeInBytes(final List tablets) { @@ -269,6 +302,48 @@ public Map, Double> deepCopyPipe2WeightMap() { return new HashMap<>(pipeName2WeightMap); } + @FunctionalInterface + public interface TabletTransformer { + + TabletTransformResult transform( + String databaseName, Tablet tablet, boolean isTableModel, boolean isAligned); + } + + public static final class TabletTransformResult { + + private final Tablet tablet; + private final String databaseName; + private final boolean tableModel; + private final boolean aligned; + + public TabletTransformResult( + final Tablet tablet, + final String databaseName, + final boolean tableModel, + final boolean aligned) { + this.tablet = tablet; + this.databaseName = databaseName; + this.tableModel = tableModel; + this.aligned = aligned; + } + + public Tablet getTablet() { + return tablet; + } + + public String getDatabaseName() { + return databaseName; + } + + public boolean isTableModel() { + return tableModel; + } + + public boolean isAligned() { + return aligned; + } + } + /** * Converts a Tablet to a TSFile and returns the generated TSFile along with its corresponding * database name. diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/header/DatasetHeaderFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/header/DatasetHeaderFactory.java index e623f81b29bf7..671c817d8f4ee 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/header/DatasetHeaderFactory.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/header/DatasetHeaderFactory.java @@ -178,6 +178,10 @@ public static DatasetHeader getShowTopicHeader() { return new DatasetHeader(ColumnHeaderConstant.showTopicColumnHeaders, true); } + public static DatasetHeader getShowTableTopicHeader() { + return new DatasetHeader(ColumnHeaderConstant.showTableTopicColumnHeaders, true); + } + public static DatasetHeader getShowSubscriptionHeader() { return new DatasetHeader(ColumnHeaderConstant.showSubscriptionColumnHeaders, true); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java index d2f81429d4578..68d75818ae10b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java @@ -265,6 +265,7 @@ import org.apache.iotdb.db.queryengine.plan.statement.sys.StartRepairDataStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.StopRepairDataStatement; import org.apache.iotdb.db.subscription.columnfilter.ColumnFilterParser; +import org.apache.iotdb.db.subscription.tagfilter.TagFilterParser; import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters; import org.apache.iotdb.rpc.TSStatusCode; import org.apache.iotdb.rpc.subscription.config.TopicConstant; @@ -311,6 +312,7 @@ public class TableConfigTaskVisitor implements AstVisitor topicAttributes) { - String columnFilterKey = null; - String columnFilter = null; - boolean hasColumnFilter = false; + validateAndNormalizeSubscriptionFilter( + topicAttributes, TopicConstant.COLUMN_FILTER_KEY, COLUMN_FILTER_PARSER::parseAndValidate); + } + + private static void validateAndNormalizeTagFilter(final Map topicAttributes) { + validateAndNormalizeSubscriptionFilter( + topicAttributes, TopicConstant.TAG_FILTER_KEY, TAG_FILTER_PARSER::parseAndValidate); + } + + private static void validateAndNormalizeSubscriptionFilter( + final Map topicAttributes, + final String expectedKey, + final SubscriptionFilterValidator validator) { + String filterKey = null; + String filter = null; + boolean hasFilter = false; for (final Map.Entry entry : topicAttributes.entrySet()) { - if (TopicConstant.COLUMN_FILTER_KEY.equalsIgnoreCase(entry.getKey())) { - if (hasColumnFilter) { + if (expectedKey.equalsIgnoreCase(entry.getKey())) { + if (hasFilter) { throw new SemanticException( String.format( - "Failed to create or alter topic, duplicate %s attributes are not allowed", - TopicConstant.COLUMN_FILTER_KEY)); + DataNodeQueryMessages + .EXCEPTION_FAILED_TO_CREATE_OR_ALTER_TOPIC_DUPLICATE_ARG_ATTRIBUTES_ARE_NOT_ALLOWED_27315578, + expectedKey)); } - hasColumnFilter = true; - columnFilterKey = entry.getKey(); - columnFilter = entry.getValue(); + hasFilter = true; + filterKey = entry.getKey(); + filter = entry.getValue(); } } - if (!hasColumnFilter) { + if (!hasFilter) { return; } - if (!TopicConstant.COLUMN_FILTER_KEY.equals(columnFilterKey)) { - topicAttributes.remove(columnFilterKey); - topicAttributes.put(TopicConstant.COLUMN_FILTER_KEY, columnFilter); + if (!expectedKey.equals(filterKey)) { + topicAttributes.remove(filterKey); + topicAttributes.put(expectedKey, filter); } try { - COLUMN_FILTER_PARSER.parseAndValidate(columnFilter); + validator.validate(filter); } catch (final SubscriptionException e) { throw new SemanticException(e.getMessage()); } } + @FunctionalInterface + private interface SubscriptionFilterValidator { + + void validate(String filter) throws SubscriptionException; + } + @Override public IConfigTask visitDropTopic(DropTopic node, MPPQueryContext context) { context.setQueryType(QueryType.OTHER); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TreeConfigTaskVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TreeConfigTaskVisitor.java index d1e363f81e1c8..269db0cb85f50 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TreeConfigTaskVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TreeConfigTaskVisitor.java @@ -783,7 +783,7 @@ public IConfigTask visitCreateTopic( createTopicStatement .getTopicAttributes() .put(SystemConstant.SQL_DIALECT_KEY, SystemConstant.SQL_DIALECT_TREE_VALUE); - rejectColumnFilterForTreeTopic(createTopicStatement.getTopicAttributes()); + rejectTableOnlyFiltersForTreeTopic(createTopicStatement.getTopicAttributes()); return new CreateTopicTask(createTopicStatement); } @@ -794,18 +794,21 @@ public IConfigTask visitAlterTopic( alterTopicStatement .getTopicAttributes() .put(SystemConstant.SQL_DIALECT_KEY, SystemConstant.SQL_DIALECT_TREE_VALUE); - rejectColumnFilterForTreeTopic(alterTopicStatement.getTopicAttributes()); + rejectTableOnlyFiltersForTreeTopic(alterTopicStatement.getTopicAttributes()); return new AlterTopicTask(alterTopicStatement); } - private static void rejectColumnFilterForTreeTopic(final Map topicAttributes) { + private static void rejectTableOnlyFiltersForTreeTopic( + final Map topicAttributes) { for (final String key : topicAttributes.keySet()) { - if (TopicConstant.COLUMN_FILTER_KEY.equalsIgnoreCase(key)) { + if (TopicConstant.COLUMN_FILTER_KEY.equalsIgnoreCase(key) + || TopicConstant.TAG_FILTER_KEY.equalsIgnoreCase(key)) { throw new SemanticException( String.format( - "Failed to create or alter topic, %s is only supported for table topics", - TopicConstant.COLUMN_FILTER_KEY)); + DataNodeQueryMessages + .EXCEPTION_FAILED_TO_CREATE_OR_ALTER_TOPIC_ARG_IS_ONLY_SUPPORTED_FOR_TABLE_TOPICS_A5126607, + key)); } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java index 7a3580fe66bf3..de26a3c383772 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java @@ -3301,6 +3301,7 @@ public SettableFuture showTopics( showTopicResp.isSetTopicInfoList() ? showTopicResp.getTopicInfoList() : Collections.emptyList(), + showTopicsStatement.isTableModel(), future); } catch (final Exception e) { future.setException(e); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/subscription/ShowTopicsTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/subscription/ShowTopicsTask.java index 1871ef66a5026..e7ffe776d2a61 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/subscription/ShowTopicsTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/subscription/ShowTopicsTask.java @@ -28,6 +28,8 @@ import org.apache.iotdb.db.queryengine.plan.execution.config.executor.IConfigTaskExecutor; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowTopics; import org.apache.iotdb.db.queryengine.plan.statement.metadata.subscription.ShowTopicsStatement; +import org.apache.iotdb.db.subscription.agent.SubscriptionAgent; +import org.apache.iotdb.db.subscription.tagfilter.TagFilterMatcher; import org.apache.iotdb.rpc.TSStatusCode; import com.google.common.util.concurrent.ListenableFuture; @@ -37,6 +39,7 @@ import org.apache.tsfile.utils.Binary; import java.util.List; +import java.util.function.Function; import java.util.stream.Collectors; public class ShowTopicsTask implements IConfigTask { @@ -60,12 +63,28 @@ public ListenableFuture execute(final IConfigTaskExecutor conf } public static void buildTSBlock( - final List topicInfoList, final SettableFuture future) { + final List topicInfoList, + final boolean isTableModel, + final SettableFuture future) { + buildTSBlock( + topicInfoList, + isTableModel, + topicName -> SubscriptionAgent.broker().getTagFilterMatcher(topicName, true), + future); + } + + static void buildTSBlock( + final List topicInfoList, + final boolean isTableModel, + final Function matcherProvider, + final SettableFuture future) { + final List columnHeaders = + isTableModel + ? ColumnHeaderConstant.showTableTopicColumnHeaders + : ColumnHeaderConstant.showTopicColumnHeaders; final TsBlockBuilder builder = new TsBlockBuilder( - ColumnHeaderConstant.showTopicColumnHeaders.stream() - .map(ColumnHeader::getColumnType) - .collect(Collectors.toList())); + columnHeaders.stream().map(ColumnHeader::getColumnType).collect(Collectors.toList())); for (final TShowTopicInfo topicInfo : topicInfoList) { builder.getTimeColumnBuilder().writeLong(0L); @@ -75,6 +94,17 @@ public static void buildTSBlock( builder .getColumnBuilder(1) .writeBinary(new Binary(topicInfo.getTopicAttributes(), TSFileConfig.STRING_CHARSET)); + if (isTableModel) { + final TagFilterMatcher matcher; + try { + matcher = matcherProvider.apply(topicInfo.getTopicName()); + } catch (final RuntimeException e) { + writeTagFilterDiagnostic(builder, TagFilterMatcher.failure(e)); + builder.declarePosition(); + continue; + } + writeTagFilterDiagnostic(builder, matcher); + } builder.declarePosition(); } @@ -82,6 +112,22 @@ public static void buildTSBlock( new ConfigTaskResult( TSStatusCode.SUCCESS_STATUS, builder.build(), - DatasetHeaderFactory.getShowTopicHeader())); + isTableModel + ? DatasetHeaderFactory.getShowTableTopicHeader() + : DatasetHeaderFactory.getShowTopicHeader())); + } + + private static void writeTagFilterDiagnostic( + final TsBlockBuilder builder, final TagFilterMatcher matcher) { + builder + .getColumnBuilder(2) + .writeBinary(new Binary(matcher.getRuntimeStatus().name(), TSFileConfig.STRING_CHARSET)); + if (matcher.isFailure()) { + builder + .getColumnBuilder(3) + .writeBinary(new Binary(matcher.getFailureMessage(), TSFileConfig.STRING_CHARSET)); + } else { + builder.getColumnBuilder(3).appendNull(); + } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/ITableCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/ITableCache.java index 13b0505895519..60adbd4632972 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/ITableCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/ITableCache.java @@ -62,6 +62,11 @@ TsTable getTable( Map> getTableSnapshot(); + /** Returns the schema-cache version, or a negative value when unavailable. */ + default long getInstanceVersion() { + return -1L; + } + String tryGetInternColumnName( final @Nonnull String database, final @Nonnull String tableName, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionBrokerAgent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionBrokerAgent.java index e65f1d8804197..ea0f2d3624e6f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionBrokerAgent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionBrokerAgent.java @@ -32,6 +32,7 @@ import org.apache.iotdb.db.i18n.DataNodeMiscMessages; import org.apache.iotdb.db.i18n.DataNodePipeMessages; import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; +import org.apache.iotdb.db.schemaengine.table.ITableCache; import org.apache.iotdb.db.subscription.broker.ConsensusSubscriptionBroker; import org.apache.iotdb.db.subscription.broker.ISubscriptionBroker; import org.apache.iotdb.db.subscription.broker.SubscriptionBroker; @@ -44,6 +45,7 @@ import org.apache.iotdb.db.subscription.columnfilter.ColumnFilterMatcher; import org.apache.iotdb.db.subscription.event.SubscriptionEvent; import org.apache.iotdb.db.subscription.resource.SubscriptionDataNodeResourceManager; +import org.apache.iotdb.db.subscription.tagfilter.TagFilterMatcher; import org.apache.iotdb.db.subscription.task.execution.ConsensusSubscriptionPrefetchExecutorManager; import org.apache.iotdb.db.subscription.task.subtask.SubscriptionSinkSubtask; import org.apache.iotdb.rpc.subscription.config.ConsumerConfig; @@ -80,6 +82,7 @@ public class SubscriptionBrokerAgent { private static final ColumnFilterMatcher EMPTY_COLUMN_FILTER_MATCHER = ColumnFilterMatcher.ofSelectedColumnNames(Collections.emptySet()); + private static final int TAG_FILTER_SCHEMA_SNAPSHOT_MAX_RETRIES = 3; /** Subscription brokers grouped by consumer group. */ private final Map> consumerGroupIdToBrokers = @@ -91,6 +94,9 @@ public class SubscriptionBrokerAgent { private final Map topicNameToColumnFilterMatcher = new ConcurrentHashMap<>(); private final ColumnFilterBinder columnFilterBinder = new ColumnFilterBinder(); + private final Map topicNameToTagFilterMatcher = + new ConcurrentHashMap<>(); + private final Map topicNameToTagFilterMatcherVersion = new ConcurrentHashMap<>(); //////////////////////////// provided for subscription agent //////////////////////////// @@ -815,6 +821,156 @@ public void dropColumnFilter(final String topicName) { LOGGER.info(DataNodeMiscMessages.SUBSCRIPTION_DROP_COLUMN_FILTER, topicName); } + public void refreshTagFilter(final String topicName, final TopicConfig topicConfig) { + final ITableCache tableCache = DataNodeTableCache.getInstance(); + if (Objects.isNull(topicConfig) + || !topicConfig.isTableTopic() + || topicConfig.isTagFilterTrivial()) { + cacheTagFilterMatcher( + topicName, TagFilterMatcher.matchAll(), tableCache.getInstanceVersion()); + return; + } + + for (int attempt = 0; attempt < TAG_FILTER_SCHEMA_SNAPSHOT_MAX_RETRIES; attempt++) { + final long versionBeforeBinding = tableCache.getInstanceVersion(); + final TagFilterMatcher matcher; + try { + final Map> bindingTables = + getTagFilterBindingTables(topicConfig); + if (bindingTables == null) { + if (versionBeforeBinding != tableCache.getInstanceVersion()) { + continue; + } + cacheTagFilterMatcher( + topicName, + TagFilterMatcher.failure( + new SubscriptionException( + DataNodeMiscMessages + .EXCEPTION_TABLE_SCHEMA_IS_NOT_AVAILABLE_FOR_TAG_FILTER_993AB728)), + versionBeforeBinding); + LOGGER.info( + DataNodeMiscMessages + .LOG_SUBSCRIPTION_TABLE_SCHEMA_IS_NOT_AVAILABLE_FOR_TAG_FILTER_TOPIC_ARG_E864C98F, + topicName); + return; + } + matcher = TagFilterMatcher.fromTopicConfig(topicConfig, bindingTables); + } catch (final Exception e) { + if (versionBeforeBinding != tableCache.getInstanceVersion()) { + continue; + } + LOGGER.warn( + DataNodeMiscMessages + .LOG_SUBSCRIPTION_FAILED_TO_REFRESH_TAG_FILTER_MATCHER_FOR_TOPIC_ARG_USE_EMPTY_MATCHER_TO_FAIL_CLOSED_3CB76D70, + topicName, + e); + cacheTagFilterMatcher(topicName, TagFilterMatcher.failure(e), versionBeforeBinding); + return; + } + + if (versionBeforeBinding != tableCache.getInstanceVersion()) { + continue; + } + cacheTagFilterMatcher(topicName, matcher, versionBeforeBinding); + LOGGER.info( + DataNodeMiscMessages.LOG_SUBSCRIPTION_REFRESHED_TAG_FILTER_MATCHER_FOR_TOPIC_ARG_71598849, + topicName); + return; + } + + // Leave the matcher uncached after repeated schema changes. The caller fails closed for this + // attempt and retries against a fresh schema snapshot on the next access. + topicNameToTagFilterMatcher.remove(topicName); + topicNameToTagFilterMatcherVersion.remove(topicName); + } + + private void cacheTagFilterMatcher( + final String topicName, final TagFilterMatcher matcher, final long tableCacheVersion) { + topicNameToTagFilterMatcher.put(topicName, matcher); + topicNameToTagFilterMatcherVersion.put(topicName, tableCacheVersion); + } + + public TagFilterMatcher getTagFilterMatcher(final String topicName) { + return getTagFilterMatcher(topicName, true); + } + + public TagFilterMatcher getTagFilterMatcher(final String topicName, final boolean isTableModel) { + if (!isTableModel) { + return TagFilterMatcher.matchAll(); + } + + final TagFilterMatcher matcher = topicNameToTagFilterMatcher.get(topicName); + final long tableCacheVersion = DataNodeTableCache.getInstance().getInstanceVersion(); + if (Objects.nonNull(matcher) + && Objects.equals(topicNameToTagFilterMatcherVersion.get(topicName), tableCacheVersion)) { + return matcher; + } + if (Objects.nonNull(matcher)) { + topicNameToTagFilterMatcher.remove(topicName, matcher); + topicNameToTagFilterMatcherVersion.remove(topicName); + } + + final TopicConfig topicConfig = + SubscriptionAgent.topic() + .getTopicConfigs(Collections.singleton(topicName), true) + .get(topicName); + if (Objects.isNull(topicConfig)) { + return TagFilterMatcher.failure( + new SubscriptionException( + DataNodeMiscMessages + .EXCEPTION_TABLE_SCHEMA_IS_NOT_AVAILABLE_FOR_TAG_FILTER_993AB728)); + } + + refreshTagFilter(topicName, topicConfig); + return topicNameToTagFilterMatcher.getOrDefault( + topicName, getDefaultTagFilterMatcher(topicConfig)); + } + + private static TagFilterMatcher getDefaultTagFilterMatcher(final TopicConfig topicConfig) { + return Objects.nonNull(topicConfig) + && topicConfig.isTableTopic() + && !topicConfig.isTagFilterTrivial() + ? TagFilterMatcher.failure( + new SubscriptionException( + DataNodeMiscMessages + .EXCEPTION_TABLE_SCHEMA_IS_NOT_AVAILABLE_FOR_TAG_FILTER_993AB728)) + : TagFilterMatcher.matchAll(); + } + + private static Map> getTagFilterBindingTables( + final TopicConfig topicConfig) { + if (Objects.isNull(topicConfig) + || !topicConfig.isTableTopic() + || topicConfig.isTagFilterTrivial()) { + return Collections.emptyMap(); + } + + final String database = + topicConfig.getStringOrDefault( + TopicConstant.DATABASE_KEY, TopicConstant.DATABASE_DEFAULT_VALUE); + final String tableName = + topicConfig.getStringOrDefault(TopicConstant.TABLE_KEY, TopicConstant.TABLE_DEFAULT_VALUE); + if (isDefaultTopicPattern(database, TopicConstant.DATABASE_DEFAULT_VALUE) + || isDefaultTopicPattern(tableName, TopicConstant.TABLE_DEFAULT_VALUE) + || !isLiteralTopicPattern(database) + || !isLiteralTopicPattern(tableName)) { + return DataNodeTableCache.getInstance().getTableSnapshot(); + } + + final TsTable table = DataNodeTableCache.getInstance().getTable(database, tableName, false); + return Objects.isNull(table) + ? null + : Collections.singletonMap(database, Collections.singletonMap(tableName, table)); + } + + public void dropTagFilter(final String topicName) { + topicNameToTagFilterMatcher.remove(topicName); + topicNameToTagFilterMatcherVersion.remove(topicName); + LOGGER.info( + DataNodeMiscMessages.LOG_SUBSCRIPTION_DROPPED_TAG_FILTER_MATCHER_FOR_TOPIC_ARG_9EA52458, + topicName); + } + public void unbindConsensusPrefetchingQueue( final String consumerGroupId, final String topicName) { final ConsensusSubscriptionBroker broker = getConsensusBroker(consumerGroupId); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionTopicAgent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionTopicAgent.java index 256c87e521dc7..df189554d6cdf 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionTopicAgent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionTopicAgent.java @@ -112,11 +112,16 @@ private void handleSingleTopicMetaChangesInternal(final TopicMeta metaFromCoordi topicMetaKeeper.addTopicMeta(topicName, metaFromCoordinator); if (shouldRefreshColumnFilter(oldMeta, metaFromCoordinator)) { SubscriptionAgent.broker().refreshColumnFilter(topicName, metaFromCoordinator.getConfig()); - } else if (!metaFromCoordinator.getConfig().isTableTopic() + } + if (shouldRefreshTagFilter(oldMeta, metaFromCoordinator)) { + SubscriptionAgent.broker().refreshTagFilter(topicName, metaFromCoordinator.getConfig()); + } + if (!metaFromCoordinator.getConfig().isTableTopic() && !topicMetaKeeper.containsTopicMeta(topicName, true)) { // ConfigNode rejects column-filter on tree topics. Drop defensively in case stale or replayed // topic metadata reaches this DataNode after a table-topic to tree-topic transition. SubscriptionAgent.broker().dropColumnFilter(topicName); + SubscriptionAgent.broker().dropTagFilter(topicName); } SubscriptionAgent.broker() .refreshConsensusQueueOrderMode( @@ -152,6 +157,35 @@ static boolean shouldRefreshColumnFilter(final TopicMeta oldMeta, final TopicMet newConfig, TopicConstant.TABLE_KEY, TopicConstant.TABLE_DEFAULT_VALUE))); } + static boolean shouldRefreshTagFilter(final TopicMeta oldMeta, final TopicMeta newMeta) { + if (Objects.isNull(newMeta) || !newMeta.getConfig().isTableTopic()) { + return false; + } + if (Objects.isNull(oldMeta) || !oldMeta.getConfig().isTableTopic()) { + return true; + } + + final TopicConfig oldConfig = oldMeta.getConfig(); + final TopicConfig newConfig = newMeta.getConfig(); + return !Objects.equals( + normalizeTagFilterValue(oldConfig.getTagFilter()), + normalizeTagFilterValue(newConfig.getTagFilter())) + || !Objects.equals( + normalizeTagFilterValue( + getAttributeIgnoreCase( + oldConfig, TopicConstant.DATABASE_KEY, TopicConstant.DATABASE_DEFAULT_VALUE)), + normalizeTagFilterValue( + getAttributeIgnoreCase( + newConfig, TopicConstant.DATABASE_KEY, TopicConstant.DATABASE_DEFAULT_VALUE))) + || !Objects.equals( + normalizeTagFilterValue( + getAttributeIgnoreCase( + oldConfig, TopicConstant.TABLE_KEY, TopicConstant.TABLE_DEFAULT_VALUE)), + normalizeTagFilterValue( + getAttributeIgnoreCase( + newConfig, TopicConstant.TABLE_KEY, TopicConstant.TABLE_DEFAULT_VALUE))); + } + private static String getAttributeIgnoreCase( final TopicConfig topicConfig, final String key, final String defaultValue) { return topicConfig.getAttribute().entrySet().stream() @@ -166,6 +200,10 @@ private static String normalizeColumnFilterBindingValue(final String value) { return Objects.nonNull(value) ? value.trim().toLowerCase(Locale.ROOT) : ""; } + private static String normalizeTagFilterValue(final String value) { + return Objects.nonNull(value) ? value.trim() : ""; + } + public TPushTopicMetaRespExceptionMessage handleTopicMetaChanges( final List topicMetasFromCoordinator) { acquireWriteLock(); @@ -232,6 +270,7 @@ private void handleDropTopicInternal(final String topicName, final Boolean isTab } if (Objects.nonNull(topicMeta) && topicMeta.visibleUnderTableModel()) { SubscriptionAgent.broker().dropColumnFilter(topicName); + SubscriptionAgent.broker().dropTagFilter(topicName); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingQueue.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingQueue.java index 801fc03d3b8f5..29207e27d329f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingQueue.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingQueue.java @@ -677,12 +677,11 @@ private void constructToTabletIterator(final TsFileInsertionEvent event) { } private boolean canPassThroughTsFile(final PipeTsFileInsertionEvent event) { + final boolean isTableModelSubscription = SubscriptionAgent.consumer().isTableModel(brokerId); return PipeEventCollector.canSkipParsing4TsFileEvent(event) - && (!event.isTableModelEvent() - || SubscriptionAgent.broker() - .getColumnFilterMatcher( - topicName, SubscriptionAgent.consumer().isTableModel(brokerId)) - .isMatchAll()); + && (!isTableModelSubscription + || (SubscriptionAgent.broker().getColumnFilterMatcher(topicName, true).isMatchAll() + && SubscriptionAgent.broker().getTagFilterMatcher(topicName, true).isMatchAll())); } private RetryableState onRetryableTabletInsertionEvent( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusLogToTabletConverter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusLogToTabletConverter.java index 0b27de77b123a..df823db248fb7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusLogToTabletConverter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusLogToTabletConverter.java @@ -42,6 +42,8 @@ import org.apache.iotdb.db.subscription.agent.SubscriptionAgent; import org.apache.iotdb.db.subscription.columnfilter.ColumnFilterMatcher; import org.apache.iotdb.db.subscription.columnfilter.TabletColumnPruner; +import org.apache.iotdb.db.subscription.tagfilter.TabletTagFilter; +import org.apache.iotdb.db.subscription.tagfilter.TagFilterMatcher; import org.apache.tsfile.enums.ColumnCategory; import org.apache.tsfile.enums.TSDataType; @@ -71,6 +73,7 @@ public class ConsensusLogToTabletConverter { private final TablePattern tablePattern; private final String topicName; private final ColumnFilterMatcher fallbackColumnFilterMatcher; + private final TagFilterMatcher fallbackTagFilterMatcher; /** * The actual database name of the DataRegion this converter processes (table-model format without @@ -83,7 +86,13 @@ public ConsensusLogToTabletConverter( final TablePattern tablePattern, final ColumnFilterMatcher columnFilterMatcher, final String databaseName) { - this(treePattern, tablePattern, null, columnFilterMatcher, databaseName); + this( + treePattern, + tablePattern, + null, + columnFilterMatcher, + TagFilterMatcher.matchAll(), + databaseName); } public ConsensusLogToTabletConverter( @@ -92,11 +101,38 @@ public ConsensusLogToTabletConverter( final String topicName, final ColumnFilterMatcher columnFilterMatcher, final String databaseName) { + this( + treePattern, + tablePattern, + topicName, + columnFilterMatcher, + TagFilterMatcher.matchAll(), + databaseName); + } + + public ConsensusLogToTabletConverter( + final TreePattern treePattern, + final TablePattern tablePattern, + final ColumnFilterMatcher columnFilterMatcher, + final TagFilterMatcher tagFilterMatcher, + final String databaseName) { + this(treePattern, tablePattern, null, columnFilterMatcher, tagFilterMatcher, databaseName); + } + + public ConsensusLogToTabletConverter( + final TreePattern treePattern, + final TablePattern tablePattern, + final String topicName, + final ColumnFilterMatcher columnFilterMatcher, + final TagFilterMatcher tagFilterMatcher, + final String databaseName) { this.treePattern = treePattern; this.tablePattern = tablePattern; this.topicName = topicName; this.fallbackColumnFilterMatcher = Objects.nonNull(columnFilterMatcher) ? columnFilterMatcher : ColumnFilterMatcher.matchAll(); + this.fallbackTagFilterMatcher = + Objects.nonNull(tagFilterMatcher) ? tagFilterMatcher : TagFilterMatcher.matchAll(); this.databaseName = databaseName; } @@ -492,7 +528,10 @@ private List convertRelationalInsertRowNode(final RelationalInsertRowNod } final Tablet prunedTablet = - TabletColumnPruner.pruneTableModelTablet(tablet, databaseName, getColumnFilterMatcher()); + TabletColumnPruner.pruneTableModelTablet( + TabletTagFilter.filter(tablet, getTagFilterMatcher(), databaseName), + databaseName, + getColumnFilterMatcher()); return Objects.nonNull(prunedTablet) ? Collections.singletonList(prunedTablet) : Collections.emptyList(); @@ -564,7 +603,10 @@ private List convertRelationalInsertTabletNode(final RelationalInsertTab node.getRowCount()); final Tablet prunedTablet = - TabletColumnPruner.pruneTableModelTablet(tablet, databaseName, getColumnFilterMatcher()); + TabletColumnPruner.pruneTableModelTablet( + TabletTagFilter.filter(tablet, getTagFilterMatcher(), databaseName), + databaseName, + getColumnFilterMatcher()); return Objects.nonNull(prunedTablet) ? Collections.singletonList(prunedTablet) : Collections.emptyList(); @@ -681,6 +723,12 @@ private ColumnFilterMatcher getColumnFilterMatcher() { : fallbackColumnFilterMatcher; } + private TagFilterMatcher getTagFilterMatcher() { + return Objects.nonNull(topicName) + ? SubscriptionAgent.broker().getTagFilterMatcher(topicName, true) + : fallbackTagFilterMatcher; + } + private boolean isValidColumn( final String[] measurements, final TSDataType[] dataTypes, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusSubscriptionSetupHandler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusSubscriptionSetupHandler.java index 87b3c66f698f1..066817c9539a8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusSubscriptionSetupHandler.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusSubscriptionSetupHandler.java @@ -632,6 +632,7 @@ private static ConsensusLogToTabletConverter buildConverter( if (isTableTopic) { SubscriptionAgent.broker().refreshColumnFilter(topicName, topicConfig); + SubscriptionAgent.broker().refreshTagFilter(topicName, topicConfig); // Table model: database + table name pattern tablePattern = buildTablePattern(topicConfig); return new ConsensusLogToTabletConverter( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/columnfilter/ColumnFilterParser.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/columnfilter/ColumnFilterParser.java index 2e7200a33c480..bb32b82d59c51 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/columnfilter/ColumnFilterParser.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/columnfilter/ColumnFilterParser.java @@ -90,7 +90,7 @@ public Expression parseAndValidate(final String rawColumnFilter) throws Subscrip } } - Expression parse(final String rawColumnFilter) { + public Expression parse(final String rawColumnFilter) { if (rawColumnFilter == null || rawColumnFilter.trim().isEmpty()) { throw new ParsingException( DataNodeMiscMessages.COLUMN_FILTER_SHOULD_NOT_BE_EMPTY, null, 1, 1); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/batch/SubscriptionFilterSnapshot.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/batch/SubscriptionFilterSnapshot.java new file mode 100644 index 0000000000000..0294dcaebbb68 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/batch/SubscriptionFilterSnapshot.java @@ -0,0 +1,160 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.subscription.event.batch; + +import org.apache.iotdb.db.i18n.DataNodeMiscMessages; +import org.apache.iotdb.db.subscription.agent.SubscriptionAgent; +import org.apache.iotdb.db.subscription.broker.SubscriptionPrefetchingQueue; +import org.apache.iotdb.db.subscription.columnfilter.ColumnFilterMatcher; +import org.apache.iotdb.db.subscription.tagfilter.TagFilterEvaluationException; +import org.apache.iotdb.db.subscription.tagfilter.TagFilterMatcher; +import org.apache.iotdb.rpc.subscription.config.TopicConfig; +import org.apache.iotdb.rpc.subscription.config.TopicConstant; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; + +/** Immutable topic-filter state used by every event in one subscription batch. */ +final class SubscriptionFilterSnapshot { + + private static final int MAX_CAPTURE_ATTEMPTS = 3; + + private final boolean tableModel; + private final TopicConfig topicConfig; + private final Map filteringAttributes; + private final TagFilterMatcher tagFilterMatcher; + private final ColumnFilterMatcher columnFilterMatcher; + + private SubscriptionFilterSnapshot( + final boolean tableModel, + final TopicConfig topicConfig, + final TagFilterMatcher tagFilterMatcher, + final ColumnFilterMatcher columnFilterMatcher) { + this.tableModel = tableModel; + this.topicConfig = topicConfig; + this.filteringAttributes = filteringAttributes(topicConfig); + this.tagFilterMatcher = tagFilterMatcher; + this.columnFilterMatcher = columnFilterMatcher; + } + + static SubscriptionFilterSnapshot capture(final SubscriptionPrefetchingQueue queue) { + final boolean tableModel = + SubscriptionAgent.consumer().isTableModel(queue.getConsumerGroupId()); + for (int attempt = 0; attempt < MAX_CAPTURE_ATTEMPTS; attempt++) { + final TopicConfig topicConfig = getCurrentTopicConfig(queue, tableModel); + if (Objects.isNull(topicConfig)) { + throw new TagFilterEvaluationException( + DataNodeMiscMessages + .EXCEPTION_TOPIC_CONFIGURATION_IS_NOT_AVAILABLE_FOR_TAG_FILTER_1E023E3A); + } + + final TagFilterMatcher tagFilterMatcher = + SubscriptionAgent.broker().getTagFilterMatcher(queue.getTopicName(), tableModel); + tagFilterMatcher.throwIfFailure(); + final ColumnFilterMatcher columnFilterMatcher = + SubscriptionAgent.broker().getColumnFilterMatcher(queue.getTopicName(), tableModel); + + final TopicConfig currentTopicConfig = getCurrentTopicConfig(queue, tableModel); + if (Objects.nonNull(currentTopicConfig) + && filteringAttributes(topicConfig).equals(filteringAttributes(currentTopicConfig)) + && tagFilterMatcher + == SubscriptionAgent.broker().getTagFilterMatcher(queue.getTopicName(), tableModel) + && columnFilterMatcher + == SubscriptionAgent.broker() + .getColumnFilterMatcher(queue.getTopicName(), tableModel)) { + return new SubscriptionFilterSnapshot( + tableModel, + new TopicConfig(new HashMap<>(topicConfig.getAttribute())), + tagFilterMatcher, + columnFilterMatcher); + } + } + + throw new TagFilterEvaluationException( + DataNodeMiscMessages + .EXCEPTION_TOPIC_CONFIGURATION_CHANGED_WHILE_CAPTURING_TAG_FILTER_SNAPSHOT_984CEB1B); + } + + boolean isCurrent(final SubscriptionPrefetchingQueue queue) { + if (!tableModel) { + return true; + } + final TopicConfig currentTopicConfig = getCurrentTopicConfig(queue, true); + return Objects.nonNull(currentTopicConfig) + && filteringAttributes.equals(filteringAttributes(currentTopicConfig)) + && tagFilterMatcher + == SubscriptionAgent.broker().getTagFilterMatcher(queue.getTopicName(), true) + && columnFilterMatcher + == SubscriptionAgent.broker().getColumnFilterMatcher(queue.getTopicName(), true); + } + + TopicConfig getTopicConfig() { + return topicConfig; + } + + TagFilterMatcher getTagFilterMatcher() { + return tagFilterMatcher; + } + + ColumnFilterMatcher getColumnFilterMatcher() { + return columnFilterMatcher; + } + + boolean hasNonTrivialFilter() { + return tableModel && (!tagFilterMatcher.isMatchAll() || !columnFilterMatcher.isMatchAll()); + } + + private static TopicConfig getCurrentTopicConfig( + final SubscriptionPrefetchingQueue queue, final boolean tableModel) { + return SubscriptionAgent.topic() + .getTopicConfigs(Collections.singleton(queue.getTopicName()), tableModel) + .get(queue.getTopicName()); + } + + private static Map filteringAttributes(final TopicConfig topicConfig) { + if (Objects.isNull(topicConfig) || Objects.isNull(topicConfig.getAttribute())) { + return Collections.emptyMap(); + } + final Map result = new HashMap<>(); + topicConfig + .getAttribute() + .forEach( + (key, value) -> { + if (Objects.isNull(key)) { + return; + } + final String normalizedKey = key.trim().toLowerCase(Locale.ROOT); + if (!isOwnerAttribute(normalizedKey)) { + result.put(normalizedKey, value); + } + }); + return result; + } + + private static boolean isOwnerAttribute(final String key) { + return TopicConstant.OWNER_ID_KEY.equals(key) + || TopicConstant.OWNER_EPOCH_KEY.equals(key) + || TopicConstant.MAX_OWNER_EPOCH_KEY.equals(key) + || TopicConstant.OWNER_LEASE_DURATION_MS_KEY.equals(key); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/batch/SubscriptionPipeEventBatch.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/batch/SubscriptionPipeEventBatch.java index d9361bf089e1d..741baba6bc32a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/batch/SubscriptionPipeEventBatch.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/batch/SubscriptionPipeEventBatch.java @@ -114,7 +114,7 @@ protected synchronized boolean emit(final Consumer consumer) /////////////////////////////// utility /////////////////////////////// - protected abstract void onTabletInsertionEvent(final TabletInsertionEvent event); + protected abstract void onTabletInsertionEvent(final TabletInsertionEvent event) throws Exception; protected abstract void onTsFileInsertionEvent(final TsFileInsertionEvent event); @@ -122,6 +122,10 @@ protected synchronized boolean emit(final Consumer consumer) protected abstract List generateSubscriptionEvents() throws Exception; + protected boolean isCompatibleWithCurrentTopicConfig() { + return true; + } + //////////////////////////// APIs provided for metric framework //////////////////////////// public int getPipeEventCount() { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/batch/SubscriptionPipeEventBatches.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/batch/SubscriptionPipeEventBatches.java index 0cb81ca43e699..d16b357b34152 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/batch/SubscriptionPipeEventBatches.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/batch/SubscriptionPipeEventBatches.java @@ -25,6 +25,7 @@ import org.apache.iotdb.db.subscription.broker.SubscriptionPrefetchingTabletQueue; import org.apache.iotdb.db.subscription.broker.SubscriptionPrefetchingTsFileQueue; import org.apache.iotdb.db.subscription.event.SubscriptionEvent; +import org.apache.iotdb.rpc.subscription.exception.SubscriptionException; import com.google.common.collect.ImmutableList; import org.slf4j.Logger; @@ -77,6 +78,7 @@ public boolean onEvent(final Consumer consumer) { } } catch (final Exception e) { LOGGER.warn(DataNodeMiscMessages.EXCEPTION_SEALING_EVENTS, batch, e); + throw propagate(e); } if (hasNew.get()) { regionIdToBatch.remove(regionId); @@ -103,33 +105,35 @@ public boolean onEvent(final EnrichedEvent event, final Consumer 0) { + if (!batch.emit(consumer)) { + throw new SubscriptionException( + DataNodeMiscMessages.EXCEPTION_FAILED_TO_SEAL_SUBSCRIPTION_EVENT_BATCH_1FB7E92C); + } + hasNew.set(true); + regionIdToBatch.remove(regionId); + batch = createBatch(regionId); + } + + final boolean emittedCurrentBatch; try { - if (batch.onEvent(event, consumer)) { + emittedCurrentBatch = batch.onEvent(event, consumer); + if (emittedCurrentBatch) { hasNew.set(true); } } catch (final Exception e) { LOGGER.warn(DataNodeMiscMessages.EXCEPTION_SEALING_EVENTS, batch, e); + throw e; } - if (hasNew.get()) { + if (emittedCurrentBatch) { regionIdToBatch.remove(regionId); } else { regionIdToBatch.put(regionId, batch); @@ -175,4 +179,26 @@ public void cleanUp() { regionIdToBatch.values().forEach(batch -> batch.cleanUp(true)); regionIdToBatch.clear(); } + + private SubscriptionPipeEventBatch createBatch(final int regionId) { + return prefetchingQueue instanceof SubscriptionPrefetchingTabletQueue + ? new SubscriptionPipeTabletEventBatch( + regionId, + (SubscriptionPrefetchingTabletQueue) prefetchingQueue, + maxDelayInMs, + maxBatchSizeInBytes) + : new SubscriptionPipeTsFileEventBatch( + regionId, + (SubscriptionPrefetchingTsFileQueue) prefetchingQueue, + maxDelayInMs, + maxBatchSizeInBytes); + } + + private static RuntimeException propagate(final Exception exception) { + return exception instanceof RuntimeException + ? (RuntimeException) exception + : new SubscriptionException( + DataNodeMiscMessages.EXCEPTION_FAILED_TO_SEAL_SUBSCRIPTION_EVENT_BATCH_1FB7E92C, + exception); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/batch/SubscriptionPipeTabletEventBatch.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/batch/SubscriptionPipeTabletEventBatch.java index 67321b0465168..ad6601073f359 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/batch/SubscriptionPipeTabletEventBatch.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/batch/SubscriptionPipeTabletEventBatch.java @@ -19,27 +19,24 @@ package org.apache.iotdb.db.subscription.event.batch; -import org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant; import org.apache.iotdb.commons.pipe.event.EnrichedEvent; -import org.apache.iotdb.commons.schema.table.TreeViewSchema; -import org.apache.iotdb.commons.schema.table.TsTable; import org.apache.iotdb.commons.subscription.config.SubscriptionConfig; +import org.apache.iotdb.db.i18n.DataNodeMiscMessages; import org.apache.iotdb.db.i18n.DataNodePipeMessages; import org.apache.iotdb.db.pipe.event.common.tablet.PipeInsertNodeTabletInsertionEvent; import org.apache.iotdb.db.pipe.event.common.tablet.PipeRawTabletInsertionEvent; import org.apache.iotdb.db.pipe.event.common.tsfile.PipeTsFileInsertionEvent; import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryWeightUtil; -import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; import org.apache.iotdb.db.subscription.agent.SubscriptionAgent; import org.apache.iotdb.db.subscription.broker.SubscriptionPrefetchingTabletQueue; import org.apache.iotdb.db.subscription.columnfilter.ColumnFilterMatcher; import org.apache.iotdb.db.subscription.columnfilter.TabletColumnPruner; -import org.apache.iotdb.db.subscription.columnfilter.TreeViewTabletProjector; import org.apache.iotdb.db.subscription.event.SubscriptionEvent; +import org.apache.iotdb.db.subscription.tagfilter.TabletTagFilter; +import org.apache.iotdb.db.subscription.tagfilter.TagFilterEvaluationException; import org.apache.iotdb.metrics.core.utils.IoTDBMovingAverage; import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent; import org.apache.iotdb.pipe.api.event.dml.insertion.TsFileInsertionEvent; -import org.apache.iotdb.rpc.subscription.config.TopicConfig; import org.apache.iotdb.rpc.subscription.config.TopicConstant; import com.codahale.metrics.Clock; @@ -50,7 +47,6 @@ import org.slf4j.LoggerFactory; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.Iterator; import java.util.List; @@ -73,8 +69,8 @@ public class SubscriptionPipeTabletEventBatch extends SubscriptionPipeEventBatch private final Meter insertNodeTabletInsertionEventSizeEstimator; private final Meter rawTabletInsertionEventSizeEstimator; - private volatile boolean treeViewTabletProjectorInitialized; - private volatile TreeViewTabletProjector treeViewTabletProjector; + private volatile SubscriptionFilterSnapshot filterSnapshot; + private volatile SubscriptionTreeViewProjector treeViewProjector; private volatile SubscriptionPipeTabletIterationSnapshot iterationSnapshot; private final AtomicInteger referenceCount = new AtomicInteger(); @@ -132,6 +128,8 @@ public synchronized void cleanUp(final boolean force) { @Override protected void onTabletInsertionEvent(final TabletInsertionEvent event) { + ensureFilterSnapshot(); + // update processing time if (firstEventProcessingTime == Long.MIN_VALUE) { firstEventProcessingTime = System.currentTimeMillis(); @@ -148,6 +146,8 @@ protected void onTabletInsertionEvent(final TabletInsertionEvent event) { @Override protected void onTsFileInsertionEvent(final TsFileInsertionEvent event) { + ensureFilterSnapshot(); + // update processing time if (firstEventProcessingTime == Long.MIN_VALUE) { firstEventProcessingTime = System.currentTimeMillis(); @@ -163,6 +163,7 @@ protected void onTsFileInsertionEvent(final TsFileInsertionEvent event) { @Override protected List generateSubscriptionEvents() { + ensureFilterSnapshot(); if (!prepareTreeViewTabletProjectorForEmission()) { return null; } @@ -226,105 +227,46 @@ private Pair> projectTreeViewTabletsIfNecessary( return tablets; } - final TreeViewTabletProjector projector = getTreeViewTabletProjector(); - if (Objects.isNull(projector)) { - return tablets; + if (!prepareTreeViewTabletProjectorForEmission() || !treeViewProjector.isAvailable()) { + if (!ensureFilterSnapshot().hasNonTrivialFilter()) { + return tablets; + } + throw new TagFilterEvaluationException( + DataNodeMiscMessages + .EXCEPTION_TREE_VIEW_PROJECTOR_IS_UNAVAILABLE_FOR_FILTERED_SUBSCRIPTION_DATA_B5F396A5); } final List projectedTablets = new ArrayList<>(tablets.right.size()); for (final Tablet tablet : tablets.right) { - final Tablet projectedTablet = projector.project(tablet); + final Tablet projectedTablet = treeViewProjector.project(tablet); if (Objects.nonNull(projectedTablet)) { projectedTablets.add(projectedTablet); } } return projectedTablets.isEmpty() ? null - : new Pair<>(projector.getDatabaseName(), projectedTablets); - } - - private TreeViewTabletProjector getTreeViewTabletProjector() { - return prepareTreeViewTabletProjectorForEmission() ? treeViewTabletProjector : null; + : new Pair<>(treeViewProjector.getDatabaseName(), projectedTablets); } private boolean prepareTreeViewTabletProjectorForEmission() { - if (treeViewTabletProjectorInitialized) { - return true; + final SubscriptionFilterSnapshot snapshot = ensureFilterSnapshot(); + if (Objects.isNull(treeViewProjector)) { + treeViewProjector = new SubscriptionTreeViewProjector(snapshot.getTopicConfig()); } - - synchronized (this) { - if (treeViewTabletProjectorInitialized) { - return true; - } - - final TopicConfig topicConfig = - SubscriptionAgent.topic() - .getTopicConfigs( - Collections.singleton(prefetchingQueue.getTopicName()), - SubscriptionAgent.consumer().isTableModel(prefetchingQueue.getConsumerGroupId())) - .get(prefetchingQueue.getTopicName()); - if (Objects.isNull(topicConfig)) { - return false; - } - if (!topicConfig.isTableTopic()) { - treeViewTabletProjectorInitialized = true; - return true; - } - - final String database = - topicConfig.getStringOrDefault( - TopicConstant.DATABASE_KEY, TopicConstant.DATABASE_DEFAULT_VALUE); - final String tableName = - topicConfig.getStringOrDefault( - TopicConstant.TABLE_KEY, TopicConstant.TABLE_DEFAULT_VALUE); - if (isDefaultTopicPattern(database, TopicConstant.DATABASE_DEFAULT_VALUE) - || isDefaultTopicPattern(tableName, TopicConstant.TABLE_DEFAULT_VALUE) - || !isLiteralTopicPattern(database) - || !isLiteralTopicPattern(tableName)) { - treeViewTabletProjectorInitialized = true; - return true; - } - - if (!isTreeCapturedByTopic(topicConfig) && topicConfig.isColumnFilterTrivial()) { - treeViewTabletProjectorInitialized = true; - return true; - } - - final TsTable table = DataNodeTableCache.getInstance().getTable(database, tableName, false); - if (Objects.isNull(table)) { - LOGGER.debug( - DataNodePipeMessages - .PIPE_LOG_SUBSCRIPTIONPIPETABLETEVENTBATCH_POSTPONE_EMITTING_SUBSCRIPTION_TABLET_BATCH_FOR_TOPIC_ARG_BECAUSE_TABLE_SCHEMA_ARG_ARG_IS_NOT_AVAILABLE_LOCALLY_996C618D, - prefetchingQueue.getTopicName(), - database, - tableName); - return false; - } - if (TreeViewSchema.isTreeViewTable(table)) { - treeViewTabletProjector = new TreeViewTabletProjector(database, table); - } - - treeViewTabletProjectorInitialized = true; - return true; + final boolean prepared = treeViewProjector.prepare(); + if (!prepared) { + LOGGER.debug( + DataNodePipeMessages + .PIPE_LOG_SUBSCRIPTIONPIPETABLETEVENTBATCH_POSTPONE_EMITTING_SUBSCRIPTION_TABLET_BATCH_FOR_TOPIC_ARG_BECAUSE_TABLE_SCHEMA_ARG_ARG_IS_NOT_AVAILABLE_LOCALLY_996C618D, + prefetchingQueue.getTopicName(), + snapshot + .getTopicConfig() + .getStringOrDefault(TopicConstant.DATABASE_KEY, TopicConstant.DATABASE_DEFAULT_VALUE), + snapshot + .getTopicConfig() + .getStringOrDefault(TopicConstant.TABLE_KEY, TopicConstant.TABLE_DEFAULT_VALUE)); } - } - - private static boolean isDefaultTopicPattern(final String pattern, final String defaultPattern) { - return Objects.isNull(pattern) || defaultPattern.equals(pattern.trim()); - } - - private static boolean isLiteralTopicPattern(final String pattern) { - final String regexMetaCharacters = ".*+?[](){}\\|^$"; - return Objects.nonNull(pattern) - && pattern.chars().noneMatch(c -> regexMetaCharacters.indexOf((char) c) >= 0); - } - - private static boolean isTreeCapturedByTopic(final TopicConfig topicConfig) { - return topicConfig.getBooleanOrDefault( - Arrays.asList( - PipeSourceConstant.EXTRACTOR_CAPTURE_TREE_KEY, - PipeSourceConstant.SOURCE_CAPTURE_TREE_KEY), - false); + return prepared; } private Pair> pruneTablets(final Pair> tablets) { @@ -332,16 +274,15 @@ private Pair> pruneTablets(final Pair> return tablets; } - final ColumnFilterMatcher matcher = - SubscriptionAgent.broker() - .getColumnFilterMatcher( - prefetchingQueue.getTopicName(), - SubscriptionAgent.consumer().isTableModel(prefetchingQueue.getConsumerGroupId())); + final SubscriptionFilterSnapshot snapshot = ensureFilterSnapshot(); final List prunedTablets = new ArrayList<>(tablets.right.size()); for (final Tablet tablet : tablets.right) { final Tablet prunedTablet = - TabletColumnPruner.pruneTableModelTablet(tablet, tablets.left, matcher); + TabletColumnPruner.pruneTableModelTablet( + TabletTagFilter.filter(tablet, snapshot.getTagFilterMatcher(), tablets.left), + tablets.left, + snapshot.getColumnFilterMatcher()); if (Objects.nonNull(prunedTablet)) { prunedTablets.add(prunedTablet); } @@ -349,6 +290,18 @@ private Pair> pruneTablets(final Pair> return prunedTablets.isEmpty() ? null : new Pair<>(tablets.left, prunedTablets); } + @Override + protected boolean isCompatibleWithCurrentTopicConfig() { + return Objects.isNull(filterSnapshot) || filterSnapshot.isCurrent(prefetchingQueue); + } + + private synchronized SubscriptionFilterSnapshot ensureFilterSnapshot() { + if (Objects.isNull(filterSnapshot)) { + filterSnapshot = SubscriptionFilterSnapshot.capture(prefetchingQueue); + } + return filterSnapshot; + } + /////////////////////////////// estimator /////////////////////////////// private long getEstimatedInsertNodeTabletInsertionEventSize() { @@ -381,6 +334,10 @@ public synchronized SubscriptionPipeTabletIterationSnapshot sendIterationSnapsho return result; } + public ColumnFilterMatcher getColumnFilterMatcher() { + return ensureFilterSnapshot().getColumnFilterMatcher(); + } + public synchronized void resetForIteration() { currentEnrichedEventsIterator = enrichedEvents.iterator(); currentTabletInsertionEventsIterator = null; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/batch/SubscriptionPipeTsFileEventBatch.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/batch/SubscriptionPipeTsFileEventBatch.java index 7a27f464ebbe5..35f9b1fe77a8b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/batch/SubscriptionPipeTsFileEventBatch.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/batch/SubscriptionPipeTsFileEventBatch.java @@ -21,16 +21,19 @@ import org.apache.iotdb.commons.pipe.event.EnrichedEvent; import org.apache.iotdb.commons.utils.TestOnly; +import org.apache.iotdb.db.i18n.DataNodeMiscMessages; import org.apache.iotdb.db.i18n.DataNodePipeMessages; import org.apache.iotdb.db.pipe.sink.payload.evolvable.batch.PipeTabletEventTsFileBatch; -import org.apache.iotdb.db.subscription.agent.SubscriptionAgent; +import org.apache.iotdb.db.pipe.sink.payload.evolvable.batch.PipeTabletEventTsFileBatch.TabletTransformResult; import org.apache.iotdb.db.subscription.broker.SubscriptionPrefetchingTsFileQueue; -import org.apache.iotdb.db.subscription.columnfilter.ColumnFilterMatcher; import org.apache.iotdb.db.subscription.columnfilter.TabletColumnPruner; import org.apache.iotdb.db.subscription.event.SubscriptionEvent; import org.apache.iotdb.db.subscription.event.pipe.SubscriptionPipeTsFileBatchEvents; +import org.apache.iotdb.db.subscription.tagfilter.TabletTagFilter; +import org.apache.iotdb.db.subscription.tagfilter.TagFilterEvaluationException; import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent; import org.apache.iotdb.pipe.api.event.dml.insertion.TsFileInsertionEvent; +import org.apache.iotdb.rpc.subscription.config.TopicConstant; import org.apache.iotdb.rpc.subscription.payload.poll.SubscriptionCommitContext; import org.apache.tsfile.utils.Pair; @@ -42,6 +45,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Objects; import java.util.concurrent.atomic.AtomicInteger; public class SubscriptionPipeTsFileEventBatch extends SubscriptionPipeEventBatch { @@ -51,6 +55,8 @@ public class SubscriptionPipeTsFileEventBatch extends SubscriptionPipeEventBatch private final PipeTabletEventTsFileBatch batch; private final List> sealedFilePairs = new ArrayList<>(); + private volatile SubscriptionFilterSnapshot filterSnapshot; + private volatile SubscriptionTreeViewProjector treeViewProjector; public SubscriptionPipeTsFileEventBatch( final int regionId, @@ -59,8 +65,7 @@ public SubscriptionPipeTsFileEventBatch( final long maxBatchSizeInBytes) { super(regionId, prefetchingQueue, maxDelayInMs, maxBatchSizeInBytes); this.batch = - new PipeTabletEventTsFileBatch( - maxDelayInMs, maxBatchSizeInBytes, this::pruneTableModelTablet); + new PipeTabletEventTsFileBatch(maxDelayInMs, maxBatchSizeInBytes, this::transformTablet); } @TestOnly @@ -99,12 +104,11 @@ public synchronized void cleanUp(final boolean force) { /////////////////////////////// utility /////////////////////////////// @Override - protected void onTabletInsertionEvent(final TabletInsertionEvent event) { - try { - batch.onEvent(event); - } catch (final Exception ignored) { - // no exceptions will be thrown - } + protected void onTabletInsertionEvent(final TabletInsertionEvent event) throws Exception { + ensureFilterSnapshot(); + // Keep the queue's reference when transformation fails: the prefetching queue retries this + // same event without acquiring another reference. Release it only after successful batching. + batch.onEvent(event); ((EnrichedEvent) event) .decreaseReferenceCount( SubscriptionPipeTsFileEventBatch.class.getName(), @@ -123,8 +127,7 @@ protected void onTsFileInsertionEvent(final TsFileInsertionEvent event) { @Override protected List generateSubscriptionEvents() throws Exception { if (batch.isEmpty()) { - enrichedEvents.clear(); - return Collections.emptyList(); + return discardEmptyBatch(); } final List events = new ArrayList<>(); @@ -132,8 +135,7 @@ protected List generateSubscriptionEvents() throws Exception if (dbTsFilePairs.isEmpty()) { batch.decreaseEventsReferenceCount(this.getClass().getName(), true); batch.onSuccess(); - enrichedEvents.clear(); - return Collections.emptyList(); + return discardEmptyBatch(); } sealedFilePairs.addAll(dbTsFilePairs); final AtomicInteger ackReferenceCount = new AtomicInteger(dbTsFilePairs.size()); @@ -151,17 +153,95 @@ protected List generateSubscriptionEvents() throws Exception return events; } + /** Closes the inner conversion batch when filtering produced no TsFile payload. */ + private List discardEmptyBatch() { + try { + batch.close(); + } finally { + enrichedEvents.clear(); + } + return Collections.emptyList(); + } + @Override protected boolean shouldEmit() { return (!enrichedEvents.isEmpty() && batch.isEmpty()) || batch.shouldEmit(); } - private Tablet pruneTableModelTablet(final String databaseName, final Tablet tablet) { - final ColumnFilterMatcher matcher = - SubscriptionAgent.broker() - .getColumnFilterMatcher( - prefetchingQueue.getTopicName(), - SubscriptionAgent.consumer().isTableModel(prefetchingQueue.getConsumerGroupId())); - return TabletColumnPruner.pruneTableModelTablet(tablet, databaseName, matcher); + private TabletTransformResult transformTablet( + final String databaseName, + final Tablet tablet, + final boolean isTableModel, + final boolean isAligned) { + final SubscriptionFilterSnapshot snapshot = ensureFilterSnapshot(); + if (isTableModel) { + return new TabletTransformResult( + pruneTableModelTablet(databaseName, tablet, snapshot), databaseName, true, isAligned); + } + if (!snapshot.getTopicConfig().isTableTopic()) { + return new TabletTransformResult(tablet, databaseName, false, isAligned); + } + + if (!prepareTreeViewProjector(snapshot)) { + throw new TagFilterEvaluationException( + DataNodeMiscMessages.EXCEPTION_TABLE_SCHEMA_IS_NOT_AVAILABLE_FOR_TAG_FILTER_993AB728); + } + if (!treeViewProjector.isAvailable()) { + if (snapshot.hasNonTrivialFilter()) { + throw new TagFilterEvaluationException( + DataNodeMiscMessages + .EXCEPTION_TREE_VIEW_PROJECTOR_IS_UNAVAILABLE_FOR_FILTERED_SUBSCRIPTION_DATA_B5F396A5); + } + return new TabletTransformResult(tablet, databaseName, false, isAligned); + } + + final Tablet projectedTablet = treeViewProjector.project(tablet); + return Objects.isNull(projectedTablet) + ? null + : new TabletTransformResult( + pruneTableModelTablet(treeViewProjector.getDatabaseName(), projectedTablet, snapshot), + treeViewProjector.getDatabaseName(), + true, + false); + } + + private Tablet pruneTableModelTablet( + final String databaseName, final Tablet tablet, final SubscriptionFilterSnapshot snapshot) { + return TabletColumnPruner.pruneTableModelTablet( + TabletTagFilter.filter(tablet, snapshot.getTagFilterMatcher(), databaseName), + databaseName, + snapshot.getColumnFilterMatcher()); + } + + private boolean prepareTreeViewProjector(final SubscriptionFilterSnapshot snapshot) { + if (Objects.isNull(treeViewProjector)) { + treeViewProjector = new SubscriptionTreeViewProjector(snapshot.getTopicConfig()); + } + final boolean prepared = treeViewProjector.prepare(); + if (!prepared) { + LOGGER.debug( + DataNodePipeMessages + .PIPE_LOG_SUBSCRIPTIONPIPETABLETEVENTBATCH_POSTPONE_EMITTING_SUBSCRIPTION_TABLET_BATCH_FOR_TOPIC_ARG_BECAUSE_TABLE_SCHEMA_ARG_ARG_IS_NOT_AVAILABLE_LOCALLY_996C618D, + prefetchingQueue.getTopicName(), + snapshot + .getTopicConfig() + .getStringOrDefault(TopicConstant.DATABASE_KEY, TopicConstant.DATABASE_DEFAULT_VALUE), + snapshot + .getTopicConfig() + .getStringOrDefault(TopicConstant.TABLE_KEY, TopicConstant.TABLE_DEFAULT_VALUE)); + } + return prepared; + } + + @Override + protected boolean isCompatibleWithCurrentTopicConfig() { + return Objects.isNull(filterSnapshot) || filterSnapshot.isCurrent(prefetchingQueue); + } + + private synchronized SubscriptionFilterSnapshot ensureFilterSnapshot() { + if (Objects.isNull(filterSnapshot)) { + filterSnapshot = SubscriptionFilterSnapshot.capture(prefetchingQueue); + } + return filterSnapshot; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/batch/SubscriptionTreeViewProjector.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/batch/SubscriptionTreeViewProjector.java new file mode 100644 index 0000000000000..b0b6ef1fa516c --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/batch/SubscriptionTreeViewProjector.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.subscription.event.batch; + +import org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant; +import org.apache.iotdb.commons.schema.table.TreeViewSchema; +import org.apache.iotdb.commons.schema.table.TsTable; +import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; +import org.apache.iotdb.db.subscription.columnfilter.TreeViewTabletProjector; +import org.apache.iotdb.rpc.subscription.config.TopicConfig; +import org.apache.iotdb.rpc.subscription.config.TopicConstant; + +import org.apache.tsfile.write.record.Tablet; + +import java.util.Arrays; +import java.util.Objects; + +/** + * Resolves and applies the table projection for tree-model events captured by a Tree View topic. + */ +final class SubscriptionTreeViewProjector { + + private final TopicConfig topicConfig; + + private boolean initialized; + private TreeViewTabletProjector projector; + + SubscriptionTreeViewProjector(final TopicConfig topicConfig) { + this.topicConfig = topicConfig; + } + + synchronized boolean prepare() { + if (initialized) { + return true; + } + if (Objects.isNull(topicConfig) || !topicConfig.isTableTopic()) { + initialized = true; + return true; + } + + final String database = + topicConfig.getStringOrDefault( + TopicConstant.DATABASE_KEY, TopicConstant.DATABASE_DEFAULT_VALUE); + final String tableName = + topicConfig.getStringOrDefault(TopicConstant.TABLE_KEY, TopicConstant.TABLE_DEFAULT_VALUE); + if (isDefaultTopicPattern(database, TopicConstant.DATABASE_DEFAULT_VALUE) + || isDefaultTopicPattern(tableName, TopicConstant.TABLE_DEFAULT_VALUE) + || !isLiteralTopicPattern(database) + || !isLiteralTopicPattern(tableName)) { + initialized = true; + return true; + } + + if (!isTreeCapturedByTopic(topicConfig) + && topicConfig.isColumnFilterTrivial() + && topicConfig.isTagFilterTrivial()) { + initialized = true; + return true; + } + + final TsTable table = DataNodeTableCache.getInstance().getTable(database, tableName, false); + if (Objects.isNull(table)) { + return false; + } + if (TreeViewSchema.isTreeViewTable(table)) { + projector = new TreeViewTabletProjector(database, table); + } + initialized = true; + return true; + } + + Tablet project(final Tablet tablet) { + return Objects.nonNull(projector) ? projector.project(tablet) : null; + } + + boolean isAvailable() { + return Objects.nonNull(projector); + } + + String getDatabaseName() { + return Objects.nonNull(projector) ? projector.getDatabaseName() : null; + } + + private static boolean isDefaultTopicPattern(final String pattern, final String defaultPattern) { + return Objects.isNull(pattern) || defaultPattern.equals(pattern.trim()); + } + + private static boolean isLiteralTopicPattern(final String pattern) { + final String regexMetaCharacters = ".*+?[](){}\\|^$"; + return Objects.nonNull(pattern) + && pattern.chars().noneMatch(c -> regexMetaCharacters.indexOf((char) c) >= 0); + } + + private static boolean isTreeCapturedByTopic(final TopicConfig topicConfig) { + return topicConfig.getBooleanOrDefault( + Arrays.asList( + PipeSourceConstant.EXTRACTOR_CAPTURE_TREE_KEY, + PipeSourceConstant.SOURCE_CAPTURE_TREE_KEY), + false); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/response/SubscriptionEventTabletResponse.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/response/SubscriptionEventTabletResponse.java index 1392c0cbb2865..3065f586871bf 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/response/SubscriptionEventTabletResponse.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/response/SubscriptionEventTabletResponse.java @@ -26,7 +26,6 @@ import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryManager; import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryWeightUtil; import org.apache.iotdb.db.pipe.resource.memory.PipeTabletMemoryBlock; -import org.apache.iotdb.db.subscription.agent.SubscriptionAgent; import org.apache.iotdb.db.subscription.broker.SubscriptionPrefetchingQueue; import org.apache.iotdb.db.subscription.columnfilter.ColumnFilterMatcher; import org.apache.iotdb.db.subscription.event.SubscriptionEvent; @@ -95,11 +94,7 @@ public SubscriptionEventTabletResponse( this.commitContext = commitContext; this.rootCommitContext = rootCommitContext; - this.columnFilterMatcher = - SubscriptionAgent.broker() - .getColumnFilterMatcher( - queue.getTopicName(), - SubscriptionAgent.consumer().isTableModel(queue.getConsumerGroupId())); + this.columnFilterMatcher = batch.getColumnFilterMatcher(); init(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/tagfilter/TabletTagFilter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/tagfilter/TabletTagFilter.java new file mode 100644 index 0000000000000..bdad533b38619 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/tagfilter/TabletTagFilter.java @@ -0,0 +1,362 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.subscription.tagfilter; + +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Identifier; +import org.apache.iotdb.db.i18n.DataNodeMiscMessages; +import org.apache.iotdb.db.pipe.event.common.tablet.PipeTabletUtils; + +import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; + +/** Filters rows of a table-model Tablet according to TAG column values. */ +public class TabletTagFilter { + + private static final String DELIMITED_FIELD_PREFIX = "D:"; + private static final String UNDELIMITED_FIELD_PREFIX = "U:"; + + private TabletTagFilter() { + // utility class + } + + public static Tablet filter(final Tablet tablet, final TagFilterMatcher matcher) { + return filter(tablet, matcher, null); + } + + public static Tablet filter( + final Tablet tablet, final TagFilterMatcher matcher, final String databaseName) { + if (Objects.isNull(tablet)) { + return null; + } + + final TagFilterMatcher effectiveMatcher = + Objects.nonNull(matcher) ? matcher : TagFilterMatcher.matchAll(); + effectiveMatcher.throwIfFailure(); + if (effectiveMatcher.isMatchAll()) { + return tablet; + } + if (effectiveMatcher.isMatchNone() || tablet.getRowSize() <= 0) { + return null; + } + + final List schemas = tablet.getSchemas(); + final Object[] values = tablet.getValues(); + if (Objects.isNull(schemas) || schemas.isEmpty()) { + throw TagFilterEvaluationException.schemaBinding( + DataNodeMiscMessages.EXCEPTION_TABLET_SCHEMA_IS_MISSING_164075B0); + } + if (Objects.isNull(values) || values.length < schemas.size()) { + throw new TagFilterEvaluationException( + DataNodeMiscMessages.EXCEPTION_TABLET_VALUE_COLUMNS_ARE_INCOMPLETE_DCD09F3C); + } + + final List categories = getColumnCategories(tablet, schemas.size()); + validateTabletStructure(tablet, schemas, values, categories); + validateBinding(effectiveMatcher, databaseName, tablet.getTableName(), schemas, categories); + final Map resolvedFields = + resolveReferencedTagColumns(effectiveMatcher.getReferencedFields(), schemas, categories); + + final List selectedRows = new ArrayList<>(); + try { + for (int rowIndex = 0; rowIndex < tablet.getRowSize(); rowIndex++) { + final int currentRowIndex = rowIndex; + if (effectiveMatcher.matches( + identifier -> + getTagValue(tablet, currentRowIndex, resolvedFields.get(toFieldKey(identifier))))) { + selectedRows.add(rowIndex); + } + } + } catch (final RuntimeException e) { + if (e instanceof TagFilterEvaluationException) { + throw e; + } + throw new TagFilterEvaluationException( + DataNodeMiscMessages.EXCEPTION_FAILED_TO_EVALUATE_A_TABLET_ROW_7E4E94CE, e); + } + + if (selectedRows.isEmpty()) { + return null; + } + if (selectedRows.size() == tablet.getRowSize()) { + return tablet; + } + return copySelectedRows(tablet, schemas, categories, selectedRows); + } + + private static void validateBinding( + final TagFilterMatcher matcher, + final String databaseName, + final String tableName, + final List schemas, + final List categories) { + if (!matcher.isBindingEnforced()) { + return; + } + final TagFilterMatcher.TableBinding binding = + matcher.getTableBindings().get(TagFilterMatcher.TableKey.of(databaseName, tableName)); + if (binding == null) { + throw TagFilterEvaluationException.schemaBinding( + DataNodeMiscMessages.EXCEPTION_TABLE_BINDING_IS_NOT_AVAILABLE_882C5F2F); + } + if (binding.isFailed()) { + throw TagFilterEvaluationException.schemaBinding(binding.getFailureReason()); + } + for (final Identifier identifier : matcher.getReferencedFields()) { + final String normalizedName = identifier.getValue().toLowerCase(Locale.ROOT); + if (!binding.getTagNames().contains(normalizedName)) { + throw TagFilterEvaluationException.schemaBinding( + DataNodeMiscMessages.EXCEPTION_REFERENCED_COLUMN_IS_NOT_A_TAG_COLUMN_33FA34BC); + } + } + // A runtime tablet must carry category metadata consistent with the binding. Missing or + // changed categories are structural/schema failures, never SQL NULL values. + for (int i = 0; i < schemas.size(); i++) { + final String name = schemas.get(i).getMeasurementName(); + if (binding.getAllNames().contains(name.toLowerCase(Locale.ROOT)) + && categories.get(i) != ColumnCategory.TAG + && matcher.getReferencedFields().stream() + .anyMatch(identifier -> identifier.getValue().equalsIgnoreCase(name))) { + throw TagFilterEvaluationException.schemaBinding( + DataNodeMiscMessages.EXCEPTION_REFERENCED_COLUMN_CATEGORY_CHANGED_564594D8); + } + } + } + + private static Map resolveReferencedTagColumns( + final List referencedFields, + final List schemas, + final List categories) { + final Map exactTagColumns = new HashMap<>(); + final Map normalizedTagColumns = new HashMap<>(); + final Map exactColumns = new HashMap<>(); + final Map normalizedColumns = new HashMap<>(); + for (int i = 0; i < schemas.size(); i++) { + final IMeasurementSchema schema = schemas.get(i); + if (Objects.isNull(schema) || Objects.isNull(schema.getMeasurementName())) { + continue; + } + exactColumns.put(schema.getMeasurementName(), categories.get(i)); + normalizedColumns.put( + schema.getMeasurementName().toLowerCase(Locale.ROOT), categories.get(i)); + if (categories.get(i) != ColumnCategory.TAG) { + continue; + } + exactTagColumns.put(schema.getMeasurementName(), i); + normalizedTagColumns.put(schema.getMeasurementName().toLowerCase(Locale.ROOT), i); + } + + final Map result = new HashMap<>(); + for (final Identifier identifier : referencedFields) { + final Integer columnIndex = + identifier.isDelimited() + ? exactTagColumns.get(identifier.getValue()) + : normalizedTagColumns.get(identifier.getValue().toLowerCase(Locale.ROOT)); + if (Objects.isNull(columnIndex)) { + final ColumnCategory category = + identifier.isDelimited() + ? exactColumns.get(identifier.getValue()) + : normalizedColumns.get(identifier.getValue().toLowerCase(Locale.ROOT)); + if (category != null && category != ColumnCategory.TAG) { + throw TagFilterEvaluationException.schemaBinding( + DataNodeMiscMessages.EXCEPTION_REFERENCED_COLUMN_IS_NOT_A_TAG_COLUMN_33FA34BC); + } + throw TagFilterEvaluationException.schemaBinding( + DataNodeMiscMessages.EXCEPTION_REFERENCED_TAG_COLUMN_IS_MISSING_55E07377); + } + result.put(toFieldKey(identifier), columnIndex); + } + return result; + } + + private static String getTagValue( + final Tablet tablet, final int rowIndex, final Integer columnIndex) { + if (Objects.isNull(columnIndex) || tablet.isNull(rowIndex, columnIndex)) { + return null; + } + final Object value = tablet.getValue(rowIndex, columnIndex); + if (value instanceof Binary) { + return ((Binary) value).getStringValue(TSFileConfig.STRING_CHARSET); + } + return Objects.nonNull(value) ? String.valueOf(value) : null; + } + + private static Tablet copySelectedRows( + final Tablet tablet, + final List schemas, + final List categories, + final List selectedRows) { + final List columnNames = new ArrayList<>(schemas.size()); + final List dataTypes = new ArrayList<>(schemas.size()); + for (final IMeasurementSchema schema : schemas) { + if (Objects.isNull(schema) + || Objects.isNull(schema.getMeasurementName()) + || Objects.isNull(schema.getType())) { + throw TagFilterEvaluationException.schemaBinding( + DataNodeMiscMessages.EXCEPTION_TABLET_MEASUREMENT_SCHEMA_IS_INCOMPLETE_6A813472); + } + columnNames.add(schema.getMeasurementName()); + dataTypes.add(schema.getType()); + } + + final Tablet result = + new Tablet(tablet.getTableName(), columnNames, dataTypes, categories, selectedRows.size()); + try { + for (int targetRow = 0; targetRow < selectedRows.size(); targetRow++) { + final int sourceRow = selectedRows.get(targetRow); + PipeTabletUtils.putTimestamp(result, targetRow, tablet.getTimestamp(sourceRow)); + for (int columnIndex = 0; columnIndex < schemas.size(); columnIndex++) { + if (tablet.isNull(sourceRow, columnIndex)) { + PipeTabletUtils.markNullValue(result, targetRow, columnIndex); + } else { + final Object value = tablet.getValue(sourceRow, columnIndex); + if (Objects.isNull(value)) { + PipeTabletUtils.markNullValue(result, targetRow, columnIndex); + } else { + PipeTabletUtils.putValue( + result, targetRow, columnIndex, dataTypes.get(columnIndex), value); + } + } + } + } + return result; + } catch (final RuntimeException e) { + if (e instanceof TagFilterEvaluationException) { + throw e; + } + throw new TagFilterEvaluationException( + DataNodeMiscMessages.EXCEPTION_FAILED_TO_COMPACT_FILTERED_TABLET_5D4AD8AA, e); + } + } + + private static List getColumnCategories( + final Tablet tablet, final int columnCount) { + final List categories = tablet.getColumnTypes(); + if (Objects.isNull(categories) || categories.size() < columnCount) { + throw TagFilterEvaluationException.schemaBinding( + DataNodeMiscMessages.EXCEPTION_TABLET_COLUMN_CATEGORIES_ARE_MISSING_2C660532); + } + final List result = new ArrayList<>(columnCount); + for (int i = 0; i < columnCount; i++) { + if (Objects.isNull(categories.get(i))) { + throw TagFilterEvaluationException.schemaBinding( + DataNodeMiscMessages.EXCEPTION_TABLET_COLUMN_CATEGORY_IS_MISSING_A812B500); + } + result.add(categories.get(i)); + } + return result; + } + + private static void validateTabletStructure( + final Tablet tablet, + final List schemas, + final Object[] values, + final List categories) { + if (tablet.getTimestamps() == null || tablet.getTimestamps().length < tablet.getRowSize()) { + throw new TagFilterEvaluationException( + DataNodeMiscMessages.EXCEPTION_TABLET_TIMESTAMPS_ARE_INCOMPLETE_24F8CE6F); + } + final org.apache.tsfile.utils.BitMap[] bitMaps = tablet.getBitMaps(); + if (bitMaps != null && bitMaps.length < schemas.size()) { + throw new TagFilterEvaluationException( + DataNodeMiscMessages.EXCEPTION_TABLET_BITMAPS_ARE_INCOMPLETE_7BBC8035); + } + for (int i = 0; i < schemas.size(); i++) { + final IMeasurementSchema schema = schemas.get(i); + if (schema == null || schema.getMeasurementName() == null || schema.getType() == null) { + throw TagFilterEvaluationException.schemaBinding( + DataNodeMiscMessages.EXCEPTION_TABLET_MEASUREMENT_SCHEMA_IS_INCOMPLETE_6A813472); + } + if (values[i] == null) { + // A column whose every row is NULL may legitimately omit its value array and represent + // the values solely through a bitmap. It still has a complete schema and must remain + // distinguishable from a missing or malformed column. + if (!isEntirelyNull(bitMaps, i, tablet.getRowSize())) { + throw new TagFilterEvaluationException( + DataNodeMiscMessages.EXCEPTION_TABLET_VALUE_COLUMN_IS_INCOMPLETE_845721FE); + } + continue; + } + if (!hasArrayLength(values[i], tablet.getRowSize())) { + throw new TagFilterEvaluationException( + DataNodeMiscMessages.EXCEPTION_TABLET_VALUE_COLUMN_IS_INCOMPLETE_845721FE); + } + if (categories.get(i) == null) { + throw TagFilterEvaluationException.schemaBinding( + DataNodeMiscMessages.EXCEPTION_TABLET_COLUMN_CATEGORY_IS_MISSING_A812B500); + } + } + } + + private static boolean hasArrayLength(final Object value, final int rowSize) { + if (value instanceof boolean[]) { + return ((boolean[]) value).length >= rowSize; + } + if (value instanceof int[]) { + return ((int[]) value).length >= rowSize; + } + if (value instanceof long[]) { + return ((long[]) value).length >= rowSize; + } + if (value instanceof float[]) { + return ((float[]) value).length >= rowSize; + } + if (value instanceof double[]) { + return ((double[]) value).length >= rowSize; + } + if (value instanceof Binary[]) { + return ((Binary[]) value).length >= rowSize; + } + if (value instanceof java.time.LocalDate[]) { + return ((java.time.LocalDate[]) value).length >= rowSize; + } + return false; + } + + private static boolean isEntirelyNull( + final org.apache.tsfile.utils.BitMap[] bitMaps, final int columnIndex, final int rowSize) { + if (bitMaps == null || columnIndex >= bitMaps.length || bitMaps[columnIndex] == null) { + return false; + } + for (int rowIndex = 0; rowIndex < rowSize; rowIndex++) { + if (!bitMaps[columnIndex].isMarked(rowIndex)) { + return false; + } + } + return true; + } + + private static String toFieldKey(final Identifier identifier) { + return (identifier.isDelimited() ? DELIMITED_FIELD_PREFIX : UNDELIMITED_FIELD_PREFIX) + + (identifier.isDelimited() + ? identifier.getValue() + : identifier.getValue().toLowerCase(Locale.ROOT)); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/tagfilter/TagFilterEvaluationException.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/tagfilter/TagFilterEvaluationException.java new file mode 100644 index 0000000000000..420427303b85d --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/tagfilter/TagFilterEvaluationException.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.subscription.tagfilter; + +import org.apache.iotdb.db.i18n.DataNodeMiscMessages; + +/** Raised when a TAG filter cannot be evaluated safely for an input event. */ +public class TagFilterEvaluationException extends RuntimeException { + + public TagFilterEvaluationException(final String reason) { + super( + String.format( + DataNodeMiscMessages.EXCEPTION_TAG_FILTER_EVALUATION_FAILED_ARG_1B239B2F, reason)); + } + + public TagFilterEvaluationException(final String reason, final Throwable cause) { + super( + String.format( + DataNodeMiscMessages.EXCEPTION_TAG_FILTER_EVALUATION_FAILED_ARG_1B239B2F, reason), + cause); + } + + public static TagFilterEvaluationException schemaBinding(final String reason) { + return new TagFilterEvaluationException( + String.format( + DataNodeMiscMessages.EXCEPTION_TAG_FILTER_SCHEMA_BINDING_FAILED_ARG_7A5D2D47, reason)); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/tagfilter/TagFilterEvaluator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/tagfilter/TagFilterEvaluator.java new file mode 100644 index 0000000000000..e112f16c8e905 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/tagfilter/TagFilterEvaluator.java @@ -0,0 +1,226 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.subscription.tagfilter; + +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.BooleanLiteral; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.CommonQueryAstVisitor; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.ComparisonExpression; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Expression; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.FunctionCall; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Identifier; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.InListExpression; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.InPredicate; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.IsNullPredicate; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.LikePredicate; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.LogicalExpression; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Node; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.NotExpression; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.StringLiteral; +import org.apache.iotdb.db.i18n.DataNodeMiscMessages; + +import java.util.Collections; +import java.util.Map; +import java.util.Objects; +import java.util.regex.Pattern; + +/** Evaluates a validated tag-filter with SQL three-valued NULL semantics. */ +public class TagFilterEvaluator + implements CommonQueryAstVisitor< + TagFilterEvaluator.TruthValue, TagFilterEvaluator.ValueProvider> { + + enum TruthValue { + TRUE, + FALSE, + UNKNOWN + } + + private final Map compiledPatterns; + + private TagFilterEvaluator() { + this(Collections.emptyMap()); + } + + private TagFilterEvaluator(final Map compiledPatterns) { + this.compiledPatterns = compiledPatterns; + } + + @FunctionalInterface + public interface ValueProvider { + + String getValue(Identifier identifier); + } + + public static boolean evaluate(final Expression expression, final ValueProvider valueProvider) { + final TagFilterValidator.ValidationResult validation = + TagFilterValidator.validateAndCompile(expression); + return evaluate(expression, valueProvider, validation.getCompiledPatterns()); + } + + public static boolean evaluate( + final Expression expression, + final ValueProvider valueProvider, + final Map compiledPatterns) { + return new TagFilterEvaluator(compiledPatterns).process(expression, valueProvider) + == TruthValue.TRUE; + } + + @Override + public TruthValue visitNode(final Node node, final ValueProvider context) { + throw new IllegalArgumentException( + String.format( + DataNodeMiscMessages.UNSUPPORTED_EXPRESSION_FMT, node.getClass().getSimpleName())); + } + + @Override + public TruthValue visitBooleanLiteral(final BooleanLiteral node, final ValueProvider context) { + return node.getValue() ? TruthValue.TRUE : TruthValue.FALSE; + } + + @Override + public TruthValue visitLogicalExpression( + final LogicalExpression node, final ValueProvider context) { + boolean hasUnknown = false; + if (node.getOperator() == LogicalExpression.Operator.AND) { + for (final Expression term : node.getTerms()) { + final TruthValue result = process(term, context); + if (result == TruthValue.FALSE) { + return TruthValue.FALSE; + } + hasUnknown |= result == TruthValue.UNKNOWN; + } + return hasUnknown ? TruthValue.UNKNOWN : TruthValue.TRUE; + } + + for (final Expression term : node.getTerms()) { + final TruthValue result = process(term, context); + if (result == TruthValue.TRUE) { + return TruthValue.TRUE; + } + hasUnknown |= result == TruthValue.UNKNOWN; + } + return hasUnknown ? TruthValue.UNKNOWN : TruthValue.FALSE; + } + + @Override + public TruthValue visitNotExpression(final NotExpression node, final ValueProvider context) { + final TruthValue result = process(node.getValue(), context); + if (result == TruthValue.UNKNOWN) { + return TruthValue.UNKNOWN; + } + return result == TruthValue.TRUE ? TruthValue.FALSE : TruthValue.TRUE; + } + + @Override + public TruthValue visitComparisonExpression( + final ComparisonExpression node, final ValueProvider context) { + final String left = context.getValue((Identifier) node.getLeft()); + if (Objects.isNull(left)) { + return TruthValue.UNKNOWN; + } + final boolean equals = left.equals(((StringLiteral) node.getRight()).getValue()); + return booleanValue( + node.getOperator() == ComparisonExpression.Operator.EQUAL ? equals : !equals); + } + + @Override + public TruthValue visitInPredicate(final InPredicate node, final ValueProvider context) { + final String left = context.getValue((Identifier) node.getValue()); + if (Objects.isNull(left)) { + return TruthValue.UNKNOWN; + } + for (final Expression expression : ((InListExpression) node.getValueList()).getValues()) { + if (left.equals(((StringLiteral) expression).getValue())) { + return TruthValue.TRUE; + } + } + return TruthValue.FALSE; + } + + @Override + public TruthValue visitLikePredicate(final LikePredicate node, final ValueProvider context) { + final String left = context.getValue((Identifier) node.getValue()); + if (Objects.isNull(left)) { + return TruthValue.UNKNOWN; + } + final Pattern pattern = compiledPatterns.get(node); + if (pattern == null) { + throw new IllegalArgumentException( + DataNodeMiscMessages.EXCEPTION_UNCOMPILED_LIKE_PREDICATE_01AEE439); + } + return booleanValue(pattern.matcher(left).matches()); + } + + @Override + public TruthValue visitFunctionCall(final FunctionCall node, final ValueProvider context) { + final String left = context.getValue((Identifier) node.getArguments().get(0)); + if (Objects.isNull(left)) { + return TruthValue.UNKNOWN; + } + final Pattern pattern = compiledPatterns.get(node); + if (pattern == null) { + throw new IllegalArgumentException( + DataNodeMiscMessages.EXCEPTION_UNCOMPILED_REGEXP_PREDICATE_2B0DD646); + } + return booleanValue(pattern.matcher(left).matches()); + } + + @Override + public TruthValue visitIsNullPredicate(final IsNullPredicate node, final ValueProvider context) { + return booleanValue(Objects.isNull(context.getValue((Identifier) node.getValue()))); + } + + static Pattern compileLikePattern(final String pattern, final String escape) { + final Character escapeChar; + if (Objects.isNull(escape)) { + escapeChar = null; + } else if (escape.length() == 1) { + escapeChar = escape.charAt(0); + } else { + throw new IllegalArgumentException(DataNodeMiscMessages.LIKE_ESCAPE_MUST_BE_SINGLE_CHARACTER); + } + + final StringBuilder regex = new StringBuilder(); + boolean escaping = false; + for (int i = 0; i < pattern.length(); i++) { + final char ch = pattern.charAt(i); + if (Objects.nonNull(escapeChar) && ch == escapeChar && !escaping) { + escaping = true; + continue; + } + if (!escaping && ch == '%') { + regex.append(".*"); + } else if (!escaping && ch == '_') { + regex.append('.'); + } else { + regex.append(Pattern.quote(String.valueOf(ch))); + } + escaping = false; + } + if (escaping) { + throw new IllegalArgumentException( + DataNodeMiscMessages.LIKE_PATTERN_ENDS_WITH_ESCAPE_CHARACTER); + } + return Pattern.compile(regex.toString(), Pattern.DOTALL); + } + + private static TruthValue booleanValue(final boolean value) { + return value ? TruthValue.TRUE : TruthValue.FALSE; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/tagfilter/TagFilterMatcher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/tagfilter/TagFilterMatcher.java new file mode 100644 index 0000000000000..ffff51d7a7b8a --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/tagfilter/TagFilterMatcher.java @@ -0,0 +1,421 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.subscription.tagfilter; + +import org.apache.iotdb.commons.pipe.datastructure.pattern.TablePattern; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Expression; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Identifier; +import org.apache.iotdb.commons.schema.table.TsTable; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; +import org.apache.iotdb.db.i18n.DataNodeMiscMessages; +import org.apache.iotdb.rpc.subscription.config.TopicConfig; +import org.apache.iotdb.rpc.subscription.config.TopicConstant; +import org.apache.iotdb.rpc.subscription.exception.SubscriptionException; + +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.regex.Pattern; + +/** Immutable parsed tag-filter cached per topic. */ +public class TagFilterMatcher { + + public enum RuntimeStatus { + MATCH_ALL, + MATCH_NONE, + ACTIVE, + ERROR + } + + private static final TagFilterMatcher MATCH_NONE = + new TagFilterMatcher( + null, Collections.emptyList(), Collections.emptyMap(), State.MATCH_NONE, null, false); + + private final Expression expression; + private final List referencedFields; + private final Map compiledPatterns; + private final State state; + private final Throwable failure; + private final Map tableBindings; + private final boolean bindingEnforced; + + private enum State { + MATCH_ALL, + MATCH_NONE, + ACTIVE, + FAILURE + } + + private TagFilterMatcher( + final Expression expression, + final List referencedFields, + final Map compiledPatterns, + final State state, + final Throwable failure, + final boolean bindingEnforced) { + this( + expression, + referencedFields, + compiledPatterns, + state, + failure, + Collections.emptyMap(), + bindingEnforced); + } + + private TagFilterMatcher( + final Expression expression, + final List referencedFields, + final Map compiledPatterns, + final State state, + final Throwable failure, + final Map tableBindings, + final boolean bindingEnforced) { + this.expression = expression; + this.referencedFields = referencedFields; + this.compiledPatterns = compiledPatterns; + this.state = state; + this.failure = failure; + this.tableBindings = tableBindings; + this.bindingEnforced = bindingEnforced; + } + + private static final TagFilterMatcher MATCH_ALL = + new TagFilterMatcher( + null, Collections.emptyList(), Collections.emptyMap(), State.MATCH_ALL, null, false); + + public static TagFilterMatcher matchAll() { + return MATCH_ALL; + } + + public static TagFilterMatcher matchNone() { + return MATCH_NONE; + } + + public static TagFilterMatcher failure(final Throwable failure) { + return new TagFilterMatcher( + null, + Collections.emptyList(), + Collections.emptyMap(), + State.FAILURE, + Objects.nonNull(failure) ? failure : new IllegalStateException(), + false); + } + + public static TagFilterMatcher fromTopicConfig(final TopicConfig topicConfig) + throws SubscriptionException { + if (Objects.isNull(topicConfig) + || !topicConfig.isTableTopic() + || topicConfig.isTagFilterTrivial()) { + return matchAll(); + } + + final Expression expression; + final TagFilterValidator.ValidationResult validation; + try { + expression = new TagFilterParser().parse(topicConfig.getTagFilter()); + validation = TagFilterValidator.validateAndCompile(expression); + } catch (final IllegalArgumentException e) { + throw new SubscriptionException( + String.format( + DataNodeMiscMessages.EXCEPTION_INVALID_TAG_FILTER_ARG_E4B1C1C6, e.getMessage()), + e); + } + return new TagFilterMatcher( + expression, + validation.getReferencedFields(), + validation.getCompiledPatterns(), + State.ACTIVE, + null, + false); + } + + /** + * Parses a filter and validates every currently visible table selected by the topic pattern. An + * empty table map is deliberately allowed for a wildcard topic because the table may be created + * after the topic; a null map means that a required concrete schema is unavailable. + */ + public static TagFilterMatcher fromTopicConfig( + final TopicConfig topicConfig, final Map> tables) + throws SubscriptionException { + final TagFilterMatcher matcher = fromTopicConfig(topicConfig); + if (matcher.isMatchAll()) { + return matcher; + } + if (tables == null) { + throw new SubscriptionException( + DataNodeMiscMessages.EXCEPTION_TABLE_SCHEMA_IS_NOT_AVAILABLE_FOR_TAG_FILTER_993AB728); + } + + final TablePattern tablePattern = + new TablePattern( + true, + topicConfig.getStringOrDefault( + TopicConstant.DATABASE_KEY, TopicConstant.DATABASE_DEFAULT_VALUE), + topicConfig.getStringOrDefault( + TopicConstant.TABLE_KEY, TopicConstant.TABLE_DEFAULT_VALUE)); + final Map bindings = new HashMap<>(); + for (final Map.Entry> databaseEntry : tables.entrySet()) { + if (!tablePattern.matchesDatabase(databaseEntry.getKey())) { + continue; + } + for (final TsTable table : databaseEntry.getValue().values()) { + if (table == null || !tablePattern.matchesTable(table.getTableName())) { + continue; + } + final TableBinding binding = bindTable(matcher.referencedFields, table); + if (binding.isFailed()) { + throw new SubscriptionException(binding.getFailureReason()); + } + bindings.put(TableKey.of(databaseEntry.getKey(), table.getTableName()), binding); + } + } + final boolean bindingEnforced = + isLiteralTopicPattern( + topicConfig.getStringOrDefault( + TopicConstant.DATABASE_KEY, TopicConstant.DATABASE_DEFAULT_VALUE)) + && isLiteralTopicPattern( + topicConfig.getStringOrDefault( + TopicConstant.TABLE_KEY, TopicConstant.TABLE_DEFAULT_VALUE)); + return matcher.withBindings(bindings, bindingEnforced); + } + + private static boolean isLiteralTopicPattern(final String pattern) { + final String regexMetaCharacters = ".*+?[](){}\\|^$"; + return Objects.nonNull(pattern) + && pattern.chars().noneMatch(c -> regexMetaCharacters.indexOf((char) c) >= 0); + } + + private static TableBinding bindTable( + final List referencedFields, final TsTable table) { + final java.util.Set tagNames = new java.util.HashSet<>(); + final java.util.Set allNames = new java.util.HashSet<>(); + for (final TsTableColumnSchema schema : table.getColumnList()) { + if (schema == null || schema.getColumnName() == null) { + continue; + } + allNames.add(schema.getColumnName().toLowerCase(java.util.Locale.ROOT)); + if (schema.getColumnCategory() == TsTableColumnCategory.TAG) { + tagNames.add(schema.getColumnName().toLowerCase(java.util.Locale.ROOT)); + } + } + for (final Identifier identifier : referencedFields) { + final String identifierName = identifier.getValue(); + TsTableColumnSchema matched = null; + for (final TsTableColumnSchema schema : table.getColumnList()) { + if (schema == null || schema.getColumnName() == null) { + continue; + } + if (identifier.isDelimited() + ? schema.getColumnName().equals(identifierName) + : schema.getColumnName().equalsIgnoreCase(identifierName)) { + matched = schema; + break; + } + } + if (matched == null) { + return new TableBinding( + true, + String.format( + DataNodeMiscMessages.EXCEPTION_REFERENCED_TAG_COLUMN_IS_MISSING_ARG_F5300BEA, + identifierName), + tagNames, + allNames); + } + if (matched.getColumnCategory() != TsTableColumnCategory.TAG) { + return new TableBinding( + true, + String.format( + DataNodeMiscMessages.EXCEPTION_REFERENCED_COLUMN_IS_NOT_A_TAG_COLUMN_ARG_34D60881, + identifierName), + tagNames, + allNames); + } + } + return new TableBinding(false, null, tagNames, allNames); + } + + TagFilterMatcher withBindings(final Map bindings) { + return withBindings(bindings, true); + } + + TagFilterMatcher withBindings( + final Map bindings, final boolean bindingEnforced) { + return new TagFilterMatcher( + expression, + referencedFields, + compiledPatterns, + state, + failure, + Objects.nonNull(bindings) + ? Collections.unmodifiableMap(new HashMap<>(bindings)) + : Collections.emptyMap(), + bindingEnforced); + } + + public boolean isMatchAll() { + return state == State.MATCH_ALL; + } + + public boolean isMatchNone() { + return state == State.MATCH_NONE; + } + + public boolean isFailure() { + return state == State.FAILURE; + } + + public RuntimeStatus getRuntimeStatus() { + switch (state) { + case MATCH_ALL: + return RuntimeStatus.MATCH_ALL; + case MATCH_NONE: + return RuntimeStatus.MATCH_NONE; + case ACTIVE: + return RuntimeStatus.ACTIVE; + case FAILURE: + default: + return RuntimeStatus.ERROR; + } + } + + public String getFailureMessage() { + if (!isFailure() || Objects.isNull(failure)) { + return null; + } + Throwable current = failure; + while (Objects.isNull(current.getMessage()) + && Objects.nonNull(current.getCause()) + && current != current.getCause()) { + current = current.getCause(); + } + return Objects.nonNull(current.getMessage()) + ? current.getMessage() + : current.getClass().getSimpleName(); + } + + public void throwIfFailure() { + if (isFailure()) { + throw new TagFilterEvaluationException( + DataNodeMiscMessages.EXCEPTION_MATCHER_IS_UNAVAILABLE_1A659D47, failure); + } + } + + List getReferencedFields() { + return referencedFields; + } + + Map getTableBindings() { + return tableBindings; + } + + boolean isBindingEnforced() { + return bindingEnforced; + } + + boolean matches(final TagFilterEvaluator.ValueProvider valueProvider) { + throwIfFailure(); + return !isMatchNone() + && (Objects.isNull(expression) + || TagFilterEvaluator.evaluate(expression, valueProvider, compiledPatterns)); + } + + static final class TableKey { + + private final String database; + private final String table; + + private TableKey(final String database, final String table) { + this.database = normalize(database); + this.table = normalize(table); + } + + static TableKey of(final String database, final String table) { + return new TableKey(database, table); + } + + String getDatabase() { + return database; + } + + String getTable() { + return table; + } + + @Override + public boolean equals(final Object object) { + if (this == object) { + return true; + } + if (!(object instanceof TableKey)) { + return false; + } + final TableKey that = (TableKey) object; + return Objects.equals(database, that.database) && Objects.equals(table, that.table); + } + + @Override + public int hashCode() { + return Objects.hash(database, table); + } + + private static String normalize(final String value) { + return Objects.nonNull(value) ? value.trim().toLowerCase(java.util.Locale.ROOT) : ""; + } + } + + static final class TableBinding { + + private final boolean failed; + private final String failureReason; + private final java.util.Set tagNames; + private final java.util.Set allNames; + + TableBinding( + final boolean failed, + final String failureReason, + final java.util.Set tagNames, + final java.util.Set allNames) { + this.failed = failed; + this.failureReason = failureReason; + this.tagNames = tagNames; + this.allNames = allNames; + } + + boolean isFailed() { + return failed; + } + + String getFailureReason() { + return failureReason; + } + + java.util.Set getTagNames() { + return tagNames; + } + + java.util.Set getAllNames() { + return allNames; + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/tagfilter/TagFilterParser.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/tagfilter/TagFilterParser.java new file mode 100644 index 0000000000000..1a08394b332eb --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/tagfilter/TagFilterParser.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.subscription.tagfilter; + +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Expression; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.parser.ParsingException; +import org.apache.iotdb.db.i18n.DataNodeMiscMessages; +import org.apache.iotdb.db.subscription.columnfilter.ColumnFilterParser; +import org.apache.iotdb.rpc.subscription.exception.SubscriptionException; + +import java.nio.charset.StandardCharsets; + +/** Parses the restricted subscription filter expression syntax for table-model TAG values. */ +public class TagFilterParser { + + private static final int MAX_UTF8_BYTES = 4096; + + private final ColumnFilterParser expressionParser = new ColumnFilterParser(); + + public Expression parse(final String rawTagFilter) throws SubscriptionException { + try { + if (rawTagFilter == null || rawTagFilter.trim().isEmpty()) { + throw new IllegalArgumentException( + DataNodeMiscMessages.EXCEPTION_TAG_FILTER_SHOULD_NOT_BE_EMPTY_507CA5B0); + } + validateText(rawTagFilter); + return expressionParser.parse(rawTagFilter); + } catch (final ParsingException | IllegalArgumentException e) { + throw new SubscriptionException( + String.format( + DataNodeMiscMessages.EXCEPTION_INVALID_TAG_FILTER_ARG_E4B1C1C6, e.getMessage()), + e); + } + } + + public Expression parseAndValidate(final String rawTagFilter) throws SubscriptionException { + final Expression expression = parse(rawTagFilter); + try { + TagFilterValidator.validate(expression); + } catch (final IllegalArgumentException e) { + throw new SubscriptionException( + String.format( + DataNodeMiscMessages.EXCEPTION_INVALID_TAG_FILTER_ARG_E4B1C1C6, e.getMessage()), + e); + } + return expression; + } + + private static void validateText(final String text) { + if (text.getBytes(StandardCharsets.UTF_8).length > MAX_UTF8_BYTES) { + throw new IllegalArgumentException( + String.format( + DataNodeMiscMessages + .EXCEPTION_TAG_FILTER_EXCEEDS_MAXIMUM_UTF_8_LENGTH_OF_ARG_BYTES_9C8400F2, + MAX_UTF8_BYTES)); + } + for (int i = 0; i < text.length(); i++) { + final char current = text.charAt(i); + if (Character.isISOControl(current) && !Character.isWhitespace(current)) { + throw new IllegalArgumentException( + String.format( + DataNodeMiscMessages.UNEXPECTED_CHARACTER_FMT, Integer.toHexString(current))); + } + if (Character.isHighSurrogate(current)) { + if (i + 1 >= text.length() || !Character.isLowSurrogate(text.charAt(++i))) { + throw new IllegalArgumentException( + String.format( + DataNodeMiscMessages.UNEXPECTED_CHARACTER_FMT, Integer.toHexString(current))); + } + } else if (Character.isLowSurrogate(current)) { + throw new IllegalArgumentException( + String.format( + DataNodeMiscMessages.UNEXPECTED_CHARACTER_FMT, Integer.toHexString(current))); + } + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/tagfilter/TagFilterValidator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/tagfilter/TagFilterValidator.java new file mode 100644 index 0000000000000..2b532056a6054 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/tagfilter/TagFilterValidator.java @@ -0,0 +1,252 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.subscription.tagfilter; + +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.BooleanLiteral; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.CommonQueryAstVisitor; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.ComparisonExpression; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Expression; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.FunctionCall; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Identifier; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.InListExpression; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.InPredicate; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.IsNullPredicate; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.LikePredicate; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.LogicalExpression; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Node; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.NotExpression; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.StringLiteral; +import org.apache.iotdb.db.i18n.DataNodeMiscMessages; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.IdentityHashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; + +/** Validates the expression shape and collects the TAG columns referenced by a tag-filter. */ +public class TagFilterValidator implements CommonQueryAstVisitor { + + static final int MAX_AST_DEPTH = 32; + static final int MAX_AST_NODES = 256; + static final int MAX_IN_VALUES = 256; + static final int MAX_REGEXP_LENGTH = 1024; + + private static final String REGEXP_LIKE = "regexp_like"; + + private final List referencedFields = new ArrayList<>(); + private final Map compiledPatterns = new IdentityHashMap<>(); + private int depth; + private int nodeCount; + + public static List validate(final Expression expression) { + return validateAndCompile(expression).getReferencedFields(); + } + + public static ValidationResult validateAndCompile(final Expression expression) { + final TagFilterValidator validator = new TagFilterValidator(); + validator.process(expression); + return new ValidationResult( + Collections.unmodifiableList(new ArrayList<>(validator.referencedFields)), + Collections.unmodifiableMap(new IdentityHashMap<>(validator.compiledPatterns))); + } + + @Override + public Void process(final Node node, final Void context) { + if (node == null) { + throw invalid(DataNodeMiscMessages.UNSUPPORTED_EXPRESSION_FMT.replace("%s", "null")); + } + accountNode(node); + if (++depth > MAX_AST_DEPTH) { + throw invalid( + String.format( + DataNodeMiscMessages.EXCEPTION_TAG_FILTER_AST_DEPTH_EXCEEDS_MAXIMUM_OF_ARG_64C92731, + MAX_AST_DEPTH)); + } + try { + return CommonQueryAstVisitor.super.process(node, context); + } finally { + depth--; + } + } + + @Override + public Void visitNode(final Node node, final Void context) { + throw invalid( + String.format( + DataNodeMiscMessages.UNSUPPORTED_EXPRESSION_FMT, node.getClass().getSimpleName())); + } + + @Override + public Void visitBooleanLiteral(final BooleanLiteral node, final Void context) { + return null; + } + + @Override + public Void visitLogicalExpression(final LogicalExpression node, final Void context) { + node.getTerms().forEach(this::process); + return null; + } + + @Override + public Void visitNotExpression(final NotExpression node, final Void context) { + process(node.getValue()); + return null; + } + + @Override + public Void visitComparisonExpression(final ComparisonExpression node, final Void context) { + if (node.getOperator() != ComparisonExpression.Operator.EQUAL + && node.getOperator() != ComparisonExpression.Operator.NOT_EQUAL) { + throw invalid( + DataNodeMiscMessages.EXCEPTION_ONLY_AND_COMPARISONS_ARE_SUPPORTED_IN_TAG_FILTER_19946958); + } + requireTagField(node.getLeft()); + requireStringLiteral( + node.getRight(), DataNodeMiscMessages.COLUMN_FILTER_COMPARISON_RIGHT_OPERAND); + return null; + } + + @Override + public Void visitInPredicate(final InPredicate node, final Void context) { + requireTagField(node.getValue()); + if (!(node.getValueList() instanceof InListExpression)) { + throw invalid(DataNodeMiscMessages.IN_PREDICATE_MUST_USE_STRING_LITERAL_LIST); + } + final List values = ((InListExpression) node.getValueList()).getValues(); + if (values.isEmpty() || values.size() > MAX_IN_VALUES) { + throw invalid( + String.format( + DataNodeMiscMessages + .EXCEPTION_TAG_FILTER_IN_LIST_EXCEEDS_MAXIMUM_OF_ARG_VALUES_07E7FC6C, + MAX_IN_VALUES)); + } + accountNode(node.getValueList()); + for (final Expression expression : values) { + accountNode(expression); + requireStringLiteral(expression, DataNodeMiscMessages.COLUMN_FILTER_IN_ELEMENT); + } + return null; + } + + @Override + public Void visitLikePredicate(final LikePredicate node, final Void context) { + requireTagField(node.getValue()); + final StringLiteral pattern = + requireStringLiteral(node.getPattern(), DataNodeMiscMessages.COLUMN_FILTER_LIKE_PATTERN); + final String escape = + node.getEscape() + .map( + expression -> + requireStringLiteral(expression, DataNodeMiscMessages.COLUMN_FILTER_LIKE_ESCAPE) + .getValue()) + .orElse(null); + compiledPatterns.put(node, TagFilterEvaluator.compileLikePattern(pattern.getValue(), escape)); + return null; + } + + @Override + public Void visitFunctionCall(final FunctionCall node, final Void context) { + if (!REGEXP_LIKE.equalsIgnoreCase(node.getName().toString()) + || node.isDistinct() + || node.getProcessingMode().isPresent() + || node.getArguments().size() != 2) { + throw invalid(DataNodeMiscMessages.ONLY_REGEXP_SUPPORTED_AS_REGEXP_LIKE); + } + + requireTagField(node.getArguments().get(0)); + final String pattern = + requireStringLiteral( + node.getArguments().get(1), DataNodeMiscMessages.COLUMN_FILTER_REGEXP_PATTERN) + .getValue(); + if (pattern.length() > MAX_REGEXP_LENGTH) { + throw invalid( + String.format( + DataNodeMiscMessages + .EXCEPTION_TAG_FILTER_REGEXP_PATTERN_EXCEEDS_MAXIMUM_LENGTH_OF_ARG_CHARACTERS_67590971, + MAX_REGEXP_LENGTH)); + } + try { + compiledPatterns.put(node, Pattern.compile(pattern)); + } catch (final PatternSyntaxException e) { + throw invalid(String.format(DataNodeMiscMessages.ILLEGAL_REGEXP_PATTERN_FMT, e.getMessage())); + } + return null; + } + + @Override + public Void visitIsNullPredicate(final IsNullPredicate node, final Void context) { + requireTagField(node.getValue()); + return null; + } + + private Identifier requireTagField(final Expression expression) { + if (!(expression instanceof Identifier)) { + throw invalid(DataNodeMiscMessages.EXCEPTION_LEFT_OPERAND_MUST_BE_A_TAG_COLUMN_F9D4548B); + } + final Identifier identifier = (Identifier) expression; + referencedFields.add(identifier); + return identifier; + } + + private static StringLiteral requireStringLiteral( + final Expression expression, final String description) { + if (!(expression instanceof StringLiteral)) { + throw invalid(String.format(DataNodeMiscMessages.MUST_BE_STRING_LITERAL_FMT, description)); + } + return (StringLiteral) expression; + } + + private static IllegalArgumentException invalid(final String message) { + return new IllegalArgumentException(message); + } + + private void accountNode(final Node node) { + if (++nodeCount > MAX_AST_NODES) { + throw invalid( + String.format( + DataNodeMiscMessages + .EXCEPTION_TAG_FILTER_AST_NODE_COUNT_EXCEEDS_MAXIMUM_OF_ARG_40CCE694, + MAX_AST_NODES)); + } + } + + public static final class ValidationResult { + + private final List referencedFields; + private final Map compiledPatterns; + + private ValidationResult( + final List referencedFields, final Map compiledPatterns) { + this.referencedFields = referencedFields; + this.compiledPatterns = compiledPatterns; + } + + public List getReferencedFields() { + return referencedFields; + } + + public Map getCompiledPatterns() { + return compiledPatterns; + } + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/subscription/ShowTopicsTaskTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/subscription/ShowTopicsTaskTest.java new file mode 100644 index 0000000000000..0d7ff2e7ca353 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/subscription/ShowTopicsTaskTest.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.execution.config.sys.subscription; + +import org.apache.iotdb.commons.pipe.config.constant.SystemConstant; +import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant; +import org.apache.iotdb.confignode.rpc.thrift.TShowTopicInfo; +import org.apache.iotdb.db.queryengine.plan.execution.config.ConfigTaskResult; +import org.apache.iotdb.db.subscription.tagfilter.TagFilterMatcher; +import org.apache.iotdb.rpc.TSStatusCode; +import org.apache.iotdb.rpc.subscription.config.TopicConfig; +import org.apache.iotdb.rpc.subscription.config.TopicConstant; + +import com.google.common.util.concurrent.SettableFuture; +import org.apache.tsfile.read.common.block.TsBlock; +import org.junit.Test; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class ShowTopicsTaskTest { + + @Test + public void testBuildTSBlockWritesTagFilterDiagnostics() throws Exception { + final Map matchers = new HashMap<>(); + matchers.put("all", TagFilterMatcher.matchAll()); + matchers.put("none", TagFilterMatcher.matchNone()); + final Map attributes = new HashMap<>(); + attributes.put(SystemConstant.SQL_DIALECT_KEY, SystemConstant.SQL_DIALECT_TABLE_VALUE); + attributes.put(TopicConstant.TAG_FILTER_KEY, "region = \"east\""); + matchers.put("active", TagFilterMatcher.fromTopicConfig(new TopicConfig(attributes))); + matchers.put("error", TagFilterMatcher.failure(new IllegalStateException("binding failed"))); + final SettableFuture future = SettableFuture.create(); + + ShowTopicsTask.buildTSBlock( + Arrays.asList(topic("all"), topic("none"), topic("active"), topic("error")), + true, + matchers::get, + future); + + final ConfigTaskResult result = future.get(); + final TsBlock resultSet = result.getResultSet(); + assertEquals(TSStatusCode.SUCCESS_STATUS, result.getStatusCode()); + assertEquals( + ColumnHeaderConstant.TAG_FILTER_STATUS, + result.getResultSetHeader().getRespColumns().get(2)); + assertEquals( + ColumnHeaderConstant.TAG_FILTER_MESSAGE, + result.getResultSetHeader().getRespColumns().get(3)); + assertEquals("MATCH_ALL", resultSet.getColumn(2).getBinary(0).toString()); + assertEquals("MATCH_NONE", resultSet.getColumn(2).getBinary(1).toString()); + assertEquals("ACTIVE", resultSet.getColumn(2).getBinary(2).toString()); + assertEquals("ERROR", resultSet.getColumn(2).getBinary(3).toString()); + assertTrue(resultSet.getColumn(3).isNull(0)); + assertTrue(resultSet.getColumn(3).isNull(1)); + assertTrue(resultSet.getColumn(3).isNull(2)); + assertEquals("binding failed", resultSet.getColumn(3).getBinary(3).toString()); + } + + @Test + public void testBuildTreeModelTSBlockLeavesTagFilterDiagnosticsNull() throws Exception { + final SettableFuture future = SettableFuture.create(); + + ShowTopicsTask.buildTSBlock( + Arrays.asList(topic("tree")), + false, + ignored -> TagFilterMatcher.failure(new IllegalStateException()), + future); + + final TsBlock resultSet = future.get().getResultSet(); + assertEquals(2, resultSet.getValueColumnCount()); + } + + private static TShowTopicInfo topic(final String name) { + return new TShowTopicInfo(name, 1L).setTopicAttributes("{}"); + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/agent/SubscriptionTopicAgentTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/agent/SubscriptionTopicAgentTest.java index c31535b5812fc..e59d2de752691 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/agent/SubscriptionTopicAgentTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/agent/SubscriptionTopicAgentTest.java @@ -66,11 +66,74 @@ public void testColumnFilterRefreshTriggeredForNewTableTopicOnly() { SubscriptionTopicAgent.shouldRefreshColumnFilter(null, createTreeTopicMeta())); } + @Test + public void testTagFilterRefreshSkippedForOwnerOnlyUpdate() { + final TopicMeta oldMeta = + createTableTopicMeta("column_name = \"temperature\"", "region = \"north\"", "db", "t1"); + final Map updatedAttributes = new HashMap<>(); + updatedAttributes.put(TopicConstant.OWNER_ID_KEY, "owner-1"); + updatedAttributes.put(TopicConstant.OWNER_EPOCH_KEY, "1"); + + Assert.assertFalse( + SubscriptionTopicAgent.shouldRefreshTagFilter( + oldMeta, oldMeta.deepCopyWithUpdatedAttributes(updatedAttributes))); + } + + @Test + public void testTagFilterRefreshTriggeredForBindingInputUpdate() { + final TopicMeta oldMeta = + createTableTopicMeta("column_name = \"temperature\"", "region = \"north\"", "db", "t1"); + + Assert.assertTrue( + SubscriptionTopicAgent.shouldRefreshTagFilter( + oldMeta, + createTableTopicMeta( + "column_name = \"temperature\"", "region = \"north\"", "db2", "t1"))); + Assert.assertTrue( + SubscriptionTopicAgent.shouldRefreshTagFilter( + oldMeta, + createTableTopicMeta( + "column_name = \"temperature\"", "region = \"north\"", "db", "t2"))); + } + + @Test + public void testTagFilterRefreshTriggeredForCaseSensitiveExpressionUpdate() { + final TopicMeta oldMeta = + createTableTopicMeta("column_name = \"temperature\"", "region = \"north\"", "db", "t1"); + + Assert.assertTrue( + SubscriptionTopicAgent.shouldRefreshTagFilter( + oldMeta, + createTableTopicMeta( + "column_name = \"temperature\"", "region = \"North\"", "db", "t1"))); + } + + @Test + public void testTagFilterRefreshTriggeredForNewTableTopicOnly() { + Assert.assertTrue( + SubscriptionTopicAgent.shouldRefreshTagFilter( + null, + createTableTopicMeta( + "column_name = \"temperature\"", "region = \"north\"", "db", "t1"))); + Assert.assertFalse(SubscriptionTopicAgent.shouldRefreshTagFilter(null, createTreeTopicMeta())); + } + private static TopicMeta createTableTopicMeta( final String columnFilter, final String database, final String table) { + return createTableTopicMeta(columnFilter, null, database, table); + } + + private static TopicMeta createTableTopicMeta( + final String columnFilter, + final String tagFilter, + final String database, + final String table) { final Map attributes = new HashMap<>(); attributes.put("__system.sql-dialect", "table"); attributes.put(TopicConstant.COLUMN_FILTER_KEY, columnFilter); + if (tagFilter != null) { + attributes.put(TopicConstant.TAG_FILTER_KEY, tagFilter); + } attributes.put(TopicConstant.DATABASE_KEY, database); attributes.put(TopicConstant.TABLE_KEY, table); return new TopicMeta("topic", 1L, attributes); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusLogToTabletConverterTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusLogToTabletConverterTest.java index 1b8ad96836532..c8ecb3650ed2d 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusLogToTabletConverterTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusLogToTabletConverterTest.java @@ -33,6 +33,9 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertTabletNode; import org.apache.iotdb.db.queryengine.plan.statement.StatementTestUtils; import org.apache.iotdb.db.subscription.columnfilter.ColumnFilterMatcher; +import org.apache.iotdb.db.subscription.tagfilter.TagFilterMatcher; +import org.apache.iotdb.rpc.subscription.config.TopicConfig; +import org.apache.iotdb.rpc.subscription.config.TopicConstant; import org.apache.tsfile.enums.ColumnCategory; import org.apache.tsfile.enums.TSDataType; @@ -50,6 +53,7 @@ import java.util.Arrays; import java.util.HashSet; import java.util.List; +import java.util.Map; public class ConsensusLogToTabletConverterTest { @@ -252,6 +256,30 @@ public void testConvertRelationalInsertNodeReturnsEmptyWhenNoColumnsMatch() { Assert.assertTrue(converter.convert(StatementTestUtils.genInsertTabletNode(2, 0)).isEmpty()); } + @Test + public void testConvertRelationalInsertTabletFiltersRowsBeforeColumns() throws Exception { + final ConsensusLogToTabletConverter converter = + createConverterWithTagFilter("id1 IN (\"id:10\", \"id:12\")", "m1"); + + final List tablets = converter.convert(StatementTestUtils.genInsertTabletNode(3, 10)); + + Assert.assertEquals(1, tablets.size()); + final Tablet tablet = tablets.get(0); + Assert.assertEquals(2, tablet.getRowSize()); + Assert.assertArrayEquals(new long[] {10L, 12L}, tablet.getTimestamps()); + Assert.assertEquals("id:10", toUtf8(((Binary[]) tablet.getValues()[0])[0])); + Assert.assertEquals("id:12", toUtf8(((Binary[]) tablet.getValues()[0])[1])); + Assert.assertArrayEquals(new double[] {10.0, 12.0}, (double[]) tablet.getValues()[1], 0.0); + } + + @Test + public void testConvertRelationalInsertRowReturnsEmptyWhenTagDoesNotMatch() throws Exception { + final ConsensusLogToTabletConverter converter = + createConverterWithTagFilter("id1 = \"missing\"", "m1"); + + Assert.assertTrue(converter.convert(StatementTestUtils.genInsertRowNode(7)).isEmpty()); + } + @Test public void testConvertInsertRowsOfOneDeviceNodeGroupsRowsWithSameSchema() throws IllegalPathException { @@ -337,6 +365,19 @@ private static ConsensusLogToTabletConverter createConverter(final String... sel DATABASE_NAME); } + private static ConsensusLogToTabletConverter createConverterWithTagFilter( + final String tagFilter, final String... selectedColumns) throws Exception { + final TopicConfig topicConfig = + new TopicConfig( + Map.of("__system.sql-dialect", "table", TopicConstant.TAG_FILTER_KEY, tagFilter)); + return new ConsensusLogToTabletConverter( + null, + new TablePattern(true, DATABASE_NAME, StatementTestUtils.tableName()), + ColumnFilterMatcher.ofSelectedColumnNames(new HashSet<>(Arrays.asList(selectedColumns))), + TagFilterMatcher.fromTopicConfig(topicConfig), + DATABASE_NAME); + } + private static ConsensusLogToTabletConverter createTreeConverter() { return new ConsensusLogToTabletConverter(new IoTDBTreePattern("root.sg.**"), null, null, null); } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/event/batch/SubscriptionPipeTsFileEventBatchCleanupTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/event/batch/SubscriptionPipeTsFileEventBatchCleanupTest.java index d4d3609bbdbbd..5723170862f3f 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/event/batch/SubscriptionPipeTsFileEventBatchCleanupTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/event/batch/SubscriptionPipeTsFileEventBatchCleanupTest.java @@ -33,10 +33,42 @@ import java.io.File; import java.nio.file.Files; import java.util.Arrays; +import java.util.Collections; import java.util.List; public class SubscriptionPipeTsFileEventBatchCleanupTest { + @Test + public void testClosesInnerBatchWhenFilteringProducesNoPayload() throws Exception { + final PipeTabletEventTsFileBatch innerBatch = Mockito.mock(PipeTabletEventTsFileBatch.class); + final SubscriptionPrefetchingTsFileQueue queue = + Mockito.mock(SubscriptionPrefetchingTsFileQueue.class); + Mockito.when(innerBatch.isEmpty()).thenReturn(true); + + final SubscriptionPipeTsFileEventBatch batch = + new SubscriptionPipeTsFileEventBatch(1, queue, 1, 1, innerBatch); + + Assert.assertTrue(batch.generateSubscriptionEvents().isEmpty()); + Mockito.verify(innerBatch).close(); + } + + @Test + public void testClosesInnerBatchWhenSealingProducesNoFile() throws Exception { + final PipeTabletEventTsFileBatch innerBatch = Mockito.mock(PipeTabletEventTsFileBatch.class); + final SubscriptionPrefetchingTsFileQueue queue = + Mockito.mock(SubscriptionPrefetchingTsFileQueue.class); + Mockito.when(innerBatch.isEmpty()).thenReturn(false); + Mockito.when(innerBatch.sealTsFiles()).thenReturn(Collections.emptyList()); + + final SubscriptionPipeTsFileEventBatch batch = + new SubscriptionPipeTsFileEventBatch(1, queue, 1, 1, innerBatch); + + Assert.assertTrue(batch.generateSubscriptionEvents().isEmpty()); + Mockito.verify(innerBatch).decreaseEventsReferenceCount(batch.getClass().getName(), true); + Mockito.verify(innerBatch).onSuccess(); + Mockito.verify(innerBatch).close(); + } + @Test public void testDeletesSealedFilesAfterAllSharedEventsAreCleaned() throws Exception { final File temporaryDirectory = diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/tagfilter/TabletTagFilterTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/tagfilter/TabletTagFilterTest.java new file mode 100644 index 0000000000000..83cb162808613 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/tagfilter/TabletTagFilterTest.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.subscription.tagfilter; + +import org.apache.iotdb.rpc.subscription.config.TopicConfig; +import org.apache.iotdb.rpc.subscription.config.TopicConstant; + +import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.write.record.Tablet; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class TabletTagFilterTest { + + @Test + public void testFilterRowsAndPreserveValuesAndNulls() throws Exception { + final Tablet filtered = + TabletTagFilter.filter(createTablet(), matcher("region = \"south\" OR region IS NULL")); + + Assert.assertNotNull(filtered); + Assert.assertEquals(2, filtered.getRowSize()); + Assert.assertEquals(2L, filtered.getTimestamp(0)); + Assert.assertEquals(3L, filtered.getTimestamp(1)); + Assert.assertEquals("south", filtered.getValue(0, 0).toString()); + Assert.assertTrue(filtered.isNull(1, 0)); + Assert.assertEquals("d2", filtered.getValue(0, 1).toString()); + Assert.assertEquals("d3", filtered.getValue(1, 1).toString()); + Assert.assertEquals(20.0, (Double) filtered.getValue(0, 2), 0.0); + Assert.assertEquals(30.0, (Double) filtered.getValue(1, 2), 0.0); + Assert.assertEquals(createTablet().getColumnTypes(), filtered.getColumnTypes()); + } + + @Test + public void testComparisonIsCaseSensitive() throws Exception { + final Tablet filtered = TabletTagFilter.filter(createTablet(), matcher("region = \"north\"")); + + Assert.assertNotNull(filtered); + Assert.assertEquals(1, filtered.getRowSize()); + Assert.assertEquals(1L, filtered.getTimestamp(0)); + } + + @Test + public void testUnknownOrNonTagColumnFailsClosed() throws Exception { + assertFailsClosed("missing = \"x\""); + assertFailsClosed("temperature = \"10\""); + } + + @Test + public void testNoMatchedRowReturnsNull() throws Exception { + Assert.assertNull(TabletTagFilter.filter(createTablet(), matcher("region = \"east\""))); + } + + @Test + public void testTrivialFilterReturnsOriginalTablet() throws Exception { + final Tablet tablet = createTablet(); + + Assert.assertSame(tablet, TabletTagFilter.filter(tablet, matcher(" TRUE "))); + Assert.assertNull(TabletTagFilter.filter(tablet, TagFilterMatcher.matchNone())); + } + + private static TagFilterMatcher matcher(final String filter) throws Exception { + final Map attributes = new HashMap<>(); + attributes.put("__system.sql-dialect", "table"); + attributes.put(TopicConstant.TAG_FILTER_KEY, filter); + return TagFilterMatcher.fromTopicConfig(new TopicConfig(attributes)); + } + + private static void assertFailsClosed(final String filter) throws Exception { + try { + TabletTagFilter.filter(createTablet(), matcher(filter)); + Assert.fail("Expected tag-filter schema binding failure: " + filter); + } catch (final TagFilterEvaluationException e) { + Assert.assertTrue(e.getMessage(), e.getMessage().contains("Tag-filter")); + } + } + + private static Tablet createTablet() { + final List columnNames = Arrays.asList("region", "device", "temperature"); + final List dataTypes = + Arrays.asList(TSDataType.STRING, TSDataType.STRING, TSDataType.DOUBLE); + final List categories = + Arrays.asList(ColumnCategory.TAG, ColumnCategory.TAG, ColumnCategory.FIELD); + final Tablet tablet = new Tablet("weather", columnNames, dataTypes, categories, 4); + + tablet.addTimestamp(0, 1L); + tablet.addValue(0, 0, "north"); + tablet.addValue(0, 1, "d1"); + tablet.addValue(0, 2, 10.0); + + tablet.addTimestamp(1, 2L); + tablet.addValue(1, 0, "south"); + tablet.addValue(1, 1, "d2"); + tablet.addValue(1, 2, 20.0); + + tablet.addTimestamp(2, 3L); + tablet.addValue(2, 1, "d3"); + tablet.addValue(2, 2, 30.0); + + tablet.addTimestamp(3, 4L); + tablet.addValue(3, 0, "North"); + tablet.addValue(3, 1, "d4"); + tablet.addValue(3, 2, 40.0); + tablet.setRowSize(4); + return tablet; + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/tagfilter/TagFilterParserTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/tagfilter/TagFilterParserTest.java new file mode 100644 index 0000000000000..1d8d82613a128 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/tagfilter/TagFilterParserTest.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.subscription.tagfilter; + +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Expression; +import org.apache.iotdb.rpc.subscription.exception.SubscriptionException; + +import org.junit.Assert; +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; + +public class TagFilterParserTest { + + private static final TagFilterParser PARSER = new TagFilterParser(); + + @Test + public void testComparisonIsCaseSensitive() throws Exception { + Assert.assertTrue(evaluate("region = \"north\"", Map.of("region", "north"))); + Assert.assertFalse(evaluate("region = \"north\"", Map.of("region", "North"))); + Assert.assertTrue(evaluate("region != \"north\"", Map.of("region", "North"))); + } + + @Test + public void testInLikeRegexpAndBooleanOperators() throws Exception { + Assert.assertTrue( + evaluate( + "region IN (\"north\", \"west\") AND device LIKE \"d_%\"", + Map.of("region", "north", "device", "d_1"))); + Assert.assertTrue( + evaluate( + "region NOT REGEXP \"south|east\" OR NOT device = \"d2\"", + Map.of("region", "north", "device", "d2"))); + Assert.assertTrue(evaluate("device LIKE \"d!_%\" ESCAPE \"!\"", Map.of("device", "d_sensor"))); + Assert.assertTrue(evaluate("device LIKE \"d_%\"", Map.of("device", "d\n1"))); + } + + @Test + public void testNullUsesThreeValuedLogic() throws Exception { + final Map values = new HashMap<>(); + values.put("region", null); + + Assert.assertTrue(evaluate("region IS NULL", values)); + Assert.assertFalse(evaluate("region IS NOT NULL", values)); + Assert.assertFalse(evaluate("region = \"north\"", values)); + Assert.assertFalse(evaluate("NOT region = \"north\"", values)); + Assert.assertTrue(evaluate("region = \"north\" OR true", values)); + Assert.assertFalse(evaluate("region = \"north\" AND true", values)); + } + + @Test + public void testRejectInvalidExpressions() { + assertRejected("", "tag-filter should not be empty"); + assertRejected("region > \"north\"", "unsupported comparison operator"); + assertRejected("lower(region) = \"north\"", "expected column predicate operator"); + assertRejected("region = other_tag", "expected string literal"); + assertRejected("region REGEXP \"[\"", "illegal REGEXP pattern"); + } + + private static boolean evaluate(final String filter, final Map values) + throws SubscriptionException { + final Expression expression = PARSER.parseAndValidate(filter); + return TagFilterEvaluator.evaluate(expression, identifier -> values.get(identifier.getValue())); + } + + private static void assertRejected(final String filter, final String expectedMessagePart) { + try { + PARSER.parseAndValidate(filter); + Assert.fail("Expected tag-filter to be rejected: " + filter); + } catch (final SubscriptionException e) { + Assert.assertTrue(e.getMessage(), e.getMessage().contains(expectedMessagePart)); + } + } +} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/column/ColumnHeaderConstant.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/column/ColumnHeaderConstant.java index e4c365818e842..f999ab16e2e54 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/column/ColumnHeaderConstant.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/column/ColumnHeaderConstant.java @@ -133,6 +133,8 @@ private ColumnHeaderConstant() { // column names for show topics statement public static final String TOPIC_NAME = "TopicName"; public static final String TOPIC_CONFIGS = "TopicConfigs"; + public static final String TAG_FILTER_STATUS = "TagFilterStatus"; + public static final String TAG_FILTER_MESSAGE = "TagFilterMessage"; public static final String TOPIC = "Topic"; public static final String CREATE_TOPIC = "Create Topic"; @@ -675,6 +677,13 @@ private ColumnHeaderConstant() { new ColumnHeader(TOPIC_NAME, TSDataType.TEXT), new ColumnHeader(TOPIC_CONFIGS, TSDataType.TEXT)); + public static final List showTableTopicColumnHeaders = + ImmutableList.of( + new ColumnHeader(TOPIC_NAME, TSDataType.TEXT), + new ColumnHeader(TOPIC_CONFIGS, TSDataType.TEXT), + new ColumnHeader(TAG_FILTER_STATUS, TSDataType.TEXT), + new ColumnHeader(TAG_FILTER_MESSAGE, TSDataType.TEXT)); + public static final List showCreateTopicColumnHeaders = ImmutableList.of( new ColumnHeader(TOPIC, TSDataType.TEXT), diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/subscription/meta/topic/TopicMeta.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/subscription/meta/topic/TopicMeta.java index c68bcdd8dd71b..7b518a580c10f 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/subscription/meta/topic/TopicMeta.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/subscription/meta/topic/TopicMeta.java @@ -37,6 +37,7 @@ import java.nio.ByteBuffer; import java.util.HashMap; import java.util.HashSet; +import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.Set; @@ -73,11 +74,11 @@ public TopicMeta( final String topicName, final long creationTime, final Map topicAttributes) { this.topicName = topicName; this.creationTime = creationTime; - this.config = new TopicConfig(topicAttributes); + this.config = new TopicConfig(canonicalizeTopicAttributes(topicAttributes)); this.ownerEpoch = -1L; this.maxOwnerEpoch = -1L; this.ownerLastTransferTimeMs = -1L; - initOwnerFromTopicAttributes(topicAttributes); + initOwnerFromTopicAttributes(this.config.getAttribute()); this.subscribedConsumerGroupIds = new HashSet<>(); } @@ -99,12 +100,18 @@ public TopicMeta deepCopy() { } public TopicMeta deepCopyWithUpdatedAttributes(final Map updatedAttributes) { - final Map copiedAttributes = new HashMap<>(config.getAttribute()); + final Map copiedAttributes = canonicalizeTopicAttributes(config.getAttribute()); if (Objects.nonNull(updatedAttributes)) { - copiedAttributes.putAll(updatedAttributes); - if ((updatedAttributes.containsKey(TopicConstant.OWNER_ID_KEY) - || updatedAttributes.containsKey(TopicConstant.OWNER_EPOCH_KEY)) - && !updatedAttributes.containsKey(TopicConstant.OWNER_LEASE_DURATION_MS_KEY)) { + boolean ownerChanged = false; + boolean leaseUpdated = false; + for (final Map.Entry entry : updatedAttributes.entrySet()) { + final String key = canonicalizeTopicAttributeKey(entry.getKey()); + copiedAttributes.put(key, entry.getValue()); + ownerChanged |= + TopicConstant.OWNER_ID_KEY.equals(key) || TopicConstant.OWNER_EPOCH_KEY.equals(key); + leaseUpdated |= TopicConstant.OWNER_LEASE_DURATION_MS_KEY.equals(key); + } + if (ownerChanged && !leaseUpdated) { copiedAttributes.remove(TopicConstant.OWNER_LEASE_DURATION_MS_KEY); } } @@ -310,6 +317,7 @@ public static TopicMeta deserialize(final InputStream inputStream) throws IOExce final String value = ReadWriteIOUtils.readString(inputStream); topicMeta.config.getAttribute().put(key, value); } + canonicalizeTopicAttributesInPlace(topicMeta.config.getAttribute()); size = ReadWriteIOUtils.readInt(inputStream); for (int i = 0; i < size; i++) { @@ -333,6 +341,7 @@ public static TopicMeta deserialize(final ByteBuffer byteBuffer) { final String value = ReadWriteIOUtils.readString(byteBuffer); topicMeta.config.getAttribute().put(key, value); } + canonicalizeTopicAttributesInPlace(topicMeta.config.getAttribute()); size = ReadWriteIOUtils.readInt(byteBuffer); for (int i = 0; i < size; i++) { @@ -396,6 +405,74 @@ public static void validateOwnerProgression( } } + private static Map canonicalizeTopicAttributes( + final Map topicAttributes) { + final Map canonicalAttributes = new HashMap<>(); + if (Objects.isNull(topicAttributes)) { + return canonicalAttributes; + } + topicAttributes.forEach( + (key, value) -> canonicalAttributes.put(canonicalizeTopicAttributeKey(key), value)); + return canonicalAttributes; + } + + private static void canonicalizeTopicAttributesInPlace(final Map attributes) { + final Map canonicalAttributes = canonicalizeTopicAttributes(attributes); + attributes.clear(); + attributes.putAll(canonicalAttributes); + } + + private static String canonicalizeTopicAttributeKey(final String key) { + if (Objects.isNull(key)) { + return null; + } + final String normalizedKey = key.trim().toLowerCase(Locale.ROOT); + switch (normalizedKey) { + case TopicConstant.PATH_KEY: + return TopicConstant.PATH_KEY; + case TopicConstant.PATTERN_KEY: + return TopicConstant.PATTERN_KEY; + case TopicConstant.DATABASE_KEY: + return TopicConstant.DATABASE_KEY; + case TopicConstant.TABLE_KEY: + return TopicConstant.TABLE_KEY; + case TopicConstant.COLUMN_FILTER_KEY: + return TopicConstant.COLUMN_FILTER_KEY; + case TopicConstant.TAG_FILTER_KEY: + return TopicConstant.TAG_FILTER_KEY; + case TopicConstant.RETENTION_BYTES_KEY: + return TopicConstant.RETENTION_BYTES_KEY; + case TopicConstant.RETENTION_MS_KEY: + return TopicConstant.RETENTION_MS_KEY; + case TopicConstant.START_TIME_KEY: + return TopicConstant.START_TIME_KEY; + case TopicConstant.END_TIME_KEY: + return TopicConstant.END_TIME_KEY; + case TopicConstant.MODE_KEY: + return TopicConstant.MODE_KEY; + case TopicConstant.ORDER_MODE_KEY: + return TopicConstant.ORDER_MODE_KEY; + case TopicConstant.FORMAT_KEY: + return TopicConstant.FORMAT_KEY; + case TopicConstant.LOOSE_RANGE_KEY: + return TopicConstant.LOOSE_RANGE_KEY; + case TopicConstant.STRICT_KEY: + return TopicConstant.STRICT_KEY; + case TopicConstant.OWNER_ID_KEY: + return TopicConstant.OWNER_ID_KEY; + case TopicConstant.OWNER_EPOCH_KEY: + return TopicConstant.OWNER_EPOCH_KEY; + case TopicConstant.MAX_OWNER_EPOCH_KEY: + return TopicConstant.MAX_OWNER_EPOCH_KEY; + case TopicConstant.OWNER_LEASE_DURATION_MS_KEY: + return TopicConstant.OWNER_LEASE_DURATION_MS_KEY; + case "__system.sql-dialect": + return "__system.sql-dialect"; + default: + return key; + } + } + private void initOwnerFromTopicAttributes(final Map topicAttributes) { final TopicConfig topicConfig = new TopicConfig(topicAttributes); @@ -455,6 +532,8 @@ public Map generateExtractorAttributes( extractorAttributes.putAll(config.getAttributesWithSourceDatabaseAndTableName()); // column-filter is evaluated by subscription runtime on DataNode. extractorAttributes.putAll(config.getAttributesWithSourceColumnFilter()); + // tag-filter is evaluated by subscription runtime on DataNode. + extractorAttributes.putAll(config.getAttributesWithSourceTagFilter()); } else { // tree model: path or pattern extractorAttributes.putAll(config.getAttributesWithSourcePathOrPattern());