From 5c9e2bf8bb463db4efdf94564d7ca8360bdee739 Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Tue, 4 Jun 2024 10:50:02 +0800 Subject: [PATCH 1/2] Pipe: fix uninitialized config when validate PipeHistoricalDataRegionTsFileExtractor parameters --- .../it/autocreate/IoTDBPipeAutoDropIT.java | 62 ++++++++++++++++++- ...peHistoricalDataRegionTsFileExtractor.java | 56 ++++++++++------- 2 files changed, 95 insertions(+), 23 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAutoDropIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAutoDropIT.java index ecd0b2d5775a6..60c0ba37d4b22 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAutoDropIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAutoDropIT.java @@ -34,6 +34,7 @@ import java.util.Map; public class IoTDBPipeAutoDropIT extends AbstractPipeDualAutoIT { + @Test public void testAutoDropInHistoricalTransfer() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); @@ -70,10 +71,69 @@ public void testAutoDropInHistoricalTransfer() throws Exception { Assert.assertEquals( TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select count(*) from root.**", + "count(root.db.d1.s1),", + Collections.singleton("1,")); + + TestUtils.assertDataEventuallyOnEnv( + senderEnv, + "show pipes", + "ID,CreationTime,State,PipeSource,PipeProcessor,PipeSink,ExceptionMessage,RemainingEventCount,EstimatedRemainingSeconds,", + Collections.emptySet()); + } + } + + @Test + public void testAutoDropInHistoricalTransferWithTimeRange() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + + if (!TestUtils.tryExecuteNonQueryWithRetry( + senderEnv, + "insert into root.db.d1(time, s1) values (1000, 1), (2000, 2), (3000, 3), (4000, 4), (5000, 5)")) { + return; + } + + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("extractor.mode", "query"); + extractorAttributes.put("extractor.start-time", "1970-01-01T08:00:02+08:00"); + extractorAttributes.put("extractor.end-time", "1970-01-01T08:00:04+08:00"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select count(*) from root.**", + "count(root.db.d1.s1),", + Collections.singleton("3,")); + TestUtils.assertDataEventuallyOnEnv( senderEnv, "show pipes", - "ID,CreationTime,State,PipeSource,PipeProcessor,PipeSink,ExceptionMessage,", + "ID,CreationTime,State,PipeSource,PipeProcessor,PipeSink,ExceptionMessage,RemainingEventCount,EstimatedRemainingSeconds,", Collections.emptySet()); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileExtractor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileExtractor.java index 1c150dd54093c..b5bff97ec98a0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileExtractor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileExtractor.java @@ -45,6 +45,7 @@ import org.apache.iotdb.pipe.api.event.Event; import org.apache.iotdb.pipe.api.exception.PipeParameterNotValidException; +import org.apache.tsfile.utils.Pair; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -101,6 +102,7 @@ public class PipeHistoricalDataRegionTsFileExtractor implements PipeHistoricalDa private boolean sloppyTimeRange; // true to disable time range filter after extraction + private Pair listeningOptionPair; private boolean shouldExtractInsertion; private boolean shouldTransferModFile; // Whether to transfer mods @@ -113,6 +115,14 @@ public class PipeHistoricalDataRegionTsFileExtractor implements PipeHistoricalDa public void validate(final PipeParameterValidator validator) { final PipeParameters parameters = validator.getParameters(); + try { + listeningOptionPair = + DataRegionListeningFilter.parseInsertionDeletionListeningOptionPair(parameters); + } catch (final Exception e) { + // compatible with the current validation framework + throw new PipeParameterNotValidException(e.getMessage()); + } + if (parameters.hasAnyAttributes( SOURCE_START_TIME_KEY, EXTRACTOR_START_TIME_KEY, @@ -140,11 +150,13 @@ public void validate(final PipeParameterValidator validator) { SOURCE_END_TIME_KEY, EXTRACTOR_END_TIME_KEY)); } - return; } catch (final Exception e) { // compatible with the current validation framework throw new PipeParameterNotValidException(e.getMessage()); } + + // return here + return; } // Historical data extraction is enabled in the following cases: @@ -185,23 +197,6 @@ public void validate(final PipeParameterValidator validator) { EXTRACTOR_HISTORY_END_TIME_KEY, SOURCE_HISTORY_END_TIME_KEY)); } - - shouldTransferModFile = - parameters.getBooleanOrDefault( - Arrays.asList(SOURCE_MODS_ENABLE_KEY, EXTRACTOR_MODS_ENABLE_KEY), - EXTRACTOR_MODS_ENABLE_DEFAULT_VALUE - || // Should extract deletion - DataRegionListeningFilter.parseInsertionDeletionListeningOptionPair(parameters) - .getRight()); - - shouldTerminatePipeOnAllHistoricalEventsConsumed = - parameters - .getStringOrDefault( - Arrays.asList( - PipeExtractorConstant.EXTRACTOR_MODE_KEY, - PipeExtractorConstant.SOURCE_MODE_KEY), - PipeExtractorConstant.EXTRACTOR_MODE_DEFAULT_VALUE) - .equalsIgnoreCase(PipeExtractorConstant.EXTRACTOR_MODE_QUERY_VALUE); } catch (final Exception e) { // Compatible with the current validation framework throw new PipeParameterNotValidException(e.getMessage()); @@ -212,8 +207,7 @@ public void validate(final PipeParameterValidator validator) { public void customize( final PipeParameters parameters, final PipeExtractorRuntimeConfiguration configuration) throws IllegalPathException { - shouldExtractInsertion = - DataRegionListeningFilter.parseInsertionDeletionListeningOptionPair(parameters).getLeft(); + shouldExtractInsertion = listeningOptionPair.getLeft(); // Do nothing if only extract deletion if (!shouldExtractInsertion) { return; @@ -295,15 +289,33 @@ public void customize( .collect(Collectors.toSet()) .contains("time"); + shouldTransferModFile = + parameters.getBooleanOrDefault( + Arrays.asList(SOURCE_MODS_ENABLE_KEY, EXTRACTOR_MODS_ENABLE_KEY), + EXTRACTOR_MODS_ENABLE_DEFAULT_VALUE + || // Should extract deletion + listeningOptionPair.getRight()); + + shouldTerminatePipeOnAllHistoricalEventsConsumed = + parameters + .getStringOrDefault( + Arrays.asList( + PipeExtractorConstant.EXTRACTOR_MODE_KEY, + PipeExtractorConstant.SOURCE_MODE_KEY), + PipeExtractorConstant.EXTRACTOR_MODE_DEFAULT_VALUE) + .equalsIgnoreCase(PipeExtractorConstant.EXTRACTOR_MODE_QUERY_VALUE); + LOGGER.info( - "Pipe {}@{}: historical data extraction time range, start time {}({}), end time {}({}), sloppy time range {}", + "Pipe {}@{}: historical data extraction time range, start time {}({}), end time {}({}), sloppy time range {}, should transfer mod file {}, should terminate pipe on all historical events consumed {}", pipeName, dataRegionId, DateTimeUtils.convertLongToDate(historicalDataExtractionStartTime), historicalDataExtractionStartTime, DateTimeUtils.convertLongToDate(historicalDataExtractionEndTime), historicalDataExtractionEndTime, - sloppyTimeRange); + sloppyTimeRange, + shouldTransferModFile, + shouldTerminatePipeOnAllHistoricalEventsConsumed); } private void flushDataRegionAllTsFiles() { From 85c87d554d855d2b0a1ba555b0dd98354069e3a5 Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Tue, 4 Jun 2024 10:50:42 +0800 Subject: [PATCH 2/2] Subscription: avoid poll messages from unsubscribed topics --- .../it/local/IoTDBSubscriptionBasicIT.java | 118 ++++++++++++++++-- .../receiver/SubscriptionReceiverV1.java | 11 +- 2 files changed, 118 insertions(+), 11 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/IoTDBSubscriptionBasicIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/IoTDBSubscriptionBasicIT.java index 46b697e44db62..13a5e8abc2617 100644 --- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/IoTDBSubscriptionBasicIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/IoTDBSubscriptionBasicIT.java @@ -60,6 +60,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.LockSupport; import java.util.stream.Collectors; @@ -181,7 +182,7 @@ public void testBasicSubscribeTsFile() throws Exception { } // Create topic - final String topicName = "topic2"; + final String topicName = "topic1"; final String host = EnvFactory.getEnv().getIP(); final int port = Integer.parseInt(EnvFactory.getEnv().getPort()); try (final SubscriptionSession session = new SubscriptionSession(host, port)) { @@ -297,11 +298,12 @@ public void testBasicPullConsumerWithCommitAsync() throws Exception { } // Create topic + final String topicName = "topic1"; final String host = EnvFactory.getEnv().getIP(); final int port = Integer.parseInt(EnvFactory.getEnv().getPort()); try (final SubscriptionSession session = new SubscriptionSession(host, port)) { session.open(); - session.createTopic("topic1"); + session.createTopic(topicName); } catch (final Exception e) { e.printStackTrace(); fail(e.getMessage()); @@ -325,7 +327,7 @@ public void testBasicPullConsumerWithCommitAsync() throws Exception { .autoCommit(false) .buildPullConsumer()) { consumer.open(); - consumer.subscribe("topic1"); + consumer.subscribe(topicName); while (!isClosed.get()) { LockSupport.parkNanos(IoTDBSubscriptionITConstant.SLEEP_NS); // wait some time final List messages = @@ -370,7 +372,7 @@ public void onFailure(final Throwable e) { } }); } - consumer.unsubscribe("topic1"); + consumer.unsubscribe(topicName); } catch (final Exception e) { e.printStackTrace(); // avoid fail @@ -450,11 +452,12 @@ public void testBasicPushConsumer() { } // Create topic + final String topicName = "topic1"; final String host = EnvFactory.getEnv().getIP(); final int port = Integer.parseInt(EnvFactory.getEnv().getPort()); try (final SubscriptionSession session = new SubscriptionSession(host, port)) { session.open(); - session.createTopic("topic1"); + session.createTopic(topicName); } catch (final Exception e) { e.printStackTrace(); fail(e.getMessage()); @@ -480,7 +483,7 @@ public void testBasicPushConsumer() { .buildPushConsumer()) { consumer.open(); - consumer.subscribe("topic1"); + consumer.subscribe(topicName); // The push consumer should automatically poll 10 rows of data by 1 onReceive() Awaitility.await() @@ -537,11 +540,112 @@ public void testBasicPushConsumer() { Assert.assertTrue(onReceiveCount.get() > lastOnReceiveCount.get()); }); - consumer.unsubscribe("topic1"); + consumer.unsubscribe(topicName); + } catch (final Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + @Test + public void testPollUnsubscribedTopics() throws Exception { + // Insert some historical data + try (final ISession session = EnvFactory.getEnv().getSessionConnection()) { + for (int i = 0; i < 100; ++i) { + session.executeNonQueryStatement( + String.format("insert into root.db.d1(time, s1) values (%s, 1)", i)); + } + for (int i = 100; i < 200; ++i) { + session.executeNonQueryStatement( + String.format("insert into root.db.d1(time, s1) values (%s, 1)", i)); + } + session.executeNonQueryStatement("flush"); } catch (final Exception e) { e.printStackTrace(); fail(e.getMessage()); } + + // Create topic + final String host = EnvFactory.getEnv().getIP(); + final int port = Integer.parseInt(EnvFactory.getEnv().getPort()); + try (final SubscriptionSession session = new SubscriptionSession(host, port)) { + session.open(); + { + final Properties properties = new Properties(); + properties.put(TopicConstant.END_TIME_KEY, 99); + session.createTopic("topic1", properties); + } + { + final Properties properties = new Properties(); + properties.put(TopicConstant.START_TIME_KEY, 100); + session.createTopic("topic2", properties); + } + } catch (final Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + // Subscription + final AtomicInteger rowCount = new AtomicInteger(); + final AtomicLong timestampSum = new AtomicLong(); + final AtomicBoolean isClosed = new AtomicBoolean(false); + final Thread thread = + new Thread( + () -> { + try (final SubscriptionPullConsumer consumer = + new SubscriptionPullConsumer.Builder() + .host(host) + .port(port) + .consumerId("c1") + .consumerGroupId("cg1") + .autoCommit(false) + .buildPullConsumer()) { + consumer.open(); + consumer.subscribe("topic2"); // only subscribe topic2 + while (!isClosed.get()) { + LockSupport.parkNanos(IoTDBSubscriptionITConstant.SLEEP_NS); // wait some time + final List messages = + consumer.poll(IoTDBSubscriptionITConstant.POLL_TIMEOUT_MS); + for (final SubscriptionMessage message : messages) { + for (final SubscriptionSessionDataSet dataSet : + message.getSessionDataSetsHandler()) { + while (dataSet.hasNext()) { + timestampSum.getAndAdd(dataSet.next().getTimestamp()); + rowCount.addAndGet(1); + } + } + } + consumer.commitSync(messages); + } + // automatically unsubscribe topics when closing + } catch (final Exception e) { + e.printStackTrace(); + // Avoid failure + } finally { + LOGGER.info("consumer exiting..."); + } + }); + thread.start(); + + // Check row count + try { + // Keep retrying if there are execution failures + Awaitility.await() + .pollDelay(IoTDBSubscriptionITConstant.AWAITILITY_POLL_DELAY_SECOND, TimeUnit.SECONDS) + .pollInterval( + IoTDBSubscriptionITConstant.AWAITILITY_POLL_INTERVAL_SECOND, TimeUnit.SECONDS) + .atMost(IoTDBSubscriptionITConstant.AWAITILITY_AT_MOST_SECOND, TimeUnit.SECONDS) + .untilAsserted( + () -> { + Assert.assertEquals(100, rowCount.get()); + Assert.assertEquals((100 + 199) * 100 / 2, timestampSum.get()); + }); + } catch (final Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } finally { + isClosed.set(true); + thread.join(); + } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiverV1.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiverV1.java index 16d157e184797..8339c3d3c8812 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiverV1.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiverV1.java @@ -399,18 +399,21 @@ private TPipeSubscribeResp handlePipeSubscribePoll(final PipeSubscribePollReq re private List handlePipeSubscribePollInternal( final ConsumerConfig consumerConfig, final PollPayload messagePayload) { + final Set subscribedTopicNames = + SubscriptionAgent.consumer() + .getTopicsSubscribedByConsumer( + consumerConfig.getConsumerGroupId(), consumerConfig.getConsumerId()); final Set topicNames; if (messagePayload.getTopicNames().isEmpty()) { // poll all subscribed topics - topicNames = - SubscriptionAgent.consumer() - .getTopicsSubscribedByConsumer( - consumerConfig.getConsumerGroupId(), consumerConfig.getConsumerId()); + topicNames = subscribedTopicNames; } else { topicNames = messagePayload.getTopicNames().stream() .map(ASTVisitor::parseIdentifier) .collect(Collectors.toSet()); + // filter unsubscribed topics + topicNames.removeIf((topicName) -> !subscribedTopicNames.contains(topicName)); } return SubscriptionAgent.broker().poll(consumerConfig, topicNames);