From 00a3a2bb067b4465b5e8173333fa1cf2052877b9 Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Mon, 3 Jun 2024 20:57:03 +0800 Subject: [PATCH 01/36] basic setup --- .../SubscriptionTerminationException.java | 40 ++++++++++++++++++ .../poll/SubscriptionPollResponse.java | 3 ++ .../poll/SubscriptionPollResponseType.java | 2 + .../payload/poll/TerminationPayload.java | 42 +++++++++++++++++++ .../consumer/SubscriptionConsumer.java | 21 ++++++---- ...peHistoricalDataRegionTsFileExtractor.java | 41 +++++++++--------- .../broker/SubscriptionBroker.java | 9 ++-- .../broker/SubscriptionPrefetchingQueue.java | 36 +++++++++++++++- .../SubscriptionPrefetchingTabletsQueue.java | 14 ++++++- .../SubscriptionPrefetchingTsFileQueue.java | 22 +++++++++- .../receiver/SubscriptionReceiverV1.java | 10 +++-- .../subscription/meta/topic/TopicMeta.java | 2 + 12 files changed, 204 insertions(+), 38 deletions(-) create mode 100644 iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/exception/SubscriptionTerminationException.java create mode 100644 iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/TerminationPayload.java diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/exception/SubscriptionTerminationException.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/exception/SubscriptionTerminationException.java new file mode 100644 index 0000000000000..f75ec4ff78915 --- /dev/null +++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/exception/SubscriptionTerminationException.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.rpc.subscription.exception; + +import java.util.Objects; + +public class SubscriptionTerminationException extends SubscriptionRuntimeCriticalException { + + public SubscriptionTerminationException(final String message) { + super(message); + } + + public SubscriptionTerminationException(final String message, final Throwable cause) { + super(message, cause); + } + + @Override + public boolean equals(final Object obj) { + return obj instanceof SubscriptionTerminationException + && Objects.equals(getMessage(), ((SubscriptionTerminationException) obj).getMessage()) + && Objects.equals(getTimeStamp(), ((SubscriptionTerminationException) obj).getTimeStamp()); + } +} diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/SubscriptionPollResponse.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/SubscriptionPollResponse.java index d386903f124d0..01d173d274283 100644 --- a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/SubscriptionPollResponse.java +++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/SubscriptionPollResponse.java @@ -95,6 +95,9 @@ public static SubscriptionPollResponse deserialize(final ByteBuffer buffer) { case ERROR: payload = new ErrorPayload().deserialize(buffer); break; + case TERMINATION: + payload = new TerminationPayload().deserialize(buffer); + break; default: LOGGER.warn("unexpected response type: {}, payload will be null", responseType); break; diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/SubscriptionPollResponseType.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/SubscriptionPollResponseType.java index a22c9590c65fc..b27791b36c538 100644 --- a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/SubscriptionPollResponseType.java +++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/SubscriptionPollResponseType.java @@ -31,6 +31,8 @@ public enum SubscriptionPollResponseType { FILE_INIT((short) 2), FILE_PIECE((short) 3), FILE_SEAL((short) 4), + + TERMINATION((short) 5), ; private final short type; diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/TerminationPayload.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/TerminationPayload.java new file mode 100644 index 0000000000000..c2da3f92e6dd8 --- /dev/null +++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/TerminationPayload.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.rpc.subscription.payload.poll; + +import java.io.DataOutputStream; +import java.nio.ByteBuffer; + +public class TerminationPayload implements SubscriptionPollPayload { + + @Override + public void serialize(final DataOutputStream stream) { + // do nothing + } + + @Override + public SubscriptionPollPayload deserialize(final ByteBuffer buffer) { + // do nothing + return null; + } + + @Override + public String toString() { + return "TerminationPayload"; + } +} diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionConsumer.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionConsumer.java index 2bbd4a129a91b..e4ac1f6811b50 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionConsumer.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionConsumer.java @@ -336,18 +336,23 @@ protected List poll(final Set topicNames, final lon case ERROR: final ErrorPayload payload = (ErrorPayload) pollResponse.getPayload(); final String errorMessage = payload.getErrorMessage(); - final boolean critical = payload.isCritical(); - LOGGER.warn( - "Error occurred when SubscriptionConsumer {} polling topics {}: {}, critical: {}", - this, - topicNames, - errorMessage, - critical); - if (critical) { + if (payload.isCritical()) { throw new SubscriptionRuntimeCriticalException(errorMessage); } else { throw new SubscriptionRuntimeNonCriticalException(errorMessage); } + case TERMINATION: + final SubscriptionCommitContext commitContext = pollResponse.getCommitContext(); + final String topicNameToUnsubscribe = commitContext.getTopicName(); + LOGGER.info( + "Termination occurred when SubscriptionConsumer {} polling topics {}, unsubscribe topic {} on DN {} automatically", + this, + topicNames, + topicNameToUnsubscribe, + commitContext.getDataNodeId()); + unsubscribe(topicNameToUnsubscribe); + // TODO: notify user by ex? + break; default: LOGGER.warn("unexpected response type: {}", responseType); break; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileExtractor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileExtractor.java index 1c150dd54093c..5e116fa037c93 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileExtractor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileExtractor.java @@ -140,7 +140,6 @@ public void validate(final PipeParameterValidator validator) { SOURCE_END_TIME_KEY, EXTRACTOR_END_TIME_KEY)); } - return; } catch (final Exception e) { // compatible with the current validation framework throw new PipeParameterNotValidException(e.getMessage()); @@ -185,23 +184,6 @@ public void validate(final PipeParameterValidator validator) { EXTRACTOR_HISTORY_END_TIME_KEY, SOURCE_HISTORY_END_TIME_KEY)); } - - shouldTransferModFile = - parameters.getBooleanOrDefault( - Arrays.asList(SOURCE_MODS_ENABLE_KEY, EXTRACTOR_MODS_ENABLE_KEY), - EXTRACTOR_MODS_ENABLE_DEFAULT_VALUE - || // Should extract deletion - DataRegionListeningFilter.parseInsertionDeletionListeningOptionPair(parameters) - .getRight()); - - shouldTerminatePipeOnAllHistoricalEventsConsumed = - parameters - .getStringOrDefault( - Arrays.asList( - PipeExtractorConstant.EXTRACTOR_MODE_KEY, - PipeExtractorConstant.SOURCE_MODE_KEY), - PipeExtractorConstant.EXTRACTOR_MODE_DEFAULT_VALUE) - .equalsIgnoreCase(PipeExtractorConstant.EXTRACTOR_MODE_QUERY_VALUE); } catch (final Exception e) { // Compatible with the current validation framework throw new PipeParameterNotValidException(e.getMessage()); @@ -295,15 +277,34 @@ public void customize( .collect(Collectors.toSet()) .contains("time"); + shouldTransferModFile = + parameters.getBooleanOrDefault( + Arrays.asList(SOURCE_MODS_ENABLE_KEY, EXTRACTOR_MODS_ENABLE_KEY), + EXTRACTOR_MODS_ENABLE_DEFAULT_VALUE + || // Should extract deletion + DataRegionListeningFilter.parseInsertionDeletionListeningOptionPair(parameters) + .getRight()); + + shouldTerminatePipeOnAllHistoricalEventsConsumed = + parameters + .getStringOrDefault( + Arrays.asList( + PipeExtractorConstant.EXTRACTOR_MODE_KEY, + PipeExtractorConstant.SOURCE_MODE_KEY), + PipeExtractorConstant.EXTRACTOR_MODE_DEFAULT_VALUE) + .equalsIgnoreCase(PipeExtractorConstant.EXTRACTOR_MODE_QUERY_VALUE); + LOGGER.info( - "Pipe {}@{}: historical data extraction time range, start time {}({}), end time {}({}), sloppy time range {}", + "Pipe {}@{}: historical data extraction time range, start time {}({}), end time {}({}), sloppy time range {}, should transfer mod file {}, should terminate pipe on all historical events consumed {}", pipeName, dataRegionId, DateTimeUtils.convertLongToDate(historicalDataExtractionStartTime), historicalDataExtractionStartTime, DateTimeUtils.convertLongToDate(historicalDataExtractionEndTime), historicalDataExtractionEndTime, - sloppyTimeRange); + sloppyTimeRange, + shouldTransferModFile, + shouldTerminatePipeOnAllHistoricalEventsConsumed); } private void flushDataRegionAllTsFiles() { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionBroker.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionBroker.java index 90785c1602e62..4655b268cda84 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionBroker.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionBroker.java @@ -164,9 +164,12 @@ public void unbindPrefetchingQueue(final String topicName) { } // clean up uncommitted events prefetchingQueue.cleanup(); - topicNameToPrefetchingQueue.remove(topicName); - SubscriptionPrefetchingQueueMetrics.getInstance() - .deregister(prefetchingQueue.getPrefetchingQueueId()); + prefetchingQueue.markCompleted(); + + // TODO + // topicNameToPrefetchingQueue.remove(topicName); + // SubscriptionPrefetchingQueueMetrics.getInstance() + // .deregister(prefetchingQueue.getPrefetchingQueueId()); } public void executePrefetch(final String topicName) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingQueue.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingQueue.java index 43aef19568148..6dae4d8706232 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingQueue.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingQueue.java @@ -24,12 +24,17 @@ import org.apache.iotdb.db.pipe.agent.PipeAgent; import org.apache.iotdb.db.subscription.event.SubscriptionEvent; import org.apache.iotdb.db.subscription.event.SubscriptionEventBinaryCache; +import org.apache.iotdb.db.subscription.event.SubscriptionTsFileEvent; import org.apache.iotdb.pipe.api.event.Event; import org.apache.iotdb.rpc.subscription.payload.poll.SubscriptionCommitContext; +import org.apache.iotdb.rpc.subscription.payload.poll.SubscriptionPollResponse; +import org.apache.iotdb.rpc.subscription.payload.poll.SubscriptionPollResponseType; +import org.apache.iotdb.rpc.subscription.payload.poll.TerminationPayload; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.Collections; import java.util.Map; import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; @@ -46,6 +51,8 @@ public abstract class SubscriptionPrefetchingQueue { protected final Map uncommittedEvents; private final AtomicLong subscriptionCommitIdGenerator = new AtomicLong(0); + private volatile boolean isCompleted = false; + public SubscriptionPrefetchingQueue( final String brokerId, final String topicName, @@ -57,7 +64,15 @@ public SubscriptionPrefetchingQueue( this.uncommittedEvents = new ConcurrentHashMap<>(); } - public abstract SubscriptionEvent poll(final String consumerId); + public SubscriptionEvent poll(final String consumerId) { + if (isCompleted()) { + return generateSubscriptionPollTerminationResponse(); + } + + return pollInternal(consumerId); + } + + public abstract SubscriptionEvent pollInternal(final String consumerId); public abstract void executePrefetch(); @@ -150,4 +165,23 @@ public long getUncommittedEventCount() { public long getCurrentCommitId() { return subscriptionCommitIdGenerator.get(); } + + /////////////////////////////// termination /////////////////////////////// + + public boolean isCompleted() { + return isCompleted; + } + + public void markCompleted() { + isCompleted = true; + } + + private SubscriptionTsFileEvent generateSubscriptionPollTerminationResponse() { + return new SubscriptionTsFileEvent( + Collections.emptyList(), + new SubscriptionPollResponse( + SubscriptionPollResponseType.TERMINATION.getType(), + new TerminationPayload(), + generateInvalidSubscriptionCommitContext())); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingTabletsQueue.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingTabletsQueue.java index 195194ef3c899..7ca1f1cdf505b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingTabletsQueue.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingTabletsQueue.java @@ -25,6 +25,7 @@ import org.apache.iotdb.db.pipe.event.UserDefinedEnrichedEvent; import org.apache.iotdb.db.pipe.event.common.tablet.PipeInsertNodeTabletInsertionEvent; import org.apache.iotdb.db.pipe.event.common.tablet.PipeRawTabletInsertionEvent; +import org.apache.iotdb.db.pipe.event.common.terminate.PipeTerminateEvent; import org.apache.iotdb.db.pipe.event.common.tsfile.PipeTsFileInsertionEvent; import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryManager; import org.apache.iotdb.db.subscription.event.SubscriptionEvent; @@ -64,7 +65,7 @@ public SubscriptionPrefetchingTabletsQueue( } @Override - public SubscriptionEvent poll(final String consumerId) { + public SubscriptionEvent pollInternal(final String consumerId) { if (prefetchingQueue.isEmpty()) { prefetchOnce(); } @@ -129,6 +130,17 @@ private void prefetchOnce() { continue; } + if (event instanceof PipeTerminateEvent) { + LOGGER.info( + "Subscription: SubscriptionPrefetchingTabletsQueue {} commit PipeTerminateEvent {}", + this, + event); + // commit directly + ((PipeTerminateEvent) event) + .decreaseReferenceCount(SubscriptionPrefetchingTsFileQueue.class.getName(), true); + continue; + } + if (event instanceof TabletInsertionEvent) { final List currentTablets = convertToTablets((TabletInsertionEvent) event); if (currentTablets.isEmpty()) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingTsFileQueue.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingTsFileQueue.java index fc5a1a8a81eeb..24fd5d4013855 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingTsFileQueue.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingTsFileQueue.java @@ -22,6 +22,7 @@ import org.apache.iotdb.commons.pipe.event.EnrichedEvent; import org.apache.iotdb.commons.pipe.task.connection.UnboundedBlockingPendingQueue; import org.apache.iotdb.db.pipe.event.UserDefinedEnrichedEvent; +import org.apache.iotdb.db.pipe.event.common.terminate.PipeTerminateEvent; import org.apache.iotdb.db.pipe.event.common.tsfile.PipeTsFileInsertionEvent; import org.apache.iotdb.db.subscription.event.SubscriptionTsFileEvent; import org.apache.iotdb.pipe.api.event.Event; @@ -64,7 +65,7 @@ public SubscriptionPrefetchingTsFileQueue( } @Override - public SubscriptionTsFileEvent poll(final String consumerId) { + public SubscriptionTsFileEvent pollInternal(final String consumerId) { if (hasUnPollableOnTheFlySubscriptionTsFileEvent(consumerId)) { return null; } @@ -78,6 +79,25 @@ public SubscriptionTsFileEvent poll(final String consumerId) { Event event; while (Objects.nonNull( event = UserDefinedEnrichedEvent.maybeOf(inputPendingQueue.waitedPoll()))) { + if (!(event instanceof EnrichedEvent)) { + LOGGER.warn( + "Subscription: SubscriptionPrefetchingTsFileQueue {} only support poll EnrichedEvent. Ignore {}.", + this, + event); + continue; + } + + if (event instanceof PipeTerminateEvent) { + LOGGER.info( + "Subscription: SubscriptionPrefetchingTsFileQueue {} commit PipeTerminateEvent {}", + this, + event); + // commit directly + ((PipeTerminateEvent) event) + .decreaseReferenceCount(SubscriptionPrefetchingTsFileQueue.class.getName(), true); + continue; + } + if (event instanceof TabletInsertionEvent) { final String errorMessage = String.format( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiverV1.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiverV1.java index 16d157e184797..9e89767dcc74d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiverV1.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiverV1.java @@ -399,18 +399,20 @@ private TPipeSubscribeResp handlePipeSubscribePoll(final PipeSubscribePollReq re private List handlePipeSubscribePollInternal( final ConsumerConfig consumerConfig, final PollPayload messagePayload) { + final Set subscribedTopicNames = + SubscriptionAgent.consumer() + .getTopicsSubscribedByConsumer( + consumerConfig.getConsumerGroupId(), consumerConfig.getConsumerId()); final Set topicNames; if (messagePayload.getTopicNames().isEmpty()) { // poll all subscribed topics - topicNames = - SubscriptionAgent.consumer() - .getTopicsSubscribedByConsumer( - consumerConfig.getConsumerGroupId(), consumerConfig.getConsumerId()); + topicNames = subscribedTopicNames; } else { topicNames = messagePayload.getTopicNames().stream() .map(ASTVisitor::parseIdentifier) .collect(Collectors.toSet()); + topicNames.removeIf((topicName) -> !subscribedTopicNames.contains(topicName)); } return SubscriptionAgent.broker().poll(consumerConfig, topicNames); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/subscription/meta/topic/TopicMeta.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/subscription/meta/topic/TopicMeta.java index 33f5065cbae65..a8ef2a80003e5 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/subscription/meta/topic/TopicMeta.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/subscription/meta/topic/TopicMeta.java @@ -185,6 +185,8 @@ public Map generateExtractorAttributes() { extractorAttributes.putAll(config.getAttributesWithTimeRange(creationTime)); // realtime mode extractorAttributes.putAll(config.getAttributesWithRealtimeMode()); + // mode + extractorAttributes.put("mode", "query"); return extractorAttributes; } From 30bc70c1a8b2864041e0bab65d41b4694b25402c Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Mon, 3 Jun 2024 21:17:02 +0800 Subject: [PATCH 02/36] conf source mode on topic --- .../apache/iotdb/rpc/subscription/config/TopicConfig.java | 7 +++++++ .../iotdb/rpc/subscription/config/TopicConstant.java | 5 +++++ .../iotdb/commons/subscription/meta/topic/TopicMeta.java | 4 ++-- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/config/TopicConfig.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/config/TopicConfig.java index fe62ba622a965..453f363c8f752 100644 --- a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/config/TopicConfig.java +++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/config/TopicConfig.java @@ -101,6 +101,13 @@ public Map getAttributesWithRealtimeMode() { : Collections.singletonMap("realtime.mode", "hybrid"); } + public Map getAttributesWithSourceMode() { + return TopicConstant.MODE_QUERY_VALUE.equals( + attributes.getOrDefault(TopicConstant.MODE_KEY, TopicConstant.MODE_DEFAULT_VALUE)) + ? Collections.singletonMap("mode", "query") + : Collections.singletonMap("mode", "subscribe"); + } + public Map getAttributesWithProcessorPrefix() { final Map attributesWithProcessorPrefix = new HashMap<>(); attributes.forEach( diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/config/TopicConstant.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/config/TopicConstant.java index bd44cf1bcc95d..a6e8ec901bf9f 100644 --- a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/config/TopicConstant.java +++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/config/TopicConstant.java @@ -30,6 +30,11 @@ public class TopicConstant { public static final String END_TIME_KEY = "end-time"; public static final String NOW_TIME_VALUE = "now"; + public static final String MODE_KEY = "mode"; + public static final String MODE_SUBSCRIBE_VALUE = "subscribe"; + public static final String MODE_QUERY_VALUE = "query"; + public static final String MODE_DEFAULT_VALUE = MODE_SUBSCRIBE_VALUE; + public static final String FORMAT_KEY = "format"; public static final String FORMAT_SESSION_DATA_SETS_HANDLER_VALUE = "SessionDataSetsHandler"; public static final String FORMAT_TS_FILE_HANDLER_VALUE = "TsFileHandler"; diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/subscription/meta/topic/TopicMeta.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/subscription/meta/topic/TopicMeta.java index a8ef2a80003e5..1ccf4d66cd392 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/subscription/meta/topic/TopicMeta.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/subscription/meta/topic/TopicMeta.java @@ -185,8 +185,8 @@ public Map generateExtractorAttributes() { extractorAttributes.putAll(config.getAttributesWithTimeRange(creationTime)); // realtime mode extractorAttributes.putAll(config.getAttributesWithRealtimeMode()); - // mode - extractorAttributes.put("mode", "query"); + // source mode + extractorAttributes.putAll(config.getAttributesWithSourceMode()); return extractorAttributes; } From 873edc1e2fa8836dad00140b830d526b955812a1 Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Tue, 4 Jun 2024 00:30:29 +0800 Subject: [PATCH 03/36] add IT --- .../env/cluster/config/MppCommonConfig.java | 26 ++++ .../cluster/config/MppSharedCommonConfig.java | 25 ++++ .../env/remote/config/RemoteCommonConfig.java | 17 +++ .../apache/iotdb/itbase/env/CommonConfig.java | 7 + .../it/dual/AbstractSubscriptionDualIT.java | 12 +- .../it/dual/IoTDBSubscriptionTopicIT.java | 127 +++++++++++++++++- 6 files changed, 205 insertions(+), 9 deletions(-) diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppCommonConfig.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppCommonConfig.java index 95fed66d08b21..095562db5a51f 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppCommonConfig.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppCommonConfig.java @@ -438,6 +438,32 @@ public CommonConfig setCnConnectionTimeoutMs(int connectionTimeoutMs) { return this; } + @Override + public CommonConfig setPipeHeartbeatIntervalSecondsForCollectingPipeMeta( + int pipeHeartbeatIntervalSecondsForCollectingPipeMeta) { + setProperty( + "pipe_heartbeat_interval_seconds_for_collecting_pipe_meta", + String.valueOf(pipeHeartbeatIntervalSecondsForCollectingPipeMeta)); + return this; + } + + @Override + public CommonConfig setPipeMetaSyncerInitialSyncDelayMinutes( + long pipeMetaSyncerInitialSyncDelayMinutes) { + setProperty( + "pipe_meta_syncer_initial_sync_delay_minutes", + String.valueOf(pipeMetaSyncerInitialSyncDelayMinutes)); + return this; + } + + @Override + public CommonConfig setPipeMetaSyncerSyncIntervalMinutes(long pipeMetaSyncerSyncIntervalMinutes) { + setProperty( + "pipe_meta_syncer_sync_interval_minutes", + String.valueOf(pipeMetaSyncerSyncIntervalMinutes)); + return this; + } + // For part of the log directory public String getClusterConfigStr() { return fromConsensusFullNameToAbbr(properties.getProperty(CONFIG_NODE_CONSENSUS_PROTOCOL_CLASS)) diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppSharedCommonConfig.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppSharedCommonConfig.java index 1851590dd411e..7724bcf09c0af 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppSharedCommonConfig.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppSharedCommonConfig.java @@ -445,4 +445,29 @@ public CommonConfig setCnConnectionTimeoutMs(int connectionTimeoutMs) { cnConfig.setCnConnectionTimeoutMs(connectionTimeoutMs); return this; } + + @Override + public CommonConfig setPipeHeartbeatIntervalSecondsForCollectingPipeMeta( + int pipeHeartbeatIntervalSecondsForCollectingPipeMeta) { + dnConfig.setPipeHeartbeatIntervalSecondsForCollectingPipeMeta( + pipeHeartbeatIntervalSecondsForCollectingPipeMeta); + cnConfig.setPipeHeartbeatIntervalSecondsForCollectingPipeMeta( + pipeHeartbeatIntervalSecondsForCollectingPipeMeta); + return this; + } + + @Override + public CommonConfig setPipeMetaSyncerInitialSyncDelayMinutes( + long pipeMetaSyncerInitialSyncDelayMinutes) { + dnConfig.setPipeMetaSyncerInitialSyncDelayMinutes(pipeMetaSyncerInitialSyncDelayMinutes); + cnConfig.setPipeMetaSyncerInitialSyncDelayMinutes(pipeMetaSyncerInitialSyncDelayMinutes); + return this; + } + + @Override + public CommonConfig setPipeMetaSyncerSyncIntervalMinutes(long pipeMetaSyncerSyncIntervalMinutes) { + dnConfig.setPipeMetaSyncerSyncIntervalMinutes(pipeMetaSyncerSyncIntervalMinutes); + cnConfig.setPipeMetaSyncerSyncIntervalMinutes(pipeMetaSyncerSyncIntervalMinutes); + return this; + } } diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteCommonConfig.java b/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteCommonConfig.java index 0ac82fc871373..36675e646865d 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteCommonConfig.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteCommonConfig.java @@ -313,4 +313,21 @@ public CommonConfig setTagAttributeTotalSize(int tagAttributeTotalSize) { public CommonConfig setCnConnectionTimeoutMs(int connectionTimeoutMs) { return this; } + + @Override + public CommonConfig setPipeHeartbeatIntervalSecondsForCollectingPipeMeta( + int pipeHeartbeatIntervalSecondsForCollectingPipeMeta) { + return this; + } + + @Override + public CommonConfig setPipeMetaSyncerInitialSyncDelayMinutes( + long pipeMetaSyncerInitialSyncDelayMinutes) { + return this; + } + + @Override + public CommonConfig setPipeMetaSyncerSyncIntervalMinutes(long pipeMetaSyncerSyncIntervalMinutes) { + return this; + } } diff --git a/integration-test/src/main/java/org/apache/iotdb/itbase/env/CommonConfig.java b/integration-test/src/main/java/org/apache/iotdb/itbase/env/CommonConfig.java index af47c8254e532..c06c45ddcd42d 100644 --- a/integration-test/src/main/java/org/apache/iotdb/itbase/env/CommonConfig.java +++ b/integration-test/src/main/java/org/apache/iotdb/itbase/env/CommonConfig.java @@ -139,4 +139,11 @@ CommonConfig setEnableAutoLeaderBalanceForIoTConsensus( CommonConfig setTagAttributeTotalSize(int tagAttributeTotalSize); CommonConfig setCnConnectionTimeoutMs(int connectionTimeoutMs); + + CommonConfig setPipeHeartbeatIntervalSecondsForCollectingPipeMeta( + int pipeHeartbeatIntervalSecondsForCollectingPipeMeta); + + CommonConfig setPipeMetaSyncerInitialSyncDelayMinutes(long pipeMetaSyncerInitialSyncDelayMinutes); + + CommonConfig setPipeMetaSyncerSyncIntervalMinutes(long pipeMetaSyncerSyncIntervalMinutes); } diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/AbstractSubscriptionDualIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/AbstractSubscriptionDualIT.java index 1d0ef260ed488..e03658ccf81aa 100644 --- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/AbstractSubscriptionDualIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/AbstractSubscriptionDualIT.java @@ -44,8 +44,16 @@ public void setUp() { receiverEnv.getConfig().getCommonConfig().setPipeAirGapReceiverEnabled(true); // 10 min, assert that the operations will not time out - senderEnv.getConfig().getCommonConfig().setCnConnectionTimeoutMs(600000); - receiverEnv.getConfig().getCommonConfig().setCnConnectionTimeoutMs(600000); + senderEnv.getConfig().getCommonConfig().setCnConnectionTimeoutMs(600_000); + receiverEnv.getConfig().getCommonConfig().setCnConnectionTimeoutMs(600_000); + + // shorten heartbeat interval to avoid timeout of query mode test + senderEnv + .getConfig() + .getCommonConfig() + .setPipeHeartbeatIntervalSecondsForCollectingPipeMeta(30); + senderEnv.getConfig().getCommonConfig().setPipeMetaSyncerInitialSyncDelayMinutes(1); + senderEnv.getConfig().getCommonConfig().setPipeMetaSyncerSyncIntervalMinutes(1); senderEnv.initClusterEnvironment(); receiverEnv.initClusterEnvironment(); diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionTopicIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionTopicIT.java index 7091c93b4dbc3..f3ec09aa638a4 100644 --- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionTopicIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionTopicIT.java @@ -20,12 +20,15 @@ package org.apache.iotdb.subscription.it.dual; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; +import org.apache.iotdb.confignode.rpc.thrift.TShowSubscriptionReq; +import org.apache.iotdb.confignode.rpc.thrift.TShowSubscriptionResp; import org.apache.iotdb.confignode.rpc.thrift.TShowTopicInfo; import org.apache.iotdb.confignode.rpc.thrift.TShowTopicReq; import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.isession.ISession; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.MultiClusterIT2Subscription; +import org.apache.iotdb.rpc.RpcUtils; import org.apache.iotdb.rpc.subscription.config.TopicConstant; import org.apache.iotdb.rpc.subscription.exception.SubscriptionRuntimeCriticalException; import org.apache.iotdb.session.subscription.SubscriptionSession; @@ -33,6 +36,10 @@ import org.apache.iotdb.session.subscription.payload.SubscriptionMessage; import org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant; +import org.apache.tsfile.read.TsFileReader; +import org.apache.tsfile.read.common.Path; +import org.apache.tsfile.read.expression.QueryExpression; +import org.apache.tsfile.read.query.dataset.QueryDataSet; import org.apache.tsfile.write.record.Tablet; import org.awaitility.Awaitility; import org.junit.Assert; @@ -45,6 +52,7 @@ import java.sql.Connection; import java.sql.Statement; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -392,9 +400,9 @@ public void testTopicNameWithBackQuote() throws Exception { } // Create topic on sender - final String topic1 = "`topic1`"; - final String topic2 = "`'topic2'`"; - final String topic3 = "`\"topic3\"`"; + final String topic1 = "`topic4`"; + final String topic2 = "`'topic5'`"; + final String topic3 = "`\"topic6\"`"; final String host = senderEnv.getIP(); final int port = Integer.parseInt(senderEnv.getPort()); try (final SubscriptionSession session = new SubscriptionSession(host, port)) { @@ -507,7 +515,7 @@ public void testTopicInvalidTimeRangeConfig() throws Exception { final Properties properties = new Properties(); properties.put(TopicConstant.START_TIME_KEY, "2024-01-32"); properties.put(TopicConstant.END_TIME_KEY, "now"); - session.createTopic("topic1", properties); + session.createTopic("topic7", properties); fail(); } catch (final Exception ignored) { } @@ -519,7 +527,7 @@ public void testTopicInvalidTimeRangeConfig() throws Exception { final Properties properties = new Properties(); properties.put(TopicConstant.START_TIME_KEY, "2001.01.01T08:00:00"); properties.put(TopicConstant.END_TIME_KEY, "2000.01.01T08:00:00"); - session.createTopic("topic2", properties); + session.createTopic("topic8", properties); fail(); } catch (final Exception ignored) { } @@ -532,7 +540,7 @@ public void testTopicInvalidPathConfig() throws Exception { final Properties config = new Properties(); config.put(TopicConstant.FORMAT_KEY, TopicConstant.FORMAT_TS_FILE_HANDLER_VALUE); config.put(TopicConstant.PATH_KEY, "root.db.*.s"); - testTopicInvalidRuntimeConfigTemplate("topic3", config); + testTopicInvalidRuntimeConfigTemplate("topic9", config); } @Test @@ -543,7 +551,112 @@ public void testTopicInvalidProcessorConfig() throws Exception { config.put("processor", "tumbling-time-sampling-processor"); config.put("processor.tumbling-time.interval-seconds", "1"); config.put("processor.down-sampling.split-file", "true"); - testTopicInvalidRuntimeConfigTemplate("topic4", config); + testTopicInvalidRuntimeConfigTemplate("topic10", config); + } + + @Test + public void testTopicWithQueryMode() throws Exception { + // Insert some historical data + try (final ISession session = senderEnv.getSessionConnection()) { + for (int i = 0; i < 100; ++i) { + session.executeNonQueryStatement( + String.format("insert into root.db.d1(time, s1) values (%s, 1)", i)); + } + session.executeNonQueryStatement("flush"); + } catch (final Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + // Create topic + final String topicName = "topic11"; + final String host = senderEnv.getIP(); + final int port = Integer.parseInt(senderEnv.getPort()); + try (final SubscriptionSession session = new SubscriptionSession(host, port)) { + session.open(); + final Properties config = new Properties(); + config.put(TopicConstant.FORMAT_KEY, TopicConstant.FORMAT_TS_FILE_HANDLER_VALUE); + config.put(TopicConstant.MODE_KEY, TopicConstant.MODE_QUERY_VALUE); + session.createTopic(topicName, config); + } catch (final Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + // Subscription + final AtomicInteger rowCount = new AtomicInteger(); + final AtomicBoolean isClosed = new AtomicBoolean(false); + final Thread thread = + new Thread( + () -> { + try (final SubscriptionPullConsumer consumer = + new SubscriptionPullConsumer.Builder() + .host(host) + .port(port) + .consumerId("c1") + .consumerGroupId("cg1") + .autoCommit(false) + .buildPullConsumer()) { + consumer.open(); + consumer.subscribe(topicName); + while (!isClosed.get()) { + LockSupport.parkNanos(IoTDBSubscriptionITConstant.SLEEP_NS); // wait some time + final List messages = + consumer.poll(IoTDBSubscriptionITConstant.POLL_TIMEOUT_MS); + for (final SubscriptionMessage message : messages) { + try (final TsFileReader tsFileReader = + message.getTsFileHandler().openReader()) { + final Path path = new Path("root.db.d1", "s1", true); + final QueryDataSet dataSet = + tsFileReader.query( + QueryExpression.create(Collections.singletonList(path), null)); + while (dataSet.hasNext()) { + dataSet.next(); + rowCount.addAndGet(1); + } + } + } + consumer.commitSync(messages); + } + consumer.unsubscribe(topicName); + } catch (final Exception e) { + e.printStackTrace(); + // Avoid failure + } finally { + LOGGER.info("consumer exiting..."); + } + }); + thread.start(); + + // Check row count + try { + // Keep retrying if there are execution failures + Awaitility.await() + .pollDelay(IoTDBSubscriptionITConstant.AWAITILITY_POLL_DELAY_SECOND, TimeUnit.SECONDS) + .pollInterval( + IoTDBSubscriptionITConstant.AWAITILITY_POLL_INTERVAL_SECOND, TimeUnit.SECONDS) + .atMost(IoTDBSubscriptionITConstant.AWAITILITY_AT_MOST_SECOND, TimeUnit.SECONDS) + .untilAsserted( + () -> { + Assert.assertEquals(100, rowCount.get()); + // empty subscription + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final TShowSubscriptionResp showSubscriptionResp = + client.showSubscription(new TShowSubscriptionReq()); + Assert.assertEquals( + RpcUtils.SUCCESS_STATUS.getCode(), showSubscriptionResp.status.getCode()); + Assert.assertNotNull(showSubscriptionResp.subscriptionInfoList); + Assert.assertEquals(0, showSubscriptionResp.subscriptionInfoList.size()); + } + }); + } catch (final Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } finally { + isClosed.set(true); + thread.join(); + } } private void testTopicInvalidRuntimeConfigTemplate( From 6334c8bcd9a0241173e6b0c97d18f5973a3332d8 Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Tue, 4 Jun 2024 06:30:15 +0800 Subject: [PATCH 04/36] fix IT --- ...peHistoricalDataRegionTsFileExtractor.java | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileExtractor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileExtractor.java index 5e116fa037c93..b5bff97ec98a0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileExtractor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileExtractor.java @@ -45,6 +45,7 @@ import org.apache.iotdb.pipe.api.event.Event; import org.apache.iotdb.pipe.api.exception.PipeParameterNotValidException; +import org.apache.tsfile.utils.Pair; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -101,6 +102,7 @@ public class PipeHistoricalDataRegionTsFileExtractor implements PipeHistoricalDa private boolean sloppyTimeRange; // true to disable time range filter after extraction + private Pair listeningOptionPair; private boolean shouldExtractInsertion; private boolean shouldTransferModFile; // Whether to transfer mods @@ -113,6 +115,14 @@ public class PipeHistoricalDataRegionTsFileExtractor implements PipeHistoricalDa public void validate(final PipeParameterValidator validator) { final PipeParameters parameters = validator.getParameters(); + try { + listeningOptionPair = + DataRegionListeningFilter.parseInsertionDeletionListeningOptionPair(parameters); + } catch (final Exception e) { + // compatible with the current validation framework + throw new PipeParameterNotValidException(e.getMessage()); + } + if (parameters.hasAnyAttributes( SOURCE_START_TIME_KEY, EXTRACTOR_START_TIME_KEY, @@ -144,6 +154,9 @@ public void validate(final PipeParameterValidator validator) { // compatible with the current validation framework throw new PipeParameterNotValidException(e.getMessage()); } + + // return here + return; } // Historical data extraction is enabled in the following cases: @@ -194,8 +207,7 @@ public void validate(final PipeParameterValidator validator) { public void customize( final PipeParameters parameters, final PipeExtractorRuntimeConfiguration configuration) throws IllegalPathException { - shouldExtractInsertion = - DataRegionListeningFilter.parseInsertionDeletionListeningOptionPair(parameters).getLeft(); + shouldExtractInsertion = listeningOptionPair.getLeft(); // Do nothing if only extract deletion if (!shouldExtractInsertion) { return; @@ -282,8 +294,7 @@ public void customize( Arrays.asList(SOURCE_MODS_ENABLE_KEY, EXTRACTOR_MODS_ENABLE_KEY), EXTRACTOR_MODS_ENABLE_DEFAULT_VALUE || // Should extract deletion - DataRegionListeningFilter.parseInsertionDeletionListeningOptionPair(parameters) - .getRight()); + listeningOptionPair.getRight()); shouldTerminatePipeOnAllHistoricalEventsConsumed = parameters From 40d2699afcc6323d73844cc1dcbe52460ccf86af Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Tue, 4 Jun 2024 09:11:13 +0800 Subject: [PATCH 05/36] improve --- .../it/dual/IoTDBSubscriptionTopicIT.java | 4 +-- .../agent/SubscriptionAgentLauncher.java | 8 ++--- .../agent/SubscriptionBrokerAgent.java | 9 ++++++ .../agent/SubscriptionConsumerAgent.java | 29 ++++++++++++------- .../agent/SubscriptionReceiverAgent.java | 6 ++-- .../agent/SubscriptionTopicAgent.java | 20 ++++++------- .../broker/SubscriptionBroker.java | 27 ++++++++++++++--- .../meta/consumer/ConsumerGroupMeta.java | 25 ++++++++++++++++ 8 files changed, 95 insertions(+), 33 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionTopicIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionTopicIT.java index f3ec09aa638a4..0ddb29985573a 100644 --- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionTopicIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionTopicIT.java @@ -628,7 +628,6 @@ public void testTopicWithQueryMode() throws Exception { }); thread.start(); - // Check row count try { // Keep retrying if there are execution failures Awaitility.await() @@ -638,8 +637,9 @@ public void testTopicWithQueryMode() throws Exception { .atMost(IoTDBSubscriptionITConstant.AWAITILITY_AT_MOST_SECOND, TimeUnit.SECONDS) .untilAsserted( () -> { + // Check row count Assert.assertEquals(100, rowCount.get()); - // empty subscription + // Check empty subscription try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { final TShowSubscriptionResp showSubscriptionResp = diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionAgentLauncher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionAgentLauncher.java index 32dd078dd7805..d83996cda470b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionAgentLauncher.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionAgentLauncher.java @@ -87,7 +87,7 @@ public static synchronized void launchSubscriptionTopicAgent() throws StartupExc } return; - } catch (SubscriptionException | ClientManagerException | TException e) { + } catch (final SubscriptionException | ClientManagerException | TException e) { retry++; LOGGER.warn( "Failed to get topic meta from config node for {} times, will retry at most {} times.", @@ -97,7 +97,7 @@ public static synchronized void launchSubscriptionTopicAgent() throws StartupExc try { Thread.sleep( retry * SubscriptionConfig.getInstance().getSubscriptionLaunchRetryIntervalMs()); - } catch (InterruptedException interruptedException) { + } catch (final InterruptedException interruptedException) { LOGGER.info( "Interrupted while sleeping, will retry to get topic meta from config node.", interruptedException); @@ -146,7 +146,7 @@ public static synchronized void launchSubscriptionConsumerAgent() throws Startup } return; - } catch (SubscriptionException | ClientManagerException | TException e) { + } catch (final SubscriptionException | ClientManagerException | TException e) { retry++; LOGGER.warn( "Failed to get consumer group meta from config node for {} times, will retry at most {} times.", @@ -156,7 +156,7 @@ public static synchronized void launchSubscriptionConsumerAgent() throws Startup try { Thread.sleep( retry * SubscriptionConfig.getInstance().getSubscriptionLaunchRetryIntervalMs()); - } catch (InterruptedException interruptedException) { + } catch (final InterruptedException interruptedException) { LOGGER.info( "Interrupted while sleeping, will retry to get consumer group meta from config node.", interruptedException); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionBrokerAgent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionBrokerAgent.java index aff4893c61a03..82fbc4d28d89e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionBrokerAgent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionBrokerAgent.java @@ -151,6 +151,15 @@ public void unbindPrefetchingQueue(final SubscriptionConnectorSubtask subtask) { broker.unbindPrefetchingQueue(subtask.getTopicName()); } + public void removePrefetchingQueue(final String consumerGroupId, final String topicName) { + final SubscriptionBroker broker = consumerGroupIdToSubscriptionBroker.get(consumerGroupId); + if (Objects.isNull(broker)) { + LOGGER.warn("Subscription: consumer group [{}] does not exist", consumerGroupId); + return; + } + broker.removePrefetchingQueue(topicName); + } + public void executePrefetch(final SubscriptionConnectorSubtask subtask) { final String consumerGroupId = subtask.getConsumerGroupId(); final SubscriptionBroker broker = consumerGroupIdToSubscriptionBroker.get(consumerGroupId); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionConsumerAgent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionConsumerAgent.java index d6572923635c4..abf4f4bdd4da9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionConsumerAgent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionConsumerAgent.java @@ -62,7 +62,7 @@ protected void releaseWriteLock() { ////////////////////////// ConsumerGroupMeta Management Entry ////////////////////////// public TPushConsumerGroupMetaRespExceptionMessage handleSingleConsumerGroupMetaChanges( - ConsumerGroupMeta consumerGroupMetaFromCoordinator) { + final ConsumerGroupMeta consumerGroupMetaFromCoordinator) { acquireWriteLock(); try { if (consumerGroupMetaFromCoordinator.isEmpty()) { @@ -71,7 +71,7 @@ public TPushConsumerGroupMetaRespExceptionMessage handleSingleConsumerGroupMetaC handleSingleConsumerGroupMetaChangesInternal(consumerGroupMetaFromCoordinator); } return null; - } catch (Exception e) { + } catch (final Exception e) { final String consumerGroupId = consumerGroupMetaFromCoordinator.getConsumerGroupId(); final String exceptionMessage = String.format( @@ -114,6 +114,13 @@ private void handleSingleConsumerGroupMetaChangesInternal( return; } + // remove unbounded prefetching queue + final Set topicsUnsubByGroup = + ConsumerGroupMeta.getTopicsUnsubByGroup(metaInAgent, metaFromCoordinator); + for (final String topicName : topicsUnsubByGroup) { + SubscriptionAgent.broker().removePrefetchingQueue(consumerGroupId, topicName); + } + // TODO: Currently we fully replace the entire ConsumerGroupMeta without carefully checking the // changes in its fields. consumerGroupMetaKeeper.removeConsumerGroupMeta(consumerGroupId); @@ -121,14 +128,15 @@ private void handleSingleConsumerGroupMetaChangesInternal( } public TPushConsumerGroupMetaRespExceptionMessage handleConsumerGroupMetaChanges( - List consumerGroupMetasFromCoordinator) { + final List consumerGroupMetasFromCoordinator) { acquireWriteLock(); try { - for (ConsumerGroupMeta consumerGroupMetaFromCoordinator : consumerGroupMetasFromCoordinator) { + for (final ConsumerGroupMeta consumerGroupMetaFromCoordinator : + consumerGroupMetasFromCoordinator) { try { handleSingleConsumerGroupMetaChangesInternal(consumerGroupMetaFromCoordinator); return null; - } catch (Exception e) { + } catch (final Exception e) { final String consumerGroupId = consumerGroupMetaFromCoordinator.getConsumerGroupId(); final String exceptionMessage = String.format( @@ -146,12 +154,12 @@ public TPushConsumerGroupMetaRespExceptionMessage handleConsumerGroupMetaChanges } public TPushConsumerGroupMetaRespExceptionMessage handleDropConsumerGroup( - String consumerGroupId) { + final String consumerGroupId) { acquireWriteLock(); try { handleDropConsumerGroupInternal(consumerGroupId); return null; - } catch (Exception e) { + } catch (final Exception e) { final String exceptionMessage = String.format( "Subscription: Failed to drop consumer group %s, because %s", consumerGroupId, e); @@ -163,7 +171,7 @@ public TPushConsumerGroupMetaRespExceptionMessage handleDropConsumerGroup( } } - private void handleDropConsumerGroupInternal(String consumerGroupId) { + private void handleDropConsumerGroupInternal(final String consumerGroupId) { if (SubscriptionAgent.broker().isBrokerExist(consumerGroupId)) { if (!SubscriptionAgent.broker().dropBroker(consumerGroupId)) { final String exceptionMessage = @@ -180,7 +188,7 @@ private void handleDropConsumerGroupInternal(String consumerGroupId) { consumerGroupMetaKeeper.removeConsumerGroupMeta(consumerGroupId); } - public boolean isConsumerExisted(String consumerGroupId, String consumerId) { + public boolean isConsumerExisted(final String consumerGroupId, final String consumerId) { acquireReadLock(); try { final ConsumerGroupMeta consumerGroupMeta = @@ -191,7 +199,8 @@ public boolean isConsumerExisted(String consumerGroupId, String consumerId) { } } - public Set getTopicsSubscribedByConsumer(String consumerGroupId, String consumerId) { + public Set getTopicsSubscribedByConsumer( + final String consumerGroupId, final String consumerId) { acquireReadLock(); try { return consumerGroupMetaKeeper.getTopicsSubscribedByConsumer(consumerGroupId, consumerId); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionReceiverAgent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionReceiverAgent.java index 1bb0735b019a3..71f319374cd53 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionReceiverAgent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionReceiverAgent.java @@ -51,7 +51,7 @@ public class SubscriptionReceiverAgent { PipeSubscribeRequestVersion.VERSION_1.getVersion(), SubscriptionReceiverV1::new); } - public TPipeSubscribeResp handle(TPipeSubscribeReq req) { + public TPipeSubscribeResp handle(final TPipeSubscribeReq req) { final byte reqVersion = req.getVersion(); if (RECEIVER_CONSTRUCTORS.containsKey(reqVersion)) { return getReceiver(reqVersion).handle(req); @@ -69,7 +69,7 @@ public TPipeSubscribeResp handle(TPipeSubscribeReq req) { } } - private SubscriptionReceiver getReceiver(byte reqVersion) { + private SubscriptionReceiver getReceiver(final byte reqVersion) { if (receiverThreadLocal.get() == null) { return setAndGetReceiver(reqVersion); } @@ -88,7 +88,7 @@ private SubscriptionReceiver getReceiver(byte reqVersion) { return receiverThreadLocal.get(); } - private SubscriptionReceiver setAndGetReceiver(byte reqVersion) { + private SubscriptionReceiver setAndGetReceiver(final byte reqVersion) { if (RECEIVER_CONSTRUCTORS.containsKey(reqVersion)) { receiverThreadLocal.set(RECEIVER_CONSTRUCTORS.get(reqVersion).get()); } else { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionTopicAgent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionTopicAgent.java index fb27e8825b8e8..45145bd2b67e0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionTopicAgent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionTopicAgent.java @@ -60,12 +60,12 @@ protected void releaseWriteLock() { ////////////////////////// Topic Management Entry ////////////////////////// public TPushTopicMetaRespExceptionMessage handleSingleTopicMetaChanges( - TopicMeta topicMetaFromCoordinator) { + final TopicMeta topicMetaFromCoordinator) { acquireWriteLock(); try { handleSingleTopicMetaChangesInternal(topicMetaFromCoordinator); return null; - } catch (Exception e) { + } catch (final Exception e) { final String topicName = topicMetaFromCoordinator.getTopicName(); final String exceptionMessage = String.format( @@ -86,13 +86,13 @@ private void handleSingleTopicMetaChangesInternal(final TopicMeta metaFromCoordi } public TPushTopicMetaRespExceptionMessage handleTopicMetaChanges( - List topicMetasFromCoordinator) { + final List topicMetasFromCoordinator) { acquireWriteLock(); try { - for (TopicMeta topicMetaFromCoordinator : topicMetasFromCoordinator) { + for (final TopicMeta topicMetaFromCoordinator : topicMetasFromCoordinator) { try { handleSingleTopicMetaChangesInternal(topicMetaFromCoordinator); - } catch (Exception e) { + } catch (final Exception e) { final String topicName = topicMetaFromCoordinator.getTopicName(); final String exceptionMessage = String.format( @@ -109,12 +109,12 @@ public TPushTopicMetaRespExceptionMessage handleTopicMetaChanges( } } - public TPushTopicMetaRespExceptionMessage handleDropTopic(String topicName) { + public TPushTopicMetaRespExceptionMessage handleDropTopic(final String topicName) { acquireWriteLock(); try { handleDropTopicInternal(topicName); return null; - } catch (Exception e) { + } catch (final Exception e) { final String exceptionMessage = String.format("Subscription: Failed to drop topic %s, because %s", topicName, e); LOGGER.warn(exceptionMessage); @@ -125,11 +125,11 @@ public TPushTopicMetaRespExceptionMessage handleDropTopic(String topicName) { } } - private void handleDropTopicInternal(String topicName) { + private void handleDropTopicInternal(final String topicName) { topicMetaKeeper.removeTopicMeta(topicName); } - public boolean isTopicExisted(String topicName) { + public boolean isTopicExisted(final String topicName) { acquireReadLock(); try { return topicMetaKeeper.containsTopicMeta(topicName); @@ -138,7 +138,7 @@ public boolean isTopicExisted(String topicName) { } } - public String getTopicFormat(String topicName) { + public String getTopicFormat(final String topicName) { acquireReadLock(); try { return topicMetaKeeper diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionBroker.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionBroker.java index 4655b268cda84..2a4b0d7b13004 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionBroker.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionBroker.java @@ -162,14 +162,33 @@ public void unbindPrefetchingQueue(final String topicName) { LOGGER.warn("Subscription: prefetching queue bound to topic [{}] does not exist", topicName); return; } + // clean up uncommitted events prefetchingQueue.cleanup(); prefetchingQueue.markCompleted(); - // TODO - // topicNameToPrefetchingQueue.remove(topicName); - // SubscriptionPrefetchingQueueMetrics.getInstance() - // .deregister(prefetchingQueue.getPrefetchingQueueId()); + // we remove this prefetching queue and deregister metrics when consumers dropping related + // subscription... + } + + public void removePrefetchingQueue(final String topicName) { + final SubscriptionPrefetchingQueue prefetchingQueue = + topicNameToPrefetchingQueue.get(topicName); + if (Objects.isNull(prefetchingQueue)) { + LOGGER.warn("Subscription: prefetching queue bound to topic [{}] does not exist", topicName); + return; + } + + if (!prefetchingQueue.isCompleted()) { + LOGGER.warn( + "Subscription: prefetching queue bound to topic [{}] is not completed when removing", + topicName); + return; + } + + topicNameToPrefetchingQueue.remove(topicName); + SubscriptionPrefetchingQueueMetrics.getInstance() + .deregister(prefetchingQueue.getPrefetchingQueueId()); } public void executePrefetch(final String topicName) { diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/subscription/meta/consumer/ConsumerGroupMeta.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/subscription/meta/consumer/ConsumerGroupMeta.java index d53868208bf5e..4c60a87c4847c 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/subscription/meta/consumer/ConsumerGroupMeta.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/subscription/meta/consumer/ConsumerGroupMeta.java @@ -74,6 +74,31 @@ public long getCreationTime() { return creationTime; } + public static /* @NonNull */ Set getTopicsUnsubByGroup( + final ConsumerGroupMeta currentMeta, final ConsumerGroupMeta updatedMeta) { + if (!Objects.equals(currentMeta.consumerGroupId, updatedMeta.consumerGroupId)) { + return Collections.emptySet(); + } + if (!Objects.equals(currentMeta.creationTime, updatedMeta.creationTime)) { + return Collections.emptySet(); + } + if (!Objects.equals( + currentMeta.consumerIdToConsumerMeta, updatedMeta.consumerIdToConsumerMeta)) { + return Collections.emptySet(); + } + final Set unsubscribedTopicNames = new HashSet<>(); + currentMeta + .topicNameToSubscribedConsumerIdSet + .keySet() + .forEach( + (topicName) -> { + if (!updatedMeta.topicNameToSubscribedConsumerIdSet.containsKey(topicName)) { + unsubscribedTopicNames.add(topicName); + } + }); + return unsubscribedTopicNames; + } + /////////////////////////////// consumer /////////////////////////////// public void addConsumer(ConsumerMeta consumerMeta) { From 097ce6d28d044ee0348f01220a1a7182f297faf9 Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Tue, 4 Jun 2024 09:51:02 +0800 Subject: [PATCH 06/36] improve comment --- .../iotdb/subscription/it/dual/AbstractSubscriptionDualIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/AbstractSubscriptionDualIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/AbstractSubscriptionDualIT.java index e03658ccf81aa..e0b05f5d3faf4 100644 --- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/AbstractSubscriptionDualIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/AbstractSubscriptionDualIT.java @@ -47,7 +47,7 @@ public void setUp() { senderEnv.getConfig().getCommonConfig().setCnConnectionTimeoutMs(600_000); receiverEnv.getConfig().getCommonConfig().setCnConnectionTimeoutMs(600_000); - // shorten heartbeat interval to avoid timeout of query mode test + // shorten heartbeat and sync interval to avoid timeout of query mode test senderEnv .getConfig() .getCommonConfig() From c1bf2948f5a34ff4ee3c0ca53f992e9bb045dbdf Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Thu, 6 Jun 2024 11:50:48 +0800 Subject: [PATCH 07/36] improve --- .../DropSubscriptionProcedure.java | 167 +++++++++--------- .../agent/SubscriptionBrokerAgent.java | 19 +- .../agent/SubscriptionConsumerAgent.java | 2 +- .../broker/SubscriptionBroker.java | 27 +-- .../subtask/SubscriptionConnectorSubtask.java | 18 +- ...SubscriptionConnectorSubtaskLifeCycle.java | 53 ++---- 6 files changed, 119 insertions(+), 167 deletions(-) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/subscription/subscription/DropSubscriptionProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/subscription/subscription/DropSubscriptionProcedure.java index efb5aa228272d..674500761b0ff 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/subscription/subscription/DropSubscriptionProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/subscription/subscription/DropSubscriptionProcedure.java @@ -62,9 +62,10 @@ public class DropSubscriptionProcedure extends AbstractOperateSubscriptionAndPip private TUnsubscribeReq unsubscribeReq; - private AlterConsumerGroupProcedure alterConsumerGroupProcedure; - private List alterTopicProcedures = new ArrayList<>(); + // NOTE: The 'drop pipe' operation should be performed before 'alter consumer group'. private List dropPipeProcedures = new ArrayList<>(); + private List alterTopicProcedures = new ArrayList<>(); + private AlterConsumerGroupProcedure alterConsumerGroupProcedure; // Record failed index of procedures to rollback properly. // We only record fail index when executing on config nodes, because when executing on data nodes @@ -76,7 +77,7 @@ public DropSubscriptionProcedure() { super(); } - public DropSubscriptionProcedure(TUnsubscribeReq unsubscribeReq) { + public DropSubscriptionProcedure(final TUnsubscribeReq unsubscribeReq) { this.unsubscribeReq = unsubscribeReq; } @@ -86,27 +87,27 @@ protected SubscriptionOperation getOperation() { } @Override - protected void executeFromValidate(ConfigNodeProcedureEnv env) throws SubscriptionException { + protected void executeFromValidate(final ConfigNodeProcedureEnv env) throws SubscriptionException { LOGGER.info("DropSubscriptionProcedure: executeFromValidate"); subscriptionInfo.get().validateBeforeUnsubscribe(unsubscribeReq); // Construct AlterConsumerGroupProcedure - ConsumerGroupMeta updatedConsumerGroupMeta = + final ConsumerGroupMeta updatedConsumerGroupMeta = subscriptionInfo.get().deepCopyConsumerGroupMeta(unsubscribeReq.getConsumerGroupId()); // Get topics subscribed by no consumers in this group after this un-subscription - Set topicsUnsubByGroup = + final Set topicsUnsubByGroup = updatedConsumerGroupMeta.removeSubscription( unsubscribeReq.getConsumerId(), unsubscribeReq.getTopicNames()); alterConsumerGroupProcedure = new AlterConsumerGroupProcedure(updatedConsumerGroupMeta, subscriptionInfo); - for (String topic : unsubscribeReq.getTopicNames()) { + for (final String topic : unsubscribeReq.getTopicNames()) { if (topicsUnsubByGroup.contains(topic)) { // Topic will be subscribed by no consumers in this group - TopicMeta updatedTopicMeta = subscriptionInfo.get().deepCopyTopicMeta(topic); + final TopicMeta updatedTopicMeta = subscriptionInfo.get().deepCopyTopicMeta(topic); updatedTopicMeta.removeSubscribedConsumerGroup(unsubscribeReq.getConsumerGroupId()); alterTopicProcedures.add(new AlterTopicProcedure(updatedTopicMeta, subscriptionInfo)); @@ -118,38 +119,39 @@ protected void executeFromValidate(ConfigNodeProcedureEnv env) throws Subscripti } } - alterConsumerGroupProcedure.executeFromValidate(env); + // Validate DropPipeProcedureV2s + for (final DropPipeProcedureV2 dropPipeProcedure : dropPipeProcedures) { + dropPipeProcedure.executeFromValidateTask(env); + dropPipeProcedure.executeFromCalculateInfoForTask(env); + } - for (AlterTopicProcedure alterTopicProcedure : alterTopicProcedures) { + // Validate AlterTopicProcedures + for (final AlterTopicProcedure alterTopicProcedure : alterTopicProcedures) { alterTopicProcedure.executeFromValidate(env); } - for (DropPipeProcedureV2 dropPipeProcedure : dropPipeProcedures) { - dropPipeProcedure.executeFromValidateTask(env); - dropPipeProcedure.executeFromCalculateInfoForTask(env); - } + // Validate AlterConsumerGroupProcedure + alterConsumerGroupProcedure.executeFromValidate(env); } @Override - protected void executeFromOperateOnConfigNodes(ConfigNodeProcedureEnv env) + protected void executeFromOperateOnConfigNodes(final ConfigNodeProcedureEnv env) throws SubscriptionException { LOGGER.info("DropSubscriptionProcedure: executeFromOperateOnConfigNodes"); - alterConsumerGroupProcedure.executeFromOperateOnConfigNodes(env); - TSStatus response; - List alterTopicPlans = - alterTopicProcedures.stream() - .map(AlterTopicProcedure::getUpdatedTopicMeta) - .map(AlterTopicPlan::new) + // Execute DropPipeProcedureV2s + final List dropPipePlans = + dropPipeProcedures.stream() + .map(proc -> new DropPipePlanV2(proc.getPipeName())) .collect(Collectors.toList()); try { response = env.getConfigManager() .getConsensusManager() - .write(new AlterMultipleTopicsPlan(alterTopicPlans)); - } catch (ConsensusException e) { + .write(new OperateMultiplePipesPlanV2(dropPipePlans)); + } catch (final ConsensusException e) { LOGGER.warn("Failed in the write API executing the consensus layer due to: ", e); response = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()); response.setMessage(e.getMessage()); @@ -157,19 +159,21 @@ protected void executeFromOperateOnConfigNodes(ConfigNodeProcedureEnv env) if (response.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode() && response.getSubStatusSize() > 0) { // Record the failed index for rollback - alterTopicProcedureFailIndexOnCN = response.getSubStatusSize() - 1; + dropPipeProcedureFailIndexOnCN = response.getSubStatusSize() - 1; } - List dropPipePlans = - dropPipeProcedures.stream() - .map(proc -> new DropPipePlanV2(proc.getPipeName())) + // Execute AlterTopicProcedures + final List alterTopicPlans = + alterTopicProcedures.stream() + .map(AlterTopicProcedure::getUpdatedTopicMeta) + .map(AlterTopicPlan::new) .collect(Collectors.toList()); try { response = env.getConfigManager() .getConsensusManager() - .write(new OperateMultiplePipesPlanV2(dropPipePlans)); - } catch (ConsensusException e) { + .write(new AlterMultipleTopicsPlan(alterTopicPlans)); + } catch (final ConsensusException e) { LOGGER.warn("Failed in the write API executing the consensus layer due to: ", e); response = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()); response.setMessage(e.getMessage()); @@ -177,34 +181,24 @@ protected void executeFromOperateOnConfigNodes(ConfigNodeProcedureEnv env) if (response.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode() && response.getSubStatusSize() > 0) { // Record the failed index for rollback - dropPipeProcedureFailIndexOnCN = response.getSubStatusSize() - 1; + alterTopicProcedureFailIndexOnCN = response.getSubStatusSize() - 1; } + + // Execute AlterConsumerGroupProcedure + alterConsumerGroupProcedure.executeFromOperateOnConfigNodes(env); } @Override - protected void executeFromOperateOnDataNodes(ConfigNodeProcedureEnv env) + protected void executeFromOperateOnDataNodes(final ConfigNodeProcedureEnv env) throws SubscriptionException, IOException { LOGGER.info("DropSubscriptionProcedure: executeFromOperateOnDataNodes"); - alterConsumerGroupProcedure.executeFromOperateOnDataNodes(env); - - // push topic meta to data nodes - List topicMetaBinaryList = new ArrayList<>(); - for (AlterTopicProcedure alterTopicProcedure : alterTopicProcedures) { - topicMetaBinaryList.add(alterTopicProcedure.getUpdatedTopicMeta().serialize()); - } - if (pushTopicMetaHasException(env.pushMultiTopicMetaToDataNodes(topicMetaBinaryList))) { - // If not all topic meta are pushed successfully, the meta can be pushed during meta sync. - LOGGER.warn( - "Failed to alter topics when creating subscription, metadata will be synchronized later."); - } - - // push pipe meta to data nodes - List pipeNames = + // Push pipe meta to data nodes + final List pipeNames = dropPipeProcedures.stream() .map(DropPipeProcedureV2::getPipeName) .collect(Collectors.toList()); - String exceptionMessage = + final String exceptionMessage = AbstractOperatePipeProcedureV2.parsePushPipeMetaExceptionForPipe( null, dropMultiPipeOnDataNodes(pipeNames, env)); if (!exceptionMessage.isEmpty()) { @@ -214,23 +208,36 @@ protected void executeFromOperateOnDataNodes(ConfigNodeProcedureEnv env) pipeNames, exceptionMessage); } + + // Push topic meta to data nodes + final List topicMetaBinaryList = new ArrayList<>(); + for (final AlterTopicProcedure alterTopicProcedure : alterTopicProcedures) { + topicMetaBinaryList.add(alterTopicProcedure.getUpdatedTopicMeta().serialize()); + } + if (pushTopicMetaHasException(env.pushMultiTopicMetaToDataNodes(topicMetaBinaryList))) { + // If not all topic meta are pushed successfully, the meta can be pushed during meta sync. + LOGGER.warn( + "Failed to alter topics when creating subscription, metadata will be synchronized later."); + } + + // Push consumer group meta to data nodes + alterConsumerGroupProcedure.executeFromOperateOnDataNodes(env); } @Override - protected void rollbackFromValidate(ConfigNodeProcedureEnv env) { + protected void rollbackFromValidate(final ConfigNodeProcedureEnv env) { LOGGER.info("DropSubscriptionProcedure: rollbackFromLock"); } @Override - protected void rollbackFromOperateOnConfigNodes(ConfigNodeProcedureEnv env) { + protected void rollbackFromOperateOnConfigNodes(final ConfigNodeProcedureEnv env) { LOGGER.info("DropSubscriptionProcedure: rollbackFromOperateOnConfigNodes"); - alterConsumerGroupProcedure.rollbackFromOperateOnConfigNodes(env); + // Do nothing to rollback DropPipeProcedureV2s + // Rollback AlterTopicProcedures TSStatus response; - - // rollback alterTopicProcedures - List alterTopicRollbackPlans = new ArrayList<>(); + final List alterTopicRollbackPlans = new ArrayList<>(); for (int i = 0; i <= Math.min(alterTopicProcedureFailIndexOnCN, alterTopicProcedures.size()); i++) { @@ -242,33 +249,26 @@ protected void rollbackFromOperateOnConfigNodes(ConfigNodeProcedureEnv env) { env.getConfigManager() .getConsensusManager() .write(new AlterMultipleTopicsPlan(alterTopicRollbackPlans)); - } catch (ConsensusException e) { + } catch (final ConsensusException e) { LOGGER.warn("Failed in the write API executing the consensus layer due to: ", e); response = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()); response.setMessage(e.getMessage()); } - // if failed to rollback, throw exception + // If failed to rollback, throw exception if (response.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { throw new SubscriptionException(response.getMessage()); } - // Do nothing to rollback dropPipeProcedures + // Rollback AlterConsumerGroupProcedure + alterConsumerGroupProcedure.rollbackFromOperateOnConfigNodes(env); } @Override - protected void rollbackFromOperateOnDataNodes(ConfigNodeProcedureEnv env) throws IOException { + protected void rollbackFromOperateOnDataNodes(final ConfigNodeProcedureEnv env) throws IOException { LOGGER.info("DropSubscriptionProcedure: rollbackFromOperateOnDataNodes"); - alterConsumerGroupProcedure.rollbackFromOperateOnDataNodes(env); - - // Push all topic metas to datanode, may be time-consuming - if (pushTopicMetaHasException(pushTopicMetaToDataNodes(env))) { - LOGGER.warn( - "Failed to rollback alter topics when dropping subscription, metadata will be synchronized later."); - } - // Push all pipe metas to datanode, may be time-consuming - String exceptionMessage = + final String exceptionMessage = AbstractOperatePipeProcedureV2.parsePushPipeMetaExceptionForPipe( null, AbstractOperatePipeProcedureV2.pushPipeMetaToDataNodes(env, pipeTaskInfo)); if (!exceptionMessage.isEmpty()) { @@ -276,10 +276,19 @@ protected void rollbackFromOperateOnDataNodes(ConfigNodeProcedureEnv env) throws "Failed to rollback create pipes when dropping subscription, details: {}, metadata will be synchronized later.", exceptionMessage); } + + // Push all topic metas to datanode, may be time-consuming + if (pushTopicMetaHasException(pushTopicMetaToDataNodes(env))) { + LOGGER.warn( + "Failed to rollback alter topics when dropping subscription, metadata will be synchronized later."); + } + + // Rollback AlterConsumerGroupProcedure + alterConsumerGroupProcedure.rollbackFromOperateOnDataNodes(env); } @Override - public void serialize(DataOutputStream stream) throws IOException { + public void serialize(final DataOutputStream stream) throws IOException { stream.writeShort(ProcedureType.DROP_SUBSCRIPTION_PROCEDURE.getTypeCode()); super.serialize(stream); @@ -289,7 +298,7 @@ public void serialize(DataOutputStream stream) throws IOException { final int size = unsubscribeReq.getTopicNamesSize(); ReadWriteIOUtils.write(size, stream); if (size != 0) { - for (String topicName : unsubscribeReq.getTopicNames()) { + for (final String topicName : unsubscribeReq.getTopicNames()) { ReadWriteIOUtils.write(topicName, stream); } } @@ -306,7 +315,7 @@ public void serialize(DataOutputStream stream) throws IOException { if (alterTopicProcedures != null) { ReadWriteIOUtils.write(true, stream); ReadWriteIOUtils.write(alterTopicProcedures.size(), stream); - for (AlterTopicProcedure topicProcedure : alterTopicProcedures) { + for (final AlterTopicProcedure topicProcedure : alterTopicProcedures) { topicProcedure.serialize(stream); } } else { @@ -317,7 +326,7 @@ public void serialize(DataOutputStream stream) throws IOException { if (dropPipeProcedures != null) { ReadWriteIOUtils.write(true, stream); ReadWriteIOUtils.write(dropPipeProcedures.size(), stream); - for (AbstractOperatePipeProcedureV2 pipeProcedure : dropPipeProcedures) { + for (final AbstractOperatePipeProcedureV2 pipeProcedure : dropPipeProcedures) { pipeProcedure.serialize(stream); } } else { @@ -326,7 +335,7 @@ public void serialize(DataOutputStream stream) throws IOException { } @Override - public void deserialize(ByteBuffer byteBuffer) { + public void deserialize(final ByteBuffer byteBuffer) { super.deserialize(byteBuffer); unsubscribeReq = @@ -355,7 +364,7 @@ public void deserialize(ByteBuffer byteBuffer) { // This readShort should return ALTER_TOPIC_PROCEDURE, and we ignore it. ReadWriteIOUtils.readShort(byteBuffer); - AlterTopicProcedure topicProcedure = new AlterTopicProcedure(); + final AlterTopicProcedure topicProcedure = new AlterTopicProcedure(); topicProcedure.deserialize(byteBuffer); alterTopicProcedures.add(topicProcedure); } @@ -366,9 +375,9 @@ public void deserialize(ByteBuffer byteBuffer) { size = ReadWriteIOUtils.readInt(byteBuffer); for (int i = 0; i < size; ++i) { // This readShort should return DROP_PIPE_PROCEDURE. - short typeCode = ReadWriteIOUtils.readShort(byteBuffer); + final short typeCode = ReadWriteIOUtils.readShort(byteBuffer); if (typeCode == ProcedureType.DROP_PIPE_PROCEDURE_V2.getTypeCode()) { - DropPipeProcedureV2 dropPipeProcedureV2 = new DropPipeProcedureV2(); + final DropPipeProcedureV2 dropPipeProcedureV2 = new DropPipeProcedureV2(); dropPipeProcedureV2.deserialize(byteBuffer); dropPipeProcedures.add(dropPipeProcedureV2); } @@ -377,14 +386,14 @@ public void deserialize(ByteBuffer byteBuffer) { } @Override - public boolean equals(Object o) { + public boolean equals(final Object o) { if (this == o) { return true; } if (o == null || getClass() != o.getClass()) { return false; } - DropSubscriptionProcedure that = (DropSubscriptionProcedure) o; + final DropSubscriptionProcedure that = (DropSubscriptionProcedure) o; return Objects.equals(getProcId(), that.getProcId()) && Objects.equals(getCurrentState(), that.getCurrentState()) && getCycles() == that.getCycles() @@ -408,7 +417,7 @@ public int hashCode() { @TestOnly public void setAlterConsumerGroupProcedure( - AlterConsumerGroupProcedure alterConsumerGroupProcedure) { + final AlterConsumerGroupProcedure alterConsumerGroupProcedure) { this.alterConsumerGroupProcedure = alterConsumerGroupProcedure; } @@ -418,7 +427,7 @@ public AlterConsumerGroupProcedure getAlterConsumerGroupProcedure() { } @TestOnly - public void setAlterTopicProcedures(List alterTopicProcedures) { + public void setAlterTopicProcedures(final List alterTopicProcedures) { this.alterTopicProcedures = alterTopicProcedures; } @@ -428,7 +437,7 @@ public List getAlterTopicProcedures() { } @TestOnly - public void setDropPipeProcedures(List dropPipeProcedures) { + public void setDropPipeProcedures(final List dropPipeProcedures) { this.dropPipeProcedures = dropPipeProcedures; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionBrokerAgent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionBrokerAgent.java index 82fbc4d28d89e..d1f6ad63c6fcf 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionBrokerAgent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionBrokerAgent.java @@ -141,32 +141,21 @@ public void bindPrefetchingQueue(final SubscriptionConnectorSubtask subtask) { broker.bindPrefetchingQueue(subtask.getTopicName(), subtask.getInputPendingQueue()); } - public void unbindPrefetchingQueue(final SubscriptionConnectorSubtask subtask) { - final String consumerGroupId = subtask.getConsumerGroupId(); + public void unbindPrefetchingQueue(final String consumerGroupId, final String topicName, final boolean doRemove) { final SubscriptionBroker broker = consumerGroupIdToSubscriptionBroker.get(consumerGroupId); if (Objects.isNull(broker)) { LOGGER.warn("Subscription: consumer group [{}] does not exist", consumerGroupId); return; } - broker.unbindPrefetchingQueue(subtask.getTopicName()); + broker.unbindPrefetchingQueue(topicName, doRemove); } - public void removePrefetchingQueue(final String consumerGroupId, final String topicName) { - final SubscriptionBroker broker = consumerGroupIdToSubscriptionBroker.get(consumerGroupId); - if (Objects.isNull(broker)) { - LOGGER.warn("Subscription: consumer group [{}] does not exist", consumerGroupId); - return; - } - broker.removePrefetchingQueue(topicName); - } - - public void executePrefetch(final SubscriptionConnectorSubtask subtask) { - final String consumerGroupId = subtask.getConsumerGroupId(); + public void executePrefetch(final String consumerGroupId, final String topicName) { final SubscriptionBroker broker = consumerGroupIdToSubscriptionBroker.get(consumerGroupId); if (Objects.isNull(broker)) { LOGGER.warn("Subscription: consumer group [{}] does not exist", consumerGroupId); return; } - broker.executePrefetch(subtask.getTopicName()); + broker.executePrefetch(topicName); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionConsumerAgent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionConsumerAgent.java index abf4f4bdd4da9..61fdb3e2b9123 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionConsumerAgent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionConsumerAgent.java @@ -118,7 +118,7 @@ private void handleSingleConsumerGroupMetaChangesInternal( final Set topicsUnsubByGroup = ConsumerGroupMeta.getTopicsUnsubByGroup(metaInAgent, metaFromCoordinator); for (final String topicName : topicsUnsubByGroup) { - SubscriptionAgent.broker().removePrefetchingQueue(consumerGroupId, topicName); + SubscriptionAgent.broker().unbindPrefetchingQueue(consumerGroupId, topicName, true); } // TODO: Currently we fully replace the entire ConsumerGroupMeta without carefully checking the diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionBroker.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionBroker.java index 2a4b0d7b13004..845688cadd997 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionBroker.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionBroker.java @@ -155,7 +155,7 @@ public void bindPrefetchingQueue( } } - public void unbindPrefetchingQueue(final String topicName) { + public void unbindPrefetchingQueue(final String topicName, final boolean doRemove) { final SubscriptionPrefetchingQueue prefetchingQueue = topicNameToPrefetchingQueue.get(topicName); if (Objects.isNull(prefetchingQueue)) { @@ -167,28 +167,11 @@ public void unbindPrefetchingQueue(final String topicName) { prefetchingQueue.cleanup(); prefetchingQueue.markCompleted(); - // we remove this prefetching queue and deregister metrics when consumers dropping related - // subscription... - } - - public void removePrefetchingQueue(final String topicName) { - final SubscriptionPrefetchingQueue prefetchingQueue = - topicNameToPrefetchingQueue.get(topicName); - if (Objects.isNull(prefetchingQueue)) { - LOGGER.warn("Subscription: prefetching queue bound to topic [{}] does not exist", topicName); - return; - } - - if (!prefetchingQueue.isCompleted()) { - LOGGER.warn( - "Subscription: prefetching queue bound to topic [{}] is not completed when removing", - topicName); - return; + if (doRemove) { + topicNameToPrefetchingQueue.remove(topicName); + SubscriptionPrefetchingQueueMetrics.getInstance() + .deregister(prefetchingQueue.getPrefetchingQueueId()); } - - topicNameToPrefetchingQueue.remove(topicName); - SubscriptionPrefetchingQueueMetrics.getInstance() - .deregister(prefetchingQueue.getPrefetchingQueueId()); } public void executePrefetch(final String topicName) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/task/subtask/SubscriptionConnectorSubtask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/task/subtask/SubscriptionConnectorSubtask.java index bb4838b903e7e..b429b8399420e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/task/subtask/SubscriptionConnectorSubtask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/task/subtask/SubscriptionConnectorSubtask.java @@ -31,14 +31,14 @@ public class SubscriptionConnectorSubtask extends PipeConnectorSubtask { private final String consumerGroupId; public SubscriptionConnectorSubtask( - String taskID, - long creationTime, - String attributeSortedString, - int connectorIndex, - UnboundedBlockingPendingQueue inputPendingQueue, - PipeConnector outputPipeConnector, - String topicName, - String consumerGroupId) { + final String taskID, + final long creationTime, + final String attributeSortedString, + final int connectorIndex, + final UnboundedBlockingPendingQueue inputPendingQueue, + final PipeConnector outputPipeConnector, + final String topicName, + final String consumerGroupId) { super( taskID, creationTime, @@ -56,7 +56,7 @@ protected boolean executeOnce() { return false; } - SubscriptionAgent.broker().executePrefetch(this); + SubscriptionAgent.broker().executePrefetch(consumerGroupId, topicName); // always return true return true; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/task/subtask/SubscriptionConnectorSubtaskLifeCycle.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/task/subtask/SubscriptionConnectorSubtaskLifeCycle.java index 14633dfafefe9..5c75399876a36 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/task/subtask/SubscriptionConnectorSubtaskLifeCycle.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/task/subtask/SubscriptionConnectorSubtaskLifeCycle.java @@ -38,9 +38,9 @@ public class SubscriptionConnectorSubtaskLifeCycle extends PipeConnectorSubtaskL private int registeredTaskCount; public SubscriptionConnectorSubtaskLifeCycle( - PipeConnectorSubtaskExecutor executor, // SubscriptionSubtaskExecutor - PipeConnectorSubtask subtask, // SubscriptionConnectorSubtask - UnboundedBlockingPendingQueue pendingQueue) { + final PipeConnectorSubtaskExecutor executor, // SubscriptionSubtaskExecutor + final PipeConnectorSubtask subtask, // SubscriptionConnectorSubtask + final UnboundedBlockingPendingQueue pendingQueue) { super(executor, subtask, pendingQueue); runningTaskCount = 0; @@ -54,6 +54,7 @@ public synchronized void register() { } if (registeredTaskCount == 0) { + // bind prefetching queue SubscriptionAgent.broker().bindPrefetchingQueue((SubscriptionConnectorSubtask) subtask); executor.register(subtask); runningTaskCount = 0; @@ -73,6 +74,8 @@ public synchronized boolean deregister(String ignored) { throw new IllegalStateException("registeredTaskCount <= 0"); } + // no need to discard events of pipe + try { if (registeredTaskCount > 1) { return false; @@ -91,45 +94,13 @@ public synchronized boolean deregister(String ignored) { } } - @Override - public synchronized void start() { - if (runningTaskCount < 0) { - throw new IllegalStateException("runningTaskCount < 0"); - } - - if (runningTaskCount == 0) { - executor.start(subtask.getTaskID()); - } - - runningTaskCount++; - LOGGER.info( - "Start subtask {}. runningTaskCount: {}, registeredTaskCount: {}", - subtask, - runningTaskCount, - registeredTaskCount); - } - - @Override - public synchronized void stop() { - if (runningTaskCount <= 0) { - throw new IllegalStateException("runningTaskCount <= 0"); - } - - if (runningTaskCount == 1) { - executor.stop(subtask.getTaskID()); - } - - runningTaskCount--; - LOGGER.info( - "Stop subtask {}. runningTaskCount: {}, registeredTaskCount: {}", - subtask, - runningTaskCount, - registeredTaskCount); - } - @Override public synchronized void close() { - executor.deregister(subtask.getTaskID()); - SubscriptionAgent.broker().unbindPrefetchingQueue((SubscriptionConnectorSubtask) subtask); + super.close(); + + // unbind prefetching queue + final String consumerGroupId = ((SubscriptionConnectorSubtask) subtask).getConsumerGroupId(); + final String topicName = ((SubscriptionConnectorSubtask) subtask).getTopicName(); + SubscriptionAgent.broker().unbindPrefetchingQueue(consumerGroupId, topicName, false); } } From 898eae83fa5a69cafc2d3c04a8071514d4d6dcce Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Thu, 6 Jun 2024 12:02:10 +0800 Subject: [PATCH 08/36] spotless & fix review --- .../subscription/it/dual/IoTDBSubscriptionTopicIT.java | 2 +- .../subscription/DropSubscriptionProcedure.java | 6 ++++-- .../db/subscription/agent/SubscriptionBrokerAgent.java | 3 ++- .../task/subtask/SubscriptionConnectorSubtaskLifeCycle.java | 2 +- 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionTopicIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionTopicIT.java index 2dee490ac2a95..9bc485b789dc4 100644 --- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionTopicIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionTopicIT.java @@ -631,7 +631,7 @@ public void testTopicWithQueryMode() throws Exception { } consumer.commitSync(messages); } - consumer.unsubscribe(topicName); + // Already unsubscribe the topic implicitly } catch (final Exception e) { e.printStackTrace(); // Avoid failure diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/subscription/subscription/DropSubscriptionProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/subscription/subscription/DropSubscriptionProcedure.java index 674500761b0ff..e29e1896302d2 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/subscription/subscription/DropSubscriptionProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/subscription/subscription/DropSubscriptionProcedure.java @@ -87,7 +87,8 @@ protected SubscriptionOperation getOperation() { } @Override - protected void executeFromValidate(final ConfigNodeProcedureEnv env) throws SubscriptionException { + protected void executeFromValidate(final ConfigNodeProcedureEnv env) + throws SubscriptionException { LOGGER.info("DropSubscriptionProcedure: executeFromValidate"); subscriptionInfo.get().validateBeforeUnsubscribe(unsubscribeReq); @@ -264,7 +265,8 @@ protected void rollbackFromOperateOnConfigNodes(final ConfigNodeProcedureEnv env } @Override - protected void rollbackFromOperateOnDataNodes(final ConfigNodeProcedureEnv env) throws IOException { + protected void rollbackFromOperateOnDataNodes(final ConfigNodeProcedureEnv env) + throws IOException { LOGGER.info("DropSubscriptionProcedure: rollbackFromOperateOnDataNodes"); // Push all pipe metas to datanode, may be time-consuming diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionBrokerAgent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionBrokerAgent.java index d1f6ad63c6fcf..21064ac34c89a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionBrokerAgent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionBrokerAgent.java @@ -141,7 +141,8 @@ public void bindPrefetchingQueue(final SubscriptionConnectorSubtask subtask) { broker.bindPrefetchingQueue(subtask.getTopicName(), subtask.getInputPendingQueue()); } - public void unbindPrefetchingQueue(final String consumerGroupId, final String topicName, final boolean doRemove) { + public void unbindPrefetchingQueue( + final String consumerGroupId, final String topicName, final boolean doRemove) { final SubscriptionBroker broker = consumerGroupIdToSubscriptionBroker.get(consumerGroupId); if (Objects.isNull(broker)) { LOGGER.warn("Subscription: consumer group [{}] does not exist", consumerGroupId); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/task/subtask/SubscriptionConnectorSubtaskLifeCycle.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/task/subtask/SubscriptionConnectorSubtaskLifeCycle.java index 5c75399876a36..a7eebd3aaea0a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/task/subtask/SubscriptionConnectorSubtaskLifeCycle.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/task/subtask/SubscriptionConnectorSubtaskLifeCycle.java @@ -69,7 +69,7 @@ public synchronized void register() { } @Override - public synchronized boolean deregister(String ignored) { + public synchronized boolean deregister(final String ignored) { if (registeredTaskCount <= 0) { throw new IllegalStateException("registeredTaskCount <= 0"); } From 3d8087f856264e5d256ebe9843ebf097717e514b Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Thu, 6 Jun 2024 12:28:36 +0800 Subject: [PATCH 09/36] fix review --- .../it/dual/IoTDBSubscriptionTopicIT.java | 3 +- .../meta/consumer/ConsumerGroupMeta.java | 55 ++++++++++--------- 2 files changed, 32 insertions(+), 26 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionTopicIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionTopicIT.java index 9bc485b789dc4..382b93ac3223c 100644 --- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionTopicIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionTopicIT.java @@ -631,7 +631,8 @@ public void testTopicWithQueryMode() throws Exception { } consumer.commitSync(messages); } - // Already unsubscribe the topic implicitly + // Exiting the loop represents passing the awaitility test, at this point the result + // of 'show subscription' is empty, so there is no need to explicitly unsubscribe. } catch (final Exception e) { e.printStackTrace(); // Avoid failure diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/subscription/meta/consumer/ConsumerGroupMeta.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/subscription/meta/consumer/ConsumerGroupMeta.java index 4c60a87c4847c..bd2d300594956 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/subscription/meta/consumer/ConsumerGroupMeta.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/subscription/meta/consumer/ConsumerGroupMeta.java @@ -48,7 +48,7 @@ public ConsumerGroupMeta() { } public ConsumerGroupMeta( - String consumerGroupId, long creationTime, ConsumerMeta firstConsumerMeta) { + final String consumerGroupId, final long creationTime, final ConsumerMeta firstConsumerMeta) { this.consumerGroupId = consumerGroupId; this.creationTime = creationTime; this.topicNameToSubscribedConsumerIdSet = new HashMap<>(); @@ -101,18 +101,22 @@ public long getCreationTime() { /////////////////////////////// consumer /////////////////////////////// - public void addConsumer(ConsumerMeta consumerMeta) { + public void addConsumer(final ConsumerMeta consumerMeta) { consumerIdToConsumerMeta.put(consumerMeta.getConsumerId(), consumerMeta); } - public void removeConsumer(String consumerId) { + public void removeConsumer(final String consumerId) { consumerIdToConsumerMeta.remove(consumerId); - for (Set subscribedConsumers : topicNameToSubscribedConsumerIdSet.values()) { - subscribedConsumers.remove(consumerId); + for (final Map.Entry> entry : + topicNameToSubscribedConsumerIdSet.entrySet()) { + entry.getValue().remove(consumerId); + if (entry.getValue().isEmpty()) { + topicNameToSubscribedConsumerIdSet.remove(entry.getKey()); + } } } - public boolean containsConsumer(String consumerId) { + public boolean containsConsumer(final String consumerId) { return consumerIdToConsumerMeta.containsKey(consumerId); } @@ -131,13 +135,13 @@ public boolean isEmpty() { * @return The set of consumer IDs subscribing the given topic in this group. If no consumer is * subscribing the topic, return an empty set. */ - public Set getConsumersSubscribingTopic(String topic) { + public Set getConsumersSubscribingTopic(final String topic) { return topicNameToSubscribedConsumerIdSet.getOrDefault(topic, Collections.emptySet()); } - public Set getTopicsSubscribedByConsumer(String consumerId) { - Set topics = new HashSet<>(); - for (Map.Entry> topicNameToSubscribedConsumerId : + public Set getTopicsSubscribedByConsumer(final String consumerId) { + final Set topics = new HashSet<>(); + for (final Map.Entry> topicNameToSubscribedConsumerId : topicNameToSubscribedConsumerIdSet.entrySet()) { if (topicNameToSubscribedConsumerId.getValue().contains(consumerId)) { topics.add(topicNameToSubscribedConsumerId.getKey()); @@ -146,7 +150,7 @@ public Set getTopicsSubscribedByConsumer(String consumerId) { return topics; } - public void addSubscription(String consumerId, Set topics) { + public void addSubscription(final String consumerId, final Set topics) { if (!consumerIdToConsumerMeta.containsKey(consumerId)) { throw new SubscriptionException( String.format( @@ -154,7 +158,7 @@ public void addSubscription(String consumerId, Set topics) { consumerId, consumerGroupId)); } - for (String topic : topics) { + for (final String topic : topics) { topicNameToSubscribedConsumerIdSet .computeIfAbsent(topic, k -> new HashSet<>()) .add(consumerId); @@ -164,7 +168,7 @@ public void addSubscription(String consumerId, Set topics) { /** * @return topics subscribed by no consumers in this group after this removal. */ - public Set removeSubscription(String consumerId, Set topics) { + public Set removeSubscription(final String consumerId, final Set topics) { if (!consumerIdToConsumerMeta.containsKey(consumerId)) { throw new SubscriptionException( String.format( @@ -172,8 +176,8 @@ public Set removeSubscription(String consumerId, Set topics) { consumerId, consumerGroupId)); } - Set noSubscriptionTopicAfterRemoval = new HashSet<>(); - for (String topic : topics) { + final Set noSubscriptionTopicAfterRemoval = new HashSet<>(); + for (final String topic : topics) { if (topicNameToSubscribedConsumerIdSet.containsKey(topic)) { topicNameToSubscribedConsumerIdSet.get(topic).remove(consumerId); if (topicNameToSubscribedConsumerIdSet.get(topic).isEmpty()) { @@ -188,33 +192,34 @@ public Set removeSubscription(String consumerId, Set topics) { /////////////////////////////// de/ser /////////////////////////////// public ByteBuffer serialize() throws IOException { - PublicBAOS byteArrayOutputStream = new PublicBAOS(); - DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream); + final PublicBAOS byteArrayOutputStream = new PublicBAOS(); + final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream); serialize(outputStream); return ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size()); } - public void serialize(OutputStream outputStream) throws IOException { + public void serialize(final OutputStream outputStream) throws IOException { ReadWriteIOUtils.write(consumerGroupId, outputStream); ReadWriteIOUtils.write(creationTime, outputStream); ReadWriteIOUtils.write(topicNameToSubscribedConsumerIdSet.size(), outputStream); - for (Map.Entry> entry : topicNameToSubscribedConsumerIdSet.entrySet()) { + for (final Map.Entry> entry : + topicNameToSubscribedConsumerIdSet.entrySet()) { ReadWriteIOUtils.write(entry.getKey(), outputStream); ReadWriteIOUtils.write(entry.getValue().size(), outputStream); - for (String id : entry.getValue()) { + for (final String id : entry.getValue()) { ReadWriteIOUtils.write(id, outputStream); } } ReadWriteIOUtils.write(consumerIdToConsumerMeta.size(), outputStream); - for (Map.Entry entry : consumerIdToConsumerMeta.entrySet()) { + for (final Map.Entry entry : consumerIdToConsumerMeta.entrySet()) { ReadWriteIOUtils.write(entry.getKey(), outputStream); entry.getValue().serialize(outputStream); } } - public static ConsumerGroupMeta deserialize(InputStream inputStream) throws IOException { + public static ConsumerGroupMeta deserialize(final InputStream inputStream) throws IOException { final ConsumerGroupMeta consumerGroupMeta = new ConsumerGroupMeta(); consumerGroupMeta.consumerGroupId = ReadWriteIOUtils.readString(inputStream); @@ -245,7 +250,7 @@ public static ConsumerGroupMeta deserialize(InputStream inputStream) throws IOEx return consumerGroupMeta; } - public static ConsumerGroupMeta deserialize(ByteBuffer byteBuffer) { + public static ConsumerGroupMeta deserialize(final ByteBuffer byteBuffer) { final ConsumerGroupMeta consumerGroupMeta = new ConsumerGroupMeta(); consumerGroupMeta.consumerGroupId = ReadWriteIOUtils.readString(byteBuffer); @@ -279,14 +284,14 @@ public static ConsumerGroupMeta deserialize(ByteBuffer byteBuffer) { /////////////////////////////// Object /////////////////////////////// @Override - public boolean equals(Object obj) { + public boolean equals(final Object obj) { if (this == obj) { return true; } if (obj == null || getClass() != obj.getClass()) { return false; } - ConsumerGroupMeta that = (ConsumerGroupMeta) obj; + final ConsumerGroupMeta that = (ConsumerGroupMeta) obj; return Objects.equals(consumerGroupId, that.consumerGroupId) && creationTime == that.creationTime && Objects.equals( From 250c4c3769e194c59bd3bc44b408151051bd1da1 Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Thu, 6 Jun 2024 13:08:13 +0800 Subject: [PATCH 10/36] add a boolean return value to dropPipe to avoid warn log in PipeDataNodeRemainingEventAndTimeMetrics#deregister --- .../agent/task/PipeDataNodeTaskAgent.java | 16 ++++++++++---- .../pipe/agent/task/PipeTaskAgent.java | 22 ++++++++++++++----- 2 files changed, 28 insertions(+), 10 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java index 37c04e26f5085..91f28e21aac8a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java @@ -238,16 +238,22 @@ private void closeSchemaRegionListeningQueueIfNecessary( } @Override - protected void dropPipe(final String pipeName, final long creationTime) { - super.dropPipe(pipeName, creationTime); + protected boolean dropPipe(final String pipeName, final long creationTime) { + if (!super.dropPipe(pipeName, creationTime)) { + return false; + } PipeDataNodeRemainingEventAndTimeMetrics.getInstance() .deregister(pipeName + "_" + creationTime); + + return true; } @Override - protected void dropPipe(final String pipeName) { - super.dropPipe(pipeName); + protected boolean dropPipe(final String pipeName) { + if (!super.dropPipe(pipeName)) { + return false; + } final PipeMeta pipeMeta = pipeMetaKeeper.getPipeMeta(pipeName); if (Objects.nonNull(pipeMeta)) { @@ -255,6 +261,8 @@ protected void dropPipe(final String pipeName) { PipeDataNodeRemainingEventAndTimeMetrics.getInstance() .deregister(pipeName + "_" + creationTime); } + + return true; } public void stopAllPipesWithCriticalException() { diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgent.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgent.java index a1844bef5722e..a5630c820f66d 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgent.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgent.java @@ -462,11 +462,14 @@ private boolean createPipe(final PipeMeta pipeMetaFromCoordinator) throws Illega protected abstract Map buildPipeTasks(final PipeMeta pipeMetaFromCoordinator) throws IllegalPathException; - protected void dropPipe(final String pipeName, final long creationTime) { + /** + * @return {@code true} if a pipe has indeed been dropped, otherwise {@code false}. + */ + protected boolean dropPipe(final String pipeName, final long creationTime) { final PipeMeta existedPipeMeta = pipeMetaKeeper.getPipeMeta(pipeName); if (!checkBeforeDropPipe(existedPipeMeta, pipeName, creationTime)) { - return; + return false; } // Mark pipe meta as dropped first. This will help us detect if the pipe meta has been dropped @@ -483,7 +486,7 @@ protected void dropPipe(final String pipeName, final long creationTime) { + "Skip dropping.", pipeName, creationTime); - return; + return false; } // Trigger drop() method for each pipe task by parallel stream @@ -496,13 +499,18 @@ protected void dropPipe(final String pipeName, final long creationTime) { // Remove pipe meta from pipe meta keeper pipeMetaKeeper.removePipeMeta(pipeName); + + return true; } - protected void dropPipe(final String pipeName) { + /** + * @return {@code true} if a pipe has indeed been dropped, otherwise {@code false}. + */ + protected boolean dropPipe(final String pipeName) { final PipeMeta existedPipeMeta = pipeMetaKeeper.getPipeMeta(pipeName); if (!checkBeforeDropPipe(existedPipeMeta, pipeName)) { - return; + return false; } // Mark pipe meta as dropped first. This will help us detect if the pipe meta has been dropped @@ -516,7 +524,7 @@ protected void dropPipe(final String pipeName) { if (pipeTasks == null) { LOGGER.info( "Pipe {} has already been dropped or has not been created. Skip dropping.", pipeName); - return; + return false; } // Trigger drop() method for each pipe task by parallel stream @@ -529,6 +537,8 @@ protected void dropPipe(final String pipeName) { // Remove pipe meta from pipe meta keeper pipeMetaKeeper.removePipeMeta(pipeName); + + return true; } private void startPipe(final String pipeName, final long creationTime) { From 5bff7dd505518110c7d5c31b6fe03449d29a64bb Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Thu, 6 Jun 2024 13:18:49 +0800 Subject: [PATCH 11/36] improve --- .../db/subscription/agent/SubscriptionConsumerAgent.java | 2 +- .../subtask/SubscriptionConnectorSubtaskLifeCycle.java | 4 +++- .../subscription/meta/consumer/ConsumerGroupMeta.java | 7 +++---- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionConsumerAgent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionConsumerAgent.java index 61fdb3e2b9123..983abb0364269 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionConsumerAgent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionConsumerAgent.java @@ -114,7 +114,7 @@ private void handleSingleConsumerGroupMetaChangesInternal( return; } - // remove unbounded prefetching queue + // unbind and remove prefetching queue final Set topicsUnsubByGroup = ConsumerGroupMeta.getTopicsUnsubByGroup(metaInAgent, metaFromCoordinator); for (final String topicName : topicsUnsubByGroup) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/task/subtask/SubscriptionConnectorSubtaskLifeCycle.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/task/subtask/SubscriptionConnectorSubtaskLifeCycle.java index a7eebd3aaea0a..41e690cbce918 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/task/subtask/SubscriptionConnectorSubtaskLifeCycle.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/task/subtask/SubscriptionConnectorSubtaskLifeCycle.java @@ -98,7 +98,9 @@ public synchronized boolean deregister(final String ignored) { public synchronized void close() { super.close(); - // unbind prefetching queue + // Here, the prefetching queue is not actually removed, because it's uncertain whether the + // corresponding underlying pipe is automatically terminated. The actual removal is carried out + // when dropping the subscription. final String consumerGroupId = ((SubscriptionConnectorSubtask) subtask).getConsumerGroupId(); final String topicName = ((SubscriptionConnectorSubtask) subtask).getTopicName(); SubscriptionAgent.broker().unbindPrefetchingQueue(consumerGroupId, topicName, false); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/subscription/meta/consumer/ConsumerGroupMeta.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/subscription/meta/consumer/ConsumerGroupMeta.java index bd2d300594956..764fbfca50121 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/subscription/meta/consumer/ConsumerGroupMeta.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/subscription/meta/consumer/ConsumerGroupMeta.java @@ -82,10 +82,9 @@ public long getCreationTime() { if (!Objects.equals(currentMeta.creationTime, updatedMeta.creationTime)) { return Collections.emptySet(); } - if (!Objects.equals( - currentMeta.consumerIdToConsumerMeta, updatedMeta.consumerIdToConsumerMeta)) { - return Collections.emptySet(); - } + + // no need to check consumerIdToConsumerMeta here to avoid potential inconsistent meta + final Set unsubscribedTopicNames = new HashSet<>(); currentMeta .topicNameToSubscribedConsumerIdSet From a802ee2e99cd5688de972241a253b9d3c4196a0d Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Thu, 6 Jun 2024 13:19:34 +0800 Subject: [PATCH 12/36] more final --- .../apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java | 2 +- .../org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgent.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java index 91f28e21aac8a..e50f1edcc6919 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java @@ -620,7 +620,7 @@ public Set getPipeTaskRegionIdSet(final String pipeName, final long cre ///////////////////////// Pipe Consensus ///////////////////////// - public ProgressIndex getPipeTaskProgressIndex(String pipeName, int consensusGroupId) { + public ProgressIndex getPipeTaskProgressIndex(final String pipeName, final int consensusGroupId) { if (!tryReadLockWithTimeOut(10)) { throw new PipeException( String.format( diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgent.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgent.java index a5630c820f66d..65c1a95aaa316 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgent.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgent.java @@ -103,7 +103,7 @@ protected void acquireWriteLock() { pipeMetaKeeper.acquireWriteLock(); } - protected boolean tryWriteLockWithTimeOut(long timeOutInSeconds) { + protected boolean tryWriteLockWithTimeOut(final long timeOutInSeconds) { try { return pipeMetaKeeper.tryWriteLock(timeOutInSeconds); } catch (final InterruptedException e) { From 4fc65594ef36b506c0f230f95b3cd74b5bb99505 Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Thu, 6 Jun 2024 18:16:13 +0800 Subject: [PATCH 13/36] intro getSubscribedTopics API & improve example --- .../iotdb/SubscriptionSessionExample.java | 60 ++++++++++++++++++- .../response/PipeSubscribeSubscribeResp.java | 52 +++++++++++++++- .../PipeSubscribeUnsubscribeResp.java | 52 +++++++++++++++- .../consumer/SubscriptionConsumer.java | 28 +++++++-- .../consumer/SubscriptionProvider.java | 12 +++- .../consumer/SubscriptionPushConsumer.java | 4 +- .../receiver/SubscriptionReceiverV1.java | 21 +++++-- 7 files changed, 205 insertions(+), 24 deletions(-) diff --git a/example/session/src/main/java/org/apache/iotdb/SubscriptionSessionExample.java b/example/session/src/main/java/org/apache/iotdb/SubscriptionSessionExample.java index 62c869e02bf0a..4d5adecec5a0c 100644 --- a/example/session/src/main/java/org/apache/iotdb/SubscriptionSessionExample.java +++ b/example/session/src/main/java/org/apache/iotdb/SubscriptionSessionExample.java @@ -25,7 +25,10 @@ import org.apache.iotdb.rpc.subscription.config.TopicConstant; import org.apache.iotdb.session.Session; import org.apache.iotdb.session.subscription.SubscriptionSession; +import org.apache.iotdb.session.subscription.consumer.AckStrategy; +import org.apache.iotdb.session.subscription.consumer.ConsumeResult; import org.apache.iotdb.session.subscription.consumer.SubscriptionPullConsumer; +import org.apache.iotdb.session.subscription.consumer.SubscriptionPushConsumer; import org.apache.iotdb.session.subscription.payload.SubscriptionMessage; import org.apache.iotdb.session.subscription.payload.SubscriptionSessionDataSet; @@ -47,6 +50,7 @@ public class SubscriptionSessionExample { private static final String TOPIC_1 = "topic1"; private static final String TOPIC_2 = "`topic2`"; + private static final String TOPIC_3 = "`\"topic3\"`"; public static final long SLEEP_NS = 1_000_000_000L; public static final long POLL_TIMEOUT_MS = 10_000L; @@ -104,6 +108,7 @@ private static void dataQuery() throws Exception { session = null; } + /** single SubscriptionPullConsumer */ private static void subscriptionExample1() throws Exception { // Create topics try (final SubscriptionSession subscriptionSession = new SubscriptionSession(HOST, PORT)) { @@ -151,6 +156,7 @@ private static void subscriptionExample1() throws Exception { consumer1.close(); } + /** multi SubscriptionPullConsumer */ private static void subscriptionExample2() throws Exception { try (final SubscriptionSession subscriptionSession = new SubscriptionSession(HOST, PORT)) { subscriptionSession.open(); @@ -206,10 +212,58 @@ private static void subscriptionExample2() throws Exception { } } + /** multi SubscriptionPushConsumer */ + private static void subscriptionExample3() throws Exception { + try (final SubscriptionSession subscriptionSession = new SubscriptionSession(HOST, PORT)) { + subscriptionSession.open(); + final Properties config = new Properties(); + config.put(TopicConstant.FORMAT_KEY, TopicConstant.FORMAT_TS_FILE_HANDLER_VALUE); + config.put(TopicConstant.MODE_KEY, TopicConstant.MODE_QUERY_VALUE); + subscriptionSession.createTopic(TOPIC_1, config); + } + + final List threads = new ArrayList<>(); + for (int i = 0; i < 8; ++i) { + final int idx = i; + final Thread thread = + new Thread( + () -> { + // Subscription: builder-style ctor + try (final SubscriptionPushConsumer consumer = + new SubscriptionPushConsumer.Builder() + .consumerId("c" + idx) + .consumerGroupId("cg2") + .fileSaveDir(System.getProperty("java.io.tmpdir")) + .ackStrategy(AckStrategy.AFTER_CONSUME) + .consumeListener( + message -> { + // do something for SubscriptionTsFileHandler + System.out.println( + message.getTsFileHandler().getFile().getAbsolutePath()); + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer()) { + consumer.open(); + consumer.subscribe(TOPIC_1); + while (!consumer.getSubscribedTopicNames().isEmpty()) { + LockSupport.parkNanos(SLEEP_NS); // wait some time + } + } + }); + thread.start(); + threads.add(thread); + } + + for (final Thread thread : threads) { + thread.join(); + } + } + public static void main(final String[] args) throws Exception { prepareData(); - dataQuery(); - subscriptionExample1(); - subscriptionExample2(); + // dataQuery(); + // subscriptionExample1(); + // subscriptionExample2(); + subscriptionExample3(); } } diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/response/PipeSubscribeSubscribeResp.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/response/PipeSubscribeSubscribeResp.java index a5540150898d2..1d1c1b307d189 100644 --- a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/response/PipeSubscribeSubscribeResp.java +++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/response/PipeSubscribeSubscribeResp.java @@ -22,10 +22,25 @@ import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.service.rpc.thrift.TPipeSubscribeResp; +import org.apache.tsfile.utils.PublicBAOS; +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Collections; +import java.util.HashSet; import java.util.Objects; +import java.util.Set; public class PipeSubscribeSubscribeResp extends TPipeSubscribeResp { + private transient Set topicNames = new HashSet<>(); // subscribed topic names + + public Set getTopicNames() { + return topicNames; + } + /////////////////////////////// Thrift /////////////////////////////// /** @@ -42,11 +57,43 @@ public static PipeSubscribeSubscribeResp toTPipeSubscribeResp(final TSStatus sta return resp; } + /** + * Serialize the incoming parameters into `PipeSubscribeSubscribeResp`, called by the subscription + * server. + */ + public static PipeSubscribeSubscribeResp toTPipeSubscribeResp( + final TSStatus status, final Set topicNames) throws IOException { + final PipeSubscribeSubscribeResp resp = new PipeSubscribeSubscribeResp(); + + resp.status = status; + resp.version = PipeSubscribeResponseVersion.VERSION_1.getVersion(); + resp.type = PipeSubscribeResponseType.ACK.getType(); + + try (final PublicBAOS byteArrayOutputStream = new PublicBAOS(); + final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) { + ReadWriteIOUtils.writeObjectSet(topicNames, outputStream); + resp.body = + Collections.singletonList( + ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size())); + } + + return resp; + } + /** Deserialize `TPipeSubscribeResp` to obtain parameters, called by the subscription client. */ public static PipeSubscribeSubscribeResp fromTPipeSubscribeResp( final TPipeSubscribeResp subscribeResp) { final PipeSubscribeSubscribeResp resp = new PipeSubscribeSubscribeResp(); + if (Objects.nonNull(subscribeResp.body)) { + for (final ByteBuffer byteBuffer : subscribeResp.body) { + if (Objects.nonNull(byteBuffer) && byteBuffer.hasRemaining()) { + resp.topicNames = ReadWriteIOUtils.readObjectSet(byteBuffer); + break; + } + } + } + resp.status = subscribeResp.status; resp.version = subscribeResp.version; resp.type = subscribeResp.type; @@ -66,7 +113,8 @@ public boolean equals(final Object obj) { return false; } final PipeSubscribeSubscribeResp that = (PipeSubscribeSubscribeResp) obj; - return Objects.equals(this.status, that.status) + return Objects.equals(this.topicNames, that.topicNames) + && Objects.equals(this.status, that.status) && this.version == that.version && this.type == that.type && Objects.equals(this.body, that.body); @@ -74,6 +122,6 @@ public boolean equals(final Object obj) { @Override public int hashCode() { - return Objects.hash(status, version, type, body); + return Objects.hash(topicNames, status, version, type, body); } } diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/response/PipeSubscribeUnsubscribeResp.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/response/PipeSubscribeUnsubscribeResp.java index a93e38013acc6..f94700ebd0339 100644 --- a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/response/PipeSubscribeUnsubscribeResp.java +++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/response/PipeSubscribeUnsubscribeResp.java @@ -22,10 +22,25 @@ import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.service.rpc.thrift.TPipeSubscribeResp; +import org.apache.tsfile.utils.PublicBAOS; +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Collections; +import java.util.HashSet; import java.util.Objects; +import java.util.Set; public class PipeSubscribeUnsubscribeResp extends TPipeSubscribeResp { + private transient Set topicNames = new HashSet<>(); // subscribed topic names + + public Set getTopicNames() { + return topicNames; + } + /////////////////////////////// Thrift /////////////////////////////// /** @@ -42,11 +57,43 @@ public static PipeSubscribeUnsubscribeResp toTPipeSubscribeResp(final TSStatus s return resp; } + /** + * Serialize the incoming parameters into `PipeSubscribeUnsubscribeResp`, called by the + * subscription server. + */ + public static PipeSubscribeUnsubscribeResp toTPipeSubscribeResp( + final TSStatus status, final Set topicNames) throws IOException { + final PipeSubscribeUnsubscribeResp resp = new PipeSubscribeUnsubscribeResp(); + + resp.status = status; + resp.version = PipeSubscribeResponseVersion.VERSION_1.getVersion(); + resp.type = PipeSubscribeResponseType.ACK.getType(); + + try (final PublicBAOS byteArrayOutputStream = new PublicBAOS(); + final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) { + ReadWriteIOUtils.writeObjectSet(topicNames, outputStream); + resp.body = + Collections.singletonList( + ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size())); + } + + return resp; + } + /** Deserialize `TPipeSubscribeResp` to obtain parameters, called by the subscription client. */ public static PipeSubscribeUnsubscribeResp fromTPipeSubscribeResp( final TPipeSubscribeResp unsubscribeResp) { final PipeSubscribeUnsubscribeResp resp = new PipeSubscribeUnsubscribeResp(); + if (Objects.nonNull(unsubscribeResp.body)) { + for (final ByteBuffer byteBuffer : unsubscribeResp.body) { + if (Objects.nonNull(byteBuffer) && byteBuffer.hasRemaining()) { + resp.topicNames = ReadWriteIOUtils.readObjectSet(byteBuffer); + break; + } + } + } + resp.status = unsubscribeResp.status; resp.version = unsubscribeResp.version; resp.type = unsubscribeResp.type; @@ -66,7 +113,8 @@ public boolean equals(final Object obj) { return false; } final PipeSubscribeUnsubscribeResp that = (PipeSubscribeUnsubscribeResp) obj; - return Objects.equals(this.status, that.status) + return Objects.equals(this.topicNames, that.topicNames) + && Objects.equals(this.status, that.status) && this.version == that.version && this.type == that.type && Objects.equals(this.body, that.body); @@ -74,6 +122,6 @@ public boolean equals(final Object obj) { @Override public int hashCode() { - return Objects.hash(status, version, type, body); + return Objects.hash(topicNames, status, version, type, body); } } diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionConsumer.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionConsumer.java index e4ac1f6811b50..f7db8f8650143 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionConsumer.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionConsumer.java @@ -92,6 +92,10 @@ abstract class SubscriptionConsumer implements AutoCloseable { private final String fileSaveDir; private final boolean fileSaveFsync; + protected volatile Set subscribedTopicNames = new HashSet<>(); + + /////////////////////////////// getter /////////////////////////////// + public String getConsumerId() { return consumerId; } @@ -100,6 +104,10 @@ public String getConsumerGroupId() { return consumerGroupId; } + public Set getSubscribedTopicNames() { + return subscribedTopicNames; + } + /////////////////////////////// ctor /////////////////////////////// protected SubscriptionConsumer(final Builder builder) { @@ -305,7 +313,8 @@ private Path getFilePath(final String topicName, String fileName) throws Subscri /////////////////////////////// poll /////////////////////////////// - protected List poll(final Set topicNames, final long timeoutMs) + protected List poll( + /* @NotNull */ final Set topicNames, final long timeoutMs) throws SubscriptionException { final List messages = new ArrayList<>(); final SubscriptionPollTimer timer = @@ -313,6 +322,14 @@ protected List poll(final Set topicNames, final lon do { try { + // check topic names + topicNames.stream() + .filter(topicName -> !subscribedTopicNames.contains(topicName)) + .forEach( + topicName -> + LOGGER.warn( + "SubscriptionConsumer {} does not subscribe to topic {}", this, topicName)); + // poll tablets or file for (final SubscriptionPollResponse pollResponse : pollInternal(topicNames)) { final short responseType = pollResponse.getResponseType(); @@ -345,11 +362,10 @@ protected List poll(final Set topicNames, final lon final SubscriptionCommitContext commitContext = pollResponse.getCommitContext(); final String topicNameToUnsubscribe = commitContext.getTopicName(); LOGGER.info( - "Termination occurred when SubscriptionConsumer {} polling topics {}, unsubscribe topic {} on DN {} automatically", + "Termination occurred when SubscriptionConsumer {} polling topics {}, unsubscribe topic {} automatically", this, topicNames, - topicNameToUnsubscribe, - commitContext.getDataNodeId()); + topicNameToUnsubscribe); unsubscribe(topicNameToUnsubscribe); // TODO: notify user by ex? break; @@ -780,7 +796,7 @@ private void subscribeWithRedirection(final Set topicNames) throws Subsc } for (final SubscriptionProvider provider : providers) { try { - provider.subscribe(topicNames); + subscribedTopicNames = provider.subscribe(topicNames); return; } catch (final Exception e) { LOGGER.warn( @@ -810,7 +826,7 @@ private void unsubscribeWithRedirection(final Set topicNames) } for (final SubscriptionProvider provider : providers) { try { - provider.unsubscribe(topicNames); + subscribedTopicNames = provider.unsubscribe(topicNames); return; } catch (final Exception e) { LOGGER.warn( diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionProvider.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionProvider.java index 70589472f8287..9bce0d6774faf 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionProvider.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionProvider.java @@ -40,6 +40,8 @@ import org.apache.iotdb.rpc.subscription.payload.request.PipeSubscribeUnsubscribeReq; import org.apache.iotdb.rpc.subscription.payload.response.PipeSubscribeHandshakeResp; import org.apache.iotdb.rpc.subscription.payload.response.PipeSubscribePollResp; +import org.apache.iotdb.rpc.subscription.payload.response.PipeSubscribeSubscribeResp; +import org.apache.iotdb.rpc.subscription.payload.response.PipeSubscribeUnsubscribeResp; import org.apache.iotdb.service.rpc.thrift.TPipeSubscribeResp; import org.apache.iotdb.session.subscription.SubscriptionSession; import org.apache.iotdb.session.subscription.SubscriptionSessionConnection; @@ -215,7 +217,7 @@ void heartbeat() throws SubscriptionException { verifyPipeSubscribeSuccess(resp.status); } - void subscribe(final Set topicNames) throws SubscriptionException { + Set subscribe(final Set topicNames) throws SubscriptionException { final PipeSubscribeSubscribeReq req; try { req = PipeSubscribeSubscribeReq.toTPipeSubscribeReq(topicNames); @@ -241,9 +243,12 @@ void subscribe(final Set topicNames) throws SubscriptionException { throw new SubscriptionConnectionException(e.getMessage(), e); } verifyPipeSubscribeSuccess(resp.status); + final PipeSubscribeSubscribeResp subscribeResp = + PipeSubscribeSubscribeResp.fromTPipeSubscribeResp(resp); + return subscribeResp.getTopicNames(); } - void unsubscribe(final Set topicNames) throws SubscriptionException { + Set unsubscribe(final Set topicNames) throws SubscriptionException { final PipeSubscribeUnsubscribeReq req; try { req = PipeSubscribeUnsubscribeReq.toTPipeSubscribeReq(topicNames); @@ -269,6 +274,9 @@ void unsubscribe(final Set topicNames) throws SubscriptionException { throw new SubscriptionConnectionException(e.getMessage(), e); } verifyPipeSubscribeSuccess(resp.status); + final PipeSubscribeUnsubscribeResp unsubscribeResp = + PipeSubscribeUnsubscribeResp.fromTPipeSubscribeResp(resp); + return unsubscribeResp.getTopicNames(); } List poll(final SubscriptionPollRequest pollMessage) diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionPushConsumer.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionPushConsumer.java index 7ff1dd39b15bd..1e460707ff809 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionPushConsumer.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionPushConsumer.java @@ -27,7 +27,6 @@ import org.slf4j.LoggerFactory; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Objects; import java.util.Properties; @@ -163,8 +162,7 @@ public void run() { } try { - // Poll all subscribed topics by passing an empty set - final List messages = poll(Collections.emptySet(), autoPollTimeoutMs); + final List messages = poll(subscribedTopicNames, autoPollTimeoutMs); if (ackStrategy.equals(AckStrategy.BEFORE_CONSUME)) { ack(messages); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiverV1.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiverV1.java index 8339c3d3c8812..4bf3e0c43453d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiverV1.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiverV1.java @@ -72,6 +72,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; import java.nio.ByteBuffer; import java.util.Collections; import java.util.List; @@ -247,7 +248,7 @@ private TPipeSubscribeResp handlePipeSubscribeHeartbeatInternal( private TPipeSubscribeResp handlePipeSubscribeSubscribe(final PipeSubscribeSubscribeReq req) { try { return handlePipeSubscribeSubscribeInternal(req); - } catch (final SubscriptionException e) { + } catch (final SubscriptionException | IOException e) { final String exceptionMessage = String.format( "Subscription: something unexpected happened when subscribing: %s, req: %s", e, req); @@ -258,7 +259,7 @@ private TPipeSubscribeResp handlePipeSubscribeSubscribe(final PipeSubscribeSubsc } private TPipeSubscribeResp handlePipeSubscribeSubscribeInternal( - final PipeSubscribeSubscribeReq req) { + final PipeSubscribeSubscribeReq req) throws IOException { // check consumer config thread local final ConsumerConfig consumerConfig = consumerConfigThreadLocal.get(); if (Objects.isNull(consumerConfig)) { @@ -273,13 +274,17 @@ private TPipeSubscribeResp handlePipeSubscribeSubscribeInternal( subscribe(consumerConfig, topicNames); LOGGER.info("Subscription: consumer {} subscribe {} successfully", consumerConfig, topicNames); - return PipeSubscribeSubscribeResp.toTPipeSubscribeResp(RpcUtils.SUCCESS_STATUS); + return PipeSubscribeSubscribeResp.toTPipeSubscribeResp( + RpcUtils.SUCCESS_STATUS, + SubscriptionAgent.consumer() + .getTopicsSubscribedByConsumer( + consumerConfig.getConsumerGroupId(), consumerConfig.getConsumerId())); } private TPipeSubscribeResp handlePipeSubscribeUnsubscribe(final PipeSubscribeUnsubscribeReq req) { try { return handlePipeSubscribeUnsubscribeInternal(req); - } catch (final SubscriptionException e) { + } catch (final SubscriptionException | IOException e) { final String exceptionMessage = String.format( "Subscription: something unexpected happened when unsubscribing: %s, req: %s", @@ -291,7 +296,7 @@ private TPipeSubscribeResp handlePipeSubscribeUnsubscribe(final PipeSubscribeUns } private TPipeSubscribeResp handlePipeSubscribeUnsubscribeInternal( - final PipeSubscribeUnsubscribeReq req) { + final PipeSubscribeUnsubscribeReq req) throws IOException { // check consumer config thread local final ConsumerConfig consumerConfig = consumerConfigThreadLocal.get(); if (Objects.isNull(consumerConfig)) { @@ -308,7 +313,11 @@ private TPipeSubscribeResp handlePipeSubscribeUnsubscribeInternal( LOGGER.info( "Subscription: consumer {} unsubscribe {} successfully", consumerConfig, topicNames); - return PipeSubscribeUnsubscribeResp.toTPipeSubscribeResp(RpcUtils.SUCCESS_STATUS); + return PipeSubscribeUnsubscribeResp.toTPipeSubscribeResp( + RpcUtils.SUCCESS_STATUS, + SubscriptionAgent.consumer() + .getTopicsSubscribedByConsumer( + consumerConfig.getConsumerGroupId(), consumerConfig.getConsumerId())); } private TPipeSubscribeResp handlePipeSubscribePoll(final PipeSubscribePollReq req) { From 7e1a3fa60b61647a4834e4a226b557cdb7dfb216 Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Thu, 6 Jun 2024 19:14:01 +0800 Subject: [PATCH 14/36] improve sub identifier parse --- .../iotdb/SubscriptionSessionExample.java | 7 ++- ...bscriptionIdentifierSemanticException.java | 42 +++++++++++++++ .../consumer/SubscriptionConsumer.java | 18 +++++-- .../consumer/SubscriptionPullConsumer.java | 9 +++- .../subscription/util/IdentifierUtils.java | 51 +++++++++++++++++++ .../receiver/SubscriptionReceiverV1.java | 15 ++---- 6 files changed, 122 insertions(+), 20 deletions(-) create mode 100644 iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/exception/SubscriptionIdentifierSemanticException.java create mode 100644 iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/util/IdentifierUtils.java diff --git a/example/session/src/main/java/org/apache/iotdb/SubscriptionSessionExample.java b/example/session/src/main/java/org/apache/iotdb/SubscriptionSessionExample.java index 4d5adecec5a0c..5a6562deb57d6 100644 --- a/example/session/src/main/java/org/apache/iotdb/SubscriptionSessionExample.java +++ b/example/session/src/main/java/org/apache/iotdb/SubscriptionSessionExample.java @@ -49,7 +49,7 @@ public class SubscriptionSessionExample { private static final int PORT = 6667; private static final String TOPIC_1 = "topic1"; - private static final String TOPIC_2 = "`topic2`"; + private static final String TOPIC_2 = "`'topic2'`"; private static final String TOPIC_3 = "`\"topic3\"`"; public static final long SLEEP_NS = 1_000_000_000L; @@ -219,7 +219,7 @@ private static void subscriptionExample3() throws Exception { final Properties config = new Properties(); config.put(TopicConstant.FORMAT_KEY, TopicConstant.FORMAT_TS_FILE_HANDLER_VALUE); config.put(TopicConstant.MODE_KEY, TopicConstant.MODE_QUERY_VALUE); - subscriptionSession.createTopic(TOPIC_1, config); + subscriptionSession.createTopic(TOPIC_3, config); } final List threads = new ArrayList<>(); @@ -233,7 +233,6 @@ private static void subscriptionExample3() throws Exception { new SubscriptionPushConsumer.Builder() .consumerId("c" + idx) .consumerGroupId("cg2") - .fileSaveDir(System.getProperty("java.io.tmpdir")) .ackStrategy(AckStrategy.AFTER_CONSUME) .consumeListener( message -> { @@ -244,7 +243,7 @@ private static void subscriptionExample3() throws Exception { }) .buildPushConsumer()) { consumer.open(); - consumer.subscribe(TOPIC_1); + consumer.subscribe(TOPIC_3); while (!consumer.getSubscribedTopicNames().isEmpty()) { LockSupport.parkNanos(SLEEP_NS); // wait some time } diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/exception/SubscriptionIdentifierSemanticException.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/exception/SubscriptionIdentifierSemanticException.java new file mode 100644 index 0000000000000..6eb9ba6d73484 --- /dev/null +++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/exception/SubscriptionIdentifierSemanticException.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.rpc.subscription.exception; + +import java.util.Objects; + +public class SubscriptionIdentifierSemanticException extends SubscriptionException { + + public SubscriptionIdentifierSemanticException(final String message) { + super(message); + } + + public SubscriptionIdentifierSemanticException(final String message, final Throwable cause) { + super(message, cause); + } + + @Override + public boolean equals(final Object obj) { + return obj instanceof SubscriptionIdentifierSemanticException + && Objects.equals( + getMessage(), ((SubscriptionIdentifierSemanticException) obj).getMessage()) + && Objects.equals( + getTimeStamp(), ((SubscriptionIdentifierSemanticException) obj).getTimeStamp()); + } +} diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionConsumer.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionConsumer.java index f7db8f8650143..234416b35df5e 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionConsumer.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionConsumer.java @@ -40,6 +40,7 @@ import org.apache.iotdb.rpc.subscription.payload.poll.SubscriptionPollResponseType; import org.apache.iotdb.rpc.subscription.payload.poll.TabletsPayload; import org.apache.iotdb.session.subscription.payload.SubscriptionMessage; +import org.apache.iotdb.session.subscription.util.IdentifierUtils; import org.apache.iotdb.session.subscription.util.RandomStringGenerator; import org.apache.iotdb.session.subscription.util.SubscriptionPollTimer; import org.apache.iotdb.session.util.SessionUtils; @@ -69,6 +70,7 @@ import java.util.concurrent.ScheduledFuture; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.LockSupport; +import java.util.stream.Collectors; abstract class SubscriptionConsumer implements AutoCloseable { @@ -229,9 +231,13 @@ public void subscribe(final String... topicNames) throws SubscriptionException { } public void subscribe(final Set topicNames) throws SubscriptionException { + // parse topic names from external source + final Set parsedTopicNames = + topicNames.stream().map(IdentifierUtils::parseIdentifier).collect(Collectors.toSet()); + providers.acquireReadLock(); try { - subscribeWithRedirection(topicNames); + subscribeWithRedirection(parsedTopicNames); } finally { providers.releaseReadLock(); } @@ -246,9 +252,13 @@ public void unsubscribe(final String... topicNames) throws SubscriptionException } public void unsubscribe(final Set topicNames) throws SubscriptionException { + // parse topic names from external source + final Set parsedTopicNames = + topicNames.stream().map(IdentifierUtils::parseIdentifier).collect(Collectors.toSet()); + providers.acquireReadLock(); try { - unsubscribeWithRedirection(topicNames); + unsubscribeWithRedirection(parsedTopicNames); } finally { providers.releaseReadLock(); } @@ -918,12 +928,12 @@ public Builder password(final String password) { } public Builder consumerId(final String consumerId) { - this.consumerId = consumerId; + this.consumerId = IdentifierUtils.parseIdentifier(consumerId); return this; } public Builder consumerGroupId(final String consumerGroupId) { - this.consumerGroupId = consumerGroupId; + this.consumerGroupId = IdentifierUtils.parseIdentifier(consumerGroupId); return this; } diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionPullConsumer.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionPullConsumer.java index 2306e4d05c1e2..38dfc530ee352 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionPullConsumer.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionPullConsumer.java @@ -22,6 +22,7 @@ import org.apache.iotdb.rpc.subscription.config.ConsumerConstant; import org.apache.iotdb.rpc.subscription.exception.SubscriptionException; import org.apache.iotdb.session.subscription.payload.SubscriptionMessage; +import org.apache.iotdb.session.subscription.util.IdentifierUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -39,6 +40,7 @@ import java.util.concurrent.ConcurrentSkipListSet; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Collectors; /** * The {@link SubscriptionPullConsumer} corresponds to the pull consumption mode in the message @@ -150,7 +152,12 @@ public List poll(final Set topicNames, final Durati @Override public List poll(final Set topicNames, final long timeoutMs) throws SubscriptionException { - final List messages = super.poll(topicNames, timeoutMs); + // parse topic names from external source + final Set parsedTopicNames = + topicNames.stream().map(IdentifierUtils::parseIdentifier).collect(Collectors.toSet()); + + // poll messages + final List messages = super.poll(parsedTopicNames, timeoutMs); // add to uncommitted messages if (autoCommit) { diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/util/IdentifierUtils.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/util/IdentifierUtils.java new file mode 100644 index 0000000000000..9f6d09ef44e0f --- /dev/null +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/util/IdentifierUtils.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.session.subscription.util; + +import org.apache.iotdb.rpc.subscription.exception.SubscriptionIdentifierSemanticException; + +import org.apache.tsfile.common.constant.TsFileConstant; +import org.apache.tsfile.read.common.parser.PathVisitor; + +public class IdentifierUtils { + + /** + * refer org.apache.iotdb.db.queryengine.plan.parser.ASTVisitor#parseIdentifier(java.lang.String) + */ + public static String parseIdentifier(final String src) { + if (src.startsWith(TsFileConstant.BACK_QUOTE_STRING) + && src.endsWith(TsFileConstant.BACK_QUOTE_STRING)) { + return src.substring(1, src.length() - 1) + .replace(TsFileConstant.DOUBLE_BACK_QUOTE_STRING, TsFileConstant.BACK_QUOTE_STRING); + } + checkIdentifier(src); + return src; + } + + private static void checkIdentifier(final String src) { + if (!TsFileConstant.IDENTIFIER_PATTERN.matcher(src).matches() + || PathVisitor.isRealNumber(src)) { + throw new SubscriptionIdentifierSemanticException( + String.format( + "%s is illegal, identifier not enclosed with backticks can only consist of digits, characters and underscore.", + src)); + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiverV1.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiverV1.java index 4bf3e0c43453d..9876339101aef 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiverV1.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiverV1.java @@ -31,7 +31,6 @@ import org.apache.iotdb.db.protocol.client.ConfigNodeClient; import org.apache.iotdb.db.protocol.client.ConfigNodeClientManager; import org.apache.iotdb.db.protocol.client.ConfigNodeInfo; -import org.apache.iotdb.db.queryengine.plan.parser.ASTVisitor; import org.apache.iotdb.db.subscription.agent.SubscriptionAgent; import org.apache.iotdb.db.subscription.broker.SubscriptionPrefetchingQueue; import org.apache.iotdb.db.subscription.event.SubscriptionEvent; @@ -269,8 +268,7 @@ private TPipeSubscribeResp handlePipeSubscribeSubscribeInternal( } // subscribe topics - Set topicNames = req.getTopicNames(); - topicNames = topicNames.stream().map(ASTVisitor::parseIdentifier).collect(Collectors.toSet()); + final Set topicNames = req.getTopicNames(); subscribe(consumerConfig, topicNames); LOGGER.info("Subscription: consumer {} subscribe {} successfully", consumerConfig, topicNames); @@ -307,8 +305,7 @@ private TPipeSubscribeResp handlePipeSubscribeUnsubscribeInternal( } // unsubscribe topics - Set topicNames = req.getTopicNames(); - topicNames = topicNames.stream().map(ASTVisitor::parseIdentifier).collect(Collectors.toSet()); + final Set topicNames = req.getTopicNames(); unsubscribe(consumerConfig, topicNames); LOGGER.info( @@ -412,15 +409,11 @@ private List handlePipeSubscribePollInternal( SubscriptionAgent.consumer() .getTopicsSubscribedByConsumer( consumerConfig.getConsumerGroupId(), consumerConfig.getConsumerId()); - final Set topicNames; - if (messagePayload.getTopicNames().isEmpty()) { + Set topicNames = messagePayload.getTopicNames(); + if (topicNames.isEmpty()) { // poll all subscribed topics topicNames = subscribedTopicNames; } else { - topicNames = - messagePayload.getTopicNames().stream() - .map(ASTVisitor::parseIdentifier) - .collect(Collectors.toSet()); // filter unsubscribed topics topicNames.removeIf((topicName) -> !subscribedTopicNames.contains(topicName)); } From e9423c6ddc933a28764ae985d82b858db24d14fd Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Thu, 6 Jun 2024 20:14:08 +0800 Subject: [PATCH 15/36] improve --- .../iotdb/SubscriptionSessionExample.java | 2 +- .../consumer/SubscriptionConsumer.java | 25 +++++++++++++------ .../SubscriptionExecutorServiceManager.java | 2 +- .../consumer/SubscriptionPushConsumer.java | 4 +++ 4 files changed, 23 insertions(+), 10 deletions(-) diff --git a/example/session/src/main/java/org/apache/iotdb/SubscriptionSessionExample.java b/example/session/src/main/java/org/apache/iotdb/SubscriptionSessionExample.java index 5a6562deb57d6..4225b34f12782 100644 --- a/example/session/src/main/java/org/apache/iotdb/SubscriptionSessionExample.java +++ b/example/session/src/main/java/org/apache/iotdb/SubscriptionSessionExample.java @@ -232,7 +232,7 @@ private static void subscriptionExample3() throws Exception { try (final SubscriptionPushConsumer consumer = new SubscriptionPushConsumer.Builder() .consumerId("c" + idx) - .consumerGroupId("cg2") + .consumerGroupId("cg3") .ackStrategy(AckStrategy.AFTER_CONSUME) .consumeListener( message -> { diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionConsumer.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionConsumer.java index 234416b35df5e..f72ab309491a1 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionConsumer.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionConsumer.java @@ -330,16 +330,16 @@ protected List poll( final SubscriptionPollTimer timer = new SubscriptionPollTimer(System.currentTimeMillis(), timeoutMs); + // check topic names + topicNames.stream() + .filter(topicName -> !subscribedTopicNames.contains(topicName)) + .forEach( + topicName -> + LOGGER.warn( + "SubscriptionConsumer {} does not subscribe to topic {}", this, topicName)); + do { try { - // check topic names - topicNames.stream() - .filter(topicName -> !subscribedTopicNames.contains(topicName)) - .forEach( - topicName -> - LOGGER.warn( - "SubscriptionConsumer {} does not subscribe to topic {}", this, topicName)); - // poll tablets or file for (final SubscriptionPollResponse pollResponse : pollInternal(topicNames)) { final short responseType = pollResponse.getResponseType(); @@ -607,6 +607,9 @@ private List pollInternal(final Set topicNames try { final SubscriptionProvider provider = providers.getNextAvailableProvider(); if (Objects.isNull(provider) || !provider.isAvailable()) { + if (isClosed()) { + return Collections.emptyList(); + } throw new SubscriptionConnectionException( String.format( "Cluster has no available subscription providers when %s poll topic %s", @@ -632,6 +635,9 @@ private List pollFileInternal( try { final SubscriptionProvider provider = providers.getProvider(dataNodeId); if (Objects.isNull(provider) || !provider.isAvailable()) { + if (isClosed()) { + return Collections.emptyList(); + } throw new SubscriptionConnectionException( String.format( "something unexpected happened when %s poll file from subscription provider with data node id %s, the subscription provider may be unavailable or not existed", @@ -691,6 +697,9 @@ private void commitInternal( try { final SubscriptionProvider provider = providers.getProvider(dataNodeId); if (Objects.isNull(provider) || !provider.isAvailable()) { + if (isClosed()) { + return; + } throw new SubscriptionConnectionException( String.format( "something unexpected happened when %s commit (nack: %s) messages to subscription provider with data node id %s, the subscription provider may be unavailable or not existed", diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionExecutorServiceManager.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionExecutorServiceManager.java index 5a587ff96fba6..d9586329197b0 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionExecutorServiceManager.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionExecutorServiceManager.java @@ -34,7 +34,7 @@ final class SubscriptionExecutorServiceManager { private static final Logger LOGGER = LoggerFactory.getLogger(SubscriptionExecutorServiceManager.class); - private static final long AWAIT_TERMINATION_TIMEOUT_MS = 10_000L; + private static final long AWAIT_TERMINATION_TIMEOUT_MS = 15_000L; private static final String CONTROL_FLOW_EXECUTOR_NAME = "SubscriptionControlFlowExecutor"; private static final String UPSTREAM_DATA_FLOW_EXECUTOR_NAME = diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionPushConsumer.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionPushConsumer.java index 1e460707ff809..7323473552caf 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionPushConsumer.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionPushConsumer.java @@ -161,6 +161,10 @@ public void run() { return; } + if (subscribedTopicNames.isEmpty()) { + return; + } + try { final List messages = poll(subscribedTopicNames, autoPollTimeoutMs); From b36e7ab63b0ba0f4739416b7ae924041b99de358 Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Fri, 7 Jun 2024 11:04:36 +0800 Subject: [PATCH 16/36] improve --- .../consumer/SubscriptionConsumer.java | 60 +++++++++++++------ 1 file changed, 42 insertions(+), 18 deletions(-) diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionConsumer.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionConsumer.java index f72ab309491a1..83b6b7713d987 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionConsumer.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionConsumer.java @@ -53,6 +53,7 @@ import java.io.RandomAccessFile; import java.nio.file.FileAlreadyExistsException; import java.nio.file.Files; +import java.nio.file.InvalidPathException; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; @@ -232,12 +233,19 @@ public void subscribe(final String... topicNames) throws SubscriptionException { public void subscribe(final Set topicNames) throws SubscriptionException { // parse topic names from external source - final Set parsedTopicNames = - topicNames.stream().map(IdentifierUtils::parseIdentifier).collect(Collectors.toSet()); + subscribe(topicNames, true); + } + + private void subscribe(Set topicNames, final boolean needParse) + throws SubscriptionException { + if (needParse) { + topicNames = + topicNames.stream().map(IdentifierUtils::parseIdentifier).collect(Collectors.toSet()); + } providers.acquireReadLock(); try { - subscribeWithRedirection(parsedTopicNames); + subscribeWithRedirection(topicNames); } finally { providers.releaseReadLock(); } @@ -253,12 +261,19 @@ public void unsubscribe(final String... topicNames) throws SubscriptionException public void unsubscribe(final Set topicNames) throws SubscriptionException { // parse topic names from external source - final Set parsedTopicNames = - topicNames.stream().map(IdentifierUtils::parseIdentifier).collect(Collectors.toSet()); + unsubscribe(topicNames, true); + } + + private void unsubscribe(Set topicNames, final boolean needParse) + throws SubscriptionException { + if (needParse) { + topicNames = + topicNames.stream().map(IdentifierUtils::parseIdentifier).collect(Collectors.toSet()); + } providers.acquireReadLock(); try { - unsubscribeWithRedirection(parsedTopicNames); + unsubscribeWithRedirection(topicNames); } finally { providers.releaseReadLock(); } @@ -302,23 +317,32 @@ private Path getFileDir(final String topicName) throws IOException { return dirPath; } - private Path getFilePath(final String topicName, String fileName) throws SubscriptionException { - Path filePath; + private Path getFilePath( + final String topicName, + final String fileName, + final boolean allowFileAlreadyExistsException, + final boolean allowInvalidPathException) + throws SubscriptionException { try { - filePath = getFileDir(topicName).resolve(fileName); + final Path filePath = getFileDir(topicName).resolve(fileName); Files.createFile(filePath); + return filePath; } catch (final FileAlreadyExistsException fileAlreadyExistsException) { - fileName += "." + RandomStringGenerator.generate(16); - try { - filePath = getFileDir(topicName).resolve(fileName); - Files.createFile(filePath); - } catch (final IOException e) { - throw new SubscriptionRuntimeNonCriticalException(e.getMessage(), e); + if (allowFileAlreadyExistsException) { + return getFilePath( + topicName, fileName + "." + RandomStringGenerator.generate(16), false, true); + } + throw new SubscriptionRuntimeNonCriticalException( + fileAlreadyExistsException.getMessage(), fileAlreadyExistsException); + } catch (final InvalidPathException invalidPathException) { + if (allowInvalidPathException) { + return getFilePath(String.valueOf(topicName.hashCode()), fileName, true, false); } + throw new SubscriptionRuntimeNonCriticalException( + invalidPathException.getMessage(), invalidPathException); } catch (final IOException e) { throw new SubscriptionRuntimeNonCriticalException(e.getMessage(), e); } - return filePath; } /////////////////////////////// poll /////////////////////////////// @@ -376,7 +400,7 @@ protected List poll( this, topicNames, topicNameToUnsubscribe); - unsubscribe(topicNameToUnsubscribe); + unsubscribe(Collections.singleton(topicNameToUnsubscribe), false); // TODO: notify user by ex? break; default: @@ -429,7 +453,7 @@ private Optional pollFile( final SubscriptionCommitContext commitContext, final String fileName) throws SubscriptionException { final String topicName = commitContext.getTopicName(); - final Path filePath = getFilePath(topicName, fileName); + final Path filePath = getFilePath(topicName, fileName, true, true); final File file = filePath.toFile(); try (final RandomAccessFile fileWriter = new RandomAccessFile(file, "rw")) { return Optional.of(pollFileInternal(commitContext, file, fileWriter)); From 14e1ba0eebf998bad7f90bc9b44d54ce1ed25076 Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Fri, 7 Jun 2024 17:33:18 +0800 Subject: [PATCH 17/36] encode topic name with URLEncoder --- .../session/subscription/consumer/SubscriptionConsumer.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionConsumer.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionConsumer.java index 83b6b7713d987..76d85c69f3488 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionConsumer.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionConsumer.java @@ -51,6 +51,7 @@ import java.io.File; import java.io.IOException; import java.io.RandomAccessFile; +import java.net.URLEncoder; import java.nio.file.FileAlreadyExistsException; import java.nio.file.Files; import java.nio.file.InvalidPathException; @@ -336,7 +337,7 @@ private Path getFilePath( fileAlreadyExistsException.getMessage(), fileAlreadyExistsException); } catch (final InvalidPathException invalidPathException) { if (allowInvalidPathException) { - return getFilePath(String.valueOf(topicName.hashCode()), fileName, true, false); + return getFilePath(URLEncoder.encode(topicName), fileName, true, false); } throw new SubscriptionRuntimeNonCriticalException( invalidPathException.getMessage(), invalidPathException); From bbe30d644247e74b123ba94af2b02ec0435f4332 Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Fri, 7 Jun 2024 17:33:26 +0800 Subject: [PATCH 18/36] fix SubscriptionFileHandler --- .../payload/SubscriptionFileHandler.java | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/payload/SubscriptionFileHandler.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/payload/SubscriptionFileHandler.java index 60c269b86370c..0ec121f96992a 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/payload/SubscriptionFileHandler.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/payload/SubscriptionFileHandler.java @@ -39,14 +39,14 @@ public SubscriptionFileHandler(final String absolutePath) { /** * @return a new File instance of the corresponding file */ - public File getFile() { + public synchronized File getFile() { return new File(absolutePath); } /** * @return a new Path instance of the corresponding file */ - public Path getPath() { + public synchronized Path getPath() { return Paths.get(absolutePath); } @@ -54,7 +54,7 @@ public Path getPath() { * @return the path to the source file * @throws IOException if an I/O error occurs */ - public Path deleteFile() throws IOException { + public synchronized Path deleteFile() throws IOException { final Path sourcePath = getPath(); Files.delete(sourcePath); return sourcePath; @@ -65,7 +65,7 @@ public Path deleteFile() throws IOException { * @return the path to the target file * @throws IOException if an I/O error occurs */ - public Path moveFile(final String target) throws IOException { + public synchronized Path moveFile(final String target) throws IOException { return this.moveFile(Paths.get(target)); } @@ -74,7 +74,10 @@ public Path moveFile(final String target) throws IOException { * @return the path to the target file * @throws IOException if an I/O error occurs */ - public Path moveFile(final Path target) throws IOException { + public synchronized Path moveFile(final Path target) throws IOException { + if (!Files.exists(target.getParent())) { + Files.createDirectories(target.getParent()); + } return Files.move(getPath(), target, StandardCopyOption.REPLACE_EXISTING); } @@ -83,7 +86,7 @@ public Path moveFile(final Path target) throws IOException { * @return the path to the target file * @throws IOException if an I/O error occurs */ - public Path copyFile(final String target) throws IOException { + public synchronized Path copyFile(final String target) throws IOException { return this.copyFile(Paths.get(target)); } @@ -92,7 +95,10 @@ public Path copyFile(final String target) throws IOException { * @return the path to the target file * @throws IOException if an I/O error occurs */ - public Path copyFile(final Path target) throws IOException { + public synchronized Path copyFile(final Path target) throws IOException { + if (!Files.exists(target.getParent())) { + Files.createDirectories(target.getParent()); + } return Files.copy( getPath(), target, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.COPY_ATTRIBUTES); } From 6019ca70fe405c6523074cc2e3dfd8222226827d Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Fri, 7 Jun 2024 17:33:37 +0800 Subject: [PATCH 19/36] improve SubscriptionSessionExample --- .../iotdb/SubscriptionSessionExample.java | 117 ++++++++++++++---- 1 file changed, 95 insertions(+), 22 deletions(-) diff --git a/example/session/src/main/java/org/apache/iotdb/SubscriptionSessionExample.java b/example/session/src/main/java/org/apache/iotdb/SubscriptionSessionExample.java index 4225b34f12782..dcb93ab528ebb 100644 --- a/example/session/src/main/java/org/apache/iotdb/SubscriptionSessionExample.java +++ b/example/session/src/main/java/org/apache/iotdb/SubscriptionSessionExample.java @@ -31,11 +31,17 @@ import org.apache.iotdb.session.subscription.consumer.SubscriptionPushConsumer; import org.apache.iotdb.session.subscription.payload.SubscriptionMessage; import org.apache.iotdb.session.subscription.payload.SubscriptionSessionDataSet; +import org.apache.iotdb.session.subscription.payload.SubscriptionTsFileHandler; import org.apache.tsfile.read.TsFileReader; +import org.apache.tsfile.read.common.Path; +import org.apache.tsfile.read.expression.QueryExpression; +import org.apache.tsfile.read.query.dataset.QueryDataSet; import java.io.IOException; +import java.nio.file.Paths; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Properties; @@ -51,10 +57,12 @@ public class SubscriptionSessionExample { private static final String TOPIC_1 = "topic1"; private static final String TOPIC_2 = "`'topic2'`"; private static final String TOPIC_3 = "`\"topic3\"`"; + private static final String TOPIC_4 = "`\"top \\.i.c4\"`"; - public static final long SLEEP_NS = 1_000_000_000L; - public static final long POLL_TIMEOUT_MS = 10_000L; + private static final long SLEEP_NS = 1_000_000_000L; + private static final long POLL_TIMEOUT_MS = 10_000L; private static final int MAX_RETRY_TIMES = 3; + private static final int PARALLELISM = 8; private static void prepareData() throws Exception { // Open session @@ -70,13 +78,13 @@ private static void prepareData() throws Exception { // Insert some historical data final long currentTime = System.currentTimeMillis(); - for (int i = 0; i < 10000; ++i) { + for (int i = 0; i < 100; ++i) { session.executeNonQueryStatement( String.format("insert into root.db.d1(time, s1, s2) values (%s, 1, 2)", i)); session.executeNonQueryStatement( - String.format("insert into root.db.d2(time, s3, s4) values (%s, 3, 4)", currentTime + i)); + String.format("insert into root.db.d2(time, s1, s2) values (%s, 3, 4)", currentTime + i)); session.executeNonQueryStatement( - String.format("insert into root.sg.d3(time, s5) values (%s, 5)", currentTime + 2 * i)); + String.format("insert into root.sg.d3(time, s1) values (%s, 5)", currentTime + 2 * i)); } session.executeNonQueryStatement("flush"); @@ -108,12 +116,16 @@ private static void dataQuery() throws Exception { session = null; } - /** single SubscriptionPullConsumer */ - private static void subscriptionExample1() throws Exception { + /** single pull consumer subscribe topic with path and time range */ + private static void dataSubscription1() throws Exception { // Create topics try (final SubscriptionSession subscriptionSession = new SubscriptionSession(HOST, PORT)) { subscriptionSession.open(); - subscriptionSession.createTopic(TOPIC_1); + final Properties config = new Properties(); + config.put(TopicConstant.PATH_KEY, "root.db.d1.s1"); + config.put(TopicConstant.START_TIME_KEY, 25); + config.put(TopicConstant.END_TIME_KEY, 75); + subscriptionSession.createTopic(TOPIC_1, config); } int retryCount = 0; @@ -156,8 +168,8 @@ private static void subscriptionExample1() throws Exception { consumer1.close(); } - /** multi SubscriptionPullConsumer */ - private static void subscriptionExample2() throws Exception { + /** multi pull consumer subscribe topic with tsfile format */ + private static void dataSubscription2() throws Exception { try (final SubscriptionSession subscriptionSession = new SubscriptionSession(HOST, PORT)) { subscriptionSession.open(); final Properties config = new Properties(); @@ -166,7 +178,7 @@ private static void subscriptionExample2() throws Exception { } final List threads = new ArrayList<>(); - for (int i = 0; i < 8; ++i) { + for (int i = 0; i < PARALLELISM; ++i) { final int idx = i; final Thread thread = new Thread( @@ -193,7 +205,16 @@ private static void subscriptionExample2() throws Exception { } for (final SubscriptionMessage message : messages) { try (final TsFileReader reader = message.getTsFileHandler().openReader()) { - // do something... + final QueryDataSet dataSet = + reader.query( + QueryExpression.create( + Arrays.asList( + new Path("root.db.d2", "s2", true), + new Path("root.db.d3", "s1", true)), + null)); + while (dataSet.hasNext()) { + System.out.println(dataSet.next()); + } } } consumer2.commitSync(messages); @@ -212,8 +233,8 @@ private static void subscriptionExample2() throws Exception { } } - /** multi SubscriptionPushConsumer */ - private static void subscriptionExample3() throws Exception { + /** multi push consumer subscribe topic with tsfile format and query mode */ + private static void dataSubscription3() throws Exception { try (final SubscriptionSession subscriptionSession = new SubscriptionSession(HOST, PORT)) { subscriptionSession.open(); final Properties config = new Properties(); @@ -223,13 +244,13 @@ private static void subscriptionExample3() throws Exception { } final List threads = new ArrayList<>(); - for (int i = 0; i < 8; ++i) { + for (int i = 0; i < PARALLELISM; ++i) { final int idx = i; final Thread thread = new Thread( () -> { // Subscription: builder-style ctor - try (final SubscriptionPushConsumer consumer = + try (final SubscriptionPushConsumer consumer3 = new SubscriptionPushConsumer.Builder() .consumerId("c" + idx) .consumerGroupId("cg3") @@ -242,9 +263,9 @@ private static void subscriptionExample3() throws Exception { return ConsumeResult.SUCCESS; }) .buildPushConsumer()) { - consumer.open(); - consumer.subscribe(TOPIC_3); - while (!consumer.getSubscribedTopicNames().isEmpty()) { + consumer3.open(); + consumer3.subscribe(TOPIC_3); + while (!consumer3.getSubscribedTopicNames().isEmpty()) { LockSupport.parkNanos(SLEEP_NS); // wait some time } } @@ -258,11 +279,63 @@ private static void subscriptionExample3() throws Exception { } } + /** multi pull consumer subscribe topic with tsfile format and query mode */ + private static void dataSubscription4() throws Exception { + try (final SubscriptionSession subscriptionSession = new SubscriptionSession(HOST, PORT)) { + subscriptionSession.open(); + final Properties config = new Properties(); + config.put(TopicConstant.FORMAT_KEY, TopicConstant.FORMAT_TS_FILE_HANDLER_VALUE); + config.put(TopicConstant.MODE_KEY, TopicConstant.MODE_QUERY_VALUE); + subscriptionSession.createTopic(TOPIC_4, config); + } + + final List threads = new ArrayList<>(); + for (int i = 0; i < PARALLELISM; ++i) { + final int idx = i; + final Thread thread = + new Thread( + () -> { + // Subscription: builder-style ctor + try (final SubscriptionPullConsumer consumer4 = + new SubscriptionPullConsumer.Builder() + .consumerId("c" + idx) + .consumerGroupId("cg4") + .autoCommit(true) + .fileSaveFsync(true) + .buildPullConsumer()) { + consumer4.open(); + consumer4.subscribe(TOPIC_4); + while (true) { + LockSupport.parkNanos(SLEEP_NS); // wait some time + if (consumer4.getSubscribedTopicNames().isEmpty()) { + break; + } + for (final SubscriptionMessage message : consumer4.poll(POLL_TIMEOUT_MS)) { + final SubscriptionTsFileHandler handler = message.getTsFileHandler(); + handler.moveFile( + Paths.get(System.getProperty("user.dir"), "new-subscription-dir") + .resolve(handler.getPath().getFileName())); + } + } + } catch (final IOException e) { + throw new RuntimeException(e); + } + }); + thread.start(); + threads.add(thread); + } + + for (final Thread thread : threads) { + thread.join(); + } + } + public static void main(final String[] args) throws Exception { prepareData(); // dataQuery(); - // subscriptionExample1(); - // subscriptionExample2(); - subscriptionExample3(); + // dataSubscription1(); + // dataSubscription2(); + // dataSubscription3(); + dataSubscription4(); } } From 46f40191ec09a454cb8071864761e719b12162b3 Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Tue, 11 Jun 2024 11:57:58 +0800 Subject: [PATCH 20/36] roll back from the last executed procedure to the first executed --- .../CreateSubscriptionProcedure.java | 2 ++ .../DropSubscriptionProcedure.java | 24 +++++++++---------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/subscription/subscription/CreateSubscriptionProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/subscription/subscription/CreateSubscriptionProcedure.java index 8710eec85225e..0894feb62ad9e 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/subscription/subscription/CreateSubscriptionProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/subscription/subscription/CreateSubscriptionProcedure.java @@ -229,6 +229,7 @@ protected void rollbackFromValidate(ConfigNodeProcedureEnv env) { protected void rollbackFromOperateOnConfigNodes(ConfigNodeProcedureEnv env) { LOGGER.info("CreateSubscriptionProcedure: rollbackFromOperateOnConfigNodes"); + // TODO: roll back from the last executed procedure to the first executed alterConsumerGroupProcedure.rollbackFromOperateOnConfigNodes(env); TSStatus response; @@ -283,6 +284,7 @@ protected void rollbackFromOperateOnConfigNodes(ConfigNodeProcedureEnv env) { protected void rollbackFromOperateOnDataNodes(ConfigNodeProcedureEnv env) throws IOException { LOGGER.info("CreateSubscriptionProcedure: rollbackFromOperateOnDataNodes"); + // TODO: roll back from the last executed procedure to the first executed alterConsumerGroupProcedure.rollbackFromOperateOnDataNodes(env); // Push all topic metas to datanode, may be time-consuming diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/subscription/subscription/DropSubscriptionProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/subscription/subscription/DropSubscriptionProcedure.java index e29e1896302d2..89da284ac0d95 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/subscription/subscription/DropSubscriptionProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/subscription/subscription/DropSubscriptionProcedure.java @@ -234,7 +234,8 @@ protected void rollbackFromValidate(final ConfigNodeProcedureEnv env) { protected void rollbackFromOperateOnConfigNodes(final ConfigNodeProcedureEnv env) { LOGGER.info("DropSubscriptionProcedure: rollbackFromOperateOnConfigNodes"); - // Do nothing to rollback DropPipeProcedureV2s + // Rollback AlterConsumerGroupProcedure + alterConsumerGroupProcedure.rollbackFromOperateOnConfigNodes(env); // Rollback AlterTopicProcedures TSStatus response; @@ -260,8 +261,7 @@ protected void rollbackFromOperateOnConfigNodes(final ConfigNodeProcedureEnv env throw new SubscriptionException(response.getMessage()); } - // Rollback AlterConsumerGroupProcedure - alterConsumerGroupProcedure.rollbackFromOperateOnConfigNodes(env); + // Do nothing to rollback DropPipeProcedureV2s } @Override @@ -269,6 +269,15 @@ protected void rollbackFromOperateOnDataNodes(final ConfigNodeProcedureEnv env) throws IOException { LOGGER.info("DropSubscriptionProcedure: rollbackFromOperateOnDataNodes"); + // Rollback AlterConsumerGroupProcedure + alterConsumerGroupProcedure.rollbackFromOperateOnDataNodes(env); + + // Push all topic metas to datanode, may be time-consuming + if (pushTopicMetaHasException(pushTopicMetaToDataNodes(env))) { + LOGGER.warn( + "Failed to rollback alter topics when dropping subscription, metadata will be synchronized later."); + } + // Push all pipe metas to datanode, may be time-consuming final String exceptionMessage = AbstractOperatePipeProcedureV2.parsePushPipeMetaExceptionForPipe( @@ -278,15 +287,6 @@ protected void rollbackFromOperateOnDataNodes(final ConfigNodeProcedureEnv env) "Failed to rollback create pipes when dropping subscription, details: {}, metadata will be synchronized later.", exceptionMessage); } - - // Push all topic metas to datanode, may be time-consuming - if (pushTopicMetaHasException(pushTopicMetaToDataNodes(env))) { - LOGGER.warn( - "Failed to rollback alter topics when dropping subscription, metadata will be synchronized later."); - } - - // Rollback AlterConsumerGroupProcedure - alterConsumerGroupProcedure.rollbackFromOperateOnDataNodes(env); } @Override From e1a93963a2314e369ebb201ccea55d7ec9c5e181 Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Tue, 11 Jun 2024 12:11:40 +0800 Subject: [PATCH 21/36] enlarge awaitility at most second --- .../iotdb/subscription/it/IoTDBSubscriptionITConstant.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/IoTDBSubscriptionITConstant.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/IoTDBSubscriptionITConstant.java index d2dc8ee49a5fc..7d16110cd1f00 100644 --- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/IoTDBSubscriptionITConstant.java +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/IoTDBSubscriptionITConstant.java @@ -23,7 +23,7 @@ public class IoTDBSubscriptionITConstant { public static final long AWAITILITY_POLL_DELAY_SECOND = 1L; public static final long AWAITILITY_POLL_INTERVAL_SECOND = 2L; - public static final long AWAITILITY_AT_MOST_SECOND = 240L; + public static final long AWAITILITY_AT_MOST_SECOND = 600L; public static final long SLEEP_NS = 1_000_000_000L; public static final long POLL_TIMEOUT_MS = 10_000L; From d580a559cbcd8a12dfb25baf338a90726515d87f Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Tue, 11 Jun 2024 23:58:01 +0800 Subject: [PATCH 22/36] improve cleanup and completed judgement logic --- .../agent/SubscriptionTopicAgent.java | 12 +++++++ .../broker/SubscriptionBroker.java | 35 +++++++++++++++++-- .../broker/SubscriptionPrefetchingQueue.java | 28 +++++++++------ .../SubscriptionPrefetchingTabletsQueue.java | 11 +++++- .../SubscriptionPrefetchingTsFileQueue.java | 18 +++++++++- .../subscription/event/SubscriptionEvent.java | 6 ++++ 6 files changed, 95 insertions(+), 15 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionTopicAgent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionTopicAgent.java index 45145bd2b67e0..bc1741704224f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionTopicAgent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionTopicAgent.java @@ -149,4 +149,16 @@ public String getTopicFormat(final String topicName) { releaseReadLock(); } } + + public String getTopicMode(final String topicName) { + acquireReadLock(); + try { + return topicMetaKeeper + .getTopicMeta(topicName) + .getConfig() + .getStringOrDefault(TopicConstant.MODE_KEY, TopicConstant.MODE_DEFAULT_VALUE); + } finally { + releaseReadLock(); + } + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionBroker.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionBroker.java index 845688cadd997..f3bd0371fdd2c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionBroker.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionBroker.java @@ -65,6 +65,18 @@ public List poll(final String consumerId, final Set t final String topicName = entry.getKey(); final SubscriptionPrefetchingQueue prefetchingQueue = entry.getValue(); if (topicNames.contains(topicName)) { + // before determining if it is closed + if (prefetchingQueue.isCompleted()) { + LOGGER.info( + "Subscription: prefetching queue bound to topic [{}] is completed, return termination response to client", + topicName); + events.add(prefetchingQueue.generateSubscriptionPollTerminationResponse()); + continue; + } + if (prefetchingQueue.isClosed()) { + LOGGER.warn("Subscription: prefetching queue bound to topic [{}] is closed", topicName); + continue; + } final SubscriptionEvent event = prefetchingQueue.poll(consumerId); if (Objects.nonNull(event)) { events.add(event); @@ -95,6 +107,10 @@ public List pollTsFile( LOGGER.warn(errorMessage); throw new SubscriptionException(errorMessage); } + if (prefetchingQueue.isClosed()) { + LOGGER.warn("Subscription: prefetching queue bound to topic [{}] is closed", topicName); + return Collections.emptyList(); + } final SubscriptionEvent event = ((SubscriptionPrefetchingTsFileQueue) prefetchingQueue) .pollTsFile(consumerId, fileName, writingOffset); @@ -117,6 +133,10 @@ public List commit( "Subscription: prefetching queue bound to topic [{}] does not exist", topicName); continue; } + if (prefetchingQueue.isClosed()) { + LOGGER.warn("Subscription: prefetching queue bound to topic [{}] is closed", topicName); + continue; + } if (!nack) { if (prefetchingQueue.ack(commitContext)) { successfulCommitContexts.add(commitContext); @@ -163,9 +183,16 @@ public void unbindPrefetchingQueue(final String topicName, final boolean doRemov return; } - // clean up uncommitted events + // mark prefetching queue closed first + prefetchingQueue.markClosed(); + + // clean up events in prefetching queue, this operation is idempotent prefetchingQueue.cleanup(); - prefetchingQueue.markCompleted(); + + // mark prefetching queue completed only for topic of query mode + if (SubscriptionAgent.topic().getTopicMode(topicName).equals(TopicConstant.MODE_QUERY_VALUE)) { + prefetchingQueue.markCompleted(); + } if (doRemove) { topicNameToPrefetchingQueue.remove(topicName); @@ -181,6 +208,10 @@ public void executePrefetch(final String topicName) { LOGGER.warn("Subscription: prefetching queue bound to topic [{}] does not exist", topicName); return; } + if (prefetchingQueue.isClosed()) { + LOGGER.warn("Subscription: prefetching queue bound to topic [{}] is closed", topicName); + return; + } prefetchingQueue.executePrefetch(); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingQueue.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingQueue.java index 6dae4d8706232..c1d100c8266a4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingQueue.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingQueue.java @@ -52,6 +52,7 @@ public abstract class SubscriptionPrefetchingQueue { private final AtomicLong subscriptionCommitIdGenerator = new AtomicLong(0); private volatile boolean isCompleted = false; + private volatile boolean isClosed = false; public SubscriptionPrefetchingQueue( final String brokerId, @@ -64,23 +65,20 @@ public SubscriptionPrefetchingQueue( this.uncommittedEvents = new ConcurrentHashMap<>(); } - public SubscriptionEvent poll(final String consumerId) { - if (isCompleted()) { - return generateSubscriptionPollTerminationResponse(); - } - - return pollInternal(consumerId); - } - - public abstract SubscriptionEvent pollInternal(final String consumerId); + public abstract SubscriptionEvent poll(final String consumerId); public abstract void executePrefetch(); - /** clean up uncommitted events */ public void cleanup() { + // clean up uncommitted events for (final SubscriptionEvent event : uncommittedEvents.values()) { + event.clearReferenceCount(); SubscriptionEventBinaryCache.getInstance().resetByteBuffer(event, true); } + uncommittedEvents.clear(); + + // no need to clean up events in inputPendingQueue, see + // org.apache.iotdb.db.pipe.task.subtask.connector.PipeConnectorSubtask.close } /////////////////////////////// commit /////////////////////////////// @@ -168,6 +166,14 @@ public long getCurrentCommitId() { /////////////////////////////// termination /////////////////////////////// + public boolean isClosed() { + return isClosed; + } + + public void markClosed() { + isClosed = true; + } + public boolean isCompleted() { return isCompleted; } @@ -176,7 +182,7 @@ public void markCompleted() { isCompleted = true; } - private SubscriptionTsFileEvent generateSubscriptionPollTerminationResponse() { + public SubscriptionTsFileEvent generateSubscriptionPollTerminationResponse() { return new SubscriptionTsFileEvent( Collections.emptyList(), new SubscriptionPollResponse( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingTabletsQueue.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingTabletsQueue.java index 7ca1f1cdf505b..935494537b3c3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingTabletsQueue.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingTabletsQueue.java @@ -65,7 +65,16 @@ public SubscriptionPrefetchingTabletsQueue( } @Override - public SubscriptionEvent pollInternal(final String consumerId) { + public void cleanup() { + super.cleanup(); + + // no need to clean up events in prefetchingQueue, since all events in prefetchingQueue are also + // in uncommittedEvents + prefetchingQueue.clear(); + } + + @Override + public SubscriptionEvent poll(final String consumerId) { if (prefetchingQueue.isEmpty()) { prefetchOnce(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingTsFileQueue.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingTsFileQueue.java index 24fd5d4013855..5dcca455428ee 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingTsFileQueue.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingTsFileQueue.java @@ -24,6 +24,7 @@ import org.apache.iotdb.db.pipe.event.UserDefinedEnrichedEvent; import org.apache.iotdb.db.pipe.event.common.terminate.PipeTerminateEvent; import org.apache.iotdb.db.pipe.event.common.tsfile.PipeTsFileInsertionEvent; +import org.apache.iotdb.db.subscription.event.SubscriptionEventBinaryCache; import org.apache.iotdb.db.subscription.event.SubscriptionTsFileEvent; import org.apache.iotdb.pipe.api.event.Event; import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent; @@ -65,7 +66,22 @@ public SubscriptionPrefetchingTsFileQueue( } @Override - public SubscriptionTsFileEvent pollInternal(final String consumerId) { + public void cleanup() { + super.cleanup(); + + // clean up events in consumerIdToCurrentEventMap + consumerIdToCurrentEventMap + .values() + .forEach( + (event) -> { + event.clearReferenceCount(); + SubscriptionEventBinaryCache.getInstance().resetByteBuffer(event, true); + }); + consumerIdToCurrentEventMap.clear(); + } + + @Override + public SubscriptionTsFileEvent poll(final String consumerId) { if (hasUnPollableOnTheFlySubscriptionTsFileEvent(consumerId)) { return null; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/SubscriptionEvent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/SubscriptionEvent.java index c327b7e058d73..0f7feb08e95e4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/SubscriptionEvent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/SubscriptionEvent.java @@ -76,6 +76,12 @@ public void decreaseReferenceCount() { } } + public void clearReferenceCount() { + for (final EnrichedEvent enrichedEvent : enrichedEvents) { + enrichedEvent.clearReferenceCount(this.getClass().getName()); + } + } + //////////////////////////// pollable //////////////////////////// public void recordLastPolledTimestamp() { From 62d50b8b3fecb18e13a1d9d838a83e9d82c17259 Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Wed, 12 Jun 2024 01:04:04 +0800 Subject: [PATCH 23/36] improve IT --- .../it/dual/AbstractSubscriptionDualIT.java | 11 +++++++++++ .../it/dual/IoTDBSubscriptionConsumerGroupIT.java | 3 ++- .../consumer/SubscriptionExecutorServiceManager.java | 6 +++--- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/AbstractSubscriptionDualIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/AbstractSubscriptionDualIT.java index 9aa01ad4acfba..67399bfcecd04 100644 --- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/AbstractSubscriptionDualIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/AbstractSubscriptionDualIT.java @@ -21,17 +21,28 @@ import org.apache.iotdb.it.env.MultiEnvFactory; import org.apache.iotdb.itbase.env.BaseEnv; +import org.apache.iotdb.session.subscription.consumer.SubscriptionExecutorServiceManager; import org.junit.After; import org.junit.Before; +import org.junit.Rule; +import org.junit.rules.TestName; abstract class AbstractSubscriptionDualIT { protected BaseEnv senderEnv; protected BaseEnv receiverEnv; + @Rule public TestName testName = new TestName(); + @Before public void setUp() { + // set thread names + Thread.currentThread().setName(String.format("%s - main", testName.getMethodName())); + SubscriptionExecutorServiceManager.setControlFlowExecutorCorePoolSize(1); + SubscriptionExecutorServiceManager.setUpstreamDataFlowExecutorCorePoolSize(1); + SubscriptionExecutorServiceManager.setDownstreamDataFlowExecutorCorePoolSize(1); + MultiEnvFactory.createEnv(2); senderEnv = MultiEnvFactory.getEnv(0); receiverEnv = MultiEnvFactory.getEnv(1); diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionConsumerGroupIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionConsumerGroupIT.java index 992d151520f90..db7b9e7ca3a51 100644 --- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionConsumerGroupIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionConsumerGroupIT.java @@ -1005,7 +1005,7 @@ private void pollMessagesAndCheck( LOGGER.info("consumer {} exiting...", consumers.get(index)); } }, - consumers.get(index).toString()); + String.format("%s - %s", testName.getMethodName(), consumers.get(index).toString())); t.start(); threads.add(t); } @@ -1016,6 +1016,7 @@ private void pollMessagesAndCheck( final Statement statement = connection.createStatement()) { // Keep retrying if there are execution failures Awaitility.await() + .pollInSameThread() .pollDelay(IoTDBSubscriptionITConstant.AWAITILITY_POLL_DELAY_SECOND, TimeUnit.SECONDS) .pollInterval( IoTDBSubscriptionITConstant.AWAITILITY_POLL_INTERVAL_SECOND, TimeUnit.SECONDS) diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionExecutorServiceManager.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionExecutorServiceManager.java index d9586329197b0..6ce5946d4fcc6 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionExecutorServiceManager.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionExecutorServiceManager.java @@ -29,7 +29,7 @@ import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; -final class SubscriptionExecutorServiceManager { +public final class SubscriptionExecutorServiceManager { private static final Logger LOGGER = LoggerFactory.getLogger(SubscriptionExecutorServiceManager.class); @@ -172,9 +172,9 @@ boolean isShutdown() { } void setCorePoolSize(final int corePoolSize) { - if (!isShutdown()) { + if (isShutdown()) { synchronized (this) { - if (!isShutdown()) { + if (isShutdown()) { this.corePoolSize = corePoolSize; return; } From bb7b0655707d56c77ad7806dd62593076554a261 Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Wed, 12 Jun 2024 10:31:23 +0800 Subject: [PATCH 24/36] sync with #12710 --- .../subscription/it/dual/AbstractSubscriptionDualIT.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/AbstractSubscriptionDualIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/AbstractSubscriptionDualIT.java index 67399bfcecd04..3d8eb45a7fdfe 100644 --- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/AbstractSubscriptionDualIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/AbstractSubscriptionDualIT.java @@ -37,8 +37,10 @@ abstract class AbstractSubscriptionDualIT { @Before public void setUp() { - // set thread names + // set thread name Thread.currentThread().setName(String.format("%s - main", testName.getMethodName())); + + // set thread pools core size SubscriptionExecutorServiceManager.setControlFlowExecutorCorePoolSize(1); SubscriptionExecutorServiceManager.setUpstreamDataFlowExecutorCorePoolSize(1); SubscriptionExecutorServiceManager.setDownstreamDataFlowExecutorCorePoolSize(1); From 564de6220971816f8ef22add24dee64614a1fdeb Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Wed, 12 Jun 2024 15:37:04 +0800 Subject: [PATCH 25/36] refactor sub IT --- .../it/AbstractSubscriptionIT.java | 49 +++++++ .../it/IoTDBSubscriptionITConstant.java | 19 ++- .../IoTDBSubscriptionRestartIT.java | 42 ++---- .../it/dual/AbstractSubscriptionDualIT.java | 22 +-- .../IoTDBSubscriptionConsumerGroupIT.java | 31 ++-- .../IoTDBSubscriptionTimePrecisionIT.java | 30 ++-- .../it/dual/IoTDBSubscriptionTopicIT.java | 133 +++++++----------- .../it/local/AbstractSubscriptionLocalIT.java | 43 ++++++ .../it/local/IoTDBSubscriptionBasicIT.java | 105 ++++---------- .../local/IoTDBSubscriptionIdempotentIT.java | 14 +- 10 files changed, 231 insertions(+), 257 deletions(-) create mode 100644 integration-test/src/test/java/org/apache/iotdb/subscription/it/AbstractSubscriptionIT.java rename integration-test/src/test/java/org/apache/iotdb/subscription/it/{local => cluster}/IoTDBSubscriptionRestartIT.java (91%) create mode 100644 integration-test/src/test/java/org/apache/iotdb/subscription/it/local/AbstractSubscriptionLocalIT.java diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/AbstractSubscriptionIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/AbstractSubscriptionIT.java new file mode 100644 index 0000000000000..a168dbc78ba72 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/AbstractSubscriptionIT.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it; + +import org.apache.iotdb.session.subscription.consumer.SubscriptionExecutorServiceManager; + +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.rules.TestName; +import org.junit.rules.TestRule; + +public abstract class AbstractSubscriptionIT { + + @Rule public TestName testName = new TestName(); + + @Rule public final TestRule skipOnSetUpFailure = new SkipOnSetUpFailure("setUp"); + + @Before + public void setUp() { + // set thread name + Thread.currentThread().setName(String.format("%s - main", testName.getMethodName())); + + // set thread pools core size + SubscriptionExecutorServiceManager.setControlFlowExecutorCorePoolSize(1); + SubscriptionExecutorServiceManager.setUpstreamDataFlowExecutorCorePoolSize(1); + SubscriptionExecutorServiceManager.setDownstreamDataFlowExecutorCorePoolSize(1); + } + + @After + public void tearDown() {} +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/IoTDBSubscriptionITConstant.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/IoTDBSubscriptionITConstant.java index 7d16110cd1f00..840abb4ccb5b5 100644 --- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/IoTDBSubscriptionITConstant.java +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/IoTDBSubscriptionITConstant.java @@ -19,11 +19,24 @@ package org.apache.iotdb.subscription.it; +import org.awaitility.Awaitility; +import org.awaitility.core.ConditionFactory; + +import java.util.concurrent.TimeUnit; + public class IoTDBSubscriptionITConstant { - public static final long AWAITILITY_POLL_DELAY_SECOND = 1L; - public static final long AWAITILITY_POLL_INTERVAL_SECOND = 2L; - public static final long AWAITILITY_AT_MOST_SECOND = 600L; + private static final long AWAITILITY_POLL_DELAY_SECOND = 1L; + private static final long AWAITILITY_POLL_INTERVAL_SECOND = 2L; + private static final long AWAITILITY_AT_MOST_SECOND = 600L; + + public static final ConditionFactory AWAIT = + Awaitility.await() + .pollInSameThread() + .pollDelay(IoTDBSubscriptionITConstant.AWAITILITY_POLL_DELAY_SECOND, TimeUnit.SECONDS) + .pollInterval( + IoTDBSubscriptionITConstant.AWAITILITY_POLL_INTERVAL_SECOND, TimeUnit.SECONDS) + .atMost(IoTDBSubscriptionITConstant.AWAITILITY_AT_MOST_SECOND, TimeUnit.SECONDS); public static final long SLEEP_NS = 1_000_000_000L; public static final long POLL_TIMEOUT_MS = 10_000L; diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/IoTDBSubscriptionRestartIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/cluster/IoTDBSubscriptionRestartIT.java similarity index 91% rename from integration-test/src/test/java/org/apache/iotdb/subscription/it/local/IoTDBSubscriptionRestartIT.java rename to integration-test/src/test/java/org/apache/iotdb/subscription/it/cluster/IoTDBSubscriptionRestartIT.java index d3680219578ab..231bf8a1a1cb8 100644 --- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/IoTDBSubscriptionRestartIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/cluster/IoTDBSubscriptionRestartIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.subscription.it.local; +package org.apache.iotdb.subscription.it.cluster; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; import org.apache.iotdb.confignode.rpc.thrift.TShowSubscriptionReq; @@ -36,17 +36,14 @@ import org.apache.iotdb.session.subscription.consumer.SubscriptionPullConsumer; import org.apache.iotdb.session.subscription.payload.SubscriptionMessage; import org.apache.iotdb.session.subscription.payload.SubscriptionSessionDataSet; +import org.apache.iotdb.subscription.it.AbstractSubscriptionIT; import org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant; -import org.apache.iotdb.subscription.it.SkipOnSetUpFailure; -import org.awaitility.Awaitility; import org.junit.After; import org.junit.Assert; import org.junit.Before; -import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; -import org.junit.rules.TestRule; import org.junit.runner.RunWith; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -54,22 +51,22 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.LockSupport; +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) @Category({ClusterIT.class}) -public class IoTDBSubscriptionRestartIT { +public class IoTDBSubscriptionRestartIT extends AbstractSubscriptionIT { private static final Logger LOGGER = LoggerFactory.getLogger(IoTDBSubscriptionRestartIT.class); - @Rule public final TestRule skipOnSetUpFailure = new SkipOnSetUpFailure("setUp"); - @Before - public void setUp() throws Exception { + public void setUp() { + super.setUp(); + EnvFactory.getEnv() .getConfig() .getCommonConfig() @@ -83,7 +80,9 @@ public void setUp() throws Exception { } @After - public void tearDown() throws Exception { + public void tearDown() { + super.tearDown(); + EnvFactory.getEnv().cleanClusterEnvironment(); } @@ -201,12 +200,7 @@ public void testSubscriptionAfterRestartCluster() throws Exception { // Check timestamps size try { // Keep retrying if there are execution failures - Awaitility.await() - .pollDelay(IoTDBSubscriptionITConstant.AWAITILITY_POLL_DELAY_SECOND, TimeUnit.SECONDS) - .pollInterval( - IoTDBSubscriptionITConstant.AWAITILITY_POLL_INTERVAL_SECOND, TimeUnit.SECONDS) - .atMost(IoTDBSubscriptionITConstant.AWAITILITY_AT_MOST_SECOND, TimeUnit.SECONDS) - .untilAsserted(() -> Assert.assertEquals(100, timestamps.size())); + AWAIT.untilAsserted(() -> Assert.assertEquals(100, timestamps.size())); } catch (final Exception e) { e.printStackTrace(); fail(e.getMessage()); @@ -343,12 +337,7 @@ public void testSubscriptionAfterRestartDataNode() throws Exception { // Check timestamps size try { // Keep retrying if there are execution failures - Awaitility.await() - .pollDelay(IoTDBSubscriptionITConstant.AWAITILITY_POLL_DELAY_SECOND, TimeUnit.SECONDS) - .pollInterval( - IoTDBSubscriptionITConstant.AWAITILITY_POLL_INTERVAL_SECOND, TimeUnit.SECONDS) - .atMost(IoTDBSubscriptionITConstant.AWAITILITY_AT_MOST_SECOND, TimeUnit.SECONDS) - .untilAsserted(() -> Assert.assertEquals(200, timestamps.size())); + AWAIT.untilAsserted(() -> Assert.assertEquals(200, timestamps.size())); } catch (final Exception e) { e.printStackTrace(); fail(e.getMessage()); @@ -486,12 +475,7 @@ public void testSubscriptionWhenConfigNodeLeaderChange() throws Exception { // Check timestamps size try { // Keep retrying if there are execution failures - Awaitility.await() - .pollDelay(IoTDBSubscriptionITConstant.AWAITILITY_POLL_DELAY_SECOND, TimeUnit.SECONDS) - .pollInterval( - IoTDBSubscriptionITConstant.AWAITILITY_POLL_INTERVAL_SECOND, TimeUnit.SECONDS) - .atMost(IoTDBSubscriptionITConstant.AWAITILITY_AT_MOST_SECOND, TimeUnit.SECONDS) - .untilAsserted(() -> Assert.assertEquals(200, timestamps.size())); + AWAIT.untilAsserted(() -> Assert.assertEquals(200, timestamps.size())); } catch (final Exception e) { e.printStackTrace(); fail(e.getMessage()); diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/AbstractSubscriptionDualIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/AbstractSubscriptionDualIT.java index 3d8eb45a7fdfe..c8a98bc342bbb 100644 --- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/AbstractSubscriptionDualIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/AbstractSubscriptionDualIT.java @@ -21,29 +21,19 @@ import org.apache.iotdb.it.env.MultiEnvFactory; import org.apache.iotdb.itbase.env.BaseEnv; -import org.apache.iotdb.session.subscription.consumer.SubscriptionExecutorServiceManager; +import org.apache.iotdb.subscription.it.AbstractSubscriptionIT; import org.junit.After; import org.junit.Before; -import org.junit.Rule; -import org.junit.rules.TestName; -abstract class AbstractSubscriptionDualIT { +abstract class AbstractSubscriptionDualIT extends AbstractSubscriptionIT { protected BaseEnv senderEnv; protected BaseEnv receiverEnv; - @Rule public TestName testName = new TestName(); - @Before public void setUp() { - // set thread name - Thread.currentThread().setName(String.format("%s - main", testName.getMethodName())); - - // set thread pools core size - SubscriptionExecutorServiceManager.setControlFlowExecutorCorePoolSize(1); - SubscriptionExecutorServiceManager.setUpstreamDataFlowExecutorCorePoolSize(1); - SubscriptionExecutorServiceManager.setDownstreamDataFlowExecutorCorePoolSize(1); + super.setUp(); MultiEnvFactory.createEnv(2); senderEnv = MultiEnvFactory.getEnv(0); @@ -55,7 +45,7 @@ public void setUp() { receiverEnv.initClusterEnvironment(); } - void setUpConfig() { + protected void setUpConfig() { // enable auto create schema senderEnv.getConfig().getCommonConfig().setAutoCreateSchemaEnabled(true); receiverEnv.getConfig().getCommonConfig().setAutoCreateSchemaEnabled(true); @@ -66,7 +56,9 @@ void setUpConfig() { } @After - public final void tearDown() { + public void tearDown() { + super.tearDown(); + senderEnv.cleanClusterEnvironment(); receiverEnv.cleanClusterEnvironment(); } diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionConsumerGroupIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionConsumerGroupIT.java index db7b9e7ca3a51..650f861f498d0 100644 --- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionConsumerGroupIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionConsumerGroupIT.java @@ -42,7 +42,6 @@ import org.apache.tsfile.read.expression.QueryExpression; import org.apache.tsfile.read.query.dataset.QueryDataSet; import org.apache.tsfile.utils.Pair; -import org.awaitility.Awaitility; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -62,11 +61,11 @@ import java.util.Map; import java.util.Properties; import java.util.Set; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.LockSupport; import java.util.stream.Collectors; +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) @@ -109,7 +108,7 @@ static final class SubscriptionInfo { } @Override - void setUpConfig() { + protected void setUpConfig() { super.setUpConfig(); // Enable air gap receiver @@ -1015,22 +1014,16 @@ private void pollMessagesAndCheck( try (final Connection connection = receiverEnv.getConnection(); final Statement statement = connection.createStatement()) { // Keep retrying if there are execution failures - Awaitility.await() - .pollInSameThread() - .pollDelay(IoTDBSubscriptionITConstant.AWAITILITY_POLL_DELAY_SECOND, TimeUnit.SECONDS) - .pollInterval( - IoTDBSubscriptionITConstant.AWAITILITY_POLL_INTERVAL_SECOND, TimeUnit.SECONDS) - .atMost(IoTDBSubscriptionITConstant.AWAITILITY_AT_MOST_SECOND, TimeUnit.SECONDS) - .untilAsserted( - () -> { - if (receiverCrashed.get()) { - LOGGER.info("detect receiver crashed, skipping this test..."); - return; - } - TestUtils.assertSingleResultSetEqual( - TestUtils.executeQueryWithRetry(statement, "select count(*) from root.**"), - expectedHeaderWithResult); - }); + AWAIT.untilAsserted( + () -> { + if (receiverCrashed.get()) { + LOGGER.info("detect receiver crashed, skipping this test..."); + return; + } + TestUtils.assertSingleResultSetEqual( + TestUtils.executeQueryWithRetry(statement, "select count(*) from root.**"), + expectedHeaderWithResult); + }); } } catch (final Exception e) { e.printStackTrace(); diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionTimePrecisionIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionTimePrecisionIT.java index 2b1cc407b7f79..9829c0f3c1acc 100644 --- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionTimePrecisionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionTimePrecisionIT.java @@ -30,7 +30,6 @@ import org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant; import org.apache.tsfile.write.record.Tablet; -import org.awaitility.Awaitility; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; @@ -43,10 +42,10 @@ import java.util.Iterator; import java.util.List; import java.util.Properties; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.LockSupport; +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) @@ -57,7 +56,7 @@ public class IoTDBSubscriptionTimePrecisionIT extends AbstractSubscriptionDualIT LoggerFactory.getLogger(IoTDBSubscriptionTimePrecisionIT.class); @Override - void setUpConfig() { + protected void setUpConfig() { super.setUpConfig(); // Set timestamp precision to nanosecond @@ -168,21 +167,16 @@ public void testTopicTimePrecision() throws Exception { try (final Connection connection = receiverEnv.getConnection(); final Statement statement = connection.createStatement()) { // Keep retrying if there are execution failures - Awaitility.await() - .pollDelay(IoTDBSubscriptionITConstant.AWAITILITY_POLL_DELAY_SECOND, TimeUnit.SECONDS) - .pollInterval( - IoTDBSubscriptionITConstant.AWAITILITY_POLL_INTERVAL_SECOND, TimeUnit.SECONDS) - .atMost(IoTDBSubscriptionITConstant.AWAITILITY_AT_MOST_SECOND, TimeUnit.SECONDS) - .untilAsserted( - () -> - TestUtils.assertSingleResultSetEqual( - TestUtils.executeQueryWithRetry(statement, "select count(*) from root.**"), - new HashMap() { - { - put("count(root.db.d1.s2)", "100"); - put("count(root.db.d2.s1)", "100"); - } - })); + AWAIT.untilAsserted( + () -> + TestUtils.assertSingleResultSetEqual( + TestUtils.executeQueryWithRetry(statement, "select count(*) from root.**"), + new HashMap() { + { + put("count(root.db.d1.s2)", "100"); + put("count(root.db.d2.s1)", "100"); + } + })); } } catch (final Exception e) { e.printStackTrace(); diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionTopicIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionTopicIT.java index 382b93ac3223c..402a17669bfcf 100644 --- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionTopicIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionTopicIT.java @@ -41,7 +41,6 @@ import org.apache.tsfile.read.expression.QueryExpression; import org.apache.tsfile.read.query.dataset.QueryDataSet; import org.apache.tsfile.write.record.Tablet; -import org.awaitility.Awaitility; import org.junit.Assert; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -59,11 +58,11 @@ import java.util.List; import java.util.Properties; import java.util.Set; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.locks.LockSupport; +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) @@ -73,7 +72,7 @@ public class IoTDBSubscriptionTopicIT extends AbstractSubscriptionDualIT { private static final Logger LOGGER = LoggerFactory.getLogger(IoTDBSubscriptionTopicIT.class); @Override - void setUpConfig() { + protected void setUpConfig() { super.setUpConfig(); // Shorten heartbeat and sync interval to avoid timeout of query mode test @@ -165,21 +164,16 @@ public void testTopicPathSubscription() throws Exception { try (final Connection connection = receiverEnv.getConnection(); final Statement statement = connection.createStatement()) { // Keep retrying if there are execution failures - Awaitility.await() - .pollDelay(IoTDBSubscriptionITConstant.AWAITILITY_POLL_DELAY_SECOND, TimeUnit.SECONDS) - .pollInterval( - IoTDBSubscriptionITConstant.AWAITILITY_POLL_INTERVAL_SECOND, TimeUnit.SECONDS) - .atMost(IoTDBSubscriptionITConstant.AWAITILITY_AT_MOST_SECOND, TimeUnit.SECONDS) - .untilAsserted( - () -> - TestUtils.assertSingleResultSetEqual( - TestUtils.executeQueryWithRetry(statement, "select count(*) from root.**"), - new HashMap() { - { - put("count(root.db.d1.s)", "100"); - put("count(root.db.d2.s)", "100"); - } - })); + AWAIT.untilAsserted( + () -> + TestUtils.assertSingleResultSetEqual( + TestUtils.executeQueryWithRetry(statement, "select count(*) from root.**"), + new HashMap() { + { + put("count(root.db.d1.s)", "100"); + put("count(root.db.d2.s)", "100"); + } + })); } } catch (final Exception e) { e.printStackTrace(); @@ -267,20 +261,15 @@ public void testTopicTimeSubscription() throws Exception { try (final Connection connection = receiverEnv.getConnection(); final Statement statement = connection.createStatement()) { // Keep retrying if there are execution failures - Awaitility.await() - .pollDelay(IoTDBSubscriptionITConstant.AWAITILITY_POLL_DELAY_SECOND, TimeUnit.SECONDS) - .pollInterval( - IoTDBSubscriptionITConstant.AWAITILITY_POLL_INTERVAL_SECOND, TimeUnit.SECONDS) - .atMost(IoTDBSubscriptionITConstant.AWAITILITY_AT_MOST_SECOND, TimeUnit.SECONDS) - .untilAsserted( - () -> - TestUtils.assertSingleResultSetEqual( - TestUtils.executeQueryWithRetry(statement, "select count(*) from root.**"), - new HashMap() { - { - put("count(root.db.d2.s)", "100"); - } - })); + AWAIT.untilAsserted( + () -> + TestUtils.assertSingleResultSetEqual( + TestUtils.executeQueryWithRetry(statement, "select count(*) from root.**"), + new HashMap() { + { + put("count(root.db.d2.s)", "100"); + } + })); } } catch (final Exception e) { e.printStackTrace(); @@ -369,17 +358,12 @@ public void testTopicProcessorSubscription() throws Exception { try (final Connection connection = receiverEnv.getConnection(); final Statement statement = connection.createStatement()) { // Keep retrying if there are execution failures - Awaitility.await() - .pollDelay(IoTDBSubscriptionITConstant.AWAITILITY_POLL_DELAY_SECOND, TimeUnit.SECONDS) - .pollInterval( - IoTDBSubscriptionITConstant.AWAITILITY_POLL_INTERVAL_SECOND, TimeUnit.SECONDS) - .atMost(IoTDBSubscriptionITConstant.AWAITILITY_AT_MOST_SECOND, TimeUnit.SECONDS) - .untilAsserted( - () -> - TestUtils.assertResultSetEqual( - TestUtils.executeQueryWithRetry(statement, "select * from root.**"), - "Time,root.db.d1.at1,", - expectedResSet)); + AWAIT.untilAsserted( + () -> + TestUtils.assertResultSetEqual( + TestUtils.executeQueryWithRetry(statement, "select * from root.**"), + "Time,root.db.d1.at1,", + expectedResSet)); } } catch (final Exception e) { e.printStackTrace(); @@ -493,20 +477,15 @@ public void testTopicNameWithBackQuote() throws Exception { try (final Connection connection = receiverEnv.getConnection(); final Statement statement = connection.createStatement()) { // Keep retrying if there are execution failures - Awaitility.await() - .pollDelay(IoTDBSubscriptionITConstant.AWAITILITY_POLL_DELAY_SECOND, TimeUnit.SECONDS) - .pollInterval( - IoTDBSubscriptionITConstant.AWAITILITY_POLL_INTERVAL_SECOND, TimeUnit.SECONDS) - .atMost(IoTDBSubscriptionITConstant.AWAITILITY_AT_MOST_SECOND, TimeUnit.SECONDS) - .untilAsserted( - () -> - TestUtils.assertSingleResultSetEqual( - TestUtils.executeQueryWithRetry(statement, "select count(*) from root.**"), - new HashMap() { - { - put("count(root.db.d1.s)", "300"); - } - })); + AWAIT.untilAsserted( + () -> + TestUtils.assertSingleResultSetEqual( + TestUtils.executeQueryWithRetry(statement, "select count(*) from root.**"), + new HashMap() { + { + put("count(root.db.d1.s)", "300"); + } + })); } } catch (final Exception e) { e.printStackTrace(); @@ -644,26 +623,21 @@ public void testTopicWithQueryMode() throws Exception { try { // Keep retrying if there are execution failures - Awaitility.await() - .pollDelay(IoTDBSubscriptionITConstant.AWAITILITY_POLL_DELAY_SECOND, TimeUnit.SECONDS) - .pollInterval( - IoTDBSubscriptionITConstant.AWAITILITY_POLL_INTERVAL_SECOND, TimeUnit.SECONDS) - .atMost(IoTDBSubscriptionITConstant.AWAITILITY_AT_MOST_SECOND, TimeUnit.SECONDS) - .untilAsserted( - () -> { - // Check row count - Assert.assertEquals(100, rowCount.get()); - // Check empty subscription - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final TShowSubscriptionResp showSubscriptionResp = - client.showSubscription(new TShowSubscriptionReq()); - Assert.assertEquals( - RpcUtils.SUCCESS_STATUS.getCode(), showSubscriptionResp.status.getCode()); - Assert.assertNotNull(showSubscriptionResp.subscriptionInfoList); - Assert.assertEquals(0, showSubscriptionResp.subscriptionInfoList.size()); - } - }); + AWAIT.untilAsserted( + () -> { + // Check row count + Assert.assertEquals(100, rowCount.get()); + // Check empty subscription + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final TShowSubscriptionResp showSubscriptionResp = + client.showSubscription(new TShowSubscriptionReq()); + Assert.assertEquals( + RpcUtils.SUCCESS_STATUS.getCode(), showSubscriptionResp.status.getCode()); + Assert.assertNotNull(showSubscriptionResp.subscriptionInfoList); + Assert.assertEquals(0, showSubscriptionResp.subscriptionInfoList.size()); + } + }); } catch (final Exception e) { e.printStackTrace(); fail(e.getMessage()); @@ -761,10 +735,7 @@ private void testTopicInvalidRuntimeConfigTemplate( thread.start(); } - Awaitility.await() - .pollDelay(IoTDBSubscriptionITConstant.AWAITILITY_POLL_DELAY_SECOND, TimeUnit.SECONDS) - .pollInterval(IoTDBSubscriptionITConstant.AWAITILITY_POLL_INTERVAL_SECOND, TimeUnit.SECONDS) - .atMost(IoTDBSubscriptionITConstant.AWAITILITY_AT_MOST_SECOND, TimeUnit.SECONDS) + AWAIT // The expected SubscriptionRuntimeCriticalException was not thrown if result is false .untilTrue(result); diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/AbstractSubscriptionLocalIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/AbstractSubscriptionLocalIT.java new file mode 100644 index 0000000000000..60d1bad3039d2 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/AbstractSubscriptionLocalIT.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.local; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.subscription.it.AbstractSubscriptionIT; + +import org.junit.After; +import org.junit.Before; + +abstract class AbstractSubscriptionLocalIT extends AbstractSubscriptionIT { + + @Before + public void setUp() { + super.setUp(); + + EnvFactory.getEnv().initClusterEnvironment(); + } + + @After + public void tearDown() { + super.tearDown(); + + EnvFactory.getEnv().cleanClusterEnvironment(); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/IoTDBSubscriptionBasicIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/IoTDBSubscriptionBasicIT.java index 13a5e8abc2617..37a1050e9a303 100644 --- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/IoTDBSubscriptionBasicIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/IoTDBSubscriptionBasicIT.java @@ -40,10 +40,7 @@ import org.apache.tsfile.read.common.Path; import org.apache.tsfile.read.expression.QueryExpression; import org.apache.tsfile.read.query.dataset.QueryDataSet; -import org.awaitility.Awaitility; -import org.junit.After; import org.junit.Assert; -import org.junit.Before; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; @@ -57,31 +54,21 @@ import java.util.Collections; import java.util.List; import java.util.Properties; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.LockSupport; import java.util.stream.Collectors; +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) @Category({LocalStandaloneIT.class}) -public class IoTDBSubscriptionBasicIT { +public class IoTDBSubscriptionBasicIT extends AbstractSubscriptionLocalIT { private static final Logger LOGGER = LoggerFactory.getLogger(IoTDBSubscriptionBasicIT.class); - @Before - public void setUp() throws Exception { - EnvFactory.getEnv().initClusterEnvironment(); - } - - @After - public void tearDown() throws Exception { - EnvFactory.getEnv().cleanClusterEnvironment(); - } - @Test public void testBasicSubscribeTablets() throws Exception { // Insert some historical data @@ -152,12 +139,7 @@ public void testBasicSubscribeTablets() throws Exception { // Check row count try { // Keep retrying if there are execution failures - Awaitility.await() - .pollDelay(IoTDBSubscriptionITConstant.AWAITILITY_POLL_DELAY_SECOND, TimeUnit.SECONDS) - .pollInterval( - IoTDBSubscriptionITConstant.AWAITILITY_POLL_INTERVAL_SECOND, TimeUnit.SECONDS) - .atMost(IoTDBSubscriptionITConstant.AWAITILITY_AT_MOST_SECOND, TimeUnit.SECONDS) - .untilAsserted(() -> Assert.assertEquals(100, rowCount.get())); + AWAIT.untilAsserted(() -> Assert.assertEquals(100, rowCount.get())); } catch (final Exception e) { e.printStackTrace(); fail(e.getMessage()); @@ -248,12 +230,7 @@ public void testBasicSubscribeTsFile() throws Exception { // Check row count try { // Keep retrying if there are execution failures - Awaitility.await() - .pollDelay(IoTDBSubscriptionITConstant.AWAITILITY_POLL_DELAY_SECOND, TimeUnit.SECONDS) - .pollInterval( - IoTDBSubscriptionITConstant.AWAITILITY_POLL_INTERVAL_SECOND, TimeUnit.SECONDS) - .atMost(IoTDBSubscriptionITConstant.AWAITILITY_AT_MOST_SECOND, TimeUnit.SECONDS) - .untilAsserted(() -> Assert.assertEquals(100, rowCount.get())); + AWAIT.untilAsserted(() -> Assert.assertEquals(100, rowCount.get())); } catch (final Exception e) { e.printStackTrace(); fail(e.getMessage()); @@ -385,12 +362,7 @@ public void onFailure(final Throwable e) { // Check row count try { // Keep retrying if there are execution failures - Awaitility.await() - .pollDelay(IoTDBSubscriptionITConstant.AWAITILITY_POLL_DELAY_SECOND, TimeUnit.SECONDS) - .pollInterval( - IoTDBSubscriptionITConstant.AWAITILITY_POLL_INTERVAL_SECOND, TimeUnit.SECONDS) - .atMost(IoTDBSubscriptionITConstant.AWAITILITY_AT_MOST_SECOND, TimeUnit.SECONDS) - .untilAsserted(() -> Assert.assertEquals(100, rowCount.get())); + AWAIT.untilAsserted(() -> Assert.assertEquals(100, rowCount.get())); Assert.assertTrue(commitSuccessCount.get() > lastCommitSuccessCount.get()); Assert.assertEquals(0, commitFailureCount.get()); } catch (final Exception e) { @@ -416,12 +388,7 @@ public void onFailure(final Throwable e) { // Check row count try { // Keep retrying if there are execution failures - Awaitility.await() - .pollDelay(IoTDBSubscriptionITConstant.AWAITILITY_POLL_DELAY_SECOND, TimeUnit.SECONDS) - .pollInterval( - IoTDBSubscriptionITConstant.AWAITILITY_POLL_INTERVAL_SECOND, TimeUnit.SECONDS) - .atMost(IoTDBSubscriptionITConstant.AWAITILITY_AT_MOST_SECOND, TimeUnit.SECONDS) - .untilAsserted(() -> Assert.assertEquals(200, rowCount.get())); + AWAIT.untilAsserted(() -> Assert.assertEquals(200, rowCount.get())); Assert.assertTrue(commitSuccessCount.get() > lastCommitSuccessCount.get()); Assert.assertEquals(0, commitFailureCount.get()); } catch (final Exception e) { @@ -486,16 +453,11 @@ public void testBasicPushConsumer() { consumer.subscribe(topicName); // The push consumer should automatically poll 10 rows of data by 1 onReceive() - Awaitility.await() - .pollDelay(IoTDBSubscriptionITConstant.AWAITILITY_POLL_DELAY_SECOND, TimeUnit.SECONDS) - .pollInterval( - IoTDBSubscriptionITConstant.AWAITILITY_POLL_INTERVAL_SECOND, TimeUnit.SECONDS) - .atMost(IoTDBSubscriptionITConstant.AWAITILITY_AT_MOST_SECOND, TimeUnit.SECONDS) - .untilAsserted( - () -> { - Assert.assertEquals(10, rowCount.get()); - Assert.assertTrue(onReceiveCount.get() > lastOnReceiveCount.get()); - }); + AWAIT.untilAsserted( + () -> { + Assert.assertEquals(10, rowCount.get()); + Assert.assertTrue(onReceiveCount.get() > lastOnReceiveCount.get()); + }); lastOnReceiveCount.set(onReceiveCount.get()); @@ -508,16 +470,11 @@ public void testBasicPushConsumer() { session.executeNonQueryStatement("flush"); } - Awaitility.await() - .pollDelay(IoTDBSubscriptionITConstant.AWAITILITY_POLL_DELAY_SECOND, TimeUnit.SECONDS) - .pollInterval( - IoTDBSubscriptionITConstant.AWAITILITY_POLL_INTERVAL_SECOND, TimeUnit.SECONDS) - .atMost(IoTDBSubscriptionITConstant.AWAITILITY_AT_MOST_SECOND, TimeUnit.SECONDS) - .untilAsserted( - () -> { - Assert.assertEquals(20, rowCount.get()); - Assert.assertTrue(onReceiveCount.get() > lastOnReceiveCount.get()); - }); + AWAIT.untilAsserted( + () -> { + Assert.assertEquals(20, rowCount.get()); + Assert.assertTrue(onReceiveCount.get() > lastOnReceiveCount.get()); + }); lastOnReceiveCount.set(onReceiveCount.get()); @@ -529,16 +486,11 @@ public void testBasicPushConsumer() { session.executeNonQueryStatement("flush"); } - Awaitility.await() - .pollDelay(IoTDBSubscriptionITConstant.AWAITILITY_POLL_DELAY_SECOND, TimeUnit.SECONDS) - .pollInterval( - IoTDBSubscriptionITConstant.AWAITILITY_POLL_INTERVAL_SECOND, TimeUnit.SECONDS) - .atMost(IoTDBSubscriptionITConstant.AWAITILITY_AT_MOST_SECOND, TimeUnit.SECONDS) - .untilAsserted( - () -> { - Assert.assertEquals(30, rowCount.get()); - Assert.assertTrue(onReceiveCount.get() > lastOnReceiveCount.get()); - }); + AWAIT.untilAsserted( + () -> { + Assert.assertEquals(30, rowCount.get()); + Assert.assertTrue(onReceiveCount.get() > lastOnReceiveCount.get()); + }); consumer.unsubscribe(topicName); } catch (final Exception e) { @@ -630,16 +582,11 @@ public void testPollUnsubscribedTopics() throws Exception { // Check row count try { // Keep retrying if there are execution failures - Awaitility.await() - .pollDelay(IoTDBSubscriptionITConstant.AWAITILITY_POLL_DELAY_SECOND, TimeUnit.SECONDS) - .pollInterval( - IoTDBSubscriptionITConstant.AWAITILITY_POLL_INTERVAL_SECOND, TimeUnit.SECONDS) - .atMost(IoTDBSubscriptionITConstant.AWAITILITY_AT_MOST_SECOND, TimeUnit.SECONDS) - .untilAsserted( - () -> { - Assert.assertEquals(100, rowCount.get()); - Assert.assertEquals((100 + 199) * 100 / 2, timestampSum.get()); - }); + AWAIT.untilAsserted( + () -> { + Assert.assertEquals(100, rowCount.get()); + Assert.assertEquals((100 + 199) * 100 / 2, timestampSum.get()); + }); } catch (final Exception e) { e.printStackTrace(); fail(e.getMessage()); diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/IoTDBSubscriptionIdempotentIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/IoTDBSubscriptionIdempotentIT.java index b671acd6709f4..85fff437f18e6 100644 --- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/IoTDBSubscriptionIdempotentIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/IoTDBSubscriptionIdempotentIT.java @@ -25,9 +25,7 @@ import org.apache.iotdb.session.subscription.SubscriptionSession; import org.apache.iotdb.session.subscription.consumer.SubscriptionPullConsumer; -import org.junit.After; import org.junit.Assert; -import org.junit.Before; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; @@ -38,20 +36,10 @@ @RunWith(IoTDBTestRunner.class) @Category({LocalStandaloneIT.class}) -public class IoTDBSubscriptionIdempotentIT { +public class IoTDBSubscriptionIdempotentIT extends AbstractSubscriptionLocalIT { private static final Logger LOGGER = LoggerFactory.getLogger(IoTDBSubscriptionIdempotentIT.class); - @Before - public void setUp() throws Exception { - EnvFactory.getEnv().initClusterEnvironment(); - } - - @After - public void tearDown() throws Exception { - EnvFactory.getEnv().cleanClusterEnvironment(); - } - @Test public void testSubscribeOrUnsubscribeNonExistedTopicTest() { final String host = EnvFactory.getEnv().getIP(); From 75fe0affddd490a7f163ab940e38ed014ed8d9fa Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Wed, 12 Jun 2024 15:39:45 +0800 Subject: [PATCH 26/36] minor improve --- .../iotdb/subscription/it/dual/IoTDBSubscriptionTopicIT.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionTopicIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionTopicIT.java index 402a17669bfcf..4b02f68ed1cbc 100644 --- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionTopicIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionTopicIT.java @@ -735,9 +735,8 @@ private void testTopicInvalidRuntimeConfigTemplate( thread.start(); } - AWAIT - // The expected SubscriptionRuntimeCriticalException was not thrown if result is false - .untilTrue(result); + // The expected SubscriptionRuntimeCriticalException was not thrown if result is false + AWAIT.untilTrue(result); for (final Thread thread : threads) { thread.join(); From ef288084531fa58b366476f7c8ea1e504b024cf3 Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Wed, 12 Jun 2024 15:41:30 +0800 Subject: [PATCH 27/36] add override annotation --- .../subscription/it/cluster/IoTDBSubscriptionRestartIT.java | 2 ++ .../iotdb/subscription/it/dual/AbstractSubscriptionDualIT.java | 2 ++ .../subscription/it/local/AbstractSubscriptionLocalIT.java | 2 ++ 3 files changed, 6 insertions(+) diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/cluster/IoTDBSubscriptionRestartIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/cluster/IoTDBSubscriptionRestartIT.java index 231bf8a1a1cb8..144f7a7513a4b 100644 --- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/cluster/IoTDBSubscriptionRestartIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/cluster/IoTDBSubscriptionRestartIT.java @@ -63,6 +63,7 @@ public class IoTDBSubscriptionRestartIT extends AbstractSubscriptionIT { private static final Logger LOGGER = LoggerFactory.getLogger(IoTDBSubscriptionRestartIT.class); + @Override @Before public void setUp() { super.setUp(); @@ -79,6 +80,7 @@ public void setUp() { EnvFactory.getEnv().initClusterEnvironment(3, 3); } + @Override @After public void tearDown() { super.tearDown(); diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/AbstractSubscriptionDualIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/AbstractSubscriptionDualIT.java index c8a98bc342bbb..fb25cdc3a6c09 100644 --- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/AbstractSubscriptionDualIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/AbstractSubscriptionDualIT.java @@ -31,6 +31,7 @@ abstract class AbstractSubscriptionDualIT extends AbstractSubscriptionIT { protected BaseEnv senderEnv; protected BaseEnv receiverEnv; + @Override @Before public void setUp() { super.setUp(); @@ -55,6 +56,7 @@ protected void setUpConfig() { receiverEnv.getConfig().getCommonConfig().setCnConnectionTimeoutMs(600000); } + @Override @After public void tearDown() { super.tearDown(); diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/AbstractSubscriptionLocalIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/AbstractSubscriptionLocalIT.java index 60d1bad3039d2..bb248f745d4ed 100644 --- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/AbstractSubscriptionLocalIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/AbstractSubscriptionLocalIT.java @@ -27,6 +27,7 @@ abstract class AbstractSubscriptionLocalIT extends AbstractSubscriptionIT { + @Override @Before public void setUp() { super.setUp(); @@ -34,6 +35,7 @@ public void setUp() { EnvFactory.getEnv().initClusterEnvironment(); } + @Override @After public void tearDown() { super.tearDown(); From 3cb65fe35f16686ed9fa9db3ebce1137c4e74793 Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Wed, 12 Jun 2024 15:49:13 +0800 Subject: [PATCH 28/36] improve thread names --- .../cluster/IoTDBSubscriptionRestartIT.java | 9 +++++--- .../IoTDBSubscriptionTimePrecisionIT.java | 3 ++- .../it/dual/IoTDBSubscriptionTopicIT.java | 21 ++++++++++++------- .../it/local/IoTDBSubscriptionBasicIT.java | 12 +++++++---- 4 files changed, 30 insertions(+), 15 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/cluster/IoTDBSubscriptionRestartIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/cluster/IoTDBSubscriptionRestartIT.java index 144f7a7513a4b..41dd654383d66 100644 --- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/cluster/IoTDBSubscriptionRestartIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/cluster/IoTDBSubscriptionRestartIT.java @@ -196,7 +196,8 @@ public void testSubscriptionAfterRestartCluster() throws Exception { } finally { LOGGER.info("consumer exiting..."); } - }); + }, + String.format("%s - %s", testName.getMethodName(), consumer)); thread.start(); // Check timestamps size @@ -308,7 +309,8 @@ public void testSubscriptionAfterRestartDataNode() throws Exception { } finally { LOGGER.info("consumer exiting..."); } - }); + }, + String.format("%s - %s", testName.getMethodName(), consumer)); thread.start(); // Start DN 1 & DN 2 @@ -434,7 +436,8 @@ public void testSubscriptionWhenConfigNodeLeaderChange() throws Exception { } finally { LOGGER.info("consumer exiting..."); } - }); + }, + String.format("%s - %s", testName.getMethodName(), consumer)); thread.start(); // Shutdown leader CN diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionTimePrecisionIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionTimePrecisionIT.java index 9829c0f3c1acc..39a9f2225f6bb 100644 --- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionTimePrecisionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionTimePrecisionIT.java @@ -159,7 +159,8 @@ public void testTopicTimePrecision() throws Exception { } finally { LOGGER.info("consumer exiting..."); } - }); + }, + String.format("%s - consumer", testName.getMethodName())); thread.start(); // Check data on receiver diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionTopicIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionTopicIT.java index 4b02f68ed1cbc..19c033af151ba 100644 --- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionTopicIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionTopicIT.java @@ -156,7 +156,8 @@ public void testTopicPathSubscription() throws Exception { } finally { LOGGER.info("consumer exiting..."); } - }); + }, + String.format("%s - consumer", testName.getMethodName())); thread.start(); // Check data on receiver @@ -253,7 +254,8 @@ public void testTopicTimeSubscription() throws Exception { } finally { LOGGER.info("consumer exiting..."); } - }); + }, + String.format("%s - consumer", testName.getMethodName())); thread.start(); // Check data on receiver @@ -346,7 +348,8 @@ public void testTopicProcessorSubscription() throws Exception { } finally { LOGGER.info("consumer exiting..."); } - }); + }, + String.format("%s - consumer", testName.getMethodName())); thread.start(); // Check data on receiver @@ -469,7 +472,8 @@ public void testTopicNameWithBackQuote() throws Exception { } finally { LOGGER.info("consumer exiting..."); } - }); + }, + String.format("%s - consumer", testName.getMethodName())); thread.start(); // Check data on receiver @@ -618,7 +622,8 @@ public void testTopicWithQueryMode() throws Exception { } finally { LOGGER.info("consumer exiting..."); } - }); + }, + String.format("%s - consumer", testName.getMethodName())); thread.start(); try { @@ -701,7 +706,8 @@ private void testTopicInvalidRuntimeConfigTemplate( } finally { LOGGER.info("consumer exiting..."); } - })); + }, + String.format("%s - consumer", testName.getMethodName()))); // Insert some realtime data on sender threads.add( @@ -729,7 +735,8 @@ private void testTopicInvalidRuntimeConfigTemplate( fail(e.getMessage()); } dataPrepared.set(true); - })); + }, + String.format("%s - data inserter", testName.getMethodName()))); for (final Thread thread : threads) { thread.start(); diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/IoTDBSubscriptionBasicIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/IoTDBSubscriptionBasicIT.java index 37a1050e9a303..fccbee6d95da7 100644 --- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/IoTDBSubscriptionBasicIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/IoTDBSubscriptionBasicIT.java @@ -133,7 +133,8 @@ public void testBasicSubscribeTablets() throws Exception { } finally { LOGGER.info("consumer exiting..."); } - }); + }, + String.format("%s - consumer", testName.getMethodName())); thread.start(); // Check row count @@ -224,7 +225,8 @@ public void testBasicSubscribeTsFile() throws Exception { } finally { LOGGER.info("consumer exiting..."); } - }); + }, + String.format("%s - consumer", testName.getMethodName())); thread.start(); // Check row count @@ -356,7 +358,8 @@ public void onFailure(final Throwable e) { } finally { LOGGER.info("consumer exiting..."); } - }); + }, + String.format("%s - consumer", testName.getMethodName())); thread.start(); // Check row count @@ -576,7 +579,8 @@ public void testPollUnsubscribedTopics() throws Exception { } finally { LOGGER.info("consumer exiting..."); } - }); + }, + String.format("%s - consumer", testName.getMethodName())); thread.start(); // Check row count From b2d49e3a58a016b1423593e7f932f1a12b910792 Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Wed, 12 Jun 2024 19:38:51 +0800 Subject: [PATCH 29/36] fixup --- .../response/PipeSubscribeSubscribeResp.java | 23 ------------------- .../PipeSubscribeUnsubscribeResp.java | 23 ------------------- 2 files changed, 46 deletions(-) diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/response/PipeSubscribeSubscribeResp.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/response/PipeSubscribeSubscribeResp.java index 36bfbd65476f8..1d1571b0122ee 100644 --- a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/response/PipeSubscribeSubscribeResp.java +++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/response/PipeSubscribeSubscribeResp.java @@ -78,29 +78,6 @@ public static PipeSubscribeSubscribeResp toTPipeSubscribeResp( return resp; } - /** - * Serialize the incoming parameters into `PipeSubscribeSubscribeResp`, called by the subscription - * server. - */ - public static PipeSubscribeSubscribeResp toTPipeSubscribeResp( - final TSStatus status, final Set topicNames) throws IOException { - final PipeSubscribeSubscribeResp resp = new PipeSubscribeSubscribeResp(); - - resp.status = status; - resp.version = PipeSubscribeResponseVersion.VERSION_1.getVersion(); - resp.type = PipeSubscribeResponseType.ACK.getType(); - - try (final PublicBAOS byteArrayOutputStream = new PublicBAOS(); - final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) { - ReadWriteIOUtils.writeObjectSet(topicNames, outputStream); - resp.body = - Collections.singletonList( - ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size())); - } - - return resp; - } - /** Deserialize `TPipeSubscribeResp` to obtain parameters, called by the subscription client. */ public static PipeSubscribeSubscribeResp fromTPipeSubscribeResp( final TPipeSubscribeResp subscribeResp) { diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/response/PipeSubscribeUnsubscribeResp.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/response/PipeSubscribeUnsubscribeResp.java index b831a44268428..e5d8ccf69cf4b 100644 --- a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/response/PipeSubscribeUnsubscribeResp.java +++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/response/PipeSubscribeUnsubscribeResp.java @@ -78,29 +78,6 @@ public static PipeSubscribeUnsubscribeResp toTPipeSubscribeResp( return resp; } - /** - * Serialize the incoming parameters into `PipeSubscribeUnsubscribeResp`, called by the - * subscription server. - */ - public static PipeSubscribeUnsubscribeResp toTPipeSubscribeResp( - final TSStatus status, final Set topicNames) throws IOException { - final PipeSubscribeUnsubscribeResp resp = new PipeSubscribeUnsubscribeResp(); - - resp.status = status; - resp.version = PipeSubscribeResponseVersion.VERSION_1.getVersion(); - resp.type = PipeSubscribeResponseType.ACK.getType(); - - try (final PublicBAOS byteArrayOutputStream = new PublicBAOS(); - final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) { - ReadWriteIOUtils.writeObjectSet(topicNames, outputStream); - resp.body = - Collections.singletonList( - ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size())); - } - - return resp; - } - /** Deserialize `TPipeSubscribeResp` to obtain parameters, called by the subscription client. */ public static PipeSubscribeUnsubscribeResp fromTPipeSubscribeResp( final TPipeSubscribeResp unsubscribeResp) { From 827a430dd3ada244cf41ef348a0c0635c15560fe Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Thu, 13 Jun 2024 20:36:15 +0800 Subject: [PATCH 30/36] return subscription event for error and termination payload --- .../broker/SubscriptionPrefetchingQueue.java | 18 +++++++++++---- .../SubscriptionPrefetchingTsFileQueue.java | 23 +++++-------------- 2 files changed, 20 insertions(+), 21 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingQueue.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingQueue.java index c1d100c8266a4..d93f64e2f1848 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingQueue.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingQueue.java @@ -24,8 +24,8 @@ import org.apache.iotdb.db.pipe.agent.PipeAgent; import org.apache.iotdb.db.subscription.event.SubscriptionEvent; import org.apache.iotdb.db.subscription.event.SubscriptionEventBinaryCache; -import org.apache.iotdb.db.subscription.event.SubscriptionTsFileEvent; import org.apache.iotdb.pipe.api.event.Event; +import org.apache.iotdb.rpc.subscription.payload.poll.ErrorPayload; import org.apache.iotdb.rpc.subscription.payload.poll.SubscriptionCommitContext; import org.apache.iotdb.rpc.subscription.payload.poll.SubscriptionPollResponse; import org.apache.iotdb.rpc.subscription.payload.poll.SubscriptionPollResponseType; @@ -129,7 +129,7 @@ protected SubscriptionCommitContext generateSubscriptionCommitContext() { subscriptionCommitIdGenerator.getAndIncrement()); } - protected SubscriptionCommitContext generateInvalidSubscriptionCommitContext() { + private SubscriptionCommitContext generateInvalidSubscriptionCommitContext() { return new SubscriptionCommitContext( IoTDBDescriptor.getInstance().getConfig().getDataNodeId(), PipeAgent.runtime().getRebootTimes(), @@ -182,12 +182,22 @@ public void markCompleted() { isCompleted = true; } - public SubscriptionTsFileEvent generateSubscriptionPollTerminationResponse() { - return new SubscriptionTsFileEvent( + public SubscriptionEvent generateSubscriptionPollTerminationResponse() { + return new SubscriptionEvent( Collections.emptyList(), new SubscriptionPollResponse( SubscriptionPollResponseType.TERMINATION.getType(), new TerminationPayload(), generateInvalidSubscriptionCommitContext())); } + + public SubscriptionEvent generateSubscriptionPollErrorResponse( + final String errorMessage, final boolean critical) { + return new SubscriptionEvent( + Collections.emptyList(), + new SubscriptionPollResponse( + SubscriptionPollResponseType.ERROR.getType(), + new ErrorPayload(errorMessage, critical), + generateInvalidSubscriptionCommitContext())); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingTsFileQueue.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingTsFileQueue.java index 5dcca455428ee..dbb6880968113 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingTsFileQueue.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingTsFileQueue.java @@ -24,11 +24,11 @@ import org.apache.iotdb.db.pipe.event.UserDefinedEnrichedEvent; import org.apache.iotdb.db.pipe.event.common.terminate.PipeTerminateEvent; import org.apache.iotdb.db.pipe.event.common.tsfile.PipeTsFileInsertionEvent; +import org.apache.iotdb.db.subscription.event.SubscriptionEvent; import org.apache.iotdb.db.subscription.event.SubscriptionEventBinaryCache; import org.apache.iotdb.db.subscription.event.SubscriptionTsFileEvent; import org.apache.iotdb.pipe.api.event.Event; import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent; -import org.apache.iotdb.rpc.subscription.payload.poll.ErrorPayload; import org.apache.iotdb.rpc.subscription.payload.poll.FileInitPayload; import org.apache.iotdb.rpc.subscription.payload.poll.FilePiecePayload; import org.apache.iotdb.rpc.subscription.payload.poll.FileSealPayload; @@ -43,7 +43,6 @@ import org.slf4j.LoggerFactory; import java.io.IOException; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Objects; @@ -81,7 +80,7 @@ public void cleanup() { } @Override - public SubscriptionTsFileEvent poll(final String consumerId) { + public SubscriptionEvent poll(final String consumerId) { if (hasUnPollableOnTheFlySubscriptionTsFileEvent(consumerId)) { return null; } @@ -148,7 +147,7 @@ public SubscriptionTsFileEvent poll(final String consumerId) { return null; } - public synchronized @NonNull SubscriptionTsFileEvent pollTsFile( + public synchronized @NonNull SubscriptionEvent pollTsFile( final String consumerId, final String fileName, final long writingOffset) { // 1. Extract current event and check it final SubscriptionTsFileEvent event = consumerIdToCurrentEventMap.get(consumerId); @@ -293,7 +292,7 @@ public SubscriptionTsFileEvent poll(final String consumerId) { return pollTsFile(consumerId, writingOffset, event); } - private synchronized @NonNull SubscriptionTsFileEvent pollTsFile( + private synchronized @NonNull SubscriptionEvent pollTsFile( final String consumerId, final long writingOffset, final SubscriptionTsFileEvent event) { Pair newEventWithCommittable = event.matchOrResetNext(writingOffset); @@ -396,18 +395,8 @@ private synchronized SubscriptionTsFileEvent getPollableOnTheFlySubscriptionTsFi return null; } - private SubscriptionTsFileEvent generateSubscriptionPollErrorResponse( - final String errorMessage, final boolean critical) { - return new SubscriptionTsFileEvent( - Collections.emptyList(), - new SubscriptionPollResponse( - SubscriptionPollResponseType.ERROR.getType(), - new ErrorPayload(errorMessage, critical), - super.generateInvalidSubscriptionCommitContext())); - } - - private SubscriptionTsFileEvent generateSubscriptionPollErrorResponse(final String errorMessage) { + private SubscriptionEvent generateSubscriptionPollErrorResponse(final String errorMessage) { // consider non-critical by default, meaning the client can retry - return generateSubscriptionPollErrorResponse(errorMessage, false); + return super.generateSubscriptionPollErrorResponse(errorMessage, false); } } From 371ddf66067e0294eb59d4bbba20f3ff6e76a6c9 Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Thu, 13 Jun 2024 20:42:28 +0800 Subject: [PATCH 31/36] static final vars for configs --- .../rpc/subscription/config/TopicConfig.java | 29 +++++++++++++++---- 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/config/TopicConfig.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/config/TopicConfig.java index 453f363c8f752..b0a64eea674a3 100644 --- a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/config/TopicConfig.java +++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/config/TopicConfig.java @@ -40,6 +40,24 @@ public TopicConfig(final Map attributes) { super(attributes); } + private static final Map LOOSE_RANGE_CONFIG = + new HashMap() { + { + put("history.loose-range", "time"); + put("realtime.loose-range", "time"); + } + }; + + private static final Map REALTIME_BATCH_MODE_CONFIG = + Collections.singletonMap("realtime.mode", "batch"); + private static final Map REALTIME_HYBRID_MODE_CONFIG = + Collections.singletonMap("realtime.mode", "hybrid"); + + private static final Map QUERY_MODE_CONFIG = + Collections.singletonMap("mode", "query"); + private static final Map SUBSCRIBE_MODE_CONFIG = + Collections.singletonMap("mode", "subscribe"); + /////////////////////////////// de/ser /////////////////////////////// public void serialize(final DataOutputStream stream) throws IOException { @@ -87,8 +105,7 @@ public Map getAttributesWithTimeRange(final long creationTime) { // enable loose range when using tsfile format if (TopicConstant.FORMAT_TS_FILE_HANDLER_VALUE.equals( attributes.getOrDefault(TopicConstant.FORMAT_KEY, TopicConstant.FORMAT_DEFAULT_VALUE))) { - attributesWithTimeRange.put("history.loose-range", "time"); - attributesWithTimeRange.put("realtime.loose-range", "time"); + attributesWithTimeRange.putAll(LOOSE_RANGE_CONFIG); } return attributesWithTimeRange; @@ -97,15 +114,15 @@ public Map getAttributesWithTimeRange(final long creationTime) { public Map getAttributesWithRealtimeMode() { return TopicConstant.FORMAT_TS_FILE_HANDLER_VALUE.equals( attributes.getOrDefault(TopicConstant.FORMAT_KEY, TopicConstant.FORMAT_DEFAULT_VALUE)) - ? Collections.singletonMap("realtime.mode", "batch") - : Collections.singletonMap("realtime.mode", "hybrid"); + ? REALTIME_BATCH_MODE_CONFIG + : REALTIME_HYBRID_MODE_CONFIG; } public Map getAttributesWithSourceMode() { return TopicConstant.MODE_QUERY_VALUE.equals( attributes.getOrDefault(TopicConstant.MODE_KEY, TopicConstant.MODE_DEFAULT_VALUE)) - ? Collections.singletonMap("mode", "query") - : Collections.singletonMap("mode", "subscribe"); + ? QUERY_MODE_CONFIG + : SUBSCRIBE_MODE_CONFIG; } public Map getAttributesWithProcessorPrefix() { From 5606872bd2c08af0210c07a66c75f19e0cfe5e75 Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Thu, 13 Jun 2024 20:45:17 +0800 Subject: [PATCH 32/36] intro hasNoTopicsSubscribed --- .../java/org/apache/iotdb/SubscriptionSessionExample.java | 4 ++-- .../org/apache/iotdb/rpc/subscription/config/TopicConfig.java | 4 ++-- .../session/subscription/consumer/SubscriptionConsumer.java | 4 ++++ 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/example/session/src/main/java/org/apache/iotdb/SubscriptionSessionExample.java b/example/session/src/main/java/org/apache/iotdb/SubscriptionSessionExample.java index dcb93ab528ebb..c9bd8af83effa 100644 --- a/example/session/src/main/java/org/apache/iotdb/SubscriptionSessionExample.java +++ b/example/session/src/main/java/org/apache/iotdb/SubscriptionSessionExample.java @@ -265,7 +265,7 @@ private static void dataSubscription3() throws Exception { .buildPushConsumer()) { consumer3.open(); consumer3.subscribe(TOPIC_3); - while (!consumer3.getSubscribedTopicNames().isEmpty()) { + while (!consumer3.hasNoTopicsSubscribed()) { LockSupport.parkNanos(SLEEP_NS); // wait some time } } @@ -307,7 +307,7 @@ private static void dataSubscription4() throws Exception { consumer4.subscribe(TOPIC_4); while (true) { LockSupport.parkNanos(SLEEP_NS); // wait some time - if (consumer4.getSubscribedTopicNames().isEmpty()) { + if (consumer4.hasNoTopicsSubscribed()) { break; } for (final SubscriptionMessage message : consumer4.poll(POLL_TIMEOUT_MS)) { diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/config/TopicConfig.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/config/TopicConfig.java index b0a64eea674a3..bdebb16d10a5d 100644 --- a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/config/TopicConfig.java +++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/config/TopicConfig.java @@ -40,7 +40,7 @@ public TopicConfig(final Map attributes) { super(attributes); } - private static final Map LOOSE_RANGE_CONFIG = + private static final Map LOOSE_RANGE_TIME_CONFIG = new HashMap() { { put("history.loose-range", "time"); @@ -105,7 +105,7 @@ public Map getAttributesWithTimeRange(final long creationTime) { // enable loose range when using tsfile format if (TopicConstant.FORMAT_TS_FILE_HANDLER_VALUE.equals( attributes.getOrDefault(TopicConstant.FORMAT_KEY, TopicConstant.FORMAT_DEFAULT_VALUE))) { - attributesWithTimeRange.putAll(LOOSE_RANGE_CONFIG); + attributesWithTimeRange.putAll(LOOSE_RANGE_TIME_CONFIG); } return attributesWithTimeRange; diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionConsumer.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionConsumer.java index 76d85c69f3488..c06bda5a943b7 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionConsumer.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionConsumer.java @@ -112,6 +112,10 @@ public Set getSubscribedTopicNames() { return subscribedTopicNames; } + public boolean hasNoTopicsSubscribed() { + return subscribedTopicNames.isEmpty(); + } + /////////////////////////////// ctor /////////////////////////////// protected SubscriptionConsumer(final Builder builder) { From c660717c8aeb7f13988ebc71b7d3c6a1debabcd1 Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Fri, 14 Jun 2024 11:02:01 +0800 Subject: [PATCH 33/36] remove SubscriptionTerminationException --- .../SubscriptionTerminationException.java | 40 ------------------- .../consumer/SubscriptionConsumer.java | 1 - 2 files changed, 41 deletions(-) delete mode 100644 iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/exception/SubscriptionTerminationException.java diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/exception/SubscriptionTerminationException.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/exception/SubscriptionTerminationException.java deleted file mode 100644 index f75ec4ff78915..0000000000000 --- a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/exception/SubscriptionTerminationException.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.rpc.subscription.exception; - -import java.util.Objects; - -public class SubscriptionTerminationException extends SubscriptionRuntimeCriticalException { - - public SubscriptionTerminationException(final String message) { - super(message); - } - - public SubscriptionTerminationException(final String message, final Throwable cause) { - super(message, cause); - } - - @Override - public boolean equals(final Object obj) { - return obj instanceof SubscriptionTerminationException - && Objects.equals(getMessage(), ((SubscriptionTerminationException) obj).getMessage()) - && Objects.equals(getTimeStamp(), ((SubscriptionTerminationException) obj).getTimeStamp()); - } -} diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionConsumer.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionConsumer.java index c06bda5a943b7..5ce3a430bc09f 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionConsumer.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionConsumer.java @@ -406,7 +406,6 @@ protected List poll( topicNames, topicNameToUnsubscribe); unsubscribe(Collections.singleton(topicNameToUnsubscribe), false); - // TODO: notify user by ex? break; default: LOGGER.warn("unexpected response type: {}", responseType); From 3a1371a5286bf2a060d50c8812154007f1de7641 Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Fri, 14 Jun 2024 11:18:39 +0800 Subject: [PATCH 34/36] rename hasNoTopicsSubscribed -> hasMoreData --- .../org/apache/iotdb/SubscriptionSessionExample.java | 4 ++-- .../subscription/consumer/SubscriptionConsumer.java | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/example/session/src/main/java/org/apache/iotdb/SubscriptionSessionExample.java b/example/session/src/main/java/org/apache/iotdb/SubscriptionSessionExample.java index c9bd8af83effa..b205ae6208785 100644 --- a/example/session/src/main/java/org/apache/iotdb/SubscriptionSessionExample.java +++ b/example/session/src/main/java/org/apache/iotdb/SubscriptionSessionExample.java @@ -265,7 +265,7 @@ private static void dataSubscription3() throws Exception { .buildPushConsumer()) { consumer3.open(); consumer3.subscribe(TOPIC_3); - while (!consumer3.hasNoTopicsSubscribed()) { + while (!consumer3.hasMoreData()) { LockSupport.parkNanos(SLEEP_NS); // wait some time } } @@ -307,7 +307,7 @@ private static void dataSubscription4() throws Exception { consumer4.subscribe(TOPIC_4); while (true) { LockSupport.parkNanos(SLEEP_NS); // wait some time - if (consumer4.hasNoTopicsSubscribed()) { + if (consumer4.hasMoreData()) { break; } for (final SubscriptionMessage message : consumer4.poll(POLL_TIMEOUT_MS)) { diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionConsumer.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionConsumer.java index 5ce3a430bc09f..5b20ecf7c953c 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionConsumer.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionConsumer.java @@ -108,11 +108,11 @@ public String getConsumerGroupId() { return consumerGroupId; } - public Set getSubscribedTopicNames() { - return subscribedTopicNames; - } - - public boolean hasNoTopicsSubscribed() { + /** + * @return When only subscribing to the query mode topics, if there is no new data to + * process, return {@code false}; otherwise, return {@code true}. + */ + public boolean hasMoreData() { return subscribedTopicNames.isEmpty(); } From f9492e7aa1207d35b090e6cb8eba94854a0da30c Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Fri, 14 Jun 2024 11:38:06 +0800 Subject: [PATCH 35/36] negate --- .../java/org/apache/iotdb/SubscriptionSessionExample.java | 4 ++-- .../session/subscription/consumer/SubscriptionConsumer.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/example/session/src/main/java/org/apache/iotdb/SubscriptionSessionExample.java b/example/session/src/main/java/org/apache/iotdb/SubscriptionSessionExample.java index b205ae6208785..3c3ae618ce3ec 100644 --- a/example/session/src/main/java/org/apache/iotdb/SubscriptionSessionExample.java +++ b/example/session/src/main/java/org/apache/iotdb/SubscriptionSessionExample.java @@ -265,7 +265,7 @@ private static void dataSubscription3() throws Exception { .buildPushConsumer()) { consumer3.open(); consumer3.subscribe(TOPIC_3); - while (!consumer3.hasMoreData()) { + while (consumer3.hasMoreData()) { LockSupport.parkNanos(SLEEP_NS); // wait some time } } @@ -307,7 +307,7 @@ private static void dataSubscription4() throws Exception { consumer4.subscribe(TOPIC_4); while (true) { LockSupport.parkNanos(SLEEP_NS); // wait some time - if (consumer4.hasMoreData()) { + if (!consumer4.hasMoreData()) { break; } for (final SubscriptionMessage message : consumer4.poll(POLL_TIMEOUT_MS)) { diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionConsumer.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionConsumer.java index 5b20ecf7c953c..410348c991b38 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionConsumer.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionConsumer.java @@ -113,7 +113,7 @@ public String getConsumerGroupId() { * process, return {@code false}; otherwise, return {@code true}. */ public boolean hasMoreData() { - return subscribedTopicNames.isEmpty(); + return !subscribedTopicNames.isEmpty(); } /////////////////////////////// ctor /////////////////////////////// From b434d3a832f031bc9c64690c377d907eb1ca5846 Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Fri, 14 Jun 2024 11:58:49 +0800 Subject: [PATCH 36/36] rename hybrid -> stream --- .../apache/iotdb/rpc/subscription/config/TopicConfig.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/config/TopicConfig.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/config/TopicConfig.java index bdebb16d10a5d..c3b5f24622c43 100644 --- a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/config/TopicConfig.java +++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/config/TopicConfig.java @@ -50,8 +50,8 @@ public TopicConfig(final Map attributes) { private static final Map REALTIME_BATCH_MODE_CONFIG = Collections.singletonMap("realtime.mode", "batch"); - private static final Map REALTIME_HYBRID_MODE_CONFIG = - Collections.singletonMap("realtime.mode", "hybrid"); + private static final Map REALTIME_STREAM_MODE_CONFIG = + Collections.singletonMap("realtime.mode", "stream"); private static final Map QUERY_MODE_CONFIG = Collections.singletonMap("mode", "query"); @@ -115,7 +115,7 @@ public Map getAttributesWithRealtimeMode() { return TopicConstant.FORMAT_TS_FILE_HANDLER_VALUE.equals( attributes.getOrDefault(TopicConstant.FORMAT_KEY, TopicConstant.FORMAT_DEFAULT_VALUE)) ? REALTIME_BATCH_MODE_CONFIG - : REALTIME_HYBRID_MODE_CONFIG; + : REALTIME_STREAM_MODE_CONFIG; } public Map getAttributesWithSourceMode() {